Skip to content

Commit

Permalink
address comments
Browse files Browse the repository at this point in the history
  • Loading branch information
@mengxr
mengxr committed Aug 19, 2014
1 parent aef6d07 commit ffde810
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 19 deletions.
19 changes: 9 additions & 10 deletions mllib/src/main/scala/org/apache/spark/mllib/random/RandomRDDs.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,16 +35,16 @@ import org.apache.spark.util.Utils
object RandomRDDs {

/**
* Generates an RDD comprised of i.i.d. samples from the uniform distribution on [0.0, 1.0].
* Generates an RDD comprised of i.i.d. samples from the uniform distribution `U(0.0, 1.0)`.
*
* To transform the distribution in the generated RDD from U[0.0, 1.0] to U[a, b], use
* `RandomRDDGenerators.uniformRDD(sc, n, p, seed).map(v => a + (b - a) * v)`.
* To transform the distribution in the generated RDD from `U(0.0, 1.0)` to `U(a, b)`, use
* `RandomRDDs.uniformRDD(sc, n, p, seed).map(v => a + (b - a) * v)`.
*
* @param sc SparkContext used to create the RDD.
* @param size Size of the RDD.
* @param numPartitions Number of partitions in the RDD (default: `sc.defaultParallelism`).
* @param seed Random seed (default: a random long integer).
* @return RDD[Double] comprised of i.i.d. samples ~ U[0.0, 1.0].
* @return RDD[Double] comprised of i.i.d. samples ~ `U(0.0, 1.0)`.
*/
def uniformRDD(
sc: SparkContext,
Expand Down Expand Up @@ -84,7 +84,7 @@ object RandomRDDs {
* Generates an RDD comprised of i.i.d. samples from the standard normal distribution.
*
* To transform the distribution in the generated RDD from standard normal to some other normal
* N(mean, sigma), use `RandomRDDGenerators.normalRDD(sc, n, p, seed).map(v => mean + sigma * v)`.
* `N(mean, sigma^2^)`, use `RandomRDDs.normalRDD(sc, n, p, seed).map(v => mean + sigma * v)`.
*
* @param sc SparkContext used to create the RDD.
* @param size Size of the RDD.
Expand All @@ -97,9 +97,8 @@ object RandomRDDs {
size: Long,
numPartitions: Int = 0,
seed: Long = Utils.random.nextLong()): RDD[Double] = {
val p = if (numPartitions > 0) numPartitions else sc.defaultParallelism
val normal = new StandardNormalGenerator()
randomRDD(sc, normal, size, p, seed)
randomRDD(sc, normal, size, numPartitionsOrDefault(sc, numPartitions), seed)
}

/**
Expand Down Expand Up @@ -202,14 +201,14 @@ object RandomRDDs {

/**
* Generates an RDD[Vector] with vectors containing i.i.d. samples drawn from the
* uniform distribution on [0.0 1.0].
* uniform distribution on `U(0.0 1.0)`.
*
* @param sc SparkContext used to create the RDD.
* @param numRows Number of Vectors in the RDD.
* @param numCols Number of elements in each Vector.
* @param numPartitions Number of partitions in the RDD.
* @param seed Seed for the RNG that generates the seed for the generator in each partition.
* @return RDD[Vector] with vectors containing i.i.d samples ~ U[0.0, 1.0].
* @return RDD[Vector] with vectors containing i.i.d samples ~ `U(0.0, 1.0)`.
*/
def uniformVectorRDD(
sc: SparkContext,
Expand Down Expand Up @@ -263,7 +262,7 @@ object RandomRDDs {
* @param numCols Number of elements in each Vector.
* @param numPartitions Number of partitions in the RDD (default: `sc.defaultParallelism`).
* @param seed Random seed (default: a random long integer).
* @return RDD[Vector] with vectors containing i.i.d. samples ~ N(0.0, 1.0).
* @return RDD[Vector] with vectors containing i.i.d. samples ~ `N(0.0, 1.0)`.
*/
def normalVectorRDD(
sc: SparkContext,
Expand Down
18 changes: 9 additions & 9 deletions python/pyspark/mllib/random.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ def uniformRDD(sc, size, numPartitions=None, seed=None):
Generates an RDD comprised of i.i.d. samples from the
uniform distribution on [0.0, 1.0].
To transform the distribution in the generated RDD from U[0.0, 1.0]
to U[a, b], use
To transform the distribution in the generated RDD from U(0.0, 1.0)
to U(a, b), use
C{RandomRDDs.uniformRDD(sc, n, p, seed)\
.map(lambda v: a + (b - a) * v)}
Expand All @@ -60,11 +60,11 @@ def uniformRDD(sc, size, numPartitions=None, seed=None):
@staticmethod
def normalRDD(sc, size, numPartitions=None, seed=None):
"""
Generates an RDD comprised of i.i.d samples from the standard normal
Generates an RDD comprised of i.i.d. samples from the standard normal
distribution.
To transform the distribution in the generated RDD from standard normal
to some other normal N(mean, sigma), use
to some other normal N(mean, sigma^2), use
C{RandomRDDs.normal(sc, n, p, seed)\
.map(lambda v: mean + sigma * v)}
Expand All @@ -84,7 +84,7 @@ def normalRDD(sc, size, numPartitions=None, seed=None):
@staticmethod
def poissonRDD(sc, mean, size, numPartitions=None, seed=None):
"""
Generates an RDD comprised of i.i.d samples from the Poisson
Generates an RDD comprised of i.i.d. samples from the Poisson
distribution with the input mean.
>>> mean = 100.0
Expand All @@ -105,8 +105,8 @@ def poissonRDD(sc, mean, size, numPartitions=None, seed=None):
@staticmethod
def uniformVectorRDD(sc, numRows, numCols, numPartitions=None, seed=None):
"""
Generates an RDD comprised of vectors containing i.i.d samples drawn
from the uniform distribution on [0.0 1.0].
Generates an RDD comprised of vectors containing i.i.d. samples drawn
from the uniform distribution U(0.0 1.0).
>>> import numpy as np
>>> mat = np.matrix(RandomRDDs.uniformVectorRDD(sc, 10, 10).collect())
Expand All @@ -125,7 +125,7 @@ def uniformVectorRDD(sc, numRows, numCols, numPartitions=None, seed=None):
@staticmethod
def normalVectorRDD(sc, numRows, numCols, numPartitions=None, seed=None):
"""
Generates an RDD comprised of vectors containing i.i.d samples drawn
Generates an RDD comprised of vectors containing i.i.d. samples drawn
from the standard normal distribution.
>>> import numpy as np
Expand All @@ -145,7 +145,7 @@ def normalVectorRDD(sc, numRows, numCols, numPartitions=None, seed=None):
@staticmethod
def poissonVectorRDD(sc, mean, numRows, numCols, numPartitions=None, seed=None):
"""
Generates an RDD comprised of vectors containing i.i.d samples drawn
Generates an RDD comprised of vectors containing i.i.d. samples drawn
from the Poisson distribution with the input mean.
>>> import numpy as np
Expand Down

0 comments on commit ffde810

Please sign in to comment.