Skip to content

Commit

Permalink
[SPARK-10279] [MLLIB] [PYSPARK] [DOCS] Add @SInCE annotation to pyspa…
Browse files Browse the repository at this point in the history
…rk.mllib.util

Author: Yu ISHIKAWA <yuu.ishikawa@gmail.com>

Closes #8689 from yu-iskw/SPARK-10279.
  • Loading branch information
yu-iskw authored and mengxr committed Sep 17, 2015
1 parent 39b44cb commit 4a0b56e
Showing 1 changed file with 26 additions and 2 deletions.
28 changes: 26 additions & 2 deletions python/pyspark/mllib/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
xrange = range
basestring = str

from pyspark import SparkContext
from pyspark import SparkContext, since
from pyspark.mllib.common import callMLlibFunc, inherit_doc
from pyspark.mllib.linalg import Vectors, SparseVector, _convert_to_vector

Expand All @@ -32,6 +32,8 @@ class MLUtils(object):

"""
Helper methods to load, save and pre-process data used in MLlib.
.. versionadded:: 1.0.0
"""

@staticmethod
Expand Down Expand Up @@ -69,6 +71,7 @@ def _convert_labeled_point_to_libsvm(p):
return " ".join(items)

@staticmethod
@since("1.0.0")
def loadLibSVMFile(sc, path, numFeatures=-1, minPartitions=None, multiclass=None):
"""
Loads labeled data in the LIBSVM format into an RDD of
Expand Down Expand Up @@ -123,6 +126,7 @@ def loadLibSVMFile(sc, path, numFeatures=-1, minPartitions=None, multiclass=None
return parsed.map(lambda x: LabeledPoint(x[0], Vectors.sparse(numFeatures, x[1], x[2])))

@staticmethod
@since("1.0.0")
def saveAsLibSVMFile(data, dir):
"""
Save labeled data in LIBSVM format.
Expand All @@ -147,6 +151,7 @@ def saveAsLibSVMFile(data, dir):
lines.saveAsTextFile(dir)

@staticmethod
@since("1.1.0")
def loadLabeledPoints(sc, path, minPartitions=None):
"""
Load labeled points saved using RDD.saveAsTextFile.
Expand All @@ -172,6 +177,7 @@ def loadLabeledPoints(sc, path, minPartitions=None):
return callMLlibFunc("loadLabeledPoints", sc, path, minPartitions)

@staticmethod
@since("1.5.0")
def appendBias(data):
"""
Returns a new vector with `1.0` (bias) appended to
Expand All @@ -186,6 +192,7 @@ def appendBias(data):
return _convert_to_vector(np.append(vec.toArray(), 1.0))

@staticmethod
@since("1.5.0")
def loadVectors(sc, path):
"""
Loads vectors saved using `RDD[Vector].saveAsTextFile`
Expand All @@ -197,6 +204,8 @@ def loadVectors(sc, path):
class Saveable(object):
"""
Mixin for models and transformers which may be saved as files.
.. versionadded:: 1.3.0
"""

def save(self, sc, path):
Expand All @@ -222,9 +231,13 @@ class JavaSaveable(Saveable):
"""
Mixin for models that provide save() through their Scala
implementation.
.. versionadded:: 1.3.0
"""

@since("1.3.0")
def save(self, sc, path):
"""Save this model to the given path."""
if not isinstance(sc, SparkContext):
raise TypeError("sc should be a SparkContext, got type %s" % type(sc))
if not isinstance(path, basestring):
Expand All @@ -235,6 +248,8 @@ def save(self, sc, path):
class Loader(object):
"""
Mixin for classes which can load saved models from files.
.. versionadded:: 1.3.0
"""

@classmethod
Expand All @@ -256,6 +271,8 @@ class JavaLoader(Loader):
"""
Mixin for classes which can load saved models using its Scala
implementation.
.. versionadded:: 1.3.0
"""

@classmethod
Expand All @@ -280,15 +297,21 @@ def _load_java(cls, sc, path):
return java_obj.load(sc._jsc.sc(), path)

@classmethod
@since("1.3.0")
def load(cls, sc, path):
"""Load a model from the given path."""
java_model = cls._load_java(sc, path)
return cls(java_model)


class LinearDataGenerator(object):
"""Utils for generating linear data"""
"""Utils for generating linear data.
.. versionadded:: 1.5.0
"""

@staticmethod
@since("1.5.0")
def generateLinearInput(intercept, weights, xMean, xVariance,
nPoints, seed, eps):
"""
Expand All @@ -311,6 +334,7 @@ def generateLinearInput(intercept, weights, xMean, xVariance,
xVariance, int(nPoints), int(seed), float(eps)))

@staticmethod
@since("1.5.0")
def generateLinearRDD(sc, nexamples, nfeatures, eps,
nParts=2, intercept=0.0):
"""
Expand Down

0 comments on commit 4a0b56e

Please sign in to comment.