postgresql
test
+
+
prob=$prob, prediction=$prediction")
- }
-
-{% endhighlight %}
+{% include_example scala/org/apache/spark/examples/ml/PipelineExample.scala %}
-{% highlight java %}
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.spark.ml.Pipeline;
-import org.apache.spark.ml.PipelineModel;
-import org.apache.spark.ml.PipelineStage;
-import org.apache.spark.ml.classification.LogisticRegression;
-import org.apache.spark.ml.feature.HashingTF;
-import org.apache.spark.ml.feature.Tokenizer;
-import org.apache.spark.sql.DataFrame;
-import org.apache.spark.sql.Row;
-
-// Labeled and unlabeled instance types.
-// Spark SQL can infer schema from Java Beans.
-public class Document implements Serializable {
- private long id;
- private String text;
-
- public Document(long id, String text) {
- this.id = id;
- this.text = text;
- }
-
- public long getId() { return this.id; }
- public void setId(long id) { this.id = id; }
-
- public String getText() { return this.text; }
- public void setText(String text) { this.text = text; }
-}
-
-public class LabeledDocument extends Document implements Serializable {
- private double label;
-
- public LabeledDocument(long id, String text, double label) {
- super(id, text);
- this.label = label;
- }
-
- public double getLabel() { return this.label; }
- public void setLabel(double label) { this.label = label; }
-}
-
-// Prepare training documents, which are labeled.
-DataFrame training = sqlContext.createDataFrame(Arrays.asList(
- new LabeledDocument(0L, "a b c d e spark", 1.0),
- new LabeledDocument(1L, "b d", 0.0),
- new LabeledDocument(2L, "spark f g h", 1.0),
- new LabeledDocument(3L, "hadoop mapreduce", 0.0)
-), LabeledDocument.class);
-
-// Configure an ML pipeline, which consists of three stages: tokenizer, hashingTF, and lr.
-Tokenizer tokenizer = new Tokenizer()
- .setInputCol("text")
- .setOutputCol("words");
-HashingTF hashingTF = new HashingTF()
- .setNumFeatures(1000)
- .setInputCol(tokenizer.getOutputCol())
- .setOutputCol("features");
-LogisticRegression lr = new LogisticRegression()
- .setMaxIter(10)
- .setRegParam(0.01);
-Pipeline pipeline = new Pipeline()
- .setStages(new PipelineStage[] {tokenizer, hashingTF, lr});
-
-// Fit the pipeline to training documents.
-PipelineModel model = pipeline.fit(training);
-
-// Prepare test documents, which are unlabeled.
-DataFrame test = sqlContext.createDataFrame(Arrays.asList(
- new Document(4L, "spark i j k"),
- new Document(5L, "l m n"),
- new Document(6L, "mapreduce spark"),
- new Document(7L, "apache hadoop")
-), Document.class);
-
-// Make predictions on test documents.
-DataFrame predictions = model.transform(test);
-for (Row r: predictions.select("id", "text", "probability", "prediction").collect()) {
- System.out.println("(" + r.get(0) + ", " + r.get(1) + ") --> prob=" + r.get(2)
- + ", prediction=" + r.get(3));
-}
-
-{% endhighlight %}
+{% include_example java/org/apache/spark/examples/ml/JavaPipelineExample.java %}
-{% highlight python %}
-from pyspark.ml import Pipeline
-from pyspark.ml.classification import LogisticRegression
-from pyspark.ml.feature import HashingTF, Tokenizer
-from pyspark.sql import Row
-
-# Prepare training documents from a list of (id, text, label) tuples.
-LabeledDocument = Row("id", "text", "label")
-training = sqlContext.createDataFrame([
- (0L, "a b c d e spark", 1.0),
- (1L, "b d", 0.0),
- (2L, "spark f g h", 1.0),
- (3L, "hadoop mapreduce", 0.0)], ["id", "text", "label"])
-
-# Configure an ML pipeline, which consists of tree stages: tokenizer, hashingTF, and lr.
-tokenizer = Tokenizer(inputCol="text", outputCol="words")
-hashingTF = HashingTF(inputCol=tokenizer.getOutputCol(), outputCol="features")
-lr = LogisticRegression(maxIter=10, regParam=0.01)
-pipeline = Pipeline(stages=[tokenizer, hashingTF, lr])
-
-# Fit the pipeline to training documents.
-model = pipeline.fit(training)
-
-# Prepare test documents, which are unlabeled (id, text) tuples.
-test = sqlContext.createDataFrame([
- (4L, "spark i j k"),
- (5L, "l m n"),
- (6L, "mapreduce spark"),
- (7L, "apache hadoop")], ["id", "text"])
-
-# Make predictions on test documents and print columns of interest.
-prediction = model.transform(test)
-selected = prediction.select("id", "text", "prediction")
-for row in selected.collect():
- print(row)
-
-{% endhighlight %}
+{% include_example python/ml/pipeline_example.py %}
@@ -646,201 +276,16 @@ However, it is also a well-established method for choosing parameters which is m
-{% highlight scala %}
-import org.apache.spark.ml.Pipeline
-import org.apache.spark.ml.classification.LogisticRegression
-import org.apache.spark.ml.evaluation.BinaryClassificationEvaluator
-import org.apache.spark.ml.feature.{HashingTF, Tokenizer}
-import org.apache.spark.ml.tuning.{ParamGridBuilder, CrossValidator}
-import org.apache.spark.mllib.linalg.Vector
-import org.apache.spark.sql.Row
-
-// Prepare training data from a list of (id, text, label) tuples.
-val training = sqlContext.createDataFrame(Seq(
- (0L, "a b c d e spark", 1.0),
- (1L, "b d", 0.0),
- (2L, "spark f g h", 1.0),
- (3L, "hadoop mapreduce", 0.0),
- (4L, "b spark who", 1.0),
- (5L, "g d a y", 0.0),
- (6L, "spark fly", 1.0),
- (7L, "was mapreduce", 0.0),
- (8L, "e spark program", 1.0),
- (9L, "a e c l", 0.0),
- (10L, "spark compile", 1.0),
- (11L, "hadoop software", 0.0)
-)).toDF("id", "text", "label")
-
-// Configure an ML pipeline, which consists of three stages: tokenizer, hashingTF, and lr.
-val tokenizer = new Tokenizer()
- .setInputCol("text")
- .setOutputCol("words")
-val hashingTF = new HashingTF()
- .setInputCol(tokenizer.getOutputCol)
- .setOutputCol("features")
-val lr = new LogisticRegression()
- .setMaxIter(10)
-val pipeline = new Pipeline()
- .setStages(Array(tokenizer, hashingTF, lr))
-
-// We use a ParamGridBuilder to construct a grid of parameters to search over.
-// With 3 values for hashingTF.numFeatures and 2 values for lr.regParam,
-// this grid will have 3 x 2 = 6 parameter settings for CrossValidator to choose from.
-val paramGrid = new ParamGridBuilder()
- .addGrid(hashingTF.numFeatures, Array(10, 100, 1000))
- .addGrid(lr.regParam, Array(0.1, 0.01))
- .build()
-
-// We now treat the Pipeline as an Estimator, wrapping it in a CrossValidator instance.
-// This will allow us to jointly choose parameters for all Pipeline stages.
-// A CrossValidator requires an Estimator, a set of Estimator ParamMaps, and an Evaluator.
-// Note that the evaluator here is a BinaryClassificationEvaluator and its default metric
-// is areaUnderROC.
-val cv = new CrossValidator()
- .setEstimator(pipeline)
- .setEvaluator(new BinaryClassificationEvaluator)
- .setEstimatorParamMaps(paramGrid)
- .setNumFolds(2) // Use 3+ in practice
-
-// Run cross-validation, and choose the best set of parameters.
-val cvModel = cv.fit(training)
-
-// Prepare test documents, which are unlabeled (id, text) tuples.
-val test = sqlContext.createDataFrame(Seq(
- (4L, "spark i j k"),
- (5L, "l m n"),
- (6L, "mapreduce spark"),
- (7L, "apache hadoop")
-)).toDF("id", "text")
-
-// Make predictions on test documents. cvModel uses the best model found (lrModel).
-cvModel.transform(test)
- .select("id", "text", "probability", "prediction")
- .collect()
- .foreach { case Row(id: Long, text: String, prob: Vector, prediction: Double) =>
- println(s"($id, $text) --> prob=$prob, prediction=$prediction")
- }
-
-{% endhighlight %}
+{% include_example scala/org/apache/spark/examples/ml/ModelSelectionViaCrossValidationExample.scala %}
-{% highlight java %}
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.spark.ml.Pipeline;
-import org.apache.spark.ml.PipelineStage;
-import org.apache.spark.ml.classification.LogisticRegression;
-import org.apache.spark.ml.evaluation.BinaryClassificationEvaluator;
-import org.apache.spark.ml.feature.HashingTF;
-import org.apache.spark.ml.feature.Tokenizer;
-import org.apache.spark.ml.param.ParamMap;
-import org.apache.spark.ml.tuning.CrossValidator;
-import org.apache.spark.ml.tuning.CrossValidatorModel;
-import org.apache.spark.ml.tuning.ParamGridBuilder;
-import org.apache.spark.sql.DataFrame;
-import org.apache.spark.sql.Row;
-
-// Labeled and unlabeled instance types.
-// Spark SQL can infer schema from Java Beans.
-public class Document implements Serializable {
- private long id;
- private String text;
-
- public Document(long id, String text) {
- this.id = id;
- this.text = text;
- }
-
- public long getId() { return this.id; }
- public void setId(long id) { this.id = id; }
-
- public String getText() { return this.text; }
- public void setText(String text) { this.text = text; }
-}
-
-public class LabeledDocument extends Document implements Serializable {
- private double label;
-
- public LabeledDocument(long id, String text, double label) {
- super(id, text);
- this.label = label;
- }
-
- public double getLabel() { return this.label; }
- public void setLabel(double label) { this.label = label; }
-}
-
-
-// Prepare training documents, which are labeled.
-DataFrame training = sqlContext.createDataFrame(Arrays.asList(
- new LabeledDocument(0L, "a b c d e spark", 1.0),
- new LabeledDocument(1L, "b d", 0.0),
- new LabeledDocument(2L, "spark f g h", 1.0),
- new LabeledDocument(3L, "hadoop mapreduce", 0.0),
- new LabeledDocument(4L, "b spark who", 1.0),
- new LabeledDocument(5L, "g d a y", 0.0),
- new LabeledDocument(6L, "spark fly", 1.0),
- new LabeledDocument(7L, "was mapreduce", 0.0),
- new LabeledDocument(8L, "e spark program", 1.0),
- new LabeledDocument(9L, "a e c l", 0.0),
- new LabeledDocument(10L, "spark compile", 1.0),
- new LabeledDocument(11L, "hadoop software", 0.0)
-), LabeledDocument.class);
-
-// Configure an ML pipeline, which consists of three stages: tokenizer, hashingTF, and lr.
-Tokenizer tokenizer = new Tokenizer()
- .setInputCol("text")
- .setOutputCol("words");
-HashingTF hashingTF = new HashingTF()
- .setNumFeatures(1000)
- .setInputCol(tokenizer.getOutputCol())
- .setOutputCol("features");
-LogisticRegression lr = new LogisticRegression()
- .setMaxIter(10)
- .setRegParam(0.01);
-Pipeline pipeline = new Pipeline()
- .setStages(new PipelineStage[] {tokenizer, hashingTF, lr});
-
-// We use a ParamGridBuilder to construct a grid of parameters to search over.
-// With 3 values for hashingTF.numFeatures and 2 values for lr.regParam,
-// this grid will have 3 x 2 = 6 parameter settings for CrossValidator to choose from.
-ParamMap[] paramGrid = new ParamGridBuilder()
- .addGrid(hashingTF.numFeatures(), new int[]{10, 100, 1000})
- .addGrid(lr.regParam(), new double[]{0.1, 0.01})
- .build();
-
-// We now treat the Pipeline as an Estimator, wrapping it in a CrossValidator instance.
-// This will allow us to jointly choose parameters for all Pipeline stages.
-// A CrossValidator requires an Estimator, a set of Estimator ParamMaps, and an Evaluator.
-// Note that the evaluator here is a BinaryClassificationEvaluator and its default metric
-// is areaUnderROC.
-CrossValidator cv = new CrossValidator()
- .setEstimator(pipeline)
- .setEvaluator(new BinaryClassificationEvaluator())
- .setEstimatorParamMaps(paramGrid)
- .setNumFolds(2); // Use 3+ in practice
-
-// Run cross-validation, and choose the best set of parameters.
-CrossValidatorModel cvModel = cv.fit(training);
-
-// Prepare test documents, which are unlabeled.
-DataFrame test = sqlContext.createDataFrame(Arrays.asList(
- new Document(4L, "spark i j k"),
- new Document(5L, "l m n"),
- new Document(6L, "mapreduce spark"),
- new Document(7L, "apache hadoop")
-), Document.class);
-
-// Make predictions on test documents. cvModel uses the best model found (lrModel).
-DataFrame predictions = cvModel.transform(test);
-for (Row r: predictions.select("id", "text", "probability", "prediction").collect()) {
- System.out.println("(" + r.get(0) + ", " + r.get(1) + ") --> prob=" + r.get(2)
- + ", prediction=" + r.get(3));
-}
-
-{% endhighlight %}
+{% include_example java/org/apache/spark/examples/ml/JavaModelSelectionViaCrossValidationExample.java %}
+
+
+
+
+{% include_example python/ml/cross_validator.py %}
@@ -864,91 +309,11 @@ The `ParamMap` which produces the best evaluation metric is selected as the best
-{% highlight scala %}
-import org.apache.spark.ml.evaluation.RegressionEvaluator
-import org.apache.spark.ml.regression.LinearRegression
-import org.apache.spark.ml.tuning.{ParamGridBuilder, TrainValidationSplit}
-
-// Prepare training and test data.
-val data = sqlContext.read.format("libsvm").load("data/mllib/sample_linear_regression_data.txt")
-val Array(training, test) = data.randomSplit(Array(0.9, 0.1), seed = 12345)
-
-val lr = new LinearRegression()
-
-// We use a ParamGridBuilder to construct a grid of parameters to search over.
-// TrainValidationSplit will try all combinations of values and determine best model using
-// the evaluator.
-val paramGrid = new ParamGridBuilder()
- .addGrid(lr.regParam, Array(0.1, 0.01))
- .addGrid(lr.fitIntercept)
- .addGrid(lr.elasticNetParam, Array(0.0, 0.5, 1.0))
- .build()
-
-// In this case the estimator is simply the linear regression.
-// A TrainValidationSplit requires an Estimator, a set of Estimator ParamMaps, and an Evaluator.
-val trainValidationSplit = new TrainValidationSplit()
- .setEstimator(lr)
- .setEvaluator(new RegressionEvaluator)
- .setEstimatorParamMaps(paramGrid)
- // 80% of the data will be used for training and the remaining 20% for validation.
- .setTrainRatio(0.8)
-
-// Run train validation split, and choose the best set of parameters.
-val model = trainValidationSplit.fit(training)
-
-// Make predictions on test data. model is the model with combination of parameters
-// that performed best.
-model.transform(test)
- .select("features", "label", "prediction")
- .show()
-
-{% endhighlight %}
+{% include_example scala/org/apache/spark/examples/ml/ModelSelectionViaTrainValidationSplitExample.scala %}
-{% highlight java %}
-import org.apache.spark.ml.evaluation.RegressionEvaluator;
-import org.apache.spark.ml.param.ParamMap;
-import org.apache.spark.ml.regression.LinearRegression;
-import org.apache.spark.ml.tuning.*;
-import org.apache.spark.sql.DataFrame;
-
-DataFrame data = jsql.read().format("libsvm").load("data/mllib/sample_linear_regression_data.txt");
-
-// Prepare training and test data.
-DataFrame[] splits = data.randomSplit(new double[] {0.9, 0.1}, 12345);
-DataFrame training = splits[0];
-DataFrame test = splits[1];
-
-LinearRegression lr = new LinearRegression();
-
-// We use a ParamGridBuilder to construct a grid of parameters to search over.
-// TrainValidationSplit will try all combinations of values and determine best model using
-// the evaluator.
-ParamMap[] paramGrid = new ParamGridBuilder()
- .addGrid(lr.regParam(), new double[] {0.1, 0.01})
- .addGrid(lr.fitIntercept())
- .addGrid(lr.elasticNetParam(), new double[] {0.0, 0.5, 1.0})
- .build();
-
-// In this case the estimator is simply the linear regression.
-// A TrainValidationSplit requires an Estimator, a set of Estimator ParamMaps, and an Evaluator.
-TrainValidationSplit trainValidationSplit = new TrainValidationSplit()
- .setEstimator(lr)
- .setEvaluator(new RegressionEvaluator())
- .setEstimatorParamMaps(paramGrid)
- .setTrainRatio(0.8); // 80% for training and the remaining 20% for validation
-
-// Run train validation split, and choose the best set of parameters.
-TrainValidationSplitModel model = trainValidationSplit.fit(training);
-
-// Make predictions on test data. model is the model with combination of parameters
-// that performed best.
-model.transform(test)
- .select("features", "label", "prediction")
- .show();
-
-{% endhighlight %}
+{% include_example java/org/apache/spark/examples/ml/JavaModelSelectionViaTrainValidationSplitExample.java %}
diff --git a/docs/mllib-decision-tree.md b/docs/mllib-decision-tree.md
index a8612b6c84fe9..9af48357b3dfc 100644
--- a/docs/mllib-decision-tree.md
+++ b/docs/mllib-decision-tree.md
@@ -121,12 +121,12 @@ The parameters are listed below roughly in order of descending importance. New
These parameters describe the problem you want to solve and your dataset.
They should be specified and do not require tuning.
-* **`algo`**: `Classification` or `Regression`
+* **`algo`**: Type of decision tree, either `Classification` or `Regression`.
-* **`numClasses`**: Number of classes (for `Classification` only)
+* **`numClasses`**: Number of classes (for `Classification` only).
* **`categoricalFeaturesInfo`**: Specifies which features are categorical and how many categorical values each of those features can take. This is given as a map from feature indices to feature arity (number of categories). Any features not in this map are treated as continuous.
- * E.g., `Map(0 -> 2, 4 -> 10)` specifies that feature `0` is binary (taking values `0` or `1`) and that feature `4` has 10 categories (values `{0, 1, ..., 9}`). Note that feature indices are 0-based: features `0` and `4` are the 1st and 5th elements of an instance's feature vector.
+ * For example, `Map(0 -> 2, 4 -> 10)` specifies that feature `0` is binary (taking values `0` or `1`) and that feature `4` has 10 categories (values `{0, 1, ..., 9}`). Note that feature indices are 0-based: features `0` and `4` are the 1st and 5th elements of an instance's feature vector.
* Note that you do not have to specify `categoricalFeaturesInfo`. The algorithm will still run and may get reasonable results. However, performance should be better if categorical features are properly designated.
### Stopping criteria
diff --git a/docs/mllib-dimensionality-reduction.md b/docs/mllib-dimensionality-reduction.md
index 11d8e0bd1d23d..cceddce9f79a6 100644
--- a/docs/mllib-dimensionality-reduction.md
+++ b/docs/mllib-dimensionality-reduction.md
@@ -64,19 +64,7 @@ passes, $O(n)$ storage on each executor, and $O(n k)$ storage on the driver.
Refer to the [`SingularValueDecomposition` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.SingularValueDecomposition) for details on the API.
-{% highlight scala %}
-import org.apache.spark.mllib.linalg.Matrix
-import org.apache.spark.mllib.linalg.distributed.RowMatrix
-import org.apache.spark.mllib.linalg.SingularValueDecomposition
-
-val mat: RowMatrix = ...
-
-// Compute the top 20 singular values and corresponding singular vectors.
-val svd: SingularValueDecomposition[RowMatrix, Matrix] = mat.computeSVD(20, computeU = true)
-val U: RowMatrix = svd.U // The U factor is a RowMatrix.
-val s: Vector = svd.s // The singular values are stored in a local dense vector.
-val V: Matrix = svd.V // The V factor is a local dense matrix.
-{% endhighlight %}
+{% include_example scala/org/apache/spark/examples/mllib/SVDExample.scala %}
The same code applies to `IndexedRowMatrix` if `U` is defined as an
`IndexedRowMatrix`.
@@ -84,43 +72,7 @@ The same code applies to `IndexedRowMatrix` if `U` is defined as an
Refer to the [`SingularValueDecomposition` Java docs](api/java/org/apache/spark/mllib/linalg/SingularValueDecomposition.html) for details on the API.
-{% highlight java %}
-import java.util.LinkedList;
-
-import org.apache.spark.api.java.*;
-import org.apache.spark.mllib.linalg.distributed.RowMatrix;
-import org.apache.spark.mllib.linalg.Matrix;
-import org.apache.spark.mllib.linalg.SingularValueDecomposition;
-import org.apache.spark.mllib.linalg.Vector;
-import org.apache.spark.mllib.linalg.Vectors;
-import org.apache.spark.rdd.RDD;
-import org.apache.spark.SparkConf;
-import org.apache.spark.SparkContext;
-
-public class SVD {
- public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("SVD Example");
- SparkContext sc = new SparkContext(conf);
-
- double[][] array = ...
- LinkedList rowsList = new LinkedList();
- for (int i = 0; i < array.length; i++) {
- Vector currentRow = Vectors.dense(array[i]);
- rowsList.add(currentRow);
- }
- JavaRDD rows = JavaSparkContext.fromSparkContext(sc).parallelize(rowsList);
-
- // Create a RowMatrix from JavaRDD.
- RowMatrix mat = new RowMatrix(rows.rdd());
-
- // Compute the top 4 singular values and corresponding singular vectors.
- SingularValueDecomposition svd = mat.computeSVD(4, true, 1.0E-9d);
- RowMatrix U = svd.U();
- Vector s = svd.s();
- Matrix V = svd.V();
- }
-}
-{% endhighlight %}
+{% include_example java/org/apache/spark/examples/mllib/JavaSVDExample.java %}
The same code applies to `IndexedRowMatrix` if `U` is defined as an
`IndexedRowMatrix`.
@@ -151,36 +103,14 @@ and use them to project the vectors into a low-dimensional space.
Refer to the [`RowMatrix` Scala docs](api/scala/index.html#org.apache.spark.mllib.linalg.distributed.RowMatrix) for details on the API.
-{% highlight scala %}
-import org.apache.spark.mllib.linalg.Matrix
-import org.apache.spark.mllib.linalg.distributed.RowMatrix
-
-val mat: RowMatrix = ...
-
-// Compute the top 10 principal components.
-val pc: Matrix = mat.computePrincipalComponents(10) // Principal components are stored in a local dense matrix.
-
-// Project the rows to the linear space spanned by the top 10 principal components.
-val projected: RowMatrix = mat.multiply(pc)
-{% endhighlight %}
+{% include_example scala/org/apache/spark/examples/mllib/PCAOnRowMatrixExample.scala %}
The following code demonstrates how to compute principal components on source vectors
and use them to project the vectors into a low-dimensional space while keeping associated labels:
Refer to the [`PCA` Scala docs](api/scala/index.html#org.apache.spark.mllib.feature.PCA) for details on the API.
-{% highlight scala %}
-import org.apache.spark.mllib.regression.LabeledPoint
-import org.apache.spark.mllib.feature.PCA
-
-val data: RDD[LabeledPoint] = ...
-
-// Compute the top 10 principal components.
-val pca = new PCA(10).fit(data.map(_.features))
-
-// Project vectors to the linear space spanned by the top 10 principal components, keeping the label
-val projected = data.map(p => p.copy(features = pca.transform(p.features)))
-{% endhighlight %}
+{% include_example scala/org/apache/spark/examples/mllib/PCAOnSourceVectorExample.scala %}
@@ -192,40 +122,7 @@ The number of columns should be small, e.g, less than 1000.
Refer to the [`RowMatrix` Java docs](api/java/org/apache/spark/mllib/linalg/distributed/RowMatrix.html) for details on the API.
-{% highlight java %}
-import java.util.LinkedList;
-
-import org.apache.spark.api.java.*;
-import org.apache.spark.mllib.linalg.distributed.RowMatrix;
-import org.apache.spark.mllib.linalg.Matrix;
-import org.apache.spark.mllib.linalg.Vector;
-import org.apache.spark.mllib.linalg.Vectors;
-import org.apache.spark.rdd.RDD;
-import org.apache.spark.SparkConf;
-import org.apache.spark.SparkContext;
-
-public class PCA {
- public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("PCA Example");
- SparkContext sc = new SparkContext(conf);
-
- double[][] array = ...
- LinkedList
rowsList = new LinkedList();
- for (int i = 0; i < array.length; i++) {
- Vector currentRow = Vectors.dense(array[i]);
- rowsList.add(currentRow);
- }
- JavaRDD rows = JavaSparkContext.fromSparkContext(sc).parallelize(rowsList);
-
- // Create a RowMatrix from JavaRDD.
- RowMatrix mat = new RowMatrix(rows.rdd());
-
- // Compute the top 3 principal components.
- Matrix pc = mat.computePrincipalComponents(3);
- RowMatrix projected = mat.multiply(pc);
- }
-}
-{% endhighlight %}
+{% include_example java/org/apache/spark/examples/mllib/JavaPCAExample.java %}
@@ -516,30 +320,7 @@ will in the future.
Refer to the [`LogisticRegressionWithLBFGS` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.classification.LogisticRegressionWithLBFGS) and [`LogisticRegressionModel` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.classification.LogisticRegressionModel) for more details on the API.
-{% highlight python %}
-from pyspark.mllib.classification import LogisticRegressionWithLBFGS, LogisticRegressionModel
-from pyspark.mllib.regression import LabeledPoint
-
-# Load and parse the data
-def parsePoint(line):
- values = [float(x) for x in line.split(' ')]
- return LabeledPoint(values[0], values[1:])
-
-data = sc.textFile("data/mllib/sample_svm_data.txt")
-parsedData = data.map(parsePoint)
-
-# Build the model
-model = LogisticRegressionWithLBFGS.train(parsedData)
-
-# Evaluating the model on training data
-labelsAndPreds = parsedData.map(lambda p: (p.label, model.predict(p.features)))
-trainErr = labelsAndPreds.filter(lambda (v, p): v != p).count() / float(parsedData.count())
-print("Training Error = " + str(trainErr))
-
-# Save and load model
-model.save(sc, "myModelPath")
-sameModel = LogisticRegressionModel.load(sc, "myModelPath")
-{% endhighlight %}
+{% include_example python/mllib/logistic_regression_with_lbfgs_example.py %}
@@ -575,36 +356,7 @@ values. We compute the mean squared error at the end to evaluate
Refer to the [`LinearRegressionWithSGD` Scala docs](api/scala/index.html#org.apache.spark.mllib.regression.LinearRegressionWithSGD) and [`LinearRegressionModel` Scala docs](api/scala/index.html#org.apache.spark.mllib.regression.LinearRegressionModel) for details on the API.
-{% highlight scala %}
-import org.apache.spark.mllib.regression.LabeledPoint
-import org.apache.spark.mllib.regression.LinearRegressionModel
-import org.apache.spark.mllib.regression.LinearRegressionWithSGD
-import org.apache.spark.mllib.linalg.Vectors
-
-// Load and parse the data
-val data = sc.textFile("data/mllib/ridge-data/lpsa.data")
-val parsedData = data.map { line =>
- val parts = line.split(',')
- LabeledPoint(parts(0).toDouble, Vectors.dense(parts(1).split(' ').map(_.toDouble)))
-}.cache()
-
-// Building the model
-val numIterations = 100
-val stepSize = 0.00000001
-val model = LinearRegressionWithSGD.train(parsedData, numIterations, stepSize)
-
-// Evaluate model on training examples and compute training error
-val valuesAndPreds = parsedData.map { point =>
- val prediction = model.predict(point.features)
- (point.label, prediction)
-}
-val MSE = valuesAndPreds.map{case(v, p) => math.pow((v - p), 2)}.mean()
-println("training Mean Squared Error = " + MSE)
-
-// Save and load model
-model.save(sc, "myModelPath")
-val sameModel = LinearRegressionModel.load(sc, "myModelPath")
-{% endhighlight %}
+{% include_example scala/org/apache/spark/examples/mllib/LinearRegressionWithSGDExample.scala %}
[`RidgeRegressionWithSGD`](api/scala/index.html#org.apache.spark.mllib.regression.RidgeRegressionWithSGD)
and [`LassoWithSGD`](api/scala/index.html#org.apache.spark.mllib.regression.LassoWithSGD) can be used in a similar fashion as `LinearRegressionWithSGD`.
@@ -620,70 +372,7 @@ the Scala snippet provided, is presented below:
Refer to the [`LinearRegressionWithSGD` Java docs](api/java/org/apache/spark/mllib/regression/LinearRegressionWithSGD.html) and [`LinearRegressionModel` Java docs](api/java/org/apache/spark/mllib/regression/LinearRegressionModel.html) for details on the API.
-{% highlight java %}
-import scala.Tuple2;
-
-import org.apache.spark.api.java.*;
-import org.apache.spark.api.java.function.Function;
-import org.apache.spark.mllib.linalg.Vector;
-import org.apache.spark.mllib.linalg.Vectors;
-import org.apache.spark.mllib.regression.LabeledPoint;
-import org.apache.spark.mllib.regression.LinearRegressionModel;
-import org.apache.spark.mllib.regression.LinearRegressionWithSGD;
-import org.apache.spark.SparkConf;
-
-public class LinearRegression {
- public static void main(String[] args) {
- SparkConf conf = new SparkConf().setAppName("Linear Regression Example");
- JavaSparkContext sc = new JavaSparkContext(conf);
-
- // Load and parse the data
- String path = "data/mllib/ridge-data/lpsa.data";
- JavaRDD data = sc.textFile(path);
- JavaRDD parsedData = data.map(
- new Function() {
- public LabeledPoint call(String line) {
- String[] parts = line.split(",");
- String[] features = parts[1].split(" ");
- double[] v = new double[features.length];
- for (int i = 0; i < features.length - 1; i++)
- v[i] = Double.parseDouble(features[i]);
- return new LabeledPoint(Double.parseDouble(parts[0]), Vectors.dense(v));
- }
- }
- );
- parsedData.cache();
-
- // Building the model
- int numIterations = 100;
- double stepSize = 0.00000001;
- final LinearRegressionModel model =
- LinearRegressionWithSGD.train(JavaRDD.toRDD(parsedData), numIterations, stepSize);
-
- // Evaluate model on training examples and compute training error
- JavaRDD> valuesAndPreds = parsedData.map(
- new Function>() {
- public Tuple2 call(LabeledPoint point) {
- double prediction = model.predict(point.features());
- return new Tuple2(prediction, point.label());
- }
- }
- );
- double MSE = new JavaDoubleRDD(valuesAndPreds.map(
- new Function, Object>() {
- public Object call(Tuple2 pair) {
- return Math.pow(pair._1() - pair._2(), 2.0);
- }
- }
- ).rdd()).mean();
- System.out.println("training Mean Squared Error = " + MSE);
-
- // Save and load model
- model.save(sc.sc(), "myModelPath");
- LinearRegressionModel sameModel = LinearRegressionModel.load(sc.sc(), "myModelPath");
- }
-}
-{% endhighlight %}
+{% include_example java/org/apache/spark/examples/mllib/JavaLinearRegressionWithSGDExample.java %}
@@ -696,29 +385,7 @@ Note that the Python API does not yet support model save/load but will in the fu
Refer to the [`LinearRegressionWithSGD` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.regression.LinearRegressionWithSGD) and [`LinearRegressionModel` Python docs](api/python/pyspark.mllib.html#pyspark.mllib.regression.LinearRegressionModel) for more details on the API.
-{% highlight python %}
-from pyspark.mllib.regression import LabeledPoint, LinearRegressionWithSGD, LinearRegressionModel
-
-# Load and parse the data
-def parsePoint(line):
- values = [float(x) for x in line.replace(',', ' ').split(' ')]
- return LabeledPoint(values[0], values[1:])
-
-data = sc.textFile("data/mllib/ridge-data/lpsa.data")
-parsedData = data.map(parsePoint)
-
-# Build the model
-model = LinearRegressionWithSGD.train(parsedData, iterations=100, step=0.00000001)
-
-# Evaluate the model on training data
-valuesAndPreds = parsedData.map(lambda p: (p.label, model.predict(p.features)))
-MSE = valuesAndPreds.map(lambda (v, p): (v - p)**2).reduce(lambda x, y: x + y) / valuesAndPreds.count()
-print("Mean Squared Error = " + str(MSE))
-
-# Save and load model
-model.save(sc, "myModelPath")
-sameModel = LinearRegressionModel.load(sc, "myModelPath")
-{% endhighlight %}
+{% include_example python/mllib/linear_regression_with_sgd_example.py %}
@@ -748,108 +415,50 @@ online to the first stream, and make predictions on the second stream.
First, we import the necessary classes for parsing our input data and creating the model.
-{% highlight scala %}
-
-import org.apache.spark.mllib.linalg.Vectors
-import org.apache.spark.mllib.regression.LabeledPoint
-import org.apache.spark.mllib.regression.StreamingLinearRegressionWithSGD
-
-{% endhighlight %}
-
Then we make input streams for training and testing data. We assume a StreamingContext `ssc`
has already been created, see [Spark Streaming Programming Guide](streaming-programming-guide.html#initializing)
for more info. For this example, we use labeled points in training and testing streams,
but in practice you will likely want to use unlabeled vectors for test data.
-{% highlight scala %}
-
-val trainingData = ssc.textFileStream("/training/data/dir").map(LabeledPoint.parse).cache()
-val testData = ssc.textFileStream("/testing/data/dir").map(LabeledPoint.parse)
-
-{% endhighlight %}
+We create our model by initializing the weights to zero and register the streams for training and
+testing then start the job. Printing predictions alongside true labels lets us easily see the
+result.
-We create our model by initializing the weights to 0
-
-{% highlight scala %}
-
-val numFeatures = 3
-val model = new StreamingLinearRegressionWithSGD()
- .setInitialWeights(Vectors.zeros(numFeatures))
-
-{% endhighlight %}
-
-Now we register the streams for training and testing and start the job.
-Printing predictions alongside true labels lets us easily see the result.
-
-{% highlight scala %}
-
-model.trainOn(trainingData)
-model.predictOnValues(testData.map(lp => (lp.label, lp.features))).print()
-
-ssc.start()
-ssc.awaitTermination()
-
-{% endhighlight %}
-
-We can now save text files with data to the training or testing folders.
+Finally we can save text files with data to the training or testing folders.
Each line should be a data point formatted as `(y,[x1,x2,x3])` where `y` is the label
-and `x1,x2,x3` are the features. Anytime a text file is placed in `/training/data/dir`
-the model will update. Anytime a text file is placed in `/testing/data/dir` you will see predictions.
+and `x1,x2,x3` are the features. Anytime a text file is placed in `args(0)`
+the model will update. Anytime a text file is placed in `args(1)` you will see predictions.
As you feed more data to the training directory, the predictions
will get better!
+Here is a complete example:
+{% include_example scala/org/apache/spark/examples/mllib/StreamingLinearRegressionExample.scala %}
+
First, we import the necessary classes for parsing our input data and creating the model.
-{% highlight python %}
-from pyspark.mllib.linalg import Vectors
-from pyspark.mllib.regression import LabeledPoint
-from pyspark.mllib.regression import StreamingLinearRegressionWithSGD
-{% endhighlight %}
-
Then we make input streams for training and testing data. We assume a StreamingContext `ssc`
has already been created, see [Spark Streaming Programming Guide](streaming-programming-guide.html#initializing)
for more info. For this example, we use labeled points in training and testing streams,
but in practice you will likely want to use unlabeled vectors for test data.
-{% highlight python %}
-def parse(lp):
- label = float(lp[lp.find('(') + 1: lp.find(',')])
- vec = Vectors.dense(lp[lp.find('[') + 1: lp.find(']')].split(','))
- return LabeledPoint(label, vec)
-
-trainingData = ssc.textFileStream("/training/data/dir").map(parse).cache()
-testData = ssc.textFileStream("/testing/data/dir").map(parse)
-{% endhighlight %}
-
-We create our model by initializing the weights to 0
-
-{% highlight python %}
-numFeatures = 3
-model = StreamingLinearRegressionWithSGD()
-model.setInitialWeights([0.0, 0.0, 0.0])
-{% endhighlight %}
+We create our model by initializing the weights to 0.
Now we register the streams for training and testing and start the job.
-{% highlight python %}
-model.trainOn(trainingData)
-print(model.predictOnValues(testData.map(lambda lp: (lp.label, lp.features))))
-
-ssc.start()
-ssc.awaitTermination()
-{% endhighlight %}
-
We can now save text files with data to the training or testing folders.
Each line should be a data point formatted as `(y,[x1,x2,x3])` where `y` is the label
-and `x1,x2,x3` are the features. Anytime a text file is placed in `/training/data/dir`
-the model will update. Anytime a text file is placed in `/testing/data/dir` you will see predictions.
+and `x1,x2,x3` are the features. Anytime a text file is placed in `sys.argv[1]`
+the model will update. Anytime a text file is placed in `sys.argv[2]` you will see predictions.
As you feed more data to the training directory, the predictions
will get better!
+Here a complete example:
+{% include_example python/mllib/streaming_linear_regression_example.py %}
+
diff --git a/docs/programming-guide.md b/docs/programming-guide.md
index 5ebafa40b0c7a..2f0ed5eca2b2b 100644
--- a/docs/programming-guide.md
+++ b/docs/programming-guide.md
@@ -1177,7 +1177,7 @@ that originally created it.
In addition, each persisted RDD can be stored using a different *storage level*, allowing you, for example,
to persist the dataset on disk, persist it in memory but as serialized Java objects (to save space),
-replicate it across nodes, or store it off-heap in [Tachyon](http://tachyon-project.org/).
+replicate it across nodes.
These levels are set by passing a
`StorageLevel` object ([Scala](api/scala/index.html#org.apache.spark.storage.StorageLevel),
[Java](api/java/index.html?org/apache/spark/storage/StorageLevel.html),
@@ -1218,24 +1218,11 @@ storage levels is:
MEMORY_ONLY_2, MEMORY_AND_DISK_2, etc. |
Same as the levels above, but replicate each partition on two cluster nodes. |