diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockHandler.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockHandler.java index fb1226c09e246..22fd592a321d2 100644 --- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockHandler.java +++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ExternalShuffleBlockHandler.java @@ -87,8 +87,11 @@ protected void handleMessage( blocks.add(blockManager.getBlockData(msg.appId, msg.execId, blockId)); } long streamId = streamManager.registerStream(client.getClientId(), blocks.iterator()); - logger.trace("Registered streamId {} with {} buffers for client {} from host {}", streamId, - msg.blockIds.length, client.getClientId(), NettyUtils.getRemoteAddress(client.getChannel())); + logger.trace("Registered streamId {} with {} buffers for client {} from host {}", + streamId, + msg.blockIds.length, + client.getClientId(), + NettyUtils.getRemoteAddress(client.getChannel())); callback.onSuccess(new StreamHandle(streamId, msg.blockIds.length).toByteBuffer()); } else if (msgObj instanceof RegisterExecutor) { diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaAFTSurvivalRegressionExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaAFTSurvivalRegressionExample.java index ecb7084e03122..2c2aa6df47c77 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaAFTSurvivalRegressionExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaAFTSurvivalRegressionExample.java @@ -33,7 +33,10 @@ public class JavaAFTSurvivalRegressionExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaAFTSurvivalRegressionExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaAFTSurvivalRegressionExample") + .getOrCreate(); // $example on$ List data = Arrays.asList( diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java index 9a9a10489b506..4b13ba6f9cea3 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java @@ -81,7 +81,10 @@ public static Rating parseRating(String str) { // $example off$ public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaALSExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaALSExample") + .getOrCreate(); // $example on$ JavaRDD ratingsRDD = spark diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaBinarizerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaBinarizerExample.java index 88e4298a6191e..5f964aca92096 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaBinarizerExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaBinarizerExample.java @@ -17,8 +17,6 @@ package org.apache.spark.examples.ml; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.SparkSession; @@ -26,7 +24,6 @@ import java.util.Arrays; import java.util.List; -import org.apache.spark.api.java.JavaRDD; import org.apache.spark.ml.feature.Binarizer; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; @@ -38,7 +35,10 @@ public class JavaBinarizerExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaBinarizerExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaBinarizerExample") + .getOrCreate(); // $example on$ List data = Arrays.asList( diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaBisectingKMeansExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaBisectingKMeansExample.java index 51aa35084e845..810ad905c56af 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaBisectingKMeansExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaBisectingKMeansExample.java @@ -42,7 +42,10 @@ public class JavaBisectingKMeansExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaBisectingKMeansExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaBisectingKMeansExample") + .getOrCreate(); // $example on$ List data = Arrays.asList( diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaBucketizerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaBucketizerExample.java index 0c24f52cf58ab..691df3887a9bb 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaBucketizerExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaBucketizerExample.java @@ -35,7 +35,10 @@ public class JavaBucketizerExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaBucketizerExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaBucketizerExample") + .getOrCreate(); // $example on$ double[] splits = {Double.NEGATIVE_INFINITY, -0.5, 0.0, 0.5, Double.POSITIVE_INFINITY}; diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaChiSqSelectorExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaChiSqSelectorExample.java index 684cf9a7145d2..f8f2fb14be1f1 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaChiSqSelectorExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaChiSqSelectorExample.java @@ -17,9 +17,6 @@ package org.apache.spark.examples.ml; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.SparkSession; @@ -40,7 +37,10 @@ public class JavaChiSqSelectorExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaChiSqSelectorExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaChiSqSelectorExample") + .getOrCreate(); // $example on$ List data = Arrays.asList( diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaCountVectorizerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaCountVectorizerExample.java index 0631f9d6d5d74..0a6b13601425b 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaCountVectorizerExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaCountVectorizerExample.java @@ -32,7 +32,10 @@ public class JavaCountVectorizerExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaCountVectorizerExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaCountVectorizerExample") + .getOrCreate(); // $example on$ // Input data: Each row is a bag of words from a sentence or document. diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaDCTExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaDCTExample.java index ec57a2445138a..eee92c77a8c58 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaDCTExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaDCTExample.java @@ -17,8 +17,6 @@ package org.apache.spark.examples.ml; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.SparkSession; @@ -26,7 +24,6 @@ import java.util.Arrays; import java.util.List; -import org.apache.spark.api.java.JavaRDD; import org.apache.spark.ml.feature.DCT; import org.apache.spark.mllib.linalg.VectorUDT; import org.apache.spark.mllib.linalg.Vectors; @@ -39,7 +36,10 @@ public class JavaDCTExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaDCTExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaDCTExample") + .getOrCreate(); // $example on$ List data = Arrays.asList( diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaDeveloperApiExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaDeveloperApiExample.java index 90023ac06b79b..49bad0afc0b71 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaDeveloperApiExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaDeveloperApiExample.java @@ -49,7 +49,10 @@ public class JavaDeveloperApiExample { public static void main(String[] args) throws Exception { - SparkSession spark = SparkSession.builder().appName("JavaDeveloperApiExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaDeveloperApiExample") + .getOrCreate(); // Prepare training data. List localTraining = Lists.newArrayList( diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaElementwiseProductExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaElementwiseProductExample.java index a062a6fcd0c19..9126242f9eb78 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaElementwiseProductExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaElementwiseProductExample.java @@ -17,8 +17,6 @@ package org.apache.spark.examples.ml; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.SparkSession; @@ -27,7 +25,6 @@ import java.util.Arrays; import java.util.List; -import org.apache.spark.api.java.JavaRDD; import org.apache.spark.ml.feature.ElementwiseProduct; import org.apache.spark.mllib.linalg.Vector; import org.apache.spark.mllib.linalg.VectorUDT; @@ -42,7 +39,9 @@ public class JavaElementwiseProductExample { public static void main(String[] args) { SparkSession spark = SparkSession - .builder().appName("JavaElementwiseProductExample").getOrCreate(); + .builder() + .appName("JavaElementwiseProductExample") + .getOrCreate(); // $example on$ // Create some vector data; also works for sparse vectors diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaGradientBoostedTreeClassifierExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaGradientBoostedTreeClassifierExample.java index a7c89b9d19974..baacd796a058f 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaGradientBoostedTreeClassifierExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaGradientBoostedTreeClassifierExample.java @@ -17,8 +17,6 @@ package org.apache.spark.examples.ml; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaSparkContext; // $example on$ import org.apache.spark.ml.Pipeline; import org.apache.spark.ml.PipelineModel; @@ -35,11 +33,15 @@ public class JavaGradientBoostedTreeClassifierExample { public static void main(String[] args) { SparkSession spark = SparkSession - .builder().appName("JavaGradientBoostedTreeClassifierExample").getOrCreate(); + .builder() + .appName("JavaGradientBoostedTreeClassifierExample") + .getOrCreate(); // $example on$ // Load and parse the data file, converting it to a DataFrame. - Dataset data = spark.read().format("libsvm") + Dataset data = spark + .read() + .format("libsvm") .load("data/mllib/sample_libsvm_data.txt"); // Index labels, adding metadata to the label column. diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaIndexToStringExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaIndexToStringExample.java index ccd74f2920ce3..0064beb8c8f33 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaIndexToStringExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaIndexToStringExample.java @@ -37,7 +37,10 @@ public class JavaIndexToStringExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaIndexToStringExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaIndexToStringExample") + .getOrCreate(); // $example on$ List data = Arrays.asList( diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaKMeansExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaKMeansExample.java index e6d82a0513a30..65e29ade299d1 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaKMeansExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaKMeansExample.java @@ -70,7 +70,10 @@ public static void main(String[] args) { int k = Integer.parseInt(args[1]); // Parses the arguments - SparkSession spark = SparkSession.builder().appName("JavaKMeansExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaKMeansExample") + .getOrCreate(); // $example on$ // Loads data diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaLDAExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaLDAExample.java index b8baca5920e8d..1c52f37867a85 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaLDAExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaLDAExample.java @@ -65,7 +65,10 @@ public static void main(String[] args) { String inputFile = "data/mllib/sample_lda_data.txt"; // Parses the arguments - SparkSession spark = SparkSession.builder().appName("JavaLDAExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaLDAExample") + .getOrCreate(); // Loads data JavaRDD points = spark.read().text(inputFile).javaRDD().map(new ParseVector()); diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaMaxAbsScalerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaMaxAbsScalerExample.java index 80cdd364b9376..9a27b0e9e23b7 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaMaxAbsScalerExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaMaxAbsScalerExample.java @@ -28,13 +28,19 @@ public class JavaMaxAbsScalerExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaMaxAbsScalerExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaMaxAbsScalerExample") + .getOrCreate(); // $example on$ - Dataset dataFrame = spark.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt"); + Dataset dataFrame = spark + .read() + .format("libsvm") + .load("data/mllib/sample_libsvm_data.txt"); MaxAbsScaler scaler = new MaxAbsScaler() - .setInputCol("features") - .setOutputCol("scaledFeatures"); + .setInputCol("features") + .setOutputCol("scaledFeatures"); // Compute summary statistics and generate MaxAbsScalerModel MaxAbsScalerModel scalerModel = scaler.fit(dataFrame); diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaMinMaxScalerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaMinMaxScalerExample.java index 022940fd1e67c..37fa1c5434ea6 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaMinMaxScalerExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaMinMaxScalerExample.java @@ -28,10 +28,16 @@ public class JavaMinMaxScalerExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaMinMaxScalerExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaMinMaxScalerExample") + .getOrCreate(); // $example on$ - Dataset dataFrame = spark.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt"); + Dataset dataFrame = spark + .read() + .format("libsvm") + .load("data/mllib/sample_libsvm_data.txt"); MinMaxScaler scaler = new MinMaxScaler() .setInputCol("features") .setOutputCol("scaledFeatures"); diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaNGramExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaNGramExample.java index 325b7b5874800..899815f57c84b 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaNGramExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaNGramExample.java @@ -35,7 +35,10 @@ public class JavaNGramExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaNGramExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaNGramExample") + .getOrCreate(); // $example on$ List data = Arrays.asList( diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaNaiveBayesExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaNaiveBayesExample.java index 1f24a23609ebe..50a46a5774fcc 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaNaiveBayesExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaNaiveBayesExample.java @@ -32,7 +32,10 @@ public class JavaNaiveBayesExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaNaiveBayesExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaNaiveBayesExample") + .getOrCreate(); // $example on$ // Load training data diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaNormalizerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaNormalizerExample.java index 4b3a718ea92c8..abc38f85ea774 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaNormalizerExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaNormalizerExample.java @@ -27,7 +27,10 @@ public class JavaNormalizerExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaNormalizerExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaNormalizerExample") + .getOrCreate(); // $example on$ Dataset dataFrame = diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaOneHotEncoderExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaOneHotEncoderExample.java index d6e4d21ead161..5d29e54549213 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaOneHotEncoderExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaOneHotEncoderExample.java @@ -37,7 +37,10 @@ public class JavaOneHotEncoderExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaOneHotEncoderExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaOneHotEncoderExample") + .getOrCreate(); // $example on$ List data = Arrays.asList( diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaOneVsRestExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaOneVsRestExample.java index 9cc983bd115df..e0cb752224f75 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaOneVsRestExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaOneVsRestExample.java @@ -58,7 +58,10 @@ private static class Params { public static void main(String[] args) { // parse the arguments Params params = parse(args); - SparkSession spark = SparkSession.builder().appName("JavaOneVsRestExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaOneVsRestExample") + .getOrCreate(); // $example on$ // configure the base classifier diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaPCAExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaPCAExample.java index 6b1dcb68ba304..ffa979ee013ad 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaPCAExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaPCAExample.java @@ -37,7 +37,10 @@ public class JavaPCAExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaPCAExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaPCAExample") + .getOrCreate(); // $example on$ List data = Arrays.asList( diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaPipelineExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaPipelineExample.java index 556a4573263f1..9a43189c91463 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaPipelineExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaPipelineExample.java @@ -36,7 +36,10 @@ */ public class JavaPipelineExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaPipelineExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaPipelineExample") + .getOrCreate(); // $example on$ // Prepare training documents, which are labeled. diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaPolynomialExpansionExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaPolynomialExpansionExample.java index e328454c70591..7afcd0e50cd95 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaPolynomialExpansionExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaPolynomialExpansionExample.java @@ -36,7 +36,10 @@ public class JavaPolynomialExpansionExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaPolynomialExpansionExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaPolynomialExpansionExample") + .getOrCreate(); // $example on$ PolynomialExpansion polyExpansion = new PolynomialExpansion() diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaRFormulaExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaRFormulaExample.java index 8282ce01d36f3..428067e0f7efe 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaRFormulaExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaRFormulaExample.java @@ -35,7 +35,10 @@ public class JavaRFormulaExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaRFormulaExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaRFormulaExample") + .getOrCreate(); // $example on$ StructType schema = createStructType(new StructField[]{ diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaSQLTransformerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaSQLTransformerExample.java index 492718bbdba81..2a3d62de41ab7 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaSQLTransformerExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaSQLTransformerExample.java @@ -31,7 +31,10 @@ public class JavaSQLTransformerExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaSQLTransformerExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaSQLTransformerExample") + .getOrCreate(); // $example on$ List data = Arrays.asList( diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaSimpleParamsExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaSimpleParamsExample.java index f906843640eda..0787079ba4e55 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaSimpleParamsExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaSimpleParamsExample.java @@ -40,7 +40,10 @@ public class JavaSimpleParamsExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaSimpleParamsExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaSimpleParamsExample") + .getOrCreate(); // Prepare training data. // We use LabeledPoint, which is a JavaBean. Spark SQL can convert RDDs of JavaBeans diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaStandardScalerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaStandardScalerExample.java index 10f82f223386e..08ea285a0d53d 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaStandardScalerExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaStandardScalerExample.java @@ -28,7 +28,10 @@ public class JavaStandardScalerExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaStandardScalerExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaStandardScalerExample") + .getOrCreate(); // $example on$ Dataset dataFrame = diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaStopWordsRemoverExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaStopWordsRemoverExample.java index 23ed071c9f6e5..def5994429124 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaStopWordsRemoverExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaStopWordsRemoverExample.java @@ -36,7 +36,10 @@ public class JavaStopWordsRemoverExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaStopWordsRemoverExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaStopWordsRemoverExample") + .getOrCreate(); // $example on$ StopWordsRemover remover = new StopWordsRemover() diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaStringIndexerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaStringIndexerExample.java index d4c2cf96a7cf3..7533c1835e325 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaStringIndexerExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaStringIndexerExample.java @@ -35,7 +35,10 @@ public class JavaStringIndexerExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaStringIndexerExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaStringIndexerExample") + .getOrCreate(); // $example on$ List data = Arrays.asList( diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaTfIdfExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaTfIdfExample.java index a8169917772c1..6e0753959efd6 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaTfIdfExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaTfIdfExample.java @@ -38,7 +38,10 @@ public class JavaTfIdfExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaTfIdfExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaTfIdfExample") + .getOrCreate(); // $example on$ List data = Arrays.asList( diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaTokenizerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaTokenizerExample.java index a65735a5e5143..1cc16bb60d172 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaTokenizerExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaTokenizerExample.java @@ -36,7 +36,10 @@ public class JavaTokenizerExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaTokenizerExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaTokenizerExample") + .getOrCreate(); // $example on$ List data = Arrays.asList( diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorAssemblerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorAssemblerExample.java index 9569bc2412def..41f1d8750ac40 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorAssemblerExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorAssemblerExample.java @@ -35,7 +35,10 @@ public class JavaVectorAssemblerExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaVectorAssemblerExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaVectorAssemblerExample") + .getOrCreate(); // $example on$ StructType schema = createStructType(new StructField[]{ diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorIndexerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorIndexerExample.java index 217d5a06d10a4..dd9d757dd6831 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorIndexerExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorIndexerExample.java @@ -30,7 +30,10 @@ public class JavaVectorIndexerExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaVectorIndexerExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaVectorIndexerExample") + .getOrCreate(); // $example on$ Dataset data = spark.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt"); diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorSlicerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorSlicerExample.java index 4f1ea824a3a9f..24959c0e10f2b 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorSlicerExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorSlicerExample.java @@ -37,7 +37,10 @@ public class JavaVectorSlicerExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaVectorSlicerExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaVectorSlicerExample") + .getOrCreate(); // $example on$ Attribute[] attrs = new Attribute[]{ diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaWord2VecExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaWord2VecExample.java index d9b1a79b52548..9be6e6353adcf 100644 --- a/examples/src/main/java/org/apache/spark/examples/ml/JavaWord2VecExample.java +++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaWord2VecExample.java @@ -32,7 +32,10 @@ public class JavaWord2VecExample { public static void main(String[] args) { - SparkSession spark = SparkSession.builder().appName("JavaWord2VecExample").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaWord2VecExample") + .getOrCreate(); // $example on$ // Input data: Each row is a bag of words from a sentence or document. diff --git a/examples/src/main/java/org/apache/spark/examples/sql/JavaSparkSQL.java b/examples/src/main/java/org/apache/spark/examples/sql/JavaSparkSQL.java index ec2142e756955..755b4f538104a 100644 --- a/examples/src/main/java/org/apache/spark/examples/sql/JavaSparkSQL.java +++ b/examples/src/main/java/org/apache/spark/examples/sql/JavaSparkSQL.java @@ -51,7 +51,10 @@ public void setAge(int age) { } public static void main(String[] args) throws Exception { - SparkSession spark = SparkSession.builder().appName("JavaSparkSQL").getOrCreate(); + SparkSession spark = SparkSession + .builder() + .appName("JavaSparkSQL") + .getOrCreate(); System.out.println("=== Data source: RDD ==="); // Load a text file and convert each line to a Java Bean. @@ -147,7 +150,8 @@ public String call(Row row) { // a RDD[String] storing one JSON object per string. List jsonData = Arrays.asList( "{\"name\":\"Yin\",\"address\":{\"city\":\"Columbus\",\"state\":\"Ohio\"}}"); - JavaRDD anotherPeopleRDD = spark.createDataFrame(jsonData, String.class).toJSON().javaRDD(); + JavaRDD anotherPeopleRDD = spark + .createDataFrame(jsonData, String.class).toJSON().javaRDD(); Dataset peopleFromJsonRDD = spark.read().json(anotherPeopleRDD); // Take a look at the schema of this new DataFrame. diff --git a/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java b/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java index 44f1e800fe448..57953ef74f795 100644 --- a/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java +++ b/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java @@ -115,7 +115,10 @@ class JavaSparkSessionSingleton { private static transient SparkSession instance = null; public static SparkSession getInstance(SparkConf sparkConf) { if (instance == null) { - instance = SparkSession.builder().config(sparkConf).getOrCreate(); + instance = SparkSession + .builder() + .config(sparkConf) + .getOrCreate(); } return instance; } diff --git a/examples/src/main/python/ml/als_example.py b/examples/src/main/python/ml/als_example.py index e36444f18506e..ff0829b0dd45a 100644 --- a/examples/src/main/python/ml/als_example.py +++ b/examples/src/main/python/ml/als_example.py @@ -30,7 +30,10 @@ # $example off$ if __name__ == "__main__": - spark = SparkSession.builder.appName("ALSExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("ALSExample")\ + .getOrCreate() # $example on$ lines = spark.read.text("data/mllib/als/sample_movielens_ratings.txt").rdd diff --git a/examples/src/main/python/ml/binarizer_example.py b/examples/src/main/python/ml/binarizer_example.py index 072187e64564c..4224a27dbef0c 100644 --- a/examples/src/main/python/ml/binarizer_example.py +++ b/examples/src/main/python/ml/binarizer_example.py @@ -23,7 +23,10 @@ # $example off$ if __name__ == "__main__": - spark = SparkSession.builder.appName("BinarizerExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("BinarizerExample")\ + .getOrCreate() # $example on$ continuousDataFrame = spark.createDataFrame([ diff --git a/examples/src/main/python/ml/bisecting_k_means_example.py b/examples/src/main/python/ml/bisecting_k_means_example.py index 836a89cde0d60..540a4bc3e4bf9 100644 --- a/examples/src/main/python/ml/bisecting_k_means_example.py +++ b/examples/src/main/python/ml/bisecting_k_means_example.py @@ -30,7 +30,10 @@ """ if __name__ == "__main__": - spark = SparkSession.builder.appName("PythonBisectingKMeansExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("PythonBisectingKMeansExample")\ + .getOrCreate() # $example on$ data = spark.read.text("data/mllib/kmeans_data.txt").rdd diff --git a/examples/src/main/python/ml/bucketizer_example.py b/examples/src/main/python/ml/bucketizer_example.py index 288ec62bdfbb5..8177e560ddef1 100644 --- a/examples/src/main/python/ml/bucketizer_example.py +++ b/examples/src/main/python/ml/bucketizer_example.py @@ -23,7 +23,10 @@ # $example off$ if __name__ == "__main__": - spark = SparkSession.builder.appName("BucketizerExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("BucketizerExample")\ + .getOrCreate() # $example on$ splits = [-float("inf"), -0.5, 0.0, 0.5, float("inf")] diff --git a/examples/src/main/python/ml/chisq_selector_example.py b/examples/src/main/python/ml/chisq_selector_example.py index 8f58fc28deb9b..8bafb942e0d27 100644 --- a/examples/src/main/python/ml/chisq_selector_example.py +++ b/examples/src/main/python/ml/chisq_selector_example.py @@ -24,7 +24,10 @@ # $example off$ if __name__ == "__main__": - spark = SparkSession.builder.appName("ChiSqSelectorExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("ChiSqSelectorExample")\ + .getOrCreate() # $example on$ df = spark.createDataFrame([ diff --git a/examples/src/main/python/ml/count_vectorizer_example.py b/examples/src/main/python/ml/count_vectorizer_example.py index 9dbf9959d17ef..38cfac82fbe20 100644 --- a/examples/src/main/python/ml/count_vectorizer_example.py +++ b/examples/src/main/python/ml/count_vectorizer_example.py @@ -23,7 +23,10 @@ # $example off$ if __name__ == "__main__": - spark = SparkSession.builder.appName("CountVectorizerExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("CountVectorizerExample")\ + .getOrCreate() # $example on$ # Input data: Each row is a bag of words with a ID. diff --git a/examples/src/main/python/ml/cross_validator.py b/examples/src/main/python/ml/cross_validator.py index a61d0f63d277c..a41df6cf946fb 100644 --- a/examples/src/main/python/ml/cross_validator.py +++ b/examples/src/main/python/ml/cross_validator.py @@ -35,7 +35,10 @@ """ if __name__ == "__main__": - spark = SparkSession.builder.appName("CrossValidatorExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("CrossValidatorExample")\ + .getOrCreate() # $example on$ # Prepare training documents, which are labeled. training = spark.createDataFrame([ diff --git a/examples/src/main/python/ml/dataframe_example.py b/examples/src/main/python/ml/dataframe_example.py index b3e671038eb7f..a7d8b9056dffe 100644 --- a/examples/src/main/python/ml/dataframe_example.py +++ b/examples/src/main/python/ml/dataframe_example.py @@ -33,7 +33,10 @@ if len(sys.argv) > 2: print("Usage: dataframe_example.py ", file=sys.stderr) exit(-1) - spark = SparkSession.builder.appName("DataFrameExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("DataFrameExample")\ + .getOrCreate() if len(sys.argv) == 2: input = sys.argv[1] else: diff --git a/examples/src/main/python/ml/dct_example.py b/examples/src/main/python/ml/dct_example.py index 1bf8fc6d140c9..e36fcdeaeed28 100644 --- a/examples/src/main/python/ml/dct_example.py +++ b/examples/src/main/python/ml/dct_example.py @@ -24,7 +24,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("DCTExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("DCTExample")\ + .getOrCreate() # $example on$ df = spark.createDataFrame([ diff --git a/examples/src/main/python/ml/decision_tree_classification_example.py b/examples/src/main/python/ml/decision_tree_classification_example.py index d2318e24369d8..9b40b701ecaaf 100644 --- a/examples/src/main/python/ml/decision_tree_classification_example.py +++ b/examples/src/main/python/ml/decision_tree_classification_example.py @@ -29,7 +29,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("decision_tree_classification_example").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("decision_tree_classification_example")\ + .getOrCreate() # $example on$ # Load the data stored in LIBSVM format as a DataFrame. diff --git a/examples/src/main/python/ml/decision_tree_regression_example.py b/examples/src/main/python/ml/decision_tree_regression_example.py index 9e8cb382a9bfe..b734d4974a4f6 100644 --- a/examples/src/main/python/ml/decision_tree_regression_example.py +++ b/examples/src/main/python/ml/decision_tree_regression_example.py @@ -29,7 +29,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("decision_tree_classification_example").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("decision_tree_classification_example")\ + .getOrCreate() # $example on$ # Load the data stored in LIBSVM format as a DataFrame. diff --git a/examples/src/main/python/ml/elementwise_product_example.py b/examples/src/main/python/ml/elementwise_product_example.py index 6fa641b772d95..41727edcdb09e 100644 --- a/examples/src/main/python/ml/elementwise_product_example.py +++ b/examples/src/main/python/ml/elementwise_product_example.py @@ -24,7 +24,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("ElementwiseProductExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("ElementwiseProductExample")\ + .getOrCreate() # $example on$ data = [(Vectors.dense([1.0, 2.0, 3.0]),), (Vectors.dense([4.0, 5.0, 6.0]),)] diff --git a/examples/src/main/python/ml/estimator_transformer_param_example.py b/examples/src/main/python/ml/estimator_transformer_param_example.py index 4993b5a9846c2..0fcae0e3fc225 100644 --- a/examples/src/main/python/ml/estimator_transformer_param_example.py +++ b/examples/src/main/python/ml/estimator_transformer_param_example.py @@ -26,7 +26,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("EstimatorTransformerParamExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("EstimatorTransformerParamExample")\ + .getOrCreate() # $example on$ # Prepare training data from a list of (label, features) tuples. diff --git a/examples/src/main/python/ml/gradient_boosted_tree_classifier_example.py b/examples/src/main/python/ml/gradient_boosted_tree_classifier_example.py index b09ad41da313c..50026d7b7e3e8 100644 --- a/examples/src/main/python/ml/gradient_boosted_tree_classifier_example.py +++ b/examples/src/main/python/ml/gradient_boosted_tree_classifier_example.py @@ -29,7 +29,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("gradient_boosted_tree_classifier_example").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("gradient_boosted_tree_classifier_example")\ + .getOrCreate() # $example on$ # Load and parse the data file, converting it to a DataFrame. diff --git a/examples/src/main/python/ml/gradient_boosted_tree_regressor_example.py b/examples/src/main/python/ml/gradient_boosted_tree_regressor_example.py index caa7cfc4e11f5..5dd2272748d70 100644 --- a/examples/src/main/python/ml/gradient_boosted_tree_regressor_example.py +++ b/examples/src/main/python/ml/gradient_boosted_tree_regressor_example.py @@ -29,7 +29,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("gradient_boosted_tree_regressor_example").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("gradient_boosted_tree_regressor_example")\ + .getOrCreate() # $example on$ # Load and parse the data file, converting it to a DataFrame. diff --git a/examples/src/main/python/ml/index_to_string_example.py b/examples/src/main/python/ml/index_to_string_example.py index dd04b2c4b0bfd..523caac00c18a 100644 --- a/examples/src/main/python/ml/index_to_string_example.py +++ b/examples/src/main/python/ml/index_to_string_example.py @@ -23,7 +23,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("IndexToStringExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("IndexToStringExample")\ + .getOrCreate() # $example on$ df = spark.createDataFrame( diff --git a/examples/src/main/python/ml/kmeans_example.py b/examples/src/main/python/ml/kmeans_example.py index 7d9d80e6452bd..73823969554fa 100644 --- a/examples/src/main/python/ml/kmeans_example.py +++ b/examples/src/main/python/ml/kmeans_example.py @@ -49,7 +49,10 @@ def parseVector(row): path = sys.argv[1] k = sys.argv[2] - spark = SparkSession.builder.appName("PythonKMeansExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("PythonKMeansExample")\ + .getOrCreate() lines = spark.read.text(path).rdd data = lines.map(parseVector) diff --git a/examples/src/main/python/ml/linear_regression_with_elastic_net.py b/examples/src/main/python/ml/linear_regression_with_elastic_net.py index 99b7f7fe99de6..620ab5b87e594 100644 --- a/examples/src/main/python/ml/linear_regression_with_elastic_net.py +++ b/examples/src/main/python/ml/linear_regression_with_elastic_net.py @@ -23,7 +23,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("LinearRegressionWithElasticNet").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("LinearRegressionWithElasticNet")\ + .getOrCreate() # $example on$ # Load training data diff --git a/examples/src/main/python/ml/logistic_regression_with_elastic_net.py b/examples/src/main/python/ml/logistic_regression_with_elastic_net.py index 0d7112e72354f..33d0689f75cd5 100644 --- a/examples/src/main/python/ml/logistic_regression_with_elastic_net.py +++ b/examples/src/main/python/ml/logistic_regression_with_elastic_net.py @@ -23,7 +23,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("LogisticRegressionWithElasticNet").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("LogisticRegressionWithElasticNet")\ + .getOrCreate() # $example on$ # Load training data diff --git a/examples/src/main/python/ml/max_abs_scaler_example.py b/examples/src/main/python/ml/max_abs_scaler_example.py index 1cb95a98f08dc..ab91198b083d1 100644 --- a/examples/src/main/python/ml/max_abs_scaler_example.py +++ b/examples/src/main/python/ml/max_abs_scaler_example.py @@ -23,7 +23,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("MaxAbsScalerExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("MaxAbsScalerExample")\ + .getOrCreate() # $example on$ dataFrame = spark.read.format("libsvm").load("data/mllib/sample_libsvm_data.txt") diff --git a/examples/src/main/python/ml/min_max_scaler_example.py b/examples/src/main/python/ml/min_max_scaler_example.py index 8d91a59e2b0bd..e3e7bc205b1ec 100644 --- a/examples/src/main/python/ml/min_max_scaler_example.py +++ b/examples/src/main/python/ml/min_max_scaler_example.py @@ -23,7 +23,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("MinMaxScalerExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("MinMaxScalerExample")\ + .getOrCreate() # $example on$ dataFrame = spark.read.format("libsvm").load("data/mllib/sample_libsvm_data.txt") diff --git a/examples/src/main/python/ml/n_gram_example.py b/examples/src/main/python/ml/n_gram_example.py index b7fecf0d685c0..9ac07f2c8ee20 100644 --- a/examples/src/main/python/ml/n_gram_example.py +++ b/examples/src/main/python/ml/n_gram_example.py @@ -23,7 +23,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("NGramExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("NGramExample")\ + .getOrCreate() # $example on$ wordDataFrame = spark.createDataFrame([ diff --git a/examples/src/main/python/ml/naive_bayes_example.py b/examples/src/main/python/ml/naive_bayes_example.py index e37035542c7e8..89255a2bae64c 100644 --- a/examples/src/main/python/ml/naive_bayes_example.py +++ b/examples/src/main/python/ml/naive_bayes_example.py @@ -24,7 +24,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("naive_bayes_example").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("naive_bayes_example")\ + .getOrCreate() # $example on$ # Load training data diff --git a/examples/src/main/python/ml/normalizer_example.py b/examples/src/main/python/ml/normalizer_example.py index ae2553761928c..19012f51f4023 100644 --- a/examples/src/main/python/ml/normalizer_example.py +++ b/examples/src/main/python/ml/normalizer_example.py @@ -23,7 +23,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("NormalizerExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("NormalizerExample")\ + .getOrCreate() # $example on$ dataFrame = spark.read.format("libsvm").load("data/mllib/sample_libsvm_data.txt") diff --git a/examples/src/main/python/ml/onehot_encoder_example.py b/examples/src/main/python/ml/onehot_encoder_example.py index 9acc363dc9e25..b9fceef68e703 100644 --- a/examples/src/main/python/ml/onehot_encoder_example.py +++ b/examples/src/main/python/ml/onehot_encoder_example.py @@ -23,7 +23,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("OneHotEncoderExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("OneHotEncoderExample")\ + .getOrCreate() # $example on$ df = spark.createDataFrame([ diff --git a/examples/src/main/python/ml/pca_example.py b/examples/src/main/python/ml/pca_example.py index adab151734aa8..f1b3cdec7bd77 100644 --- a/examples/src/main/python/ml/pca_example.py +++ b/examples/src/main/python/ml/pca_example.py @@ -24,7 +24,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("PCAExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("PCAExample")\ + .getOrCreate() # $example on$ data = [(Vectors.sparse(5, [(1, 1.0), (3, 7.0)]),), diff --git a/examples/src/main/python/ml/pipeline_example.py b/examples/src/main/python/ml/pipeline_example.py index ed9765d96165e..bd10cfd7a252b 100644 --- a/examples/src/main/python/ml/pipeline_example.py +++ b/examples/src/main/python/ml/pipeline_example.py @@ -27,7 +27,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("PipelineExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("PipelineExample")\ + .getOrCreate() # $example on$ # Prepare training documents from a list of (id, text, label) tuples. diff --git a/examples/src/main/python/ml/polynomial_expansion_example.py b/examples/src/main/python/ml/polynomial_expansion_example.py index 328b559320f85..08882bcb256f4 100644 --- a/examples/src/main/python/ml/polynomial_expansion_example.py +++ b/examples/src/main/python/ml/polynomial_expansion_example.py @@ -24,7 +24,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("PolynomialExpansionExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("PolynomialExpansionExample")\ + .getOrCreate() # $example on$ df = spark\ diff --git a/examples/src/main/python/ml/random_forest_classifier_example.py b/examples/src/main/python/ml/random_forest_classifier_example.py index b0a93e050c54b..c618eaf60c2e9 100644 --- a/examples/src/main/python/ml/random_forest_classifier_example.py +++ b/examples/src/main/python/ml/random_forest_classifier_example.py @@ -29,7 +29,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("random_forest_classifier_example").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("random_forest_classifier_example")\ + .getOrCreate() # $example on$ # Load and parse the data file, converting it to a DataFrame. diff --git a/examples/src/main/python/ml/random_forest_regressor_example.py b/examples/src/main/python/ml/random_forest_regressor_example.py index 4bb84f0de8ff3..3a793737dba89 100644 --- a/examples/src/main/python/ml/random_forest_regressor_example.py +++ b/examples/src/main/python/ml/random_forest_regressor_example.py @@ -29,7 +29,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("random_forest_regressor_example").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("random_forest_regressor_example")\ + .getOrCreate() # $example on$ # Load and parse the data file, converting it to a DataFrame. diff --git a/examples/src/main/python/ml/rformula_example.py b/examples/src/main/python/ml/rformula_example.py index 45cc116ac2f61..d5df3ce4f5915 100644 --- a/examples/src/main/python/ml/rformula_example.py +++ b/examples/src/main/python/ml/rformula_example.py @@ -23,7 +23,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("RFormulaExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("RFormulaExample")\ + .getOrCreate() # $example on$ dataset = spark.createDataFrame( diff --git a/examples/src/main/python/ml/simple_text_classification_pipeline.py b/examples/src/main/python/ml/simple_text_classification_pipeline.py index 3600c12211d05..886f43c0b08e8 100644 --- a/examples/src/main/python/ml/simple_text_classification_pipeline.py +++ b/examples/src/main/python/ml/simple_text_classification_pipeline.py @@ -33,7 +33,10 @@ if __name__ == "__main__": - spark = SparkSession.builder.appName("SimpleTextClassificationPipeline").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("SimpleTextClassificationPipeline")\ + .getOrCreate() # Prepare training documents, which are labeled. training = spark.createDataFrame([ diff --git a/examples/src/main/python/ml/sql_transformer.py b/examples/src/main/python/ml/sql_transformer.py index 26045db4be6bc..0bf8f35720c95 100644 --- a/examples/src/main/python/ml/sql_transformer.py +++ b/examples/src/main/python/ml/sql_transformer.py @@ -23,7 +23,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("SQLTransformerExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("SQLTransformerExample")\ + .getOrCreate() # $example on$ df = spark.createDataFrame([ diff --git a/examples/src/main/python/ml/standard_scaler_example.py b/examples/src/main/python/ml/standard_scaler_example.py index c50804f6bf4e0..c0027480e69b3 100644 --- a/examples/src/main/python/ml/standard_scaler_example.py +++ b/examples/src/main/python/ml/standard_scaler_example.py @@ -23,7 +23,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("StandardScalerExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("StandardScalerExample")\ + .getOrCreate() # $example on$ dataFrame = spark.read.format("libsvm").load("data/mllib/sample_libsvm_data.txt") diff --git a/examples/src/main/python/ml/stopwords_remover_example.py b/examples/src/main/python/ml/stopwords_remover_example.py index 57362673df786..395fdeffc5379 100644 --- a/examples/src/main/python/ml/stopwords_remover_example.py +++ b/examples/src/main/python/ml/stopwords_remover_example.py @@ -23,7 +23,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("StopWordsRemoverExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("StopWordsRemoverExample")\ + .getOrCreate() # $example on$ sentenceData = spark.createDataFrame([ diff --git a/examples/src/main/python/ml/string_indexer_example.py b/examples/src/main/python/ml/string_indexer_example.py index aacd4f999bb1b..a328e040f5636 100644 --- a/examples/src/main/python/ml/string_indexer_example.py +++ b/examples/src/main/python/ml/string_indexer_example.py @@ -23,7 +23,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("StringIndexerExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("StringIndexerExample")\ + .getOrCreate() # $example on$ df = spark.createDataFrame( diff --git a/examples/src/main/python/ml/tf_idf_example.py b/examples/src/main/python/ml/tf_idf_example.py index 25df8166efc2e..fb4ad992fb809 100644 --- a/examples/src/main/python/ml/tf_idf_example.py +++ b/examples/src/main/python/ml/tf_idf_example.py @@ -23,7 +23,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("TfIdfExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("TfIdfExample")\ + .getOrCreate() # $example on$ sentenceData = spark.createDataFrame([ diff --git a/examples/src/main/python/ml/tokenizer_example.py b/examples/src/main/python/ml/tokenizer_example.py index 5be4b4cfe3a52..e61ec920d2281 100644 --- a/examples/src/main/python/ml/tokenizer_example.py +++ b/examples/src/main/python/ml/tokenizer_example.py @@ -23,7 +23,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("TokenizerExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("TokenizerExample")\ + .getOrCreate() # $example on$ sentenceDataFrame = spark.createDataFrame([ diff --git a/examples/src/main/python/ml/train_validation_split.py b/examples/src/main/python/ml/train_validation_split.py index 2e43a0f8aec54..5f5c52aca8c42 100644 --- a/examples/src/main/python/ml/train_validation_split.py +++ b/examples/src/main/python/ml/train_validation_split.py @@ -31,7 +31,10 @@ """ if __name__ == "__main__": - spark = SparkSession.builder.appName("TrainValidationSplit").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("TrainValidationSplit")\ + .getOrCreate() # $example on$ # Prepare training and test data. data = spark.read.format("libsvm")\ diff --git a/examples/src/main/python/ml/vector_assembler_example.py b/examples/src/main/python/ml/vector_assembler_example.py index 019a9ea6f7503..b955ff00a8195 100644 --- a/examples/src/main/python/ml/vector_assembler_example.py +++ b/examples/src/main/python/ml/vector_assembler_example.py @@ -24,7 +24,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("VectorAssemblerExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("VectorAssemblerExample")\ + .getOrCreate() # $example on$ dataset = spark.createDataFrame( diff --git a/examples/src/main/python/ml/vector_indexer_example.py b/examples/src/main/python/ml/vector_indexer_example.py index 3cf5b8ebf17ee..9b00e0f84136c 100644 --- a/examples/src/main/python/ml/vector_indexer_example.py +++ b/examples/src/main/python/ml/vector_indexer_example.py @@ -23,7 +23,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("VectorIndexerExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("VectorIndexerExample")\ + .getOrCreate() # $example on$ data = spark.read.format("libsvm").load("data/mllib/sample_libsvm_data.txt") diff --git a/examples/src/main/python/ml/vector_slicer_example.py b/examples/src/main/python/ml/vector_slicer_example.py index 0531bcdb06675..b833a894eb841 100644 --- a/examples/src/main/python/ml/vector_slicer_example.py +++ b/examples/src/main/python/ml/vector_slicer_example.py @@ -25,7 +25,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("VectorSlicerExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("VectorSlicerExample")\ + .getOrCreate() # $example on$ df = spark.createDataFrame([ diff --git a/examples/src/main/python/ml/word2vec_example.py b/examples/src/main/python/ml/word2vec_example.py index 6766a7b6aa454..66500bee152f7 100644 --- a/examples/src/main/python/ml/word2vec_example.py +++ b/examples/src/main/python/ml/word2vec_example.py @@ -23,7 +23,10 @@ from pyspark.sql import SparkSession if __name__ == "__main__": - spark = SparkSession.builder.appName("Word2VecExample").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("Word2VecExample")\ + .getOrCreate() # $example on$ # Input data: Each row is a bag of words from a sentence or document. diff --git a/examples/src/main/python/mllib/binary_classification_metrics_example.py b/examples/src/main/python/mllib/binary_classification_metrics_example.py index 8f0fc9d45da2b..daf000e38dcd0 100644 --- a/examples/src/main/python/mllib/binary_classification_metrics_example.py +++ b/examples/src/main/python/mllib/binary_classification_metrics_example.py @@ -18,20 +18,25 @@ Binary Classification Metrics Example. """ from __future__ import print_function -from pyspark import SparkContext +from pyspark.sql import SparkSession # $example on$ from pyspark.mllib.classification import LogisticRegressionWithLBFGS from pyspark.mllib.evaluation import BinaryClassificationMetrics -from pyspark.mllib.util import MLUtils +from pyspark.mllib.regression import LabeledPoint # $example off$ if __name__ == "__main__": - sc = SparkContext(appName="BinaryClassificationMetricsExample") + spark = SparkSession\ + .builder\ + .appName("BinaryClassificationMetricsExample")\ + .getOrCreate() # $example on$ # Several of the methods available in scala are currently missing from pyspark # Load training data in LIBSVM format - data = MLUtils.loadLibSVMFile(sc, "data/mllib/sample_binary_classification_data.txt") + data = spark\ + .read.format("libsvm").load("data/mllib/sample_binary_classification_data.txt")\ + .rdd.map(lambda row: LabeledPoint(row[0], row[1])) # Split data into training (60%) and test (40%) training, test = data.randomSplit([0.6, 0.4], seed=11L) @@ -53,4 +58,4 @@ print("Area under ROC = %s" % metrics.areaUnderROC) # $example off$ - sc.stop() + spark.stop() diff --git a/examples/src/main/python/sql.py b/examples/src/main/python/sql.py index 59a46cb283c49..5594223465f87 100644 --- a/examples/src/main/python/sql.py +++ b/examples/src/main/python/sql.py @@ -25,7 +25,10 @@ if __name__ == "__main__": - spark = SparkSession.builder.appName("PythonSQL").getOrCreate() + spark = SparkSession\ + .builder\ + .appName("PythonSQL")\ + .getOrCreate() # A list of Rows. Infer schema from the first row, create a DataFrame and print the schema rows = [Row(name="John", age=19), Row(name="Smith", age=23), Row(name="Sarah", age=18)] diff --git a/examples/src/main/python/streaming/sql_network_wordcount.py b/examples/src/main/python/streaming/sql_network_wordcount.py index 588cbfee14baf..f8801d4ea63e2 100644 --- a/examples/src/main/python/streaming/sql_network_wordcount.py +++ b/examples/src/main/python/streaming/sql_network_wordcount.py @@ -38,8 +38,10 @@ def getSparkSessionInstance(sparkConf): if ('sparkSessionSingletonInstance' not in globals()): - globals()['sparkSessionSingletonInstance'] =\ - SparkSession.builder.config(conf=sparkConf).getOrCreate() + globals()['sparkSessionSingletonInstance'] = SparkSession\ + .builder\ + .config(conf=sparkConf)\ + .getOrCreate() return globals()['sparkSessionSingletonInstance'] diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/AFTSurvivalRegressionExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/AFTSurvivalRegressionExample.scala index 3795af83094a6..2b224d50a0a3c 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/AFTSurvivalRegressionExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/AFTSurvivalRegressionExample.scala @@ -30,7 +30,10 @@ import org.apache.spark.sql.SparkSession object AFTSurvivalRegressionExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("AFTSurvivalRegressionExample").getOrCreate() + val spark = SparkSession + .builder + .appName("AFTSurvivalRegressionExample") + .getOrCreate() // $example on$ val training = spark.createDataFrame(Seq( diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/ALSExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/ALSExample.scala index 41750ca779146..7c1cfe293717a 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/ALSExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/ALSExample.scala @@ -42,7 +42,10 @@ object ALSExample { // $example off$ def main(args: Array[String]) { - val spark = SparkSession.builder.appName("ALSExample").getOrCreate() + val spark = SparkSession + .builder + .appName("ALSExample") + .getOrCreate() import spark.implicits._ // $example on$ diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/BinarizerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/BinarizerExample.scala index 93c153f923232..82bc14789b461 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/BinarizerExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/BinarizerExample.scala @@ -25,7 +25,10 @@ import org.apache.spark.sql.{DataFrame, SparkSession} object BinarizerExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("BinarizerExample").getOrCreate() + val spark = SparkSession + .builder + .appName("BinarizerExample") + .getOrCreate() // $example on$ val data = Array((0, 0.1), (1, 0.8), (2, 0.2)) val dataFrame: DataFrame = spark.createDataFrame(data).toDF("label", "feature") diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/BucketizerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/BucketizerExample.scala index 779ad33dbda0a..38cce34bb5091 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/BucketizerExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/BucketizerExample.scala @@ -25,7 +25,10 @@ import org.apache.spark.sql.SparkSession object BucketizerExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("BucketizerExample").getOrCreate() + val spark = SparkSession + .builder + .appName("BucketizerExample") + .getOrCreate() // $example on$ val splits = Array(Double.NegativeInfinity, -0.5, 0.0, 0.5, Double.PositiveInfinity) diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/ChiSqSelectorExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/ChiSqSelectorExample.scala index 84ca1f0b564d2..80f50cd3556af 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/ChiSqSelectorExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/ChiSqSelectorExample.scala @@ -26,7 +26,10 @@ import org.apache.spark.sql.SparkSession object ChiSqSelectorExample { def main(args: Array[String]) { - val spark = SparkSession.builder.appName("ChiSqSelectorExample").getOrCreate() + val spark = SparkSession + .builder + .appName("ChiSqSelectorExample") + .getOrCreate() import spark.implicits._ // $example on$ diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/CountVectorizerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/CountVectorizerExample.scala index 9ab43a48bff8e..51aa5179fa4a8 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/CountVectorizerExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/CountVectorizerExample.scala @@ -25,7 +25,10 @@ import org.apache.spark.sql.SparkSession object CountVectorizerExample { def main(args: Array[String]) { - val spark = SparkSession.builder.appName("CounterVectorizerExample").getOrCreate() + val spark = SparkSession + .builder + .appName("CounterVectorizerExample") + .getOrCreate() // $example on$ val df = spark.createDataFrame(Seq( diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/DCTExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/DCTExample.scala index b415333c710c7..5a888b15eb1c9 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/DCTExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/DCTExample.scala @@ -26,7 +26,10 @@ import org.apache.spark.sql.SparkSession object DCTExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("DCTExample").getOrCreate() + val spark = SparkSession + .builder + .appName("DCTExample") + .getOrCreate() // $example on$ val data = Seq( diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/DataFrameExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/DataFrameExample.scala index 2f892f8d72c1a..6cb81cde6f795 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/DataFrameExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/DataFrameExample.scala @@ -61,7 +61,10 @@ object DataFrameExample { } def run(params: Params) { - val spark = SparkSession.builder.appName(s"DataFrameExample with $params").getOrCreate() + val spark = SparkSession + .builder + .appName(s"DataFrameExample with $params") + .getOrCreate() // Load input data println(s"Loading LIBSVM file with UDT from ${params.input}.") diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeClassificationExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeClassificationExample.scala index a0a2e1fb33dce..7f6c8de9679f9 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeClassificationExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeClassificationExample.scala @@ -29,7 +29,10 @@ import org.apache.spark.sql.SparkSession object DecisionTreeClassificationExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("DecisionTreeClassificationExample").getOrCreate() + val spark = SparkSession + .builder + .appName("DecisionTreeClassificationExample") + .getOrCreate() // $example on$ // Load the data stored in LIBSVM format as a DataFrame. val data = spark.read.format("libsvm").load("data/mllib/sample_libsvm_data.txt") diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeExample.scala index cea1d801aa73a..eadb02ab0d7f7 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeExample.scala @@ -167,7 +167,9 @@ object DecisionTreeExample { testInput: String, algo: String, fracTest: Double): (DataFrame, DataFrame) = { - val spark = SparkSession.builder.getOrCreate() + val spark = SparkSession + .builder + .getOrCreate() // Load training data val origExamples: DataFrame = loadData(spark, input, dataFormat) diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeRegressionExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeRegressionExample.scala index 26b52d0489e1c..799070ef47da6 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeRegressionExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeRegressionExample.scala @@ -29,7 +29,10 @@ import org.apache.spark.sql.SparkSession object DecisionTreeRegressionExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("DecisionTreeRegressionExample").getOrCreate() + val spark = SparkSession + .builder + .appName("DecisionTreeRegressionExample") + .getOrCreate() // $example on$ // Load the data stored in LIBSVM format as a DataFrame. diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/DeveloperApiExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/DeveloperApiExample.scala index 2aa1ab1ec855f..a522d2127edca 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/DeveloperApiExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/DeveloperApiExample.scala @@ -37,7 +37,10 @@ import org.apache.spark.sql.{Dataset, Row, SparkSession} object DeveloperApiExample { def main(args: Array[String]) { - val spark = SparkSession.builder.appName("DeveloperApiExample").getOrCreate() + val spark = SparkSession + .builder + .appName("DeveloperApiExample") + .getOrCreate() import spark.implicits._ // Prepare training data. diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/ElementwiseProductExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/ElementwiseProductExample.scala index f289c28df9b53..b99b76e58cef7 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/ElementwiseProductExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/ElementwiseProductExample.scala @@ -26,7 +26,10 @@ import org.apache.spark.sql.SparkSession object ElementwiseProductExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("ElementwiseProductExample").getOrCreate() + val spark = SparkSession + .builder + .appName("ElementwiseProductExample") + .getOrCreate() // $example on$ // Create some vector data; also works for sparse vectors diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/EstimatorTransformerParamExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/EstimatorTransformerParamExample.scala index 91076ccbc14d4..972241e76922e 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/EstimatorTransformerParamExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/EstimatorTransformerParamExample.scala @@ -29,7 +29,10 @@ import org.apache.spark.sql.SparkSession object EstimatorTransformerParamExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("EstimatorTransformerParamExample").getOrCreate() + val spark = SparkSession + .builder + .appName("EstimatorTransformerParamExample") + .getOrCreate() // $example on$ // Prepare training data from a list of (label, features) tuples. diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/GradientBoostedTreeClassifierExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/GradientBoostedTreeClassifierExample.scala index 412c54db7d5c1..b6a8baba2d95f 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/GradientBoostedTreeClassifierExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/GradientBoostedTreeClassifierExample.scala @@ -28,7 +28,10 @@ import org.apache.spark.sql.SparkSession object GradientBoostedTreeClassifierExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("GradientBoostedTreeClassifierExample").getOrCreate() + val spark = SparkSession + .builder + .appName("GradientBoostedTreeClassifierExample") + .getOrCreate() // $example on$ // Load and parse the data file, converting it to a DataFrame. diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/GradientBoostedTreeRegressorExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/GradientBoostedTreeRegressorExample.scala index fd43553cc69df..62285b83cbb9d 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/GradientBoostedTreeRegressorExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/GradientBoostedTreeRegressorExample.scala @@ -28,7 +28,10 @@ import org.apache.spark.sql.SparkSession object GradientBoostedTreeRegressorExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("GradientBoostedTreeRegressorExample").getOrCreate() + val spark = SparkSession + .builder + .appName("GradientBoostedTreeRegressorExample") + .getOrCreate() // $example on$ // Load and parse the data file, converting it to a DataFrame. diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/IndexToStringExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/IndexToStringExample.scala index d873618726c6e..950733831c3d5 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/IndexToStringExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/IndexToStringExample.scala @@ -25,7 +25,10 @@ import org.apache.spark.sql.SparkSession object IndexToStringExample { def main(args: Array[String]) { - val spark = SparkSession.builder.appName("IndexToStringExample").getOrCreate() + val spark = SparkSession + .builder + .appName("IndexToStringExample") + .getOrCreate() // $example on$ val df = spark.createDataFrame(Seq( diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/KMeansExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/KMeansExample.scala index d2573fad3596f..2abd588c6f0e4 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/KMeansExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/KMeansExample.scala @@ -36,7 +36,10 @@ object KMeansExample { def main(args: Array[String]): Unit = { // Creates a Spark context and a SQL context - val spark = SparkSession.builder.appName(s"${this.getClass.getSimpleName}").getOrCreate() + val spark = SparkSession + .builder + .appName(s"${this.getClass.getSimpleName}") + .getOrCreate() // $example on$ // Crates a DataFrame diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/LDAExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/LDAExample.scala index c23adee1a3ead..c2920f6a5d4a1 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/LDAExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/LDAExample.scala @@ -40,7 +40,10 @@ object LDAExample { val input = "data/mllib/sample_lda_data.txt" // Creates a Spark context and a SQL context - val spark = SparkSession.builder.appName(s"${this.getClass.getSimpleName}").getOrCreate() + val spark = SparkSession + .builder + .appName(s"${this.getClass.getSimpleName}") + .getOrCreate() // $example on$ // Loads data diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/LinearRegressionWithElasticNetExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/LinearRegressionWithElasticNetExample.scala index cb6e2492f513f..94cf2866238b9 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/LinearRegressionWithElasticNetExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/LinearRegressionWithElasticNetExample.scala @@ -26,7 +26,10 @@ import org.apache.spark.sql.SparkSession object LinearRegressionWithElasticNetExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("LinearRegressionWithElasticNetExample").getOrCreate() + val spark = SparkSession + .builder + .appName("LinearRegressionWithElasticNetExample") + .getOrCreate() // $example on$ // Load training data diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/LogisticRegressionSummaryExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/LogisticRegressionSummaryExample.scala index 50670d7b38416..cd8775c942162 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/LogisticRegressionSummaryExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/LogisticRegressionSummaryExample.scala @@ -27,7 +27,10 @@ import org.apache.spark.sql.functions.max object LogisticRegressionSummaryExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("LogisticRegressionSummaryExample").getOrCreate() + val spark = SparkSession + .builder + .appName("LogisticRegressionSummaryExample") + .getOrCreate() import spark.implicits._ // Load training data diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/MaxAbsScalerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/MaxAbsScalerExample.scala index 896d8fadbe67c..572adce657081 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/MaxAbsScalerExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/MaxAbsScalerExample.scala @@ -24,7 +24,10 @@ import org.apache.spark.sql.SparkSession object MaxAbsScalerExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("MaxAbsScalerExample").getOrCreate() + val spark = SparkSession + .builder + .appName("MaxAbsScalerExample") + .getOrCreate() // $example on$ val dataFrame = spark.read.format("libsvm").load("data/mllib/sample_libsvm_data.txt") diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/MinMaxScalerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/MinMaxScalerExample.scala index bcdca0fa0440d..d728019a621d4 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/MinMaxScalerExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/MinMaxScalerExample.scala @@ -25,7 +25,10 @@ import org.apache.spark.sql.SparkSession object MinMaxScalerExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("MinMaxScalerExample").getOrCreate() + val spark = SparkSession + .builder + .appName("MinMaxScalerExample") + .getOrCreate() // $example on$ val dataFrame = spark.read.format("libsvm").load("data/mllib/sample_libsvm_data.txt") diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/MultilayerPerceptronClassifierExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/MultilayerPerceptronClassifierExample.scala index a11fe1b4b2627..0e780fb7d342e 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/MultilayerPerceptronClassifierExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/MultilayerPerceptronClassifierExample.scala @@ -30,7 +30,10 @@ import org.apache.spark.sql.SparkSession object MultilayerPerceptronClassifierExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("MultilayerPerceptronClassifierExample").getOrCreate() + val spark = SparkSession + .builder + .appName("MultilayerPerceptronClassifierExample") + .getOrCreate() // $example on$ // Load the data stored in LIBSVM format as a DataFrame. diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/NGramExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/NGramExample.scala index 1b71a398905e0..e0b52e7a367fc 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/NGramExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/NGramExample.scala @@ -25,7 +25,10 @@ import org.apache.spark.sql.SparkSession object NGramExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("NGramExample").getOrCreate() + val spark = SparkSession + .builder + .appName("NGramExample") + .getOrCreate() // $example on$ val wordDataFrame = spark.createDataFrame(Seq( diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/NaiveBayesExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/NaiveBayesExample.scala index 8d54555cd3c65..90cdebfcb08de 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/NaiveBayesExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/NaiveBayesExample.scala @@ -26,7 +26,10 @@ import org.apache.spark.sql.SparkSession object NaiveBayesExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("NaiveBayesExample").getOrCreate() + val spark = SparkSession + .builder + .appName("NaiveBayesExample") + .getOrCreate() // $example on$ // Load the data stored in LIBSVM format as a DataFrame. val data = spark.read.format("libsvm").load("data/mllib/sample_libsvm_data.txt") diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/NormalizerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/NormalizerExample.scala index 4622d69ef98cc..75ba33a7e7fc1 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/NormalizerExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/NormalizerExample.scala @@ -25,7 +25,10 @@ import org.apache.spark.sql.SparkSession object NormalizerExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("NormalizerExample").getOrCreate() + val spark = SparkSession + .builder + .appName("NormalizerExample") + .getOrCreate() // $example on$ val dataFrame = spark.read.format("libsvm").load("data/mllib/sample_libsvm_data.txt") diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/OneHotEncoderExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/OneHotEncoderExample.scala index 338436100c5ce..4aa649b1332c6 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/OneHotEncoderExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/OneHotEncoderExample.scala @@ -25,7 +25,10 @@ import org.apache.spark.sql.SparkSession object OneHotEncoderExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("OneHotEncoderExample").getOrCreate() + val spark = SparkSession + .builder + .appName("OneHotEncoderExample") + .getOrCreate() // $example on$ val df = spark.createDataFrame(Seq( diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/OneVsRestExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/OneVsRestExample.scala index e2351c682d753..fc73ae07ff6c6 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/OneVsRestExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/OneVsRestExample.scala @@ -109,7 +109,10 @@ object OneVsRestExample { } private def run(params: Params) { - val spark = SparkSession.builder.appName(s"OneVsRestExample with $params").getOrCreate() + val spark = SparkSession + .builder + .appName(s"OneVsRestExample with $params") + .getOrCreate() // $example on$ val inputData = spark.read.format("libsvm").load(params.input) diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/PCAExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/PCAExample.scala index 14394d562468b..7927323b42852 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/PCAExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/PCAExample.scala @@ -26,7 +26,10 @@ import org.apache.spark.sql.SparkSession object PCAExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("PCAExample").getOrCreate() + val spark = SparkSession + .builder + .appName("PCAExample") + .getOrCreate() // $example on$ val data = Array( diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/PipelineExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/PipelineExample.scala index 61b34aebd9c32..e5e916ac166fb 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/PipelineExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/PipelineExample.scala @@ -30,7 +30,10 @@ import org.apache.spark.sql.SparkSession object PipelineExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("PipelineExample").getOrCreate() + val spark = SparkSession + .builder + .appName("PipelineExample") + .getOrCreate() // $example on$ // Prepare training documents from a list of (id, text, label) tuples. diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/PolynomialExpansionExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/PolynomialExpansionExample.scala index 4d8c672a55c34..94b17a3cd7064 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/PolynomialExpansionExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/PolynomialExpansionExample.scala @@ -26,7 +26,10 @@ import org.apache.spark.sql.SparkSession object PolynomialExpansionExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("PolynomialExpansionExample").getOrCreate() + val spark = SparkSession + .builder + .appName("PolynomialExpansionExample") + .getOrCreate() // $example on$ val data = Array( diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/QuantileDiscretizerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/QuantileDiscretizerExample.scala index 0839c609f1d95..1a16515594161 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/QuantileDiscretizerExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/QuantileDiscretizerExample.scala @@ -24,7 +24,10 @@ import org.apache.spark.sql.SparkSession object QuantileDiscretizerExample { def main(args: Array[String]) { - val spark = SparkSession.builder.appName("QuantileDiscretizerExample").getOrCreate() + val spark = SparkSession + .builder + .appName("QuantileDiscretizerExample") + .getOrCreate() import spark.implicits._ // $example on$ diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/RFormulaExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/RFormulaExample.scala index 699b621db90af..9ea4920146448 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/RFormulaExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/RFormulaExample.scala @@ -25,7 +25,10 @@ import org.apache.spark.sql.SparkSession object RFormulaExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("RFormulaExample").getOrCreate() + val spark = SparkSession + .builder + .appName("RFormulaExample") + .getOrCreate() // $example on$ val dataset = spark.createDataFrame(Seq( diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/RandomForestClassifierExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/RandomForestClassifierExample.scala index 4192a9c737045..ae0bd945d8fed 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/RandomForestClassifierExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/RandomForestClassifierExample.scala @@ -28,7 +28,10 @@ import org.apache.spark.sql.SparkSession object RandomForestClassifierExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("RandomForestClassifierExample").getOrCreate() + val spark = SparkSession + .builder + .appName("RandomForestClassifierExample") + .getOrCreate() // $example on$ // Load and parse the data file, converting it to a DataFrame. diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/RandomForestRegressorExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/RandomForestRegressorExample.scala index 5632f0419a80c..96dc2f05be974 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/RandomForestRegressorExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/RandomForestRegressorExample.scala @@ -28,7 +28,10 @@ import org.apache.spark.sql.SparkSession object RandomForestRegressorExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("RandomForestRegressorExample").getOrCreate() + val spark = SparkSession + .builder + .appName("RandomForestRegressorExample") + .getOrCreate() // $example on$ // Load and parse the data file, converting it to a DataFrame. diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/SQLTransformerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/SQLTransformerExample.scala index f03b29ba327c6..bb4587b82cb37 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/SQLTransformerExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/SQLTransformerExample.scala @@ -25,7 +25,10 @@ import org.apache.spark.sql.SparkSession object SQLTransformerExample { def main(args: Array[String]) { - val spark = SparkSession.builder.appName("SQLTransformerExample").getOrCreate() + val spark = SparkSession + .builder + .appName("SQLTransformerExample") + .getOrCreate() // $example on$ val df = spark.createDataFrame( diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/SimpleParamsExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/SimpleParamsExample.scala index dff771950715a..3547dd95bdced 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/SimpleParamsExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/SimpleParamsExample.scala @@ -34,7 +34,10 @@ import org.apache.spark.sql.{Row, SparkSession} object SimpleParamsExample { def main(args: Array[String]) { - val spark = SparkSession.builder.appName("SimpleParamsExample").getOrCreate() + val spark = SparkSession + .builder + .appName("SimpleParamsExample") + .getOrCreate() import spark.implicits._ // Prepare training data. diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/SimpleTextClassificationPipeline.scala b/examples/src/main/scala/org/apache/spark/examples/ml/SimpleTextClassificationPipeline.scala index 05199007f015a..c78ff2378bc1f 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/SimpleTextClassificationPipeline.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/SimpleTextClassificationPipeline.scala @@ -42,7 +42,10 @@ case class Document(id: Long, text: String) object SimpleTextClassificationPipeline { def main(args: Array[String]) { - val spark = SparkSession.builder.appName("SimpleTextClassificationPipeline").getOrCreate() + val spark = SparkSession + .builder + .appName("SimpleTextClassificationPipeline") + .getOrCreate() import spark.implicits._ // Prepare training documents, which are labeled. diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/StandardScalerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/StandardScalerExample.scala index 55f777c6e228a..4d668e8ab9670 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/StandardScalerExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/StandardScalerExample.scala @@ -25,7 +25,10 @@ import org.apache.spark.sql.SparkSession object StandardScalerExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("StandardScalerExample").getOrCreate() + val spark = SparkSession + .builder + .appName("StandardScalerExample") + .getOrCreate() // $example on$ val dataFrame = spark.read.format("libsvm").load("data/mllib/sample_libsvm_data.txt") diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/StopWordsRemoverExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/StopWordsRemoverExample.scala index 85e79c8cb38ae..fb1a43e962cd5 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/StopWordsRemoverExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/StopWordsRemoverExample.scala @@ -25,7 +25,10 @@ import org.apache.spark.sql.SparkSession object StopWordsRemoverExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("StopWordsRemoverExample").getOrCreate() + val spark = SparkSession + .builder + .appName("StopWordsRemoverExample") + .getOrCreate() // $example on$ val remover = new StopWordsRemover() diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/StringIndexerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/StringIndexerExample.scala index e01a768da988d..63f273e87a209 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/StringIndexerExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/StringIndexerExample.scala @@ -25,7 +25,10 @@ import org.apache.spark.sql.SparkSession object StringIndexerExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("StringIndexerExample").getOrCreate() + val spark = SparkSession + .builder + .appName("StringIndexerExample") + .getOrCreate() // $example on$ val df = spark.createDataFrame( diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/TfIdfExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/TfIdfExample.scala index 910ef62a2670a..33b5daec59783 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/TfIdfExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/TfIdfExample.scala @@ -26,7 +26,10 @@ import org.apache.spark.sql.SparkSession object TfIdfExample { def main(args: Array[String]) { - val spark = SparkSession.builder.appName("TfIdfExample").getOrCreate() + val spark = SparkSession + .builder + .appName("TfIdfExample") + .getOrCreate() // $example on$ val sentenceData = spark.createDataFrame(Seq( diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/TokenizerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/TokenizerExample.scala index 4f0c47b3c84c7..1c70dc700b91c 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/TokenizerExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/TokenizerExample.scala @@ -25,7 +25,10 @@ import org.apache.spark.sql.SparkSession object TokenizerExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("TokenizerExample").getOrCreate() + val spark = SparkSession + .builder + .appName("TokenizerExample") + .getOrCreate() // $example on$ val sentenceDataFrame = spark.createDataFrame(Seq( diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/VectorAssemblerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/VectorAssemblerExample.scala index 56b7263b192e0..8e382ccc4844a 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/VectorAssemblerExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/VectorAssemblerExample.scala @@ -26,7 +26,10 @@ import org.apache.spark.sql.SparkSession object VectorAssemblerExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("VectorAssemblerExample").getOrCreate() + val spark = SparkSession + .builder + .appName("VectorAssemblerExample") + .getOrCreate() // $example on$ val dataset = spark.createDataFrame( diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/VectorIndexerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/VectorIndexerExample.scala index 214ad91634ee3..afa761aee0b98 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/VectorIndexerExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/VectorIndexerExample.scala @@ -25,7 +25,10 @@ import org.apache.spark.sql.SparkSession object VectorIndexerExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("VectorIndexerExample").getOrCreate() + val spark = SparkSession + .builder + .appName("VectorIndexerExample") + .getOrCreate() // $example on$ val data = spark.read.format("libsvm").load("data/mllib/sample_libsvm_data.txt") diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/VectorSlicerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/VectorSlicerExample.scala index 716bf023a8087..b1a3997f48c91 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/VectorSlicerExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/VectorSlicerExample.scala @@ -31,7 +31,10 @@ import org.apache.spark.sql.SparkSession object VectorSlicerExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("VectorSlicerExample").getOrCreate() + val spark = SparkSession + .builder + .appName("VectorSlicerExample") + .getOrCreate() // $example on$ val data = Arrays.asList(Row(Vectors.dense(-2.0, 2.3, 0.0))) diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/Word2VecExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/Word2VecExample.scala index 292b6d9f77763..9ac5623607296 100644 --- a/examples/src/main/scala/org/apache/spark/examples/ml/Word2VecExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/ml/Word2VecExample.scala @@ -25,7 +25,10 @@ import org.apache.spark.sql.SparkSession object Word2VecExample { def main(args: Array[String]) { - val spark = SparkSession.builder.appName("Word2Vec example").getOrCreate() + val spark = SparkSession + .builder + .appName("Word2Vec example") + .getOrCreate() // $example on$ // Input data: Each row is a bag of words from a sentence or document. diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/LDAExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/LDAExample.scala index c2bf1548b5685..7651aade493a0 100644 --- a/examples/src/main/scala/org/apache/spark/examples/mllib/LDAExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/mllib/LDAExample.scala @@ -189,7 +189,9 @@ object LDAExample { vocabSize: Int, stopwordFile: String): (RDD[(Long, Vector)], Array[String], Long) = { - val spark = SparkSession.builder.getOrCreate() + val spark = SparkSession + .builder + .getOrCreate() import spark.implicits._ // Get dataset of document texts diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/RankingMetricsExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/RankingMetricsExample.scala index cd4f0bb0de601..781a934df6637 100644 --- a/examples/src/main/scala/org/apache/spark/examples/mllib/RankingMetricsExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/mllib/RankingMetricsExample.scala @@ -26,7 +26,10 @@ import org.apache.spark.sql.SparkSession object RankingMetricsExample { def main(args: Array[String]) { - val spark = SparkSession.builder.appName("RankingMetricsExample").getOrCreate() + val spark = SparkSession + .builder + .appName("RankingMetricsExample") + .getOrCreate() import spark.implicits._ // $example on$ // Read in the ratings data diff --git a/examples/src/main/scala/org/apache/spark/examples/mllib/RegressionMetricsExample.scala b/examples/src/main/scala/org/apache/spark/examples/mllib/RegressionMetricsExample.scala index 22c47a694d346..abeaaa00b5a4f 100644 --- a/examples/src/main/scala/org/apache/spark/examples/mllib/RegressionMetricsExample.scala +++ b/examples/src/main/scala/org/apache/spark/examples/mllib/RegressionMetricsExample.scala @@ -27,7 +27,10 @@ import org.apache.spark.sql.SparkSession object RegressionMetricsExample { def main(args: Array[String]): Unit = { - val spark = SparkSession.builder.appName("RegressionMetricsExample").getOrCreate() + val spark = SparkSession + .builder + .appName("RegressionMetricsExample") + .getOrCreate() // $example on$ // Load the data val data = spark diff --git a/examples/src/main/scala/org/apache/spark/examples/sql/RDDRelation.scala b/examples/src/main/scala/org/apache/spark/examples/sql/RDDRelation.scala index b4118b16e2918..94c378ae4b911 100644 --- a/examples/src/main/scala/org/apache/spark/examples/sql/RDDRelation.scala +++ b/examples/src/main/scala/org/apache/spark/examples/sql/RDDRelation.scala @@ -26,7 +26,10 @@ case class Record(key: Int, value: String) object RDDRelation { def main(args: Array[String]) { - val spark = SparkSession.builder.appName("RDDRelation").getOrCreate() + val spark = SparkSession + .builder + .appName("RDDRelation") + .getOrCreate() // Importing the SparkSession gives access to all the SQL functions and implicit conversions. import spark.implicits._ diff --git a/examples/src/main/scala/org/apache/spark/examples/streaming/SqlNetworkWordCount.scala b/examples/src/main/scala/org/apache/spark/examples/streaming/SqlNetworkWordCount.scala index 2f0fe704f709b..9aba4a05a89ad 100644 --- a/examples/src/main/scala/org/apache/spark/examples/streaming/SqlNetworkWordCount.scala +++ b/examples/src/main/scala/org/apache/spark/examples/streaming/SqlNetworkWordCount.scala @@ -93,7 +93,10 @@ object SparkSessionSingleton { def getInstance(sparkConf: SparkConf): SparkSession = { if (instance == null) { - instance = SparkSession.builder.config(sparkConf).getOrCreate() + instance = SparkSession + .builder + .config(sparkConf) + .getOrCreate() } instance } diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java index 9475c853a03ff..9def4559d214e 100644 --- a/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java +++ b/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/VectorizedPlainValuesReader.java @@ -34,8 +34,9 @@ public class VectorizedPlainValuesReader extends ValuesReader implements Vectori private int offset; private int bitOffset; // Only used for booleans. private ByteBuffer byteBuffer; // used to wrap the byte array buffer - - private final static boolean bigEndianPlatform = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); + + private static final boolean bigEndianPlatform = + ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); public VectorizedPlainValuesReader() { } diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java index b8dd16227ec17..70b4a683311ca 100644 --- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java +++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java @@ -19,8 +19,6 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; -import org.apache.commons.lang.NotImplementedException; - import org.apache.spark.memory.MemoryMode; import org.apache.spark.sql.types.*; import org.apache.spark.unsafe.Platform; @@ -29,9 +27,10 @@ * Column data backed using offheap memory. */ public final class OffHeapColumnVector extends ColumnVector { - - private final static boolean bigEndianPlatform = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); - + + private static final boolean bigEndianPlatform = + ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); + // The data stored in these two allocations need to maintain binary compatible. We can // directly pass this buffer to external components. private long nulls; @@ -230,7 +229,8 @@ public void putIntsLittleEndian(int rowId, int count, byte[] src, int srcIndex) int srcOffset = srcIndex + Platform.BYTE_ARRAY_OFFSET; long offset = data + 4 * rowId; for (int i = 0; i < count; ++i, offset += 4, srcOffset += 4) { - Platform.putInt(null, offset, java.lang.Integer.reverseBytes(Platform.getInt(src, srcOffset))); + Platform.putInt(null, offset, + java.lang.Integer.reverseBytes(Platform.getInt(src, srcOffset))); } } } @@ -276,7 +276,8 @@ public void putLongsLittleEndian(int rowId, int count, byte[] src, int srcIndex) int srcOffset = srcIndex + Platform.BYTE_ARRAY_OFFSET; long offset = data + 8 * rowId; for (int i = 0; i < count; ++i, offset += 8, srcOffset += 8) { - Platform.putLong(null, offset, java.lang.Long.reverseBytes(Platform.getLong(src, srcOffset))); + Platform.putLong(null, offset, + java.lang.Long.reverseBytes(Platform.getLong(src, srcOffset))); } } } diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java index b1ffe4c21049b..7fb7617050f2d 100644 --- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java +++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java @@ -29,9 +29,10 @@ * and a java array for the values. */ public final class OnHeapColumnVector extends ColumnVector { - - private final static boolean bigEndianPlatform = ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); - + + private static final boolean bigEndianPlatform = + ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN); + // The data stored in these arrays need to maintain binary compatible. We can // directly pass this buffer to external components. diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/SessionManager.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/SessionManager.java index 0457b3781ebf0..de066dd406c7a 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/SessionManager.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/session/SessionManager.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Date; -import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Future; @@ -35,7 +34,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.ql.hooks.HookUtils; import org.apache.hive.service.CompositeService; import org.apache.hive.service.cli.HiveSQLException; import org.apache.hive.service.cli.SessionHandle;