Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SPARK-15134][EXAMPLE] Indent SparkSession builder patterns and update binary_classification_metrics_example.py #12911

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,11 @@ protected void handleMessage(
blocks.add(blockManager.getBlockData(msg.appId, msg.execId, blockId));
}
long streamId = streamManager.registerStream(client.getClientId(), blocks.iterator());
logger.trace("Registered streamId {} with {} buffers for client {} from host {}", streamId,
msg.blockIds.length, client.getClientId(), NettyUtils.getRemoteAddress(client.getChannel()));
logger.trace("Registered streamId {} with {} buffers for client {} from host {}",
streamId,
msg.blockIds.length,
client.getClientId(),
NettyUtils.getRemoteAddress(client.getChannel()));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, but in the future I would keep unrelated changes separate

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, I see. I'll keep it in mind.

callback.onSuccess(new StreamHandle(streamId, msg.blockIds.length).toByteBuffer());

} else if (msgObj instanceof RegisterExecutor) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,10 @@

public class JavaAFTSurvivalRegressionExample {
public static void main(String[] args) {
SparkSession spark = SparkSession.builder().appName("JavaAFTSurvivalRegressionExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaAFTSurvivalRegressionExample")
.getOrCreate();

// $example on$
List<Row> data = Arrays.asList(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,10 @@ public static Rating parseRating(String str) {
// $example off$

public static void main(String[] args) {
SparkSession spark = SparkSession.builder().appName("JavaALSExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaALSExample")
.getOrCreate();

// $example on$
JavaRDD<Rating> ratingsRDD = spark
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,13 @@

package org.apache.spark.examples.ml;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.SparkSession;

// $example on$
import java.util.Arrays;
import java.util.List;

import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.ml.feature.Binarizer;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.RowFactory;
Expand All @@ -38,7 +35,10 @@

public class JavaBinarizerExample {
public static void main(String[] args) {
SparkSession spark = SparkSession.builder().appName("JavaBinarizerExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaBinarizerExample")
.getOrCreate();

// $example on$
List<Row> data = Arrays.asList(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,10 @@
public class JavaBisectingKMeansExample {

public static void main(String[] args) {
SparkSession spark = SparkSession.builder().appName("JavaBisectingKMeansExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaBisectingKMeansExample")
.getOrCreate();

// $example on$
List<Row> data = Arrays.asList(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,10 @@

public class JavaBucketizerExample {
public static void main(String[] args) {
SparkSession spark = SparkSession.builder().appName("JavaBucketizerExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaBucketizerExample")
.getOrCreate();

// $example on$
double[] splits = {Double.NEGATIVE_INFINITY, -0.5, 0.0, 0.5, Double.POSITIVE_INFINITY};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,6 @@

package org.apache.spark.examples.ml;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.SparkSession;

Expand All @@ -40,7 +37,10 @@

public class JavaChiSqSelectorExample {
public static void main(String[] args) {
SparkSession spark = SparkSession.builder().appName("JavaChiSqSelectorExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaChiSqSelectorExample")
.getOrCreate();

// $example on$
List<Row> data = Arrays.asList(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,10 @@

public class JavaCountVectorizerExample {
public static void main(String[] args) {
SparkSession spark = SparkSession.builder().appName("JavaCountVectorizerExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaCountVectorizerExample")
.getOrCreate();

// $example on$
// Input data: Each row is a bag of words from a sentence or document.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,13 @@

package org.apache.spark.examples.ml;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.SparkSession;

// $example on$
import java.util.Arrays;
import java.util.List;

import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.ml.feature.DCT;
import org.apache.spark.mllib.linalg.VectorUDT;
import org.apache.spark.mllib.linalg.Vectors;
Expand All @@ -39,7 +36,10 @@

public class JavaDCTExample {
public static void main(String[] args) {
SparkSession spark = SparkSession.builder().appName("JavaDCTExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaDCTExample")
.getOrCreate();

// $example on$
List<Row> data = Arrays.asList(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,10 @@
public class JavaDeveloperApiExample {

public static void main(String[] args) throws Exception {
SparkSession spark = SparkSession.builder().appName("JavaDeveloperApiExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaDeveloperApiExample")
.getOrCreate();

// Prepare training data.
List<LabeledPoint> localTraining = Lists.newArrayList(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@

package org.apache.spark.examples.ml;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.SparkSession;

Expand All @@ -27,7 +25,6 @@
import java.util.Arrays;
import java.util.List;

import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.ml.feature.ElementwiseProduct;
import org.apache.spark.mllib.linalg.Vector;
import org.apache.spark.mllib.linalg.VectorUDT;
Expand All @@ -42,7 +39,9 @@
public class JavaElementwiseProductExample {
public static void main(String[] args) {
SparkSession spark = SparkSession
.builder().appName("JavaElementwiseProductExample").getOrCreate();
.builder()
.appName("JavaElementwiseProductExample")
.getOrCreate();

// $example on$
// Create some vector data; also works for sparse vectors
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@

package org.apache.spark.examples.ml;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
// $example on$
import org.apache.spark.ml.Pipeline;
import org.apache.spark.ml.PipelineModel;
Expand All @@ -35,11 +33,15 @@
public class JavaGradientBoostedTreeClassifierExample {
public static void main(String[] args) {
SparkSession spark = SparkSession
.builder().appName("JavaGradientBoostedTreeClassifierExample").getOrCreate();
.builder()
.appName("JavaGradientBoostedTreeClassifierExample")
.getOrCreate();

// $example on$
// Load and parse the data file, converting it to a DataFrame.
Dataset<Row> data = spark.read().format("libsvm")
Dataset<Row> data = spark
.read()
.format("libsvm")
.load("data/mllib/sample_libsvm_data.txt");

// Index labels, adding metadata to the label column.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,10 @@

public class JavaIndexToStringExample {
public static void main(String[] args) {
SparkSession spark = SparkSession.builder().appName("JavaIndexToStringExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaIndexToStringExample")
.getOrCreate();

// $example on$
List<Row> data = Arrays.asList(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,10 @@ public static void main(String[] args) {
int k = Integer.parseInt(args[1]);

// Parses the arguments
SparkSession spark = SparkSession.builder().appName("JavaKMeansExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaKMeansExample")
.getOrCreate();

// $example on$
// Loads data
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,10 @@ public static void main(String[] args) {
String inputFile = "data/mllib/sample_lda_data.txt";

// Parses the arguments
SparkSession spark = SparkSession.builder().appName("JavaLDAExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaLDAExample")
.getOrCreate();

// Loads data
JavaRDD<Row> points = spark.read().text(inputFile).javaRDD().map(new ParseVector());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,19 @@
public class JavaMaxAbsScalerExample {

public static void main(String[] args) {
SparkSession spark = SparkSession.builder().appName("JavaMaxAbsScalerExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaMaxAbsScalerExample")
.getOrCreate();

// $example on$
Dataset<Row> dataFrame = spark.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
Dataset<Row> dataFrame = spark
.read()
.format("libsvm")
.load("data/mllib/sample_libsvm_data.txt");
MaxAbsScaler scaler = new MaxAbsScaler()
.setInputCol("features")
.setOutputCol("scaledFeatures");
.setInputCol("features")
.setOutputCol("scaledFeatures");

// Compute summary statistics and generate MaxAbsScalerModel
MaxAbsScalerModel scalerModel = scaler.fit(dataFrame);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,16 @@

public class JavaMinMaxScalerExample {
public static void main(String[] args) {
SparkSession spark = SparkSession.builder().appName("JavaMinMaxScalerExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaMinMaxScalerExample")
.getOrCreate();

// $example on$
Dataset<Row> dataFrame = spark.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
Dataset<Row> dataFrame = spark
.read()
.format("libsvm")
.load("data/mllib/sample_libsvm_data.txt");
MinMaxScaler scaler = new MinMaxScaler()
.setInputCol("features")
.setOutputCol("scaledFeatures");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,10 @@

public class JavaNGramExample {
public static void main(String[] args) {
SparkSession spark = SparkSession.builder().appName("JavaNGramExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaNGramExample")
.getOrCreate();

// $example on$
List<Row> data = Arrays.asList(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,10 @@
public class JavaNaiveBayesExample {

public static void main(String[] args) {
SparkSession spark = SparkSession.builder().appName("JavaNaiveBayesExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaNaiveBayesExample")
.getOrCreate();

// $example on$
// Load training data
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,10 @@

public class JavaNormalizerExample {
public static void main(String[] args) {
SparkSession spark = SparkSession.builder().appName("JavaNormalizerExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaNormalizerExample")
.getOrCreate();

// $example on$
Dataset<Row> dataFrame =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,10 @@

public class JavaOneHotEncoderExample {
public static void main(String[] args) {
SparkSession spark = SparkSession.builder().appName("JavaOneHotEncoderExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaOneHotEncoderExample")
.getOrCreate();

// $example on$
List<Row> data = Arrays.asList(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,10 @@ private static class Params {
public static void main(String[] args) {
// parse the arguments
Params params = parse(args);
SparkSession spark = SparkSession.builder().appName("JavaOneVsRestExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaOneVsRestExample")
.getOrCreate();

// $example on$
// configure the base classifier
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,10 @@

public class JavaPCAExample {
public static void main(String[] args) {
SparkSession spark = SparkSession.builder().appName("JavaPCAExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaPCAExample")
.getOrCreate();

// $example on$
List<Row> data = Arrays.asList(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,10 @@
*/
public class JavaPipelineExample {
public static void main(String[] args) {
SparkSession spark = SparkSession.builder().appName("JavaPipelineExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaPipelineExample")
.getOrCreate();

// $example on$
// Prepare training documents, which are labeled.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,10 @@

public class JavaPolynomialExpansionExample {
public static void main(String[] args) {
SparkSession spark = SparkSession.builder().appName("JavaPolynomialExpansionExample").getOrCreate();
SparkSession spark = SparkSession
.builder()
.appName("JavaPolynomialExpansionExample")
.getOrCreate();

// $example on$
PolynomialExpansion polyExpansion = new PolynomialExpansion()
Expand Down