From 816e3ff27d665c75e3f560b6b89a1af80fe16ef3 Mon Sep 17 00:00:00 2001 From: Thomas Groh Date: Fri, 17 Jun 2016 11:42:19 -0700 Subject: [PATCH] Rename DataflowPipelineRunner to DataflowRunner --- .../beam/examples/DebuggingWordCount.java | 2 +- .../beam/examples/MinimalWordCount.java | 4 +- .../beam/examples/WindowedWordCount.java | 2 +- .../org/apache/beam/examples/WordCount.java | 2 +- .../examples/common/DataflowExampleUtils.java | 18 +- .../beam/examples/complete/AutoComplete.java | 4 +- .../apache/beam/examples/complete/TfIdf.java | 2 +- .../complete/TopWikipediaSessions.java | 2 +- .../examples/cookbook/BigQueryTornadoes.java | 2 +- .../cookbook/CombinePerKeyExamples.java | 2 +- .../examples/cookbook/DatastoreWordCount.java | 4 +- .../beam/examples/cookbook/DeDupExample.java | 2 +- .../examples/cookbook/FilterExamples.java | 2 +- .../beam/examples/cookbook/JoinExamples.java | 2 +- .../examples/cookbook/MaxPerKeyExamples.java | 2 +- .../beam/examples/MinimalWordCountJava8.java | 4 +- .../examples/complete/game/GameStats.java | 2 +- .../complete/game/HourlyTeamScore.java | 2 +- .../examples/complete/game/LeaderBoard.java | 2 +- .../examples/complete/game/UserScore.java | 2 +- .../runners/flink/FlinkPipelineRunner.java | 2 +- ...unner.java => BlockingDataflowRunner.java} | 36 ++-- .../DataflowJobCancelledException.java | 2 +- .../DataflowJobExecutionException.java | 2 +- .../dataflow/DataflowJobUpdatedException.java | 2 +- .../runners/dataflow/DataflowPipelineJob.java | 2 +- .../dataflow/DataflowPipelineRegistrar.java | 8 +- .../dataflow/DataflowPipelineTranslator.java | 10 +- ...ipelineRunner.java => DataflowRunner.java} | 168 +++++++++--------- ...nerHooks.java => DataflowRunnerHooks.java} | 4 +- .../dataflow/internal/package-info.java | 2 +- .../BlockingDataflowPipelineOptions.java | 6 +- .../options/DataflowPipelineOptions.java | 4 +- .../DataflowPipelineWorkerPoolOptions.java | 6 +- .../testing/TestDataflowPipelineRunner.java | 10 +- ...t.java => BlockingDataflowRunnerTest.java} | 34 ++-- .../DataflowPipelineRegistrarTest.java | 4 +- .../DataflowPipelineTranslatorTest.java | 50 +++--- ...unnerTest.java => DataflowRunnerTest.java} | 106 +++++------ .../dataflow/io/DataflowAvroIOTest.java | 4 +- .../dataflow/io/DataflowPubsubIOTest.java | 4 +- .../dataflow/io/DataflowTextIOTest.java | 4 +- ...rTest.java => TestDataflowRunnerTest.java} | 28 +-- .../DataflowDisplayDataEvaluator.java | 10 +- .../transforms/DataflowGroupByKeyTest.java | 8 +- .../dataflow/transforms/DataflowViewTest.java | 8 +- .../util/DataflowPathValidatorTest.java | 8 +- .../runners/spark/SimpleWordCountTest.java | 2 +- .../spark/translation/SerializationTest.java | 2 +- .../apache/beam/sdk/runners/package-info.java | 12 +- .../apache/beam/sdk/testing/TestPipeline.java | 2 +- .../beam/sdk/transforms/Aggregator.java | 2 +- .../sdk/util/BigQueryTableRowIterator.java | 2 +- .../beam/sdk/runners/PipelineRunnerTest.java | 2 +- .../src/main/java/DebuggingWordCount.java | 2 +- .../src/main/java/MinimalWordCount.java | 4 +- .../src/main/java/WindowedWordCount.java | 2 +- .../src/main/java/WordCount.java | 2 +- .../java/common/DataflowExampleUtils.java | 14 +- .../src/main/java/StarterPipeline.java | 2 +- .../src/main/java/it/pkg/StarterPipeline.java | 2 +- sdks/java/pom.xml | 2 +- 62 files changed, 325 insertions(+), 325 deletions(-) rename runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/{BlockingDataflowPipelineRunner.java => BlockingDataflowRunner.java} (86%) rename runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/{DataflowPipelineRunner.java => DataflowRunner.java} (96%) rename runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/{DataflowPipelineRunnerHooks.java => DataflowRunnerHooks.java} (93%) rename runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/{BlockingDataflowPipelineRunnerTest.java => BlockingDataflowRunnerTest.java} (89%) rename runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/{DataflowPipelineRunnerTest.java => DataflowRunnerTest.java} (94%) rename runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/{TestDataflowPipelineRunnerTest.java => TestDataflowRunnerTest.java} (95%) diff --git a/examples/java/src/main/java/org/apache/beam/examples/DebuggingWordCount.java b/examples/java/src/main/java/org/apache/beam/examples/DebuggingWordCount.java index 11c56d132b0d..85823c2005cb 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/DebuggingWordCount.java +++ b/examples/java/src/main/java/org/apache/beam/examples/DebuggingWordCount.java @@ -69,7 +69,7 @@ *
{@code
  *   --project=YOUR_PROJECT_ID
  *   --tempLocation=gs://YOUR_TEMP_DIRECTORY
- *   --runner=BlockingDataflowPipelineRunner
+ *   --runner=BlockingDataflowRunner
  *   --workerLogLevelOverrides={"org.apache.beam.examples":"DEBUG"}
  * }
  * 
diff --git a/examples/java/src/main/java/org/apache/beam/examples/MinimalWordCount.java b/examples/java/src/main/java/org/apache/beam/examples/MinimalWordCount.java index 65d450dfce23..6d4bfd4702a4 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/MinimalWordCount.java +++ b/examples/java/src/main/java/org/apache/beam/examples/MinimalWordCount.java @@ -17,7 +17,7 @@ */ package org.apache.beam.examples; -import org.apache.beam.runners.dataflow.BlockingDataflowPipelineRunner; +import org.apache.beam.runners.dataflow.BlockingDataflowRunner; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.io.TextIO; @@ -65,7 +65,7 @@ public static void main(String[] args) { // in Google Cloud Storage to stage files. DataflowPipelineOptions options = PipelineOptionsFactory.create() .as(DataflowPipelineOptions.class); - options.setRunner(BlockingDataflowPipelineRunner.class); + options.setRunner(BlockingDataflowRunner.class); // CHANGE 1/3: Your project ID is required in order to run your pipeline on the Google Cloud. options.setProject("SET_YOUR_PROJECT_ID_HERE"); // CHANGE 2/3: Your Google Cloud Storage path is required for staging local files. diff --git a/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java b/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java index 9ca26bf5fea4..d4f128024bf7 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java +++ b/examples/java/src/main/java/org/apache/beam/examples/WindowedWordCount.java @@ -84,7 +84,7 @@ *
{@code
  *   --project=YOUR_PROJECT_ID
  *   --tempLocation=gs://YOUR_TEMP_DIRECTORY
- *   --runner=BlockingDataflowPipelineRunner
+ *   --runner=BlockingDataflowRunner
  * }
  * 
* diff --git a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java index 03ed5d787edc..cf6c45aac4a0 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java +++ b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java @@ -82,7 +82,7 @@ *
{@code
  *   --project=YOUR_PROJECT_ID
  *   --tempLocation=gs://YOUR_TEMP_DIRECTORY
- *   --runner=BlockingDataflowPipelineRunner
+ *   --runner=BlockingDataflowRunner
  * }
  * 
* and an output prefix on GCS: diff --git a/examples/java/src/main/java/org/apache/beam/examples/common/DataflowExampleUtils.java b/examples/java/src/main/java/org/apache/beam/examples/common/DataflowExampleUtils.java index a0b7319db81c..e635a3f62da1 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/common/DataflowExampleUtils.java +++ b/examples/java/src/main/java/org/apache/beam/examples/common/DataflowExampleUtils.java @@ -17,9 +17,9 @@ */ package org.apache.beam.examples.common; -import org.apache.beam.runners.dataflow.BlockingDataflowPipelineRunner; +import org.apache.beam.runners.dataflow.BlockingDataflowRunner; import org.apache.beam.runners.dataflow.DataflowPipelineJob; -import org.apache.beam.runners.dataflow.DataflowPipelineRunner; +import org.apache.beam.runners.dataflow.DataflowRunner; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.runners.dataflow.util.MonitoringUtil; import org.apache.beam.sdk.Pipeline; @@ -316,16 +316,16 @@ public void startInjectorIfNeeded(String inputFile) { /** * Do some runner setup: check that the DirectPipelineRunner is not used in conjunction with - * streaming, and if streaming is specified, use the DataflowPipelineRunner. + * streaming, and if streaming is specified, use the DataflowRunner. */ public void setupRunner() { Class> runner = options.getRunner(); if (options.isStreaming() - && (runner.equals(DataflowPipelineRunner.class) - || runner.equals(BlockingDataflowPipelineRunner.class))) { + && (runner.equals(DataflowRunner.class) + || runner.equals(BlockingDataflowRunner.class))) { // In order to cancel the pipelines automatically, - // {@literal DataflowPipelineRunner} is forced to be used. - options.setRunner(DataflowPipelineRunner.class); + // {@literal DataflowRunner} is forced to be used. + options.setRunner(DataflowRunner.class); } } @@ -363,7 +363,7 @@ public void runInjectorPipeline(PTransform> } copiedOptions.setStreaming(false); copiedOptions.setWorkerHarnessContainerImage( - DataflowPipelineRunner.BATCH_WORKER_HARNESS_CONTAINER_IMAGE); + DataflowRunner.BATCH_WORKER_HARNESS_CONTAINER_IMAGE); copiedOptions.setNumWorkers(options.as(DataflowExampleOptions.class).getInjectorNumWorkers()); copiedOptions.setJobName(options.getJobName() + "-injector"); Pipeline injectorPipeline = Pipeline.create(copiedOptions); @@ -396,7 +396,7 @@ public void mockUnboundedSource(String inputFile, PipelineResult result) { } /** - * If {@literal DataflowPipelineRunner} or {@literal BlockingDataflowPipelineRunner} is used, + * If {@literal DataflowRunner} or {@literal BlockingDataflowRunner} is used, * waits for the pipeline to finish and cancels it (and the injector) before the program exists. */ public void waitToFinish(PipelineResult result) { diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java b/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java index 3e4440c7a20f..ef47762ea35c 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/AutoComplete.java @@ -82,7 +82,7 @@ *
{@code
  *   --project=YOUR_PROJECT_ID
  *   --tempLocation=gs://YOUR_TEMP_DIRECTORY
- *   --runner=DataflowPipelineRunner
+ *   --runner=DataflowRunner
  *   --inputFile=gs://path/to/input*.txt
  * }
* @@ -91,7 +91,7 @@ *
{@code
  *   --project=YOUR_PROJECT_ID
  *   --tempLocation=gs://YOUR_TEMP_DIRECTORY
- *   --runner=DataflowPipelineRunner
+ *   --runner=DataflowRunner
  *   --inputFile=gs://YOUR_INPUT_DIRECTORY/*.txt
  *   --streaming
  * }
diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java b/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java index 1f1885fa0b6f..73f3323e18a9 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java @@ -80,7 +80,7 @@ *
{@code
  *   --project=YOUR_PROJECT_ID
  *   --tempLocation=gs://YOUR_TEMP_DIRECTORY
- *   --runner=BlockingDataflowPipelineRunner
+ *   --runner=BlockingDataflowRunner
  * and an output prefix on GCS:
  *   --output=gs://YOUR_OUTPUT_PREFIX
  * }
diff --git a/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java b/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java index c483608ceb0e..2c0d0ebdf72a 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java +++ b/examples/java/src/main/java/org/apache/beam/examples/complete/TopWikipediaSessions.java @@ -60,7 +60,7 @@ *
{@code
  *   --project=YOUR_PROJECT_ID
  *   --tempLocation=gs://YOUR_TEMP_DIRECTORY
- *   --runner=BlockingDataflowPipelineRunner
+ *   --runner=BlockingDataflowRunner
  * }
  * 
* and an output prefix on GCS: diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/BigQueryTornadoes.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/BigQueryTornadoes.java index 4c69efb81ca5..a0ac6c5f9945 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/BigQueryTornadoes.java +++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/BigQueryTornadoes.java @@ -61,7 +61,7 @@ *
{@code
  *   --project=YOUR_PROJECT_ID
  *   --tempLocation=gs://YOUR_TEMP_DIRECTORY
- *   --runner=BlockingDataflowPipelineRunner
+ *   --runner=BlockingDataflowRunner
  * }
  * 
* and the BigQuery table for the output: diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/CombinePerKeyExamples.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/CombinePerKeyExamples.java index 505989c6fb84..dbaa1f8122e8 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/CombinePerKeyExamples.java +++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/CombinePerKeyExamples.java @@ -68,7 +68,7 @@ *
{@code
  *   --project=YOUR_PROJECT_ID
  *   --tempLocation=gs://
- *   --runner=BlockingDataflowPipelineRunner
+ *   --runner=BlockingDataflowRunner
  * }
  * 
* and the BigQuery table for the output: diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/DatastoreWordCount.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/DatastoreWordCount.java index bfaecdf897ae..8aa3bcd6d7a6 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/DatastoreWordCount.java +++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/DatastoreWordCount.java @@ -71,7 +71,7 @@ *

To run this example using Dataflow service, you must additionally * provide either {@literal --tempLocation} or {@literal --tempLocation}, and * select one of the Dataflow pipeline runners, eg - * {@literal --runner=BlockingDataflowPipelineRunner}. + * {@literal --runner=BlockingDataflowRunner}. * *

Note: this example creates entities with Ancestor keys to ensure that all * entities created are in the same entity group. Similarly, the query used to read from the Cloud @@ -248,7 +248,7 @@ public static void readDataFromDatastore(Options options) { /** * An example to demo how to use {@link DatastoreIO}. The runner here is * customizable, which means users could pass either {@code DirectPipelineRunner} - * or {@code DataflowPipelineRunner} in the pipeline options. + * or {@code DataflowRunner} in the pipeline options. */ public static void main(String args[]) { // The options are used in two places, for Dataflow service, and diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/DeDupExample.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/DeDupExample.java index 37ddb4b8c29c..db65543101bf 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/DeDupExample.java +++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/DeDupExample.java @@ -43,7 +43,7 @@ *

To execute this pipeline using the Dataflow service, specify pipeline configuration: * --project=YOUR_PROJECT_ID * --tempLocation=gs://YOUR_TEMP_DIRECTORY - * --runner=BlockingDataflowPipelineRunner + * --runner=BlockingDataflowRunner * and an output prefix on GCS: * --output=gs://YOUR_OUTPUT_PREFIX * diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/FilterExamples.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/FilterExamples.java index 1c6a6e1e9266..a669b0cb4468 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/FilterExamples.java +++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/FilterExamples.java @@ -73,7 +73,7 @@ *

{@code
  *   --project=YOUR_PROJECT_ID
  *   --tempLocation=gs://YOUR_TEMP_DIRECTORY
- *   --runner=BlockingDataflowPipelineRunner
+ *   --runner=BlockingDataflowRunner
  * }
  * 
* and the BigQuery table for the output: diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/JoinExamples.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/JoinExamples.java index 8e642f87fc7b..e8f1f01eec9a 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/JoinExamples.java +++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/JoinExamples.java @@ -56,7 +56,7 @@ *
{@code
  *   --project=YOUR_PROJECT_ID
  *   --tempLocation=gs://YOUR_TEMP_DIRECTORY
- *   --runner=BlockingDataflowPipelineRunner
+ *   --runner=BlockingDataflowRunner
  * }
  * 
* and an output prefix on GCS: diff --git a/examples/java/src/main/java/org/apache/beam/examples/cookbook/MaxPerKeyExamples.java b/examples/java/src/main/java/org/apache/beam/examples/cookbook/MaxPerKeyExamples.java index 24809672e6dd..6630f3dfa72b 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/cookbook/MaxPerKeyExamples.java +++ b/examples/java/src/main/java/org/apache/beam/examples/cookbook/MaxPerKeyExamples.java @@ -62,7 +62,7 @@ *
{@code
  *   --project=YOUR_PROJECT_ID
  *   --tempLocation=gs://YOUR_TEMP_DIRECTORY
- *   --runner=BlockingDataflowPipelineRunner
+ *   --runner=BlockingDataflowRunner
  * }
  * 
* and the BigQuery table for the output: diff --git a/examples/java8/src/main/java/org/apache/beam/examples/MinimalWordCountJava8.java b/examples/java8/src/main/java/org/apache/beam/examples/MinimalWordCountJava8.java index 0ad1a045bad4..cf3bbf955fb5 100644 --- a/examples/java8/src/main/java/org/apache/beam/examples/MinimalWordCountJava8.java +++ b/examples/java8/src/main/java/org/apache/beam/examples/MinimalWordCountJava8.java @@ -17,7 +17,7 @@ */ package org.apache.beam.examples; -import org.apache.beam.runners.dataflow.BlockingDataflowPipelineRunner; +import org.apache.beam.runners.dataflow.BlockingDataflowRunner; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.io.TextIO; @@ -42,7 +42,7 @@ public static void main(String[] args) { DataflowPipelineOptions options = PipelineOptionsFactory.create() .as(DataflowPipelineOptions.class); - options.setRunner(BlockingDataflowPipelineRunner.class); + options.setRunner(BlockingDataflowRunner.class); // CHANGE 1 of 3: Your project ID is required in order to run your pipeline on the Google Cloud. options.setProject("SET_YOUR_PROJECT_ID_HERE"); diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java index c5579134850a..c8bcc8ce4cb1 100644 --- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java +++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/GameStats.java @@ -83,7 +83,7 @@ *
{@code
  *   --project=YOUR_PROJECT_ID
  *   --tempLocation=gs://YOUR_TEMP_DIRECTORY
- *   --runner=BlockingDataflowPipelineRunner
+ *   --runner=BlockingDataflowRunner
  *   --dataset=YOUR-DATASET
  *   --topic=projects/YOUR-PROJECT/topics/YOUR-TOPIC
  * }
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java
index ba3983d575f3..c5c2fb527311 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/HourlyTeamScore.java
@@ -60,7 +60,7 @@
  * 
{@code
  *   --project=YOUR_PROJECT_ID
  *   --tempLocation=gs://YOUR_TEMP_DIRECTORY
- *   --runner=BlockingDataflowPipelineRunner
+ *   --runner=BlockingDataflowRunner
  *   --dataset=YOUR-DATASET
  * }
  * 
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java index d79403a88794..12d272942bfa 100644 --- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java +++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/LeaderBoard.java @@ -80,7 +80,7 @@ *
{@code
  *   --project=YOUR_PROJECT_ID
  *   --tempLocation=gs://YOUR_TEMP_DIRECTORY
- *   --runner=BlockingDataflowPipelineRunner
+ *   --runner=BlockingDataflowRunner
  *   --dataset=YOUR-DATASET
  *   --topic=projects/YOUR-PROJECT/topics/YOUR-TOPIC
  * }
diff --git a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
index de049e83ea13..97b692987ff3 100644
--- a/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
+++ b/examples/java8/src/main/java/org/apache/beam/examples/complete/game/UserScore.java
@@ -63,7 +63,7 @@
  * 
{@code
  *   --project=YOUR_PROJECT_ID
  *   --tempLocation=gs://YOUR_TEMP_DIRECTORY
- *   --runner=BlockingDataflowPipelineRunner
+ *   --runner=BlockingDataflowRunner
  *   --dataset=YOUR-DATASET
  * }
  * 
diff --git a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkPipelineRunner.java b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkPipelineRunner.java index 220e8e83750a..46a4fcea626e 100644 --- a/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkPipelineRunner.java +++ b/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/FlinkPipelineRunner.java @@ -145,7 +145,7 @@ public Output apply( @Override public String toString() { - return "DataflowPipelineRunner#" + hashCode(); + return "DataflowRunner#" + hashCode(); } /** diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BlockingDataflowPipelineRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BlockingDataflowRunner.java similarity index 86% rename from runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BlockingDataflowPipelineRunner.java rename to runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BlockingDataflowRunner.java index d8ee16a25bb1..5c59bc2b0a8e 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BlockingDataflowPipelineRunner.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/BlockingDataflowRunner.java @@ -38,7 +38,7 @@ import javax.annotation.Nullable; /** - * A {@link PipelineRunner} that's like {@link DataflowPipelineRunner} + * A {@link PipelineRunner} that's like {@link DataflowRunner} * but that waits for the launched job to finish. * *

Prints out job status updates and console messages while it waits. @@ -48,42 +48,42 @@ * *

Permissions

* When reading from a Dataflow source or writing to a Dataflow sink using - * {@code BlockingDataflowPipelineRunner}, the Google cloud services account and the Google compute + * {@code BlockingDataflowRunner}, the Google cloud services account and the Google compute * engine service account of the GCP project running the Dataflow Job will need access to the * corresponding source/sink. * *

Please see Google Cloud * Dataflow Security and Permissions for more details. */ -public class BlockingDataflowPipelineRunner extends +public class BlockingDataflowRunner extends PipelineRunner { - private static final Logger LOG = LoggerFactory.getLogger(BlockingDataflowPipelineRunner.class); + private static final Logger LOG = LoggerFactory.getLogger(BlockingDataflowRunner.class); // Defaults to an infinite wait period. // TODO: make this configurable after removal of option map. private static final long BUILTIN_JOB_TIMEOUT_SEC = -1L; - private final DataflowPipelineRunner dataflowPipelineRunner; + private final DataflowRunner dataflowRunner; private final BlockingDataflowPipelineOptions options; - protected BlockingDataflowPipelineRunner( - DataflowPipelineRunner internalRunner, + protected BlockingDataflowRunner( + DataflowRunner internalRunner, BlockingDataflowPipelineOptions options) { - this.dataflowPipelineRunner = internalRunner; + this.dataflowRunner = internalRunner; this.options = options; } /** * Constructs a runner from the provided options. */ - public static BlockingDataflowPipelineRunner fromOptions( + public static BlockingDataflowRunner fromOptions( PipelineOptions options) { BlockingDataflowPipelineOptions dataflowOptions = PipelineOptionsValidator.validate(BlockingDataflowPipelineOptions.class, options); - DataflowPipelineRunner dataflowPipelineRunner = - DataflowPipelineRunner.fromOptions(dataflowOptions); + DataflowRunner dataflowRunner = + DataflowRunner.fromOptions(dataflowOptions); - return new BlockingDataflowPipelineRunner(dataflowPipelineRunner, dataflowOptions); + return new BlockingDataflowRunner(dataflowRunner, dataflowOptions); } /** @@ -94,7 +94,7 @@ public static BlockingDataflowPipelineRunner fromOptions( */ @Override public DataflowPipelineJob run(Pipeline p) { - final DataflowPipelineJob job = dataflowPipelineRunner.run(p); + final DataflowPipelineJob job = dataflowRunner.run(p); // We ignore the potential race condition here (Ctrl-C after job submission but before the // shutdown hook is registered). Even if we tried to do something smarter (eg., SettableFuture) @@ -168,19 +168,19 @@ public void run() { @Override public OutputT apply( PTransform transform, InputT input) { - return dataflowPipelineRunner.apply(transform, input); + return dataflowRunner.apply(transform, input); } /** - * Sets callbacks to invoke during execution. See {@link DataflowPipelineRunnerHooks}. + * Sets callbacks to invoke during execution. See {@link DataflowRunnerHooks}. */ @Experimental - public void setHooks(DataflowPipelineRunnerHooks hooks) { - this.dataflowPipelineRunner.setHooks(hooks); + public void setHooks(DataflowRunnerHooks hooks) { + this.dataflowRunner.setHooks(hooks); } @Override public String toString() { - return "BlockingDataflowPipelineRunner#" + options.getJobName(); + return "BlockingDataflowRunner#" + options.getJobName(); } } diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowJobCancelledException.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowJobCancelledException.java index de44a19516ae..e2edb6af50c7 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowJobCancelledException.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowJobCancelledException.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.dataflow; /** - * Signals that a job run by a {@link BlockingDataflowPipelineRunner} was updated during execution. + * Signals that a job run by a {@link BlockingDataflowRunner} was updated during execution. */ public class DataflowJobCancelledException extends DataflowJobException { /** diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowJobExecutionException.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowJobExecutionException.java index 11b8723b75bd..ccf805720209 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowJobExecutionException.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowJobExecutionException.java @@ -20,7 +20,7 @@ import javax.annotation.Nullable; /** - * Signals that a job run by a {@link BlockingDataflowPipelineRunner} fails during execution, and + * Signals that a job run by a {@link BlockingDataflowRunner} fails during execution, and * provides access to the failed job. */ public class DataflowJobExecutionException extends DataflowJobException { diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowJobUpdatedException.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowJobUpdatedException.java index e30a0e78bc16..39d1d47ad5ee 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowJobUpdatedException.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowJobUpdatedException.java @@ -18,7 +18,7 @@ package org.apache.beam.runners.dataflow; /** - * Signals that a job run by a {@link BlockingDataflowPipelineRunner} was updated during execution. + * Signals that a job run by a {@link BlockingDataflowRunner} was updated during execution. */ public class DataflowJobUpdatedException extends DataflowJobException { private DataflowPipelineJob replacedByJob; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java index 0f42148a0a4c..548d2b93d97b 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineJob.java @@ -55,7 +55,7 @@ /** * A DataflowPipelineJob represents a job submitted to Dataflow using - * {@link DataflowPipelineRunner}. + * {@link DataflowRunner}. */ public class DataflowPipelineJob implements PipelineResult { private static final Logger LOG = LoggerFactory.getLogger(DataflowPipelineJob.class); diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java index d7d243f4df83..4c88c4e0f010 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRegistrar.java @@ -29,7 +29,7 @@ /** * Contains the {@link PipelineOptionsRegistrar} and {@link PipelineRunnerRegistrar} for the - * {@link DataflowPipelineRunner}. + * {@link DataflowRunner}. */ public class DataflowPipelineRegistrar { private DataflowPipelineRegistrar() { } @@ -48,15 +48,15 @@ public Iterable> getPipelineOptions() { } /** - * Register the {@link DataflowPipelineRunner} and {@link BlockingDataflowPipelineRunner}. + * Register the {@link DataflowRunner} and {@link BlockingDataflowRunner}. */ @AutoService(PipelineRunnerRegistrar.class) public static class Runner implements PipelineRunnerRegistrar { @Override public Iterable>> getPipelineRunners() { return ImmutableList.>>of( - DataflowPipelineRunner.class, - BlockingDataflowPipelineRunner.class); + DataflowRunner.class, + BlockingDataflowRunner.class); } } } diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java index f5fefc0f6f94..e15b9d252092 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineTranslator.java @@ -30,7 +30,7 @@ import static com.google.common.base.Preconditions.checkArgument; -import org.apache.beam.runners.dataflow.DataflowPipelineRunner.GroupByKeyAndSortValuesOnly; +import org.apache.beam.runners.dataflow.DataflowRunner.GroupByKeyAndSortValuesOnly; import org.apache.beam.runners.dataflow.internal.ReadTranslator; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.runners.dataflow.util.DoFnInfo; @@ -139,7 +139,7 @@ private DataflowPipelineTranslator(DataflowPipelineOptions options) { */ public JobSpecification translate( Pipeline pipeline, - DataflowPipelineRunner runner, + DataflowRunner runner, List packages) { Translator translator = new Translator(pipeline, runner); @@ -224,7 +224,7 @@ public void translate(TransformT transform, /** * The interface provided to registered callbacks for interacting - * with the {@link DataflowPipelineRunner}, including reading and writing the + * with the {@link DataflowRunner}, including reading and writing the * values of {@link PCollection}s and side inputs ({@link PCollectionView}s). */ public interface TranslationContext { @@ -350,7 +350,7 @@ class Translator extends PipelineVisitor.Defaults implements TranslationContext private final Pipeline pipeline; /** The runner which will execute the pipeline. */ - private final DataflowPipelineRunner runner; + private final DataflowRunner runner; /** The Cloud Dataflow Job representation. */ private final Job job = new Job(); @@ -385,7 +385,7 @@ class Translator extends PipelineVisitor.Defaults implements TranslationContext * Constructs a Translator that will translate the specified * Pipeline into Dataflow objects. */ - public Translator(Pipeline pipeline, DataflowPipelineRunner runner) { + public Translator(Pipeline pipeline, DataflowRunner runner) { this.pipeline = pipeline; this.runner = runner; } diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java similarity index 96% rename from runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRunner.java rename to runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java index 1eb39ad9039e..91e34ac907f5 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRunner.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunner.java @@ -189,15 +189,15 @@ * *

Permissions

* When reading from a Dataflow source or writing to a Dataflow sink using - * {@code DataflowPipelineRunner}, the Google cloudservices account and the Google compute engine + * {@code DataflowRunner}, the Google cloudservices account and the Google compute engine * service account of the GCP project running the Dataflow Job will need access to the corresponding * source/sink. * *

Please see Google Cloud * Dataflow Security and Permissions for more details. */ -public class DataflowPipelineRunner extends PipelineRunner { - private static final Logger LOG = LoggerFactory.getLogger(DataflowPipelineRunner.class); +public class DataflowRunner extends PipelineRunner { + private static final Logger LOG = LoggerFactory.getLogger(DataflowRunner.class); /** Provided configuration options. */ private final DataflowPipelineOptions options; @@ -205,14 +205,14 @@ public class DataflowPipelineRunner extends PipelineRunner /** Client for the Dataflow service. This is used to actually submit jobs. */ private final Dataflow dataflowClient; - /** Translator for this DataflowPipelineRunner, based on options. */ + /** Translator for this DataflowRunner, based on options. */ private final DataflowPipelineTranslator translator; /** Custom transforms implementations. */ private final Map, Class> overrides; /** A set of user defined functions to invoke at different points in execution. */ - private DataflowPipelineRunnerHooks hooks; + private DataflowRunnerHooks hooks; // Environment version information. private static final String ENVIRONMENT_MAJOR_VERSION = "5"; @@ -243,7 +243,7 @@ public class DataflowPipelineRunner extends PipelineRunner * @param options Properties that configure the runner. * @return The newly created runner. */ - public static DataflowPipelineRunner fromOptions(PipelineOptions options) { + public static DataflowRunner fromOptions(PipelineOptions options) { // (Re-)register standard IO factories. Clobbers any prior credentials. IOChannelUtils.registerStandardIOFactories(options); @@ -284,7 +284,7 @@ public static DataflowPipelineRunner fromOptions(PipelineOptions options) { if (dataflowOptions.getFilesToStage() == null) { dataflowOptions.setFilesToStage(detectClassPathResourcesToStage( - DataflowPipelineRunner.class.getClassLoader())); + DataflowRunner.class.getClassLoader())); LOG.info("PipelineOptions.filesToStage was not specified. " + "Defaulting to files from the classpath: will stage {} files. " + "Enable logging at DEBUG level to see which files will be staged.", @@ -331,10 +331,10 @@ public static DataflowPipelineRunner fromOptions(PipelineOptions options) { + "' invalid. Please make sure the value is non-negative."); } - return new DataflowPipelineRunner(dataflowOptions); + return new DataflowRunner(dataflowOptions); } - @VisibleForTesting protected DataflowPipelineRunner(DataflowPipelineOptions options) { + @VisibleForTesting protected DataflowRunner(DataflowPipelineOptions options) { this.options = options; this.dataflowClient = options.getDataflowClient(); this.translator = DataflowPipelineTranslator.fromOptions(options); @@ -442,7 +442,7 @@ public OutputT apply( PTransform customTransform = InstanceBuilder.ofType(customTransformClass) - .withArg(DataflowPipelineRunner.class, this) + .withArg(DataflowRunner.class, this) .withArg(transformClass, transform) .build(); @@ -674,10 +674,10 @@ public DataflowPipelineTranslator getTranslator() { } /** - * Sets callbacks to invoke during execution see {@code DataflowPipelineRunnerHooks}. + * Sets callbacks to invoke during execution see {@code DataflowRunnerHooks}. */ @Experimental - public void setHooks(DataflowPipelineRunnerHooks hooks) { + public void setHooks(DataflowRunnerHooks hooks) { this.hooks = hooks; } @@ -750,7 +750,7 @@ private void recordViewUsesNonDeterministicKeyCoder(PTransform ptransform) } /** - * A {@link GroupByKey} transform for the {@link DataflowPipelineRunner} which sorts + * A {@link GroupByKey} transform for the {@link DataflowRunner} which sorts * values using the secondary key {@code K2}. * *

The {@link PCollection} created created by this {@link PTransform} will have values in @@ -898,13 +898,13 @@ public void processElement(ProcessContext c) throws Exception { } } - private final DataflowPipelineRunner runner; + private final DataflowRunner runner; private final View.AsSingleton transform; /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public BatchViewAsSingleton(DataflowPipelineRunner runner, View.AsSingleton transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public BatchViewAsSingleton(DataflowRunner runner, View.AsSingleton transform) { this.runner = runner; this.transform = transform; } @@ -926,7 +926,7 @@ public PCollectionView apply(PCollection input) { static PCollectionView applyForSingleton( - DataflowPipelineRunner runner, + DataflowRunner runner, PCollection input, DoFn>>>, IsmRecord>> doFn, @@ -991,12 +991,12 @@ static IsmRecordCoder> coderForSingleton( static class BatchViewAsIterable extends PTransform, PCollectionView>> { - private final DataflowPipelineRunner runner; + private final DataflowRunner runner; /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public BatchViewAsIterable(DataflowPipelineRunner runner, View.AsIterable transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public BatchViewAsIterable(DataflowRunner runner, View.AsIterable transform) { this.runner = runner; } @@ -1095,12 +1095,12 @@ public void processElement(ProcessContext c) throws Exception { } } - private final DataflowPipelineRunner runner; + private final DataflowRunner runner; /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public BatchViewAsList(DataflowPipelineRunner runner, View.AsList transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public BatchViewAsList(DataflowRunner runner, View.AsList transform) { this.runner = runner; } @@ -1112,7 +1112,7 @@ public PCollectionView> apply(PCollection input) { } static PCollectionView applyForIterableLike( - DataflowPipelineRunner runner, + DataflowRunner runner, PCollection input, PCollectionView view) { @@ -1263,12 +1263,12 @@ public void processElement(ProcessContext c) } } - private final DataflowPipelineRunner runner; + private final DataflowRunner runner; /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public BatchViewAsMap(DataflowPipelineRunner runner, View.AsMap transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public BatchViewAsMap(DataflowRunner runner, View.AsMap transform) { this.runner = runner; } @@ -1753,12 +1753,12 @@ public void processElement(ProcessContext c) } } - private final DataflowPipelineRunner runner; + private final DataflowRunner runner; /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public BatchViewAsMultimap(DataflowPipelineRunner runner, View.AsMultimap transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public BatchViewAsMultimap(DataflowRunner runner, View.AsMultimap transform) { this.runner = runner; } @@ -1825,7 +1825,7 @@ W> applyForSingleton( } private static PCollectionView applyForMapLike( - DataflowPipelineRunner runner, + DataflowRunner runner, PCollection> input, PCollectionView view, boolean uniqueKeysExpected) throws NonDeterministicException { @@ -2099,13 +2099,13 @@ public void processElement(ProcessContext c) { * Cloud Dataflow specific path validation of {@link FileBasedSink}s. */ private static class BatchWrite extends PTransform, PDone> { - private final DataflowPipelineRunner runner; + private final DataflowRunner runner; private final Write.Bound transform; /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public BatchWrite(DataflowPipelineRunner runner, Write.Bound transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public BatchWrite(DataflowRunner runner, Write.Bound transform) { this.runner = runner; this.transform = transform; } @@ -2132,8 +2132,8 @@ private static class BatchTextIOWrite extends PTransform, PDon /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public BatchTextIOWrite(DataflowPipelineRunner runner, TextIO.Write.Bound transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public BatchTextIOWrite(DataflowRunner runner, TextIO.Write.Bound transform) { this.transform = transform; } @@ -2237,8 +2237,8 @@ private static class BatchAvroIOWrite extends PTransform, PDon /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public BatchAvroIOWrite(DataflowPipelineRunner runner, AvroIO.Write.Bound transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public BatchAvroIOWrite(DataflowRunner runner, AvroIO.Write.Bound transform) { this.transform = transform; } @@ -2337,8 +2337,8 @@ private static class StreamingWrite extends PTransform, PDone> /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public StreamingWrite(DataflowPipelineRunner runner, Write.Bound transform) { } + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public StreamingWrite(DataflowRunner runner, Write.Bound transform) { } @Override public PDone apply(PCollection input) { @@ -2367,7 +2367,7 @@ private static class StreamingPubsubIORead extends PTransform transform) { + DataflowRunner runner, PubsubUnboundedSource transform) { this.transform = transform; } @@ -2445,7 +2445,7 @@ private static class StreamingPubsubIOWrite extends PTransform * Builds an instance of this class from the overridden transform. */ public StreamingPubsubIOWrite( - DataflowPipelineRunner runner, PubsubUnboundedSink transform) { + DataflowRunner runner, PubsubUnboundedSink transform) { this.transform = transform; } @@ -2521,8 +2521,8 @@ private static class StreamingUnboundedRead extends PTransform transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public StreamingUnboundedRead(DataflowRunner runner, Read.Unbounded transform) { this.source = transform.getSource(); } @@ -2634,8 +2634,8 @@ private static class StreamingCreate extends PTransform transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public StreamingCreate(DataflowRunner runner, Create.Values transform) { this.transform = transform; } @@ -2748,10 +2748,10 @@ public void processElement(ProcessContext c) throws Exception { */ private static class StreamingViewAsMap extends PTransform>, PCollectionView>> { - private final DataflowPipelineRunner runner; + private final DataflowRunner runner; - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public StreamingViewAsMap(DataflowPipelineRunner runner, View.AsMap transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public StreamingViewAsMap(DataflowRunner runner, View.AsMap transform) { this.runner = runner; } @@ -2790,13 +2790,13 @@ protected String getKindString() { */ private static class StreamingViewAsMultimap extends PTransform>, PCollectionView>>> { - private final DataflowPipelineRunner runner; + private final DataflowRunner runner; /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public StreamingViewAsMultimap(DataflowPipelineRunner runner, View.AsMultimap transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public StreamingViewAsMultimap(DataflowRunner runner, View.AsMultimap transform) { this.runner = runner; } @@ -2838,8 +2838,8 @@ private static class StreamingViewAsList /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public StreamingViewAsList(DataflowPipelineRunner runner, View.AsList transform) {} + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public StreamingViewAsList(DataflowRunner runner, View.AsList transform) {} @Override public PCollectionView> apply(PCollection input) { @@ -2870,8 +2870,8 @@ private static class StreamingViewAsIterable /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public StreamingViewAsIterable(DataflowPipelineRunner runner, View.AsIterable transform) { } + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public StreamingViewAsIterable(DataflowRunner runner, View.AsIterable transform) { } @Override public PCollectionView> apply(PCollection input) { @@ -2911,8 +2911,8 @@ private static class StreamingViewAsSingleton /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public StreamingViewAsSingleton(DataflowPipelineRunner runner, View.AsSingleton transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public StreamingViewAsSingleton(DataflowRunner runner, View.AsSingleton transform) { this.transform = transform; } @@ -2967,9 +2967,9 @@ private static class StreamingCombineGloballyAsSingletonView /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() public StreamingCombineGloballyAsSingletonView( - DataflowPipelineRunner runner, + DataflowRunner runner, Combine.GloballyAsSingletonView transform) { this.transform = transform; } @@ -3060,64 +3060,64 @@ private static class UnsupportedIO transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public UnsupportedIO(DataflowRunner runner, AvroIO.Read.Bound transform) { this.transform = transform; } /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public UnsupportedIO(DataflowPipelineRunner runner, BigQueryIO.Read.Bound transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public UnsupportedIO(DataflowRunner runner, BigQueryIO.Read.Bound transform) { this.transform = transform; } /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public UnsupportedIO(DataflowPipelineRunner runner, TextIO.Read.Bound transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public UnsupportedIO(DataflowRunner runner, TextIO.Read.Bound transform) { this.transform = transform; } /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public UnsupportedIO(DataflowPipelineRunner runner, Read.Bounded transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public UnsupportedIO(DataflowRunner runner, Read.Bounded transform) { this.transform = transform; } /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public UnsupportedIO(DataflowPipelineRunner runner, Read.Unbounded transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public UnsupportedIO(DataflowRunner runner, Read.Unbounded transform) { this.transform = transform; } /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public UnsupportedIO(DataflowPipelineRunner runner, AvroIO.Write.Bound transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public UnsupportedIO(DataflowRunner runner, AvroIO.Write.Bound transform) { this.transform = transform; } /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public UnsupportedIO(DataflowPipelineRunner runner, TextIO.Write.Bound transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public UnsupportedIO(DataflowRunner runner, TextIO.Write.Bound transform) { this.transform = transform; } /** * Builds an instance of this class from the overridden doFn. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public UnsupportedIO(DataflowPipelineRunner runner, + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public UnsupportedIO(DataflowRunner runner, PubsubIO.Read.Bound.PubsubBoundedReader doFn) { this.doFn = doFn; } @@ -3125,8 +3125,8 @@ public UnsupportedIO(DataflowPipelineRunner runner, /** * Builds an instance of this class from the overridden doFn. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public UnsupportedIO(DataflowPipelineRunner runner, + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public UnsupportedIO(DataflowRunner runner, PubsubIO.Write.Bound.PubsubBoundedWriter doFn) { this.doFn = doFn; } @@ -3134,16 +3134,16 @@ public UnsupportedIO(DataflowPipelineRunner runner, /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public UnsupportedIO(DataflowPipelineRunner runner, PubsubUnboundedSource transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public UnsupportedIO(DataflowRunner runner, PubsubUnboundedSource transform) { this.transform = transform; } /** * Builds an instance of this class from the overridden transform. */ - @SuppressWarnings("unused") // used via reflection in DataflowPipelineRunner#apply() - public UnsupportedIO(DataflowPipelineRunner runner, PubsubUnboundedSink transform) { + @SuppressWarnings("unused") // used via reflection in DataflowRunner#apply() + public UnsupportedIO(DataflowRunner runner, PubsubUnboundedSink transform) { this.transform = transform; } @@ -3157,13 +3157,13 @@ public OutputT apply(InputT input) { ? approximateSimpleName(doFn.getClass()) : approximatePTransformName(transform.getClass()); throw new UnsupportedOperationException( - String.format("The DataflowPipelineRunner in %s mode does not support %s.", mode, name)); + String.format("The DataflowRunner in %s mode does not support %s.", mode, name)); } } @Override public String toString() { - return "DataflowPipelineRunner#" + options.getJobName(); + return "DataflowRunner#" + options.getJobName(); } /** diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRunnerHooks.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunnerHooks.java similarity index 93% rename from runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRunnerHooks.java rename to runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunnerHooks.java index 4d3796631744..05297ec52046 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowPipelineRunnerHooks.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/DataflowRunnerHooks.java @@ -23,11 +23,11 @@ /** * An instance of this class can be passed to the - * {@link DataflowPipelineRunner} to add user defined hooks to be + * {@link DataflowRunner} to add user defined hooks to be * invoked at various times during pipeline execution. */ @Experimental -public class DataflowPipelineRunnerHooks { +public class DataflowRunnerHooks { /** * Allows the user to modify the environment of their job before their job is submitted * to the service for execution. diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/package-info.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/package-info.java index f2e8459bba3d..ea83bfbb6693 100755 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/package-info.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/internal/package-info.java @@ -16,6 +16,6 @@ * limitations under the License. */ /** - * Implementation of the {@link org.apache.beam.runners.dataflow.DataflowPipelineRunner}. + * Implementation of the {@link org.apache.beam.runners.dataflow.DataflowRunner}. */ package org.apache.beam.runners.dataflow.internal; diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/BlockingDataflowPipelineOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/BlockingDataflowPipelineOptions.java index 7fa5ad6a8266..809df35c33ba 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/BlockingDataflowPipelineOptions.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/BlockingDataflowPipelineOptions.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.options; -import org.apache.beam.runners.dataflow.BlockingDataflowPipelineRunner; +import org.apache.beam.runners.dataflow.BlockingDataflowRunner; import org.apache.beam.sdk.options.Default; import org.apache.beam.sdk.options.DefaultValueFactory; import org.apache.beam.sdk.options.Description; @@ -29,9 +29,9 @@ import java.io.PrintStream; /** - * Options that are used to configure the {@link BlockingDataflowPipelineRunner}. + * Options that are used to configure the {@link BlockingDataflowRunner}. */ -@Description("Configure options on the BlockingDataflowPipelineRunner.") +@Description("Configure options on the BlockingDataflowRunner.") public interface BlockingDataflowPipelineOptions extends DataflowPipelineOptions { /** * Output stream for job status messages. diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java index 6e6ad96d0a49..f665a08de778 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineOptions.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.options; -import org.apache.beam.runners.dataflow.DataflowPipelineRunner; +import org.apache.beam.runners.dataflow.DataflowRunner; import org.apache.beam.sdk.options.ApplicationNameOptions; import org.apache.beam.sdk.options.BigQueryOptions; import org.apache.beam.sdk.options.Default; @@ -38,7 +38,7 @@ import org.joda.time.format.DateTimeFormatter; /** - * Options that can be used to configure the {@link DataflowPipelineRunner}. + * Options that can be used to configure the {@link DataflowRunner}. */ @Description("Options that configure the Dataflow pipeline.") public interface DataflowPipelineOptions diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineWorkerPoolOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineWorkerPoolOptions.java index a29b328fe871..e3a1a0f5bdef 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineWorkerPoolOptions.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineWorkerPoolOptions.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.options; -import org.apache.beam.runners.dataflow.DataflowPipelineRunner; +import org.apache.beam.runners.dataflow.DataflowRunner; import org.apache.beam.sdk.annotations.Experimental; import org.apache.beam.sdk.options.Default; import org.apache.beam.sdk.options.DefaultValueFactory; @@ -131,9 +131,9 @@ public static class WorkerHarnessContainerImageFactory public String create(PipelineOptions options) { DataflowPipelineOptions dataflowOptions = options.as(DataflowPipelineOptions.class); if (dataflowOptions.isStreaming()) { - return DataflowPipelineRunner.STREAMING_WORKER_HARNESS_CONTAINER_IMAGE; + return DataflowRunner.STREAMING_WORKER_HARNESS_CONTAINER_IMAGE; } else { - return DataflowPipelineRunner.BATCH_WORKER_HARNESS_CONTAINER_IMAGE; + return DataflowRunner.BATCH_WORKER_HARNESS_CONTAINER_IMAGE; } } } diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowPipelineRunner.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowPipelineRunner.java index c940e9a5b7d0..f83a139e0ff6 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowPipelineRunner.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/testing/TestDataflowPipelineRunner.java @@ -21,7 +21,7 @@ import org.apache.beam.runners.dataflow.DataflowJobExecutionException; import org.apache.beam.runners.dataflow.DataflowPipelineJob; -import org.apache.beam.runners.dataflow.DataflowPipelineRunner; +import org.apache.beam.runners.dataflow.DataflowRunner; import org.apache.beam.runners.dataflow.util.MonitoringUtil; import org.apache.beam.runners.dataflow.util.MonitoringUtil.JobMessagesHandler; import org.apache.beam.sdk.Pipeline; @@ -54,7 +54,7 @@ /** * {@link TestDataflowPipelineRunner} is a pipeline runner that wraps a - * {@link DataflowPipelineRunner} when running tests against the {@link TestPipeline}. + * {@link DataflowRunner} when running tests against the {@link TestPipeline}. * * @see TestPipeline */ @@ -63,12 +63,12 @@ public class TestDataflowPipelineRunner extends PipelineRunner answer(InvocationOnMock invocation) throws Throwable { when(mockGcsUtil.isGcsPatternSupported(anyString())).thenCallRealMethod(); DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); options.setGcpCredential(new TestCredential()); options.setJobName("some-job-name"); options.setProject("some-project"); @@ -178,14 +178,14 @@ public List answer(InvocationOnMock invocation) throws Throwable { @Test public void testSettingOfSdkPipelineOptions() throws IOException { DataflowPipelineOptions options = buildPipelineOptions(); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); Pipeline p = buildPipeline(options); p.traverseTopologically(new RecordingPipelineVisitor()); Job job = DataflowPipelineTranslator.fromOptions(options) .translate( - p, (DataflowPipelineRunner) p.getRunner(), Collections.emptyList()) + p, (DataflowRunner) p.getRunner(), Collections.emptyList()) .getJob(); // Note that the contents of this materialized map may be changed by the act of reading an @@ -196,7 +196,7 @@ public void testSettingOfSdkPipelineOptions() throws IOException { settings.put("project", "some-project"); settings.put("pathValidatorClass", "org.apache.beam.runners.dataflow.util.DataflowPathValidator"); - settings.put("runner", "org.apache.beam.runners.dataflow.DataflowPipelineRunner"); + settings.put("runner", "org.apache.beam.runners.dataflow.DataflowRunner"); settings.put("jobName", "some-job-name"); settings.put("tempLocation", "gs://somebucket/some/path"); settings.put("stagingLocation", "gs://somebucket/some/path/staging"); @@ -222,7 +222,7 @@ public void testNetworkConfig() throws IOException { Job job = DataflowPipelineTranslator.fromOptions(options) .translate( - p, (DataflowPipelineRunner) p.getRunner(), Collections.emptyList()) + p, (DataflowRunner) p.getRunner(), Collections.emptyList()) .getJob(); assertEquals(1, job.getEnvironment().getWorkerPools().size()); @@ -239,7 +239,7 @@ public void testNetworkConfigMissing() throws IOException { Job job = DataflowPipelineTranslator.fromOptions(options) .translate( - p, (DataflowPipelineRunner) p.getRunner(), Collections.emptyList()) + p, (DataflowRunner) p.getRunner(), Collections.emptyList()) .getJob(); assertEquals(1, job.getEnvironment().getWorkerPools().size()); @@ -258,7 +258,7 @@ public void testSubnetworkConfig() throws IOException { Job job = DataflowPipelineTranslator.fromOptions(options) .translate( - p, (DataflowPipelineRunner) p.getRunner(), Collections.emptyList()) + p, (DataflowRunner) p.getRunner(), Collections.emptyList()) .getJob(); assertEquals(1, job.getEnvironment().getWorkerPools().size()); @@ -275,7 +275,7 @@ public void testSubnetworkConfigMissing() throws IOException { Job job = DataflowPipelineTranslator.fromOptions(options) .translate( - p, (DataflowPipelineRunner) p.getRunner(), Collections.emptyList()) + p, (DataflowRunner) p.getRunner(), Collections.emptyList()) .getJob(); assertEquals(1, job.getEnvironment().getWorkerPools().size()); @@ -291,7 +291,7 @@ public void testScalingAlgorithmMissing() throws IOException { Job job = DataflowPipelineTranslator.fromOptions(options) .translate( - p, (DataflowPipelineRunner) p.getRunner(), Collections.emptyList()) + p, (DataflowRunner) p.getRunner(), Collections.emptyList()) .getJob(); assertEquals(1, job.getEnvironment().getWorkerPools().size()); @@ -327,7 +327,7 @@ public void testScalingAlgorithmNone() throws IOException { Job job = DataflowPipelineTranslator.fromOptions(options) .translate( - p, (DataflowPipelineRunner) p.getRunner(), Collections.emptyList()) + p, (DataflowRunner) p.getRunner(), Collections.emptyList()) .getJob(); assertEquals(1, job.getEnvironment().getWorkerPools().size()); @@ -362,7 +362,7 @@ public void testMaxNumWorkersIsPassedWhenNoAlgorithmIsSet() throws IOException { Job job = DataflowPipelineTranslator.fromOptions(options) .translate( - p, (DataflowPipelineRunner) p.getRunner(), Collections.emptyList()) + p, (DataflowRunner) p.getRunner(), Collections.emptyList()) .getJob(); assertEquals(1, job.getEnvironment().getWorkerPools().size()); @@ -396,7 +396,7 @@ public void testZoneConfig() throws IOException { Job job = DataflowPipelineTranslator.fromOptions(options) .translate( - p, (DataflowPipelineRunner) p.getRunner(), Collections.emptyList()) + p, (DataflowRunner) p.getRunner(), Collections.emptyList()) .getJob(); assertEquals(1, job.getEnvironment().getWorkerPools().size()); @@ -416,7 +416,7 @@ public void testWorkerMachineTypeConfig() throws IOException { Job job = DataflowPipelineTranslator.fromOptions(options) .translate( - p, (DataflowPipelineRunner) p.getRunner(), Collections.emptyList()) + p, (DataflowRunner) p.getRunner(), Collections.emptyList()) .getJob(); assertEquals(1, job.getEnvironment().getWorkerPools().size()); @@ -437,7 +437,7 @@ public void testDiskSizeGbConfig() throws IOException { Job job = DataflowPipelineTranslator.fromOptions(options) .translate( - p, (DataflowPipelineRunner) p.getRunner(), Collections.emptyList()) + p, (DataflowRunner) p.getRunner(), Collections.emptyList()) .getJob(); assertEquals(1, job.getEnvironment().getWorkerPools().size()); @@ -466,7 +466,7 @@ public void testPredefinedAddStep() throws Exception { translator .translate( pipeline, - (DataflowPipelineRunner) pipeline.getRunner(), + (DataflowRunner) pipeline.getRunner(), Collections.emptyList()) .getJob(); @@ -520,7 +520,7 @@ private static Step createPredefinedStep() throws Exception { translator .translate( pipeline, - (DataflowPipelineRunner) pipeline.getRunner(), + (DataflowRunner) pipeline.getRunner(), Collections.emptyList()) .getJob(); @@ -655,7 +655,7 @@ public void testMultiGraphPipelineSerialization() throws IOException { // Check that translation doesn't fail. t.translate( - p, (DataflowPipelineRunner) p.getRunner(), Collections.emptyList()); + p, (DataflowRunner) p.getRunner(), Collections.emptyList()); } @Test @@ -698,7 +698,7 @@ public void testGoodWildcards() throws Exception { // Check that translation doesn't fail. t.translate( pipeline, - (DataflowPipelineRunner) pipeline.getRunner(), + (DataflowRunner) pipeline.getRunner(), Collections.emptyList()); } @@ -724,7 +724,7 @@ public void testBadWildcardRecursive() throws Exception { ThrowableMessageMatcher.hasMessage(containsString("Unsupported wildcard usage")))); t.translate( pipeline, - (DataflowPipelineRunner) pipeline.getRunner(), + (DataflowRunner) pipeline.getRunner(), Collections.emptyList()); } @@ -745,7 +745,7 @@ public void testToSingletonTranslation() throws Exception { translator .translate( pipeline, - (DataflowPipelineRunner) pipeline.getRunner(), + (DataflowRunner) pipeline.getRunner(), Collections.emptyList()) .getJob(); @@ -777,7 +777,7 @@ public void testToIterableTranslation() throws Exception { translator .translate( pipeline, - (DataflowPipelineRunner) pipeline.getRunner(), + (DataflowRunner) pipeline.getRunner(), Collections.emptyList()) .getJob(); @@ -807,7 +807,7 @@ public void testToSingletonTranslationWithIsmSideInput() throws Exception { translator .translate( pipeline, - (DataflowPipelineRunner) pipeline.getRunner(), + (DataflowRunner) pipeline.getRunner(), Collections.emptyList()) .getJob(); @@ -840,7 +840,7 @@ public void testToIterableTranslationWithIsmSideInput() throws Exception { translator .translate( pipeline, - (DataflowPipelineRunner) pipeline.getRunner(), + (DataflowRunner) pipeline.getRunner(), Collections.emptyList()) .getJob(); @@ -903,7 +903,7 @@ public void populateDisplayData(DisplayData.Builder builder) { translator .translate( pipeline, - (DataflowPipelineRunner) pipeline.getRunner(), + (DataflowRunner) pipeline.getRunner(), Collections.emptyList()) .getJob(); diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java similarity index 94% rename from runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineRunnerTest.java rename to runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java index 38d4c96cb377..e094d0d3ed38 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowPipelineRunnerTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/DataflowRunnerTest.java @@ -37,11 +37,11 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import org.apache.beam.runners.dataflow.DataflowPipelineRunner.BatchViewAsList; -import org.apache.beam.runners.dataflow.DataflowPipelineRunner.BatchViewAsMap; -import org.apache.beam.runners.dataflow.DataflowPipelineRunner.BatchViewAsMultimap; -import org.apache.beam.runners.dataflow.DataflowPipelineRunner.BatchViewAsSingleton; -import org.apache.beam.runners.dataflow.DataflowPipelineRunner.TransformedMap; +import org.apache.beam.runners.dataflow.DataflowRunner.BatchViewAsList; +import org.apache.beam.runners.dataflow.DataflowRunner.BatchViewAsMap; +import org.apache.beam.runners.dataflow.DataflowRunner.BatchViewAsMultimap; +import org.apache.beam.runners.dataflow.DataflowRunner.BatchViewAsSingleton; +import org.apache.beam.runners.dataflow.DataflowRunner.TransformedMap; import org.apache.beam.runners.dataflow.internal.IsmFormat; import org.apache.beam.runners.dataflow.internal.IsmFormat.IsmRecord; import org.apache.beam.runners.dataflow.internal.IsmFormat.IsmRecordCoder; @@ -127,10 +127,10 @@ import java.util.regex.Pattern; /** - * Tests for the {@link DataflowPipelineRunner}. + * Tests for the {@link DataflowRunner}. */ @RunWith(JUnit4.class) -public class DataflowPipelineRunnerTest { +public class DataflowRunnerTest { private static final String PROJECT_ID = "some-project"; @@ -148,7 +148,7 @@ private static void assertValidJob(Job job) { private Pipeline buildDataflowPipeline(DataflowPipelineOptions options) { options.setStableUniqueNames(CheckEnabled.ERROR); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); Pipeline p = Pipeline.create(options); p.apply(TextIO.Read.named("ReadMyFile").from("gs://bucket/object")) @@ -219,7 +219,7 @@ private DataflowPipelineOptions buildPipelineOptions() throws IOException { private DataflowPipelineOptions buildPipelineOptions( ArgumentCaptor jobCaptor) throws IOException { DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); options.setProject(PROJECT_ID); options.setTempLocation("gs://somebucket/some/path"); // Set FILES_PROPERTY to empty to prevent a default value calculated from classpath. @@ -237,7 +237,7 @@ public void testFromOptionsWithUppercaseConvertsToLowercase() throws Exception { DataflowPipelineOptions options = buildPipelineOptions(jobCaptor); options.setJobName(mixedCase); - DataflowPipelineRunner runner = DataflowPipelineRunner.fromOptions(options); + DataflowRunner runner = DataflowRunner.fromOptions(options); assertThat(options.getJobName(), equalTo(mixedCase.toLowerCase())); } @@ -338,7 +338,7 @@ protected boolean matchesSafely(DataflowJobAlreadyUpdatedException item) { @Test public void testRunWithFiles() throws IOException { - // Test that the function DataflowPipelineRunner.stageFiles works as + // Test that the function DataflowRunner.stageFiles works as // expected. GcsUtil mockGcsUtil = buildMockGcsUtil(true /* bucket exists */); final String gcsStaging = "gs://somebucket/some/path"; @@ -346,9 +346,9 @@ public void testRunWithFiles() throws IOException { final String cloudDataflowDataset = "somedataset"; // Create some temporary files. - File temp1 = File.createTempFile("DataflowPipelineRunnerTest", "txt"); + File temp1 = File.createTempFile("DataflowRunnerTest", "txt"); temp1.deleteOnExit(); - File temp2 = File.createTempFile("DataflowPipelineRunnerTest2", "txt"); + File temp2 = File.createTempFile("DataflowRunnerTest2", "txt"); temp2.deleteOnExit(); String overridePackageName = "alias.txt"; @@ -403,7 +403,7 @@ public void testRunWithFiles() throws IOException { public void runWithDefaultFilesToStage() throws Exception { DataflowPipelineOptions options = buildPipelineOptions(); options.setFilesToStage(null); - DataflowPipelineRunner.fromOptions(options); + DataflowRunner.fromOptions(options); assertTrue(!options.getFilesToStage().isEmpty()); } @@ -417,7 +417,7 @@ public void detectClassPathResourceWithFileResources() throws Exception { }); assertEquals(ImmutableList.of(file.getAbsolutePath(), file2.getAbsolutePath()), - DataflowPipelineRunner.detectClassPathResourcesToStage(classLoader)); + DataflowRunner.detectClassPathResourcesToStage(classLoader)); } @Test @@ -426,7 +426,7 @@ public void detectClassPathResourcesWithUnsupportedClassLoader() { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Unable to use ClassLoader to detect classpath elements."); - DataflowPipelineRunner.detectClassPathResourcesToStage(mockClassLoader); + DataflowRunner.detectClassPathResourcesToStage(mockClassLoader); } @Test @@ -438,7 +438,7 @@ public void detectClassPathResourceWithNonFileResources() throws Exception { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Unable to convert url (" + url + ") to file."); - DataflowPipelineRunner.detectClassPathResourcesToStage(classLoader); + DataflowRunner.detectClassPathResourcesToStage(classLoader); } @Test @@ -452,9 +452,9 @@ public void testGcsStagingLocationInitialization() throws Exception { options.setProject(PROJECT_ID); options.setGcpCredential(new TestCredential()); options.setGcsUtil(buildMockGcsUtil(true /* bucket exists */)); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); - DataflowPipelineRunner.fromOptions(options); + DataflowRunner.fromOptions(options); assertNotNull(options.getStagingLocation()); } @@ -518,7 +518,7 @@ public void testInvalidTempLocation() throws IOException { thrown.expect(IllegalArgumentException.class); thrown.expectMessage(containsString("expected a valid 'gs://' path but was given")); - DataflowPipelineRunner.fromOptions(options); + DataflowRunner.fromOptions(options); assertValidJob(jobCaptor.getValue()); } @@ -527,14 +527,14 @@ public void testInvalidStagingLocation() throws IOException { DataflowPipelineOptions options = buildPipelineOptions(); options.setStagingLocation("file://my/staging/location"); try { - DataflowPipelineRunner.fromOptions(options); + DataflowRunner.fromOptions(options); fail("fromOptions should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), containsString("expected a valid 'gs://' path but was given")); } options.setStagingLocation("my/staging/location"); try { - DataflowPipelineRunner.fromOptions(options); + DataflowRunner.fromOptions(options); fail("fromOptions should have failed"); } catch (IllegalArgumentException e) { assertThat(e.getMessage(), containsString("expected a valid 'gs://' path but was given")); @@ -553,7 +553,7 @@ public void testNonExistentTempLocation() throws IOException { thrown.expect(IllegalArgumentException.class); thrown.expectMessage(containsString( "Output path does not exist or is not writeable: gs://non-existent-bucket/location")); - DataflowPipelineRunner.fromOptions(options); + DataflowRunner.fromOptions(options); assertValidJob(jobCaptor.getValue()); } @@ -569,7 +569,7 @@ public void testNonExistentStagingLocation() throws IOException { thrown.expect(IllegalArgumentException.class); thrown.expectMessage(containsString( "Output path does not exist or is not writeable: gs://non-existent-bucket/location")); - DataflowPipelineRunner.fromOptions(options); + DataflowRunner.fromOptions(options); assertValidJob(jobCaptor.getValue()); } @@ -577,7 +577,7 @@ public void testNonExistentStagingLocation() throws IOException { public void testNoProjectFails() { DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); // Explicitly set to null to prevent the default instance factory from reading credentials // from a user's environment, causing this test to fail. options.setProject(null); @@ -586,39 +586,39 @@ public void testNoProjectFails() { thrown.expectMessage("Project id"); thrown.expectMessage("when running a Dataflow in the cloud"); - DataflowPipelineRunner.fromOptions(options); + DataflowRunner.fromOptions(options); } @Test public void testProjectId() throws IOException { DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); options.setProject("foo-12345"); options.setStagingLocation("gs://spam/ham/eggs"); options.setGcsUtil(buildMockGcsUtil(true /* bucket exists */)); options.setGcpCredential(new TestCredential()); - DataflowPipelineRunner.fromOptions(options); + DataflowRunner.fromOptions(options); } @Test public void testProjectPrefix() throws IOException { DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); options.setProject("google.com:some-project-12345"); options.setStagingLocation("gs://spam/ham/eggs"); options.setGcsUtil(buildMockGcsUtil(true /* bucket exists */)); options.setGcpCredential(new TestCredential()); - DataflowPipelineRunner.fromOptions(options); + DataflowRunner.fromOptions(options); } @Test public void testProjectNumber() throws IOException { DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); options.setProject("12345"); options.setStagingLocation("gs://spam/ham/eggs"); @@ -628,13 +628,13 @@ public void testProjectNumber() throws IOException { thrown.expectMessage("Project ID"); thrown.expectMessage("project number"); - DataflowPipelineRunner.fromOptions(options); + DataflowRunner.fromOptions(options); } @Test public void testProjectDescription() throws IOException { DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); options.setProject("some project"); options.setStagingLocation("gs://spam/ham/eggs"); @@ -644,13 +644,13 @@ public void testProjectDescription() throws IOException { thrown.expectMessage("Project ID"); thrown.expectMessage("project description"); - DataflowPipelineRunner.fromOptions(options); + DataflowRunner.fromOptions(options); } @Test public void testInvalidNumberOfWorkerHarnessThreads() throws IOException { DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); options.setProject("foo-12345"); options.setStagingLocation("gs://spam/ham/eggs"); @@ -662,44 +662,44 @@ public void testInvalidNumberOfWorkerHarnessThreads() throws IOException { thrown.expectMessage("Number of worker harness threads"); thrown.expectMessage("Please make sure the value is non-negative."); - DataflowPipelineRunner.fromOptions(options); + DataflowRunner.fromOptions(options); } @Test public void testNoStagingLocationAndNoTempLocationFails() { DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); options.setProject("foo-project"); thrown.expect(IllegalArgumentException.class); thrown.expectMessage( "Missing required value: at least one of tempLocation or stagingLocation must be set."); - DataflowPipelineRunner.fromOptions(options); + DataflowRunner.fromOptions(options); } @Test public void testStagingLocationAndNoTempLocationSucceeds() throws Exception { DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); options.setGcpCredential(new TestCredential()); options.setProject("foo-project"); options.setStagingLocation("gs://spam/ham/eggs"); options.setGcsUtil(buildMockGcsUtil(true /* bucket exists */)); - DataflowPipelineRunner.fromOptions(options); + DataflowRunner.fromOptions(options); } @Test public void testTempLocationAndNoStagingLocationSucceeds() throws Exception { DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); options.setGcpCredential(new TestCredential()); options.setProject("foo-project"); options.setTempLocation("gs://spam/ham/eggs"); options.setGcsUtil(buildMockGcsUtil(true /* bucket exists */)); - DataflowPipelineRunner.fromOptions(options); + DataflowRunner.fromOptions(options); } @Test @@ -718,7 +718,7 @@ public void testInvalidJobName() throws IOException { options.setJobName(invalidNames.get(i)); try { - DataflowPipelineRunner.fromOptions(options); + DataflowRunner.fromOptions(options); fail("Expected IllegalArgumentException for jobName " + options.getJobName()); } catch (IllegalArgumentException e) { @@ -737,7 +737,7 @@ public void testValidJobName() throws IOException { DataflowPipelineOptions options = buildPipelineOptions(); options.setJobName(name); - DataflowPipelineRunner runner = DataflowPipelineRunner + DataflowRunner runner = DataflowRunner .fromOptions(options); assertNotNull(runner); } @@ -779,7 +779,7 @@ public void testTransformTranslatorMissing() throws IOException { thrown.expectMessage(Matchers.containsString("no translator registered")); DataflowPipelineTranslator.fromOptions(options) .translate( - p, (DataflowPipelineRunner) p.getRunner(), Collections.emptyList()); + p, (DataflowRunner) p.getRunner(), Collections.emptyList()); assertValidJob(jobCaptor.getValue()); } @@ -793,7 +793,7 @@ public void testTransformTranslator() throws IOException { p.apply(Create.of(Arrays.asList(1, 2, 3)).withCoder(BigEndianIntegerCoder.of())) .apply(transform); - DataflowPipelineTranslator translator = DataflowPipelineRunner + DataflowPipelineTranslator translator = DataflowRunner .fromOptions(options).getTranslator(); DataflowPipelineTranslator.registerTransformTranslator( @@ -814,7 +814,7 @@ public void translate( }); translator.translate( - p, (DataflowPipelineRunner) p.getRunner(), Collections.emptyList()); + p, (DataflowRunner) p.getRunner(), Collections.emptyList()); assertTrue(transform.translated); } @@ -867,15 +867,15 @@ public void testToString() { options.setTempLocation("gs://test/temp/location"); options.setGcpCredential(new TestCredential()); options.setPathValidatorClass(NoopPathValidator.class); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); assertEquals( - "DataflowPipelineRunner#testjobname", - DataflowPipelineRunner.fromOptions(options).toString()); + "DataflowRunner#testjobname", + DataflowRunner.fromOptions(options).toString()); } private static PipelineOptions makeOptions(boolean streaming) { DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); options.setStreaming(streaming); options.setJobName("TestJobName"); options.setProject("test-project"); @@ -890,7 +890,7 @@ private void testUnsupportedSource(PTransform source, String name, bo String mode = streaming ? "streaming" : "batch"; thrown.expect(UnsupportedOperationException.class); thrown.expectMessage( - "The DataflowPipelineRunner in " + mode + " mode does not support " + name); + "The DataflowRunner in " + mode + " mode does not support " + name); Pipeline p = Pipeline.create(makeOptions(streaming)); p.apply(source); @@ -935,7 +935,7 @@ private void testUnsupportedSink( throws Exception { thrown.expect(UnsupportedOperationException.class); thrown.expectMessage( - "The DataflowPipelineRunner in streaming mode does not support " + name); + "The DataflowRunner in streaming mode does not support " + name); Pipeline p = Pipeline.create(makeOptions(streaming)); p.apply(Create.of("foo")).apply(sink); diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowAvroIOTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowAvroIOTest.java index 614affb30f62..006daa996fae 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowAvroIOTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowAvroIOTest.java @@ -22,7 +22,7 @@ import static org.hamcrest.Matchers.hasItem; import static org.junit.Assert.assertThat; -import org.apache.beam.runners.dataflow.DataflowPipelineRunner; +import org.apache.beam.runners.dataflow.DataflowRunner; import org.apache.beam.runners.dataflow.transforms.DataflowDisplayDataEvaluator; import org.apache.beam.sdk.io.AvroIO; import org.apache.beam.sdk.transforms.display.DisplayData; @@ -36,7 +36,7 @@ import java.util.Set; /** - * {@link DataflowPipelineRunner} specific tests for {@link AvroIO} transforms. + * {@link DataflowRunner} specific tests for {@link AvroIO} transforms. */ @RunWith(JUnit4.class) public class DataflowAvroIOTest { diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowPubsubIOTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowPubsubIOTest.java index 3df9cdb9d4e7..27bc2d9290e5 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowPubsubIOTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowPubsubIOTest.java @@ -22,7 +22,7 @@ import static org.hamcrest.Matchers.hasItem; import static org.junit.Assert.assertThat; -import org.apache.beam.runners.dataflow.DataflowPipelineRunner; +import org.apache.beam.runners.dataflow.DataflowRunner; import org.apache.beam.runners.dataflow.transforms.DataflowDisplayDataEvaluator; import org.apache.beam.sdk.io.PubsubIO; import org.apache.beam.sdk.transforms.display.DisplayData; @@ -35,7 +35,7 @@ import java.util.Set; /** - * {@link DataflowPipelineRunner} specific tests for {@link PubsubIO} transforms. + * {@link DataflowRunner} specific tests for {@link PubsubIO} transforms. */ @RunWith(JUnit4.class) public class DataflowPubsubIOTest { diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowTextIOTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowTextIOTest.java index 03404353e014..727ffdccab98 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowTextIOTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/io/DataflowTextIOTest.java @@ -23,7 +23,7 @@ import static org.hamcrest.Matchers.startsWith; import static org.junit.Assert.assertThat; -import org.apache.beam.runners.dataflow.DataflowPipelineRunner; +import org.apache.beam.runners.dataflow.DataflowRunner; import org.apache.beam.runners.dataflow.testing.TestDataflowPipelineOptions; import org.apache.beam.runners.dataflow.transforms.DataflowDisplayDataEvaluator; import org.apache.beam.sdk.io.TextIO; @@ -39,7 +39,7 @@ import java.util.Set; /** - * {@link DataflowPipelineRunner} specific tests for TextIO Read and Write transforms. + * {@link DataflowRunner} specific tests for TextIO Read and Write transforms. */ @RunWith(JUnit4.class) public class DataflowTextIOTest { diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowPipelineRunnerTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunnerTest.java similarity index 95% rename from runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowPipelineRunnerTest.java rename to runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunnerTest.java index fbaf116d98cb..4067f0896107 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowPipelineRunnerTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/testing/TestDataflowRunnerTest.java @@ -32,7 +32,7 @@ import static org.mockito.Mockito.when; import org.apache.beam.runners.dataflow.DataflowPipelineJob; -import org.apache.beam.runners.dataflow.DataflowPipelineRunner; +import org.apache.beam.runners.dataflow.DataflowRunner; import org.apache.beam.runners.dataflow.util.MonitoringUtil; import org.apache.beam.runners.dataflow.util.MonitoringUtil.JobMessagesHandler; import org.apache.beam.runners.dataflow.util.TimeUtil; @@ -87,7 +87,7 @@ /** Tests for {@link TestDataflowPipelineRunner}. */ @RunWith(JUnit4.class) -public class TestDataflowPipelineRunnerTest { +public class TestDataflowRunnerTest { @Rule public ExpectedException expectedException = ExpectedException.none(); @Mock private MockHttpTransport transport; @Mock private MockLowLevelHttpRequest request; @@ -132,7 +132,7 @@ public void testRunBatchJobThatSucceeds() throws Exception { when(mockJob.getProjectId()).thenReturn("test-project"); when(mockJob.getJobId()).thenReturn("test-job"); - DataflowPipelineRunner mockRunner = Mockito.mock(DataflowPipelineRunner.class); + DataflowRunner mockRunner = Mockito.mock(DataflowRunner.class); when(mockRunner.run(any(Pipeline.class))).thenReturn(mockJob); TestDataflowPipelineRunner runner = (TestDataflowPipelineRunner) p.getRunner(); @@ -153,7 +153,7 @@ public void testRunBatchJobThatFails() throws Exception { when(mockJob.getProjectId()).thenReturn("test-project"); when(mockJob.getJobId()).thenReturn("test-job"); - DataflowPipelineRunner mockRunner = Mockito.mock(DataflowPipelineRunner.class); + DataflowRunner mockRunner = Mockito.mock(DataflowRunner.class); when(mockRunner.run(any(Pipeline.class))).thenReturn(mockJob); TestDataflowPipelineRunner runner = (TestDataflowPipelineRunner) p.getRunner(); @@ -192,7 +192,7 @@ public State answer(InvocationOnMock invocation) { } }); - DataflowPipelineRunner mockRunner = Mockito.mock(DataflowPipelineRunner.class); + DataflowRunner mockRunner = Mockito.mock(DataflowRunner.class); when(mockRunner.run(any(Pipeline.class))).thenReturn(mockJob); when(request.execute()).thenReturn( @@ -223,7 +223,7 @@ public void testRunStreamingJobThatSucceeds() throws Exception { when(mockJob.getProjectId()).thenReturn("test-project"); when(mockJob.getJobId()).thenReturn("test-job"); - DataflowPipelineRunner mockRunner = Mockito.mock(DataflowPipelineRunner.class); + DataflowRunner mockRunner = Mockito.mock(DataflowRunner.class); when(mockRunner.run(any(Pipeline.class))).thenReturn(mockJob); when(request.execute()).thenReturn( @@ -245,7 +245,7 @@ public void testRunStreamingJobThatFails() throws Exception { when(mockJob.getProjectId()).thenReturn("test-project"); when(mockJob.getJobId()).thenReturn("test-job"); - DataflowPipelineRunner mockRunner = Mockito.mock(DataflowPipelineRunner.class); + DataflowRunner mockRunner = Mockito.mock(DataflowRunner.class); when(mockRunner.run(any(Pipeline.class))).thenReturn(mockJob); when(request.execute()).thenReturn( @@ -368,7 +368,7 @@ public State answer(InvocationOnMock invocation) { } }); - DataflowPipelineRunner mockRunner = Mockito.mock(DataflowPipelineRunner.class); + DataflowRunner mockRunner = Mockito.mock(DataflowRunner.class); when(mockRunner.run(any(Pipeline.class))).thenReturn(mockJob); when(request.execute()).thenReturn( @@ -398,7 +398,7 @@ public void testBatchOnCreateMatcher() throws Exception { when(mockJob.getProjectId()).thenReturn("test-project"); when(mockJob.getJobId()).thenReturn("test-job"); - DataflowPipelineRunner mockRunner = Mockito.mock(DataflowPipelineRunner.class); + DataflowRunner mockRunner = Mockito.mock(DataflowRunner.class); when(mockRunner.run(any(Pipeline.class))).thenReturn(mockJob); TestDataflowPipelineRunner runner = (TestDataflowPipelineRunner) p.getRunner(); @@ -423,7 +423,7 @@ public void testStreamingOnCreateMatcher() throws Exception { when(mockJob.getProjectId()).thenReturn("test-project"); when(mockJob.getJobId()).thenReturn("test-job"); - DataflowPipelineRunner mockRunner = Mockito.mock(DataflowPipelineRunner.class); + DataflowRunner mockRunner = Mockito.mock(DataflowRunner.class); when(mockRunner.run(any(Pipeline.class))).thenReturn(mockJob); TestDataflowPipelineRunner runner = (TestDataflowPipelineRunner) p.getRunner(); @@ -450,7 +450,7 @@ public void testBatchOnSuccessMatcherWhenPipelineSucceeds() throws Exception { when(mockJob.getProjectId()).thenReturn("test-project"); when(mockJob.getJobId()).thenReturn("test-job"); - DataflowPipelineRunner mockRunner = Mockito.mock(DataflowPipelineRunner.class); + DataflowRunner mockRunner = Mockito.mock(DataflowRunner.class); when(mockRunner.run(any(Pipeline.class))).thenReturn(mockJob); TestDataflowPipelineRunner runner = (TestDataflowPipelineRunner) p.getRunner(); @@ -475,7 +475,7 @@ public void testStreamingOnSuccessMatcherWhenPipelineSucceeds() throws Exception when(mockJob.getProjectId()).thenReturn("test-project"); when(mockJob.getJobId()).thenReturn("test-job"); - DataflowPipelineRunner mockRunner = Mockito.mock(DataflowPipelineRunner.class); + DataflowRunner mockRunner = Mockito.mock(DataflowRunner.class); when(mockRunner.run(any(Pipeline.class))).thenReturn(mockJob); TestDataflowPipelineRunner runner = (TestDataflowPipelineRunner) p.getRunner(); @@ -502,7 +502,7 @@ public void testBatchOnSuccessMatcherWhenPipelineFails() throws Exception { when(mockJob.getProjectId()).thenReturn("test-project"); when(mockJob.getJobId()).thenReturn("test-job"); - DataflowPipelineRunner mockRunner = Mockito.mock(DataflowPipelineRunner.class); + DataflowRunner mockRunner = Mockito.mock(DataflowRunner.class); when(mockRunner.run(any(Pipeline.class))).thenReturn(mockJob); TestDataflowPipelineRunner runner = (TestDataflowPipelineRunner) p.getRunner(); @@ -534,7 +534,7 @@ public void testStreamingOnSuccessMatcherWhenPipelineFails() throws Exception { when(mockJob.getProjectId()).thenReturn("test-project"); when(mockJob.getJobId()).thenReturn("test-job"); - DataflowPipelineRunner mockRunner = Mockito.mock(DataflowPipelineRunner.class); + DataflowRunner mockRunner = Mockito.mock(DataflowRunner.class); when(mockRunner.run(any(Pipeline.class))).thenReturn(mockJob); TestDataflowPipelineRunner runner = (TestDataflowPipelineRunner) p.getRunner(); diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowDisplayDataEvaluator.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowDisplayDataEvaluator.java index 0b865c3dab92..d809cc6ad92a 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowDisplayDataEvaluator.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowDisplayDataEvaluator.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.transforms; -import org.apache.beam.runners.dataflow.DataflowPipelineRunner; +import org.apache.beam.runners.dataflow.DataflowRunner; import org.apache.beam.runners.dataflow.options.DataflowPipelineDebugOptions; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.sdk.options.GcpOptions; @@ -30,7 +30,7 @@ /** * Factory methods for creating {@link DisplayDataEvaluator} instances against the - * {@link DataflowPipelineRunner}. + * {@link DataflowRunner}. */ public final class DataflowDisplayDataEvaluator { /** Do not instantiate. */ @@ -43,7 +43,7 @@ private DataflowDisplayDataEvaluator() {} public static DataflowPipelineOptions getDefaultOptions() { DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); options.setProject("foobar"); options.setTempLocation("gs://bucket/tmpLocation"); options.setFilesToStage(Lists.newArrayList()); @@ -56,7 +56,7 @@ public static DataflowPipelineOptions getDefaultOptions() { /** * Create a {@link DisplayDataEvaluator} instance to evaluate pipeline display data against - * the {@link DataflowPipelineRunner}. + * the {@link DataflowRunner}. */ public static DisplayDataEvaluator create() { return create(getDefaultOptions()); @@ -64,7 +64,7 @@ public static DisplayDataEvaluator create() { /** * Create a {@link DisplayDataEvaluator} instance to evaluate pipeline display data against - * the {@link DataflowPipelineRunner} with the specified {@code options}. + * the {@link DataflowRunner} with the specified {@code options}. */ public static DisplayDataEvaluator create(DataflowPipelineOptions options) { return DisplayDataEvaluator.create(options); diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowGroupByKeyTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowGroupByKeyTest.java index f0e677edef45..a44b8a7c96e3 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowGroupByKeyTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowGroupByKeyTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.transforms; -import org.apache.beam.runners.dataflow.DataflowPipelineRunner; +import org.apache.beam.runners.dataflow.DataflowRunner; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.coders.BigEndianIntegerCoder; @@ -46,20 +46,20 @@ import java.util.Arrays; import java.util.List; -/** Tests for {@link GroupByKey} for the {@link DataflowPipelineRunner}. */ +/** Tests for {@link GroupByKey} for the {@link DataflowRunner}. */ @RunWith(JUnit4.class) public class DataflowGroupByKeyTest { @Rule public ExpectedException thrown = ExpectedException.none(); /** - * Create a test pipeline that uses the {@link DataflowPipelineRunner} so that {@link GroupByKey} + * Create a test pipeline that uses the {@link DataflowRunner} so that {@link GroupByKey} * is not expanded. This is used for verifying that even without expansion the proper errors show * up. */ private Pipeline createTestServiceRunner() { DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); options.setProject("someproject"); options.setStagingLocation("gs://staging"); options.setPathValidatorClass(NoopPathValidator.class); diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowViewTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowViewTest.java index d78750076aeb..1b263d2e186e 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowViewTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/transforms/DataflowViewTest.java @@ -17,7 +17,7 @@ */ package org.apache.beam.runners.dataflow.transforms; -import org.apache.beam.runners.dataflow.DataflowPipelineRunner; +import org.apache.beam.runners.dataflow.DataflowRunner; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.options.PipelineOptionsFactory; @@ -44,7 +44,7 @@ import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -/** Tests for {@link View} for a {@link DataflowPipelineRunner}. */ +/** Tests for {@link View} for a {@link DataflowRunner}. */ @RunWith(JUnit4.class) public class DataflowViewTest { @Rule @@ -52,7 +52,7 @@ public class DataflowViewTest { private Pipeline createTestBatchRunner() { DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); options.setProject("someproject"); options.setStagingLocation("gs://staging"); options.setPathValidatorClass(NoopPathValidator.class); @@ -62,7 +62,7 @@ private Pipeline createTestBatchRunner() { private Pipeline createTestStreamingRunner() { DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); options.setStreaming(true); options.setProject("someproject"); options.setStagingLocation("gs://staging"); diff --git a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/DataflowPathValidatorTest.java b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/DataflowPathValidatorTest.java index 55879866aea4..a91f56cab965 100644 --- a/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/DataflowPathValidatorTest.java +++ b/runners/google-cloud-dataflow-java/src/test/java/org/apache/beam/runners/dataflow/util/DataflowPathValidatorTest.java @@ -21,7 +21,7 @@ import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.when; -import org.apache.beam.runners.dataflow.DataflowPipelineRunner; +import org.apache.beam.runners.dataflow.DataflowRunner; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.sdk.options.PipelineOptionsFactory; import org.apache.beam.sdk.util.GcsUtil; @@ -52,7 +52,7 @@ public void setUp() throws Exception { when(mockGcsUtil.isGcsPatternSupported(anyString())).thenCallRealMethod(); DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class); options.setGcpCredential(new TestCredential()); - options.setRunner(DataflowPipelineRunner.class); + options.setRunner(DataflowRunner.class); options.setGcsUtil(mockGcsUtil); validator = new DataflowPathValidator(options); } @@ -66,7 +66,7 @@ public void testValidFilePattern() { public void testInvalidFilePattern() { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage( - "DataflowPipelineRunner expected a valid 'gs://' path but was given '/local/path'"); + "DataflowRunner expected a valid 'gs://' path but was given '/local/path'"); validator.validateInputFilePatternSupported("/local/path"); } @@ -88,7 +88,7 @@ public void testValidOutputPrefix() { public void testInvalidOutputPrefix() { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage( - "DataflowPipelineRunner expected a valid 'gs://' path but was given '/local/path'"); + "DataflowRunner expected a valid 'gs://' path but was given '/local/path'"); validator.validateOutputFilePrefixSupported("/local/path"); } } diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/SimpleWordCountTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/SimpleWordCountTest.java index 61ad24ff93d9..2b4464db303a 100644 --- a/runners/spark/src/test/java/org/apache/beam/runners/spark/SimpleWordCountTest.java +++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/SimpleWordCountTest.java @@ -113,7 +113,7 @@ public void processElement(ProcessContext c) { String[] words = WORD_BOUNDARY.split(c.element()); // Keep track of the number of lines without any words encountered while tokenizing. - // This aggregator is visible in the monitoring UI when run using DataflowPipelineRunner. + // This aggregator is visible in the monitoring UI when run using DataflowRunner. if (words.length == 0) { emptyLines.addValue(1L); } diff --git a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SerializationTest.java b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SerializationTest.java index 22a22414c8b8..de3c15271df3 100644 --- a/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SerializationTest.java +++ b/runners/spark/src/test/java/org/apache/beam/runners/spark/translation/SerializationTest.java @@ -158,7 +158,7 @@ public void processElement(ProcessContext c) { String[] words = WORD_BOUNDARY.split(c.element().toString()); // Keep track of the number of lines without any words encountered while tokenizing. - // This aggregator is visible in the monitoring UI when run using DataflowPipelineRunner. + // This aggregator is visible in the monitoring UI when run using DataflowRunner. if (words.length == 0) { emptyLines.addValue(1L); } diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/package-info.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/package-info.java index 479090f67af3..8719384ce228 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/package-info.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/runners/package-info.java @@ -17,16 +17,16 @@ */ /** * Defines runners for executing Pipelines in different modes, including - * {@link org.apache.beam.sdk.runners.DirectPipelineRunner} and - * {@link org.apache.beam.sdk.runners.DataflowPipelineRunner}. + * {@link org.apache.beam.sdk.runners.DirectRunner} and + * {@link org.apache.beam.sdk.runners.DataflowRunner}. * - *

{@link org.apache.beam.sdk.runners.DirectPipelineRunner} executes a {@code Pipeline} + *

{@link org.apache.beam.sdk.runners.DirectRunner} executes a {@code Pipeline} * locally, without contacting the Dataflow service. - * {@link org.apache.beam.sdk.runners.DataflowPipelineRunner} submits a + * {@link org.apache.beam.sdk.runners.DataflowRunner} submits a * {@code Pipeline} to the Dataflow service, which executes it on Dataflow-managed Compute Engine - * instances. {@code DataflowPipelineRunner} returns + * instances. {@code DataflowRunner} returns * as soon as the {@code Pipeline} has been submitted. Use - * {@link org.apache.beam.sdk.runners.BlockingDataflowPipelineRunner} to have execution + * {@link org.apache.beam.sdk.runners.BlockingDataflowRunner} to have execution * updates printed to the console. * *

The runner is specified as part {@link org.apache.beam.sdk.options.PipelineOptions}. diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java index 0dba043b2ac1..b901268c9df2 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/testing/TestPipeline.java @@ -60,7 +60,7 @@ *

  • System property "beamTestPipelineOptions" must contain a JSON delimited list of pipeline * options. For example: *
    {@code [
    - *     "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowPipelineRunner",
    + *     "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowRunner",
      *     "--project=mygcpproject",
      *     "--stagingLocation=gs://mygcsbucket/path"
      *     ]}
    diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Aggregator.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Aggregator.java index 4c9812337b4c..329dec5bed04 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Aggregator.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/transforms/Aggregator.java @@ -28,7 +28,7 @@ * {@code Aggregator} by calling {@link Aggregator#addValue}. * *

    Aggregators are visible in the monitoring UI, when the pipeline is run - * using DataflowPipelineRunner or BlockingDataflowPipelineRunner, along with + * using DataflowRunner or BlockingDataflowRunner, along with * their current value. Aggregators may not become visible until the system * begins executing the ParDo transform that created them and/or their initial * value is changed. diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BigQueryTableRowIterator.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BigQueryTableRowIterator.java index ca1ac69c3736..11b16f26fc57 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BigQueryTableRowIterator.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/util/BigQueryTableRowIterator.java @@ -156,7 +156,7 @@ public boolean advance() throws IOException, InterruptedException { if (iteratorOverCurrentBatch != null && iteratorOverCurrentBatch.hasNext()) { // Embed schema information into the raw row, so that values have an // associated key. This matches how rows are read when using the - // DataflowPipelineRunner. + // DataflowRunner. current = getTypedTableRow(schema.getFields(), iteratorOverCurrentBatch.next()); return true; } diff --git a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/PipelineRunnerTest.java b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/PipelineRunnerTest.java index 5d2e69d796bf..7705c365b051 100644 --- a/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/PipelineRunnerTest.java +++ b/sdks/java/core/src/test/java/org/apache/beam/sdk/runners/PipelineRunnerTest.java @@ -35,7 +35,7 @@ import org.mockito.MockitoAnnotations; /** - * Tests for DataflowPipelineRunner. + * Tests for DataflowRunner. */ @RunWith(JUnit4.class) public class PipelineRunnerTest { diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/DebuggingWordCount.java b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/DebuggingWordCount.java index 43c990a9f30d..3306cb4023c2 100644 --- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/DebuggingWordCount.java +++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/DebuggingWordCount.java @@ -70,7 +70,7 @@ *

    {@code
      *   --project=YOUR_PROJECT_ID
      *   --stagingLocation=gs://YOUR_STAGING_DIRECTORY
    - *   --runner=BlockingDataflowPipelineRunner
    + *   --runner=BlockingDataflowRunner
      *   --workerLogLevelOverrides={"org.apache.beam.examples":"DEBUG"}
      * }
      * 
    diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/MinimalWordCount.java b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/MinimalWordCount.java index 3e4fc860afe3..98af2e717769 100644 --- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/MinimalWordCount.java +++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/MinimalWordCount.java @@ -17,7 +17,7 @@ */ package ${package}; -import org.apache.beam.runners.dataflow.BlockingDataflowPipelineRunner; +import org.apache.beam.runners.dataflow.BlockingDataflowRunner; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.io.TextIO; @@ -63,7 +63,7 @@ public static void main(String[] args) { // in Google Cloud Storage to stage files. DataflowPipelineOptions options = PipelineOptionsFactory.create() .as(DataflowPipelineOptions.class); - options.setRunner(BlockingDataflowPipelineRunner.class); + options.setRunner(BlockingDataflowRunner.class); // CHANGE 1/3: Your project ID is required in order to run your pipeline on the Google Cloud. options.setProject("SET_YOUR_PROJECT_ID_HERE"); // CHANGE 2/3: Your Google Cloud Storage path is required for staging local files. diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/WindowedWordCount.java b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/WindowedWordCount.java index 7dea9feb12c6..8e56b03d0cac 100644 --- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/WindowedWordCount.java +++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/WindowedWordCount.java @@ -82,7 +82,7 @@ *
    {@code
      *   --project=YOUR_PROJECT_ID
      *   --stagingLocation=gs://YOUR_STAGING_DIRECTORY
    - *   --runner=BlockingDataflowPipelineRunner
    + *   --runner=BlockingDataflowRunner
      * }
      * 
    * diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/WordCount.java b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/WordCount.java index fc1f4b55f9d2..07ed6d0ffc9c 100644 --- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/WordCount.java +++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/WordCount.java @@ -77,7 +77,7 @@ *
    {@code
      *   --project=YOUR_PROJECT_ID
      *   --stagingLocation=gs://YOUR_STAGING_DIRECTORY
    - *   --runner=BlockingDataflowPipelineRunner
    + *   --runner=BlockingDataflowRunner
      * }
      * 
    * and an output prefix on GCS: diff --git a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/DataflowExampleUtils.java b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/DataflowExampleUtils.java index 76df4d49f200..04debd302509 100644 --- a/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/DataflowExampleUtils.java +++ b/sdks/java/maven-archetypes/examples/src/main/resources/archetype-resources/src/main/java/common/DataflowExampleUtils.java @@ -17,9 +17,9 @@ */ package ${package}.common; -import org.apache.beam.runners.dataflow.BlockingDataflowPipelineRunner; +import org.apache.beam.runners.dataflow.BlockingDataflowRunner; import org.apache.beam.runners.dataflow.DataflowPipelineJob; -import org.apache.beam.runners.dataflow.DataflowPipelineRunner; +import org.apache.beam.runners.dataflow.DataflowRunner; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.runners.dataflow.util.MonitoringUtil; @@ -251,10 +251,10 @@ public void startInjectorIfNeeded(String inputFile) { } public void setupRunner() { - if (options.isStreaming() && options.getRunner().equals(BlockingDataflowPipelineRunner.class)) { + if (options.isStreaming() && options.getRunner().equals(BlockingDataflowRunner.class)) { // In order to cancel the pipelines automatically, - // {@literal DataflowPipelineRunner} is forced to be used. - options.setRunner(DataflowPipelineRunner.class); + // {@literal DataflowRunner} is forced to be used. + options.setRunner(DataflowRunner.class); } } @@ -268,7 +268,7 @@ public void runInjectorPipeline(String inputFile, String topic) { DataflowPipelineOptions copiedOptions = options.cloneAs(DataflowPipelineOptions.class); copiedOptions.setStreaming(false); copiedOptions.setWorkerHarnessContainerImage( - DataflowPipelineRunner.BATCH_WORKER_HARNESS_CONTAINER_IMAGE); + DataflowRunner.BATCH_WORKER_HARNESS_CONTAINER_IMAGE); copiedOptions.setNumWorkers( options.as(ExamplePubsubTopicOptions.class).getInjectorNumWorkers()); copiedOptions.setJobName(options.getJobName() + "-injector"); @@ -298,7 +298,7 @@ public void mockUnboundedSource(String inputFile, PipelineResult result) { } /** - * If {@literal DataflowPipelineRunner} or {@literal BlockingDataflowPipelineRunner} is used, + * If {@literal DataflowRunner} or {@literal BlockingDataflowRunner} is used, * waits for the pipeline to finish and cancels it (and the injector) before the program exists. */ public void waitToFinish(PipelineResult result) { diff --git a/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/src/main/java/StarterPipeline.java b/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/src/main/java/StarterPipeline.java index 2146b779470b..c5af620ae448 100644 --- a/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/src/main/java/StarterPipeline.java +++ b/sdks/java/maven-archetypes/starter/src/main/resources/archetype-resources/src/main/java/StarterPipeline.java @@ -40,7 +40,7 @@ * Platform, you should specify the following command-line options: * --project= * --stagingLocation= - * --runner=BlockingDataflowPipelineRunner + * --runner=BlockingDataflowRunner */ public class StarterPipeline { private static final Logger LOG = LoggerFactory.getLogger(StarterPipeline.class); diff --git a/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/src/main/java/it/pkg/StarterPipeline.java b/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/src/main/java/it/pkg/StarterPipeline.java index 6cd27e708a5f..57ad8e03fd7a 100644 --- a/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/src/main/java/it/pkg/StarterPipeline.java +++ b/sdks/java/maven-archetypes/starter/src/test/resources/projects/basic/reference/src/main/java/it/pkg/StarterPipeline.java @@ -40,7 +40,7 @@ * Platform, you should specify the following command-line options: * --project= * --stagingLocation= - * --runner=BlockingDataflowPipelineRunner + * --runner=BlockingDataflowRunner */ public class StarterPipeline { private static final Logger LOG = LoggerFactory.getLogger(StarterPipeline.class); diff --git a/sdks/java/pom.xml b/sdks/java/pom.xml index 09fe7d9f7125..55aea6a9e06c 100644 --- a/sdks/java/pom.xml +++ b/sdks/java/pom.xml @@ -37,7 +37,7 @@ core io extensions