From 35bb519398ea37c68c692c72a22b7b8f55c2b83e Mon Sep 17 00:00:00 2001 From: Pei He Date: Mon, 11 Sep 2017 17:58:45 +0800 Subject: [PATCH 1/5] mr-runner: add mapreduce-runner profile for examples, archetypes and IT tests. --- examples/java/pom.xml | 36 +++++++++++++++++++ examples/java8/pom.xml | 13 +++++++ pom.xml | 6 ++++ .../resources/archetype-resources/pom.xml | 13 +++++++ .../resources/archetype-resources/pom.xml | 13 +++++++ 5 files changed, 81 insertions(+) diff --git a/examples/java/pom.xml b/examples/java/pom.xml index e47e9a12e251d..84cc0ae0bfc68 100644 --- a/examples/java/pom.xml +++ b/examples/java/pom.xml @@ -107,6 +107,18 @@ + + + mapreduce-runner + + + org.apache.beam + beam-runners-mapreduce + runtime + + + + spark-runner @@ -235,6 +247,30 @@ + + mapreduce-runner-integration-tests + + integration-test + verify + + + + WordCountIT.java + WindowedWordCountIT.java + + all + 4 + + + [ + "--project=apache-beam-testing", + "--tempRoot=gs://temp-storage-for-end-to-end-tests", + "--runner=org.apache.beam.runners.mapreduce.MapReduceRunner" + ] + + + + spark-runner-integration-tests diff --git a/examples/java8/pom.xml b/examples/java8/pom.xml index 76518456b63ee..58b5eb28cd824 100644 --- a/examples/java8/pom.xml +++ b/examples/java8/pom.xml @@ -107,6 +107,19 @@ + + + mapreduce-runner + + + org.apache.beam + beam-runners-mapreduce + ${beam.version} + runtime + + + + spark-runner diff --git a/pom.xml b/pom.xml index 5f997a30ad58f..ff71cdd86d9de 100644 --- a/pom.xml +++ b/pom.xml @@ -736,6 +736,12 @@ ${project.version} + + org.apache.beam + beam-runners-mapreduce + ${project.version} + + org.apache.beam beam-examples-java diff --git a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml index ffdb066046667..f6e2b5b73049a 100644 --- a/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml +++ b/sdks/java/maven-archetypes/examples-java8/src/main/resources/archetype-resources/pom.xml @@ -235,6 +235,19 @@ + + mapreduce-runner + + + + org.apache.beam + beam-runners-mapreduce + ${beam.version} + runtime + + + + spark-runner + + + org.apache.beam + beam-runners-mapreduce + ${beam.version} + runtime + + + + spark-runner + examples sdks/java/javadoc From 4885c7c3649c16f41e4a89aa2366f2aa4c4d1c74 Mon Sep 17 00:00:00 2001 From: Pei He Date: Mon, 18 Sep 2017 11:02:06 +0800 Subject: [PATCH 3/5] fixup --- .test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy index 0775e2f309eb7..57a630944ca6f 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy @@ -44,7 +44,7 @@ mavenJob('beam_PreCommit_Java_MavenInstall') { goals([ '--batch-mode', '--errors', - '--activate-profiles release,jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner', + '--activate-profiles release,jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner,mapreduce-runner', '--projects sdks/java/core,runners/direct-java,sdks/java/fn-execution', '--also-make', '--also-make-dependents', From 3f35244e83a427c5482d98dc7e44a01418af6078 Mon Sep 17 00:00:00 2001 From: Pei He Date: Wed, 8 Nov 2017 10:48:27 +0800 Subject: [PATCH 4/5] fixup. --- .test-infra/jenkins/job_beam_Java_Build.groovy | 3 ++- .test-infra/jenkins/job_beam_Java_IntegrationTest.groovy | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.test-infra/jenkins/job_beam_Java_Build.groovy b/.test-infra/jenkins/job_beam_Java_Build.groovy index 704d7eb43bc20..471c32c76b692 100644 --- a/.test-infra/jenkins/job_beam_Java_Build.groovy +++ b/.test-infra/jenkins/job_beam_Java_Build.groovy @@ -49,7 +49,8 @@ mavenJob('beam_Java_Build') { 'dataflow-runner', 'spark-runner', 'flink-runner', - 'apex-runner' + 'apex-runner', + 'mapreduce-runner' ] args = [ '-B', diff --git a/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy b/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy index 56daf731d84c5..989b0906d337c 100644 --- a/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy +++ b/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy @@ -34,7 +34,8 @@ mavenJob('beam_Java_IntegrationTest') { 'dataflow-runner', 'spark-runner', 'flink-runner', - 'apex-runner' + 'apex-runner', + 'mapreduce-runner' ] // In the case of the precommit integration tests, we are currently only running the integration // tests in the examples directory. By directly invoking failsafe with an execution name (which we @@ -47,6 +48,7 @@ mavenJob('beam_Java_IntegrationTest') { 'direct-runner-integration-tests', 'flink-runner-integration-tests', 'spark-runner-integration-tests', + 'mapreduce-runner-integration-tests', ] // Arguments to provide Maven. args = [ From a3e5a05dfd5d0c6e0e0aa708349a1122cea4bac7 Mon Sep 17 00:00:00 2001 From: Pei He Date: Wed, 8 Nov 2017 12:50:56 +0800 Subject: [PATCH 5/5] fixup. --- .test-infra/jenkins/job_beam_Java_UnitTest.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.test-infra/jenkins/job_beam_Java_UnitTest.groovy b/.test-infra/jenkins/job_beam_Java_UnitTest.groovy index d3c8ed782f271..a6db45658f56e 100644 --- a/.test-infra/jenkins/job_beam_Java_UnitTest.groovy +++ b/.test-infra/jenkins/job_beam_Java_UnitTest.groovy @@ -33,7 +33,8 @@ mavenJob('beam_Java_UnitTest') { 'dataflow-runner', 'spark-runner', 'flink-runner', - 'apex-runner' + 'apex-runner', + 'mapreduce-runner' ] args = [ '-B',