[BEAM-6936] Added Jenkins jobs running Java examples on Dataflow with Java 11

.test-infra/jenkins/job_PostCommit_Java11_Dataflow_Examples.groovy

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import PostcommitJobBuilder
+import CommonJobProperties as commonJobProperties
+
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java11_Examples_Dataflow',
+        'Run Java examples on Dataflow with Java 11', 'Google Cloud Dataflow Runner Examples Java 11', this) {
+
+    description('Runs the Java Examples suite on the Java 11 enabled Dataflow runner.')
+
+    commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180)
+
+    publishers {
+        archiveJunit('**/build/test-results/**/*.xml')
+    }
+
+    steps {
+        gradle {
+            rootBuildScriptDir(commonJobProperties.checkoutDir)
+            tasks(':beam-runners-google-cloud-dataflow-java-examples:java11PostCommit')
+
+            // Increase parallel worker threads above processor limit since most time is
+            // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow
+            // because each one launches a Dataflow job with about 3 mins of overhead.
+            // 3 x num_cores strikes a good balance between maxing out parallelism without
+            // overloading the machines.
+            commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors())
+        }
+    }
+}

.test-infra/jenkins/job_PostCommit_Java11_Dataflow_Portability_Examples.groovy

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import PostcommitJobBuilder
+import CommonJobProperties as commonJobProperties
+
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java11_Examples_Dataflow_Portability',
+        'Run Java Portability examples on Dataflow with Java 11', 'Google Cloud Dataflow Portability Runner Examples Java 11', this) {
+
+    description('Runs the Java Examples suite on the Java 11 enabled Dataflow runner with Portability API.')
+
+    commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180)
+
+    publishers {
+        archiveJunit('**/build/test-results/**/*.xml')
+    }
+
+    steps {
+        gradle {
+            rootBuildScriptDir(commonJobProperties.checkoutDir)
+            tasks(':beam-runners-google-cloud-dataflow-java-examples:verifyPortabilityApi')
+            switches ('-Pdockerfile=Dockerfile-java11')
+
+            // Increase parallel worker threads above processor limit since most time is
+            // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow
+            // because each one launches a Dataflow job with about 3 mins of overhead.
+            // 3 x num_cores strikes a good balance between maxing out parallelism without
+            // overloading the machines.
+            commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors())
+        }
+    }
+}

build.gradle

@@ -145,7 +145,7 @@ task javaPreCommit() {
 
 task javaPreCommitPortabilityApi() {
   dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:build"
-  dependsOn ":beam-runners-google-cloud-dataflow-java-examples:preCommitPortabilityApi"
+  dependsOn ":beam-runners-google-cloud-dataflow-java-examples:verifyPortabilityApi"
 }
 
 task javaPostCommit() {

runners/google-cloud-dataflow-java/examples/build.gradle

@@ -42,57 +42,59 @@ def dockerImageName = project(':beam-runners-google-cloud-dataflow-java').ext.do
 // If -PuseExecutableStage is set, the use_executable_stage_bundle_execution wil be enabled.
 def fnapiExperiments = project.hasProperty('useExecutableStage') ? 'beam_fn_api,use_executable_stage_bundle_execution' : "beam_fn_api"
 
+def commonConfig = { dataflowWorkerJar, workerHarnessContainerImage = '', additionalOptions = [] ->
+   // return the preevaluated configuration closure
+   return {
+       testClassesDirs = files(project(":beam-examples-java").sourceSets.test.output.classesDirs)
+       include "**/WordCountIT.class"
+       include "**/WindowedWordCountIT.class"
+       forkEvery 1
+       maxParallelForks 4
+
+       def preCommitBeamTestPipelineOptions = [
+               "--project=${gcpProject}",
+               "--tempRoot=${gcsTempRoot}",
+               "--runner=TestDataflowRunner",
+               "--dataflowWorkerJar=${dataflowWorkerJar}",
+               workerHarnessContainerImage.isEmpty() ?'':"--workerHarnessContainerImage=${workerHarnessContainerImage}"
+               ] + additionalOptions
+       systemProperty "beamTestPipelineOptions", JsonOutput.toJson(preCommitBeamTestPipelineOptions)
+     }
+ }
+
 task preCommitLegacyWorker(type: Test) {
   dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar"
   def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath
-
-  //Set workerHarnessContainerImage to empty to make Dataflow pick up the non-versioned container
-  //image, which handles a staged worker jar.
-  def preCommitBeamTestPipelineOptions = [
-     "--project=${gcpProject}",
-     "--tempRoot=${gcsTempRoot}",
-     "--runner=TestDataflowRunner",
-     "--dataflowWorkerJar=${dataflowWorkerJar}",
-     "--workerHarnessContainerImage=",
-  ]
-  testClassesDirs = files(project(":beam-examples-java").sourceSets.test.output.classesDirs)
-  include "**/WordCountIT.class"
-  include "**/WindowedWordCountIT.class"
-  forkEvery 1
-  maxParallelForks 4
-  systemProperty "beamTestPipelineOptions", JsonOutput.toJson(preCommitBeamTestPipelineOptions)
+  with commonConfig(dataflowWorkerJar)
 }
 
-task preCommitFnApiWorker(type: Test) {
+task verifyFnApiWorker(type: Test) {
   dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar"
   dependsOn ":beam-runners-google-cloud-dataflow-java:buildAndPushDockerContainer"
-
   def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").shadowJar.archivePath
-  def preCommitBeamTestPipelineOptions = [
-          "--project=${gcpProject}",
-          "--tempRoot=${gcsTempRoot}",
-          "--runner=TestDataflowRunner",
-          "--dataflowWorkerJar=${dataflowWorkerJar}",
-          "--workerHarnessContainerImage=${dockerImageName}",
-          "--experiments=${fnapiExperiments}",
-  ]
-  testClassesDirs = files(project(":beam-examples-java").sourceSets.test.output.classesDirs)
-  include "**/WordCountIT.class"
-  include "**/WindowedWordCountIT.class"
-  forkEvery 1
-  maxParallelForks 4
-  systemProperty "beamTestPipelineOptions", JsonOutput.toJson(preCommitBeamTestPipelineOptions)
+  with commonConfig(dataflowWorkerJar, dockerImageName, ["--experiments=${fnapiExperiments}"])
   useJUnit {
     excludeCategories 'org.apache.beam.sdk.testing.StreamingIT'
   }
 }
 
+task postCommitLegacyWorkerJava11(type: Test) {
+  dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar"
+  def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath
+  systemProperty "java.specification.version", "11"
+  with commonConfig(dataflowWorkerJar)
+}
+
+task java11PostCommit() {
+  dependsOn postCommitLegacyWorkerJava11
+}
+
 task preCommit() {
   dependsOn preCommitLegacyWorker
 }
 
-task preCommitPortabilityApi() {
-  dependsOn preCommitFnApiWorker
+task verifyPortabilityApi() {
+  dependsOn verifyFnApiWorker
 }
 
 afterEvaluate {