diff --git a/.test-infra/jenkins/job_PostCommit_Java11_Dataflow_Examples.groovy b/.test-infra/jenkins/job_PostCommit_Java11_Dataflow_Examples.groovy new file mode 100644 index 000000000000..ad7546e8bfdf --- /dev/null +++ b/.test-infra/jenkins/job_PostCommit_Java11_Dataflow_Examples.groovy @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PostcommitJobBuilder +import CommonJobProperties as commonJobProperties + +PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java11_Examples_Dataflow', + 'Run Java examples on Dataflow with Java 11', 'Google Cloud Dataflow Runner Examples Java 11', this) { + + description('Runs the Java Examples suite on the Java 11 enabled Dataflow runner.') + + commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180) + + publishers { + archiveJunit('**/build/test-results/**/*.xml') + } + + steps { + gradle { + rootBuildScriptDir(commonJobProperties.checkoutDir) + tasks(':beam-runners-google-cloud-dataflow-java-examples:java11PostCommit') + + // Increase parallel worker threads above processor limit since most time is + // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow + // because each one launches a Dataflow job with about 3 mins of overhead. + // 3 x num_cores strikes a good balance between maxing out parallelism without + // overloading the machines. + commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors()) + } + } +} diff --git a/.test-infra/jenkins/job_PostCommit_Java11_Dataflow_Portability_Examples.groovy b/.test-infra/jenkins/job_PostCommit_Java11_Dataflow_Portability_Examples.groovy new file mode 100644 index 000000000000..b67035273c5c --- /dev/null +++ b/.test-infra/jenkins/job_PostCommit_Java11_Dataflow_Portability_Examples.groovy @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import PostcommitJobBuilder +import CommonJobProperties as commonJobProperties + +PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java11_Examples_Dataflow_Portability', + 'Run Java Portability examples on Dataflow with Java 11', 'Google Cloud Dataflow Portability Runner Examples Java 11', this) { + + description('Runs the Java Examples suite on the Java 11 enabled Dataflow runner with Portability API.') + + commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180) + + publishers { + archiveJunit('**/build/test-results/**/*.xml') + } + + steps { + gradle { + rootBuildScriptDir(commonJobProperties.checkoutDir) + tasks(':beam-runners-google-cloud-dataflow-java-examples:verifyPortabilityApi') + switches ('-Pdockerfile=Dockerfile-java11') + + // Increase parallel worker threads above processor limit since most time is + // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow + // because each one launches a Dataflow job with about 3 mins of overhead. + // 3 x num_cores strikes a good balance between maxing out parallelism without + // overloading the machines. + commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors()) + } + } +} diff --git a/build.gradle b/build.gradle index e34eaa2fad75..60dbcc6181f3 100644 --- a/build.gradle +++ b/build.gradle @@ -145,7 +145,7 @@ task javaPreCommit() { task javaPreCommitPortabilityApi() { dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:build" - dependsOn ":beam-runners-google-cloud-dataflow-java-examples:preCommitPortabilityApi" + dependsOn ":beam-runners-google-cloud-dataflow-java-examples:verifyPortabilityApi" } task javaPostCommit() { diff --git a/runners/google-cloud-dataflow-java/examples/build.gradle b/runners/google-cloud-dataflow-java/examples/build.gradle index b1ec60a27db2..e492fef1146b 100644 --- a/runners/google-cloud-dataflow-java/examples/build.gradle +++ b/runners/google-cloud-dataflow-java/examples/build.gradle @@ -42,57 +42,59 @@ def dockerImageName = project(':beam-runners-google-cloud-dataflow-java').ext.do // If -PuseExecutableStage is set, the use_executable_stage_bundle_execution wil be enabled. def fnapiExperiments = project.hasProperty('useExecutableStage') ? 'beam_fn_api,use_executable_stage_bundle_execution' : "beam_fn_api" +def commonConfig = { dataflowWorkerJar, workerHarnessContainerImage = '', additionalOptions = [] -> + // return the preevaluated configuration closure + return { + testClassesDirs = files(project(":beam-examples-java").sourceSets.test.output.classesDirs) + include "**/WordCountIT.class" + include "**/WindowedWordCountIT.class" + forkEvery 1 + maxParallelForks 4 + + def preCommitBeamTestPipelineOptions = [ + "--project=${gcpProject}", + "--tempRoot=${gcsTempRoot}", + "--runner=TestDataflowRunner", + "--dataflowWorkerJar=${dataflowWorkerJar}", + workerHarnessContainerImage.isEmpty() ?'':"--workerHarnessContainerImage=${workerHarnessContainerImage}" + ] + additionalOptions + systemProperty "beamTestPipelineOptions", JsonOutput.toJson(preCommitBeamTestPipelineOptions) + } + } + task preCommitLegacyWorker(type: Test) { dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar" def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath - - //Set workerHarnessContainerImage to empty to make Dataflow pick up the non-versioned container - //image, which handles a staged worker jar. - def preCommitBeamTestPipelineOptions = [ - "--project=${gcpProject}", - "--tempRoot=${gcsTempRoot}", - "--runner=TestDataflowRunner", - "--dataflowWorkerJar=${dataflowWorkerJar}", - "--workerHarnessContainerImage=", - ] - testClassesDirs = files(project(":beam-examples-java").sourceSets.test.output.classesDirs) - include "**/WordCountIT.class" - include "**/WindowedWordCountIT.class" - forkEvery 1 - maxParallelForks 4 - systemProperty "beamTestPipelineOptions", JsonOutput.toJson(preCommitBeamTestPipelineOptions) + with commonConfig(dataflowWorkerJar) } -task preCommitFnApiWorker(type: Test) { +task verifyFnApiWorker(type: Test) { dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar" dependsOn ":beam-runners-google-cloud-dataflow-java:buildAndPushDockerContainer" - def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").shadowJar.archivePath - def preCommitBeamTestPipelineOptions = [ - "--project=${gcpProject}", - "--tempRoot=${gcsTempRoot}", - "--runner=TestDataflowRunner", - "--dataflowWorkerJar=${dataflowWorkerJar}", - "--workerHarnessContainerImage=${dockerImageName}", - "--experiments=${fnapiExperiments}", - ] - testClassesDirs = files(project(":beam-examples-java").sourceSets.test.output.classesDirs) - include "**/WordCountIT.class" - include "**/WindowedWordCountIT.class" - forkEvery 1 - maxParallelForks 4 - systemProperty "beamTestPipelineOptions", JsonOutput.toJson(preCommitBeamTestPipelineOptions) + with commonConfig(dataflowWorkerJar, dockerImageName, ["--experiments=${fnapiExperiments}"]) useJUnit { excludeCategories 'org.apache.beam.sdk.testing.StreamingIT' } } +task postCommitLegacyWorkerJava11(type: Test) { + dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar" + def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath + systemProperty "java.specification.version", "11" + with commonConfig(dataflowWorkerJar) +} + +task java11PostCommit() { + dependsOn postCommitLegacyWorkerJava11 +} + task preCommit() { dependsOn preCommitLegacyWorker } -task preCommitPortabilityApi() { - dependsOn preCommitFnApiWorker +task verifyPortabilityApi() { + dependsOn verifyFnApiWorker } afterEvaluate {