From b190916d3fc5657a210923094416c3fd81bda8ed Mon Sep 17 00:00:00 2001 From: Luke Cwik Date: Fri, 17 Nov 2017 15:05:09 -0800 Subject: [PATCH 1/5] Enable Java Gradle Precommit in Jenkins --- ...job_beam_PreCommit_Java_GradleBuild.groovy | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy new file mode 100644 index 000000000000..527fb3211f8e --- /dev/null +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This is the Java precommit which runs a Gradle build, and the current set +// of precommit tests. +job('beam_PreCommit_Java_GradleBuild') { + description('Runs a build of the current GitHub Pull Request.') + + // Execute concurrent builds if necessary. + concurrentBuild() + + // Set common parameters. + common_job_properties.setTopLevelMainJobProperties( + delegate, + 'master', + 240) + + // Sets that this is a PreCommit job. + common_job_properties.setPreCommit(delegate, './gradlew --continue --rerun-tasks :beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-core:buildDependents :beam-runners-parent:beam-runners-direct-java:buildDependents :beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-fn-execution:buildDependents :beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-core:buildNeeded :beam-runners-parent:beam-runners-direct-java:buildNeeded :beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-fn-execution:buildNeeded', 'Run Java Gradle PreCommit') + + steps { + gradle { + tasks(':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-core:buildNeeded') + tasks(':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-core:buildDependents') + tasks(':beam-runners-parent:beam-runners-direct-java:buildNeeded') + tasks(':beam-runners-parent:beam-runners-direct-java:buildDependents') + tasks(':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-fn-execution:buildNeeded') + tasks(':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-fn-execution:buildDependents') + // Continue the build even if there is a failure to show as many potential failures as possible. + switches('--continue') + // Until we verify the build cache is working appropriately, force rerunning all tasks + switches('--rerun-tasks') + } + } +} From fc74b8a4eb7462b60d6a54a721cf0b9a48a68547 Mon Sep 17 00:00:00 2001 From: Luke Cwik Date: Fri, 17 Nov 2017 16:30:59 -0800 Subject: [PATCH 2/5] Simplify maintainability by generating tasks and command line from list of root Java projects --- ...job_beam_PreCommit_Java_GradleBuild.groovy | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy index 527fb3211f8e..40f64d1e0ced 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy @@ -32,21 +32,31 @@ job('beam_PreCommit_Java_GradleBuild') { 'master', 240) + def root_projects = [ + ':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-core', + ':beam-runners-parent:beam-runners-direct-java', + ':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-fn-execution', + ] + def gradle_switches = [ + // Continue the build even if there is a failure to show as many potential failures as possible. + '--continue', + // Until we verify the build cache is working appropriately, force rerunning all tasks + '--rerun-tasks', + ] + + def gradle_command_line = './gradlew ' + gradle_switches.join(' ') + ' ' + root_projects.join(':buildNeeded ') + ' ' + root_projects.join(':buildDependents ') // Sets that this is a PreCommit job. - common_job_properties.setPreCommit(delegate, './gradlew --continue --rerun-tasks :beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-core:buildDependents :beam-runners-parent:beam-runners-direct-java:buildDependents :beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-fn-execution:buildDependents :beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-core:buildNeeded :beam-runners-parent:beam-runners-direct-java:buildNeeded :beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-fn-execution:buildNeeded', 'Run Java Gradle PreCommit') + common_job_properties.setPreCommit(delegate, gradle_command_line, 'Run Java Gradle PreCommit') steps { gradle { - tasks(':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-core:buildNeeded') - tasks(':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-core:buildDependents') - tasks(':beam-runners-parent:beam-runners-direct-java:buildNeeded') - tasks(':beam-runners-parent:beam-runners-direct-java:buildDependents') - tasks(':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-fn-execution:buildNeeded') - tasks(':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-fn-execution:buildDependents') - // Continue the build even if there is a failure to show as many potential failures as possible. - switches('--continue') - // Until we verify the build cache is working appropriately, force rerunning all tasks - switches('--rerun-tasks') + for (String root_project : root_projects) { + tasks(root_project + ':buildNeeded') + tasks(root_project + ':buildDependents') + } + for (String gradle_switch : gradle_switches) { + switches(gradle_switch) + } } } } From 9bd9c85c18bb16a3e46f32fd2bcef77552ddfbf2 Mon Sep 17 00:00:00 2001 From: Luke Cwik Date: Sat, 18 Nov 2017 08:14:28 -0800 Subject: [PATCH 3/5] Limit the commit status length to 255 characters to honor Github limit --- .test-infra/jenkins/common_job_properties.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy index 2930d741608c..3b0776ba19c2 100644 --- a/.test-infra/jenkins/common_job_properties.groovy +++ b/.test-infra/jenkins/common_job_properties.groovy @@ -150,8 +150,8 @@ class common_job_properties { extensions { commitStatus { // This is the name that will show up in the GitHub pull request UI - // for this Jenkins project. - delegate.context("Jenkins: " + commitStatusContext) + // for this Jenkins project. It has a limit of 255 characters. + delegate.context(("Jenkins: " + commitStatusContext).take(255)) } // Comment messages after build completes. From af9bfe57250c24c55a70de3fea7fff00e46ae351 Mon Sep 17 00:00:00 2001 From: Luke Cwik Date: Sat, 18 Nov 2017 08:45:05 -0800 Subject: [PATCH 4/5] Revert PR/3976, the Jenkins Gradle plugin expects the Gradle Wrapper to be in the root of the directory. --- .test-infra/jenkins/common_job_properties.groovy | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy index 3b0776ba19c2..68409c0c3f40 100644 --- a/.test-infra/jenkins/common_job_properties.groovy +++ b/.test-infra/jenkins/common_job_properties.groovy @@ -22,8 +22,6 @@ // http://groovy-lang.org/style-guide.html class common_job_properties { - static String checkoutDir = 'src' - static void setSCM(def context, String repositoryName) { context.scm { git { @@ -38,7 +36,6 @@ class common_job_properties { branch('${sha1}') extensions { cleanAfterCheckout() - relativeTargetDirectory(checkoutDir) } } } @@ -174,7 +171,7 @@ class common_job_properties { // tiered compilation to make the JVM startup times faster during the tests. context.mavenOpts('-XX:+TieredCompilation') context.mavenOpts('-XX:TieredStopAtLevel=1') - context.rootPOM(checkoutDir + '/pom.xml') + context.rootPOM('pom.xml') // Use a repository local to the workspace for better isolation of jobs. context.localRepository(LocalRepositoryLocation.LOCAL_TO_WORKSPACE) // Disable archiving the built artifacts by default, as this is slow and flaky. From b565925f487a703b9190774ca8d17ce068076c11 Mon Sep 17 00:00:00 2001 From: Luke Cwik Date: Sun, 19 Nov 2017 10:38:06 -0800 Subject: [PATCH 5/5] Sync with master and add new subproject for beam-runners-local-java-core --- runners/direct-java/build.gradle | 2 ++ runners/local-java/build.gradle | 43 ++++++++++++++++++++++++++++++++ settings.gradle | 2 ++ 3 files changed, 47 insertions(+) create mode 100644 runners/local-java/build.gradle diff --git a/runners/direct-java/build.gradle b/runners/direct-java/build.gradle index 335e552056df..fa0a65f88cd1 100644 --- a/runners/direct-java/build.gradle +++ b/runners/direct-java/build.gradle @@ -36,6 +36,7 @@ dependencies { compile project(path: ":beam-model-parent:beam-model-pipeline", configuration: "shadow") compile project(path: ":beam-runners-parent:beam-runners-core-construction-java", configuration: "shadow") compile project(path: ":beam-runners-parent:beam-runners-core-java", configuration: "shadow") + compile project(path: ":beam-runners-parent:beam-runners-local-java-core", configuration: "shadow") shadow project(path: ":beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-core", configuration: "shadow") shadow library.java.joda_time shadow library.java.findbugs_jsr305 @@ -60,6 +61,7 @@ shadowJar { include(project(path: ":beam-model-parent:beam-model-pipeline", configuration: "shadow")) include(project(path: ":beam-runners-parent:beam-runners-core-construction-java", configuration: "shadow")) include(project(path: ":beam-runners-parent:beam-runners-core-java", configuration: "shadow")) + include(project(path: ":beam-runners-parent:beam-runners-local-java-core", configuration: "shadow")) } relocate "org.apache.beam.runners.core", "org.apache.beam." + project.name.replace("-", ".") + ".repackaged.runners.core" relocate "org.apache.beam.model", "org.apache.beam." + project.name.replace("-", ".") + ".repackaged.model" diff --git a/runners/local-java/build.gradle b/runners/local-java/build.gradle new file mode 100644 index 000000000000..492cb94ee7f9 --- /dev/null +++ b/runners/local-java/build.gradle @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * License); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an AS IS BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply from: project(":").file("build_rules.gradle") +applyJavaNature() + +description = "Apache Beam :: Runners :: Local Java Core" + +dependencies { + /* + * The Local Java Core artifact should carefully manage the classes made available by this + * dependency. The Java SDK should be used to provide common utilities (e.g. Coder, WindowedValue) + * but should not be used within this library to execute any UDFs. + * TODO: Add an APISurfaceTest to force this to be the case, if possible. + */ + shadow project(path: ":beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-core", configuration: "shadow") + shadow library.java.joda_time + shadow library.java.findbugs_jsr305 + shadowTest library.java.hamcrest_core + shadowTest library.java.junit +} + +task packageTests(type: Jar) { + from sourceSets.test.output + classifier = "tests" +} + +artifacts.archives packageTests diff --git a/settings.gradle b/settings.gradle index cd5a9f7db7d4..7eb25aac3620 100644 --- a/settings.gradle +++ b/settings.gradle @@ -81,6 +81,7 @@ include ':beam-runners-parent:beam-runners-reference-parent' include ':beam-runners-parent:beam-runners-direct-java' include ':beam-runners-parent:beam-runners-flink_2.10' include ':beam-runners-parent:beam-runners-google-cloud-dataflow-java' +include ':beam-runners-parent:beam-runners-local-java-core' include ':beam-runners-parent:beam-runners-spark' include ':beam-runners-parent:beam-runners-apex' include ':beam-runners-parent:beam-runners-gcp-parent:beam-runners-gcp-gcemd' @@ -157,6 +158,7 @@ project(':beam-runners-parent:beam-runners-reference-parent').projectDir = "$roo project(':beam-runners-parent:beam-runners-direct-java').projectDir = "$rootDir/runners/direct-java" as File project(':beam-runners-parent:beam-runners-flink_2.10').projectDir = "$rootDir/runners/flink" as File project(':beam-runners-parent:beam-runners-google-cloud-dataflow-java').projectDir = "$rootDir/runners/google-cloud-dataflow-java" as File +project(':beam-runners-parent:beam-runners-local-java-core').projectDir = "$rootDir/runners/local-java" as File project(':beam-runners-parent:beam-runners-spark').projectDir = "$rootDir/runners/spark" as File project(':beam-runners-parent:beam-runners-apex').projectDir = "$rootDir/runners/apex" as File project(':beam-runners-parent:beam-runners-gcp-parent:beam-runners-gcp-gcemd').projectDir = "$rootDir/runners/gcp/gcemd" as File