From 3d2149be83c9e0a764a3a2b77deb508c7dd5600d Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Wed, 28 Jun 2017 16:22:52 -0700 Subject: [PATCH 01/19] Initial set of pipeline jobs. Signed-off-by: Jason Kuster --- .test-infra/jenkins/PreCommit_Pipeline.groovy | 66 +++++++++++ .../job_beam_PreCommit_Java_Build.groovy | 107 ++++++++++++++++++ .../job_beam_PreCommit_Java_UnitTest.groovy | 78 +++++++++++++ .../job_beam_PreCommit_Pipeline.groovy | 92 +++++++++++++++ 4 files changed, 343 insertions(+) create mode 100644 .test-infra/jenkins/PreCommit_Pipeline.groovy create mode 100644 .test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy create mode 100644 .test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy create mode 100644 .test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy diff --git a/.test-infra/jenkins/PreCommit_Pipeline.groovy b/.test-infra/jenkins/PreCommit_Pipeline.groovy new file mode 100644 index 000000000000..3259ad7db8d3 --- /dev/null +++ b/.test-infra/jenkins/PreCommit_Pipeline.groovy @@ -0,0 +1,66 @@ +#!groovy +import hudson.model.Result + +try { + def javaBuildNum = -1 + def pythonBuildNum = -1 + stage('Build') { + parallel ( + java: { + def javaBuild = build job: 'beam_PreCommit_Java_Build', parameters: [string(name: 'prNum', value: "${ghprbPullId}")] + if(javaBuild.getResult() == Result.SUCCESS.toString()) { + javaBuildNum = javaBuild.getNumber() + } + }, + python: { + def pythonBuild = build job: 'beam_PreCommit_Python_Build', parameters:[string(name: 'prNum', value: "${ghprbPullId}")] + if(pythonBuild.getResult() == Result.SUCCESS.toString()) { + pythonBuildNum = pythonBuild.getNumber() + } + } + ) + } + def javaUnitPassed = false + def pythonUnitPassed = false + stage('Unit Test / Code Health') { + parallel ( + java_unit: { + if(javaBuildNum != -1) { + def javaTest = build job: 'beam_PreCommit_Java_UnitTest', parameters: [string(name: 'buildNum', value: javaBuildNum)] + if(javaTest.getResult() == Result.SUCCESS.toString()) { + javaUnitPassed = true + } + } + }, + java_codehealth: { + if(javaBuildNum != -1) { + build job: 'beam_PreCommit_Java_CodeHealth', parameters: [string(name: 'buildNum', value: javaBuildNum)] + } + }, + python_unit: { + if(pythonBuildNum != -1) { + def pythonTest = build job: 'beam_PreCommit_Python_UnitTest', parameters: [string(name: 'buildNum', value: pythonBuildNum)] + if(pythonTest.getResult() == Result.SUCCESS.toString()) { + pythonUnitPassed = true + } + } + } + ) + } + stage('Integration Test') { + parallel ( + java_integration: { + if(javaUnitPassed) { + build job: 'beam_PreCommit_Java_IntegrationTest', parameters: [string(name: 'buildNum', value: javaBuildNum)] + } + }, + python_integration: { + if(pythonUnitPassed) { + build job: 'beam_PreCommit_Python_IntegrationTest', parameters: [string(name: 'buildNum', value: pythonBuildNum)] + } + } + ) + } +} catch (Exception e) { + echo e.toString() +} diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy new file mode 100644 index 000000000000..507403bdc28c --- /dev/null +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This is the Java precommit which runs a maven install, and the current set +// of precommit tests. +mavenJob('beam_PreCommit_Java_Build') { + description('Part of the PreCommit Pipeline. Builds Java SDK and archives artifacts.') + + properties { + githubProjectUrl('https://github.com/apache/' + repositoryName + '/') + } + + configure { project -> + project / 'properties' / 'hudson.plugins.copyartifact.CopyArtifactPermissionProperty' / 'projectNameList' { + 'string' "beam_*" + } + } + + parameters { + stringParam( + 'sha1', + 'master', + 'Commit id or refname (e.g. origin/pr/9/head) you want to build.') + } + + wrappers { + timeout { + absolute(15) + abortBuild() + } + downstreamCommitStatus { + context('Jenkins: Java Build') + triggeredStatus("Java Build Pending") + startedStatus("Running Java Build") + statusUrl() + completedStatus('SUCCESS', "Java Build Passed") + completedStatus('FAILURE', "Some Java Build Failed") + completedStatus('ERROR', "Error Executing Java Build") + } + } + + // Set JDK version. + jdk('JDK 1.8 (latest)') + + // Restrict this project to run only on Jenkins executors as specified + label('beam') + + // Execute concurrent builds if necessary. + concurrentBuild() + + // Source code management. + scm { + git { + remote { + github("apache/beam") + refspec('+refs/heads/*:refs/remotes/origin/* ' + + '+refs/pull/*:refs/remotes/origin/pr/*') + } + branch('${sha1}') + extensions { + cleanAfterCheckout() + } + } + } + + // Set Maven parameters. + common_job_properties.setMavenConfig(delegate) + + // Maven goals for this job. + goals('-B -e -Prelease,include-runners,jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner clean install -pl \'!sdks/python\' -DskipTests -Dcheckstyle.skip -Dfindbugs.skip -Dmaven.javadoc.skip -Drat.skip') + + publishers { + archiveArtifacts { + pattern('.repository/org/apache/beam/**/*') + pattern('.test-infra/**/*') + pattern('.github/**/*') + pattern('examples/**/*') + pattern('runners/**/*') + pattern('sdks/**/*') + pattern('target/**/*') + pattern('pom.xml') + exclude('examples/**/*.jar') + exclude('runners/**/*.jar') + exclude('sdks/**/*.jar') + exclude('target/**/*.jar') + onlyIfSuccessful() + defaultExcludes() + } + } +} diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy new file mode 100644 index 000000000000..869b048f7bdb --- /dev/null +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This is the Java precommit which runs a maven install, and the current set +// of precommit tests. +mavenJob('beam_PreCommit_Java_UnitTest') { + description('Part of the PreCommit Pipeline. Runs Java Surefire unit tests.') + + parameters { + stringParam( + 'buildNum', + 'N/A', + 'Build number of beam_PreCommit_Java_Build to copy from.') + } + + // Set JDK version. + jdk('JDK 1.8 (latest)') + + // Restrict this project to run only on Jenkins executors as specified + label('beam') + + // Execute concurrent builds if necessary. + concurrentBuild() + + wrappers { + timeout { + absolute(20) + abortBuild() + } + credentialsBinding { + string("COVERALLS_REPO_TOKEN", "beam-coveralls-token") + } + downstreamCommitStatus { + context('Jenkins: Java Unit Tests') + triggeredStatus("Java Unit Tests Pending") + startedStatus("Running Java Unit Tests") + statusUrl() + completedStatus('SUCCESS', "Java Unit Tests Passed") + completedStatus('FAILURE', "Some Java Unit Tests Failed") + completedStatus('ERROR', "Error Executing Java Unit Tests") + } + // Set SPARK_LOCAL_IP for spark tests. + environmentVariables { + env('SPARK_LOCAL_IP', '127.0.0.1') + } + } + + // Set Maven parameters. + common_job_properties.setMavenConfig(delegate) + + preBuildSteps { + copyArtifacts("beam_PreCommit_Java_Build") { + buildSelector { + buildNumber("${buildNum}") + } + } + } + + // Maven goals for this job. + goals('-B -e -Prelease,include-runners,jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner surefire:test@default-test -pl \'!sdks/python\' -DrepoToken=$COVERALLS_REPO_TOKEN -DpullRequest=$ghprbPullId coveralls:report') +} diff --git a/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy new file mode 100644 index 000000000000..98daa1c8373e --- /dev/null +++ b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy @@ -0,0 +1,92 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This is the Java precommit which runs a maven install, and the current set +// of precommit tests. +mavenJob('beam_PreCommit_Pipeline') { + description('PreCommit Pipeline Job. Owns overall lifecycle of PreCommit tests.') + + properties { + githubProjectUrl('https://github.com/apache/' + repositoryName + '/') + } + + parameters { + stringParam( + 'sha1', + 'master', + 'Commit id or refname (e.g. origin/pr/9/head) you want to build.') + } + + wrappers { + timeout { + absolute(120) + abortBuild() + } + } + + // Restrict this project to run only on Jenkins executors as specified + label('beam') + + // Execute concurrent builds if necessary. + concurrentBuild() + + + triggers { + githubPullRequest { + admins(['asfbot']) + useGitHubHooks() + orgWhitelist(['apache']) + allowMembersOfWhitelistedOrgsAsAdmin() + permitAll() + displayBuildErrorsOnDownstreamBuilds() + extensions { + commitStatus { + context("Jenkins: PreCommit Pipeline") + } + buildStatus { + completedStatus('SUCCESS', '--none--') + completedStatus('FAILURE', '--none--') + completedStatus('ERROR', '--none--') + } + } + } + } + + definition { + cpsScm { + // Source code management. + scm { + git { + remote { + github("apache/beam") + refspec('+refs/heads/*:refs/remotes/origin/* ' + + '+refs/pull/*:refs/remotes/origin/pr/*') + } + branch('${sha1}') + extensions { + cleanAfterCheckout() + } + } + } + scriptPath('.test-infra/jenkins/Precommit_Pipeline.groovy') + } + } +} From 761c1fc6cfcefdee7a210cee8eb4644c6eb29cb8 Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Thu, 29 Jun 2017 01:56:50 -0700 Subject: [PATCH 02/19] Fixed many build options and configurations. Signed-off-by: Jason Kuster --- .test-infra/jenkins/PreCommit_Pipeline.groovy | 57 +++++++++++++++---- .../jenkins/common_job_properties.groovy | 24 +------- .../job_beam_PreCommit_Java_Build.groovy | 54 ++++++++++++++---- .../job_beam_PreCommit_Java_UnitTest.groovy | 48 ++++++++++++++-- .../job_beam_PreCommit_Pipeline.groovy | 15 ++--- 5 files changed, 138 insertions(+), 60 deletions(-) diff --git a/.test-infra/jenkins/PreCommit_Pipeline.groovy b/.test-infra/jenkins/PreCommit_Pipeline.groovy index 3259ad7db8d3..058317320f6c 100644 --- a/.test-infra/jenkins/PreCommit_Pipeline.groovy +++ b/.test-infra/jenkins/PreCommit_Pipeline.groovy @@ -2,31 +2,46 @@ import hudson.model.Result try { - def javaBuildNum = -1 - def pythonBuildNum = -1 + javaBuildNum = -1 + pythonBuildNum = -1 stage('Build') { parallel ( java: { - def javaBuild = build job: 'beam_PreCommit_Java_Build', parameters: [string(name: 'prNum', value: "${ghprbPullId}")] + def javaBuild = build job: 'beam_PreCommit_Java_Build', parameters: [ + string(name: 'sha1', value: "origin/pr/${ghprbPullId}/head"), + string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), + string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), + string(name: 'ghprbPullId', value: "${ghprbPullId}") + ] if(javaBuild.getResult() == Result.SUCCESS.toString()) { javaBuildNum = javaBuild.getNumber() } }, python: { - def pythonBuild = build job: 'beam_PreCommit_Python_Build', parameters:[string(name: 'prNum', value: "${ghprbPullId}")] + def pythonBuild = build job: 'beam_PreCommit_Python_Build', parameters:[ + string(name: 'sha1', value: "origin/pr/${ghprbPullId}/head"), + string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), + string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), + string(name: 'ghprbPullId', value: "${ghprbPullId}") + ] if(pythonBuild.getResult() == Result.SUCCESS.toString()) { pythonBuildNum = pythonBuild.getNumber() } } ) } - def javaUnitPassed = false - def pythonUnitPassed = false + javaUnitPassed = false + pythonUnitPassed = false stage('Unit Test / Code Health') { parallel ( java_unit: { if(javaBuildNum != -1) { - def javaTest = build job: 'beam_PreCommit_Java_UnitTest', parameters: [string(name: 'buildNum', value: javaBuildNum)] + def javaTest = build job: 'beam_PreCommit_Java_UnitTest', parameters: [ + string(name: 'buildNum', value: "${javaBuildNum}"), + string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), + string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), + string(name: 'ghprbPullId', value: "${ghprbPullId}") + ] if(javaTest.getResult() == Result.SUCCESS.toString()) { javaUnitPassed = true } @@ -34,12 +49,22 @@ try { }, java_codehealth: { if(javaBuildNum != -1) { - build job: 'beam_PreCommit_Java_CodeHealth', parameters: [string(name: 'buildNum', value: javaBuildNum)] + build job: 'beam_PreCommit_Java_CodeHealth', parameters: [ + string(name: 'buildNum', value: "${javaBuildNum}"), + string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), + string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), + string(name: 'ghprbPullId', value: "${ghprbPullId}") + ] } }, python_unit: { if(pythonBuildNum != -1) { - def pythonTest = build job: 'beam_PreCommit_Python_UnitTest', parameters: [string(name: 'buildNum', value: pythonBuildNum)] + def pythonTest = build job: 'beam_PreCommit_Python_UnitTest', parameters: [ + string(name: 'buildNum', value: "${pythonBuildNum}"), + string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), + string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), + string(name: 'ghprbPullId', value: "${ghprbPullId}") + ] if(pythonTest.getResult() == Result.SUCCESS.toString()) { pythonUnitPassed = true } @@ -51,12 +76,22 @@ try { parallel ( java_integration: { if(javaUnitPassed) { - build job: 'beam_PreCommit_Java_IntegrationTest', parameters: [string(name: 'buildNum', value: javaBuildNum)] + build job: 'beam_PreCommit_Java_IntegrationTest', parameters: [ + string(name: 'buildNum', value: "${javaBuildNum}"), + string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), + string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), + string(name: 'ghprbPullId', value: "${ghprbPullId}") + ] } }, python_integration: { if(pythonUnitPassed) { - build job: 'beam_PreCommit_Python_IntegrationTest', parameters: [string(name: 'buildNum', value: pythonBuildNum)] + build job: 'beam_PreCommit_Python_IntegrationTest', parameters: [ + string(name: 'buildNum', value: "${pythonBuildNum}"), + string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), + string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), + string(name: 'ghprbPullId', value: "${ghprbPullId}") + ] } } ) diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy index 70534c6ac3bd..70d48bfd2d1d 100644 --- a/.test-infra/jenkins/common_job_properties.groovy +++ b/.test-infra/jenkins/common_job_properties.groovy @@ -74,7 +74,7 @@ class common_job_properties { context.scm { git { remote { - url('https://github.com/apache/' + repositoryName + '.git') + github("apache/${repositoryName}") refspec('+refs/heads/*:refs/remotes/origin/* ' + '+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*') } @@ -141,37 +141,15 @@ class common_job_properties { delegate.context("Jenkins: " + commitStatusContext) } - /* - This section is disabled, because of jenkinsci/ghprb-plugin#417 issue. - For the time being, an equivalent configure section below is added. - // Comment messages after build completes. buildStatus { completedStatus('SUCCESS', successComment) completedStatus('FAILURE', '--none--') completedStatus('ERROR', '--none--') } - */ } } } - - // Comment messages after build completes. - context.configure { - def messages = it / triggers / 'org.jenkinsci.plugins.ghprb.GhprbTrigger' / extensions / 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildStatus' / messages - messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' { - message(successComment) - result('SUCCESS') - } - messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' { - message('--none--') - result('ERROR') - } - messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' { - message('--none--') - result('FAILURE') - } - } } // Sets common config for Maven jobs. diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy index 507403bdc28c..72ae586051a1 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy @@ -24,7 +24,7 @@ mavenJob('beam_PreCommit_Java_Build') { description('Part of the PreCommit Pipeline. Builds Java SDK and archives artifacts.') properties { - githubProjectUrl('https://github.com/apache/' + repositoryName + '/') + githubProjectUrl('https://github.com/apache/beam/') } configure { project -> @@ -38,8 +38,20 @@ mavenJob('beam_PreCommit_Java_Build') { 'sha1', 'master', 'Commit id or refname (e.g. origin/pr/9/head) you want to build.') + stringParam( + 'ghprbGhRepository', + 'N/A', + 'Repository name for use by ghprb plugin.') + stringParam( + 'ghprbActualCommit', + 'N/A', + 'Commit ID for use by ghprb plugin.') + stringParam( + 'ghprbPullId', + 'N/A', + 'PR # for use by ghprb plugin.') } - + wrappers { timeout { absolute(15) @@ -51,14 +63,14 @@ mavenJob('beam_PreCommit_Java_Build') { startedStatus("Running Java Build") statusUrl() completedStatus('SUCCESS', "Java Build Passed") - completedStatus('FAILURE', "Some Java Build Failed") + completedStatus('FAILURE', "Java Build Failed") completedStatus('ERROR', "Error Executing Java Build") } - } + } // Set JDK version. jdk('JDK 1.8 (latest)') - + // Restrict this project to run only on Jenkins executors as specified label('beam') @@ -83,8 +95,31 @@ mavenJob('beam_PreCommit_Java_Build') { // Set Maven parameters. common_job_properties.setMavenConfig(delegate) - // Maven goals for this job. - goals('-B -e -Prelease,include-runners,jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner clean install -pl \'!sdks/python\' -DskipTests -Dcheckstyle.skip -Dfindbugs.skip -Dmaven.javadoc.skip -Drat.skip') + // Construct Maven goals for this job. + profiles = [ + 'release', + 'include-runners', + 'jenkins-precommit', + 'direct-runner', + 'dataflow-runner', + 'spark-runner', + 'flink-runner', + 'apex-runner' + ] + args = [ + '-B', + '-e', + '-P' + profiles.join(',') + 'clean', + 'install', + '-pl \'!sdks/python\'', + '-DskipTests', + '-Dcheckstyle.skip', + '-Dfindbugs.skip', + '-Dmaven.javadoc.skip', + '-Drat.skip' + ] + goals(args.join(' ')) publishers { archiveArtifacts { @@ -96,10 +131,7 @@ mavenJob('beam_PreCommit_Java_Build') { pattern('sdks/**/*') pattern('target/**/*') pattern('pom.xml') - exclude('examples/**/*.jar') - exclude('runners/**/*.jar') - exclude('sdks/**/*.jar') - exclude('target/**/*.jar') + exclude('examples/**/*.jar,runners/**/*.jar,sdks/**/*.jar,target/**/*.jar') onlyIfSuccessful() defaultExcludes() } diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy index 869b048f7bdb..6fe3fa614c20 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy @@ -28,11 +28,23 @@ mavenJob('beam_PreCommit_Java_UnitTest') { 'buildNum', 'N/A', 'Build number of beam_PreCommit_Java_Build to copy from.') + stringParam( + 'ghprbGhRepository', + 'N/A', + 'Repository name for use by ghprb plugin.') + stringParam( + 'ghprbActualCommit', + 'N/A', + 'Commit ID for use by ghprb plugin.') + stringParam( + 'ghprbPullId', + 'N/A', + 'PR # for use by ghprb plugin.') } - + // Set JDK version. jdk('JDK 1.8 (latest)') - + // Restrict this project to run only on Jenkins executors as specified label('beam') @@ -53,7 +65,7 @@ mavenJob('beam_PreCommit_Java_UnitTest') { startedStatus("Running Java Unit Tests") statusUrl() completedStatus('SUCCESS', "Java Unit Tests Passed") - completedStatus('FAILURE', "Some Java Unit Tests Failed") + completedStatus('FAILURE', "Java Unit Tests Failed") completedStatus('ERROR', "Error Executing Java Unit Tests") } // Set SPARK_LOCAL_IP for spark tests. @@ -68,11 +80,35 @@ mavenJob('beam_PreCommit_Java_UnitTest') { preBuildSteps { copyArtifacts("beam_PreCommit_Java_Build") { buildSelector { - buildNumber("${buildNum}") + buildNumber('${buildNum}') } } } - // Maven goals for this job. - goals('-B -e -Prelease,include-runners,jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner surefire:test@default-test -pl \'!sdks/python\' -DrepoToken=$COVERALLS_REPO_TOKEN -DpullRequest=$ghprbPullId coveralls:report') + // Construct Maven goals for this job. + profiles = [ + 'release', + 'include-runners', + 'jenkins-precommit', + 'direct-runner', + 'dataflow-runner', + 'spark-runner', + 'flink-runner', + 'apex-runner' + ] + args = [ + '-B', + '-e', + '-P' + profiles.join(',') + 'surefire:test@default-test', + 'coveralls:report', + '-pl \'!sdks/python\'', + ' -DrepoToken=$COVERALLS_REPO_TOKEN', + '-DpullRequest=$ghprbPullId', + '-Dcheckstyle.skip', + '-Dfindbugs.skip', + '-Dmaven.javadoc.skip', + '-Drat.skip' + ] + goals(args.join(' ')) } diff --git a/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy index 98daa1c8373e..11b32d5c21b2 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy @@ -17,15 +17,13 @@ * limitations under the License. */ -import common_job_properties - // This is the Java precommit which runs a maven install, and the current set // of precommit tests. -mavenJob('beam_PreCommit_Pipeline') { +pipelineJob('beam_PreCommit_Pipeline') { description('PreCommit Pipeline Job. Owns overall lifecycle of PreCommit tests.') properties { - githubProjectUrl('https://github.com/apache/' + repositoryName + '/') + githubProjectUrl('https://github.com/apache/beam/') } parameters { @@ -34,21 +32,20 @@ mavenJob('beam_PreCommit_Pipeline') { 'master', 'Commit id or refname (e.g. origin/pr/9/head) you want to build.') } - + wrappers { timeout { absolute(120) abortBuild() } - } - + } + // Restrict this project to run only on Jenkins executors as specified label('beam') // Execute concurrent builds if necessary. concurrentBuild() - triggers { githubPullRequest { admins(['asfbot']) @@ -86,7 +83,7 @@ mavenJob('beam_PreCommit_Pipeline') { } } } - scriptPath('.test-infra/jenkins/Precommit_Pipeline.groovy') + scriptPath('.test-infra/jenkins/PreCommit_Pipeline.groovy') } } } From 7f8f00000f7d48376edf5d488b981dc2752421a8 Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Thu, 29 Jun 2017 02:14:27 -0700 Subject: [PATCH 03/19] add code health and integration test items Signed-off-by: Jason Kuster --- .../job_beam_PreCommit_Java_CodeHealth.groovy | 109 ++++++++++++++++++ ...beam_PreCommit_Java_IntegrationTest.groovy | 107 +++++++++++++++++ .../job_beam_PreCommit_Java_UnitTest.groovy | 6 +- 3 files changed, 217 insertions(+), 5 deletions(-) create mode 100644 .test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy create mode 100644 .test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy new file mode 100644 index 000000000000..18be2035b2f1 --- /dev/null +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This is the Java precommit which runs a maven install, and the current set +// of precommit tests. +mavenJob('beam_PreCommit_Java_CodeHealth') { + description('Part of the PreCommit Pipeline. Runs Java code health checks.') + + parameters { + stringParam( + 'buildNum', + 'N/A', + 'Build number of beam_PreCommit_Java_Build to copy from.') + stringParam( + 'ghprbGhRepository', + 'N/A', + 'Repository name for use by ghprb plugin.') + stringParam( + 'ghprbActualCommit', + 'N/A', + 'Commit ID for use by ghprb plugin.') + stringParam( + 'ghprbPullId', + 'N/A', + 'PR # for use by ghprb plugin.') + } + + // Set JDK version. + jdk('JDK 1.8 (latest)') + + // Restrict this project to run only on Jenkins executors as specified + label('beam') + + // Execute concurrent builds if necessary. + concurrentBuild() + + wrappers { + timeout { + absolute(20) + abortBuild() + } + credentialsBinding { + string("COVERALLS_REPO_TOKEN", "beam-coveralls-token") + } + downstreamCommitStatus { + context('Jenkins: Java Code Health') + triggeredStatus("Java Code Health Pending") + startedStatus("Running Java Code Health") + statusUrl() + completedStatus('SUCCESS', "Java Code Health Passed") + completedStatus('FAILURE', "Java Code Health Failed") + completedStatus('ERROR', "Error Executing Java Code Health") + } + // Set SPARK_LOCAL_IP for spark tests. + environmentVariables { + env('SPARK_LOCAL_IP', '127.0.0.1') + } + } + + // Set Maven parameters. + common_job_properties.setMavenConfig(delegate) + + preBuildSteps { + copyArtifacts("beam_PreCommit_Java_Build") { + buildSelector { + buildNumber('${buildNum}') + } + } + } + + // Construct Maven goals for this job. + profiles = [ + 'release', + 'include-runners', + 'jenkins-precommit', + 'direct-runner', + 'dataflow-runner', + 'spark-runner', + 'flink-runner', + 'apex-runner' + ] + args = [ + '-B', + '-e', + '-P' + profiles.join(',') + '-pl \'!sdks/python\'', + 'checkstyle:check', + 'findbugs:check', + 'rat:check' + ] + goals(args.join(' ')) +} diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy new file mode 100644 index 000000000000..c81f89d3da95 --- /dev/null +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This is the Java precommit which runs a maven install, and the current set +// of precommit tests. +mavenJob('beam_PreCommit_Java_IntegrationTest') { + description('Part of the PreCommit Pipeline. Runs Java Failsafe integration tests.') + + parameters { + stringParam( + 'buildNum', + 'N/A', + 'Build number of beam_PreCommit_Java_Build to copy from.') + stringParam( + 'ghprbGhRepository', + 'N/A', + 'Repository name for use by ghprb plugin.') + stringParam( + 'ghprbActualCommit', + 'N/A', + 'Commit ID for use by ghprb plugin.') + stringParam( + 'ghprbPullId', + 'N/A', + 'PR # for use by ghprb plugin.') + } + + // Set JDK version. + jdk('JDK 1.8 (latest)') + + // Restrict this project to run only on Jenkins executors as specified + label('beam') + + // Execute concurrent builds if necessary. + concurrentBuild() + + wrappers { + timeout { + absolute(20) + abortBuild() + } + credentialsBinding { + string("COVERALLS_REPO_TOKEN", "beam-coveralls-token") + } + downstreamCommitStatus { + context('Jenkins: Java Integration Tests') + triggeredStatus("Java Integration Tests Pending") + startedStatus("Running Java Integration Tests") + statusUrl() + completedStatus('SUCCESS', "Java Integration Tests Passed") + completedStatus('FAILURE', "Java Integration Tests Failed") + completedStatus('ERROR', "Error Executing Java Integration Tests") + } + // Set SPARK_LOCAL_IP for spark tests. + environmentVariables { + env('SPARK_LOCAL_IP', '127.0.0.1') + } + } + + // Set Maven parameters. + common_job_properties.setMavenConfig(delegate) + + preBuildSteps { + copyArtifacts("beam_PreCommit_Java_Build") { + buildSelector { + buildNumber('${buildNum}') + } + } + } + + // Construct Maven goals for this job. + profiles = [ + 'release', + 'include-runners', + 'jenkins-precommit', + 'direct-runner', + 'dataflow-runner', + 'spark-runner', + 'flink-runner', + 'apex-runner' + ] + args = [ + '-B', + '-e', + '-P' + profiles.join(',') + 'failsafe:integration-test', + '-pl \'!sdks/python\'', + ] + goals(args.join(' ')) +} diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy index 6fe3fa614c20..e71686a5523f 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy @@ -103,12 +103,8 @@ mavenJob('beam_PreCommit_Java_UnitTest') { 'surefire:test@default-test', 'coveralls:report', '-pl \'!sdks/python\'', - ' -DrepoToken=$COVERALLS_REPO_TOKEN', + '-DrepoToken=$COVERALLS_REPO_TOKEN', '-DpullRequest=$ghprbPullId', - '-Dcheckstyle.skip', - '-Dfindbugs.skip', - '-Dmaven.javadoc.skip', - '-Drat.skip' ] goals(args.join(' ')) } From c4f4b3eedab77d34256fca04f402d1a895d406c8 Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Thu, 29 Jun 2017 02:23:16 -0700 Subject: [PATCH 04/19] Stub out Python builds. Signed-off-by: Jason Kuster --- .../job_beam_PreCommit_Python_Build.groovy | 137 ++++++++++++++++++ ...am_PreCommit_Python_IntegrationTest.groovy | 107 ++++++++++++++ .../job_beam_PreCommit_Python_UnitTest.groovy | 107 ++++++++++++++ 3 files changed, 351 insertions(+) create mode 100644 .test-infra/jenkins/job_beam_PreCommit_Python_Build.groovy create mode 100644 .test-infra/jenkins/job_beam_PreCommit_Python_IntegrationTest.groovy create mode 100644 .test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy diff --git a/.test-infra/jenkins/job_beam_PreCommit_Python_Build.groovy b/.test-infra/jenkins/job_beam_PreCommit_Python_Build.groovy new file mode 100644 index 000000000000..e92b64dd4fb8 --- /dev/null +++ b/.test-infra/jenkins/job_beam_PreCommit_Python_Build.groovy @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This is the Python precommit which runs a maven install, and the current set +// of precommit tests. +mavenJob('beam_PreCommit_Python_Build') { + description('Part of the PreCommit Pipeline. Builds Python SDK and archives artifacts.') + + properties { + githubProjectUrl('https://github.com/apache/beam/') + } + + configure { project -> + project / 'properties' / 'hudson.plugins.copyartifact.CopyArtifactPermissionProperty' / 'projectNameList' { + 'string' "beam_*" + } + } + + parameters { + stringParam( + 'sha1', + 'master', + 'Commit id or refname (e.g. origin/pr/9/head) you want to build.') + stringParam( + 'ghprbGhRepository', + 'N/A', + 'Repository name for use by ghprb plugin.') + stringParam( + 'ghprbActualCommit', + 'N/A', + 'Commit ID for use by ghprb plugin.') + stringParam( + 'ghprbPullId', + 'N/A', + 'PR # for use by ghprb plugin.') + } + + wrappers { + timeout { + absolute(15) + abortBuild() + } + downstreamCommitStatus { + context('Jenkins: Python Build') + triggeredStatus("Python Build Pending") + startedStatus("Running Python Build") + statusUrl() + completedStatus('SUCCESS', "Python Build Passed") + completedStatus('FAILURE', "Python Build Failed") + completedStatus('ERROR', "Error Executing Python Build") + } + } + + // Set JDK version. + jdk('JDK 1.8 (latest)') + + // Restrict this project to run only on Jenkins executors as specified + label('beam') + + // Execute concurrent builds if necessary. + concurrentBuild() + + // Source code management. + scm { + git { + remote { + github("apache/beam") + refspec('+refs/heads/*:refs/remotes/origin/* ' + + '+refs/pull/*:refs/remotes/origin/pr/*') + } + branch('${sha1}') + extensions { + cleanAfterCheckout() + } + } + } + + // Set Maven parameters. + common_job_properties.setMavenConfig(delegate) + + // Construct Maven goals for this job. + profiles = [ + 'release', + 'include-runners', + 'jenkins-precommit', + 'direct-runner', + 'dataflow-runner', + 'spark-runner', + 'flink-runner', + 'apex-runner' + ] + args = [ + '-B', + '-e', + '-P' + profiles.join(',') + 'clean', + 'install', + '-pl sdks/python', + '-DskipTests', + '-Dcheckstyle.skip', + '-Dfindbugs.skip', + '-Dmaven.javadoc.skip', + '-Drat.skip' + ] + goals(args.join(' ')) + + publishers { + archiveArtifacts { + pattern('.test-infra/**/*') + pattern('.github/**/*') + pattern('examples/**/*') + pattern('runners/**/*') + pattern('sdks/**/*') + pattern('target/**/*') + pattern('pom.xml') + onlyIfSuccessful() + defaultExcludes() + } + } +} diff --git a/.test-infra/jenkins/job_beam_PreCommit_Python_IntegrationTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Python_IntegrationTest.groovy new file mode 100644 index 000000000000..99c5a95175e7 --- /dev/null +++ b/.test-infra/jenkins/job_beam_PreCommit_Python_IntegrationTest.groovy @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This is the Python precommit which runs a maven install, and the current set +// of precommit tests. +mavenJob('beam_PreCommit_Python_IntegrationTest') { + description('Part of the PreCommit Pipeline. Runs Python integration tests.') + + parameters { + stringParam( + 'buildNum', + 'N/A', + 'Build number of beam_PreCommit_Python_Build to copy from.') + stringParam( + 'ghprbGhRepository', + 'N/A', + 'Repository name for use by ghprb plugin.') + stringParam( + 'ghprbActualCommit', + 'N/A', + 'Commit ID for use by ghprb plugin.') + stringParam( + 'ghprbPullId', + 'N/A', + 'PR # for use by ghprb plugin.') + } + + // Set JDK version. + jdk('JDK 1.8 (latest)') + + // Restrict this project to run only on Jenkins executors as specified + label('beam') + + // Execute concurrent builds if necessary. + concurrentBuild() + + wrappers { + timeout { + absolute(20) + abortBuild() + } + credentialsBinding { + string("COVERALLS_REPO_TOKEN", "beam-coveralls-token") + } + downstreamCommitStatus { + context('Jenkins: Python Integration Tests') + triggeredStatus("Python Integration Tests Pending") + startedStatus("Running Python Integration Tests") + statusUrl() + completedStatus('SUCCESS', "Python Integration Tests Passed") + completedStatus('FAILURE', "Python Integration Tests Failed") + completedStatus('ERROR', "Error Executing Python Integration Tests") + } + // Set SPARK_LOCAL_IP for spark tests. + environmentVariables { + env('SPARK_LOCAL_IP', '127.0.0.1') + } + } + + // Set Maven parameters. + common_job_properties.setMavenConfig(delegate) + + preBuildSteps { + copyArtifacts("beam_PreCommit_Python_Build") { + buildSelector { + buildNumber('${buildNum}') + } + } + } + + // Construct Maven goals for this job. + profiles = [ + 'release', + 'include-runners', + 'jenkins-precommit', + 'direct-runner', + 'dataflow-runner', + 'spark-runner', + 'flink-runner', + 'apex-runner' + ] + args = [ + '-B', + '-e', + '-P' + profiles.join(',') + '?????????', + '-pl sdks/python', + ] + goals(args.join(' ')) +} diff --git a/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy new file mode 100644 index 000000000000..72894fd5459c --- /dev/null +++ b/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This is the Python precommit which runs a maven install, and the current set +// of precommit tests. +mavenJob('beam_PreCommit_Python_UnitTest') { + description('Part of the PreCommit Pipeline. Runs Python unit tests.') + + parameters { + stringParam( + 'buildNum', + 'N/A', + 'Build number of beam_PreCommit_Python_Build to copy from.') + stringParam( + 'ghprbGhRepository', + 'N/A', + 'Repository name for use by ghprb plugin.') + stringParam( + 'ghprbActualCommit', + 'N/A', + 'Commit ID for use by ghprb plugin.') + stringParam( + 'ghprbPullId', + 'N/A', + 'PR # for use by ghprb plugin.') + } + + // Set JDK version. + jdk('JDK 1.8 (latest)') + + // Restrict this project to run only on Jenkins executors as specified + label('beam') + + // Execute concurrent builds if necessary. + concurrentBuild() + + wrappers { + timeout { + absolute(20) + abortBuild() + } + credentialsBinding { + string("COVERALLS_REPO_TOKEN", "beam-coveralls-token") + } + downstreamCommitStatus { + context('Jenkins: Python Unit Tests') + triggeredStatus("Python Unit Tests Pending") + startedStatus("Running Python Unit Tests") + statusUrl() + completedStatus('SUCCESS', "Python Unit Tests Passed") + completedStatus('FAILURE', "Python Unit Tests Failed") + completedStatus('ERROR', "Error Executing Python Unit Tests") + } + // Set SPARK_LOCAL_IP for spark tests. + environmentVariables { + env('SPARK_LOCAL_IP', '127.0.0.1') + } + } + + // Set Maven parameters. + common_job_properties.setMavenConfig(delegate) + + preBuildSteps { + copyArtifacts("beam_PreCommit_Python_Build") { + buildSelector { + buildNumber('${buildNum}') + } + } + } + + // Construct Maven goals for this job. + profiles = [ + 'release', + 'include-runners', + 'jenkins-precommit', + 'direct-runner', + 'dataflow-runner', + 'spark-runner', + 'flink-runner', + 'apex-runner' + ] + args = [ + '-B', + '-e', + '-P' + profiles.join(',') + '???????', + '-pl sdks/python', + ] + goals(args.join(' ')) +} From 3ed3fcb49a15688583e51de8caf314890dbd96ef Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Thu, 29 Jun 2017 16:28:59 -0700 Subject: [PATCH 05/19] fix typo, remove python build, start on common job properties Signed-off-by: Jason Kuster --- .test-infra/jenkins/PreCommit_Pipeline.groovy | 40 ++--- .../jenkins/common_job_properties.groovy | 5 + .../job_beam_PreCommit_Java_Build.groovy | 2 +- .../job_beam_PreCommit_Java_CodeHealth.groovy | 2 +- ...beam_PreCommit_Java_IntegrationTest.groovy | 2 +- .../job_beam_PreCommit_Java_UnitTest.groovy | 2 +- .../job_beam_PreCommit_Python_Build.groovy | 137 ------------------ ...am_PreCommit_Python_IntegrationTest.groovy | 2 +- .../job_beam_PreCommit_Python_UnitTest.groovy | 4 +- 9 files changed, 25 insertions(+), 171 deletions(-) delete mode 100644 .test-infra/jenkins/job_beam_PreCommit_Python_Build.groovy diff --git a/.test-infra/jenkins/PreCommit_Pipeline.groovy b/.test-infra/jenkins/PreCommit_Pipeline.groovy index 058317320f6c..51234c2ffbd0 100644 --- a/.test-infra/jenkins/PreCommit_Pipeline.groovy +++ b/.test-infra/jenkins/PreCommit_Pipeline.groovy @@ -3,7 +3,6 @@ import hudson.model.Result try { javaBuildNum = -1 - pythonBuildNum = -1 stage('Build') { parallel ( java: { @@ -16,17 +15,6 @@ try { if(javaBuild.getResult() == Result.SUCCESS.toString()) { javaBuildNum = javaBuild.getNumber() } - }, - python: { - def pythonBuild = build job: 'beam_PreCommit_Python_Build', parameters:[ - string(name: 'sha1', value: "origin/pr/${ghprbPullId}/head"), - string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), - string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), - string(name: 'ghprbPullId', value: "${ghprbPullId}") - ] - if(pythonBuild.getResult() == Result.SUCCESS.toString()) { - pythonBuildNum = pythonBuild.getNumber() - } } ) } @@ -58,16 +46,14 @@ try { } }, python_unit: { - if(pythonBuildNum != -1) { - def pythonTest = build job: 'beam_PreCommit_Python_UnitTest', parameters: [ - string(name: 'buildNum', value: "${pythonBuildNum}"), - string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), - string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), - string(name: 'ghprbPullId', value: "${ghprbPullId}") - ] - if(pythonTest.getResult() == Result.SUCCESS.toString()) { - pythonUnitPassed = true - } + def pythonTest = build job: 'beam_PreCommit_Python_UnitTest', parameters: [ + string(name: 'buildNum', value: "${pythonBuildNum}"), + string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), + string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), + string(name: 'ghprbPullId', value: "${ghprbPullId}") + ] + if(pythonTest.getResult() == Result.SUCCESS.toString()) { + pythonUnitPassed = true } } ) @@ -77,11 +63,11 @@ try { java_integration: { if(javaUnitPassed) { build job: 'beam_PreCommit_Java_IntegrationTest', parameters: [ - string(name: 'buildNum', value: "${javaBuildNum}"), - string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), - string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), - string(name: 'ghprbPullId', value: "${ghprbPullId}") - ] + string(name: 'buildNum', value: "${javaBuildNum}"), + string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), + string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), + string(name: 'ghprbPullId', value: "${ghprbPullId}") + ] } }, python_integration: { diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy index 70d48bfd2d1d..fa7d55c98e10 100644 --- a/.test-infra/jenkins/common_job_properties.groovy +++ b/.test-infra/jenkins/common_job_properties.groovy @@ -111,6 +111,11 @@ class common_job_properties { } } + private static void setPipelineDownstreamJobProperties(context, + String sdkLang) { + + } + // Sets the pull request build trigger. Accessed through precommit methods // below to insulate callers from internal parameter defaults. private static void setPullRequestBuildTrigger(context, diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy index 72ae586051a1..18f259e04a31 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy @@ -109,7 +109,7 @@ mavenJob('beam_PreCommit_Java_Build') { args = [ '-B', '-e', - '-P' + profiles.join(',') + '-P' + profiles.join(','), 'clean', 'install', '-pl \'!sdks/python\'', diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy index 18be2035b2f1..45a2dd0494e1 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy @@ -99,7 +99,7 @@ mavenJob('beam_PreCommit_Java_CodeHealth') { args = [ '-B', '-e', - '-P' + profiles.join(',') + '-P' + profiles.join(','), '-pl \'!sdks/python\'', 'checkstyle:check', 'findbugs:check', diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy index c81f89d3da95..e41358ecad5c 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy @@ -99,7 +99,7 @@ mavenJob('beam_PreCommit_Java_IntegrationTest') { args = [ '-B', '-e', - '-P' + profiles.join(',') + '-P' + profiles.join(','), 'failsafe:integration-test', '-pl \'!sdks/python\'', ] diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy index e71686a5523f..9f976e88f97b 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy @@ -99,7 +99,7 @@ mavenJob('beam_PreCommit_Java_UnitTest') { args = [ '-B', '-e', - '-P' + profiles.join(',') + '-P' + profiles.join(','), 'surefire:test@default-test', 'coveralls:report', '-pl \'!sdks/python\'', diff --git a/.test-infra/jenkins/job_beam_PreCommit_Python_Build.groovy b/.test-infra/jenkins/job_beam_PreCommit_Python_Build.groovy deleted file mode 100644 index e92b64dd4fb8..000000000000 --- a/.test-infra/jenkins/job_beam_PreCommit_Python_Build.groovy +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import common_job_properties - -// This is the Python precommit which runs a maven install, and the current set -// of precommit tests. -mavenJob('beam_PreCommit_Python_Build') { - description('Part of the PreCommit Pipeline. Builds Python SDK and archives artifacts.') - - properties { - githubProjectUrl('https://github.com/apache/beam/') - } - - configure { project -> - project / 'properties' / 'hudson.plugins.copyartifact.CopyArtifactPermissionProperty' / 'projectNameList' { - 'string' "beam_*" - } - } - - parameters { - stringParam( - 'sha1', - 'master', - 'Commit id or refname (e.g. origin/pr/9/head) you want to build.') - stringParam( - 'ghprbGhRepository', - 'N/A', - 'Repository name for use by ghprb plugin.') - stringParam( - 'ghprbActualCommit', - 'N/A', - 'Commit ID for use by ghprb plugin.') - stringParam( - 'ghprbPullId', - 'N/A', - 'PR # for use by ghprb plugin.') - } - - wrappers { - timeout { - absolute(15) - abortBuild() - } - downstreamCommitStatus { - context('Jenkins: Python Build') - triggeredStatus("Python Build Pending") - startedStatus("Running Python Build") - statusUrl() - completedStatus('SUCCESS', "Python Build Passed") - completedStatus('FAILURE', "Python Build Failed") - completedStatus('ERROR', "Error Executing Python Build") - } - } - - // Set JDK version. - jdk('JDK 1.8 (latest)') - - // Restrict this project to run only on Jenkins executors as specified - label('beam') - - // Execute concurrent builds if necessary. - concurrentBuild() - - // Source code management. - scm { - git { - remote { - github("apache/beam") - refspec('+refs/heads/*:refs/remotes/origin/* ' + - '+refs/pull/*:refs/remotes/origin/pr/*') - } - branch('${sha1}') - extensions { - cleanAfterCheckout() - } - } - } - - // Set Maven parameters. - common_job_properties.setMavenConfig(delegate) - - // Construct Maven goals for this job. - profiles = [ - 'release', - 'include-runners', - 'jenkins-precommit', - 'direct-runner', - 'dataflow-runner', - 'spark-runner', - 'flink-runner', - 'apex-runner' - ] - args = [ - '-B', - '-e', - '-P' + profiles.join(',') - 'clean', - 'install', - '-pl sdks/python', - '-DskipTests', - '-Dcheckstyle.skip', - '-Dfindbugs.skip', - '-Dmaven.javadoc.skip', - '-Drat.skip' - ] - goals(args.join(' ')) - - publishers { - archiveArtifacts { - pattern('.test-infra/**/*') - pattern('.github/**/*') - pattern('examples/**/*') - pattern('runners/**/*') - pattern('sdks/**/*') - pattern('target/**/*') - pattern('pom.xml') - onlyIfSuccessful() - defaultExcludes() - } - } -} diff --git a/.test-infra/jenkins/job_beam_PreCommit_Python_IntegrationTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Python_IntegrationTest.groovy index 99c5a95175e7..3533a30f0d23 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Python_IntegrationTest.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Python_IntegrationTest.groovy @@ -99,7 +99,7 @@ mavenJob('beam_PreCommit_Python_IntegrationTest') { args = [ '-B', '-e', - '-P' + profiles.join(',') + '-P' + profiles.join(','), '?????????', '-pl sdks/python', ] diff --git a/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy index 72894fd5459c..bf27d541658f 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy @@ -99,8 +99,8 @@ mavenJob('beam_PreCommit_Python_UnitTest') { args = [ '-B', '-e', - '-P' + profiles.join(',') - '???????', + '-P' + profiles.join(','), + 'clean install', '-pl sdks/python', ] goals(args.join(' ')) From 2c0f79d4ed48893fba03c82749aaa184efacb518 Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Mon, 10 Jul 2017 17:58:11 -0700 Subject: [PATCH 06/19] update Python pipelines Signed-off-by: Jason Kuster --- .test-infra/jenkins/PreCommit_Pipeline.groovy | 58 +++------- ...am_PreCommit_Python_IntegrationTest.groovy | 107 ------------------ .../job_beam_PreCommit_Python_UnitTest.groovy | 27 +++-- 3 files changed, 31 insertions(+), 161 deletions(-) delete mode 100644 .test-infra/jenkins/job_beam_PreCommit_Python_IntegrationTest.groovy diff --git a/.test-infra/jenkins/PreCommit_Pipeline.groovy b/.test-infra/jenkins/PreCommit_Pipeline.groovy index 51234c2ffbd0..436aa85aa662 100644 --- a/.test-infra/jenkins/PreCommit_Pipeline.groovy +++ b/.test-infra/jenkins/PreCommit_Pipeline.groovy @@ -1,35 +1,33 @@ #!groovy import hudson.model.Result +def ghprbArgs = [ + string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), + string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), + string(name: 'ghprbPullId', value: "${ghprbPullId}") +] + +def sha1Arg = [string(name: 'sha1', value: "origin/pr/${ghprbPullId}/head")] + try { javaBuildNum = -1 stage('Build') { parallel ( java: { - def javaBuild = build job: 'beam_PreCommit_Java_Build', parameters: [ - string(name: 'sha1', value: "origin/pr/${ghprbPullId}/head"), - string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), - string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), - string(name: 'ghprbPullId', value: "${ghprbPullId}") - ] + def javaBuild = build job: 'beam_PreCommit_Java_Build', parameters: sha1Arg + ghprbArgs if(javaBuild.getResult() == Result.SUCCESS.toString()) { javaBuildNum = javaBuild.getNumber() } } ) } + javaBuildArg = [string(name: 'buildNum', value: "${javaBuildNum}")] javaUnitPassed = false - pythonUnitPassed = false stage('Unit Test / Code Health') { parallel ( java_unit: { if(javaBuildNum != -1) { - def javaTest = build job: 'beam_PreCommit_Java_UnitTest', parameters: [ - string(name: 'buildNum', value: "${javaBuildNum}"), - string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), - string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), - string(name: 'ghprbPullId', value: "${ghprbPullId}") - ] + def javaTest = build job: 'beam_PreCommit_Java_UnitTest', parameters: javaBuildArg + ghprbArgs if(javaTest.getResult() == Result.SUCCESS.toString()) { javaUnitPassed = true } @@ -37,24 +35,11 @@ try { }, java_codehealth: { if(javaBuildNum != -1) { - build job: 'beam_PreCommit_Java_CodeHealth', parameters: [ - string(name: 'buildNum', value: "${javaBuildNum}"), - string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), - string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), - string(name: 'ghprbPullId', value: "${ghprbPullId}") - ] + build job: 'beam_PreCommit_Java_CodeHealth', parameters: javaBuildArg + ghprbArgs } }, python_unit: { - def pythonTest = build job: 'beam_PreCommit_Python_UnitTest', parameters: [ - string(name: 'buildNum', value: "${pythonBuildNum}"), - string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), - string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), - string(name: 'ghprbPullId', value: "${ghprbPullId}") - ] - if(pythonTest.getResult() == Result.SUCCESS.toString()) { - pythonUnitPassed = true - } + build job: 'beam_PreCommit_Python_UnitTest', parameters: sha1Arg + ghprbArgs } ) } @@ -62,22 +47,7 @@ try { parallel ( java_integration: { if(javaUnitPassed) { - build job: 'beam_PreCommit_Java_IntegrationTest', parameters: [ - string(name: 'buildNum', value: "${javaBuildNum}"), - string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), - string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), - string(name: 'ghprbPullId', value: "${ghprbPullId}") - ] - } - }, - python_integration: { - if(pythonUnitPassed) { - build job: 'beam_PreCommit_Python_IntegrationTest', parameters: [ - string(name: 'buildNum', value: "${pythonBuildNum}"), - string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), - string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), - string(name: 'ghprbPullId', value: "${ghprbPullId}") - ] + build job: 'beam_PreCommit_Java_IntegrationTest', parameters: javaBuildArg + ghprbArgs } } ) diff --git a/.test-infra/jenkins/job_beam_PreCommit_Python_IntegrationTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Python_IntegrationTest.groovy deleted file mode 100644 index 3533a30f0d23..000000000000 --- a/.test-infra/jenkins/job_beam_PreCommit_Python_IntegrationTest.groovy +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import common_job_properties - -// This is the Python precommit which runs a maven install, and the current set -// of precommit tests. -mavenJob('beam_PreCommit_Python_IntegrationTest') { - description('Part of the PreCommit Pipeline. Runs Python integration tests.') - - parameters { - stringParam( - 'buildNum', - 'N/A', - 'Build number of beam_PreCommit_Python_Build to copy from.') - stringParam( - 'ghprbGhRepository', - 'N/A', - 'Repository name for use by ghprb plugin.') - stringParam( - 'ghprbActualCommit', - 'N/A', - 'Commit ID for use by ghprb plugin.') - stringParam( - 'ghprbPullId', - 'N/A', - 'PR # for use by ghprb plugin.') - } - - // Set JDK version. - jdk('JDK 1.8 (latest)') - - // Restrict this project to run only on Jenkins executors as specified - label('beam') - - // Execute concurrent builds if necessary. - concurrentBuild() - - wrappers { - timeout { - absolute(20) - abortBuild() - } - credentialsBinding { - string("COVERALLS_REPO_TOKEN", "beam-coveralls-token") - } - downstreamCommitStatus { - context('Jenkins: Python Integration Tests') - triggeredStatus("Python Integration Tests Pending") - startedStatus("Running Python Integration Tests") - statusUrl() - completedStatus('SUCCESS', "Python Integration Tests Passed") - completedStatus('FAILURE', "Python Integration Tests Failed") - completedStatus('ERROR', "Error Executing Python Integration Tests") - } - // Set SPARK_LOCAL_IP for spark tests. - environmentVariables { - env('SPARK_LOCAL_IP', '127.0.0.1') - } - } - - // Set Maven parameters. - common_job_properties.setMavenConfig(delegate) - - preBuildSteps { - copyArtifacts("beam_PreCommit_Python_Build") { - buildSelector { - buildNumber('${buildNum}') - } - } - } - - // Construct Maven goals for this job. - profiles = [ - 'release', - 'include-runners', - 'jenkins-precommit', - 'direct-runner', - 'dataflow-runner', - 'spark-runner', - 'flink-runner', - 'apex-runner' - ] - args = [ - '-B', - '-e', - '-P' + profiles.join(','), - '?????????', - '-pl sdks/python', - ] - goals(args.join(' ')) -} diff --git a/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy index bf27d541658f..548031267c59 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy @@ -25,9 +25,9 @@ mavenJob('beam_PreCommit_Python_UnitTest') { parameters { stringParam( - 'buildNum', - 'N/A', - 'Build number of beam_PreCommit_Python_Build to copy from.') + 'sha1', + 'master', + 'Commit id or refname (e.g. origin/pr/9/head) you want to build.') stringParam( 'ghprbGhRepository', 'N/A', @@ -74,17 +74,24 @@ mavenJob('beam_PreCommit_Python_UnitTest') { } } - // Set Maven parameters. - common_job_properties.setMavenConfig(delegate) - - preBuildSteps { - copyArtifacts("beam_PreCommit_Python_Build") { - buildSelector { - buildNumber('${buildNum}') + // Source code management. + scm { + git { + remote { + github("apache/beam") + refspec('+refs/heads/*:refs/remotes/origin/* ' + + '+refs/pull/*:refs/remotes/origin/pr/*') + } + branch('${sha1}') + extensions { + cleanAfterCheckout() } } } + // Set Maven parameters. + common_job_properties.setMavenConfig(delegate) + // Construct Maven goals for this job. profiles = [ 'release', From 71967c6cb324fc83b3a9cad59d6977a2788420b3 Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Mon, 10 Jul 2017 18:25:18 -0700 Subject: [PATCH 07/19] Reuse common options in common_job_properties Signed-off-by: Jason Kuster --- .test-infra/jenkins/PreCommit_Pipeline.groovy | 4 +- .../jenkins/common_job_properties.groovy | 103 ++++++++++++++++-- .../job_beam_PreCommit_Java_Build.groovy | 67 +----------- .../job_beam_PreCommit_Java_CodeHealth.groovy | 49 +-------- ...beam_PreCommit_Java_IntegrationTest.groovy | 49 +-------- .../job_beam_PreCommit_Python_UnitTest.groovy | 70 +----------- 6 files changed, 105 insertions(+), 237 deletions(-) diff --git a/.test-infra/jenkins/PreCommit_Pipeline.groovy b/.test-infra/jenkins/PreCommit_Pipeline.groovy index 436aa85aa662..5b0b9432cc7f 100644 --- a/.test-infra/jenkins/PreCommit_Pipeline.groovy +++ b/.test-infra/jenkins/PreCommit_Pipeline.groovy @@ -1,13 +1,13 @@ #!groovy import hudson.model.Result -def ghprbArgs = [ +List ghprbArgs = [ string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), string(name: 'ghprbPullId', value: "${ghprbPullId}") ] -def sha1Arg = [string(name: 'sha1', value: "origin/pr/${ghprbPullId}/head")] +List sha1Arg = [string(name: 'sha1', value: "origin/pr/${ghprbPullId}/head")] try { javaBuildNum = -1 diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy index fa7d55c98e10..81f19fe0d6a1 100644 --- a/.test-infra/jenkins/common_job_properties.groovy +++ b/.test-infra/jenkins/common_job_properties.groovy @@ -219,10 +219,19 @@ class common_job_properties { } } + static def mapToArgList(LinkedHashMap inputArgs) { + List argList = [] + inputArgs.each({ + // FYI: Replacement only works with double quotes. + key, value -> argList.add("--$key=$value") + }) + return argList.join(' ') + } + // Configures the argument list for performance tests, adding the standard // performance test job arguments. private static def genPerformanceArgs(def argMap) { - def standard_args = [ + LinkedHashMap standardArgs = [ project: 'apache-beam-testing', dpb_log_level: 'INFO', maven_binary: '/home/jenkins/tools/maven/latest/bin/mvn', @@ -231,13 +240,8 @@ class common_job_properties { official: 'true' ] // Note: in case of key collision, keys present in ArgMap win. - def joined_args = standard_args.plus(argMap) - def argList = [] - joined_args.each({ - // FYI: Replacement only works with double quotes. - key, value -> argList.add("--$key=$value") - }) - return argList.join(' ') + LinkedHashMap joinedArgs = standardArgs.plus(argMap) + return mapToArgList(joinedArgs) } // Adds the standard performance test job steps. @@ -256,4 +260,87 @@ class common_job_properties { shell("python PerfKitBenchmarker/pkb.py $pkbArgs") } } + + static def setPipelineJobProperties(def context, int tout, String descriptor) { + context.parameters { + stringParam( + 'ghprbGhRepository', + 'N/A', + 'Repository name for use by ghprb plugin.') + stringParam( + 'ghprbActualCommit', + 'N/A', + 'Commit ID for use by ghprb plugin.') + stringParam( + 'ghprbPullId', + 'N/A', + 'PR # for use by ghprb plugin.') + + } + + // Set JDK version. + jdk('JDK 1.8 (latest)') + + // Restrict this project to run only on Jenkins executors as specified + label('beam') + + // Execute concurrent builds if necessary. + concurrentBuild() + + wrappers { + timeout { + absolute(tout) + abortBuild() + } + credentialsBinding { + string("COVERALLS_REPO_TOKEN", "beam-coveralls-token") + } + downstreamCommitStatus { + context("Jenkins: ${descriptor}") + triggeredStatus("${descriptor} Pending") + startedStatus("Running ${descriptor}") + statusUrl() + completedStatus('SUCCESS', "${descriptor} Passed") + completedStatus('FAILURE', "${descriptor} Failed") + completedStatus('ERROR', "Error Executing ${descriptor}") + } + // Set SPARK_LOCAL_IP for spark tests. + environmentVariables { + env('SPARK_LOCAL_IP', '127.0.0.1') + } + } + + // Set Maven parameters. + setMavenConfig(context) + + } + + static def setPipelineBuildJobProperties(def context) { + context.properties { + githubProjectUrl('https://github.com/apache/beam/') + } + + context.parameters { + stringParam( + 'sha1', + 'master', + 'Commit id or refname (e.g. origin/pr/9/head) you want to build.') + } + + // Source code management. + context.scm { + git { + remote { + github("apache/beam") + refspec('+refs/heads/*:refs/remotes/origin/* ' + + '+refs/pull/*:refs/remotes/origin/pr/*') + } + branch('${sha1}') + extensions { + cleanAfterCheckout() + } + } + } + } + } diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy index 18f259e04a31..8cbd1ec11e2d 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy @@ -23,9 +23,8 @@ import common_job_properties mavenJob('beam_PreCommit_Java_Build') { description('Part of the PreCommit Pipeline. Builds Java SDK and archives artifacts.') - properties { - githubProjectUrl('https://github.com/apache/beam/') - } + common_job_properties.setPipelineJobProperties(delegate, 15, "Java Build") + common_job_properties.setPipelineBuildJobProperties(delegate) configure { project -> project / 'properties' / 'hudson.plugins.copyartifact.CopyArtifactPermissionProperty' / 'projectNameList' { @@ -33,68 +32,6 @@ mavenJob('beam_PreCommit_Java_Build') { } } - parameters { - stringParam( - 'sha1', - 'master', - 'Commit id or refname (e.g. origin/pr/9/head) you want to build.') - stringParam( - 'ghprbGhRepository', - 'N/A', - 'Repository name for use by ghprb plugin.') - stringParam( - 'ghprbActualCommit', - 'N/A', - 'Commit ID for use by ghprb plugin.') - stringParam( - 'ghprbPullId', - 'N/A', - 'PR # for use by ghprb plugin.') - } - - wrappers { - timeout { - absolute(15) - abortBuild() - } - downstreamCommitStatus { - context('Jenkins: Java Build') - triggeredStatus("Java Build Pending") - startedStatus("Running Java Build") - statusUrl() - completedStatus('SUCCESS', "Java Build Passed") - completedStatus('FAILURE', "Java Build Failed") - completedStatus('ERROR', "Error Executing Java Build") - } - } - - // Set JDK version. - jdk('JDK 1.8 (latest)') - - // Restrict this project to run only on Jenkins executors as specified - label('beam') - - // Execute concurrent builds if necessary. - concurrentBuild() - - // Source code management. - scm { - git { - remote { - github("apache/beam") - refspec('+refs/heads/*:refs/remotes/origin/* ' + - '+refs/pull/*:refs/remotes/origin/pr/*') - } - branch('${sha1}') - extensions { - cleanAfterCheckout() - } - } - } - - // Set Maven parameters. - common_job_properties.setMavenConfig(delegate) - // Construct Maven goals for this job. profiles = [ 'release', diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy index 45a2dd0494e1..0d03e266c0b1 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy @@ -23,60 +23,15 @@ import common_job_properties mavenJob('beam_PreCommit_Java_CodeHealth') { description('Part of the PreCommit Pipeline. Runs Java code health checks.') + common_job_properties.setPipelineJobProperties(delegate, 15, "Java Code Health") + parameters { stringParam( 'buildNum', 'N/A', 'Build number of beam_PreCommit_Java_Build to copy from.') - stringParam( - 'ghprbGhRepository', - 'N/A', - 'Repository name for use by ghprb plugin.') - stringParam( - 'ghprbActualCommit', - 'N/A', - 'Commit ID for use by ghprb plugin.') - stringParam( - 'ghprbPullId', - 'N/A', - 'PR # for use by ghprb plugin.') } - // Set JDK version. - jdk('JDK 1.8 (latest)') - - // Restrict this project to run only on Jenkins executors as specified - label('beam') - - // Execute concurrent builds if necessary. - concurrentBuild() - - wrappers { - timeout { - absolute(20) - abortBuild() - } - credentialsBinding { - string("COVERALLS_REPO_TOKEN", "beam-coveralls-token") - } - downstreamCommitStatus { - context('Jenkins: Java Code Health') - triggeredStatus("Java Code Health Pending") - startedStatus("Running Java Code Health") - statusUrl() - completedStatus('SUCCESS', "Java Code Health Passed") - completedStatus('FAILURE', "Java Code Health Failed") - completedStatus('ERROR', "Error Executing Java Code Health") - } - // Set SPARK_LOCAL_IP for spark tests. - environmentVariables { - env('SPARK_LOCAL_IP', '127.0.0.1') - } - } - - // Set Maven parameters. - common_job_properties.setMavenConfig(delegate) - preBuildSteps { copyArtifacts("beam_PreCommit_Java_Build") { buildSelector { diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy index e41358ecad5c..a23e33a3fdcf 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy @@ -23,60 +23,15 @@ import common_job_properties mavenJob('beam_PreCommit_Java_IntegrationTest') { description('Part of the PreCommit Pipeline. Runs Java Failsafe integration tests.') + common_job_properties.setPipelineJobProperties(delegate, 25, "Java Integration Tests") + parameters { stringParam( 'buildNum', 'N/A', 'Build number of beam_PreCommit_Java_Build to copy from.') - stringParam( - 'ghprbGhRepository', - 'N/A', - 'Repository name for use by ghprb plugin.') - stringParam( - 'ghprbActualCommit', - 'N/A', - 'Commit ID for use by ghprb plugin.') - stringParam( - 'ghprbPullId', - 'N/A', - 'PR # for use by ghprb plugin.') } - // Set JDK version. - jdk('JDK 1.8 (latest)') - - // Restrict this project to run only on Jenkins executors as specified - label('beam') - - // Execute concurrent builds if necessary. - concurrentBuild() - - wrappers { - timeout { - absolute(20) - abortBuild() - } - credentialsBinding { - string("COVERALLS_REPO_TOKEN", "beam-coveralls-token") - } - downstreamCommitStatus { - context('Jenkins: Java Integration Tests') - triggeredStatus("Java Integration Tests Pending") - startedStatus("Running Java Integration Tests") - statusUrl() - completedStatus('SUCCESS', "Java Integration Tests Passed") - completedStatus('FAILURE', "Java Integration Tests Failed") - completedStatus('ERROR', "Error Executing Java Integration Tests") - } - // Set SPARK_LOCAL_IP for spark tests. - environmentVariables { - env('SPARK_LOCAL_IP', '127.0.0.1') - } - } - - // Set Maven parameters. - common_job_properties.setMavenConfig(delegate) - preBuildSteps { copyArtifacts("beam_PreCommit_Java_Build") { buildSelector { diff --git a/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy index 548031267c59..fe709225ba51 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy @@ -23,74 +23,8 @@ import common_job_properties mavenJob('beam_PreCommit_Python_UnitTest') { description('Part of the PreCommit Pipeline. Runs Python unit tests.') - parameters { - stringParam( - 'sha1', - 'master', - 'Commit id or refname (e.g. origin/pr/9/head) you want to build.') - stringParam( - 'ghprbGhRepository', - 'N/A', - 'Repository name for use by ghprb plugin.') - stringParam( - 'ghprbActualCommit', - 'N/A', - 'Commit ID for use by ghprb plugin.') - stringParam( - 'ghprbPullId', - 'N/A', - 'PR # for use by ghprb plugin.') - } - - // Set JDK version. - jdk('JDK 1.8 (latest)') - - // Restrict this project to run only on Jenkins executors as specified - label('beam') - - // Execute concurrent builds if necessary. - concurrentBuild() - - wrappers { - timeout { - absolute(20) - abortBuild() - } - credentialsBinding { - string("COVERALLS_REPO_TOKEN", "beam-coveralls-token") - } - downstreamCommitStatus { - context('Jenkins: Python Unit Tests') - triggeredStatus("Python Unit Tests Pending") - startedStatus("Running Python Unit Tests") - statusUrl() - completedStatus('SUCCESS', "Python Unit Tests Passed") - completedStatus('FAILURE', "Python Unit Tests Failed") - completedStatus('ERROR', "Error Executing Python Unit Tests") - } - // Set SPARK_LOCAL_IP for spark tests. - environmentVariables { - env('SPARK_LOCAL_IP', '127.0.0.1') - } - } - - // Source code management. - scm { - git { - remote { - github("apache/beam") - refspec('+refs/heads/*:refs/remotes/origin/* ' + - '+refs/pull/*:refs/remotes/origin/pr/*') - } - branch('${sha1}') - extensions { - cleanAfterCheckout() - } - } - } - - // Set Maven parameters. - common_job_properties.setMavenConfig(delegate) + common_job_properties.setPipelineJobProperties(delegate, 25, "Python Unit Tests") + common_job_properties.setPipelineBuildJobProperties(delegate) // Construct Maven goals for this job. profiles = [ From 505c61d5f61a2dab6ef5d4354a3406d90dfaee86 Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Mon, 10 Jul 2017 18:34:24 -0700 Subject: [PATCH 08/19] Extract downstream settings into common_job_properties Signed-off-by: Jason Kuster --- .../jenkins/common_job_properties.groovy | 21 ++++++++++++++----- .../job_beam_PreCommit_Java_CodeHealth.groovy | 16 +------------- ...beam_PreCommit_Java_IntegrationTest.groovy | 16 +------------- 3 files changed, 18 insertions(+), 35 deletions(-) diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy index 81f19fe0d6a1..0c017701e905 100644 --- a/.test-infra/jenkins/common_job_properties.groovy +++ b/.test-infra/jenkins/common_job_properties.groovy @@ -111,11 +111,6 @@ class common_job_properties { } } - private static void setPipelineDownstreamJobProperties(context, - String sdkLang) { - - } - // Sets the pull request build trigger. Accessed through precommit methods // below to insulate callers from internal parameter defaults. private static void setPullRequestBuildTrigger(context, @@ -343,4 +338,20 @@ class common_job_properties { } } + static def setPipelineDownstreamJobProperties(def context, String jobName) { + context.parameters { + stringParam( + 'buildNum', + 'N/A', + "Build number of ${jobName} to copy from.") + } + + context.preBuildSteps { + copyArtifacts(jobName) { + buildSelector { + buildNumber('${buildNum}') + } + } + } + } } diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy index 0d03e266c0b1..1019084eae05 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy @@ -24,21 +24,7 @@ mavenJob('beam_PreCommit_Java_CodeHealth') { description('Part of the PreCommit Pipeline. Runs Java code health checks.') common_job_properties.setPipelineJobProperties(delegate, 15, "Java Code Health") - - parameters { - stringParam( - 'buildNum', - 'N/A', - 'Build number of beam_PreCommit_Java_Build to copy from.') - } - - preBuildSteps { - copyArtifacts("beam_PreCommit_Java_Build") { - buildSelector { - buildNumber('${buildNum}') - } - } - } + common_job_properties.setPipelineDownstreamJobProperties('beam_PreCommit_Java_Build') // Construct Maven goals for this job. profiles = [ diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy index a23e33a3fdcf..d0c23e5e9761 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy @@ -24,21 +24,7 @@ mavenJob('beam_PreCommit_Java_IntegrationTest') { description('Part of the PreCommit Pipeline. Runs Java Failsafe integration tests.') common_job_properties.setPipelineJobProperties(delegate, 25, "Java Integration Tests") - - parameters { - stringParam( - 'buildNum', - 'N/A', - 'Build number of beam_PreCommit_Java_Build to copy from.') - } - - preBuildSteps { - copyArtifacts("beam_PreCommit_Java_Build") { - buildSelector { - buildNumber('${buildNum}') - } - } - } + common_job_properties.setPipelineDownstreamJobProperties('beam_PreCommit_Java_Build') // Construct Maven goals for this job. profiles = [ From bb1e926cea721443f5b7692ea125f93521de8067 Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Tue, 11 Jul 2017 11:24:14 -0700 Subject: [PATCH 09/19] Pick up changes in Java_UnitTest, plus extracted scm into c_j_p. Signed-off-by: Jason Kuster --- .../jenkins/common_job_properties.groovy | 50 ++++++-------- .../job_beam_PreCommit_Java_Build.groovy | 7 +- .../job_beam_PreCommit_Java_CodeHealth.groovy | 2 +- ...beam_PreCommit_Java_IntegrationTest.groovy | 5 +- .../job_beam_PreCommit_Java_UnitTest.groovy | 66 +------------------ .../job_beam_PreCommit_Pipeline.groovy | 16 +---- 6 files changed, 37 insertions(+), 109 deletions(-) diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy index 0c017701e905..45aae790ccd2 100644 --- a/.test-infra/jenkins/common_job_properties.groovy +++ b/.test-infra/jenkins/common_job_properties.groovy @@ -22,8 +22,24 @@ // http://groovy-lang.org/style-guide.html class common_job_properties { + static void setBeamSCM(def context, String repositoryName) { + context.scm { + git { + remote { + github("apache/${repositoryName}") + refspec('+refs/heads/*:refs/remotes/origin/* ' + + '+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*') + } + branch('${sha1}') + extensions { + cleanAfterCheckout() + } + } + } + } + // Sets common top-level job properties for website repository jobs. - static void setTopLevelWebsiteJobProperties(context, + static void setTopLevelWebsiteJobProperties(def context, String branch = 'asf-site') { setTopLevelJobProperties( context, @@ -34,7 +50,7 @@ class common_job_properties { } // Sets common top-level job properties for main repository jobs. - static void setTopLevelMainJobProperties(context, + static void setTopLevelMainJobProperties(def context, String branch = 'master', int timeout = 100, String jenkinsExecutorLabel = 'beam') { @@ -48,7 +64,7 @@ class common_job_properties { // Sets common top-level job properties. Accessed through one of the above // methods to protect jobs from internal details of param defaults. - private static void setTopLevelJobProperties(context, + private static void setTopLevelJobProperties(def context, String repositoryName, String defaultBranch, String jenkinsExecutorLabel, @@ -71,19 +87,7 @@ class common_job_properties { } // Source code management. - context.scm { - git { - remote { - github("apache/${repositoryName}") - refspec('+refs/heads/*:refs/remotes/origin/* ' + - '+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*') - } - branch('${sha1}') - extensions { - cleanAfterCheckout() - } - } - } + setBeamSCM(context, repositoryName) context.parameters { // This is a recommended setup if you want to run the job manually. The @@ -323,19 +327,7 @@ class common_job_properties { } // Source code management. - context.scm { - git { - remote { - github("apache/beam") - refspec('+refs/heads/*:refs/remotes/origin/* ' + - '+refs/pull/*:refs/remotes/origin/pr/*') - } - branch('${sha1}') - extensions { - cleanAfterCheckout() - } - } - } + setBeamSCM(context, 'beam') } static def setPipelineDownstreamJobProperties(def context, String jobName) { diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy index 8cbd1ec11e2d..3cc4cd9a4680 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy @@ -23,20 +23,23 @@ import common_job_properties mavenJob('beam_PreCommit_Java_Build') { description('Part of the PreCommit Pipeline. Builds Java SDK and archives artifacts.') + // Set properties common to PreCommit Pipeline subjobs. common_job_properties.setPipelineJobProperties(delegate, 15, "Java Build") + // Set properties common to PreCommit initial jobs. common_job_properties.setPipelineBuildJobProperties(delegate) + // The CopyArtifact plugin doesn't support the job DSL so we have to configure it manually. configure { project -> project / 'properties' / 'hudson.plugins.copyartifact.CopyArtifactPermissionProperty' / 'projectNameList' { 'string' "beam_*" } } + // TODO: Add configuration controlling how long to keep artifacts for. // Construct Maven goals for this job. profiles = [ 'release', 'include-runners', - 'jenkins-precommit', 'direct-runner', 'dataflow-runner', 'spark-runner', @@ -44,6 +47,7 @@ mavenJob('beam_PreCommit_Java_Build') { 'apex-runner' ] args = [ + '-B', '-e', '-P' + profiles.join(','), @@ -58,6 +62,7 @@ mavenJob('beam_PreCommit_Java_Build') { ] goals(args.join(' ')) + // This job publishes artifacts so that downstream jobs can use them. publishers { archiveArtifacts { pattern('.repository/org/apache/beam/**/*') diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy index 1019084eae05..d8cf8ad78a1c 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy @@ -24,7 +24,7 @@ mavenJob('beam_PreCommit_Java_CodeHealth') { description('Part of the PreCommit Pipeline. Runs Java code health checks.') common_job_properties.setPipelineJobProperties(delegate, 15, "Java Code Health") - common_job_properties.setPipelineDownstreamJobProperties('beam_PreCommit_Java_Build') + common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_PreCommit_Java_Build') // Construct Maven goals for this job. profiles = [ diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy index d0c23e5e9761..83f098506db8 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy @@ -24,7 +24,7 @@ mavenJob('beam_PreCommit_Java_IntegrationTest') { description('Part of the PreCommit Pipeline. Runs Java Failsafe integration tests.') common_job_properties.setPipelineJobProperties(delegate, 25, "Java Integration Tests") - common_job_properties.setPipelineDownstreamJobProperties('beam_PreCommit_Java_Build') + common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_PreCommit_Java_Build') // Construct Maven goals for this job. profiles = [ @@ -40,8 +40,9 @@ mavenJob('beam_PreCommit_Java_IntegrationTest') { args = [ '-B', '-e', - '-P' + profiles.join(','), + "-P${profiles.join(',')}", 'failsafe:integration-test', + // TODO: PUT APPROPRIATE ARGS / OTHER PIECES OF THE INCANTATION HERE. '-pl \'!sdks/python\'', ] goals(args.join(' ')) diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy index 9f976e88f97b..78ebddb5222f 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy @@ -23,73 +23,13 @@ import common_job_properties mavenJob('beam_PreCommit_Java_UnitTest') { description('Part of the PreCommit Pipeline. Runs Java Surefire unit tests.') - parameters { - stringParam( - 'buildNum', - 'N/A', - 'Build number of beam_PreCommit_Java_Build to copy from.') - stringParam( - 'ghprbGhRepository', - 'N/A', - 'Repository name for use by ghprb plugin.') - stringParam( - 'ghprbActualCommit', - 'N/A', - 'Commit ID for use by ghprb plugin.') - stringParam( - 'ghprbPullId', - 'N/A', - 'PR # for use by ghprb plugin.') - } - - // Set JDK version. - jdk('JDK 1.8 (latest)') - - // Restrict this project to run only on Jenkins executors as specified - label('beam') - - // Execute concurrent builds if necessary. - concurrentBuild() - - wrappers { - timeout { - absolute(20) - abortBuild() - } - credentialsBinding { - string("COVERALLS_REPO_TOKEN", "beam-coveralls-token") - } - downstreamCommitStatus { - context('Jenkins: Java Unit Tests') - triggeredStatus("Java Unit Tests Pending") - startedStatus("Running Java Unit Tests") - statusUrl() - completedStatus('SUCCESS', "Java Unit Tests Passed") - completedStatus('FAILURE', "Java Unit Tests Failed") - completedStatus('ERROR', "Error Executing Java Unit Tests") - } - // Set SPARK_LOCAL_IP for spark tests. - environmentVariables { - env('SPARK_LOCAL_IP', '127.0.0.1') - } - } - - // Set Maven parameters. - common_job_properties.setMavenConfig(delegate) - - preBuildSteps { - copyArtifacts("beam_PreCommit_Java_Build") { - buildSelector { - buildNumber('${buildNum}') - } - } - } + common_job_properties.setPipelineJobProperties(delegate, 20, "Java Unit Tests") + common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_PreCommit_Java_Build') // Construct Maven goals for this job. profiles = [ 'release', 'include-runners', - 'jenkins-precommit', 'direct-runner', 'dataflow-runner', 'spark-runner', @@ -99,7 +39,7 @@ mavenJob('beam_PreCommit_Java_UnitTest') { args = [ '-B', '-e', - '-P' + profiles.join(','), + "-P${profiles.join(',')}", 'surefire:test@default-test', 'coveralls:report', '-pl \'!sdks/python\'', diff --git a/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy index 11b32d5c21b2..2295ec195d58 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy @@ -17,6 +17,8 @@ * limitations under the License. */ +import common_job_properties + // This is the Java precommit which runs a maven install, and the current set // of precommit tests. pipelineJob('beam_PreCommit_Pipeline') { @@ -70,19 +72,7 @@ pipelineJob('beam_PreCommit_Pipeline') { definition { cpsScm { // Source code management. - scm { - git { - remote { - github("apache/beam") - refspec('+refs/heads/*:refs/remotes/origin/* ' + - '+refs/pull/*:refs/remotes/origin/pr/*') - } - branch('${sha1}') - extensions { - cleanAfterCheckout() - } - } - } + common_job_properties.setBeamSCM(delegate, 'beam') scriptPath('.test-infra/jenkins/PreCommit_Pipeline.groovy') } } From cba593ea91d42a7dc3aa4a0eba51ad188f9c8bdc Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Tue, 11 Jul 2017 11:30:50 -0700 Subject: [PATCH 10/19] Cut Maven executions down to just what they need. Signed-off-by: Jason Kuster --- .../jenkins/job_beam_PreCommit_Java_Build.groovy | 1 - .../jenkins/job_beam_PreCommit_Python_UnitTest.groovy | 11 ----------- 2 files changed, 12 deletions(-) diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy index 3cc4cd9a4680..f8fdc3cc5628 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy @@ -38,7 +38,6 @@ mavenJob('beam_PreCommit_Java_Build') { // Construct Maven goals for this job. profiles = [ - 'release', 'include-runners', 'direct-runner', 'dataflow-runner', diff --git a/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy index fe709225ba51..6e263ee1903b 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy @@ -27,20 +27,9 @@ mavenJob('beam_PreCommit_Python_UnitTest') { common_job_properties.setPipelineBuildJobProperties(delegate) // Construct Maven goals for this job. - profiles = [ - 'release', - 'include-runners', - 'jenkins-precommit', - 'direct-runner', - 'dataflow-runner', - 'spark-runner', - 'flink-runner', - 'apex-runner' - ] args = [ '-B', '-e', - '-P' + profiles.join(','), 'clean install', '-pl sdks/python', ] From 1eeae5d93094676de4d5d190349ab423d573c3b8 Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Tue, 11 Jul 2017 11:58:53 -0700 Subject: [PATCH 11/19] fixup! Cut Maven executions down to just what they need. --- .../jenkins/job_beam_PreCommit_Java_Build.groovy | 15 +++++++-------- .../job_beam_PreCommit_Java_CodeHealth.groovy | 10 +++++----- ...job_beam_PreCommit_Java_IntegrationTest.groovy | 6 ++---- .../job_beam_PreCommit_Java_UnitTest.groovy | 8 +++----- 4 files changed, 17 insertions(+), 22 deletions(-) diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy index f8fdc3cc5628..034841a88863 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy @@ -34,11 +34,11 @@ mavenJob('beam_PreCommit_Java_Build') { 'string' "beam_*" } } + // TODO: Add configuration controlling how long to keep artifacts for. // Construct Maven goals for this job. profiles = [ - 'include-runners', 'direct-runner', 'dataflow-runner', 'spark-runner', @@ -46,18 +46,17 @@ mavenJob('beam_PreCommit_Java_Build') { 'apex-runner' ] args = [ - '-B', '-e', - '-P' + profiles.join(','), + "-P${profiles.join(',')}", 'clean', 'install', - '-pl \'!sdks/python\'', + "-pl '!sdks/python'", '-DskipTests', - '-Dcheckstyle.skip', - '-Dfindbugs.skip', - '-Dmaven.javadoc.skip', - '-Drat.skip' + '-Dcheckstyle.skip', // TODO: I think this line is necessary. Verify. + '-Dfindbugs.skip', // TODO: Do we need now that -Prelease is gone? + '-Dmaven.javadoc.skip', // TODO: Javadoc still seems to run, figure out what also to skip. + '-Drat.skip' // TODO: Do we need now that -Prelease is gone? ] goals(args.join(' ')) diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy index d8cf8ad78a1c..2dc6576353f3 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy @@ -27,9 +27,8 @@ mavenJob('beam_PreCommit_Java_CodeHealth') { common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_PreCommit_Java_Build') // Construct Maven goals for this job. - profiles = [ + profiles = [ // TODO: Some of these are definitely not necessary. 'release', - 'include-runners', 'jenkins-precommit', 'direct-runner', 'dataflow-runner', @@ -40,11 +39,12 @@ mavenJob('beam_PreCommit_Java_CodeHealth') { args = [ '-B', '-e', - '-P' + profiles.join(','), - '-pl \'!sdks/python\'', + "-P${profiles.join(',')}", + "-pl '!sdks/python'", 'checkstyle:check', 'findbugs:check', - 'rat:check' + 'rat:check', + // TODO: Need something here for javadoc. ] goals(args.join(' ')) } diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy index 83f098506db8..f170936bb6aa 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy @@ -27,9 +27,7 @@ mavenJob('beam_PreCommit_Java_IntegrationTest') { common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_PreCommit_Java_Build') // Construct Maven goals for this job. - profiles = [ - 'release', - 'include-runners', + profiles = [ // TODO: Is jenkins-precommit enough here? 'jenkins-precommit', 'direct-runner', 'dataflow-runner', @@ -43,7 +41,7 @@ mavenJob('beam_PreCommit_Java_IntegrationTest') { "-P${profiles.join(',')}", 'failsafe:integration-test', // TODO: PUT APPROPRIATE ARGS / OTHER PIECES OF THE INCANTATION HERE. - '-pl \'!sdks/python\'', + "-pl '!sdks/python'", ] goals(args.join(' ')) } diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy index 78ebddb5222f..1f1d707193ab 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy @@ -27,9 +27,7 @@ mavenJob('beam_PreCommit_Java_UnitTest') { common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_PreCommit_Java_Build') // Construct Maven goals for this job. - profiles = [ - 'release', - 'include-runners', + profiles = [ // TODO: Are all of these necessary? 'direct-runner', 'dataflow-runner', 'spark-runner', @@ -41,8 +39,8 @@ mavenJob('beam_PreCommit_Java_UnitTest') { '-e', "-P${profiles.join(',')}", 'surefire:test@default-test', - 'coveralls:report', - '-pl \'!sdks/python\'', + 'coveralls:report', // TODO: Will this work? + "-pl '!sdks/python'", '-DrepoToken=$COVERALLS_REPO_TOKEN', '-DpullRequest=$ghprbPullId', ] From c5f4cbeba8f1abd0144be410cde10899126b98fc Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Tue, 11 Jul 2017 17:30:30 -0700 Subject: [PATCH 12/19] Some additional Maven invocation changes, plus actually error pipeline out. Signed-off-by: Jason Kuster --- .test-infra/jenkins/PreCommit_Pipeline.groovy | 80 +++++++++---------- .../job_beam_PreCommit_Java_Build.groovy | 17 ++-- .../job_beam_PreCommit_Java_CodeHealth.groovy | 2 +- .../job_beam_PreCommit_Java_UnitTest.groovy | 4 +- 4 files changed, 52 insertions(+), 51 deletions(-) diff --git a/.test-infra/jenkins/PreCommit_Pipeline.groovy b/.test-infra/jenkins/PreCommit_Pipeline.groovy index 5b0b9432cc7f..4e8257a3d913 100644 --- a/.test-infra/jenkins/PreCommit_Pipeline.groovy +++ b/.test-infra/jenkins/PreCommit_Pipeline.groovy @@ -9,49 +9,47 @@ List ghprbArgs = [ List sha1Arg = [string(name: 'sha1', value: "origin/pr/${ghprbPullId}/head")] -try { - javaBuildNum = -1 - stage('Build') { - parallel ( - java: { - def javaBuild = build job: 'beam_PreCommit_Java_Build', parameters: sha1Arg + ghprbArgs - if(javaBuild.getResult() == Result.SUCCESS.toString()) { - javaBuildNum = javaBuild.getNumber() - } +int javaBuildNum = -1 +stage('Build') { + parallel ( + java: { + def javaBuild = build job: 'beam_PreCommit_Java_Build', parameters: sha1Arg + ghprbArgs + if(javaBuild.getResult() == Result.SUCCESS.toString()) { + javaBuildNum = javaBuild.getNumber() } - ) - } - javaBuildArg = [string(name: 'buildNum', value: "${javaBuildNum}")] - javaUnitPassed = false - stage('Unit Test / Code Health') { - parallel ( - java_unit: { - if(javaBuildNum != -1) { - def javaTest = build job: 'beam_PreCommit_Java_UnitTest', parameters: javaBuildArg + ghprbArgs - if(javaTest.getResult() == Result.SUCCESS.toString()) { - javaUnitPassed = true - } - } - }, - java_codehealth: { - if(javaBuildNum != -1) { - build job: 'beam_PreCommit_Java_CodeHealth', parameters: javaBuildArg + ghprbArgs + } + ) +} + +javaBuildArg = [string(name: 'buildNum', value: "${javaBuildNum}")] +javaUnitPassed = false +stage('Unit Test / Code Health') { + parallel ( + java_unit: { + if(javaBuildNum != -1) { + def javaTest = build job: 'beam_PreCommit_Java_UnitTest', parameters: javaBuildArg + ghprbArgs + if(javaTest.getResult() == Result.SUCCESS.toString()) { + javaUnitPassed = true } - }, - python_unit: { - build job: 'beam_PreCommit_Python_UnitTest', parameters: sha1Arg + ghprbArgs } - ) - } - stage('Integration Test') { - parallel ( - java_integration: { - if(javaUnitPassed) { - build job: 'beam_PreCommit_Java_IntegrationTest', parameters: javaBuildArg + ghprbArgs - } + }, + java_codehealth: { + if(javaBuildNum != -1) { + build job: 'beam_PreCommit_Java_CodeHealth', parameters: javaBuildArg + ghprbArgs + } + }, + python_unit: { + build job: 'beam_PreCommit_Python_UnitTest', parameters: sha1Arg + ghprbArgs + } + ) +} + +stage('Integration Test') { + parallel ( + java_integration: { + if(javaUnitPassed) { + build job: 'beam_PreCommit_Java_IntegrationTest', parameters: javaBuildArg + ghprbArgs } - ) - } -} catch (Exception e) { - echo e.toString() + } + ) } diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy index 034841a88863..27526949693d 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy @@ -28,15 +28,20 @@ mavenJob('beam_PreCommit_Java_Build') { // Set properties common to PreCommit initial jobs. common_job_properties.setPipelineBuildJobProperties(delegate) - // The CopyArtifact plugin doesn't support the job DSL so we have to configure it manually. configure { project -> + // The CopyArtifact plugin doesn't support the job DSL so we have to configure it manually. project / 'properties' / 'hudson.plugins.copyartifact.CopyArtifactPermissionProperty' / 'projectNameList' { 'string' "beam_*" } + // The Build Discarder also doesn't support the job DSL in the right way so we have to configure it manually. + project / 'properties' / 'jenkins.model.BuildDiscarderProperty' / 'strategy'(class:'hudson.tasks.LogRotator') { + 'daysToKeep'(-1) + 'numToKeep'(-1) + 'artifactDaysToKeep'(1) + 'artifactNumToKeep'(-1) + } } - // TODO: Add configuration controlling how long to keep artifacts for. - // Construct Maven goals for this job. profiles = [ 'direct-runner', @@ -51,12 +56,10 @@ mavenJob('beam_PreCommit_Java_Build') { "-P${profiles.join(',')}", 'clean', 'install', - "-pl '!sdks/python'", + "-pl '!sdks/python,!sdks/java/javadoc'", '-DskipTests', - '-Dcheckstyle.skip', // TODO: I think this line is necessary. Verify. - '-Dfindbugs.skip', // TODO: Do we need now that -Prelease is gone? + '-Dcheckstyle.skip', '-Dmaven.javadoc.skip', // TODO: Javadoc still seems to run, figure out what also to skip. - '-Drat.skip' // TODO: Do we need now that -Prelease is gone? ] goals(args.join(' ')) diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy index 2dc6576353f3..c33b74adf5aa 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy @@ -29,7 +29,6 @@ mavenJob('beam_PreCommit_Java_CodeHealth') { // Construct Maven goals for this job. profiles = [ // TODO: Some of these are definitely not necessary. 'release', - 'jenkins-precommit', 'direct-runner', 'dataflow-runner', 'spark-runner', @@ -41,6 +40,7 @@ mavenJob('beam_PreCommit_Java_CodeHealth') { '-e', "-P${profiles.join(',')}", "-pl '!sdks/python'", + // TODO: This incantation fails in the root -- figure out why. 'checkstyle:check', 'findbugs:check', 'rat:check', diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy index 1f1d707193ab..fecc146a1479 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy @@ -27,7 +27,7 @@ mavenJob('beam_PreCommit_Java_UnitTest') { common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_PreCommit_Java_Build') // Construct Maven goals for this job. - profiles = [ // TODO: Are all of these necessary? + profiles = [ 'direct-runner', 'dataflow-runner', 'spark-runner', @@ -39,7 +39,7 @@ mavenJob('beam_PreCommit_Java_UnitTest') { '-e', "-P${profiles.join(',')}", 'surefire:test@default-test', - 'coveralls:report', // TODO: Will this work? + 'coveralls:report', // TODO: Will this work? Can't verify on my own Jenkins due to no coveralls. "-pl '!sdks/python'", '-DrepoToken=$COVERALLS_REPO_TOKEN', '-DpullRequest=$ghprbPullId', From 4d118e91559a5eaaa6def3df64974babb0a8ff27 Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Tue, 11 Jul 2017 17:39:27 -0700 Subject: [PATCH 13/19] Add license to Pipeline job. Signed-off-by: Jason Kuster --- .test-infra/jenkins/PreCommit_Pipeline.groovy | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.test-infra/jenkins/PreCommit_Pipeline.groovy b/.test-infra/jenkins/PreCommit_Pipeline.groovy index 4e8257a3d913..20d53785b32a 100644 --- a/.test-infra/jenkins/PreCommit_Pipeline.groovy +++ b/.test-infra/jenkins/PreCommit_Pipeline.groovy @@ -1,4 +1,22 @@ #!groovy +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + import hudson.model.Result List ghprbArgs = [ From a54dd63f908c1f0666ed727babf72f718982ec34 Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Fri, 21 Jul 2017 10:19:51 -0700 Subject: [PATCH 14/19] Final Maven invocations for code health and integration test steps. Signed-off-by: Jason Kuster --- .../job_beam_PreCommit_Java_CodeHealth.groovy | 7 ++----- ...b_beam_PreCommit_Java_IntegrationTest.groovy | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy index c33b74adf5aa..eebbe8a19d27 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy @@ -27,8 +27,7 @@ mavenJob('beam_PreCommit_Java_CodeHealth') { common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_PreCommit_Java_Build') // Construct Maven goals for this job. - profiles = [ // TODO: Some of these are definitely not necessary. - 'release', + profiles = [ 'direct-runner', 'dataflow-runner', 'spark-runner', @@ -40,11 +39,9 @@ mavenJob('beam_PreCommit_Java_CodeHealth') { '-e', "-P${profiles.join(',')}", "-pl '!sdks/python'", - // TODO: This incantation fails in the root -- figure out why. 'checkstyle:check', 'findbugs:check', - 'rat:check', - // TODO: Need something here for javadoc. + 'org.apache.rat:apache-rat-plugin:check', ] goals(args.join(' ')) } diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy index f170936bb6aa..a49d8baf5731 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy @@ -27,7 +27,7 @@ mavenJob('beam_PreCommit_Java_IntegrationTest') { common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_PreCommit_Java_Build') // Construct Maven goals for this job. - profiles = [ // TODO: Is jenkins-precommit enough here? + profiles = [ 'jenkins-precommit', 'direct-runner', 'dataflow-runner', @@ -35,13 +35,22 @@ mavenJob('beam_PreCommit_Java_IntegrationTest') { 'flink-runner', 'apex-runner' ] + examples_integration_executions = [ + 'apex-runner-integration-tests', + 'dataflow-runner-integration-tests', + 'dataflow-runner-integration-tests-streaming', + 'direct-runner-integration-tests', + 'flink-runner-integration-tests', + 'spark-runner-integration-tests', + ] args = [ '-B', '-e', "-P${profiles.join(',')}", - 'failsafe:integration-test', - // TODO: PUT APPROPRIATE ARGS / OTHER PIECES OF THE INCANTATION HERE. - "-pl '!sdks/python'", + "-pl examples/java", ] + examples_integration_executions.each({ + value -> args.add("failsafe:integration-test@${value}") + }) goals(args.join(' ')) } From a8353b2841a1e67dc69281a16229940e070e9cf2 Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Tue, 25 Jul 2017 15:00:01 -0700 Subject: [PATCH 15/19] Add additional commenting and address pull request comments. Signed-off-by: Jason Kuster --- .test-infra/jenkins/PreCommit_Pipeline.groovy | 35 +++++++++++++------ .../jenkins/common_job_properties.groovy | 26 +++++++++++--- ...uild.groovy => job_beam_Java_Build.groovy} | 13 ++++--- ...groovy => job_beam_Java_CodeHealth.groovy} | 20 ++++------- ...y => job_beam_Java_IntegrationTest.groovy} | 19 ++++++---- ...t.groovy => job_beam_Java_UnitTest.groovy} | 11 +++--- .../job_beam_PreCommit_Pipeline.groovy | 4 ++- ...groovy => job_beam_Python_UnitTest.groovy} | 11 +++--- 8 files changed, 86 insertions(+), 53 deletions(-) rename .test-infra/jenkins/{job_beam_PreCommit_Java_Build.groovy => job_beam_Java_Build.groovy} (84%) rename .test-infra/jenkins/{job_beam_PreCommit_Java_CodeHealth.groovy => job_beam_Java_CodeHealth.groovy} (72%) rename .test-infra/jenkins/{job_beam_PreCommit_Java_IntegrationTest.groovy => job_beam_Java_IntegrationTest.groovy} (63%) rename .test-infra/jenkins/{job_beam_PreCommit_Java_UnitTest.groovy => job_beam_Java_UnitTest.groovy} (79%) rename .test-infra/jenkins/{job_beam_PreCommit_Python_UnitTest.groovy => job_beam_Python_UnitTest.groovy} (72%) diff --git a/.test-infra/jenkins/PreCommit_Pipeline.groovy b/.test-infra/jenkins/PreCommit_Pipeline.groovy index 20d53785b32a..e2156323d5ba 100644 --- a/.test-infra/jenkins/PreCommit_Pipeline.groovy +++ b/.test-infra/jenkins/PreCommit_Pipeline.groovy @@ -19,45 +19,58 @@ import hudson.model.Result +int NO_BUILD = -1 + +// These are args for the GitHub Pull Request Builder (ghprb) Plugin. Providing these arguments is +// necessary due to a bug in the ghprb plugin where environment variables are not correctly passed +// to jobs downstream of a Pipeline job. List ghprbArgs = [ string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), string(name: 'ghprbPullId', value: "${ghprbPullId}") ] -List sha1Arg = [string(name: 'sha1', value: "origin/pr/${ghprbPullId}/head")] +// This argument is the commit at which to build. +List commitArg = [string(name: 'commit', value: "origin/pr/${ghprbPullId}/head")] + +int javaBuildNum = NO_BUILD -int javaBuildNum = -1 +// This (and the below) define "Stages" of a pipeline. These stages run serially, and inside can +// have "parallel" blocks which execute several work steps concurrently. This work is limited to +// simple operations -- more complicated operations need to be performed on an actual node. In this +// case we are using the pipeline to trigger downstream builds. stage('Build') { parallel ( java: { - def javaBuild = build job: 'beam_PreCommit_Java_Build', parameters: sha1Arg + ghprbArgs + def javaBuild = build job: 'beam_Java_Build', parameters: commitArg + ghprbArgs if(javaBuild.getResult() == Result.SUCCESS.toString()) { javaBuildNum = javaBuild.getNumber() } + }, + python_unit: { // Python doesn't have a build phase, so we include this here. + build job: 'beam_Python_UnitTest', parameters: commitArg + ghprbArgs } ) } +// This argument is provided to downstream jobs so they know from which build to pull artifacts. javaBuildArg = [string(name: 'buildNum', value: "${javaBuildNum}")] javaUnitPassed = false + stage('Unit Test / Code Health') { parallel ( java_unit: { - if(javaBuildNum != -1) { - def javaTest = build job: 'beam_PreCommit_Java_UnitTest', parameters: javaBuildArg + ghprbArgs + if(javaBuildNum != NO_BUILD) { + def javaTest = build job: 'beam_Java_UnitTest', parameters: javaBuildArg + ghprbArgs if(javaTest.getResult() == Result.SUCCESS.toString()) { javaUnitPassed = true } } }, java_codehealth: { - if(javaBuildNum != -1) { - build job: 'beam_PreCommit_Java_CodeHealth', parameters: javaBuildArg + ghprbArgs + if(javaBuildNum != NO_BUILD) { + build job: 'beam_Java_CodeHealth', parameters: javaBuildArg + ghprbArgs } - }, - python_unit: { - build job: 'beam_PreCommit_Python_UnitTest', parameters: sha1Arg + ghprbArgs } ) } @@ -66,7 +79,7 @@ stage('Integration Test') { parallel ( java_integration: { if(javaUnitPassed) { - build job: 'beam_PreCommit_Java_IntegrationTest', parameters: javaBuildArg + ghprbArgs + build job: 'beam_Java_IntegrationTest', parameters: javaBuildArg + ghprbArgs } } ) diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy index 45aae790ccd2..b30a64a916c3 100644 --- a/.test-infra/jenkins/common_job_properties.groovy +++ b/.test-infra/jenkins/common_job_properties.groovy @@ -157,7 +157,7 @@ class common_job_properties { } // Sets common config for Maven jobs. - static void setMavenConfig(context, mavenInstallation='Maven 3.3.3') { + static void setMavenConfig(context, String mavenInstallation='Maven 3.3.3') { context.mavenInstallation(mavenInstallation) context.mavenOpts('-Dorg.slf4j.simpleLogger.showDateTime=true') context.mavenOpts('-Dorg.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd\\\'T\\\'HH:mm:ss.SSS') @@ -260,7 +260,13 @@ class common_job_properties { } } - static def setPipelineJobProperties(def context, int tout, String descriptor) { + /** + * Sets properties for all jobs which are run by a pipeline top-level (maven) job. + * @param context The delegate from the top level of a MavenJob. + * @param jobTimeout How long (in minutes) to wait for the job to finish. + * @param descriptor A short string identifying the job, e.g. "Java Unit Test". + */ + static def setPipelineJobProperties(def context, int jobTimeout, String descriptor) { context.parameters { stringParam( 'ghprbGhRepository', @@ -288,7 +294,7 @@ class common_job_properties { wrappers { timeout { - absolute(tout) + absolute(jobTimeout) abortBuild() } credentialsBinding { @@ -311,9 +317,13 @@ class common_job_properties { // Set Maven parameters. setMavenConfig(context) - } + /** + * Sets job properties common to pipeline jobs which are responsible for being the root of a + * build tree. Downstream jobs should pull artifacts from these jobs. + * @param context The delegate from the top level of a MavenJob. + */ static def setPipelineBuildJobProperties(def context) { context.properties { githubProjectUrl('https://github.com/apache/beam/') @@ -321,7 +331,7 @@ class common_job_properties { context.parameters { stringParam( - 'sha1', + 'commit', 'master', 'Commit id or refname (e.g. origin/pr/9/head) you want to build.') } @@ -330,6 +340,12 @@ class common_job_properties { setBeamSCM(context, 'beam') } + /** + * Sets job parameters common to pipeline jobs which are downstream of a job which builds + * artifacts for them. + * @param context The delegate from the top level of a MavenJob. + * @param jobName The job from which to copy artifacts. + */ static def setPipelineDownstreamJobProperties(def context, String jobName) { context.parameters { stringParam( diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy b/.test-infra/jenkins/job_beam_Java_Build.groovy similarity index 84% rename from .test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy rename to .test-infra/jenkins/job_beam_Java_Build.groovy index 27526949693d..5d9be7083ae2 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_Build.groovy +++ b/.test-infra/jenkins/job_beam_Java_Build.groovy @@ -18,14 +18,14 @@ import common_job_properties -// This is the Java precommit which runs a maven install, and the current set -// of precommit tests. -mavenJob('beam_PreCommit_Java_Build') { - description('Part of the PreCommit Pipeline. Builds Java SDK and archives artifacts.') +// This is the Java Jenkins job which builds artifacts for downstream jobs to consume. +mavenJob('beam_Java_Build') { + description('Builds Java SDK and archives artifacts. Meant to be run as part of a pipeline.') - // Set properties common to PreCommit Pipeline subjobs. + // Set standard properties for a job which is part of a pipeline. common_job_properties.setPipelineJobProperties(delegate, 15, "Java Build") - // Set properties common to PreCommit initial jobs. + // Set standard properties for a pipeline job which needs to pull from GitHub instead of an + // upstream job. common_job_properties.setPipelineBuildJobProperties(delegate) configure { project -> @@ -59,7 +59,6 @@ mavenJob('beam_PreCommit_Java_Build') { "-pl '!sdks/python,!sdks/java/javadoc'", '-DskipTests', '-Dcheckstyle.skip', - '-Dmaven.javadoc.skip', // TODO: Javadoc still seems to run, figure out what also to skip. ] goals(args.join(' ')) diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy b/.test-infra/jenkins/job_beam_Java_CodeHealth.groovy similarity index 72% rename from .test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy rename to .test-infra/jenkins/job_beam_Java_CodeHealth.groovy index eebbe8a19d27..e50bcd720bd0 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_CodeHealth.groovy +++ b/.test-infra/jenkins/job_beam_Java_CodeHealth.groovy @@ -18,26 +18,18 @@ import common_job_properties -// This is the Java precommit which runs a maven install, and the current set -// of precommit tests. -mavenJob('beam_PreCommit_Java_CodeHealth') { - description('Part of the PreCommit Pipeline. Runs Java code health checks.') +// This is the Java Jenkins job which runs the Beam code health checks. +mavenJob('beam_Java_CodeHealth') { + description('Runs Java code health checks. Meant to be run as part of a pipeline.') + // Set standard properties for a job which is part of a pipeline. common_job_properties.setPipelineJobProperties(delegate, 15, "Java Code Health") - common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_PreCommit_Java_Build') + // This job runs downstream of the beam_Java_Build job and gets artifacts from that job. + common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_Java_Build') - // Construct Maven goals for this job. - profiles = [ - 'direct-runner', - 'dataflow-runner', - 'spark-runner', - 'flink-runner', - 'apex-runner' - ] args = [ '-B', '-e', - "-P${profiles.join(',')}", "-pl '!sdks/python'", 'checkstyle:check', 'findbugs:check', diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy b/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy similarity index 63% rename from .test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy rename to .test-infra/jenkins/job_beam_Java_IntegrationTest.groovy index a49d8baf5731..87470ef59e97 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_IntegrationTest.groovy +++ b/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy @@ -18,15 +18,16 @@ import common_job_properties -// This is the Java precommit which runs a maven install, and the current set -// of precommit tests. -mavenJob('beam_PreCommit_Java_IntegrationTest') { - description('Part of the PreCommit Pipeline. Runs Java Failsafe integration tests.') +// This is the Java Jenkins job which runs the set of precommit integration tests. +mavenJob('beam_Java_IntegrationTest') { + description('Runs Java Failsafe integration tests. Designed to be run as part of a pipeline.') + // Set standard properties for a job which is part of a pipeline. common_job_properties.setPipelineJobProperties(delegate, 25, "Java Integration Tests") - common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_PreCommit_Java_Build') + // Set standard properties for a job which pulls artifacts from an upstream job. + common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_Java_Build') - // Construct Maven goals for this job. + // Profiles to activate in order to ensure runners are available at test time. profiles = [ 'jenkins-precommit', 'direct-runner', @@ -35,6 +36,10 @@ mavenJob('beam_PreCommit_Java_IntegrationTest') { 'flink-runner', 'apex-runner' ] + // In the case of the precommit integration tests, we are currently only running the integration + // tests in the examples directory. By directly invoking failsafe with an execution name (which we + // do in order to avoid building artifacts again) we are required to enumerate each execution we + // want to run, something which is feasible in this case. examples_integration_executions = [ 'apex-runner-integration-tests', 'dataflow-runner-integration-tests', @@ -43,12 +48,14 @@ mavenJob('beam_PreCommit_Java_IntegrationTest') { 'flink-runner-integration-tests', 'spark-runner-integration-tests', ] + // Arguments to provide Maven. args = [ '-B', '-e', "-P${profiles.join(',')}", "-pl examples/java", ] + // This adds executions for each of the failsafe invocations listed above to the list of goals. examples_integration_executions.each({ value -> args.add("failsafe:integration-test@${value}") }) diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy b/.test-infra/jenkins/job_beam_Java_UnitTest.groovy similarity index 79% rename from .test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy rename to .test-infra/jenkins/job_beam_Java_UnitTest.groovy index fecc146a1479..5a3d04e407f0 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_UnitTest.groovy +++ b/.test-infra/jenkins/job_beam_Java_UnitTest.groovy @@ -18,13 +18,14 @@ import common_job_properties -// This is the Java precommit which runs a maven install, and the current set -// of precommit tests. -mavenJob('beam_PreCommit_Java_UnitTest') { - description('Part of the PreCommit Pipeline. Runs Java Surefire unit tests.') +// This is the Java Jenkins job which runs the current set of standard unit tests. +mavenJob('beam_Java_UnitTest') { + description('Runs Java Surefire unit tests. Designed to be run by a pipeline job.') + // Set standard properties for a job which is part of a pipeline. common_job_properties.setPipelineJobProperties(delegate, 20, "Java Unit Tests") - common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_PreCommit_Java_Build') + // Set standard properties for a job which pulls artifacts from an upstream job. + common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_Java_Build') // Construct Maven goals for this job. profiles = [ diff --git a/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy index 2295ec195d58..cb963fc602b5 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy @@ -29,13 +29,15 @@ pipelineJob('beam_PreCommit_Pipeline') { } parameters { + // Allow building at a specific commit. stringParam( - 'sha1', + 'commit', 'master', 'Commit id or refname (e.g. origin/pr/9/head) you want to build.') } wrappers { + // Set a timeout appropriate for the precommit tests. timeout { absolute(120) abortBuild() diff --git a/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy b/.test-infra/jenkins/job_beam_Python_UnitTest.groovy similarity index 72% rename from .test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy rename to .test-infra/jenkins/job_beam_Python_UnitTest.groovy index 6e263ee1903b..299157a6cc1a 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Python_UnitTest.groovy +++ b/.test-infra/jenkins/job_beam_Python_UnitTest.groovy @@ -18,12 +18,15 @@ import common_job_properties -// This is the Python precommit which runs a maven install, and the current set -// of precommit tests. -mavenJob('beam_PreCommit_Python_UnitTest') { - description('Part of the PreCommit Pipeline. Runs Python unit tests.') +// This is the Python Jenkins job which runs a maven install, and the current set of precommit +// tests. +mavenJob('beam_Python_UnitTest') { + description('Runs Python unit tests on a specific commit. Designed to be run by a pipeline job.') + // Set standard properties for a job which is part of a pipeline. common_job_properties.setPipelineJobProperties(delegate, 25, "Python Unit Tests") + // Set standard properties for a pipeline job which needs to pull from GitHub instead of an + // upstream job. common_job_properties.setPipelineBuildJobProperties(delegate) // Construct Maven goals for this job. From a99940468fae1d23b01053b029e32c91e4901a20 Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Mon, 18 Sep 2017 16:12:19 -0700 Subject: [PATCH 16/19] Address PR comments. Signed-off-by: Jason Kuster --- .test-infra/jenkins/PreCommit_Pipeline.groovy | 1 + .../jenkins/common_job_properties.groovy | 24 ++++++++++--------- .../jenkins/job_beam_Java_Build.groovy | 3 ++- .../job_beam_PreCommit_Pipeline.groovy | 6 ++--- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.test-infra/jenkins/PreCommit_Pipeline.groovy b/.test-infra/jenkins/PreCommit_Pipeline.groovy index e2156323d5ba..9a4b86a852eb 100644 --- a/.test-infra/jenkins/PreCommit_Pipeline.groovy +++ b/.test-infra/jenkins/PreCommit_Pipeline.groovy @@ -24,6 +24,7 @@ int NO_BUILD = -1 // These are args for the GitHub Pull Request Builder (ghprb) Plugin. Providing these arguments is // necessary due to a bug in the ghprb plugin where environment variables are not correctly passed // to jobs downstream of a Pipeline job. +// Tracked by https://github.com/jenkinsci/ghprb-plugin/issues/572. List ghprbArgs = [ string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"), string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"), diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy index b30a64a916c3..062803356c67 100644 --- a/.test-infra/jenkins/common_job_properties.groovy +++ b/.test-infra/jenkins/common_job_properties.groovy @@ -22,11 +22,14 @@ // http://groovy-lang.org/style-guide.html class common_job_properties { - static void setBeamSCM(def context, String repositoryName) { + static void setSCM(def context, String repositoryName) { context.scm { git { remote { + // Double quotes here mean ${repositoryName} is interpolated. github("apache/${repositoryName}") + // Single quotes here mean that ${ghprbPullId} is not interpolated and instead passed + // through to Jenkins where it refers to the environment variable. refspec('+refs/heads/*:refs/remotes/origin/* ' + '+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*') } @@ -87,7 +90,7 @@ class common_job_properties { } // Source code management. - setBeamSCM(context, repositoryName) + setSCM(context, repositoryName) context.parameters { // This is a recommended setup if you want to run the job manually. The @@ -218,7 +221,7 @@ class common_job_properties { } } - static def mapToArgList(LinkedHashMap inputArgs) { + static def mapToArgString(LinkedHashMap inputArgs) { List argList = [] inputArgs.each({ // FYI: Replacement only works with double quotes. @@ -240,7 +243,7 @@ class common_job_properties { ] // Note: in case of key collision, keys present in ArgMap win. LinkedHashMap joinedArgs = standardArgs.plus(argMap) - return mapToArgList(joinedArgs) + return mapToArgString(joinedArgs) } // Adds the standard performance test job steps. @@ -284,15 +287,15 @@ class common_job_properties { } // Set JDK version. - jdk('JDK 1.8 (latest)') + context.jdk('JDK 1.8 (latest)') // Restrict this project to run only on Jenkins executors as specified - label('beam') + context.label('beam') // Execute concurrent builds if necessary. - concurrentBuild() + context.concurrentBuild() - wrappers { + context.wrappers { timeout { absolute(jobTimeout) abortBuild() @@ -337,12 +340,11 @@ class common_job_properties { } // Source code management. - setBeamSCM(context, 'beam') + setSCM(context, 'beam') } /** - * Sets job parameters common to pipeline jobs which are downstream of a job which builds - * artifacts for them. + * Sets common job parameters for jobs which consume artifacts built for them by an upstream job. * @param context The delegate from the top level of a MavenJob. * @param jobName The job from which to copy artifacts. */ diff --git a/.test-infra/jenkins/job_beam_Java_Build.groovy b/.test-infra/jenkins/job_beam_Java_Build.groovy index 5d9be7083ae2..7c6f4cf52c07 100644 --- a/.test-infra/jenkins/job_beam_Java_Build.groovy +++ b/.test-infra/jenkins/job_beam_Java_Build.groovy @@ -20,7 +20,7 @@ import common_job_properties // This is the Java Jenkins job which builds artifacts for downstream jobs to consume. mavenJob('beam_Java_Build') { - description('Builds Java SDK and archives artifacts. Meant to be run as part of a pipeline.') + description('Builds Beam Java SDK and archives artifacts. Meant to be run as part of a pipeline.') // Set standard properties for a job which is part of a pipeline. common_job_properties.setPipelineJobProperties(delegate, 15, "Java Build") @@ -34,6 +34,7 @@ mavenJob('beam_Java_Build') { 'string' "beam_*" } // The Build Discarder also doesn't support the job DSL in the right way so we have to configure it manually. + // -1 indicates that a property is "infinite". project / 'properties' / 'jenkins.model.BuildDiscarderProperty' / 'strategy'(class:'hudson.tasks.LogRotator') { 'daysToKeep'(-1) 'numToKeep'(-1) diff --git a/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy index cb963fc602b5..832712a47b2d 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy @@ -19,8 +19,8 @@ import common_job_properties -// This is the Java precommit which runs a maven install, and the current set -// of precommit tests. +// This job owns the overall execution of the precommit pipeline. The actual pipeline code is in +// Precommit_Pipeline.groovy. pipelineJob('beam_PreCommit_Pipeline') { description('PreCommit Pipeline Job. Owns overall lifecycle of PreCommit tests.') @@ -74,7 +74,7 @@ pipelineJob('beam_PreCommit_Pipeline') { definition { cpsScm { // Source code management. - common_job_properties.setBeamSCM(delegate, 'beam') + common_job_properties.setSCM(delegate, 'beam') scriptPath('.test-infra/jenkins/PreCommit_Pipeline.groovy') } } From 8caa4a9a4e33e41c7f8f82f857c366121bb395e8 Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Thu, 21 Sep 2017 16:00:44 -0700 Subject: [PATCH 17/19] Comment about integration tests. Signed-off-by: Jason Kuster --- .test-infra/jenkins/PreCommit_Pipeline.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.test-infra/jenkins/PreCommit_Pipeline.groovy b/.test-infra/jenkins/PreCommit_Pipeline.groovy index 9a4b86a852eb..20eaa56b76bb 100644 --- a/.test-infra/jenkins/PreCommit_Pipeline.groovy +++ b/.test-infra/jenkins/PreCommit_Pipeline.groovy @@ -78,6 +78,8 @@ stage('Unit Test / Code Health') { stage('Integration Test') { parallel ( + // Not gated on codehealth because codehealth shouldn't affect whether tests provide useful + // signal. java_integration: { if(javaUnitPassed) { build job: 'beam_Java_IntegrationTest', parameters: javaBuildArg + ghprbArgs From 74c7a3053d7038fb956511dd4c6ecb238c59af60 Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Thu, 21 Sep 2017 16:16:41 -0700 Subject: [PATCH 18/19] Context needs to be triggered on delegate. Signed-off-by: Jason Kuster --- .test-infra/jenkins/common_job_properties.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy index 062803356c67..43ed7cfc5391 100644 --- a/.test-infra/jenkins/common_job_properties.groovy +++ b/.test-infra/jenkins/common_job_properties.groovy @@ -304,7 +304,7 @@ class common_job_properties { string("COVERALLS_REPO_TOKEN", "beam-coveralls-token") } downstreamCommitStatus { - context("Jenkins: ${descriptor}") + delegate.context("Jenkins: ${descriptor}") triggeredStatus("${descriptor} Pending") startedStatus("Running ${descriptor}") statusUrl() From 58ba187881e8852a3bb4979d882d2112d72178dc Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Thu, 28 Sep 2017 10:29:52 -0700 Subject: [PATCH 19/19] Bump timeouts, make commit arg standard, trigger only on trigger phrase. Signed-off-by: Jason Kuster --- .test-infra/jenkins/PreCommit_Pipeline.groovy | 2 +- .test-infra/jenkins/common_job_properties.groovy | 2 +- .test-infra/jenkins/job_beam_Java_Build.groovy | 2 +- .test-infra/jenkins/job_beam_Java_CodeHealth.groovy | 2 +- .test-infra/jenkins/job_beam_Java_IntegrationTest.groovy | 2 +- .test-infra/jenkins/job_beam_Java_UnitTest.groovy | 2 +- .test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy | 3 +++ .test-infra/jenkins/job_beam_Python_UnitTest.groovy | 2 +- 8 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.test-infra/jenkins/PreCommit_Pipeline.groovy b/.test-infra/jenkins/PreCommit_Pipeline.groovy index 20eaa56b76bb..9abf39d46ece 100644 --- a/.test-infra/jenkins/PreCommit_Pipeline.groovy +++ b/.test-infra/jenkins/PreCommit_Pipeline.groovy @@ -32,7 +32,7 @@ List ghprbArgs = [ ] // This argument is the commit at which to build. -List commitArg = [string(name: 'commit', value: "origin/pr/${ghprbPullId}/head")] +List commitArg = [string(name: 'sha1', value: "origin/pr/${ghprbPullId}/head")] int javaBuildNum = NO_BUILD diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy index 43ed7cfc5391..87763a2550ba 100644 --- a/.test-infra/jenkins/common_job_properties.groovy +++ b/.test-infra/jenkins/common_job_properties.groovy @@ -334,7 +334,7 @@ class common_job_properties { context.parameters { stringParam( - 'commit', + 'sha1', 'master', 'Commit id or refname (e.g. origin/pr/9/head) you want to build.') } diff --git a/.test-infra/jenkins/job_beam_Java_Build.groovy b/.test-infra/jenkins/job_beam_Java_Build.groovy index 7c6f4cf52c07..704d7eb43bc2 100644 --- a/.test-infra/jenkins/job_beam_Java_Build.groovy +++ b/.test-infra/jenkins/job_beam_Java_Build.groovy @@ -23,7 +23,7 @@ mavenJob('beam_Java_Build') { description('Builds Beam Java SDK and archives artifacts. Meant to be run as part of a pipeline.') // Set standard properties for a job which is part of a pipeline. - common_job_properties.setPipelineJobProperties(delegate, 15, "Java Build") + common_job_properties.setPipelineJobProperties(delegate, 30, "Java Build") // Set standard properties for a pipeline job which needs to pull from GitHub instead of an // upstream job. common_job_properties.setPipelineBuildJobProperties(delegate) diff --git a/.test-infra/jenkins/job_beam_Java_CodeHealth.groovy b/.test-infra/jenkins/job_beam_Java_CodeHealth.groovy index e50bcd720bd0..41a45369ebd8 100644 --- a/.test-infra/jenkins/job_beam_Java_CodeHealth.groovy +++ b/.test-infra/jenkins/job_beam_Java_CodeHealth.groovy @@ -23,7 +23,7 @@ mavenJob('beam_Java_CodeHealth') { description('Runs Java code health checks. Meant to be run as part of a pipeline.') // Set standard properties for a job which is part of a pipeline. - common_job_properties.setPipelineJobProperties(delegate, 15, "Java Code Health") + common_job_properties.setPipelineJobProperties(delegate, 30, "Java Code Health") // This job runs downstream of the beam_Java_Build job and gets artifacts from that job. common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_Java_Build') diff --git a/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy b/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy index 87470ef59e97..56daf731d84c 100644 --- a/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy +++ b/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy @@ -23,7 +23,7 @@ mavenJob('beam_Java_IntegrationTest') { description('Runs Java Failsafe integration tests. Designed to be run as part of a pipeline.') // Set standard properties for a job which is part of a pipeline. - common_job_properties.setPipelineJobProperties(delegate, 25, "Java Integration Tests") + common_job_properties.setPipelineJobProperties(delegate, 30, "Java Integration Tests") // Set standard properties for a job which pulls artifacts from an upstream job. common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_Java_Build') diff --git a/.test-infra/jenkins/job_beam_Java_UnitTest.groovy b/.test-infra/jenkins/job_beam_Java_UnitTest.groovy index 5a3d04e407f0..8f03900a40ad 100644 --- a/.test-infra/jenkins/job_beam_Java_UnitTest.groovy +++ b/.test-infra/jenkins/job_beam_Java_UnitTest.groovy @@ -23,7 +23,7 @@ mavenJob('beam_Java_UnitTest') { description('Runs Java Surefire unit tests. Designed to be run by a pipeline job.') // Set standard properties for a job which is part of a pipeline. - common_job_properties.setPipelineJobProperties(delegate, 20, "Java Unit Tests") + common_job_properties.setPipelineJobProperties(delegate, 30, "Java Unit Tests") // Set standard properties for a job which pulls artifacts from an upstream job. common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_Java_Build') diff --git a/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy index 832712a47b2d..2e1ea55bfe94 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy @@ -57,6 +57,9 @@ pipelineJob('beam_PreCommit_Pipeline') { orgWhitelist(['apache']) allowMembersOfWhitelistedOrgsAsAdmin() permitAll() + // Remove once Pipeline Build is default. + triggerPhrase('Run PreCommit Pipeline') + onlyTriggerPhrase() displayBuildErrorsOnDownstreamBuilds() extensions { commitStatus { diff --git a/.test-infra/jenkins/job_beam_Python_UnitTest.groovy b/.test-infra/jenkins/job_beam_Python_UnitTest.groovy index 299157a6cc1a..89701d4474b4 100644 --- a/.test-infra/jenkins/job_beam_Python_UnitTest.groovy +++ b/.test-infra/jenkins/job_beam_Python_UnitTest.groovy @@ -24,7 +24,7 @@ mavenJob('beam_Python_UnitTest') { description('Runs Python unit tests on a specific commit. Designed to be run by a pipeline job.') // Set standard properties for a job which is part of a pipeline. - common_job_properties.setPipelineJobProperties(delegate, 25, "Python Unit Tests") + common_job_properties.setPipelineJobProperties(delegate, 35, "Python Unit Tests") // Set standard properties for a pipeline job which needs to pull from GitHub instead of an // upstream job. common_job_properties.setPipelineBuildJobProperties(delegate)