From beee6a19983a3c7ffa0a973a14159ac83f5dc727 Mon Sep 17 00:00:00 2001 From: Thomas Groh Date: Fri, 20 Oct 2017 16:27:18 -0700 Subject: [PATCH 1/2] Split Precommit Integration Tests into a separate Build Integration tests consume a significant amount of time, and are easily separable from the remaining tests. --- ..._PreCommit_Java_Example_Integration.groovy | 62 +++++++++++++++++++ ...ob_beam_PreCommit_Java_MavenInstall.groovy | 4 +- 2 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 .test-infra/jenkins/job_beam_PreCommit_Java_Example_Integration.groovy diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_Example_Integration.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_Example_Integration.groovy new file mode 100644 index 000000000000..f23c64efe21e --- /dev/null +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_Example_Integration.groovy @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import common_job_properties + +// This is the Java Precommit which executes integration tests on each runner. +// This Precommit only compiles the SDK and tests, and executes the Integration +// tests within the examples/java module. +mavenJob('beam_PreCommit_Java_Integration_Tests') { + description('Runs Java Examples Integration Tests of the current GitHub Pull Request.') + + previousNames('beam_PreCommit_Java_MavenInstall') + + // Execute concurrent builds if necessary. + concurrentBuild() + + // Set common parameters. + common_job_properties.setTopLevelMainJobProperties( + delegate, + 'master', + 60) + + // Set Maven parameters. + common_job_properties.setMavenConfig(delegate) + + // Sets that this is a PreCommit job which runs the integration-tests. + common_job_properties.setPreCommit(delegate, 'mvn failsafe:integration-test -pl examples/java -am', 'Run Java Integration PreCommit') + + // Maven goals for this job: The Java Examples and their dependencies + goals('''\ + --batch-mode \ + --errors \ + --activate-profiles jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner \ + --projects examples/java \ + --also-make \ + -D checkstyle.skip \ + -D findbugs.skip \ + -D pullRequest=$ghprbPullId \ + help:effective-settings \ + clean \ + compile \ + test-compile \ + failsafe:integration-test \ + failsafe:verify \ + ''') +} + diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy index 872146858859..d6a04dbd35f4 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy @@ -19,7 +19,7 @@ import common_job_properties // This is the Java precommit which runs a maven install, and the current set -// of precommit tests. +// of precommit tests. This precommit does not execute any integration tests. mavenJob('beam_PreCommit_Java_MavenInstall') { description('Runs an install of the current GitHub Pull Request.') @@ -44,7 +44,7 @@ mavenJob('beam_PreCommit_Java_MavenInstall') { goals('''\ --batch-mode \ --errors \ - --activate-profiles release,jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner \ + --activate-profiles release,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner \ --projects sdks/java/core,runners/direct-java \ --also-make \ --also-make-dependents \ From 59900e5bcb90f0c807c9e55c67c27719afc5dfdb Mon Sep 17 00:00:00 2001 From: Thomas Groh Date: Fri, 20 Oct 2017 17:39:13 -0700 Subject: [PATCH 2/2] fixup! Split Precommit Integration Tests into a separate Build --- ..._PreCommit_Java_Example_Integration.groovy | 31 +++++++++---------- ...ob_beam_PreCommit_Java_MavenInstall.groovy | 27 ++++++++-------- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_Example_Integration.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_Example_Integration.groovy index f23c64efe21e..4d981648044b 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_Example_Integration.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_Example_Integration.groovy @@ -42,21 +42,20 @@ mavenJob('beam_PreCommit_Java_Integration_Tests') { common_job_properties.setPreCommit(delegate, 'mvn failsafe:integration-test -pl examples/java -am', 'Run Java Integration PreCommit') // Maven goals for this job: The Java Examples and their dependencies - goals('''\ - --batch-mode \ - --errors \ - --activate-profiles jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner \ - --projects examples/java \ - --also-make \ - -D checkstyle.skip \ - -D findbugs.skip \ - -D pullRequest=$ghprbPullId \ - help:effective-settings \ - clean \ - compile \ - test-compile \ - failsafe:integration-test \ - failsafe:verify \ - ''') + goals([ + '--batch-mode', + '--errors', + '--activate-profiles jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner', + '--projects examples/java', + '--also-make', + '-D checkstyle.skip', + '-D findbugs.skip', + '-D pullRequest=$ghprbPullId', + 'help:effective-settings', + 'clean', + 'compile', + 'test-compile', + 'failsafe:integration-test', + 'failsafe:verify'].join(' ')) } diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy index d6a04dbd35f4..83fc5d6924ea 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy @@ -41,18 +41,17 @@ mavenJob('beam_PreCommit_Java_MavenInstall') { common_job_properties.setPreCommit(delegate, 'mvn clean install -pl sdks/java/core,runners/direct-java -am -amd', 'Run Java PreCommit') // Maven goals for this job: The Java SDK, its dependencies, and things that depend on it. - goals('''\ - --batch-mode \ - --errors \ - --activate-profiles release,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner \ - --projects sdks/java/core,runners/direct-java \ - --also-make \ - --also-make-dependents \ - -D repoToken=$COVERALLS_REPO_TOKEN \ - -D pullRequest=$ghprbPullId \ - help:effective-settings \ - clean \ - install \ - coveralls:report \ - ''') + goals([ + '--batch-mode', + '--errors', + '--activate-profiles release', + '--projects sdks/java/core,runners/direct-java', + '--also-make', + '--also-make-dependents', + '-D repoToken=$COVERALLS_REPO_TOKEN', + '-D pullRequest=$ghprbPullId', + 'help:effective-settings', + 'clean', + 'install', + 'coveralls:report'].join(' ')) }