diff --git a/.jenkins/common_job_properties.groovy b/.jenkins/common_job_properties.groovy index f1f2b7f590d19..eee1cdae7e255 100644 --- a/.jenkins/common_job_properties.groovy +++ b/.jenkins/common_job_properties.groovy @@ -20,14 +20,27 @@ // common properties that are shared among all Jenkins projects. class common_job_properties { - // Sets common top-level job properties. - static def setTopLevelJobProperties(def context, - def default_branch = 'master', - def default_timeout = 100) { + // Sets common top-level job properties for website repository jobs. + static def setTopLevelWebsiteJobProperties(def context) { + setTopLevelJobProperties(context, 'beam-site', 'asf-site', 30) + } + + // Sets common top-level job properties for main repository jobs. + static def setTopLevelMainJobProperties(def context, + def default_branch = 'master') { + setTopLevelJobProperties(context, 'beam', default_branch, 100) + } + + // Sets common top-level job properties. Should be accessed through one of the + // above methods to protect jobs from internal details of param defaults. + private static def setTopLevelJobProperties(def context, + def repository_name, + def default_branch, + def default_timeout) { // GitHub project. context.properties { - githubProjectUrl('https://github.com/apache/beam/') + githubProjectUrl('https://github.com/apache/' + repository_name + '/') } // Set JDK version. @@ -46,7 +59,7 @@ class common_job_properties { context.scm { git { remote { - url('https://github.com/apache/beam.git') + url('https://github.com/apache/' + repository_name + '.git') refspec('+refs/heads/*:refs/remotes/origin/* ' + '+refs/pull/*:refs/remotes/origin/pr/*') } @@ -83,10 +96,11 @@ class common_job_properties { } } - // Sets the pull request build trigger. - static def setPullRequestBuildTrigger(def context, - def commitStatusContext, - def successComment = '--none--') { + // Sets the pull request build trigger. Accessed through precommit methods + // below to insulate callers from internal parameter defaults. + private static def setPullRequestBuildTrigger(def context, + def commitStatusContext, + def successComment = '--none--') { context.triggers { githubPullRequest { admins(['asfbot']) @@ -150,9 +164,11 @@ class common_job_properties { } // Sets common config for PreCommit jobs. - static def setPreCommit(def context, comment) { + static def setPreCommit(def context, + def commitStatusName, + def successComment = '--none--') { // Set pull request build trigger. - setPullRequestBuildTrigger(context, comment) + setPullRequestBuildTrigger(context, commitStatusName, successComment) } // Sets common config for PostCommit jobs. diff --git a/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy b/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy index 31257e222e8db..a288a84482113 100644 --- a/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy +++ b/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy @@ -29,7 +29,7 @@ mavenJob('beam_PostCommit_Java_MavenInstall') { concurrentBuild() // Set common parameters. - common_job_properties.setTopLevelJobProperties(delegate) + common_job_properties.setTopLevelMainJobProperties(delegate) // Set maven parameters. common_job_properties.setMavenConfig(delegate) diff --git a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Apex.groovy b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Apex.groovy index 232c94e1d5c45..f6575963b8b92 100644 --- a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Apex.groovy +++ b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Apex.groovy @@ -23,7 +23,7 @@ mavenJob('beam_PostCommit_Java_RunnableOnService_Apex') { description('Runs the RunnableOnService suite on the Apex runner.') // Set common parameters. - common_job_properties.setTopLevelJobProperties(delegate) + common_job_properties.setTopLevelMainJobProperties(delegate) // Set maven parameters. common_job_properties.setMavenConfig(delegate) diff --git a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Dataflow.groovy b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Dataflow.groovy index 44136ade8f20c..f849c835c168c 100644 --- a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Dataflow.groovy +++ b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Dataflow.groovy @@ -26,7 +26,7 @@ mavenJob('beam_PostCommit_Java_RunnableOnService_Dataflow') { previousNames('beam_PostCommit_RunnableOnService_GoogleCloudDataflow') // Set common parameters. - common_job_properties.setTopLevelJobProperties(delegate) + common_job_properties.setTopLevelMainJobProperties(delegate) // Set maven parameters. common_job_properties.setMavenConfig(delegate) diff --git a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Flink.groovy b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Flink.groovy index 5a5863f31e45c..c0caa51b84bec 100644 --- a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Flink.groovy +++ b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Flink.groovy @@ -25,7 +25,7 @@ mavenJob('beam_PostCommit_Java_RunnableOnService_Flink') { previousNames('beam_PostCommit_RunnableOnService_FlinkLocal') // Set common parameters. - common_job_properties.setTopLevelJobProperties(delegate) + common_job_properties.setTopLevelMainJobProperties(delegate) // Set maven parameters. common_job_properties.setMavenConfig(delegate) diff --git a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Gearpump.groovy b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Gearpump.groovy index f07bbc6f7a096..1e35017308f38 100644 --- a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Gearpump.groovy +++ b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Gearpump.groovy @@ -26,7 +26,9 @@ mavenJob('beam_PostCommit_Java_RunnableOnService_Gearpump') { previousNames('beam_PostCommit_RunnableOnService_GearpumpLocal') // Set common parameters. - common_job_properties.setTopLevelJobProperties(delegate, 'gearpump-runner') + common_job_properties.setTopLevelMainJobProperties( + delegate, + 'gearpump-runner') // Set maven parameters. common_job_properties.setMavenConfig(delegate) diff --git a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Spark.groovy b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Spark.groovy index 875c6e6488b0b..97f6a70eadeda 100644 --- a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Spark.groovy +++ b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Spark.groovy @@ -25,7 +25,7 @@ mavenJob('beam_PostCommit_Java_RunnableOnService_Spark') { previousNames('beam_PostCommit_RunnableOnService_SparkLocal') // Set common parameters. - common_job_properties.setTopLevelJobProperties(delegate) + common_job_properties.setTopLevelMainJobProperties(delegate) // Set maven parameters. common_job_properties.setMavenConfig(delegate) diff --git a/.jenkins/job_beam_PostCommit_Python_Verify.groovy b/.jenkins/job_beam_PostCommit_Python_Verify.groovy index 7f40a151bcc9c..d92958c93079f 100644 --- a/.jenkins/job_beam_PostCommit_Python_Verify.groovy +++ b/.jenkins/job_beam_PostCommit_Python_Verify.groovy @@ -25,7 +25,7 @@ job('beam_PostCommit_Python_Verify') { previousNames('beam_PostCommit_PythonVerify') // Set common parameters. - common_job_properties.setTopLevelJobProperties(delegate) + common_job_properties.setTopLevelMainJobProperties(delegate) // Sets that this is a PostCommit job. common_job_properties.setPostCommit(delegate, '0 3-22/6 * * *') diff --git a/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy b/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy index 153df14671a74..ff2e476b6af12 100644 --- a/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy +++ b/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy @@ -29,7 +29,7 @@ mavenJob('beam_PreCommit_Java_MavenInstall') { concurrentBuild() // Set common parameters. - common_job_properties.setTopLevelJobProperties(delegate) + common_job_properties.setTopLevelMainJobProperties(delegate) // Set Maven parameters. common_job_properties.setMavenConfig(delegate) diff --git a/.jenkins/job_beam_PreCommit_Website_Stage.groovy b/.jenkins/job_beam_PreCommit_Website_Stage.groovy new file mode 100644 index 0000000000000..b63d40af5dc98 --- /dev/null +++ b/.jenkins/job_beam_PreCommit_Website_Stage.groovy @@ -0,0 +1,62 @@ +import common_job_properties + +// Defines a job. +job('beam_PreCommit_Website_Stage') { + description('Stages the pull requests proposed for the Apache Beam ' + + 'website to a temporary location to ease reviews.') + + // Set common parameters. + common_job_properties.setTopLevelWebsiteJobProperties(delegate) + + // Set pull request build trigger. + common_job_properties.setPreCommit( + delegate, + 'Jenkins: automatic staging of pull requests', + '\nJenkins built the site at commit id ${ghprbActualCommit} with ' + + 'Jekyll and staged it [here](http://apache-beam-website-pull-' + + 'requests.storage.googleapis.com/${ghprbPullId}/index.html). ' + + 'Happy reviewing.\n\nNote that any previous site has been deleted. ' + + 'This staged site will be automatically deleted after its TTL ' + + 'expires. Push any commit to the pull request branch or re-trigger ' + + 'the build to get it staged again.') + + steps { + // Run the following shell script as a build step. + shell ''' + # Install RVM. + gpg --keyserver hkp://keys.gnupg.net --recv-keys \\ + 409B6B1796C275462A1703113804BB82D39DC0E3 + \\curl -sSL https://get.rvm.io | bash + source /home/jenkins/.rvm/scripts/rvm + + # Install Ruby. + RUBY_VERSION_NUM=2.3.0 + rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only + + # Install Bundler gem + PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH + GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH + gem install bundler --user-install + + # Install all needed gems. + bundle install --path ~/.gem/ + + # Remove current site if it exists. + GCS_PATH="gs://apache-beam-website-pull-requests/${ghprbPullId}/" + gsutil -m rm -r -f ${GCS_PATH} || true + + # Build the new site with the baseurl specified. + rm -fr ./content/ + bundle exec jekyll build --baseurl=/${ghprbPullId} + + # Install BeautifulSoup HTML Parser for python. + pip install --user beautifulsoup4 + + # Fix links on staged website. + python .jenkins/append_index_html_to_internal_links.py + + # Upload the new site. + gsutil -m cp -R ./content/* ${GCS_PATH} + '''.stripIndent().trim() + } +} diff --git a/.jenkins/job_beam_PreCommit_Website_Test.groovy b/.jenkins/job_beam_PreCommit_Website_Test.groovy new file mode 100644 index 0000000000000..ada089f77f847 --- /dev/null +++ b/.jenkins/job_beam_PreCommit_Website_Test.groovy @@ -0,0 +1,47 @@ +import common_job_properties + +// Defines a job. +job('beam_PreCommit_Website_Test') { + description('Runs tests on the pull requests proposed for the Apache Beam ' + + 'website.') + + // Set common parameters. + common_job_properties.setTopLevelWebsiteJobProperties(delegate) + + // Execute concurrent builds. Multiple builds of this project may be executed + // in parallel. This is safe because this build does not require exclusive + // access to any shared resources. + concurrentBuild() + + // Set pull request build trigger. + common_job_properties.setPreCommit( + delegate, + 'Jenkins: test website (dead links, etc.)') + + steps { + // Run the following shell script as a build step. + shell ''' + # Install RVM. + gpg --keyserver hkp://keys.gnupg.net --recv-keys \\ + 409B6B1796C275462A1703113804BB82D39DC0E3 + \\curl -sSL https://get.rvm.io | bash + source /home/jenkins/.rvm/scripts/rvm + + # Install Ruby. + RUBY_VERSION_NUM=2.3.0 + rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only + + # Install Bundler gem + PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH + GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH + gem install bundler --user-install + + # Install all needed gems. + bundle install --path ~/.gem/ + + # Build the new site and test it. + rm -fr ./content/ + bundle exec rake test + '''.stripIndent().trim() + } +} diff --git a/.jenkins/job_beam_Release_NightlySnapshot.groovy b/.jenkins/job_beam_Release_NightlySnapshot.groovy index a19f8ff0ced81..f2c3ff0740d53 100644 --- a/.jenkins/job_beam_Release_NightlySnapshot.groovy +++ b/.jenkins/job_beam_Release_NightlySnapshot.groovy @@ -28,7 +28,7 @@ mavenJob('beam_Release_NightlySnapshot') { concurrentBuild() // Set common parameters. - common_job_properties.setTopLevelJobProperties(delegate) + common_job_properties.setTopLevelMainJobProperties(delegate) // Set maven paramaters. common_job_properties.setMavenConfig(delegate) diff --git a/.jenkins/job_seed.groovy b/.jenkins/job_seed.groovy index fcdd564d97499..b830f884160c0 100644 --- a/.jenkins/job_seed.groovy +++ b/.jenkins/job_seed.groovy @@ -19,13 +19,14 @@ import common_job_properties // Defines the seed job, which creates or updates all other Jenkins projects. -job('beam_SeedJob_Main') { - description('Automatically configures all Apache Beam main repo Jenkins ' + - 'projects based on Jenkins DSL groovy files checked into the ' + - 'code repository.') +job('beam_SeedJob') { + description('Automatically configures all Apache Beam Jenkins projects based' + + ' on Jenkins DSL groovy files checked into the code repository.') + + previousNames('beam_SeedJob_Main') // Set common parameters. - common_job_properties.setTopLevelJobProperties(delegate) + common_job_properties.setTopLevelMainJobProperties(delegate) // This is a post-commit job that runs once per day, not for every push. common_job_properties.setPostCommit(