[BEAM-1376] Move website jobs to main Beam repo to reduce duplication.

.jenkins/common_job_properties.groovy

@@ -20,14 +20,27 @@
 // common properties that are shared among all Jenkins projects.
 class common_job_properties {
 
-  // Sets common top-level job properties.
-  static def setTopLevelJobProperties(def context,
-                                      def default_branch = 'master',
-                                      def default_timeout = 100) {
+  // Sets common top-level job properties for website repository jobs.
+  static def setTopLevelWebsiteJobProperties(def context) {
+    setTopLevelJobProperties(context, 'beam-site', 'asf-site', 30)
+  }
+
+  // Sets common top-level job properties for main repository jobs.
+  static def setTopLevelMainJobProperties(def context,
+                                          def default_branch = 'master') {
+    setTopLevelJobProperties(context, 'beam', default_branch, 100)
+  }
+
+  // Sets common top-level job properties. Should be accessed through one of the
+  // above methods to protect jobs from internal details of param defaults.
+  private static def setTopLevelJobProperties(def context,
+                                              def repository_name,
+                                              def default_branch,
+                                              def default_timeout) {
 
     // GitHub project.
     context.properties {
-      githubProjectUrl('https://github.com/apache/beam/')
+      githubProjectUrl('https://github.com/apache/' + repository_name + '/')
     }
 
     // Set JDK version.
@@ -46,7 +59,7 @@ class common_job_properties {
     context.scm {
       git {
         remote {
-          url('https://github.com/apache/beam.git')
+          url('https://github.com/apache/' + repository_name + '.git')
           refspec('+refs/heads/*:refs/remotes/origin/* ' +
                   '+refs/pull/*:refs/remotes/origin/pr/*')
         }
@@ -83,10 +96,11 @@ class common_job_properties {
     }
   }
 
-  // Sets the pull request build trigger.
-  static def setPullRequestBuildTrigger(def context,
-                                        def commitStatusContext,
-                                        def successComment = '--none--') {
+  // Sets the pull request build trigger. Accessed through precommit methods
+  // below to insulate callers from internal parameter defaults.
+  private static def setPullRequestBuildTrigger(def context,
+                                                def commitStatusContext,
+                                                def successComment = '--none--') {
     context.triggers {
       githubPullRequest {
         admins(['asfbot'])
@@ -150,9 +164,11 @@ class common_job_properties {
   }
 
   // Sets common config for PreCommit jobs.
-  static def setPreCommit(def context, comment) {
+  static def setPreCommit(def context,
+                          def commitStatusName,
+                          def successComment = '--none--') {
     // Set pull request build trigger.
-    setPullRequestBuildTrigger(context, comment)
+    setPullRequestBuildTrigger(context, commitStatusName, successComment)
   }
 
   // Sets common config for PostCommit jobs.

.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy

@@ -29,7 +29,7 @@ mavenJob('beam_PostCommit_Java_MavenInstall') {
   concurrentBuild()
 
   // Set common parameters.
-  common_job_properties.setTopLevelJobProperties(delegate)
+  common_job_properties.setTopLevelMainJobProperties(delegate)
 
   // Set maven parameters.
   common_job_properties.setMavenConfig(delegate)

.jenkins/job_beam_PostCommit_Java_RunnableOnService_Apex.groovy

@@ -23,7 +23,7 @@ mavenJob('beam_PostCommit_Java_RunnableOnService_Apex') {
   description('Runs the RunnableOnService suite on the Apex runner.')
 
   // Set common parameters.
-  common_job_properties.setTopLevelJobProperties(delegate)
+  common_job_properties.setTopLevelMainJobProperties(delegate)
 
   // Set maven parameters.
   common_job_properties.setMavenConfig(delegate)

.jenkins/job_beam_PostCommit_Java_RunnableOnService_Dataflow.groovy

@@ -26,7 +26,7 @@ mavenJob('beam_PostCommit_Java_RunnableOnService_Dataflow') {
   previousNames('beam_PostCommit_RunnableOnService_GoogleCloudDataflow')
 
   // Set common parameters.
-  common_job_properties.setTopLevelJobProperties(delegate)
+  common_job_properties.setTopLevelMainJobProperties(delegate)
 
   // Set maven parameters.
   common_job_properties.setMavenConfig(delegate)

.jenkins/job_beam_PostCommit_Java_RunnableOnService_Flink.groovy

@@ -25,7 +25,7 @@ mavenJob('beam_PostCommit_Java_RunnableOnService_Flink') {
   previousNames('beam_PostCommit_RunnableOnService_FlinkLocal')
 
   // Set common parameters.
-  common_job_properties.setTopLevelJobProperties(delegate)
+  common_job_properties.setTopLevelMainJobProperties(delegate)
 
   // Set maven parameters.
   common_job_properties.setMavenConfig(delegate)

.jenkins/job_beam_PostCommit_Java_RunnableOnService_Gearpump.groovy

@@ -26,7 +26,9 @@ mavenJob('beam_PostCommit_Java_RunnableOnService_Gearpump') {
   previousNames('beam_PostCommit_RunnableOnService_GearpumpLocal')
 
   // Set common parameters.
-  common_job_properties.setTopLevelJobProperties(delegate, 'gearpump-runner')
+  common_job_properties.setTopLevelMainJobProperties(
+      delegate,
+      'gearpump-runner')
 
   // Set maven parameters.
   common_job_properties.setMavenConfig(delegate)

.jenkins/job_beam_PostCommit_Java_RunnableOnService_Spark.groovy

@@ -25,7 +25,7 @@ mavenJob('beam_PostCommit_Java_RunnableOnService_Spark') {
   previousNames('beam_PostCommit_RunnableOnService_SparkLocal')
 
   // Set common parameters.
-  common_job_properties.setTopLevelJobProperties(delegate)
+  common_job_properties.setTopLevelMainJobProperties(delegate)
 
   // Set maven parameters.
   common_job_properties.setMavenConfig(delegate)

.jenkins/job_beam_PostCommit_Python_Verify.groovy

@@ -25,7 +25,7 @@ job('beam_PostCommit_Python_Verify') {
   previousNames('beam_PostCommit_PythonVerify')
 
   // Set common parameters.
-  common_job_properties.setTopLevelJobProperties(delegate)
+  common_job_properties.setTopLevelMainJobProperties(delegate)
 
   // Sets that this is a PostCommit job.
   common_job_properties.setPostCommit(delegate, '0 3-22/6 * * *')

.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy

@@ -29,7 +29,7 @@ mavenJob('beam_PreCommit_Java_MavenInstall') {
   concurrentBuild()
 
   // Set common parameters.
-  common_job_properties.setTopLevelJobProperties(delegate)
+  common_job_properties.setTopLevelMainJobProperties(delegate)
 
   // Set Maven parameters.
   common_job_properties.setMavenConfig(delegate)

.jenkins/job_beam_PreCommit_Website_Stage.groovy

@@ -0,0 +1,62 @@
+import common_job_properties
+
+// Defines a job.
+job('beam_PreCommit_Website_Stage') {
+  description('Stages the pull requests proposed for the Apache Beam ' +
+              'website to a temporary location to ease reviews.')
+
+  // Set common parameters.
+  common_job_properties.setTopLevelWebsiteJobProperties(delegate)
+
+  // Set pull request build trigger.
+  common_job_properties.setPreCommit(
+      delegate,
+      'Jenkins: automatic staging of pull requests',
+      '\nJenkins built the site at commit id ${ghprbActualCommit} with ' +
+      'Jekyll and staged it [here](http://apache-beam-website-pull-' +
+      'requests.storage.googleapis.com/${ghprbPullId}/index.html). ' +
+      'Happy reviewing.\n\nNote that any previous site has been deleted. ' +
+      'This staged site will be automatically deleted after its TTL ' +
+      'expires. Push any commit to the pull request branch or re-trigger ' +
+      'the build to get it staged again.')
+
+  steps {
+    // Run the following shell script as a build step.
+    shell '''
+        # Install RVM.
+        gpg --keyserver hkp://keys.gnupg.net --recv-keys \\
+            409B6B1796C275462A1703113804BB82D39DC0E3
+        \\curl -sSL https://get.rvm.io | bash
+        source /home/jenkins/.rvm/scripts/rvm
+
+        # Install Ruby.
+        RUBY_VERSION_NUM=2.3.0
+        rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only
+
+        # Install Bundler gem
+        PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH
+        GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH
+        gem install bundler --user-install
+
+        # Install all needed gems.
+        bundle install --path ~/.gem/
+
+        # Remove current site if it exists.
+        GCS_PATH="gs://apache-beam-website-pull-requests/${ghprbPullId}/"
+        gsutil -m rm -r -f ${GCS_PATH} || true
+
+        # Build the new site with the baseurl specified.
+        rm -fr ./content/
+        bundle exec jekyll build --baseurl=/${ghprbPullId}
+
+        # Install BeautifulSoup HTML Parser for python.
+        pip install --user beautifulsoup4
+
+        # Fix links on staged website.
+        python .jenkins/append_index_html_to_internal_links.py
+
+        # Upload the new site.
+        gsutil -m cp -R ./content/* ${GCS_PATH}
+    '''.stripIndent().trim()
+  }
+}

.jenkins/job_beam_PreCommit_Website_Test.groovy

@@ -0,0 +1,47 @@
+import common_job_properties
+
+// Defines a job.
+job('beam_PreCommit_Website_Test') {
+  description('Runs tests on the pull requests proposed for the Apache Beam ' +
+              'website.')
+
+  // Set common parameters.
+  common_job_properties.setTopLevelWebsiteJobProperties(delegate)
+
+  // Execute concurrent builds. Multiple builds of this project may be executed
+  // in parallel. This is safe because this build does not require exclusive
+  // access to any shared resources.
+  concurrentBuild()
+
+  // Set pull request build trigger.
+  common_job_properties.setPreCommit(
+      delegate,
+      'Jenkins: test website (dead links, etc.)')
+
+  steps {
+    // Run the following shell script as a build step.
+    shell '''
+        # Install RVM.
+        gpg --keyserver hkp://keys.gnupg.net --recv-keys \\
+            409B6B1796C275462A1703113804BB82D39DC0E3
+        \\curl -sSL https://get.rvm.io | bash
+        source /home/jenkins/.rvm/scripts/rvm
+
+        # Install Ruby.
+        RUBY_VERSION_NUM=2.3.0
+        rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only
+
+        # Install Bundler gem
+        PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH
+        GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH
+        gem install bundler --user-install
+
+        # Install all needed gems.
+        bundle install --path ~/.gem/
+
+        # Build the new site and test it.
+        rm -fr ./content/
+        bundle exec rake test
+    '''.stripIndent().trim()
+  }
+}

.jenkins/job_beam_Release_NightlySnapshot.groovy

@@ -28,7 +28,7 @@ mavenJob('beam_Release_NightlySnapshot') {
   concurrentBuild()
 
   // Set common parameters.
-  common_job_properties.setTopLevelJobProperties(delegate)
+  common_job_properties.setTopLevelMainJobProperties(delegate)
 
   // Set maven paramaters.
   common_job_properties.setMavenConfig(delegate)

.jenkins/job_seed.groovy

@@ -19,13 +19,14 @@
 import common_job_properties
 
 // Defines the seed job, which creates or updates all other Jenkins projects.
-job('beam_SeedJob_Main') {
-  description('Automatically configures all Apache Beam main repo Jenkins ' +
-              'projects based on Jenkins DSL groovy files checked into the ' +
-              'code repository.')
+job('beam_SeedJob') {
+  description('Automatically configures all Apache Beam Jenkins projects based' +
+              ' on Jenkins DSL groovy files checked into the code repository.')
+
+  previousNames('beam_SeedJob_Main')
 
   // Set common parameters.
-  common_job_properties.setTopLevelJobProperties(delegate)
+  common_job_properties.setTopLevelMainJobProperties(delegate)
 
   // This is a post-commit job that runs once per day, not for every push.
   common_job_properties.setPostCommit(