From 88c76dcf27394f71a551752a66446856af4b31b6 Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Wed, 1 Feb 2017 16:45:33 -0800 Subject: [PATCH 1/2] Delete jobs due to move to main repo. Signed-off-by: Jason Kuster --- .../append_index_html_to_internal_links.py | 76 ------------- .jenkins/common_job_properties.groovy | 106 ------------------ .jenkins/job_precommit_stage.groovy | 62 ---------- .jenkins/job_precommit_test.groovy | 47 -------- .jenkins/job_seed.groovy | 32 ------ 5 files changed, 323 deletions(-) delete mode 100644 .jenkins/append_index_html_to_internal_links.py delete mode 100644 .jenkins/common_job_properties.groovy delete mode 100644 .jenkins/job_precommit_stage.groovy delete mode 100644 .jenkins/job_precommit_test.groovy delete mode 100644 .jenkins/job_seed.groovy diff --git a/.jenkins/append_index_html_to_internal_links.py b/.jenkins/append_index_html_to_internal_links.py deleted file mode 100644 index df55d597a3d..00000000000 --- a/.jenkins/append_index_html_to_internal_links.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Script to fix the links in the staged website. -Finds all internal links which do not have index.html at the end and appends -index.html in the appropriate place (preserving anchors, etc). - -Usage: - From root directory, after running the jekyll build, execute - 'python .jenkins/append_index_html_to_internal_links.py'. - -Dependencies: - beautifulsoup4 - Installable via pip as 'sudo pip install beautifulsoup4' or apt via - 'sudo apt-get install python-beautifulsoup4'. - -""" - -import fnmatch -import os -import re -from bs4 import BeautifulSoup - -# Original link match. Matches any string which starts with '/' and doesn't -# have a file extension. -linkMatch = r'^\/(.*\.(?!([^\/]+)$))?[^.]*$' - -# Regex which matches strings of type /internal/link/#anchor. Breaks into two -# groups for ease of inserting 'index.html'. -anchorMatch1 = r'(.+\/)(#[^\/]+$)' - -# Regex which matches strings of type /internal/link#anchor. Breaks into two -# groups for ease of inserting 'index.html'. -anchorMatch2 = r'(.+\/[a-zA-Z0-9]+)(#[^\/]+$)' - - -matches = [] -# Recursively walk content directory and find all html files. -for root, dirnames, filenames in os.walk('content'): - for filename in fnmatch.filter(filenames, '*.html'): - # Javadoc does not have the index.html problem, so omit it. - if 'javadoc' not in root: - matches.append(os.path.join(root, filename)) - -print 'Matches: ' + str(len(matches)) -# Iterates over each matched file looking for link matches. -for match in matches: - print 'Fixing links in: ' + match - mf = open(match) - soup = BeautifulSoup(mf) - # Iterates over every - for a in soup.findAll('a'): - try: - hr = a['href'] - if re.match(linkMatch, hr) is not None: - if hr.endswith('/'): - # /internal/link/ - a['href'] = hr + 'index.html' - elif re.match(anchorMatch1, hr) is not None: - # /internal/link/#anchor - mat = re.match(anchorMatch1, hr) - a['href'] = mat.group(1) + 'index.html' + mat.group(2) - elif re.match(anchorMatch2, hr) is not None: - # /internal/link#anchor - mat = re.match(anchorMatch2, hr) - a['href'] = mat.group(1) + '/index.html' + mat.group(2) - else: - # /internal/link - a['href'] = hr + '/index.html' - mf.close() - - html = soup.prettify("utf-8") - # Write back to the file. - with open(match, "wb") as f: - print 'Replacing ' + hr + ' with: ' + a['href'] - f.write(html) - except KeyError as e: - # Some tags don't have an href. - continue diff --git a/.jenkins/common_job_properties.groovy b/.jenkins/common_job_properties.groovy deleted file mode 100644 index 6bc3d145a9d..00000000000 --- a/.jenkins/common_job_properties.groovy +++ /dev/null @@ -1,106 +0,0 @@ -// Contains functions that help build Jenkins projects. Functions typically set -// common properties that are shared among all Jenkins projects. -class common_job_properties { - - // Sets common top-level job properties. - static def setTopLevelJobProperties(def context) { - - // GitHub project. - context.properties { - githubProjectUrl('https://github.com/apache/beam-site/') - } - - // Restrict this project to run only on Jenkins executors dedicated to the - // Apache Beam project. - context.label('beam') - - // Discard old builds. Build records are only kept up to this number of days. - context.logRotator { - daysToKeep(14) - } - - // Source code management. - context.scm { - git { - remote { - url('https://github.com/apache/beam-site.git') - refspec('+refs/heads/*:refs/remotes/origin/* ' + - '+refs/pull/*:refs/remotes/origin/pr/*') - } - branch('${sha1}') - extensions { - cleanAfterCheckout() - } - } - } - - context.parameters { - // This is a recommended setup if you want to run the job manually. The - // ${sha1} parameter needs to be provided, and defaults to the main branch. - stringParam( - 'sha1', - 'asf-site', - 'Commit id or refname (eg: origin/pr/9/head) you want to build.') - } - - context.wrappers { - // Abort the build if it's stuck for more minutes than specified. - timeout { - absolute(30) - abortBuild() - } - } - } - - // Sets the pull request build trigger. - static def setPullRequestBuildTrigger(def context, - def commitStatusContext, - def successComment = '--none--') { - context.triggers { - githubPullRequest { - admins(['asfbot']) - useGitHubHooks() - orgWhitelist(['apache']) - allowMembersOfWhitelistedOrgsAsAdmin() - permitAll() - - extensions { - commitStatus { - // This is the name that will show up in the GitHub pull request UI - // for this Jenkins project. - delegate.context(commitStatusContext) - } - - /* - This section is disabled, because of jenkinsci/ghprb-plugin#417 issue. - For the time being, an equivalent configure section below is added. - - // Comment messages after build completes. - buildStatus { - completedStatus('SUCCESS', successComment) - completedStatus('FAILURE', '--none--') - completedStatus('ERROR', '--none--') - } - */ - } - } - } - - // Comment messages after build completes. - context.configure { - def messages = it / triggers / 'org.jenkinsci.plugins.ghprb.GhprbTrigger' / extensions / 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildStatus' / messages - messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' { - message(successComment) - result('SUCCESS') - } - messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' { - message('--none--') - result('ERROR') - } - messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' { - message('--none--') - result('FAILURE') - } - } - } -} diff --git a/.jenkins/job_precommit_stage.groovy b/.jenkins/job_precommit_stage.groovy deleted file mode 100644 index 398b8e6e6a7..00000000000 --- a/.jenkins/job_precommit_stage.groovy +++ /dev/null @@ -1,62 +0,0 @@ -import common_job_properties - -// Defines a job. -job('beam_PreCommit_Website_Stage') { - description('Stages the pull requests proposed for the Apache Beam ' + - 'website to a temporary location to ease reviews.') - - // Set common parameters. - common_job_properties.setTopLevelJobProperties(delegate) - - // Set pull request build trigger. - common_job_properties.setPullRequestBuildTrigger( - delegate, - 'Jenkins: automatic staging of pull requests', - '\nJenkins built the site at commit id ${ghprbActualCommit} with ' + - 'Jekyll and staged it [here](http://apache-beam-website-pull-' + - 'requests.storage.googleapis.com/${ghprbPullId}/index.html). ' + - 'Happy reviewing.\n\nNote that any previous site has been deleted. ' + - 'This staged site will be automatically deleted after its TTL ' + - 'expires. Push any commit to the pull request branch or re-trigger ' + - 'the build to get it staged again.') - - steps { - // Run the following shell script as a build step. - shell ''' - # Install RVM. - gpg --keyserver hkp://keys.gnupg.net --recv-keys \\ - 409B6B1796C275462A1703113804BB82D39DC0E3 - \\curl -sSL https://get.rvm.io | bash - source /home/jenkins/.rvm/scripts/rvm - - # Install Ruby. - RUBY_VERSION_NUM=2.3.0 - rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only - - # Install Bundler gem - PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH - GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH - gem install bundler --user-install - - # Install all needed gems. - bundle install --path ~/.gem/ - - # Remove current site if it exists. - GCS_PATH="gs://apache-beam-website-pull-requests/${ghprbPullId}/" - gsutil -m rm -r -f ${GCS_PATH} || true - - # Build the new site with the baseurl specified. - rm -fr ./content/ - bundle exec jekyll build --baseurl=/${ghprbPullId} - - # Install BeautifulSoup HTML Parser for python. - pip install --user beautifulsoup4 - - # Fix links on staged website. - python .jenkins/append_index_html_to_internal_links.py - - # Upload the new site. - gsutil -m cp -R ./content/* ${GCS_PATH} - '''.stripIndent().trim() - } -} diff --git a/.jenkins/job_precommit_test.groovy b/.jenkins/job_precommit_test.groovy deleted file mode 100644 index c6128a82318..00000000000 --- a/.jenkins/job_precommit_test.groovy +++ /dev/null @@ -1,47 +0,0 @@ -import common_job_properties - -// Defines a job. -job('beam_PreCommit_Website_Test') { - description('Runs tests on the pull requests proposed for the Apache Beam ' + - 'website.') - - // Set common parameters. - common_job_properties.setTopLevelJobProperties(delegate) - - // Execute concurrent builds. Multiple builds of this project may be executed - // in parallel. This is safe because this build does not require exclusive - // access to any shared resources. - concurrentBuild() - - // Set pull request build trigger. - common_job_properties.setPullRequestBuildTrigger( - delegate, - 'Jenkins: test website (dead links, etc.)') - - steps { - // Run the following shell script as a build step. - shell ''' - # Install RVM. - gpg --keyserver hkp://keys.gnupg.net --recv-keys \\ - 409B6B1796C275462A1703113804BB82D39DC0E3 - \\curl -sSL https://get.rvm.io | bash - source /home/jenkins/.rvm/scripts/rvm - - # Install Ruby. - RUBY_VERSION_NUM=2.3.0 - rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only - - # Install Bundler gem - PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH - GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH - gem install bundler --user-install - - # Install all needed gems. - bundle install --path ~/.gem/ - - # Build the new site and test it. - rm -fr ./content/ - bundle exec rake test - '''.stripIndent().trim() - } -} diff --git a/.jenkins/job_seed.groovy b/.jenkins/job_seed.groovy deleted file mode 100644 index 92522d2ac9f..00000000000 --- a/.jenkins/job_seed.groovy +++ /dev/null @@ -1,32 +0,0 @@ -import common_job_properties - -// Defines the seed job, which creates or updates all other Jenkins projects. -job('beam_SeedJob_Website') { - description('Automatically configures all Apache Beam website Jenkins ' + - 'projects based on Jenkins DSL groovy files checked into the ' + - 'code repository.') - - // Set common parameters. - common_job_properties.setTopLevelJobProperties(delegate) - - // Run this job every night to revert back any accidental changes to the - // configuration. - triggers { - cron('0 6 * * *') - } - - steps { - dsl { - // A list or a glob of other groovy files to process. - external('.jenkins/job_*.groovy') - - // If a job is removed from the script, disable it (rather than deleting). - removeAction('DISABLE') - } - } - - publishers { - // Notify the mailing list for each failed build. - mailer('dev@beam.apache.org', false, false) - } -} From b36d4e3e560de7f8121df4c003a5b2fd2fe9f1be Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Wed, 1 Feb 2017 17:10:22 -0800 Subject: [PATCH 2/2] add tool back Signed-off-by: Jason Kuster --- .../append_index_html_to_internal_links.py | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 .jenkins/append_index_html_to_internal_links.py diff --git a/.jenkins/append_index_html_to_internal_links.py b/.jenkins/append_index_html_to_internal_links.py new file mode 100644 index 00000000000..df55d597a3d --- /dev/null +++ b/.jenkins/append_index_html_to_internal_links.py @@ -0,0 +1,76 @@ +"""Script to fix the links in the staged website. +Finds all internal links which do not have index.html at the end and appends +index.html in the appropriate place (preserving anchors, etc). + +Usage: + From root directory, after running the jekyll build, execute + 'python .jenkins/append_index_html_to_internal_links.py'. + +Dependencies: + beautifulsoup4 + Installable via pip as 'sudo pip install beautifulsoup4' or apt via + 'sudo apt-get install python-beautifulsoup4'. + +""" + +import fnmatch +import os +import re +from bs4 import BeautifulSoup + +# Original link match. Matches any string which starts with '/' and doesn't +# have a file extension. +linkMatch = r'^\/(.*\.(?!([^\/]+)$))?[^.]*$' + +# Regex which matches strings of type /internal/link/#anchor. Breaks into two +# groups for ease of inserting 'index.html'. +anchorMatch1 = r'(.+\/)(#[^\/]+$)' + +# Regex which matches strings of type /internal/link#anchor. Breaks into two +# groups for ease of inserting 'index.html'. +anchorMatch2 = r'(.+\/[a-zA-Z0-9]+)(#[^\/]+$)' + + +matches = [] +# Recursively walk content directory and find all html files. +for root, dirnames, filenames in os.walk('content'): + for filename in fnmatch.filter(filenames, '*.html'): + # Javadoc does not have the index.html problem, so omit it. + if 'javadoc' not in root: + matches.append(os.path.join(root, filename)) + +print 'Matches: ' + str(len(matches)) +# Iterates over each matched file looking for link matches. +for match in matches: + print 'Fixing links in: ' + match + mf = open(match) + soup = BeautifulSoup(mf) + # Iterates over every + for a in soup.findAll('a'): + try: + hr = a['href'] + if re.match(linkMatch, hr) is not None: + if hr.endswith('/'): + # /internal/link/ + a['href'] = hr + 'index.html' + elif re.match(anchorMatch1, hr) is not None: + # /internal/link/#anchor + mat = re.match(anchorMatch1, hr) + a['href'] = mat.group(1) + 'index.html' + mat.group(2) + elif re.match(anchorMatch2, hr) is not None: + # /internal/link#anchor + mat = re.match(anchorMatch2, hr) + a['href'] = mat.group(1) + '/index.html' + mat.group(2) + else: + # /internal/link + a['href'] = hr + '/index.html' + mf.close() + + html = soup.prettify("utf-8") + # Write back to the file. + with open(match, "wb") as f: + print 'Replacing ' + hr + ' with: ' + a['href'] + f.write(html) + except KeyError as e: + # Some tags don't have an href. + continue