Comprehensive CI system patch

YETUS-681. Add robots plugin type YETUS-649. Travis CI support YETUS-684. add support for Gitlab CI YETUS-687. add better docker env support Circle CI support
effectivemachines · Nov 6, 2018 · f055aa3 · f055aa3
1 parent e8046f0
commit f055aa3
Show file tree

Hide file tree

Showing 18 changed files with 870 additions and 77 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+version: 2
+jobs:
+  build:
+    docker:
+      - image: awwashere/buretoolbox
+
+    working_directory: ~/repo
+
+    environment:
+      # Customize the JVM maximum heap limit
+      MAVEN_OPTS: -Xmx3200m
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "pom.xml" }}
+            # fallback to using the latest cache if no exact match is found
+            - v1-dependencies-
+
+      - run: >
+             ~/repo/precommit/src/main/shell/test-patch.sh
+             --plugins=all
+             --patch-dir=/tmp/yetus-out
+
+      - save_cache:
+          paths:
+            - ~/.m2
+          key: v1-dependencies-{{ checksum "pom.xml" }}
+
+      - store_artifacts:
+          path: /tmp/yetus-out
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+buretoolbox-job:
+  image: awwashere/buretoolbox
+  allow_failure: true
+  script:
+    - >
+      precommit/src/main/shell/test-patch.sh
+      --patch-dir=/tmp/yetus-out
+      --plugins=all
+      --java-home=/usr/lib/jvm/java-8-openjdk-amd64
+  artifacts:
+    expire_in: 1 week
+    when: always
+    paths:
+      - yetus-out/
diff --git a/.travis.yml b/.travis.yml
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+sudo: required
+
+services:
+  - docker
+
+addons:
+  artifacts: true
+  paths:
+    - yetus-out
+
+script:
+  - >
+    precommit/src/main/shell/test-patch.sh
+    --docker
+    --patch-dir=/tmp/yetus-out
+    --plugins=all
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -0,0 +1,113 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+pipeline {
+  agent any
+  triggers {
+    cron('@daily')
+  }
+  options {
+    buildDiscarder(logRotator(numToKeepStr: '5'))
+    timeout (time: 9, unit: 'HOURS')
+    timestamps()
+    checkoutToSubdirectory('src')
+  }
+  environment {
+    //YETUS_RELEASE = '0.8.0'
+    YETUS_BASEDIR = 'src'
+    // will also need to change email section below
+    YETUS_RELATIVE_PATCHDIR = 'out'
+    YETUS_DOCKERFILE = "${YETUS_BASEDIR}/precommit/src/main/shell/test-patch-docker/Dockerfile"
+  }
+  stages {
+    stage ('precommit-run') {
+      steps {
+        sh '''#!/usr/bin/env bash
+            env
+            if [[ -d "${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}" ]]; then
+              rm -rf "${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}"
+            fi
+            mkdir -p "${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}"
+
+            # where the source is located
+            YETUS_ARGS+=("--basedir=${WORKSPACE}/${YETUS_BASEDIR}")
+
+            # nuke the src repo before working
+            YETUS_ARGS+=("--resetrepo")
+
+            # Enable maven custom repos in order to avoid multiple executor clashes
+            YETUS_ARGS+=("--mvn-custom-repos")
+
+            # run in docker mode
+            YETUS_ARGS+=("--docker")
+
+            # temp storage, etc
+            YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}")
+
+            # lots of different output formats
+            YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/brief.txt")
+            YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/console.txt")
+            YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${YETUS_RELATIVE_PATCHDIR}/report.html")
+
+            # rsync these files back into the archive dir
+            YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,findbugsXml.xml")
+
+            # URL for user-side presentation
+            YETUS_ARGS+=("--build-url-artifacts=artifact/out")
+
+            # plugins to enable
+            YETUS_ARGS+=("--plugins=all")
+
+            # run test-patch from the source tree specified up above
+            TESTPATCHBIN=${WORKSPACE}/src/precommit/src/main/shell/test-patch.sh
+
+            /usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
+
+            '''
+      }
+    }
+  }
+  post {
+    always {
+      // Has to be relative to WORKSPACE.
+      archiveArtifacts "${env.YETUS_RELATIVE_PATCHDIR}/**"
+      publishHTML target: [
+        allowMissing: true,
+        keepAll: true,
+        alwaysLinkToLastBuild: true,
+        // Has to be relative to WORKSPACE
+        reportDir: "${env.YETUS_RELATIVE_PATCHDIR}",
+        reportFiles: 'report.html',
+        reportName: 'Yetus QBT Report'
+      ]
+    }
+    failure {
+      emailext subject: '$DEFAULT_SUBJECT',
+      body: '''For more details, see ${BUILD_URL}
+
+${CHANGES, format="[%d] (%a) %m"}
+
+${FILE,path="out/brief.txt"}''',
+      recipientProviders: [
+          [$class: 'CulpritsRecipientProvider'],
+          [$class: 'DevelopersRecipientProvider'],
+          [$class: 'RequesterRecipientProvider']
+      ],
+      replyTo: '$DEFAULT_REPLYTO',
+      to: '$DEFAULT_RECIPIENTS'
+    }
+  }
+}
diff --git a/asf-site-src/source/documentation/in-progress.html.md b/asf-site-src/source/documentation/in-progress.html.md
@@ -29,6 +29,8 @@ The Yetus Precommit Patch Tester allows projects to codify their patch acceptanc
 
 For a complete guide to the Precommit API, see [the generated API documentation](precommit-apidocs/).
 
+For help with CI tools and automation, see our documentation about [robots](precommit-robots).
+
 # Yetus Release Doc Maker
 
 The Release Documentation Maker allows projects to generate nicely formated Markdown Changelogs and Release Notes based upon JIRA. You can view that

diff --git a/asf-site-src/source/documentation/in-progress/precommit-advanced.md b/asf-site-src/source/documentation/in-progress/precommit-advanced.md
@@ -28,7 +28,7 @@ test-patch
 
 # Docker Support
 
-By default, test-patch runs in the same shell where it was launched.  It can alternatively use Docker to launch itself in a container.  This is particularly useful if running under a QA environment that does not provide all the necessary binaries. For example, if the patch requires a newer version of Java than what is installed on a Jenkins instance.
+By default, test-patch runs in the same shell where it was launched.  It can alternatively use Docker to launch itself in a container.  This is particularly useful if running under a QA environment that does not provide all the necessary binaries. For example, if the patch requires a newer version of Java than what is installed on a CI instance.
 
 The `--docker` parameter tells test-patch to run in Docker mode. The `--dockerfile` parameter allows one to provide a custom Dockerfile. The Dockerfile should contain all of the necessary binaries and tooling needed to run the test.  test-patch will copy this file up until the text "YETUS CUT HERE" to a different directory and then append its necessary hooks to re-launch itself prior to executing docker.
 
@@ -273,7 +273,7 @@ This list is used if the user does not provide a list of plug-ins.
 
 There are a handful of extremely important system variables that make life easier for personality and plug-in writers.  Other variables may be provided by individual plug-ins.  Check their development documentation for more information.
 
-* BUILD\_NATIVE will be set to true if the system has requested that non-JVM-based code be built (e.g., JNI or other compiled C code). Under Jenkins, this is always true.
+* BUILD\_NATIVE will be set to true if the system has requested that non-JVM-based code be built (e.g., JNI or other compiled C code). For [robots](precommit-robots), this is always true.
 
 * BUILDTOOL specifies which tool is currently being used to drive compilation.  Additionally, many build tools define xyz\_ARGS to pass on to the build tool command line. (e.g., MAVEN\_ARGS if maven is in use).  Projects may set this in their personality.  NOTE: today, only one build tool at a time is supported.  This may change in the future.
 
@@ -295,4 +295,4 @@ There are a handful of extremely important system variables that make life easie
 
 * PATCH\_NAMING\_RULE should be a URL that points to a project's on-boarding documentation for new users. It is used to suggest a review of patch naming guidelines. Since this should be project specific information, it is useful to set in a project's personality.
 
-* TEST\_PARALLEL if parallel unit tests have been requested. Project personalities are responsible for actually enabling or ignoring the request. TEST\_THREADS is the number of threads that have been requested to run in parallel.
+* TEST\_PARALLEL if parallel unit tests have been requested. Project personalities are responsible for actually enabling or ignoring the request. TEST\_THREADS is the number of threads that have been requested to run in parallel. For [robots](precommit-robots), this is always true.
diff --git a/asf-site-src/source/documentation/in-progress/precommit-robots.md b/asf-site-src/source/documentation/in-progress/precommit-robots.md
@@ -0,0 +1,86 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+Robots: Continuous Integration Support
+======================================
+
+test-patch works hand-in-hand with various CI and other automated build systems.  test-patch will attempt to auto-determine if it is running under such a system and change its defaults to match known configuration parameters automatically. When robots are activated, there is generally some additional/changed behavior:
+
+  * display extra information in the footer
+  * change log entries from file names to URLs
+  * activate --resetrepo
+  * enable the running of unit tests and run them in parallel
+  * if possible, write comments to bug systems
+  * attempt to determine the build tool in use
+  * activate Docker maintenance when --docker is passed
+  * attempt to determine whether this is a full build (qbt-mode) or testing a patch/merge request/pull request.
+
+Circle CI
+=========
+
+TRIGGER: ${CIRCLECI}=true
+
+Circle CI support in test-patch is limited to github.com.  Artifacts (the --patch-dir directory) need to be handled directly on the command line.  Linking to the logs is not currently supported.
+
+Gitlab CI
+=========
+
+TRIGGER: ${GITLAB_CI}=true
+
+Artifacts, patch logs, etc are configured to go to a yetus-out directory in the source tree after completion. Adding this stanza to your .gitlab-ci.yml file will upload and store those components for a week in Gitlab CI's artifact retrieval system:
+
+```yaml
+  artifacts:
+    expire_in: 1 week
+    when: always
+    paths:
+      - yetus-out/
+
+```
+
+Jenkins
+=======
+
+TRIGGER: ${JENKINS_URL}=(anything)
+
+Jenkins is extremely open-ended and, given multiple executors, does not run workflows in isolation.  As a result, many more configuration options generally need to be configured as it is not safe or may be suprising to users for test-patch to autodetermine some settings.  By default, Jenkins will trigger a full build.
+
+There is some support for a few well known environment variables:
+  * ${CHANGE_URL} or ${ghprbPullLink} will set the patch location as well as trigger some extra handling if 'github' or 'gitlab' appear in the string.
+  * ${GIT_URL} will trigger the same extra handling if 'github' or 'gitlab' appear in the string.
+  * If ${ghprbPullId} is set, then test-patch will configure itself for a Github-style PR.
+
+
+See also
+  * See also the source tree's Jenkinsfile for some tips and tricks.
+  * [precommit-admin](precommit-admin), for special utilities built for Jenkins.
+  * [GitHub Branch Source Plugin](https://wiki.jenkins.io/display/JENKINS/GitHub+Branch+Source+Plugin)
+  * [GitHub Pull Request Builder Plugin](https://wiki.jenkins.io/display/JENKINS/GitHub+pull+request+builder+plugin)
+  * https://{your local server}/env-vars.html/
+
+Travis CI
+=========
+
+TRIGGER: ${TRAVIS}=true
+
+Travis CI support will update the local checked out source repository to include references to all branches and tags
+
+If ${ARTIFACTS_PATH} is configured, then '--patch-dir' is set to the first listed directory path.  However, links to the location logs must still be configured manually.
+
+Personalities will override the auto-detected Github repository information.  It may be necessary to manually configure it in your .travis.yml file.