HBASE-27935 Introduce Jenkins PR job for hbase-kustomize

dev-support/jenkins/Dockerfile

@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dockerfile for hbase-operator-tools pre-commit build.
+# https://ci-hbase.apache.org/job/HBase-Operator-Tools-PreCommit/
+
+ARG IMG_BASE='ubuntu'
+ARG IMG_BASE_TAG='22.04'
+
+FROM hadolint/hadolint:latest-debian as hadolint
+
+FROM ${IMG_BASE}:${IMG_BASE_TAG} as final
+ARG IMG_BASE
+ARG IMG_BASE_TAG
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+# hadolint ignore=DL3008
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
+       binutils \
+       git \
+       rsync \
+       shellcheck \
+       wget && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY --from=hadolint /bin/hadolint /bin/hadolint
+
+CMD ["/bin/bash"]
+
+###
+# Everything past this point is either not needed for testing or breaks Yetus.
+# So tell Yetus not to read the rest of the file:
+# YETUS CUT HERE
+###

dev-support/jenkins/Jenkinsfile

@@ -0,0 +1,140 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+pipeline {
+
+    agent {
+        label 'hbase'
+    }
+
+    options {
+        // N.B. this is per-branch, which means per PR
+        disableConcurrentBuilds()
+        buildDiscarder(logRotator(numToKeepStr: '15'))
+        timeout (time: 1, unit: 'HOURS')
+        timestamps()
+        skipDefaultCheckout()
+    }
+
+    environment {
+        SRC_REL = 'src'
+        PATCH_REL = 'output'
+        YETUS_REL = 'yetus'
+        // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
+        YETUS_VERSION = 'rel/0.14.1'
+        DOCKERFILE_REL = "${SRC_REL}/dev-support/jenkins/Dockerfile"
+        YETUS_DRIVER_REL = "${SRC_REL}/dev-support/jenkins/jenkins_precommit_github_yetus.sh"
+        ARCHIVE_PATTERN_LIST = 'TEST-*.xml'
+        BUILD_URL_ARTIFACTS = "artifact/${WORKDIR_REL}/${PATCH_REL}"
+        WORKDIR_REL = 'yetus-precommit-check'
+        WORKDIR = "${WORKSPACE}/${WORKDIR_REL}"
+        SOURCEDIR = "${WORKDIR}/${SRC_REL}"
+        PATCHDIR = "${WORKDIR}/${PATCH_REL}"
+        DOCKERFILE = "${WORKDIR}/${DOCKERFILE_REL}"
+        YETUS_DRIVER = "${WORKDIR}/${YETUS_DRIVER_REL}"
+        YETUSDIR = "${WORKDIR}/${YETUS_REL}"
+        PLUGINS = 'all'
+    }
+
+    parameters {
+        booleanParam(name: 'DEBUG',
+               defaultValue: false,
+               description: 'Print extra outputs for debugging the jenkins job and yetus')
+    }
+
+    stages {
+        stage ('precommit checks') {
+            steps {
+                dir("${SOURCEDIR}") {
+                    checkout scm
+                }
+                dir("${YETUSDIR}") {
+                    checkout([
+                      $class           : 'GitSCM',
+                      branches         : [[name: "${YETUS_VERSION}"]],
+                      userRemoteConfigs: [[url: 'https://github.com/apache/yetus.git']]]
+                    )
+                }
+                dir("${WORKDIR}") {
+                    withCredentials([
+                      usernamePassword(
+                        credentialsId: 'apache-hbase-at-github.com',
+                        passwordVariable: 'GITHUB_PASSWORD',
+                        usernameVariable: 'GITHUB_USER'
+                      )]) {
+                        sh label: 'test-patch', script: '''#!/bin/bash -e
+                            printenv 2>&1 | sort
+                            echo "[INFO] Launching Yetus via ${YETUS_DRIVER}"
+                            "${YETUS_DRIVER}"
+                        '''
+                    }
+                }
+            }
+            post {
+                always {
+                    // Has to be relative to WORKSPACE.
+                    archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit"
+                    archiveArtifacts artifacts: "${WORKDIR_REL}/${PATCH_REL}/**/*", excludes: "${WORKDIR_REL}/${PATCH_REL}/precommit/**/*"
+                    publishHTML target: [
+                      allowMissing: true,
+                      keepAll: true,
+                      alwaysLinkToLastBuild: true,
+                      // Has to be relative to WORKSPACE
+                      reportDir: "${WORKDIR_REL}/${PATCH_REL}",
+                      reportFiles: 'report.html',
+                      reportName: 'PR General Check Report'
+                    ]
+                }
+                // Jenkins pipeline jobs fill slaves on PRs without this :(
+                cleanup() {
+                    script {
+                        sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
+                            # See YETUS-764
+                            if [ -f "${PATCHDIR}/pidfile.txt" ]; then
+                              echo "test-patch process appears to still be running: killing"
+                              kill `cat "${PATCHDIR}/pidfile.txt"` || true
+                              sleep 10
+                            fi
+                            if [ -f "${PATCHDIR}/cidfile.txt" ]; then
+                              echo "test-patch container appears to still be running: killing"
+                              docker kill `cat "${PATCHDIR}/cidfile.txt"` || true
+                            fi
+                            # See HADOOP-13951
+                            chmod -R u+rxw "${WORKSPACE}"
+                        '''
+                        dir ("${WORKDIR}") {
+                            deleteDir()
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    post {
+        // Jenkins pipeline jobs fill slaves on PRs without this :(
+        cleanup() {
+            script {
+                sh label: 'Cleanup workspace', script: '''#!/bin/bash -e
+                    # See HADOOP-13951
+                    chmod -R u+rxw "${WORKSPACE}"
+                    '''
+                deleteDir()
+            }
+        }
+    }
+}

dev-support/jenkins/gather_machine_environment.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+function usage {
+  echo "Usage: ${0} /path/for/output/dir"
+  echo ""
+  echo "  Gather info about a build machine that test harnesses should poll before running."
+  echo "  presumes you'll then archive the passed output dir."
+
+  exit 1
+}
+
+if [ "$#" -lt 1 ]; then
+  usage
+fi
+
+
+declare output=$1
+
+if [ ! -d "${output}" ] || [ ! -w "${output}" ]; then
+  echo "Specified output directory must exist and be writable." >&2
+  exit 1
+fi
+
+echo "getting machine specs, find in ${BUILD_URL}/artifact/${output}/"
+echo "JAVA_HOME: ${JAVA_HOME}" >"${output}/java_home" 2>&1 || true
+ls -l "${JAVA_HOME}" >"${output}/java_home_ls" 2>&1 || true
+echo "MAVEN_HOME: ${MAVEN_HOME}" >"${output}/mvn_home" 2>&1 || true
+mvn --offline --version  >"${output}/mvn_version" 2>&1 || true
+cat /proc/cpuinfo >"${output}/cpuinfo" 2>&1 || true
+cat /proc/meminfo >"${output}/meminfo" 2>&1 || true
+cat /proc/diskstats >"${output}/diskstats" 2>&1 || true
+cat /sys/block/sda/stat >"${output}/sys-block-sda-stat" 2>&1 || true
+df -h >"${output}/df-h" 2>&1 || true
+ps -Aww >"${output}/ps-Aww" 2>&1 || true
+ifconfig -a >"${output}/ifconfig-a" 2>&1 || true
+lsblk -ta >"${output}/lsblk-ta" 2>&1 || true
+lsblk -fa >"${output}/lsblk-fa" 2>&1 || true
+ulimit -a >"${output}/ulimit-a" 2>&1 || true
+uptime >"${output}/uptime" 2>&1 || true
+hostname -a >"${output}/hostname-a" 2>&1 || true

dev-support/jenkins/jenkins_precommit_github_yetus.sh

@@ -0,0 +1,127 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+# place ourselves in the directory containing the hbase and yetus checkouts
+cd "$(dirname "$0")/../.."
+echo "executing from $(pwd)"
+
+if [[ "true" = "${DEBUG}" ]]; then
+  set -x
+  printenv 2>&1 | sort
+fi
+
+declare -i missing_env=0
+declare -a required_envs=(
+  # these ENV variables define the required API with Jenkinsfile_GitHub
+  "ARCHIVE_PATTERN_LIST"
+  "BUILD_URL_ARTIFACTS"
+  "DOCKERFILE"
+  "GITHUB_PASSWORD"
+  "GITHUB_USER"
+  "PATCHDIR"
+  "PLUGINS"
+  "SOURCEDIR"
+  "YETUSDIR"
+)
+# Validate params
+for required_env in "${required_envs[@]}"; do
+  if [ -z "${!required_env}" ]; then
+    echo "[ERROR] Required environment variable '${required_env}' is not set."
+    missing_env=${missing_env}+1
+  fi
+done
+
+if [ ${missing_env} -gt 0 ]; then
+  echo "[ERROR] Please set the required environment variables before invoking. If this error is " \
+       "on Jenkins, then please file a JIRA about the error."
+  exit 1
+fi
+
+# TODO (HBASE-23900): cannot assume test-patch runs directly from sources
+TESTPATCHBIN="${YETUSDIR}/precommit/src/main/shell/test-patch.sh"
+
+# this must be clean for every run
+rm -rf "${PATCHDIR}"
+mkdir -p "${PATCHDIR}"
+
+# Gather machine information
+mkdir "${PATCHDIR}/machine"
+"${SOURCEDIR}/dev-support/jenkins/gather_machine_environment.sh" "${PATCHDIR}/machine"
+
+# If CHANGE_URL is set (e.g., Github Branch Source plugin), process it.
+# Otherwise exit, because we don't want HBase to do a
+# full build.  We wouldn't normally do this check for smaller
+# projects. :)
+if [[ -z "${CHANGE_URL}" ]]; then
+  echo "Full build skipped" > "${PATCHDIR}/report.html"
+  exit 0
+fi
+# enable debug output for yetus
+if [[ "true" = "${DEBUG}" ]]; then
+  YETUS_ARGS+=("--debug")
+fi
+# If we're doing docker, make sure we don't accidentally pollute the image with a host java path
+if [ -n "${JAVA_HOME}" ]; then
+  unset JAVA_HOME
+fi
+YETUS_ARGS+=("--patch-dir=${PATCHDIR}")
+# where the source is located
+YETUS_ARGS+=("--basedir=${SOURCEDIR}")
+# lots of different output formats
+YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt")
+YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt")
+YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html")
+# enable writing back to Github
+YETUS_ARGS+=("--github-password=${GITHUB_PASSWORD}")
+YETUS_ARGS+=("--github-user=${GITHUB_USER}")
+# auto-kill any surefire stragglers during unit test runs
+YETUS_ARGS+=("--reapermode=kill")
+# set relatively high limits for ASF machines
+# changing these to higher values may cause problems
+# with other jobs on systemd-enabled machines
+YETUS_ARGS+=("--dockermemlimit=20g")
+# -1 spotbugs issues that show up prior to the patch being applied
+YETUS_ARGS+=("--spotbugs-strict-precheck")
+# rsync these files back into the archive dir
+YETUS_ARGS+=("--archive-list=${ARCHIVE_PATTERN_LIST}")
+# URL for user-side presentation in reports and such to our artifacts
+YETUS_ARGS+=("--build-url-artifacts=${BUILD_URL_ARTIFACTS}")
+# plugins to enable
+YETUS_ARGS+=("--plugins=${PLUGINS}")
+YETUS_ARGS+=("--tests-filter=test4tests")
+# run in docker mode and specifically point to our
+# Dockerfile since we don't want to use the auto-pulled version.
+YETUS_ARGS+=("--docker")
+YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
+# effectively treat dev-support as a custom maven module
+YETUS_ARGS+=("--skip-dirs=dev-support")
+# help keep the ASF boxes clean
+YETUS_ARGS+=("--sentinel")
+# use emoji vote so it is easier to find the broken line
+YETUS_ARGS+=("--github-use-emoji-vote")
+YETUS_ARGS+=("--github-repo=apache/hbase-kustomize")
+# increasing proc limit to avoid OOME: unable to create native threads
+YETUS_ARGS+=("--proclimit=5000")
+
+
+echo "Launching yetus with command line:"
+echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"
+
+/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"