PHOENIX-5032 add Apache Yetus to Phoenix

BUILDING.md

@@ -79,7 +79,7 @@ Running the tests
 All Unit tests  
 `$ mvn clean test`
 
-All Unit Tests and Integration tests
+All Unit Tests and Integration tests (takes a few hours)
 `$ mvn clean verify`
 
 The verify maven target will also run dependency:analyze-only, which checks if the dependencies 
@@ -89,19 +89,20 @@ generated at /target/site/jacoco/index.html
 To skip code coverage analysis
 `$ mvn verify -Dskip.code-coverage`
 
-Running OWASP Dependency-Check
-------------------------------
+Running project reports
+-----------------------
 
-To run OWASP Dependency-Check (https://owasp.org/www-project-dependency-check/)
-`$ mvn verify -DskipTests -Dowasp-check`
+Phoenix currently supports generating the standard set of Maven Project Info Reports, as well as
+Spotbugs, Apache Creadur RAT, OWASP Dependency-Check, and Jacoco Code Coverage reports.
 
-The report is generated in target/dependency-check-report.html
+To run all available reports (takes a few hours)
+`$ mvn clean verify site -Dspotbugs.site`
 
-Findbugs
---------
+To run OWASP, RAT and Spotbugs, but not Jacoco (takes ~10 minutes)
+`$ mvn clean compile test-compile site -Dspotbugs.site`
 
-Findbugs report is generated in /target/site  
-`$ mvn site`
+The reports are accessible via `target/site/index.html`, under the main project,
+as well as each of the subprojects. (not every project has all reports)
 
 Generate Apache Web Site
 ------------------------

Jenkinsfile.yetus

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+pipeline {
+    agent {
+        label 'Hadoop'
+    }
+
+    options {
+        buildDiscarder(logRotator(daysToKeepStr: '7'))
+        timestamps()
+    }
+
+    stages {
+
+        stage('Yetus') {
+
+            environment {
+                MAVEN_OPTS = '-Xmx3G'
+                DEBUG = 'true'
+                RUN_IN_DOCKER = 'true'
+            }
+
+            options {
+                timeout(time: 5, unit: 'HOURS')
+            }
+
+            steps {
+                sh "component/dev/jenkins_precommit_jira_yetus.sh"
+            }
+
+            post {
+                always {
+                    junit testResults: "component/**/target/**/TEST-*.xml", allowEmptyResults: true
+                    // Has to be relative to WORKSPACE.
+                    archiveArtifacts artifacts: "patchprocess/*", excludes: "patchprocess/precommit"
+                    archiveArtifacts artifacts: "patchprocess/**/*", excludes: "patchprocess/precommit/**/*"
+                    publishHTML target: [
+                      allowMissing: true,
+                      keepAll: true,
+                      alwaysLinkToLastBuild: true,
+                      // Has to be relative to WORKSPACE
+                      reportDir: "/",
+                      reportFiles: 'report.html',
+                      reportName: 'Precommit Report'
+                    ]
+                }
+            }
+        }
+    }
+}

dev/docker/Dockerfile.yetus

@@ -0,0 +1,16 @@
+FROM apache/yetus:0.12.0
+
+#####
+# Update SpotBugs
+#####
+RUN rm -rf /opt/spotbugs \
+    && git clone https://github.com/stoty/spotbugs.git \
+    && cd spotbugs \
+    && git checkout PHOENIX-1161-backport \
+    && ./gradlew clean build -x test \
+    && mkdir /opt/spotbugs \
+    && tar -C /opt/spotbugs --strip-components 1 -xpf spotbugs/build/distributions/spotbugs-4.1.2.tgz \
+    && chmod a+rx /opt/spotbugs/bin/* \
+    && cd .. \
+    && rm -rf ./spotbugs
+ENV SPOTBUGS_HOME /opt/spotbugs

dev/jenkins_precommit_jira_yetus.sh

@@ -0,0 +1,177 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#
+# Based on jenkins_precommit_jira_yetus.sh of the HBase project
+#
+
+if [[ "true" = "${DEBUG}" ]]; then
+  set -x
+  printenv
+fi
+
+##To set jenkins Environment Variables:
+export TOOLS_HOME=/home/jenkins/tools
+#export JAVA_HOME=${JAVA_HOME_HADOOP_MACHINES_HOME}
+export CLOVER_HOME=${TOOLS_HOME}/clover/latest
+#export MAVEN_HOME=${MAVEN_3_0_4_HOME}
+export MAVEN_HOME=/home/jenkins/tools/maven/apache-maven-3.0.5
+
+#export PATH=$PATH:${JAVA_HOME}/bin:${MAVEN_HOME}/bin:
+export PATH=$PATH:${MAVEN_HOME}/bin:
+
+YETUS_RELEASE=0.12.0
+COMPONENT=${WORKSPACE}/component
+TEST_FRAMEWORK=${WORKSPACE}/test_framework
+
+PATCHPROCESS=${WORKSPACE}/patchprocess
+if [[ -d ${PATCHPROCESS} ]]; then
+  echo "[WARN] patch process already existed '${PATCHPROCESS}'"
+  rm -rf "${PATCHPROCESS}"
+fi
+mkdir -p "${PATCHPROCESS}"
+
+
+## Checking on H* machine nonsense
+echo "JAVA_HOME: ${JAVA_HOME}"
+ls -l "${JAVA_HOME}" || true
+echo "MAVEN_HOME: ${MAVEN_HOME}"
+echo "maven version:"
+mvn --offline --version  || true
+echo "getting machine specs, find in ${BUILD_URL}/artifact/patchprocess/machine/"
+mkdir "${PATCHPROCESS}/machine"
+cat /proc/cpuinfo >"${PATCHPROCESS}/machine/cpuinfo" 2>&1 || true
+cat /proc/meminfo >"${PATCHPROCESS}/machine/meminfo" 2>&1 || true
+cat /proc/diskstats >"${PATCHPROCESS}/machine/diskstats" 2>&1 || true
+cat /sys/block/sda/stat >"${PATCHPROCESS}/machine/sys-block-sda-stat" 2>&1 || true
+df -h >"${PATCHPROCESS}/machine/df-h" 2>&1 || true
+ps -Awwf >"${PATCHPROCESS}/machine/ps-Awwf" 2>&1 || true
+ifconfig -a >"${PATCHPROCESS}/machine/ifconfig-a" 2>&1 || true
+lsblk -ta >"${PATCHPROCESS}/machine/lsblk-ta" 2>&1 || true
+lsblk -fa >"${PATCHPROCESS}/machine/lsblk-fa" 2>&1 || true
+cat /proc/loadavg >"${PATCHPROCESS}/loadavg" 2>&1 || true
+ulimit -a >"${PATCHPROCESS}/machine/ulimit-a" 2>&1 || true
+
+## /H*
+
+### Download Yetus
+if [[ "true" != "${USE_YETUS_PRERELEASE}" ]]; then
+  if [ ! -d "${TEST_FRAMEWORK}/apache-yetus-${YETUS_RELEASE}" ]; then
+    mkdir -p "${TEST_FRAMEWORK}"
+    cd "${TEST_FRAMEWORK}" || exit 1
+    # clear out any cached 'use a prerelease' versions
+    rm -rf apache-yetus-*
+
+    mkdir -p "${TEST_FRAMEWORK}/.gpg"
+    chmod -R 700 "${TEST_FRAMEWORK}/.gpg"
+
+    curl -L --fail -o "${TEST_FRAMEWORK}/KEYS_YETUS" https://dist.apache.org/repos/dist/release/yetus/KEYS
+    gpg --homedir "${TEST_FRAMEWORK}/.gpg" --import "${TEST_FRAMEWORK}/KEYS_YETUS"
+
+    ## Release
+    curl -L --fail -O "https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/apache-yetus-${YETUS_RELEASE}-bin.tar.gz"
+    curl -L --fail -O "https://dist.apache.org/repos/dist/release/yetus/${YETUS_RELEASE}/apache-yetus-${YETUS_RELEASE}-bin.tar.gz.asc"
+    gpg --homedir "${TEST_FRAMEWORK}/.gpg" --verify "apache-yetus-${YETUS_RELEASE}-bin.tar.gz.asc"
+    tar xzpf "apache-yetus-${YETUS_RELEASE}-bin.tar.gz"
+  fi
+  TESTPATCHBIN=${TEST_FRAMEWORK}/apache-yetus-${YETUS_RELEASE}/bin/test-patch
+  TESTPATCHLIB=${TEST_FRAMEWORK}/apache-yetus-${YETUS_RELEASE}/lib/precommit
+else
+  prerelease_dirs=("${TEST_FRAMEWORK}/${YETUS_PRERELEASE_GITHUB/\//-}-*")
+  if [ ! -d "${prerelease_dirs[0]}" ]; then
+    mkdir -p "${TEST_FRAMEWORK}"
+    cd "${TEST_FRAMEWORK}" || exit
+    ## from github
+    curl -L --fail "https://api.github.com/repos/${YETUS_PRERELEASE_GITHUB}/tarball/HEAD" > yetus.tar.gz
+    tar xvpf yetus.tar.gz
+    prerelease_dirs=("${TEST_FRAMEWORK}/${YETUS_PRERELEASE_GITHUB/\//-}-*")
+  fi
+  TESTPATCHBIN=${prerelease_dirs[0]}/precommit/test-patch.sh
+  TESTPATCHLIB=${prerelease_dirs[0]}/precommit
+fi
+
+if [[ "true" = "${DEBUG}" ]]; then
+  # DEBUG print the test framework
+  ls -l "${TESTPATCHBIN}"
+  ls -la "${TESTPATCHLIB}/test-patch.d/"
+  # DEBUG print the local customization
+  if [ -d "${COMPONENT}/dev/test-patch.d" ]; then
+    ls -la "${COMPONENT}/dev/test-patch.d/"
+  fi
+  YETUS_ARGS=(--debug "${YETUS_ARGS[@]}")
+fi
+
+if [ -r "${COMPONENT}/dev/phoenix-personality.sh" ] ; then
+  YETUS_ARGS=("--personality=${COMPONENT}/dev/phoenix-personality.sh" "${YETUS_ARGS[@]}")
+fi
+
+if [ ! -x "${TESTPATCHBIN}" ] && [ -n "${TEST_FRAMEWORK}" ] && [ -d "${TEST_FRAMEWORK}" ]; then
+  echo "Something is amiss with the test framework; removing it. please re-run."
+  rm -rf "${TEST_FRAMEWORK}"
+  exit 1
+fi
+
+cd "${WORKSPACE}" || exit
+
+
+#
+# Yetus *always* builds with JAVA_HOME, so no need to list it.
+#
+# non-docker-mode JDK:
+
+# docker-mode:  (openjdk 7 added for free)
+#         --docker \
+#         --multijdkdirs="/usr/lib/jvm/java-8-openjdk-amd64" \
+
+if [[ "true" = "${RUN_IN_DOCKER}" ]]; then
+  YETUS_ARGS=(
+    --docker \
+    --dockermemlimit=20g \
+    "--multijdkdirs=/usr/lib/jvm/java-8-openjdk-amd64" \
+    "${YETUS_ARGS[@]}" \
+  )
+  if [ -r "${COMPONENT}/dev/docker/Dockerfile.yetus" ]; then
+    YETUS_ARGS=("--dockerfile=${COMPONENT}/dev/docker/Dockerfile.yetus" "${YETUS_ARGS[@]}")
+  fi
+fi
+
+if [ -d "${COMPONENT}/dev/test-patch.d" ]; then
+  YETUS_ARGS=("--user-plugins=${COMPONENT}/dev/test-patch.d" "${YETUS_ARGS[@]}")
+fi
+
+#Removed some code not relevant to Phoenix
+
+YETUS_ARGS=("--skip-dirs=dev" "${YETUS_ARGS[@]}")
+
+
+/bin/bash "${TESTPATCHBIN}" \
+        "${YETUS_ARGS[@]}" \
+        --patch-dir="${PATCHPROCESS}" \
+        --basedir="${COMPONENT}" \
+        --mvn-custom-repos \
+        --mvn-custom-repos-dir="${WORKSPACE}/yetus-m2/" \
+        --whitespace-eol-ignore-list=".*/generated/.*" \
+        --whitespace-tabs-ignore-list=".*/generated/.*" \
+        --run-tests \
+        --plugins="all,-findbugs,-gitlab" \
+        --html-report-file=report.html \
+        --jira-user=hadoopqa \
+        --jira-password="${JIRA_PASSWORD}" \
+        "PHOENIX-${ISSUE_NUM}"
+
+find "${COMPONENT}" -name target -exec chmod -R u+w {} \;