Updating to Proxy Verifier v2.0.0 (#7454)

We will want to use Proxy Verifier v2.0.0 for it's delayed transaction and HTTP/2 to server capabilities. This updates the AuTest extensions to properly handle the new version. (cherry picked from commit 8dc2cbe) Conflicts: ci/jenkins/bin/autest.sh NOTE: I had to copy over all of ci/jenkins/bin from master into this PR, because it got maddening with all the merge conflicts. The CI jenkins scripts were never meant to be backported, and somehow we started including the Jenkins scripts changes into regular PRs.
apache · Feb 12, 2021 · 7eb4e48 · 7eb4e48
1 parent f6258ff
commit 7eb4e48
Show file tree

Hide file tree

Showing 18 changed files with 222 additions and 80 deletions.
diff --git a/ci/jenkins/bin/autest.sh b/ci/jenkins/bin/autest.sh
@@ -19,8 +19,10 @@
 set +x
 
 cd src
+sleep 30
 
-if [ ! -z "$ghprbActualCommit" ]; then
+git branch --contains ${ghprbActualCommit} > /dev/null
+if [ $? = 0 -a ! -z "$ghprbActualCommit" ]; then
     git diff ${ghprbActualCommit}^...${ghprbActualCommit} --name-only | egrep -E '^(build|iocore|proxy|tests|include|mgmt|plugins|proxy|src)/' > /dev/null
     if [ $? = 1 ]; then
         echo "No relevant files changed, skipping run"
@@ -44,6 +46,9 @@ URL="https://ci.trafficserver.apache.org/autest"
 JOB_ID=${ghprbPullId:-${ATS_BRANCH:-master}}
 AUSB="ausb-${JOB_ID}.${BUILD_NUMBER}"
 SANDBOX="/var/tmp/${AUSB}"
+PROXY_VERIFIER_VERSIONS="/home/jenkins/proxy-verifier"
+PROXY_VERIFIER_VERSION_FILE="tests/proxy-verifier-version.txt"
+PROXY_VERIFIER_PREPARE="tests/prepare_proxy_verifier.sh"
 
 # Optional settings
 CCACHE=""
@@ -55,6 +60,7 @@ QUIC=""
 CURL=""
 AUTEST_DEBUG=""
 AUTEST_VERBOSE=""
+PROXY_VERIFIER_ARGUMENT=""
 
 [ "1" == "$enable_ccache" ] && CCACHE="--enable-ccache"
 [ "1" == "$enable_werror" ] && WERROR="--enable-werror"
@@ -84,7 +90,8 @@ set -x
 
 # Configure
 autoreconf -if
-./configure --prefix="${INSTALL}" \
+./configure \
+    --prefix="${INSTALL}" \
     --with-user=jenkins \
     --enable-experimental-plugins \
     --enable-example-plugins \
@@ -105,7 +112,28 @@ set +x
 echo -n "=======>>>>  Started on "
 date
 
-./tests/autest.sh --sandbox "$SANDBOX" --ats-bin "${INSTALL}/bin" $AUTEST_DEBUG $AUTEST_VERBOSE
+AUTEST="/usr/bin/autest"
+[ ! -x ${AUTEST} ] && AUTEST="/usr/local/bin/autest"
+set -x
+
+pv_version=""
+if [ -f "${PROXY_VERIFIER_VERSION_FILE}" ]; then
+  pv_version=`cat "${PROXY_VERIFIER_VERSION_FILE}"`
+elif [ -f "${PROXY_VERIFIER_PREPARE}" ]; then
+  pv_version=`awk -F'"' '/^pv_version/ {print $2}' "${PROXY_VERIFIER_PREPARE}"`
+fi
+if [ "x${pv_version}" != "x" ]; then
+  PROXY_VERIFIER_BIN="${PROXY_VERIFIER_VERSIONS}/${pv_version}/bin"
+  PROXY_VERIFIER_ARGUMENT="--proxy-verifier-bin ${PROXY_VERIFIER_BIN}"
+fi
+
+${AUTEST} \
+    -D ./tests/gold_tests \
+    --sandbox "$SANDBOX" \
+    --ats-bin "${INSTALL}/bin" \
+    $PROXY_VERIFIER_ARGUMENT \
+    $AUTEST_DEBUG \
+    $AUTEST_VERBOSE
 status=$?
 
 set +x

diff --git a/ci/jenkins/bin/build.sh b/ci/jenkins/bin/build.sh
@@ -28,24 +28,33 @@ CCACHE="--enable-ccache"
 # When to enable -Werror
 WERROR="--enable-werror"
 
+# Optional settings
+ASAN=""
+[ "1" == "$enable_asan" ] && ASAN="--enable-asan"
+
 echo "DEBUG: $DEBUG"
 echo "CCACHE: $CCACHE"
 echo "WERROR: $WERROR"
+echo "ASAN: $ASAN"
 
 # Change to the build area (this is previously setup in extract.sh)
-cd "${WORKSPACE}/${BUILD_NUMBER}/build"
+cd "${ATS_BUILD_BASEDIR}/build"
 mkdir -p BUILDS && cd BUILDS
 
 # Restore verbose shell output
 set -x
 
 ../configure \
-    --prefix="${WORKSPACE}/${BUILD_NUMBER}/install" \
+    --prefix="${ATS_BUILD_BASEDIR}/install" \
     --enable-experimental-plugins \
     --enable-example-plugins \
     --with-user=jenkins \
     ${CCACHE} \
     ${WERROR} \
-    ${DEBUG}
+    ${DEBUG} \
+    ${ASAN}
 
+echo
+echo -n "Main build started at " && date
 ${ATS_MAKE} ${ATS_MAKE_FLAGS} V=1 Q= || exit 1
+echo -n "Main build finished at " && date
diff --git a/ci/jenkins/bin/cache-tests.sh b/ci/jenkins/bin/cache-tests.sh
@@ -16,7 +16,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-PREFIX="${WORKSPACE}/${BUILD_NUMBER}/install"
+PREFIX="${ATS_BUILD_BASEDIR}/install"
 REMAP="${PREFIX}/etc/trafficserver/remap.config"
 RECORDS="${PREFIX}/etc/trafficserver/records.config"
 
@@ -25,7 +25,7 @@ TWEAK=""
 
 # Change to the build area (this is previously setup in extract.sh)
 set +x
-cd "${WORKSPACE}/${BUILD_NUMBER}/build"
+cd "${ATS_BUILD_BASEDIR}/build"
 
 ./configure \
     --prefix=${PREFIX} \
@@ -37,19 +37,20 @@ ${ATS_MAKE} -i ${ATS_MAKE_FLAGS} V=1 Q=
 ${ATS_MAKE} -i install
 
 [ -x ${PREFIX}/bin/traffic_server ] || exit 1
+ldd  ${PREFIX}/bin/traffic_server
 
-# Get NPM v10
-source /opt/rh/rh-nodejs10/enable
+# Get NPM v12
+source /opt/rh/rh-nodejs12/enable
 
 # Setup and start ATS with the required remap rule
-echo "map http://127.0.0.1:8080 http://192.168.3.13:8000" >> $REMAP
+echo "map http://127.0.0.1:8080 http://192.168.3.1:8000" >> $REMAP
 
 ${PREFIX}/bin/trafficserver start
 
 set -x
 
 cd /home/jenkins/cache-tests
-npm run --silent cli --base=http://127.0.0.1:8080/ > /CA/cache-tests/${ATS_BRANCH}.json
+npm run --silent cli --base=http://127.0.0.1:8080 > /CA/cache-tests/${ATS_BRANCH}.json
 cat /CA/cache-tests/${ATS_BRANCH}.json
 
 ${PREFIX}/bin/trafficserver stop
@@ -61,7 +62,7 @@ if [ "" != "$TWEAK" ]; then
     echo "CONFIG proxy.config.http.negative_caching_enabled INT 1" >> $RECORDS
     ${PREFIX}/bin/trafficserver start
     cd /home/jenkins/cache-tests
-    npm run --silent cli --base=http://127.0.0.1:8080/ > /CA/cache-tests/${ATS_BRANCH}${TWEAK}.json
+    npm run --silent cli --base=http://127.0.0.1:8080 > /CA/cache-tests/${ATS_BRANCH}${TWEAK}.json
     echo "TWEAKED RESULTS"
     cat /CA/cache-tests/${ATS_BRANCH}${TWEAK}.json
 

diff --git a/ci/jenkins/bin/clang-analyzer.sh b/ci/jenkins/bin/clang-analyzer.sh
@@ -38,9 +38,6 @@ checkers="-enable-checker alpha.unix.cstring.BufferOverlap \
 test -z "${ATS_MAKE}" && ATS_MAKE="make"
 test ! -z "${WORKSPACE}" && cd "${WORKSPACE}/src"
 
-# Check to see if this is a Github PR build (so not a github branch per-se)
-test "${JOB_NAME#*-github}" != "${JOB_NAME}" && ATS_BRANCH="github"
-
 # Where to store the results, special case for the CI
 output="/tmp"
 
@@ -50,7 +47,7 @@ if [ "${JOB_NAME#*-github}" != "${JOB_NAME}" ]; then
     ATS_BRANCH="github"
     if [ -w "${OUTPUT_BASE}/${ATS_BRANCH}" ]; then
         output="${OUTPUT_BASE}/${ATS_BRANCH}/${ghprbPullId}"
-        [ ! -d "${output}"] && mkdir "${output}"
+        [ ! -d "${output}" ] && mkdir "${output}"
     fi
     github_pr=" PR #${ghprbPullId}"
     results_url="https://ci.trafficserver.apache.org/clang-analyzer/${ATS_BRANCH}/${ghprbPullId}/"
@@ -103,6 +100,3 @@ fi
 if [ -x "/admin/bin/clean-clang.sh" ]; then
     /admin/bin/clean-clang.sh
 fi
-
-# Exit with the scan-build exit code (thanks to --status-bugs)
-exit $status
diff --git a/ci/jenkins/bin/clang-format.sh b/ci/jenkins/bin/clang-format.sh
@@ -16,18 +16,43 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+set +x
 cd "${WORKSPACE}/src"
-autoreconf -if && ./configure
-[ "0" != "$?" ] && exit -1
 
-${ATS_MAKE} -j clang-format
-[ "0" != "$?" ] && exit -1
+# First, make sure there are no trailing WS!!!
+git grep -IE ' +$' | fgrep -v '.gold:'
+if [ "1" != "$?" ]; then
+    echo "Error: Trailing whitespaces are not allowed!"
+    echo "Error: Please run: git grep -IE ' +$'"
+    exit 1
+fi
+echo "Success! No trailing whitespace"
+
+# Make sure there are no DOS shit here.
+git grep -IE $'\r$' | fgrep -v 'lib/yamlcpp'
+if [ "1" != "$?" ]; then
+    echo "Error: Please make sure to run dos2unix on the above file(s)"
+    exit 1
+fi
+echo "Success! No DOS carriage return"
+
+set -x
+
+autoreconf -if && ./configure
+[ "0" != "$?" ] && exit 1
 
-${ATS_MAKE} autopep8
+${ATS_MAKE} clang-format
 [ "0" != "$?" ] && exit 1
 
+# Only enforce autopep8 on branches where the pre-commit hook was updated to
+# check it. Otherwise, none of the PRs for older branches will pass this check.
+if grep -q autopep8 tools/git/pre-commit; then
+    ${ATS_MAKE} autopep8
+    [ "0" != "$?" ] && exit 1
+fi
+
 git diff --exit-code
-[ "0" != "$?" ] && exit -1
+[ "0" != "$?" ] && exit 1
 
 # Normal exit
 exit 0
diff --git a/ci/jenkins/bin/cleanup.sh b/ci/jenkins/bin/cleanup.sh
@@ -17,11 +17,10 @@
 #  limitations under the License.
 
 # Do a distclean, to verify that we can actually satisfy this (common) build target
-cd "${WORKSPACE}/${BUILD_NUMBER}/build"
+cd "${ATS_BUILD_BASEDIR}/build"
 [ -d BUILDS ] && cd BUILDS
 
 ${ATS_MAKE} distclean
 
-# Final cleanup, this removes the build and install areas
-cd "${WORKSPACE}"
-rm -rf ${BUILD_NUMBER}
+# Final cleanup, this removes the build and install areas, not really needed nor right for docker builds
+cd "${WORKSPACE}" && rm -rf ${BUILD_NUMBER}
diff --git a/ci/jenkins/bin/docs.sh b/ci/jenkins/bin/docs.sh
@@ -20,6 +20,17 @@
 test -z "${ATS_MAKE}" && ATS_MAKE="make"
 test ! -z "${WORKSPACE}" && cd "${WORKSPACE}/src"
 
+# Skip if nothing in doc has changed
+INCLUDE_FILES=$(for i in $(git grep literalinclude doc/ | awk '{print $3}'); do basename $i; done | sort -u | paste -sd\|)
+echo $INCLUDE_FILES
+if [ ! -z "$ghprbActualCommit" ]; then
+    git diff ${ghprbActualCommit}^...${ghprbActualCommit} --name-only | egrep -E "(^doc/|$INCLUDE_FILES)" > /dev/null
+    if [ $? = 1 ]; then
+        echo "No relevant files changed, skipping run"
+        exit 0
+    fi
+fi
+
 # Run configure on the docs builds each time in case there have been updates
 autoreconf -fi && ./configure --enable-docs || exit 1
 
@@ -30,7 +41,7 @@ rm -rf docbuild/html
 ${ATS_MAKE} -e SPHINXOPTS="-D language='en'" html
 [ $? != 0 ] && exit 1
 
-# Only continue with the rsync and JA build if we're on the official docs updates.
+# Only continue with the rsync and JA build if we're on the official docs updates
 [ -w /home/docs ] || exit 0
 
 /usr/bin/rsync --delete -av docbuild/html/ /home/docs/en/${ATS_BRANCH}

diff --git a/ci/jenkins/bin/environment.sh b/ci/jenkins/bin/environment.sh
@@ -18,11 +18,17 @@
 
 # Show which platform we're actually building on
 set +x
-echo -n "Build platform: "
 
+# Deduct if this build is on a docker instance
+IS_DOCKER="no"
+df / | fgrep -q overlay && IS_DOCKER="yes"
+export IS_DOCKER
+
+echo -n "Build platform: "
 [ -f /etc/lsb-release ] && grep DISTRIB_RELEASE /etc/lsb-release
 [ -f /etc/debian_version ] && cat /etc/debian_version
 [ -f /etc/redhat-release ] && cat /etc/redhat-release
+echo "Build on Docker: " $IS_DOCKER
 
 # Shouldn't have to tweak this
 export ATS_SRC_HOME="/home/jenkins/src"
@@ -40,15 +46,15 @@ export TODAY=$(/bin/date +'%m%d%Y')
 
 # Extract the current branch (default to master). ToDo: Can we do this better ?
 ATS_BRANCH=master
-ATS_IS_7="yes"
 
-test "${JOB_NAME#*-5.3.x}" != "${JOB_NAME}" && ATS_BRANCH=5.3.x && ATS_IS_7="no"
-test "${JOB_NAME#*-6.2.x}" != "${JOB_NAME}" && ATS_BRANCH=6.2.x && ATS_IS_7="no"
+# Make sure to leave these, for the HTTP cache tests
+test "${JOB_NAME#*-5.3.x}" != "${JOB_NAME}" && ATS_BRANCH=5.3.x
+test "${JOB_NAME#*-6.2.x}" != "${JOB_NAME}" && ATS_BRANCH=6.2.x
+
+# These should be maintained and cleaned up as needed.
 test "${JOB_NAME#*-7.1.x}" != "${JOB_NAME}" && ATS_BRANCH=7.1.x
 test "${JOB_NAME#*-8.0.x}" != "${JOB_NAME}" && ATS_BRANCH=8.0.x
 test "${JOB_NAME#*-8.1.x}" != "${JOB_NAME}" && ATS_BRANCH=8.1.x
-test "${JOB_NAME#*-8.2.x}" != "${JOB_NAME}" && ATS_BRANCH=8.2.x
-test "${JOB_NAME#*-8.3.x}" != "${JOB_NAME}" && ATS_BRANCH=8.3.x
 test "${JOB_NAME#*-9.0.x}" != "${JOB_NAME}" && ATS_BRANCH=9.0.x
 test "${JOB_NAME#*-9.1.x}" != "${JOB_NAME}" && ATS_BRANCH=9.1.x
 test "${JOB_NAME#*-9.2.x}" != "${JOB_NAME}" && ATS_BRANCH=9.2.x
@@ -58,6 +64,9 @@ test "${JOB_NAME#*-10.1.x}" != "${JOB_NAME}" && ATS_BRANCH=10.1.x
 test "${JOB_NAME#*-10.2.x}" != "${JOB_NAME}" && ATS_BRANCH=10.2.x
 test "${JOB_NAME#*-10.3.x}" != "${JOB_NAME}" && ATS_BRANCH=10.3.x
 
+# Special case for the full build of clang analyzer
+test "${JOB_NAME}" == "clang-analyzer-full" && ATS_BRANCH=FULL
+
 export ATS_BRANCH
 echo "Branch is $ATS_BRANCH"
 
@@ -89,17 +98,14 @@ else
     # Default is gcc / g++
     export CC=gcc
     export CXX=g++
-    # Only test for non standard compilers on ATS v7.x and later. ToDo: Remove this when 6.x is EOLifed.
-    if test "$ATS_IS_7" == "yes"; then
-        if test -f "/opt/rh/devtoolset-7/enable"; then
-            # This changes the path such that gcc / g++ is the right version. This is for CentOS 6 / 7.
-            source /opt/rh/devtoolset-7/enable
-            echo "Enabling devtoolset-7"
-        elif test -x "/usr/bin/g++-7"; then
-            # This is for Debian platforms
-            export CC=/usr/bin/gcc-7
-            export CXX=/usr/bin/g++-7
-        fi
+    if test -f "/opt/rh/devtoolset-7/enable"; then
+        # This changes the path such that gcc / g++ is the right version. This is for CentOS 6 / 7.
+        source /opt/rh/devtoolset-7/enable
+        echo "Enabling devtoolset-7"
+    elif test -x "/usr/bin/g++-7"; then
+        # This is for Debian platforms
+        export CC=/usr/bin/gcc-7
+        export CXX=/usr/bin/g++-7
     fi
 fi
 
@@ -111,7 +117,16 @@ echo "CXX: $CXX"
 $CXX -v
 
 # Figure out parallelism for regular builds / bots
-ATS_MAKE_FLAGS="-j4"
+export ATS_MAKE_FLAGS="-j6"
+if [ "yes" == "$IS_DOCKER" ]; then
+    export ATS_BUILD_BASEDIR="${WORKSPACE}"
+else
+    export ATS_BUILD_BASEDIR="${WORKSPACE}/${BUILD_NUMBER}"
+fi
+
+# ccache settings
+export CCACHE_BASEDIR=${ATS_BUILD_BASEDIR}
+#export CCACHE_COMPRESS=true
 
 # Restore verbose shell output
 set -x
diff --git a/ci/jenkins/bin/extract.sh b/ci/jenkins/bin/extract.sh
@@ -18,16 +18,16 @@
 
 set +x
 # Setup the build and install area for this build
-mkdir -p "${WORKSPACE}/${BUILD_NUMBER}/build"
-mkdir -p "${WORKSPACE}/${BUILD_NUMBER}/install"
+mkdir -p "${ATS_BUILD_BASEDIR}/build"
+mkdir -p "${ATS_BUILD_BASEDIR}/install"
 
-cd "${WORKSPACE}/${BUILD_NUMBER}/build"
+cd "${ATS_BUILD_BASEDIR}/build"
 echo "Artifact: ${ATS_SRC_HOME}/trafficserver-${ATS_BRANCH}.tar.bz2"
 tar xf ${ATS_SRC_HOME}/trafficserver-${ATS_BRANCH}.tar.bz2
 mv trafficserver-*/* .
 
-echo "build: ${WORKSPACE}/${BUILD_NUMBER}/build"
-echo "install: ${WORKSPACE}/${BUILD_NUMBER}/install"
+echo "build: ${ATS_BUILD_BASEDIR}/build"
+echo "install: ${ATS_BUILD_BASEDIR}/install"
 
 # Restore verbose shell output
 set -x