diff --git a/ci/jenkins/bin/autest.sh b/ci/jenkins/bin/autest.sh index 7df0372b1e5..7ec18eec345 100755 --- a/ci/jenkins/bin/autest.sh +++ b/ci/jenkins/bin/autest.sh @@ -19,8 +19,10 @@ set +x cd src +sleep 30 -if [ ! -z "$ghprbActualCommit" ]; then +git branch --contains ${ghprbActualCommit} > /dev/null +if [ $? = 0 -a ! -z "$ghprbActualCommit" ]; then git diff ${ghprbActualCommit}^...${ghprbActualCommit} --name-only | egrep -E '^(build|iocore|proxy|tests|include|mgmt|plugins|proxy|src)/' > /dev/null if [ $? = 1 ]; then echo "No relevant files changed, skipping run" @@ -44,6 +46,9 @@ URL="https://ci.trafficserver.apache.org/autest" JOB_ID=${ghprbPullId:-${ATS_BRANCH:-master}} AUSB="ausb-${JOB_ID}.${BUILD_NUMBER}" SANDBOX="/var/tmp/${AUSB}" +PROXY_VERIFIER_VERSIONS="/home/jenkins/proxy-verifier" +PROXY_VERIFIER_VERSION_FILE="tests/proxy-verifier-version.txt" +PROXY_VERIFIER_PREPARE="tests/prepare_proxy_verifier.sh" # Optional settings CCACHE="" @@ -55,6 +60,7 @@ QUIC="" CURL="" AUTEST_DEBUG="" AUTEST_VERBOSE="" +PROXY_VERIFIER_ARGUMENT="" [ "1" == "$enable_ccache" ] && CCACHE="--enable-ccache" [ "1" == "$enable_werror" ] && WERROR="--enable-werror" @@ -84,7 +90,8 @@ set -x # Configure autoreconf -if -./configure --prefix="${INSTALL}" \ +./configure \ + --prefix="${INSTALL}" \ --with-user=jenkins \ --enable-experimental-plugins \ --enable-example-plugins \ @@ -105,7 +112,28 @@ set +x echo -n "=======>>>> Started on " date -./tests/autest.sh --sandbox "$SANDBOX" --ats-bin "${INSTALL}/bin" $AUTEST_DEBUG $AUTEST_VERBOSE +AUTEST="/usr/bin/autest" +[ ! -x ${AUTEST} ] && AUTEST="/usr/local/bin/autest" +set -x + +pv_version="" +if [ -f "${PROXY_VERIFIER_VERSION_FILE}" ]; then + pv_version=`cat "${PROXY_VERIFIER_VERSION_FILE}"` +elif [ -f "${PROXY_VERIFIER_PREPARE}" ]; then + pv_version=`awk -F'"' '/^pv_version/ {print $2}' "${PROXY_VERIFIER_PREPARE}"` +fi +if [ "x${pv_version}" != "x" ]; then + PROXY_VERIFIER_BIN="${PROXY_VERIFIER_VERSIONS}/${pv_version}/bin" + PROXY_VERIFIER_ARGUMENT="--proxy-verifier-bin ${PROXY_VERIFIER_BIN}" +fi + +${AUTEST} \ + -D ./tests/gold_tests \ + --sandbox "$SANDBOX" \ + --ats-bin "${INSTALL}/bin" \ + $PROXY_VERIFIER_ARGUMENT \ + $AUTEST_DEBUG \ + $AUTEST_VERBOSE status=$? set +x diff --git a/ci/jenkins/bin/build.sh b/ci/jenkins/bin/build.sh index 9a9fd354a01..93d23282f2c 100755 --- a/ci/jenkins/bin/build.sh +++ b/ci/jenkins/bin/build.sh @@ -28,24 +28,33 @@ CCACHE="--enable-ccache" # When to enable -Werror WERROR="--enable-werror" +# Optional settings +ASAN="" +[ "1" == "$enable_asan" ] && ASAN="--enable-asan" + echo "DEBUG: $DEBUG" echo "CCACHE: $CCACHE" echo "WERROR: $WERROR" +echo "ASAN: $ASAN" # Change to the build area (this is previously setup in extract.sh) -cd "${WORKSPACE}/${BUILD_NUMBER}/build" +cd "${ATS_BUILD_BASEDIR}/build" mkdir -p BUILDS && cd BUILDS # Restore verbose shell output set -x ../configure \ - --prefix="${WORKSPACE}/${BUILD_NUMBER}/install" \ + --prefix="${ATS_BUILD_BASEDIR}/install" \ --enable-experimental-plugins \ --enable-example-plugins \ --with-user=jenkins \ ${CCACHE} \ ${WERROR} \ - ${DEBUG} + ${DEBUG} \ + ${ASAN} +echo +echo -n "Main build started at " && date ${ATS_MAKE} ${ATS_MAKE_FLAGS} V=1 Q= || exit 1 +echo -n "Main build finished at " && date diff --git a/ci/jenkins/bin/cache-tests.sh b/ci/jenkins/bin/cache-tests.sh index 5ca9b55725d..21771f94f9d 100755 --- a/ci/jenkins/bin/cache-tests.sh +++ b/ci/jenkins/bin/cache-tests.sh @@ -16,7 +16,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -PREFIX="${WORKSPACE}/${BUILD_NUMBER}/install" +PREFIX="${ATS_BUILD_BASEDIR}/install" REMAP="${PREFIX}/etc/trafficserver/remap.config" RECORDS="${PREFIX}/etc/trafficserver/records.config" @@ -25,7 +25,7 @@ TWEAK="" # Change to the build area (this is previously setup in extract.sh) set +x -cd "${WORKSPACE}/${BUILD_NUMBER}/build" +cd "${ATS_BUILD_BASEDIR}/build" ./configure \ --prefix=${PREFIX} \ @@ -37,19 +37,20 @@ ${ATS_MAKE} -i ${ATS_MAKE_FLAGS} V=1 Q= ${ATS_MAKE} -i install [ -x ${PREFIX}/bin/traffic_server ] || exit 1 +ldd ${PREFIX}/bin/traffic_server -# Get NPM v10 -source /opt/rh/rh-nodejs10/enable +# Get NPM v12 +source /opt/rh/rh-nodejs12/enable # Setup and start ATS with the required remap rule -echo "map http://127.0.0.1:8080 http://192.168.3.13:8000" >> $REMAP +echo "map http://127.0.0.1:8080 http://192.168.3.1:8000" >> $REMAP ${PREFIX}/bin/trafficserver start set -x cd /home/jenkins/cache-tests -npm run --silent cli --base=http://127.0.0.1:8080/ > /CA/cache-tests/${ATS_BRANCH}.json +npm run --silent cli --base=http://127.0.0.1:8080 > /CA/cache-tests/${ATS_BRANCH}.json cat /CA/cache-tests/${ATS_BRANCH}.json ${PREFIX}/bin/trafficserver stop @@ -61,7 +62,7 @@ if [ "" != "$TWEAK" ]; then echo "CONFIG proxy.config.http.negative_caching_enabled INT 1" >> $RECORDS ${PREFIX}/bin/trafficserver start cd /home/jenkins/cache-tests - npm run --silent cli --base=http://127.0.0.1:8080/ > /CA/cache-tests/${ATS_BRANCH}${TWEAK}.json + npm run --silent cli --base=http://127.0.0.1:8080 > /CA/cache-tests/${ATS_BRANCH}${TWEAK}.json echo "TWEAKED RESULTS" cat /CA/cache-tests/${ATS_BRANCH}${TWEAK}.json diff --git a/ci/jenkins/bin/clang-analyzer.sh b/ci/jenkins/bin/clang-analyzer.sh index c2ebb423833..5334b984f04 100755 --- a/ci/jenkins/bin/clang-analyzer.sh +++ b/ci/jenkins/bin/clang-analyzer.sh @@ -38,9 +38,6 @@ checkers="-enable-checker alpha.unix.cstring.BufferOverlap \ test -z "${ATS_MAKE}" && ATS_MAKE="make" test ! -z "${WORKSPACE}" && cd "${WORKSPACE}/src" -# Check to see if this is a Github PR build (so not a github branch per-se) -test "${JOB_NAME#*-github}" != "${JOB_NAME}" && ATS_BRANCH="github" - # Where to store the results, special case for the CI output="/tmp" @@ -50,7 +47,7 @@ if [ "${JOB_NAME#*-github}" != "${JOB_NAME}" ]; then ATS_BRANCH="github" if [ -w "${OUTPUT_BASE}/${ATS_BRANCH}" ]; then output="${OUTPUT_BASE}/${ATS_BRANCH}/${ghprbPullId}" - [ ! -d "${output}"] && mkdir "${output}" + [ ! -d "${output}" ] && mkdir "${output}" fi github_pr=" PR #${ghprbPullId}" results_url="https://ci.trafficserver.apache.org/clang-analyzer/${ATS_BRANCH}/${ghprbPullId}/" @@ -103,6 +100,3 @@ fi if [ -x "/admin/bin/clean-clang.sh" ]; then /admin/bin/clean-clang.sh fi - -# Exit with the scan-build exit code (thanks to --status-bugs) -exit $status diff --git a/ci/jenkins/bin/clang-format.sh b/ci/jenkins/bin/clang-format.sh index 44e7c971932..dd37eca27b8 100755 --- a/ci/jenkins/bin/clang-format.sh +++ b/ci/jenkins/bin/clang-format.sh @@ -16,18 +16,43 @@ # See the License for the specific language governing permissions and # limitations under the License. +set +x cd "${WORKSPACE}/src" -autoreconf -if && ./configure -[ "0" != "$?" ] && exit -1 -${ATS_MAKE} -j clang-format -[ "0" != "$?" ] && exit -1 +# First, make sure there are no trailing WS!!! +git grep -IE ' +$' | fgrep -v '.gold:' +if [ "1" != "$?" ]; then + echo "Error: Trailing whitespaces are not allowed!" + echo "Error: Please run: git grep -IE ' +$'" + exit 1 +fi +echo "Success! No trailing whitespace" + +# Make sure there are no DOS shit here. +git grep -IE $'\r$' | fgrep -v 'lib/yamlcpp' +if [ "1" != "$?" ]; then + echo "Error: Please make sure to run dos2unix on the above file(s)" + exit 1 +fi +echo "Success! No DOS carriage return" + +set -x + +autoreconf -if && ./configure +[ "0" != "$?" ] && exit 1 -${ATS_MAKE} autopep8 +${ATS_MAKE} clang-format [ "0" != "$?" ] && exit 1 +# Only enforce autopep8 on branches where the pre-commit hook was updated to +# check it. Otherwise, none of the PRs for older branches will pass this check. +if grep -q autopep8 tools/git/pre-commit; then + ${ATS_MAKE} autopep8 + [ "0" != "$?" ] && exit 1 +fi + git diff --exit-code -[ "0" != "$?" ] && exit -1 +[ "0" != "$?" ] && exit 1 # Normal exit exit 0 diff --git a/ci/jenkins/bin/cleanup.sh b/ci/jenkins/bin/cleanup.sh index 9fa5a07b83d..39c3de03f4b 100755 --- a/ci/jenkins/bin/cleanup.sh +++ b/ci/jenkins/bin/cleanup.sh @@ -17,11 +17,10 @@ # limitations under the License. # Do a distclean, to verify that we can actually satisfy this (common) build target -cd "${WORKSPACE}/${BUILD_NUMBER}/build" +cd "${ATS_BUILD_BASEDIR}/build" [ -d BUILDS ] && cd BUILDS ${ATS_MAKE} distclean -# Final cleanup, this removes the build and install areas -cd "${WORKSPACE}" -rm -rf ${BUILD_NUMBER} +# Final cleanup, this removes the build and install areas, not really needed nor right for docker builds +cd "${WORKSPACE}" && rm -rf ${BUILD_NUMBER} diff --git a/ci/jenkins/bin/docs.sh b/ci/jenkins/bin/docs.sh index 6aa2d92fcaa..5af6ae54f09 100755 --- a/ci/jenkins/bin/docs.sh +++ b/ci/jenkins/bin/docs.sh @@ -20,6 +20,17 @@ test -z "${ATS_MAKE}" && ATS_MAKE="make" test ! -z "${WORKSPACE}" && cd "${WORKSPACE}/src" +# Skip if nothing in doc has changed +INCLUDE_FILES=$(for i in $(git grep literalinclude doc/ | awk '{print $3}'); do basename $i; done | sort -u | paste -sd\|) +echo $INCLUDE_FILES +if [ ! -z "$ghprbActualCommit" ]; then + git diff ${ghprbActualCommit}^...${ghprbActualCommit} --name-only | egrep -E "(^doc/|$INCLUDE_FILES)" > /dev/null + if [ $? = 1 ]; then + echo "No relevant files changed, skipping run" + exit 0 + fi +fi + # Run configure on the docs builds each time in case there have been updates autoreconf -fi && ./configure --enable-docs || exit 1 @@ -30,7 +41,7 @@ rm -rf docbuild/html ${ATS_MAKE} -e SPHINXOPTS="-D language='en'" html [ $? != 0 ] && exit 1 -# Only continue with the rsync and JA build if we're on the official docs updates. +# Only continue with the rsync and JA build if we're on the official docs updates [ -w /home/docs ] || exit 0 /usr/bin/rsync --delete -av docbuild/html/ /home/docs/en/${ATS_BRANCH} diff --git a/ci/jenkins/bin/environment.sh b/ci/jenkins/bin/environment.sh index e76cf55522e..b3559ebe810 100755 --- a/ci/jenkins/bin/environment.sh +++ b/ci/jenkins/bin/environment.sh @@ -18,11 +18,17 @@ # Show which platform we're actually building on set +x -echo -n "Build platform: " +# Deduct if this build is on a docker instance +IS_DOCKER="no" +df / | fgrep -q overlay && IS_DOCKER="yes" +export IS_DOCKER + +echo -n "Build platform: " [ -f /etc/lsb-release ] && grep DISTRIB_RELEASE /etc/lsb-release [ -f /etc/debian_version ] && cat /etc/debian_version [ -f /etc/redhat-release ] && cat /etc/redhat-release +echo "Build on Docker: " $IS_DOCKER # Shouldn't have to tweak this export ATS_SRC_HOME="/home/jenkins/src" @@ -40,15 +46,15 @@ export TODAY=$(/bin/date +'%m%d%Y') # Extract the current branch (default to master). ToDo: Can we do this better ? ATS_BRANCH=master -ATS_IS_7="yes" -test "${JOB_NAME#*-5.3.x}" != "${JOB_NAME}" && ATS_BRANCH=5.3.x && ATS_IS_7="no" -test "${JOB_NAME#*-6.2.x}" != "${JOB_NAME}" && ATS_BRANCH=6.2.x && ATS_IS_7="no" +# Make sure to leave these, for the HTTP cache tests +test "${JOB_NAME#*-5.3.x}" != "${JOB_NAME}" && ATS_BRANCH=5.3.x +test "${JOB_NAME#*-6.2.x}" != "${JOB_NAME}" && ATS_BRANCH=6.2.x + +# These should be maintained and cleaned up as needed. test "${JOB_NAME#*-7.1.x}" != "${JOB_NAME}" && ATS_BRANCH=7.1.x test "${JOB_NAME#*-8.0.x}" != "${JOB_NAME}" && ATS_BRANCH=8.0.x test "${JOB_NAME#*-8.1.x}" != "${JOB_NAME}" && ATS_BRANCH=8.1.x -test "${JOB_NAME#*-8.2.x}" != "${JOB_NAME}" && ATS_BRANCH=8.2.x -test "${JOB_NAME#*-8.3.x}" != "${JOB_NAME}" && ATS_BRANCH=8.3.x test "${JOB_NAME#*-9.0.x}" != "${JOB_NAME}" && ATS_BRANCH=9.0.x test "${JOB_NAME#*-9.1.x}" != "${JOB_NAME}" && ATS_BRANCH=9.1.x test "${JOB_NAME#*-9.2.x}" != "${JOB_NAME}" && ATS_BRANCH=9.2.x @@ -58,6 +64,9 @@ test "${JOB_NAME#*-10.1.x}" != "${JOB_NAME}" && ATS_BRANCH=10.1.x test "${JOB_NAME#*-10.2.x}" != "${JOB_NAME}" && ATS_BRANCH=10.2.x test "${JOB_NAME#*-10.3.x}" != "${JOB_NAME}" && ATS_BRANCH=10.3.x +# Special case for the full build of clang analyzer +test "${JOB_NAME}" == "clang-analyzer-full" && ATS_BRANCH=FULL + export ATS_BRANCH echo "Branch is $ATS_BRANCH" @@ -89,17 +98,14 @@ else # Default is gcc / g++ export CC=gcc export CXX=g++ - # Only test for non standard compilers on ATS v7.x and later. ToDo: Remove this when 6.x is EOLifed. - if test "$ATS_IS_7" == "yes"; then - if test -f "/opt/rh/devtoolset-7/enable"; then - # This changes the path such that gcc / g++ is the right version. This is for CentOS 6 / 7. - source /opt/rh/devtoolset-7/enable - echo "Enabling devtoolset-7" - elif test -x "/usr/bin/g++-7"; then - # This is for Debian platforms - export CC=/usr/bin/gcc-7 - export CXX=/usr/bin/g++-7 - fi + if test -f "/opt/rh/devtoolset-7/enable"; then + # This changes the path such that gcc / g++ is the right version. This is for CentOS 6 / 7. + source /opt/rh/devtoolset-7/enable + echo "Enabling devtoolset-7" + elif test -x "/usr/bin/g++-7"; then + # This is for Debian platforms + export CC=/usr/bin/gcc-7 + export CXX=/usr/bin/g++-7 fi fi @@ -111,7 +117,16 @@ echo "CXX: $CXX" $CXX -v # Figure out parallelism for regular builds / bots -ATS_MAKE_FLAGS="-j4" +export ATS_MAKE_FLAGS="-j6" +if [ "yes" == "$IS_DOCKER" ]; then + export ATS_BUILD_BASEDIR="${WORKSPACE}" +else + export ATS_BUILD_BASEDIR="${WORKSPACE}/${BUILD_NUMBER}" +fi + +# ccache settings +export CCACHE_BASEDIR=${ATS_BUILD_BASEDIR} +#export CCACHE_COMPRESS=true # Restore verbose shell output set -x diff --git a/ci/jenkins/bin/extract.sh b/ci/jenkins/bin/extract.sh index 5ee733d2ea1..9587825b748 100755 --- a/ci/jenkins/bin/extract.sh +++ b/ci/jenkins/bin/extract.sh @@ -18,16 +18,16 @@ set +x # Setup the build and install area for this build -mkdir -p "${WORKSPACE}/${BUILD_NUMBER}/build" -mkdir -p "${WORKSPACE}/${BUILD_NUMBER}/install" +mkdir -p "${ATS_BUILD_BASEDIR}/build" +mkdir -p "${ATS_BUILD_BASEDIR}/install" -cd "${WORKSPACE}/${BUILD_NUMBER}/build" +cd "${ATS_BUILD_BASEDIR}/build" echo "Artifact: ${ATS_SRC_HOME}/trafficserver-${ATS_BRANCH}.tar.bz2" tar xf ${ATS_SRC_HOME}/trafficserver-${ATS_BRANCH}.tar.bz2 mv trafficserver-*/* . -echo "build: ${WORKSPACE}/${BUILD_NUMBER}/build" -echo "install: ${WORKSPACE}/${BUILD_NUMBER}/install" +echo "build: ${ATS_BUILD_BASEDIR}/build" +echo "install: ${ATS_BUILD_BASEDIR}/install" # Restore verbose shell output set -x diff --git a/ci/jenkins/bin/gh-mirror.sh b/ci/jenkins/bin/gh-mirror.sh index 3f03fc6a924..9bf2e3e5f6e 100755 --- a/ci/jenkins/bin/gh-mirror.sh +++ b/ci/jenkins/bin/gh-mirror.sh @@ -69,7 +69,8 @@ function checkBuild() { REF_6_2=$(getRef "6.2.x") REF_7_1=$(getRef "7.1.x") REF_8_0=$(getRef "8.0.x") -#REF_8_1=$(getRef "8.1.x") +REF_8_1=$(getRef "8.1.x") +REF_9_0=$(getRef "9.0.x") REF_master=$(getRef "master") # Do the updates @@ -80,5 +81,6 @@ ${GIT} update-server-info checkBuild "$REF_6_2" "6.2.x" checkBuild "$REF_7_1" "7.1.x" checkBuild "$REF_8_0" "8.0.x" -#checkBuild "$REF_8_1" "8.1.x" +checkBuild "$REF_8_1" "8.1.x" +checkBuild "$REF_9_0" "9.0.x" checkBuild "$REF_master" "master" diff --git a/ci/jenkins/bin/github.sh b/ci/jenkins/bin/github.sh index 82271fe13d6..054d98e956d 100755 --- a/ci/jenkins/bin/github.sh +++ b/ci/jenkins/bin/github.sh @@ -18,7 +18,7 @@ set +x -INSTALL="${WORKSPACE}/${BUILD_NUMBER}/install" +INSTALL="${ATS_BUILD_BASEDIR}/install" # Optional settings CCACHE="" @@ -26,11 +26,13 @@ WERROR="" DEBUG="" WCCP="" LUAJIT="" +QUIC="" [ "1" == "$enable_ccache" ] && CCACHE="--enable-ccache" [ "1" == "$enable_werror" ] && WERROR="--enable-werror" [ "1" == "$enable_debug" ] && DEBUG="--enable-debug" [ "1" == "$enable_wccp" ] && WCCP="--enable-wccp" [ "1" == "$enable_luajit" ] && LUAJIT="--enable-luajit" +[ "1" == "$enable_quic" ] && QUIC="--with-openssl=/opt/openssl-quic" mkdir -p ${INSTALL} cd src @@ -40,6 +42,7 @@ echo "WERROR: $WERROR" echo "DEBUG: $DEBUG" echo "WCCP: $WCCP" echo "LUAJIT: $LUAJIT" +echo "QUIC: $QUIC" # Restore verbose shell output set -x @@ -54,14 +57,25 @@ autoreconf -if ${CCACHE} \ ${WCCP} \ ${LUAJIT} \ + ${QUIC} \ ${WERROR} \ ${DEBUG} # Build and run regressions -${ATS_MAKE} ${ATS_MAKE_FLAGS} V=1 Q= -${ATS_MAKE} check VERBOSE=Y V=1 && ${ATS_MAKE} install +echo +echo -n "Main build started at " && date +${ATS_MAKE} ${ATS_MAKE_FLAGS} V=1 Q= || exit 1 +echo -n "Main build finished at " && date +echo +echo -n "Unit tests started at " && date +${ATS_MAKE} -j 2 check VERBOSE=Y V=1 && ${ATS_MAKE} install +echo -n "Unit tests finished at " && date -${INSTALL}/bin/traffic_server -K -k -R 1 -[ "0" != "$?" ] && exit -1 +[ -x ${INSTALL}/bin/traffic_server ] || exit 1 -exit 0 +echo +echo -n "Regression tests started at " && date +${INSTALL}/bin/traffic_server -K -k -R 1 +rval=$? +echo -n "Regression tests finished at " && date +exit $rval diff --git a/ci/jenkins/bin/regression.sh b/ci/jenkins/bin/regression.sh index 62c1f3237a5..d0bf711f40b 100755 --- a/ci/jenkins/bin/regression.sh +++ b/ci/jenkins/bin/regression.sh @@ -16,10 +16,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -cd "${WORKSPACE}/${BUILD_NUMBER}/build" +cd "${ATS_BUILD_BASEDIR}/build" [ -d BUILDS ] && cd BUILDS -${ATS_MAKE} check VERBOSE=Y V=1 || exit 1 +echo +echo -n "Unit tests started at " && date +${ATS_MAKE} -j 2 check VERBOSE=Y V=1 || exit 1 +echo -n "Unit tests finished at " && date +${ATS_MAKE} install || exit 1 -${ATS_MAKE} install || exit -"${WORKSPACE}/${BUILD_NUMBER}/install/bin/traffic_server" -k -K -R 1 +echo +echo -n "Regression tests started at " && date +"${ATS_BUILD_BASEDIR}/install/bin/traffic_server" -k -K -R 1 +rval=$? +echo -n "Regression tests finished at " && date +exit $rval diff --git a/tests/gold_tests/autest-site/conditions.test.ext b/tests/gold_tests/autest-site/conditions.test.ext index c9d3a0d3221..52af993d505 100644 --- a/tests/gold_tests/autest-site/conditions.test.ext +++ b/tests/gold_tests/autest-site/conditions.test.ext @@ -40,6 +40,10 @@ def HasCurlVersion(self, version): return self.EnsureVersion(["curl", "--version"], min_version=version) +def HasProxyVerifierVersion(self, version): + return self.EnsureVersion(["verifier-client", "--version"], min_version=version) + + def HasCurlFeature(self, feature): def default(output): diff --git a/tests/gold_tests/autest-site/setup.cli.ext b/tests/gold_tests/autest-site/setup.cli.ext index 14b14fd2c60..e752fa0e097 100644 --- a/tests/gold_tests/autest-site/setup.cli.ext +++ b/tests/gold_tests/autest-site/setup.cli.ext @@ -23,6 +23,8 @@ import pprint import subprocess import shutil +PROXY_VERIFIER_VERSION_FILENAME = 'proxy-verifier-version.txt' + test_root = dirname(dirname(AutestSitePath)) repo_root = dirname(test_root) @@ -38,6 +40,9 @@ else: host.WriteVerbose(['ats'], "Test build root: {}:".format(ENV['BUILD_ROOT'])) +proxy_verifer_version_file = os.path.join(test_root, PROXY_VERIFIER_VERSION_FILENAME) +proxy_verifer_version = open(proxy_verifer_version_file, "rt").read().strip() + if Arguments.proxy_verifier_bin is not None: ENV['VERIFIER_BIN'] = Arguments.proxy_verifier_bin host.WriteVerbose( @@ -47,7 +52,7 @@ if Arguments.proxy_verifier_bin is not None: else: # No Verifier bin path was specified. First see if a Proxy Verifier was # unpacked as a part of preparing for this test. - unpack_bin = os.path.join(test_root, 'proxy-verifier', 'unpack', 'bin') + unpack_bin = os.path.join(test_root, 'proxy-verifier', 'unpack', proxy_verifer_version, 'bin') if os.path.exists(os.path.join(unpack_bin, 'verifier-client')): ENV['VERIFIER_BIN'] = unpack_bin host.WriteVerbose( @@ -68,6 +73,20 @@ else: host.WriteError("Could not find Proxy Verifier binaries. " "Try running: ", prepare_proxy_verifier_path) +required_pv_version = Version(proxy_verifer_version[1:]) +verifier_client = os.path.join(ENV['VERIFIER_BIN'], 'verifier-client') +pv_version_out = subprocess.check_output([verifier_client, "--version"]) +pv_version = Version(pv_version_out.decode("utf-8").split()[1]) +if pv_version < required_pv_version: + host.WriteError( + f"Proxy Verifier at {verifier_client} is too old. " + f"Version required: {required_pv_version}, version found: {pv_version}") +else: + host.WriteVerbose( + ['ats'], + f"Proxy Verifier at {verifier_client} has version: {pv_version}") + + if ENV['ATS_BIN'] is not None: # Add variables for Tests traffic_layout = os.path.join(ENV['ATS_BIN'], "traffic_layout") diff --git a/tests/gold_tests/autest-site/verifier_client.test.ext b/tests/gold_tests/autest-site/verifier_client.test.ext index 41ef41a4745..9ff5c0311a6 100755 --- a/tests/gold_tests/autest-site/verifier_client.test.ext +++ b/tests/gold_tests/autest-site/verifier_client.test.ext @@ -24,7 +24,7 @@ from verifier_common import create_address_argument def _configure_client(obj, process, name, replay_path, http_ports=None, https_ports=None, keys=None, ssl_cert='', ca_cert='', - other_args='--verbose diag'): + verbose=True, other_args=''): """ Configure the process for running the verifier-client. @@ -61,12 +61,14 @@ def _configure_client(obj, process, name, replay_path, http_ports=None, if not http_ports: http_ports = [8080] if http_ports: + command += "--connect-http " command += create_address_argument(http_ports) command += " " if not https_ports: https_ports = [4443] if https_ports: + command += "--connect-https " command += create_address_argument(https_ports) command += " " @@ -94,6 +96,9 @@ def _configure_client(obj, process, name, replay_path, http_ports=None, process.Setup.Copy(ca_cert, run_ca_cert, CopyLogic.SoftFiles) command += ' --ca-certs "{}" '.format(run_ca_cert) + if verbose: + command += ' --verbose diag ' + if other_args: command += " {}".format(other_args) @@ -110,7 +115,7 @@ def _configure_client(obj, process, name, replay_path, http_ports=None, def AddVerifierClientProcess(run, name, replay_path, http_ports=None, https_ports=None, keys=None, ssl_cert='', ca_cert='', - other_args='--verbose diag'): + verbose=True, other_args=''): """ Set the Default process of the test run to a verifier-client Process. @@ -134,6 +139,8 @@ def AddVerifierClientProcess(run, name, replay_path, http_ports=None, ca_cert: (path) The location of the CA for HTTPS encryption. If this is not provided a stock CA will be used. + verbose: (bool) Whether to enable debug level logging. + other_args: (str) Any other arbitrary options to pass to verifier-client. Returns: @@ -143,7 +150,7 @@ def AddVerifierClientProcess(run, name, replay_path, http_ports=None, p = run.Processes.Default _configure_client(run, p, name, replay_path, http_ports, https_ports, - keys, ssl_cert, ca_cert, other_args) + keys, ssl_cert, ca_cert, verbose, other_args) return p diff --git a/tests/gold_tests/autest-site/verifier_server.test.ext b/tests/gold_tests/autest-site/verifier_server.test.ext index 3fd7f449c5d..eab23ab4ec8 100755 --- a/tests/gold_tests/autest-site/verifier_server.test.ext +++ b/tests/gold_tests/autest-site/verifier_server.test.ext @@ -23,7 +23,7 @@ from verifier_common import create_address_argument def _configure_server(obj, process, name, replay_path, http_ports=None, https_ports=None, - ssl_cert='', ca_cert='', other_args='--verbose diag'): + ssl_cert='', ca_cert='', verbose=True, other_args=''): """ Configure the provided process to run a verifier-server command. @@ -55,7 +55,7 @@ def _configure_server(obj, process, name, replay_path, http_ports=None, https_po process.Variables['http_port'] = http_ports[0] if len(http_ports) > 0: - command += "--listen " + command += "--listen-http " command += create_address_argument(http_ports) command += " " @@ -100,6 +100,9 @@ def _configure_server(obj, process, name, replay_path, http_ports=None, https_po process.Setup.Copy(replay_path, run_replay_path, CopyLogic.SoftFiles) command += "{} ".format(run_replay_path) + if verbose: + command += ' --verbose diag ' + if other_args: command += "{} ".format(other_args) process.Command = command @@ -114,7 +117,7 @@ def _configure_server(obj, process, name, replay_path, http_ports=None, https_po def MakeVerifierServerProcess(test, name, replay_path, http_ports=None, https_ports=None, ssl_cert='', ca_cert='', - other_args='--verbose diag'): + verbose=True, other_args=''): """ Create a verifier-server process for the Test. @@ -140,6 +143,8 @@ def MakeVerifierServerProcess(test, name, replay_path, http_ports=None, ca_cert: (path) The location of the CA for HTTPS encryption. If this is not provided a stock CA will be used. + verbose: (bool) Whether to enable debug level logging. + other_args: (str) Any other arbitrary options to pass to verifier-server. Raises: @@ -148,13 +153,13 @@ def MakeVerifierServerProcess(test, name, replay_path, http_ports=None, """ server = test.Processes.Process(name) _configure_server(test, server, name, replay_path, http_ports, https_ports, - ssl_cert, ca_cert, other_args) + ssl_cert, ca_cert, verbose, other_args) return server def AddVerifierServerProcess(run, name, replay_path, http_ports=None, https_ports=None, ssl_cert='', ca_cert='', - other_args='--verbose diag'): + verbose=True, other_args=''): """ Create a verifier-server process and configure it for the given TestRun. @@ -169,7 +174,7 @@ def AddVerifierServerProcess(run, name, replay_path, http_ports=None, server = run.Processes.Process(name) _configure_server(run, server, name, replay_path, http_ports, https_ports, - ssl_cert, ca_cert, other_args) + ssl_cert, ca_cert, verbose, other_args) client = run.Processes.Default client.StartBefore(server) diff --git a/tests/prepare_proxy_verifier.sh b/tests/prepare_proxy_verifier.sh index 6690b845ec7..06a2faa9cd7 100755 --- a/tests/prepare_proxy_verifier.sh +++ b/tests/prepare_proxy_verifier.sh @@ -18,16 +18,16 @@ # limitations under the License. ROOT=${ROOT:-$(cd $(dirname $0) && git rev-parse --show-toplevel)} +pv_name="proxy-verifier" +pv_version=`cat ${ROOT}/tests/proxy-verifier-version.txt` pv_top_dir="${ROOT}/tests/proxy-verifier" -pv_unpack_dir="${pv_top_dir}/unpack" +pv_unpack_dir="${pv_top_dir}/unpack/${pv_version}" bin_dir="${pv_unpack_dir}/bin" -pv_name="proxy-verifier" -pv_version="v1.10.0" pv_dir="${pv_name}-${pv_version}" pv_tar_filename="${pv_dir}.tar.gz" pv_tar="${pv_top_dir}/${pv_tar_filename}" pv_tar_url="https://ci.trafficserver.apache.org/bintray/${pv_tar_filename}" -expected_sha1="f9ad11942f9098733e0286112a3c17975ebed363" +expected_sha1="f644ac9838ba74e674b9efe7e7b3d244037faf21" pv_client="${bin_dir}/verifier-client" pv_server="${bin_dir}/verifier-server" TAR=${TAR:-tar} diff --git a/tests/proxy-verifier-version.txt b/tests/proxy-verifier-version.txt new file mode 100644 index 00000000000..46b105a30dc --- /dev/null +++ b/tests/proxy-verifier-version.txt @@ -0,0 +1 @@ +v2.0.0