From 21816479723d8b2c24669a9144810bedcb79bbb8 Mon Sep 17 00:00:00 2001 From: Lan Xia Date: Fri, 27 Jan 2023 13:01:36 -0500 Subject: [PATCH] Add TAP pipeline resolves: https://github.com/adoptium/aqa-tests/issues/4277 Signed-off-by: Lan Xia --- buildenv/jenkins/tapVerification/aqaTap.sh | 182 ++++++++++++++++++ .../tapVerification/tapVerification.groovy | 75 ++++++++ 2 files changed, 257 insertions(+) create mode 100755 buildenv/jenkins/tapVerification/aqaTap.sh create mode 100644 buildenv/jenkins/tapVerification/tapVerification.groovy diff --git a/buildenv/jenkins/tapVerification/aqaTap.sh b/buildenv/jenkins/tapVerification/aqaTap.sh new file mode 100755 index 0000000000..26fecdb026 --- /dev/null +++ b/buildenv/jenkins/tapVerification/aqaTap.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash + +set -eo pipefail + +TAPDIR="AQAvitTapFiles" +DOWNLOAD_URL="" +PLATFORM="unknown" + +usage () +{ + echo 'Usage : aqaTap.sh --url --dir ' +} + +parseArgs() { + while [ $# -gt 0 ] && [[ ."$1" = .-* ]] ; do + opt="$1"; + shift; + case "$opt" in + "--url" | "-u" ) + DOWNLOAD_URL="$1"; shift;; + + "--dir" | "-d" ) + TAPDIR="$1"; shift;; + + "--platform" | "-p" ) + PLATFORM="$1"; shift;; + + "--username" ) + USERNAME="$1"; shift;; + + "--password" ) + PASSWORD="$1"; shift;; + + "--clean" | "-c" ) + clean; exit 0;; + + "--deleteGrinder" | "-dg" ) + deleteGrinder; exit 0;; + + "--listAll" | "-l" ) + listAll; exit 0;; + + "--listFailed" | "-lf" ) + listFailed; exit 0;; + + "--listRerun" | "-lr" ) + listRerun; exit 0;; + + "--listTAP" | "-lt" ) + listTAP; exit 0;; + + "--help" | "-h" ) + usage; exit 0;; + + *) echo >&2 "Invalid option: ${opt}"; echo "This option was unrecognized."; usage; exit 1; + esac + done + + echo "TAPDIR: $TAPDIR" + echo "PLATFORM: $PLATFORM" +} + +downloadTAP() { + rm -rf "$PLATFORM" + mkdir "$PLATFORM" + rm -rf tmp + mkdir tmp + cd tmp + if [ "$USERNAME" != "" ] && [ "$PASSWORD" != "" ]; then + curl_options="--user $USERNAME:$PASSWORD" + fi + echo "curl -OLJSks ${curl_options} ${DOWNLOAD_URL}" + curl -OLJSks ${curl_options} "${DOWNLOAD_URL}" + jar_file=$(ls ./*.zip) + jar_file_array=("${jar_file//\\n/ }") + unzip -q "${jar_file_array[0]}" -d . + jar_dirs=$(ls -d ./*/) + jar_dir_array=("${jar_dirs//\\n/ }") + mv "${jar_dir_array[0]}" "$PLATFORM" + mv "$PLATFORM" ../ + cd .. + rm -rf tmp +} + +clean() { + echo "============================" + echo "clean files in $(pwd)" + echo "============================" + echo "List and remove the above *.jck_* ..." + find . -name "*.jck_*" + find . -name "*.jck_*" -exec rm -f {} ';' + + echo "List and remove the above *_special.system_* ..." + find . -name "*_special.system_*" + find . -name "*_special.system_*" -exec rm -f {} ';' + + echo "List and remove the above *.external_* ..." + find . -name "*.external_*" + find . -name "*.external_*" -exec rm -f {} ';' + + echo "List and remove the above *_dev.* ..." + find . -name "*_dev.*" + find . -name "*_dev.*" -exec rm -f {} ';' + + echo "List and remove the above .DS_Store ..." + find . -name ".DS_Store" + find . -name ".DS_Store" -exec rm -f {} ';' + + echo "List and remove the above *_fips* ..." + find . -name "*_fips*" + find . -name "*_fips*" -exec rm -f {} ';' + + echo "Print empty dir ..." + find . -type d -empty -print + echo "Delete empty dir ..." + find . -type d -empty -delete +} + +deleteGrinder() { + echo "============================" + echo "List and remove *Grinder_* ..." + echo "============================" + find . -name "*Grinder_*" + find . -name "*Grinder_*" -exec rm -f {} ';' +} + +listAll() { + echo "============================" + echo "list all files in $(pwd)" + echo "============================" + find . -type f | sort +} + +listFailed() { + echo "============================" + echo "List failed tests in $(pwd)" + echo "============================" + grep -R 'not ok ' . || true +} + +listRerun() { + echo "============================" + echo "list rerun in Grinder..." + echo "============================" + find . -name "*Grinder_*" -exec rm -f {} ';' +} + +listTAP() { + echo "============================" + echo "list AQAvit TAP files in $(pwd)" + echo "============================" + echo "1. List sanity.openjdk ..." + printf '%s\n' *"sanity.openjdk"*.tap + echo "2. List extended.openjdk ..." + printf '%s\n' *"extended.openjdk"*.tap + echo "3. List sanity.functional ..." + printf '%s\n' *"sanity.functional"*.tap + echo "4. List extended.functional ..." + printf '%s\n' *"extended.functional"*.tap + echo "5. List special.functional ..." + printf '%s\n' *"special.functional"*.tap + echo "6. List sanity.system ..." + printf '%s\n' *"sanity.system"*.tap + echo "7. List extended.openjdk ..." + printf '%s\n' *"extended.system"*.tap + echo "8. List sanity.perf ..." + printf '%s\n' *"sanity.perf"*.tap + echo "9. List extended.perf ..." + printf '%s\n' *"extended.perf"*.tap +} + +parseArgs "$@" +mkdir -p "$TAPDIR" +cd "$TAPDIR" +if [ "$DOWNLOAD_URL" ]; then + downloadTAP +fi +cd "$PLATFORM" +listAll +clean +listTAP +listFailed diff --git a/buildenv/jenkins/tapVerification/tapVerification.groovy b/buildenv/jenkins/tapVerification/tapVerification.groovy new file mode 100644 index 0000000000..e38587389b --- /dev/null +++ b/buildenv/jenkins/tapVerification/tapVerification.groovy @@ -0,0 +1,75 @@ +#!groovy + +def AQA_REPO = params.AQA_REPO ?: "adoptium" +def AQA_BRANCH = params.AQA_BRANCH ?: "master" +def JDK_VERSIONS = params.JDK_VERSIONS.trim().split("\\s*,\\s*"); +def PLATFORMS = params.PLATFORMS.trim().split("\\s*,\\s*"); +def TARGETS = params.TARGETS.trim().split("\\s*,\\s*"); +def LABEL = (params.LABEL) ?: "" +def LABEL_ADDITION = (params.LABEL_ADDITION) ?: "" + +// Use BUILD_USER_ID if set and jdk-JDK_VERSIONS +def DEFAULT_SUFFIX = (env.BUILD_USER_ID) ? "${env.BUILD_USER_ID} - jdk-${params.JDK_VERSIONS}" : "jdk-${params.JDK_VERSIONS}" +def PIPELINE_DISPLAY_NAME = (params.PIPELINE_DISPLAY_NAME) ? "#${currentBuild.number} - ${params.PIPELINE_DISPLAY_NAME}" : "#${currentBuild.number} - ${DEFAULT_SUFFIX}" + +timestamps{ + // Set the AQA_TEST_PIPELINE Jenkins job displayName + currentBuild.setDisplayName(PIPELINE_DISPLAY_NAME) + node(LABEL) { + TIME_LIMIT = params.TIME_LIMIT ? params.TIME_LIMIT.toInteger() : 1 + timeout(time: TIME_LIMIT, unit: 'HOURS') { + // change openjdk-tests to aqa-tests + sh "curl -Os https://raw.githubusercontent.com/${AQA_REPO}/aqa-tests/${AQA_BRANCH}/buildenv/jenkins/tapVerification/aqaTap.sh" + sh "chmod 755 aqaTap.sh" + JDK_VERSIONS.each { JDK_VERSION -> + PLATFORMS.each { PLATFORM -> + String[] tokens = PLATFORM.split('_') + def os = tokens[1]; + def arch = tokens[0]; + if (arch.contains("x86-64")){ + arch = "x64" + } else if (arch.contains("x86-32")) { + arch ="x86-32" + } + + def filter = "*.tar.gz" + if (os.contains("windows")) { + filter = "*.zip" + } + def short_name = "hs" + def jdk_impl = "hotspot" + if (params.VARIANT == "openj9") { + short_name = "j9" + jdk_impl = params.VARIANT + } + def download_url = params.CUSTOMIZED_SDK_URL ? params.CUSTOMIZED_SDK_URL : "" + def sdk_resource_value = SDK_RESOURCE + + if (SDK_RESOURCE == "customized" ) { + if (params.TOP_LEVEL_SDK_URL) { + // example: /job/build-scripts/job/openjdk17-pipeline-IBM/354/artifact/target/linux/s390x/openj9/AQAvitTaps/*zip*/AQAvitTaps.zip + + download_url = params.TOP_LEVEL_SDK_URL + "artifact/target/${os}/${arch}/${params.VARIANT}/AQAvitTaps/*zip*/AQAvitTaps.zip" + dir("${WORKSPACE}") { + def aqaTapCmd = "${WORKSPACE}/aqaTap.sh -u ${download_url} -p ${PLATFORM}" + if (params.CUSTOMIZED_SDK_URL_CREDENTIAL_ID) { + // USERNAME and PASSWORD reference with a withCredentials block will not be visible within job output + withCredentials([usernamePassword(credentialsId: "${params.CUSTOMIZED_SDK_URL_CREDENTIAL_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { + sh "$aqaTapCmd" + } + } else { + sh "$aqaTapCmd" + } + + } + } + } + } + } + + // archive + sh "tar -cf - ./AQAvitTapFiles | (pigz -9 2>/dev/null || gzip -9) > AQAvitTapFiles.tar.gz" + archiveArtifacts artifacts: "AQAvitTapFiles.tar.gz", fingerprint: true, allowEmptyArchive: false + } + } +}