diff --git a/validation/Jenkinsfile.e2e b/validation/Jenkinsfile.e2e index d6f309ddb0..19ec044130 100644 --- a/validation/Jenkinsfile.e2e +++ b/validation/Jenkinsfile.e2e @@ -1,125 +1,766 @@ #!groovy -node { - def rootPath = "/root/go/src/github.com/rancher/tests/" - def workPath = "/root/go/src/github.com/rancher/tests/validation/" - def job_name = "${JOB_NAME}" - if (job_name.contains('/')) { - job_names = job_name.split('/') - job_name = job_names[job_names.size() - 1] +/** + * E2E validation pipeline with infrastructure provisioning. + * + * Full-stack pipeline that: + * 1. Provisions AWS infrastructure via OpenTofu + * 2. Deploys an airgapped RKE2 cluster via qa-infra-automation Makefile + * 3. Deploys Rancher + * 4. Runs Go validation tests + * 5. Tears down infrastructure after tests (or on failure) + * + * Consumes shared functions from qa-jenkins-library: + * airgap.standardCheckout, airgap.configureAnsible, + * airgap.teardownInfrastructure, + * tofu.initBackend, tofu.createWorkspace, tofu.apply, tofu.getOutputs, + * infrastructure.parseAndSubstituteVars, infrastructure.writeConfig, + * infrastructure.generateWorkspaceName, infrastructure.archiveWorkspaceName, + * infrastructure.writeSshKey, + * make.runTarget, property.useWithProperties, ansible.runPlaybook + * + * Make targets (run via make.runTarget in Docker): + * cluster — Deploy RKE2 via Ansible tarball playbook + * registry — Configure private registry on cluster nodes + * rancher — Deploy Rancher via Helm + * + * Replaces the original validation/Jenkinsfile.e2e which expected + * pre-existing infrastructure. This version provisions and tears down + * infrastructure automatically. + */ + +def libraryBranch = env.QA_JENKINS_LIBRARY_BRANCH ?: 'main' +library "qa-jenkins-library@${libraryBranch}" + +/** + * Shared teardown helper: re-creates tfvars from params, tears down Tofu workspace. + * Sets env.INFRA_CLEANED to 'true' on success. + */ +def destroyInfrastructure(String reason) { + def wsName = '' + try { + if (fileExists('workspace_name.txt')) { + wsName = readFile('workspace_name.txt')?.trim() + } + } catch (e) { + echo "Could not read workspace_name.txt: ${e.message}" } - def golangTestContainer = "${job_name}${env.BUILD_NUMBER}-golangtest" - def buildTestContainer = "${job_name}${env.BUILD_NUMBER}-buildtest" - def cleanupTestContainer = "${job_name}${env.BUILD_NUMBER}-cleanuptest" - def golangImageName = "rancher-validation-${job_name}${env.BUILD_NUMBER}" - def validationVolume = "ValidationSharedVolume-${job_name}${env.BUILD_NUMBER}" - def testsDir = "/root/go/src/github.com/rancher/tests/validation/${env.TEST_PACKAGE}" - def testResultsOut = "results.xml" - def testResultsJSON = "results.json" - def envFile = ".env" - def rancherConfig = "rancher_env.config" - def branch = "main" - def cleanup = env.RANCHER_CLEANUP.toLowerCase() - if ("${env.BRANCH}" != "null" && "${env.BRANCH}" != "") { - branch = "${env.BRANCH}" + + if (!wsName || env.INFRA_CLEANED == 'true') { + echo "Skipping teardown (${reason}): no workspace or already cleaned" + return } + try { + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_SSH_PEM_KEY_NAME' + ]) { + def tofuDir = "${env.INFRA_DIR}/tofu/aws/modules/airgap" + def terraformConfig = infrastructure.parseAndSubstituteVars( + content: params.TERRAFORM_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME + ] + ) + infrastructure.writeConfig( + path: "${tofuDir}/terraform.tfvars", + content: terraformConfig + ) + airgap.teardownInfrastructure( + dir: tofuDir, + name: wsName, + varFile: 'terraform.tfvars' + ) + env.INFRA_CLEANED = 'true' + } + } catch (cleanupErr) { + echo "Teardown failed (${reason}): ${cleanupErr.message}" + } +} + +pipeline { + agent any + + options { + ansiColor('xterm') + timeout(time: 180, unit: 'MINUTES') + buildDiscarder(logRotator(numToKeepStr: '30')) + } + + parameters { + // ── Repository parameters ────────────────────────────── + string( + name: 'QA_JENKINS_LIBRARY_BRANCH', + defaultValue: 'main', + description: 'Branch of qa-jenkins-library to use' + ) + string( + name: 'TESTS_REPO_URL', + defaultValue: 'https://github.com/rancher/tests', + description: 'URL of rancher/tests repository' + ) + string( + name: 'TESTS_BRANCH', + defaultValue: 'main', + description: 'Branch of rancher/tests repository' + ) + string( + name: 'QA_INFRA_REPO_URL', + defaultValue: 'https://github.com/rancher/qa-infra-automation', + description: 'URL of qa-infra-automation repository' + ) + string( + name: 'QA_INFRA_BRANCH', + defaultValue: 'main', + description: 'Branch of qa-infra-automation repository' + ) + + // ── Infrastructure parameters ────────────────────────── + string( + name: 'HOSTNAME_PREFIX', + defaultValue: '', + description: 'Hostname prefix for *.qa.rancher.space and other AWS resources' + ) + string( + name: 'RKE2_VERSION', + defaultValue: 'v1.33.6+rke2r1', + description: 'RKE2 version to deploy' + ) + string( + name: 'RANCHER_VERSION', + defaultValue: 'v2.13.0', + description: 'Rancher version to deploy' + ) + string( + name: 'PRIVATE_REGISTRY_URL', + defaultValue: '', + description: 'Private registry URL' + ) + string( + name: 'PRIVATE_REGISTRY_USERNAME', + defaultValue: '', + description: 'Private registry username' + ) + password( + name: 'PRIVATE_REGISTRY_PASSWORD', + defaultValue: '', + description: 'Private registry password' + ) + password( + name: 'RANCHER_BOOTSTRAP_PASSWORD', + defaultValue: 'rancherrocks', + description: 'Rancher bootstrap password for initial setup' + ) + password( + name: 'RANCHER_ADMIN_PASSWORD', + defaultValue: 'rancherrocks', + description: 'Rancher admin password after bootstrap' + ) + booleanParam( + name: 'DEPLOY_RANCHER', + defaultValue: true, + description: 'Deploy Rancher via Helm after RKE2 setup' + ) + booleanParam( + name: 'DESTROY_ON_FAILURE', + defaultValue: true, + description: 'Tear down infrastructure if pipeline fails' + ) + booleanParam( + name: 'DESTROY_AFTER_TESTS', + defaultValue: true, + description: 'Tear down infrastructure after tests complete' + ) + + // ── Terraform parameters ─────────────────────────────── + string( + name: 'S3_BUCKET_NAME', + defaultValue: 'jenkins-terraform-state-storage', + description: 'S3 bucket name where Terraform state is stored' + ) + string( + name: 'S3_BUCKET_REGION', + defaultValue: 'us-east-2', + description: 'AWS region where the S3 bucket is located' + ) + string( + name: 'S3_KEY_PREFIX', + defaultValue: 'terraform.tfstate', + description: 'S3 key prefix for the Terraform state files' + ) + text( + name: 'TERRAFORM_CONFIG', + description: 'Terraform config values for the VM instances', + defaultValue: '''aws_access_key = "${AWS_ACCESS_KEY_ID}" +aws_secret_key = "${AWS_SECRET_ACCESS_KEY}" +aws_ami = "ami-09457fad1d2c34c31" +instance_type = "t3a.xlarge" +aws_security_group = ["sg-08e8243a8cfbea8a0"] +aws_subnet = "subnet-ee8cac86" +aws_volume_size = 100 +aws_hostname_prefix = "${HOSTNAME_PREFIX}" +aws_region = "us-east-2" +aws_route53_zone = "qa.rancher.space" +aws_ssh_user = "ubuntu" +aws_vpc = "vpc-bfccf4d7" +user_id = "ubuntu" +ssh_key = "/root/.ssh/${AWS_SSH_PEM_KEY_NAME}.pem" +ssh_key_name = "${AWS_SSH_PEM_KEY_NAME}" +provision_registry = false''' + ) + text( + name: 'ANSIBLE_VARIABLES', + description: 'Ansible config values for the RKE2 airgap deployment', + defaultValue: '''--- +# Global variables for RKE2 airgap deployment with tarball installation + +# RKE2 Configuration +rke2_version: ${RKE2_VERSION} +rke2_server_options: | + cluster-cidr: {{ cluster_cidr }} + service-cidr: {{ service_cidr }} + cluster-dns: {{ cluster_dns }} + disable: + - rke2-snapshot-controller + - rke2-snapshot-controller-crd + - rke2-snapshot-validation-webhook +rke2_agent_options: "" +installation_method: "tarball" + +# Network Configuration +cluster_cidr: "10.42.0.0/16" +service_cidr: "10.43.0.0/16" +cluster_dns: "10.43.0.10" + +# CNI Configuration +cni: "calico" + +# Logging and Monitoring +enable_audit_log: false +audit_log_path: "/var/lib/rancher/rke2/server/logs/audit.log" +audit_log_maxage: 30 +audit_log_maxbackup: 10 +audit_log_maxsize: 100 + +# Registry mirrors configuration +private_registry_mirrors: + - registry: "docker.io" + endpoints: + - "${PRIVATE_REGISTRY_URL}" + rewrite: + - pattern: "^(.*)" + replacement: "proxycache/$1" + - registry: "quay.io" + endpoints: + - "${PRIVATE_REGISTRY_URL}" + rewrite: + - pattern: "^(.*)" + replacement: "quaycache/$1" + +# Registry authentication and TLS configuration +private_registry_configs: + - registry: ${PRIVATE_REGISTRY_URL} + auth: + username: ${PRIVATE_REGISTRY_USERNAME} + password: ${PRIVATE_REGISTRY_PASSWORD} + tls: + insecure_skip_verify: true + +# Whether to enable private registry configuration +enable_private_registry: ${ENABLE_PRIVATE_REGISTRY} + +# Deploy Rancher +deploy_rancher: ${DEPLOY_RANCHER_ANSIBLE} +install_helm: true + +rancher_hostname: "${HOSTNAME_PREFIX}.qa.rancher.space" +rancher_bootstrap_password: "${RANCHER_BOOTSTRAP_PASSWORD}" +rancher_admin_password: "${RANCHER_ADMIN_PASSWORD}" +rancher_image_tag: ${RANCHER_VERSION} +rancher_use_bundled_system_charts: true''' + ) - def rancherRepo = scm.getUserRemoteConfigs()[0].getUrl() - if ("${env.REPO}" != "null" && "${env.REPO}" != "") { - rancherRepo = "${env.REPO}" + // ── Test parameters ──────────────────────────────────── + string( + name: 'GO_TEST_PACKAGE', + defaultValue: './validation/...', + description: 'Go test package to run' + ) + string( + name: 'GO_TEST_CASE', + defaultValue: '', + description: 'Specific test case regex (-run flag). Empty = all tests.' + ) + string( + name: 'GO_TAGS', + defaultValue: 'validation', + description: 'Go build tags' + ) + string( + name: 'GO_TIMEOUT', + defaultValue: '45m', + description: 'Go test timeout duration' + ) + text( + name: 'CATTLE_TEST_CONFIG', + defaultValue: '', + description: 'Test configuration YAML (cattle-config). Token will be injected automatically.' + ) + booleanParam( + name: 'REPORT_ARTIFACTS', + defaultValue: true, + description: 'Archive test logs and results as build artifacts' + ) + string( + name: 'QASE_TEST_RUN_ID', + defaultValue: '', + description: 'Qase test run ID. Enables Qase reporting when set.' + ) + string( + name: 'QASE_PROJECT_ID', + defaultValue: '', + description: 'Qase project ID' + ) + string( + name: 'QASE_REPORTER_SCRIPT', + defaultValue: 'build_qase_reporter_v2.sh', + description: 'Qase reporter script name in pipeline/scripts/' + ) } - def timeout = "60m" - if ("${env.TIMEOUT}" != "null" && "${env.TIMEOUT}" != "") { - timeout = "${env.TIMEOUT}" + environment { + WORKSPACE_NAME = '' + INFRA_CLEANED = 'false' } - wrap([$class: 'AnsiColorBuildWrapper', 'colorMapName': 'XTerm', 'defaultFg': 2, 'defaultBg':1]) { - withFolderProperties { - paramsMap = [] - params.each { - if (it.value && it.value.trim() != "") { - paramsMap << "$it.key=$it.value" - } + + stages { + stage('Checkout') { + steps { + script { + def dirs = airgap.standardCheckout( + testsRepo: [url: params.TESTS_REPO_URL, branch: params.TESTS_BRANCH], + infraRepo: [url: params.QA_INFRA_REPO_URL, branch: params.QA_INFRA_BRANCH] + ) + env.TESTS_DIR = dirs.testsDir + env.INFRA_DIR = dirs.infraDir + + def hostnamePrefix = params.HOSTNAME_PREFIX ?: 'unknown' + currentBuild.displayName = "#${env.BUILD_NUMBER} - ${hostnamePrefix}" + currentBuild.description = "Provisioning infrastructure for ${hostnamePrefix}" + } + } } - withCredentials([ string(credentialsId: 'AWS_ACCESS_KEY_ID', variable: 'AWS_ACCESS_KEY_ID'), - string(credentialsId: 'AWS_SECRET_ACCESS_KEY', variable: 'AWS_SECRET_ACCESS_KEY'), - string(credentialsId: 'AWS_ACCESS_KEY_ID', variable: 'RANCHER_EKS_ACCESS_KEY'), - string(credentialsId: 'AWS_SECRET_ACCESS_KEY', variable: 'RANCHER_EKS_SECRET_KEY'), - string(credentialsId: 'AWS_SSH_PEM_KEY', variable: 'AWS_SSH_PEM_KEY'), - string(credentialsId: 'RANCHER_SSH_KEY', variable: 'RANCHER_SSH_KEY'), - string(credentialsId: 'RANCHER_REGISTRY_USER_NAME', variable: 'RANCHER_REGISTRY_USER_NAME'), - string(credentialsId: 'RANCHER_REGISTRY_PASSWORD', variable: 'RANCHER_REGISTRY_PASSWORD'), - string(credentialsId: 'ADMIN_PASSWORD', variable: 'ADMIN_PASSWORD'), - string(credentialsId: 'USER_PASSWORD', variable: 'USER_PASSWORD'), - string(credentialsId: 'RANCHER_VALID_TLS_CERT', variable: 'RANCHER_VALID_TLS_CERT'), - string(credentialsId: 'RANCHER_VALID_TLS_KEY', variable: 'RANCHER_VALID_TLS_KEY'), - string(credentialsId: 'RANCHER_BYO_TLS_CERT', variable: 'RANCHER_BYO_TLS_CERT'), - string(credentialsId: 'QASE_AUTOMATION_TOKEN', variable: 'QASE_AUTOMATION_TOKEN'), - string(credentialsId: 'SLACK_WEBHOOK', variable: 'SLACK_WEBHOOK'), - string(credentialsId: 'RANCHER_BYO_TLS_KEY', variable: 'RANCHER_BYO_TLS_KEY')]) { - - withEnv(paramsMap) { - stage('Checkout') { - deleteDir() - dir("./tests") { - checkout([ - $class: 'GitSCM', - branches: [[name: "*/${branch}"]], - extensions: scm.extensions + [[$class: 'CleanCheckout']], - userRemoteConfigs: [[url: rancherRepo]] - ]) + + stage('Build Docker Images') { + steps { + sh "docker build --no-cache --platform linux/amd64 -t rancher-go-test:latest -f ${env.TESTS_DIR}/validation/pipeline/Dockerfile.airgap-go-tests ." } - } - dir ("./") { - stage('Configure and Build') { - if (env.AWS_SSH_PEM_KEY && env.AWS_SSH_KEY_NAME) { - dir("./tests/.ssh") { - def decoded = new String(AWS_SSH_PEM_KEY.decodeBase64()) - writeFile file: "${AWS_SSH_KEY_NAME}", text: decoded + } + + stage('Provision Infrastructure') { + steps { + script { + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY', + 'AWS_SSH_PEM_KEY', + 'AWS_SSH_PEM_KEY_NAME', + 'PRIVATE_REGISTRY_URL', + 'PRIVATE_REGISTRY_USERNAME', + 'PRIVATE_REGISTRY_PASSWORD' + ]) { + def tofuModulePath = "${env.INFRA_DIR}/tofu/aws/modules/airgap" + def ansiblePath = "${env.INFRA_DIR}/ansible/rke2/airgap" + def workspaceName = '' + + // ── Tofu: init backend ───────────────────────── + stage('Initialize Tofu Backend') { + tofu.initBackend( + dir: tofuModulePath, + bucket: params.S3_BUCKET_NAME, + key: params.S3_KEY_PREFIX, + region: params.S3_BUCKET_REGION, + backendInitScript: './scripts/init-backend.sh' + ) + } + + // ── Tofu: create workspace ───────────────────── + stage('Create Workspace') { + workspaceName = infrastructure.generateWorkspaceName( + prefix: 'jenkins_e2e_workspace', + suffix: params.HOSTNAME_PREFIX, + includeTimestamp: false + ) + tofu.createWorkspace(dir: tofuModulePath, name: workspaceName) + infrastructure.archiveWorkspaceName(workspaceName: workspaceName) + env.WORKSPACE_NAME = workspaceName + } + + // ── SSH key setup ─────────────────────────────── + stage('Configure SSH Key') { + infrastructure.writeSshKey( + keyContent: env.AWS_SSH_PEM_KEY, + keyName: env.AWS_SSH_PEM_KEY_NAME, + dir: '.ssh' + ) + } + + // ── Tofu: configure variables ─────────────────── + stage('Configure Tofu Variables') { + def terraformConfig = infrastructure.parseAndSubstituteVars( + content: params.TERRAFORM_CONFIG, + envVars: [ + 'AWS_ACCESS_KEY_ID': env.AWS_ACCESS_KEY_ID, + 'AWS_SECRET_ACCESS_KEY': env.AWS_SECRET_ACCESS_KEY, + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'AWS_SSH_PEM_KEY_NAME': env.AWS_SSH_PEM_KEY_NAME + ] + ) + infrastructure.writeConfig( + path: "${tofuModulePath}/terraform.tfvars", + content: terraformConfig + ) + } + + // ── Tofu: apply ───────────────────────────────── + stage('Apply Tofu') { + tofu.apply( + dir: tofuModulePath, + varFile: 'terraform.tfvars', + autoApprove: true + ) + } + + // ── Inventory generation ──────────────────────── + stage('Generate Inventory') { + sh """ + docker run --rm --platform linux/amd64 \ + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \ + -v \$(pwd):/workspace -w /workspace \ + rancher-go-test:latest sh -c ' + tofu -chdir=${tofuModulePath} output -raw airgap_inventory_json > /tmp/airgap.json && + python3 /workspace/${env.INFRA_DIR}/scripts/generate_inventory.py \ + --input /tmp/airgap.json \ + --schema /workspace/${env.INFRA_DIR}/ansible/_inventory-schema.yaml \ + --distro rke2 --env airgap \ + --output-dir ${ansiblePath}/inventory + ' + """ + } + + // ── Ansible: configure variables ──────────────── + stage('Configure Ansible Variables') { + airgap.configureAnsible( + sshKey: [ + content: env.AWS_SSH_PEM_KEY, + name: env.AWS_SSH_PEM_KEY_NAME, + dir: '.ssh' + ], + inventoryVars: [ + content: params.ANSIBLE_VARIABLES, + path: "${ansiblePath}/inventory/group_vars/all.yml", + envVars: [ + 'RKE2_VERSION': params.RKE2_VERSION, + 'RANCHER_VERSION': params.RANCHER_VERSION, + 'HOSTNAME_PREFIX': params.HOSTNAME_PREFIX, + 'PRIVATE_REGISTRY_URL': params.PRIVATE_REGISTRY_URL ?: '', + 'PRIVATE_REGISTRY_USERNAME': params.PRIVATE_REGISTRY_USERNAME ?: '', + 'PRIVATE_REGISTRY_PASSWORD': params.PRIVATE_REGISTRY_PASSWORD ?: '', + 'RANCHER_BOOTSTRAP_PASSWORD': params.RANCHER_BOOTSTRAP_PASSWORD ?: '', + 'RANCHER_ADMIN_PASSWORD': params.RANCHER_ADMIN_PASSWORD ?: '', + 'ENABLE_PRIVATE_REGISTRY': params.PRIVATE_REGISTRY_URL ? 'true' : 'false', + 'DEPLOY_RANCHER_ANSIBLE': params.DEPLOY_RANCHER ? 'true' : 'false' + ] + ], + ansibleDir: ansiblePath, + inventoryFile: 'inventory/inventory.yml', + validate: false + ) + } + + // ── Ansible: copy SSH keys to nodes ───────────── + stage('Copy SSH Keys to Nodes') { + ansible.runPlaybook( + dir: ansiblePath, + inventory: 'inventory/inventory.yml', + playbook: 'playbooks/setup/setup-ssh-keys.yml' + ) + } + + // ── Make: deploy RKE2 cluster ─────────────────── + stage('Deploy RKE2 Cluster') { + make.runTarget( + target: 'cluster', + dir: env.INFRA_DIR, + makeArgs: 'ENV=airgap', + passAwsCreds: false + ) + } + + // ── Make: configure private registry ──────────── + stage('Configure Private Registry') { + if (params.PRIVATE_REGISTRY_URL?.trim()) { + make.runTarget( + target: 'registry', + dir: env.INFRA_DIR, + makeArgs: 'ENV=airgap', + passAwsCreds: false + ) + } else { + echo 'Skipping private registry configuration (PRIVATE_REGISTRY_URL not set)' + } + } + + // ── Make: deploy Rancher ──────────────────────── + stage('Deploy Rancher') { + if (params.DEPLOY_RANCHER) { + make.runTarget( + target: 'rancher', + dir: env.INFRA_DIR, + makeArgs: 'ENV=airgap', + passAwsCreds: false + ) + } else { + echo 'Skipping Rancher deployment (DEPLOY_RANCHER not checked)' + } + } + + // ── Output infrastructure details ─────────────── + stage('Output Infrastructure Details') { + try { + def rancherHostname = tofu.getOutputs( + dir: tofuModulePath, output: 'external_lb_hostname' + ) + echo "Rancher Hostname: https://${rancherHostname}" + env.RANCHER_HOSTNAME = rancherHostname + currentBuild.description = "Testing: https://${rancherHostname} | Workspace: ${workspaceName}" + } catch (e) { + echo "Could not retrieve Rancher hostname: ${e.message}" + } + } + } } - } - dir("./tests/validation") { - def filename = "config.yaml" - def configContents = env.CONFIG - - writeFile file: filename, text: configContents - env.CATTLE_TEST_CONFIG = "${workPath}"+filename - } - dir ("./") { - sh "./tests/validation/configure.sh" - sh "docker build . -f ./tests/validation/pipeline/Dockerfile.tofu_and_validation -t ${golangImageName}" - sh "docker volume create --name ${validationVolume}" - } } - stage('Run Validation Tests') { - try { - sh "docker run --name ${golangTestContainer} -t --env-file ${envFile} " + - "${golangImageName} sh -c \" pwd; eval \\\$(ssh-agent -s); ssh-add .ssh/je*; gotestsum --format standard-verbose --packages=${testsDir} --junitfile ${testResultsOut} --jsonfile ${testResultsJSON} -- -tags=${TAGS} ${GOTEST_TESTCASE} -timeout=${timeout} -v;" + - "${workPath}pipeline/scripts/build_qase_reporter.sh;" + - "if [ -f ${workPath}reporter ]; then ${workPath}reporter; fi\"" - } catch(err) { - echo 'Validation tests had failures. Aborting' - } + } + + stage('Create Test Config and Inject Admin Token') { + steps { + script { + property.useWithProperties([ + 'AWS_ACCESS_KEY_ID', + 'AWS_SECRET_ACCESS_KEY' + ]) { + // Create cattle-config if provided + if (params.CATTLE_TEST_CONFIG?.trim()) { + writeFile file: "${env.TESTS_DIR}/cattle-config.yaml", text: params.CATTLE_TEST_CONFIG.trim() + '\n' + } + + // Generate admin token via Ansible playbook + def adminPassword = params.RANCHER_ADMIN_PASSWORD ?: 'rancherrocks' + def cattleConfigPath = "/workspace/${env.TESTS_DIR}/cattle-config.yaml" + def workspace = pwd() + def tokenTtl = env.RANCHER_TOKEN_TTL ?: '0' + def tokenDescription = "jenkins-e2e-${env.BUILD_NUMBER}" + + sh """ + docker run --rm --platform linux/amd64 \ + --name generate-token \ + -e RANCHER_ADMIN_PASSWORD=${adminPassword} \ + -e ANSIBLE_CONFIG=/workspace/${env.INFRA_DIR}/ansible/ansible.cfg \ + -v ${workspace}:/workspace \ + -w /workspace/${env.INFRA_DIR}/ansible/rke2/airgap \ + rancher-go-test:latest \ + ansible-playbook -i inventory/inventory.yml /workspace/${env.INFRA_DIR}/ansible/rancher/token/generate-admin-token.yml \ + -e rancher_cattle_config_file=${cattleConfigPath} \ + -e rancher_token_ttl=${tokenTtl} \ + -e rancher_token_description=${tokenDescription} \ + -e rancher_token_output_format=json \ + -e rancher_token_output_file=/workspace/rancher-token.json + """ + + if (fileExists('rancher-token.json')) { + def tokenData = readJSON file: 'rancher-token.json' + ['token', 'bearerToken', 'accessKey', 'secretKey', 'value'].each { secretField -> + if (tokenData instanceof Map && tokenData.containsKey(secretField)) { + tokenData.remove(secretField) + } + } + writeJSON file: 'rancher-token-metadata.json', json: tokenData, pretty: 2 + archiveArtifacts artifacts: 'rancher-token-metadata.json', fingerprint: true + echo 'Archived redacted token metadata: rancher-token-metadata.json' + } + } + } } - stage('Test Report') { - try { - sh "docker cp ${golangTestContainer}:${rootPath}${testResultsOut} ." - step([$class: 'JUnitResultArchiver', testResults: "**/${testResultsOut}"]) - } catch (err) { - sh "docker stop ${golangTestContainer}" - sh "docker rm -v ${golangTestContainer}" - if (cleanup.toBoolean()) { - sh "docker stop ${cleanupTestContainer}" - sh "docker rm -v ${cleanupTestContainer}" + } + + stage('Run Go Validation Tests') { + steps { + script { + def rancherHostname = env.RANCHER_HOSTNAME ?: '' + def envFile = '.go_test_env' + def lines = [] + + if (rancherHostname) { + lines += [ + "RANCHER_URL=https://${rancherHostname}", + "RANCHER_HOSTNAME=${rancherHostname}" + ] + } + lines += "RANCHER_BOOTSTRAP_PASSWORD=${params.RANCHER_BOOTSTRAP_PASSWORD ?: 'rancherrocks'}" + + def cattleConfigPath = "${env.TESTS_DIR}/cattle-config.yaml" + if (!fileExists(cattleConfigPath) && params.CATTLE_TEST_CONFIG?.trim()) { + writeFile file: cattleConfigPath, text: params.CATTLE_TEST_CONFIG.trim() + '\n' + } + lines += "CATTLE_TEST_CONFIG=/workspace/${env.TESTS_DIR}/cattle-config.yaml" + + def envContent = lines.findAll { it?.trim() }.join('\n') + writeFile file: envFile, text: envContent ? envContent + '\n' : '' + + def workspace = pwd() + def containerName = "gotest-${env.BUILD_NUMBER}" + + def testCmd = "/root/go/bin/gotestsum --format standard-verbose --packages=${params.GO_TEST_PACKAGE} --junitfile junit.xml --jsonfile gotestsum.json -- -tags=${params.GO_TAGS} ${params.GO_TEST_CASE} -timeout=${params.GO_TIMEOUT} -v" + + catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + sh """ + docker run --rm --platform linux/amd64 \ + --name ${containerName} \ + --env-file ${envFile} \ + -v ${workspace}:/workspace \ + -w /workspace/tests \ + rancher-go-test:latest \ + sh -c 'set -e; set -o pipefail; ${testCmd} | tee go-test.log' + """ + } + + catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { + junit allowEmptyResults: true, testResults: "tests/junit.xml" + } + + if (params.REPORT_ARTIFACTS) { + archiveArtifacts allowEmptyArchive: true, artifacts: 'tests/go-test.log, tests/junit.xml, tests/gotestsum.json', fingerprint: true + } + + env.TEST_CONTAINER = containerName + } + } + } + + stage('Report to Qase') { + when { + allOf { + expression { params.QASE_TEST_RUN_ID?.trim() } + expression { params.QASE_PROJECT_ID?.trim() } } - sh "docker rmi -f ${golangImageName}" - sh "docker volume rm -f ${validationVolume}" - error 'Report had failures.' - } } - } // dir - } // withEnv - } // creds - } // folder properties - } // wrap -} // node \ No newline at end of file + steps { + script { + property.useWithProperties(['QASE_AUTOMATION_TOKEN']) { + echo "Reporting to Qase (Project: ${params.QASE_PROJECT_ID}, Run: ${params.QASE_TEST_RUN_ID})" + + def workspace = pwd() + sh """ + docker run --rm --platform linux/amd64 \ + --name qase-reporter \ + -e QASE_TEST_RUN_ID=${params.QASE_TEST_RUN_ID} \ + -e QASE_PROJECT_ID=${params.QASE_PROJECT_ID} \ + -e QASE_AUTOMATION_TOKEN=${env.QASE_AUTOMATION_TOKEN} \ + -v ${workspace}:/workspace \ + -w /workspace/tests \ + rancher-go-test:latest \ + sh -c ' + set -e + echo "Building Qase reporter..." + ./validation/pipeline/scripts/${params.QASE_REPORTER_SCRIPT} + + if [ -f ./validation/reporter ]; then + echo "Running Qase reporter..." + ./validation/reporter + echo "Qase reporting complete" + else + echo "Reporter binary not found - reporter script may have skipped build" + fi + ' + """ + } + } + } + } + } + + post { + failure { + script { + if (params.DESTROY_ON_FAILURE) { + destroyInfrastructure('DESTROY_ON_FAILURE') + } + } + } + + always { + script { + if (params.DESTROY_AFTER_TESTS) { + destroyInfrastructure('DESTROY_AFTER_TESTS') + } + + // Docker cleanup + sh 'docker rmi -f rancher-go-test:latest || true' + + // Sensitive artifact cleanup — shred or remove credentials from workspace + sh ''' + set +e + + cleanup_file() { + if [ -f "$1" ]; then + if command -v shred >/dev/null 2>&1; then + shred -u "$1" || rm -f "$1" + else + rm -f "$1" + fi + fi + } + + cleanup_dir() { + if [ -d "$1" ]; then + find "$1" -type f -exec sh -c ' + for file do + if command -v shred >/dev/null 2>&1; then + shred -u "$file" || rm -f "$file" + else + rm -f "$file" + fi + done + ' sh {} + + rm -rf "$1" + fi + } + + cleanup_file ".go_test_env" + cleanup_file "rancher-token.json" + cleanup_file "rancher-token-metadata.json" + cleanup_file "workspace_name.txt" + + if [ -n "${INFRA_DIR}" ] && [ -d "${INFRA_DIR}" ]; then + find "${INFRA_DIR}" -type f \( -name 'terraform.tfvars' -o -name '*.pem' -o -name 'id_rsa' -o -name 'id_ed25519' \) -exec sh -c ' + for file do + if command -v shred >/dev/null 2>&1; then + shred -u "$file" || rm -f "$file" + else + rm -f "$file" + fi + done + ' sh {} + + fi + + cleanup_dir ".ssh" + ''' + } + } + } +}