Skip to content

[Don't Merge] Test Jenkins #500

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Aggarwal-Raghav wants to merge 3 commits into
base: master
Choose a base branch
from
Open

[Don't Merge] Test Jenkins #500

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ac9028e
TEZ-4718: Modernize Jenkins and Yetus integration to ensure full buil…
Aggarwal-Raghav May 12, 2026
d7f20cb
TEZ-4715: Migrate Tez PB Protocols from ProtobufRPCEngine to Protobuf…
Aggarwal-Raghav Apr 16, 2026
787d45f
TEZ-4708: TezTaskUmbilicalProtocol to use a protobuf-based protocol
Aggarwal-Raghav Apr 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions .codespellrc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#

[codespell]
ignore-words-list = thirdparty
269 changes: 121 additions & 148 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,188 +23,161 @@ pipeline {

options {
buildDiscarder(logRotator(numToKeepStr: '5'))
timeout (time: 20, unit: 'HOURS')
timeout(time: 8, unit: 'HOURS')
timestamps()
checkoutToSubdirectory('src')
// Ensure only one build runs per PR at a time
disableConcurrentBuilds(abortPrevious: true)
}

parameters {
string(name: 'JIRA_ISSUE_KEY',
defaultValue: '',
description: 'The JIRA issue to comment on. Example: TEZ-1234')
}

environment {
SOURCEDIR = 'src'
// will also need to change notification section below
YETUSDIR = 'yetus'
PATCHDIR = 'out'
DOCKERFILE = "${SOURCEDIR}/build-tools/docker/Dockerfile"
YETUS='yetus'
// Branch or tag name. Yetus release tags are 'rel/X.Y.Z'
YETUS_VERSION='rel/0.15.1'

}
// Credentials IDs
GITHUB_CRED_ID = 'apache-tez-at-github.com'
JIRA_CRED_ID = 'tez-ci'

parameters {
string(name: 'JIRA_ISSUE_KEY',
defaultValue: '',
description: 'The JIRA issue that has a patch needing pre-commit testing. Example: HADOOP-1234')
// Yetus version
YETUS_VERSION = 'rel/0.15.1'
}

stages {
stage ('install yetus') {
stage('Install Yetus') {
steps {
dir("${WORKSPACE}/${YETUS}") {
dir("${WORKSPACE}/${env.YETUSDIR}") {
checkout([
$class: 'GitSCM',
branches: [[name: "${env.YETUS_VERSION}"]],
userRemoteConfigs: [[ url: 'https://github.com/apache/yetus']]]
userRemoteConfigs: [[url: 'https://github.com/apache/yetus']]]
)
}
}
}

stage ('precommit-run') {
stage('Yetus Analysis') {
steps {
withCredentials(
[usernamePassword(credentialsId: 'apache-tez-at-github.com',
passwordVariable: 'GITHUB_TOKEN',
usernameVariable: 'GITHUB_USER'),
usernamePassword(credentialsId: 'tez-ci',
passwordVariable: 'JIRA_PASSWORD',
usernameVariable: 'JIRA_USER')]) {
sh '''#!/usr/bin/env bash

set -e

TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"

# this must be clean for every run
if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then
rm -rf "${WORKSPACE}/${PATCHDIR}"
fi
mkdir -p "${WORKSPACE}/${PATCHDIR}"

# if given a JIRA issue, process it. If CHANGE_URL is set
# (e.g., Github Branch Source plugin), process it.
# otherwise exit, because we don't want Hadoop to do a
# full build. We wouldn't normally do this check for smaller
# projects. :)
if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
elif [[ -z "${CHANGE_URL}" ]]; then
echo "Full build skipped" > "${WORKSPACE}/${PATCHDIR}/report.html"
exit 0
fi

YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")

# where the source is located
YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")

# our project defaults come from a personality file
YETUS_ARGS+=("--project=tez")

# lots of different output formats
YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")

# enable writing back to Github
YETUS_ARGS+=(--github-token="${GITHUB_TOKEN}")

# auto-kill any surefire stragglers during unit test runs
YETUS_ARGS+=("--reapermode=kill")

# set relatively high limits for ASF machines
# changing these to higher values may cause problems
# with other jobs on systemd-enabled machines
YETUS_ARGS+=("--proclimit=5500")
YETUS_ARGS+=("--dockermemlimit=20g")

# -1 spotbugs issues that show up prior to the patch being applied
# YETUS_ARGS+=("--spotbugs-strict-precheck")
# rsync these files back into the archive dir
YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml")

# URL for user-side presentation in reports and such to our artifacts
# (needs to match the archive bits below)
YETUS_ARGS+=("--build-url-artifacts=artifact/out")

# plugins to enable
YETUS_ARGS+=("--plugins=all")

# use Hadoop's bundled shelldocs
YETUS_ARGS+=("--shelldocs=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/shelldocs")

# don't let these tests cause -1s because we aren't really paying that
# much attention to them
YETUS_ARGS+=("--tests-filter=checkstyle")

# run in docker mode and specifically point to our
# Dockerfile since we don't want to use the auto-pulled version.
YETUS_ARGS+=("--docker")
YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
YETUS_ARGS+=("--mvn-custom-repos")

# effectively treat dev-suport as a custom maven module
YETUS_ARGS+=("--skip-dirs=dev-support")

# help keep the ASF boxes clean
YETUS_ARGS+=("--sentinel")

# test with Java 21 eclipse-temurin docker image path
YETUS_ARGS+=("--java-home=/opt/java/openjdk")
YETUS_ARGS+=("--debug")

# write Yetus report as GitHub comment (YETUS-1102)
YETUS_ARGS+=("--github-write-comment")
YETUS_ARGS+=("--github-use-emoji-vote")

"${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
'''
withCredentials([
usernamePassword(credentialsId: env.GITHUB_CRED_ID, passwordVariable: 'GITHUB_TOKEN', usernameVariable: 'GITHUB_USER'),
usernamePassword(credentialsId: env.JIRA_CRED_ID, passwordVariable: 'JIRA_PASSWORD', usernameVariable: 'JIRA_USER')
]) {
sh '''#!/usr/bin/env bash

TESTPATCHBIN="${WORKSPACE}/${YETUSDIR}/precommit/src/main/shell/test-patch.sh"

rm -rf "${WORKSPACE}/${PATCHDIR}"
mkdir -p "${WORKSPACE}/${PATCHDIR}"

cd "${WORKSPACE}/${SOURCEDIR}"
# Ensure origin/master is available for diffing
git fetch origin master || true
git diff origin/master...HEAD > "${WORKSPACE}/${PATCHDIR}/local-pr.patch"
cd "${WORKSPACE}"

YETUS_ARGS=()

# Uncomment the following line to enable Yetus debugging
# YETUS_ARGS+=("--debug")

# 1. Target Issue/PR
if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
elif [[ -n "${CHANGE_ID}" ]]; then
echo "Processing PR: ${CHANGE_ID} using local patch"
else
echo "No PR or JIRA key provided. Skipping Yetus."
exit 0
fi

# 2. Core Paths & Project
YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
YETUS_ARGS+=("--project=tez")

# 3. Docker & Environment
YETUS_ARGS+=("--docker")
YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
YETUS_ARGS+=("--java-home=/opt/java/openjdk")
YETUS_ARGS+=("--build-url-artifacts=artifact/out")

# 4. Plugins & Personality
# Copy personality to PATCHDIR to avoid deletion by Yetus' robot mode
cp "${WORKSPACE}/${SOURCEDIR}/dev-support/tez-personality.sh" "${WORKSPACE}/${PATCHDIR}/tez-personality.sh"

YETUS_ARGS+=("--plugins=all")

YETUS_ARGS+=("--personality=${WORKSPACE}/${PATCHDIR}/tez-personality.sh")
YETUS_ARGS+=("--mvn-custom-repos")
YETUS_ARGS+=("--tests-filter=checkstyle")
YETUS_ARGS+=("--archive-list=checkstyle-errors.xml,spotbugsXml.xml")
YETUS_ARGS+=("--reapermode=kill")
YETUS_ARGS+=("--sentinel")

# 5. Reporting & GitHub Integration
YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
YETUS_ARGS+=("--github-write-comment")
YETUS_ARGS+=("--github-use-emoji-vote")
YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")

echo "Running Yetus with local patch overriding GitHub..."
"${TESTPATCHBIN}" "${YETUS_ARGS[@]}" "${WORKSPACE}/${PATCHDIR}/local-pr.patch"
'''
}
}
}

}

post {
always {
script {
// Yetus output
archiveArtifacts "${env.PATCHDIR}/**"
// Publish the HTML report so that it can be looked at
// Has to be relative to WORKSPACE.
publishHTML (target: [
allowMissing: true,
keepAll: true,
alwaysLinkToLastBuild: true,
// Has to be relative to WORKSPACE
reportDir: "${env.PATCHDIR}",
reportFiles: 'report.html',
reportName: 'Yetus Report'
])
// Publish JUnit results
try {
junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml"
} catch(e) {
echo 'junit processing: ' + e.toString()
script {
sh '''
# Fix permissions before attempting to read/zip directories created by root Docker processes
echo "Cleaning up workspace permissions to prevent HADOOP-13951 and unzipping errors..."
chmod -R u+rxw "${WORKSPACE}" || true

# Zip test logs for easier debugging and storage efficiency
cd "${SOURCEDIR}" || exit 0
find . -type d -name "surefire-reports" | while read -r dir; do
rel_path=$(echo "${dir#./}" | tr '/' '_')
zip -r -q "../${PATCHDIR}/test-logs-${rel_path}.zip" "${dir}" || true
done
'''

dir(env.SOURCEDIR) {
// Safely publish junit results
junit testResults: "**/target/surefire-reports/*.xml", allowEmptyResults: true
}

// Archive Yetus output, reports, and zipped test logs
archiveArtifacts artifacts: "${env.PATCHDIR}/**", allowEmptyArchive: true

// Publish Yetus HTML report to Jenkins UI sidebar only if it exists
if (fileExists("${env.PATCHDIR}/report.html")) {
publishHTML(target: [
allowMissing: true,
alwaysLinkToLastBuild: true,
keepAll: true,
reportDir: "${env.PATCHDIR}",
reportFiles: 'report.html',
reportName: 'Yetus Report'
])
}
}
}
}

// Jenkins pipeline jobs fill slaves on PRs without this :(
cleanup() {
cleanup {
script {
sh '''
# See YETUS-764
if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then
echo "test-patch process appears to still be running: killing"
kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true
sleep 10
fi
if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then
echo "test-patch container appears to still be running: killing"
docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true
fi
# See HADOOP-13951
chmod -R u+rxw "${WORKSPACE}"
'''
// Wipe the workspace to prevent Jenkins agent disk exhaustion
deleteDir()
}
}
Expand Down
Loading