Skip to content

Commit 92c0cb7

Browse files
michaelsembwevermichaelsembwever
committed
squash
– capture generateTestReports logs, and stream xz where possible – fix ant version in centos7-build.docker – remove docker login (if credentials exist then docker is logged in) – prefetch docker images from jfrog (to reduce dockerhub pull rate limits) – use scripts from cassandra-builds to clean and report on agents – elvis on pipelineProfiles()[params.profile] – set +x fix – parallel jfrog pulls
1 parent ed86141 commit 92c0cb7

4 files changed

Lines changed: 86 additions & 54 deletions

File tree

‎.build/docker/_docker_run.sh‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -93,9 +93,6 @@ pushd ${cassandra_dir}/.build >/dev/null
9393
image_tag="$(md5sum docker/${dockerfile} | cut -d' ' -f1)"
9494
image_name="apache/cassandra-${dockerfile/.docker/}:${image_tag}"
9595

96-
# try docker login to increase dockerhub rate limits
97-
echo "Attempting 'docker login' to increase dockerhub rate limits"
98-
timeout -k 5 5 docker login >/dev/null
9996
# Look for existing docker image, otherwise build
10097
if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
10198
echo "Build image not found locally, pulling image ${image_name}..."
@@ -111,7 +108,7 @@ if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
111108
echo "Successfully pulled build image."
112109
fi
113110
else
114-
echo "Found build image locally."
111+
echo "Found build image locally."
115112
fi
116113

117114
# Run build script through docker

‎.build/docker/centos7-build.docker‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ ENV BUILD_HOME=/home/build
2323
ENV RPM_BUILD_DIR=$BUILD_HOME/rpmbuild
2424
ENV DIST_DIR=/dist
2525
ENV CASSANDRA_DIR=$BUILD_HOME/cassandra
26-
ENV ANT_VERSION=1.10.12
26+
ENV ANT_VERSION=1.10.14
2727
ARG UID_ARG=1000
2828
ARG GID_ARG=1000
2929

‎.build/docker/run-tests.sh‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -91,9 +91,6 @@ docker_mounts="-v ${cassandra_dir}:/home/cassandra/cassandra -v "${build_dir}":/
9191
# HACK hardlinks in overlay are buggy, the following mount prevents hardlinks from being used. ref $TMP_DIR in .build/run-tests.sh
9292
docker_mounts="${docker_mounts} -v "${build_dir}/tmp":/home/cassandra/cassandra/build/tmp"
9393

94-
# try docker login to increase dockerhub rate limits
95-
echo "Attempting 'docker login' to increase dockerhub rate limits"
96-
timeout -k 5 5 docker login >/dev/null 2>/dev/null
9794
# Look for existing docker image, otherwise build
9895
if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
9996
echo "Build image not found locally, pulling image ${image_name}..."
@@ -109,7 +106,7 @@ if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
109106
echo "Successfully pulled build image."
110107
fi
111108
else
112-
echo "Found build image locally."
109+
echo "Found build image locally."
113110
fi
114111

115112
pushd ${cassandra_dir} >/dev/null

‎.jenkins/Jenkinsfile‎

Lines changed: 83 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@
4848
// will be ignored when run on other environments.
4949
// Note there are also differences when CI is being run pre- or post-commit.
5050
//
51+
// When running CI with changes in this file, ensure the "Pipeline script from SCM" scm details match
52+
// the brances being tested. These details don't honour the per-build repository and branch parameterisation.
5153
//
5254
// Validate/lint this file using the following command
5355
// `curl -X POST -F "jenkinsfile=<.jenkins/Jenkinsfile" https://ci-cassandra.apache.org/pipeline-model-converter/validate`
@@ -297,7 +299,7 @@ def isCanonical() {
297299
}
298300

299301
def isStageEnabled(stage) {
300-
return "jar" == stage || pipelineProfiles()[params.profile].contains(stage) || ("custom" == params.profile && stage ==~ params.profile_custom_regexp)
302+
return "jar" == stage || pipelineProfiles()[params.profile]?.contains(stage) || ("custom" == params.profile && stage ==~ params.profile_custom_regexp)
301303
}
302304

303305
def isArchEnabled(arch) {
@@ -322,22 +324,20 @@ def build(command, cell) {
322324
nodeExclusion = "&&!${NODE_NAME}"
323325
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
324326
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}") {
325-
cleanAgent(cell.step)
326-
cleanWs()
327327
fetchSource(cell.step, cell.arch, cell.jdk)
328328
sh """
329329
test -f .jenkins/Jenkinsfile || { echo "Invalid git fork/branch"; exit 1; }
330330
grep -q "Jenkins CI declaration" .jenkins/Jenkinsfile || { echo "Only Cassandra 5.0+ supported"; exit 1; }
331-
"""
331+
"""
332+
fetchDockerImages(['almalinux-build', 'bullseye-build', 'centos7-build'])
332333
def cell_suffix = "_jdk${cell.jdk}_${cell.arch}"
333-
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log"
334+
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
334335
def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
335336
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
336-
def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee build/${logfile}", returnStatus: true
337+
def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
337338
dir("build") {
338-
sh "xz -f *${logfile}"
339-
archiveArtifacts artifacts: "${logfile}.xz", fingerprint: true
340-
copyToNightlies("${logfile}.xz", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
339+
archiveArtifacts artifacts: "${logfile}", fingerprint: true
340+
copyToNightlies("${logfile}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
341341
}
342342
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
343343
if ("jar" == cell.step) { // TODO only stash the project built files. all dependency libraries are restored from the local maven repo using `ant resolver-dist-lib`
@@ -364,11 +364,10 @@ def test(command, cell) {
364364
nodeExclusion = "&&!${NODE_NAME}"
365365
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
366366
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}/python-${cell.python}") {
367-
cleanAgent(cell.step)
368-
cleanWs()
369367
fetchSource(cell.step, cell.arch, cell.jdk)
368+
fetchDockerImages(['ubuntu2004_test'])
370369
def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}_${cell.split}_${splits}"
371-
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log"
370+
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
372371
// pipe to tee needs pipefail
373372
def script_vars = "#!/bin/bash \n set -o pipefail ; "
374373
script_vars = "${script_vars} python_version=\'${cell.python}\'"
@@ -378,11 +377,10 @@ def test(command, cell) {
378377
}
379378
script_vars = fetchDTestsSource(command, script_vars)
380379
buildJVMDTestJars(cell, script_vars, logfile)
381-
def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh ${cell.step} '${cell.split}/${splits}' ${cell.jdk} 2>&1 | tee -a build/${logfile}", returnStatus: true
380+
def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh ${cell.step} '${cell.split}/${splits}' ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
382381
dir("build") {
383-
sh "xz -f ${logfile}"
384-
archiveArtifacts artifacts: "${logfile}.xz", fingerprint: true
385-
copyToNightlies("${logfile}.xz", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
382+
archiveArtifacts artifacts: "${logfile}", fingerprint: true
383+
copyToNightlies("${logfile}", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
386384
}
387385
if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
388386
dir("build") {
@@ -405,12 +403,13 @@ def test(command, cell) {
405403
}
406404

407405
def fetchSource(stage, arch, jdk) {
408-
if ("jar" == stage) {
409-
checkout changelog: false, scm: scmGit(branches: [[name: params.branch]], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true)], userRemoteConfigs: [[url: params.repository]])
410-
sh "mkdir -p build/stage-logs"
411-
} else {
412-
unstash name: "${arch}_${jdk}"
413-
}
406+
cleanAgent(stage)
407+
if ("jar" == stage) {
408+
checkout changelog: false, scm: scmGit(branches: [[name: params.branch]], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true)], userRemoteConfigs: [[url: params.repository]])
409+
sh "mkdir -p build/stage-logs"
410+
} else {
411+
unstash name: "${arch}_${jdk}"
412+
}
414413
}
415414

416415
def fetchDTestsSource(command, script_vars) {
@@ -427,12 +426,26 @@ def buildJVMDTestJars(cell, script_vars, logfile) {
427426
try {
428427
unstash name: "jvm_dtests_${cell.arch}_${cell.jdk}"
429428
} catch (error) {
430-
sh label: "RUNNING build_dtest_jars...", script: "${script_vars} .build/docker/run-tests.sh build_dtest_jars ${cell.jdk} 2>&1 | tee build/${logfile}"
429+
sh label: "RUNNING build_dtest_jars...", script: "${script_vars} .build/docker/run-tests.sh build_dtest_jars ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )"
431430
stash name: "jvm_dtests_${cell.arch}_${cell.jdk}", includes: '**/dtest*.jar'
432431
}
433432
}
434433
}
435434

435+
def fetchDockerImages(dockerfiles) {
436+
// prefetch, from apache jfrog, reduces risking dockerhub pull rate limits
437+
for (dockerfile in dockerfiles) {
438+
sh """#!/bin/bash
439+
image_tag="\$(md5sum .build/docker/${dockerfile}.docker | cut -d' ' -f1)"
440+
image_name="apache/cassandra-${dockerfile}:\${image_tag}"
441+
if ! ( [[ "" != "\$(docker images -q \${image_name} 2>/dev/null)" ]] ) ; then
442+
docker pull -q apache.jfrog.io/cassan-docker/\${image_name} &
443+
fi
444+
wait
445+
"""
446+
}
447+
}
448+
436449
def getNodeLabel(command, cell) {
437450
echo "using node label: cassandra-${cell.arch}-${command.size}"
438451
return "cassandra-${cell.arch}-${command.size}"
@@ -445,44 +458,68 @@ def copyToNightlies(sourceFiles, remoteDirectory='') {
445458
retry(9) {
446459
if (attempt > 1) { sleep(60 * attempt) }
447460
sshPublisher(
448-
continueOnError: true, failOnError: false,
449-
publishers: [
450-
sshPublisherDesc(
451-
configName: "Nightlies",
452-
transfers: [ sshTransfer( sourceFiles: sourceFiles, remoteDirectory: remotePath) ]
453-
)
454-
])
461+
continueOnError: true, failOnError: false,
462+
publishers: [
463+
sshPublisherDesc(
464+
configName: "Nightlies",
465+
transfers: [ sshTransfer( sourceFiles: sourceFiles, remoteDirectory: remotePath) ]
466+
)
467+
])
455468
}
456469
echo "archived to https://nightlies.apache.org/${remotePath}"
457470
}
458471
}
459472

460473
def cleanAgent(job_name) {
461474
sh "hostname"
475+
sh "test -f build.xml && git clean -qxdff -e build/test/jmh-result.json || true"
462476
if (isCanonical()) {
463-
def maxJobHours = 12
464-
echo "Cleaning project, and pruning docker for '${job_name}' on ${NODE_NAME}…" ;
465-
sh """
466-
git clean -qxdff -e build/test/jmh-result.json || true;
467-
if pgrep -xa docker || pgrep -af "build/docker" || pgrep -af "cassandra-builds/build-scripts" ; then docker system prune --all --force --filter "until=${maxJobHours}h" || true ; else docker system prune --force --volumes || true ; fi;
468-
"""
477+
def agentScriptsUrl = "https://raw.githubusercontent.com/apache/cassandra-builds/trunk/jenkins-dsl/agent_scripts/"
478+
cleanAgentDocker(job_name, agentScriptsUrl)
479+
logAgentInfo(job_name, agentScriptsUrl)
480+
sh 'rm docker_agent_cleaner.sh docker_image_pruner.py agent_report.sh *-disk-usage-stats.txt'
469481
}
470482
}
471483

484+
def cleanAgentDocker(job_name, agentScriptsUrl) {
485+
// we don't expect any build to have been running for longer than maxBuildHours
486+
def maxBuildHours = 12
487+
echo "Pruning docker for '${job_name}' on ${NODE_NAME}…" ;
488+
sh """#!/bin/bash
489+
set +e
490+
wget -q ${agentScriptsUrl}/docker_image_pruner.py
491+
wget -q ${agentScriptsUrl}//docker_agent_cleaner.sh
492+
bash docker_agent_cleaner.sh ${maxBuildHours}
493+
"""
494+
}
495+
496+
def logAgentInfo(job_name, agentScriptsUrl) {
497+
sh """#!/bin/bash
498+
set +e -o pipefail
499+
wget -q ${agentScriptsUrl}/agent_report.sh
500+
bash -x agent_report.sh | tee -a \$(date +"%Y%m%d%H%M")-disk-usage-stats.txt
501+
"""
502+
copyToNightlies("*-disk-usage-stats.txt", "cassandra/ci-cassandra.apache.org/agents/${NODE_NAME}/disk-usage/")
503+
}
504+
472505
/////////////////////////////////////////
473506
////// scripting support for summary ////
474507
/////////////////////////////////////////
475508

476509
def generateTestReports() {
477510
node("cassandra-medium") {
478-
cleanWs()
511+
cleanAgent("generateTestReports")
479512
checkout changelog: false, scm: scmGit(branches: [[name: params.branch]], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true)], userRemoteConfigs: [[url: params.repository]])
513+
def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_generateTestReports.log.xz"
514+
sh "mkdir -p build/stage-logs"
515+
def teeSuffix = "2>&1 | tee >( xz -c > build/${logfile} )"
516+
def script_vars = "#!/bin/bash -x \n "
480517
if (isCanonical()) {
481518
// copyArtifacts takes >4hrs, hack with manual download
482-
sh """
483-
mkdir -p build/test
519+
sh """${script_vars}
520+
( mkdir -p build/test
484521
wget -q ${BUILD_URL}/artifact/test/output/*zip*/output.zip
485-
unzip -x -d build/test -q output.zip
522+
unzip -x -d build/test -q output.zip ) ${teeSuffix}
486523
"""
487524
} else {
488525
copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true
@@ -491,8 +528,8 @@ def generateTestReports() {
491528
// merge splits for each target's test report, other axes are kept separate
492529
// TODO parallelised for loop
493530
// TODO results_details.tar.xz needs to include all logs for failed tests
494-
sh """
495-
find build/test/output -name *.xml.xz -exec sh -c 'xz -f --decompress {} &' ';' ; wait
531+
sh """${script_vars} (
532+
find build/test/output -name *.xml.xz -exec sh -c 'xz -f --decompress {} &' ';' ; wait
496533
497534
for target in \$(ls build/test/output/) ; do
498535
if test -d build/test/output/\${target} ; then
@@ -506,12 +543,13 @@ def generateTestReports() {
506543
507544
.build/docker/_docker_run.sh bullseye-build.docker ci/generate-ci-summary.sh || echo "failed generate-ci-summary.sh"
508545
509-
tar -cf build/results_details.tar -C build/test/ reports && xz -8f build/results_details.tar
546+
tar -cf build/results_details.tar -C build/test/ reports
547+
xz -8f build/results_details.tar ) ${teeSuffix}
510548
"""
511549

512550
dir('build/') {
513-
archiveArtifacts artifacts: "ci_summary.html,results_details.tar.xz", fingerprint: true
514-
copyToNightlies('ci_summary.html,results_details.tar.xz')
551+
archiveArtifacts artifacts: "ci_summary.html,results_details.tar.xz,${logfile}", fingerprint: true
552+
copyToNightlies('ci_summary.html,results_details.tar.xz,${logfile}')
515553
}
516554
}
517555
}

0 commit comments

Comments
 (0)