Skip to content

Commit

Permalink
Apply matrix splits to all dtest jenkins jobs
Browse files Browse the repository at this point in the history 
 patch by Mick Semb Wever; reviewed by Eduard Tudenhöfner for CASSANDRA-16006
  • Loading branch information
@michaelsembwever
michaelsembwever committed Aug 5, 2020
1 parent b59b6bf commit 6fc260a
Showing 1 changed file with 19 additions and 83 deletions.
102 changes: 19 additions & 83 deletions jenkins-dsl/cassandra_job_dsl_seed.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,12 @@ if(binding.hasVariable("MAX_JOB_HOURS")) {
maxJobHours = "${MAX_JOB_HOURS}"
}

// how many splits are dtest jobs matrixed into
def dtestSplits = 64
if(binding.hasVariable("DTEST_SPLITS")) {
dtestSplits = "${DTEST_SPLITS}"
}

////////////////////////////////////////////////////////////
//
// Job Templates
Expand Down Expand Up @@ -224,67 +230,6 @@ job('Cassandra-template-test') {
/**
* Dtest template
*/
job('Cassandra-template-dtest') {
disabled(true)
description(jobDescription)
concurrentBuild()
jdk(jdkLabel)
label(slaveLabel)
compressBuildLog()
logRotator {
numToKeep(10)
artifactNumToKeep(10)
}
wrappers {
timeout {
noActivity(1200)
}
timestamps()
}
properties {
githubProjectUrl(githubRepo)
priority(5)
}
scm {
git {
remote {
url(mainRepo)
}
branch('*/null')
extensions {
cleanAfterCheckout()
}
}
}
steps {
buildDescription('', buildDescStr)
shell("git clean -xdff ; git clone --depth 1 --single-branch -b ${buildsBranch} ${buildsRepo} ; git clone --depth 1 --single-branch ${dtestRepo}")
}
publishers {
archiveArtifacts {
pattern('**/test_stdout.txt,**/nosetests.xml,**/ccm_logs.tar.xz')
allowEmpty()
fingerprint()
}
archiveJunit('nosetests.xml') {
testDataPublishers {
publishTestStabilityData()
}
}
postBuildTask {
task('.', """
echo "Cleaning project…"; git clean -xdff ;
echo "Pruning docker…" ; if pgrep -af jenkinscommand.sh; then docker system prune -f --filter 'until=${maxJobHours}h'; else docker system prune -f --volumes ; fi;
echo "Reporting disk usage…"; df -h ; du -hs ../* ; du -hs ../../* ;
echo "Cleaning tmp…";
find . -type d -name tmp -delete 2>/dev/null ;
find /tmp -type f -atime +2 -user jenkins -and -not -exec fuser -s {} ';' -and -delete 2>/dev/null
""")
}
}
}

// wip – parallelise dtests, starting with just upgrade dtests
matrixJob('Cassandra-template-dtest-matrix') {
disabled(true)
description(jobDescription)
Expand Down Expand Up @@ -487,37 +432,21 @@ cassandraBranches.each {
// Skip dtest-offheap on cassandra-3.0 branch
if ((targetName == 'dtest-offheap') && (branchName == 'cassandra-3.0')) {
println("Skipping ${targetName} on branch ${branchName}")
} else if (targetName == 'dtest-upgrade') {
// wip – parallelise dtests, starting with just upgrade dtests
} else {
matrixJob("${jobNamePrefix}-${targetName}") {
disabled(false)
using('Cassandra-template-dtest-matrix')
axes {
splits = 64
List<String> values = new ArrayList<String>()
(1..splits).each { values << it.toString() }
(1..dtestSplits).each { values << it.toString() }
text('split', values)
label('label', slaveLabel)
}
configure { node ->
node / scm / branches / 'hudson.plugins.git.BranchSpec' / name(branchName)
}
steps {
shell("sh ./cassandra-builds/docker/jenkins/jenkinscommand.sh apache ${branchName} https://github.com/apache/cassandra-dtest.git master ${buildsRepo} ${buildsBranch} ${dtestDockerImage} ${targetName} \${split}/${splits}")
}
}
} else {
job("${jobNamePrefix}-${targetName}") {
disabled(false)
using('Cassandra-template-dtest')
if (targetName == 'dtest-large') {
label(largeSlaveLabel)
}
configure { node ->
node / scm / branches / 'hudson.plugins.git.BranchSpec' / name(branchName)
}
steps {
shell("sh ./cassandra-builds/docker/jenkins/jenkinscommand.sh apache ${branchName} https://github.com/apache/cassandra-dtest.git master ${buildsRepo} ${buildsBranch} ${dtestDockerImage} ${targetName}")
shell("sh ./cassandra-builds/docker/jenkins/jenkinscommand.sh apache ${branchName} https://github.com/apache/cassandra-dtest.git master ${buildsRepo} ${buildsBranch} ${dtestDockerImage} ${targetName} \${split}/${dtestSplits}")
}
}
}
Expand Down Expand Up @@ -725,9 +654,10 @@ testTargets.each {
dtestTargets.each {
def targetName = it

job("Cassandra-devbranch-${targetName}") {
matrixJob("Cassandra-devbranch-${targetName}") {
description(jobDescription)
concurrentBuild()
compressBuildLog()
jdk(jdkLabel)
if (targetName == 'dtest-large') {
label(largeSlaveLabel)
Expand All @@ -752,9 +682,15 @@ dtestTargets.each {
stringParam('DTEST_BRANCH', 'master', 'The branch of cassandra-dtest to checkout')
stringParam('DOCKER_IMAGE', "${dtestDockerImage}", 'Docker image for running dtests')
}
axes {
List<String> values = new ArrayList<String>()
(1..dtestSplits).each { values << it.toString() }
text('split', values)
label('label', slaveLabel)
}
properties {
githubProjectUrl(githubRepo)
priority(5)
priority(6)
}
scm {
git {
Expand All @@ -770,7 +706,7 @@ dtestTargets.each {
steps {
buildDescription('', buildDescStr)
shell("git clean -xdff ; git clone --depth 1 --single-branch -b ${buildsBranch} ${buildsRepo}")
shell("sh ./cassandra-builds/docker/jenkins/jenkinscommand.sh \$REPO \$BRANCH \$DTEST_REPO \$DTEST_BRANCH ${buildsRepo} ${buildsBranch} \$DOCKER_IMAGE ${targetName}")
shell("sh ./cassandra-builds/docker/jenkins/jenkinscommand.sh \$REPO \$BRANCH \$DTEST_REPO \$DTEST_BRANCH ${buildsRepo} ${buildsBranch} \$DOCKER_IMAGE ${targetName} \${split}/${dtestSplits}")
}
publishers {
archiveArtifacts {
Expand Down

0 comments on commit 6fc260a

Please sign in to comment.