Skip to content

Commit

Permalink
Feature/jenkins pipeline (#3218)
Browse files Browse the repository at this point in the history
  • Loading branch information
@fceller
fceller committed Sep 8, 2017
1 parent 33e3ead commit e62d0a3
Showing 1 changed file with 139 additions and 141 deletions.
280 changes: 139 additions & 141 deletions Installation/Pipeline/Jenkinsfile.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ def checkCoresAndSave(os, runDir, name, archRuns, archCores) {
}
}
else {
sh "for i in logs out tmp; do test -e \"${runDir}/\$i\" && mv \"${runDir}/\$i\" \"${archRuns}/${name}.\$i\" || true; done"
sh "for i in logs out tmp result; do test -e \"${runDir}/\$i\" && mv \"${runDir}/\$i\" \"${archRuns}/${name}.\$i\" || true; done"

def files = findFiles(glob: '${runDir}/core*')

Expand Down Expand Up @@ -400,14 +400,11 @@ def checkCommitMessages() {
]
}
else {

restrictions = [
"build-community-mac" : true,
// "build-community-windows" : true,
"build-enterprise-linux" : true,
"test-cluster-enterprise-rocksdb-linux" : true,
"test-singleserver-community-mmfiles-mac" : true
// "test-singleserver-community-rocksdb-windows" : true
"test-singleserver-enterprise-mmfiles-linux" : true
]
}
}
Expand Down Expand Up @@ -438,6 +435,10 @@ Restrictions: ${restrictions.keySet().join(", ")}
def stashBinaries(os, edition) {
def paths = ["build/etc", "etc", "Installation/Pipeline", "js", "scripts", "UnitTests"]

if (edition == "enterprise") {
paths << "enterprise/js"
}

if (os == "windows") {
paths << "build/bin/RelWithDebInfo"
paths << "build/tests/RelWithDebInfo"
Expand Down Expand Up @@ -488,75 +489,68 @@ def jslint() {
// -----------------------------------------------------------------------------

def getTests(os, edition, mode, engine) {
def tests = [
["arangobench", "arangobench" , ""],
["arangosh", "arangosh", "--skipShebang true"],
["authentication", "authentication", ""],
["authentication_parameters", "authentication_parameters", ""],
["config", "config" , ""],
["dump", "dump" , ""],
["dump_authentication", "dump_authentication" , ""],
["endpoints", "endpoints", ""],
["server_http", "server_http", ""],
["shell_client", "shell_client", ""],
["shell_server", "shell_server", ""],
["shell_server_aql_1", "shell_server_aql", "--testBuckets 4/0", ,""],
["shell_server_aql_2", "shell_server_aql", "--testBuckets 4/1", ,""],
["shell_server_aql_3", "shell_server_aql", "--testBuckets 4/2", ,""],
["shell_server_aql_4", "shell_server_aql", "--testBuckets 4/3", ,""],
["upgrade", "upgrade" , ""],
rspecify(os, "http_server"),
rspecify(os, "ssl_server")
]

if (edition == "enterprise") {
tests += [
["authentication_server", "authentication_server", ""]
]
}

if (mode == "singleserver") {
return [
tests += [
["agency", "agency", ""],
["boost", "boost", "--skipCache false"],
["arangobench", "arangobench", ""],
["arangosh", "arangosh", "--skipShebang true"],
["authentication", "authentication", ""],
["authentication_server", "authentication_server", ""],
["authentication_parameters", "authentication_parameters", ""],
["cluster_sync", "cluster_sync", ""],
["config", "config", ""],
["dfdb", "dfdb", ""],
//"dump",
//"dump_authentication",
["endpoints", "endpoints", ""],
rspecify(os, "http_replication"),
rspecify(os, "http_server"),
["replication_sync", "replication_sync", ""],
["replication_static", "replication_static", ""],
["replication_ongoing", "replication_ongoing", ""],
["server_http", "server_http", ""],
["shell_client", "shell_client", ""],
["replication_static", "replication_static", ""],
["replication_sync", "replication_sync", ""],
["shell_replication", "shell_replication", ""],
["shell_server", "shell_server", ""],
["shell_server_aql_1", "shell_server_aql", "--testBuckets 4/0", ,""],
["shell_server_aql_2", "shell_server_aql", "--testBuckets 4/1", ,""],
["shell_server_aql_3", "shell_server_aql", "--testBuckets 4/2", ,""],
["shell_server_aql_4", "shell_server_aql", "--testBuckets 4/3", ,""],
rspecify(os, "ssl_server"),
["upgrade", "upgrade" , ""]
]
}
else {
return [
["arangobench", "arangobench" , ""],
["arangosh", "arangosh" , "--skipShebang true"],
["authentication", "authentication" , ""],
["authentication_server", "authentication_server", ""],
["authentication_parameters", "authentication_parameters" , ""],
["config", "config" , ""],
["dump", "dump" , ""],
["dump_authentication", "dump_authentication" , ""],
["endpoints", "endpoints" , ""],
rspecify(os, "http_server"),
["server_http", "server_http", ""],
["shell_client", "shell_client", ""],
["shell_server", "shell_server", ""],
["shell_server_aql_1", "shell_server_aql", "--testBuckets 4/0"],
["shell_server_aql_2", "shell_server_aql", "--testBuckets 4/1"],
["shell_server_aql_3", "shell_server_aql", "--testBuckets 4/2"],
["shell_server_aql_4", "shell_server_aql", "--testBuckets 4/3"],
rspecify(os, "ssl_server"),
["upgrade", "upgrade" , ""]
rspecify(os, "http_replication")
]
}

return tests
}

def setupTestEnvironment(os, logFile, runDir) {
def setupTestEnvironment(os, edition, logFile, runDir) {
fileOperations([
folderCreateOperation("${runDir}/tmp"),
])

def subdirs = ['build', 'etc', 'js', 'UnitTests']

if (edition == "enterprise") {
subdirs << "enterprise"
}

if (os == "windows") {
for (file in ['build', 'etc', 'js', 'UnitTests']) {
for (file in subdirs) {
powershell "cd ${runDir} ; New-Item -Path ${file} -ItemType SymbolicLink -Value ..\\${file} | Out-Null"
}
}
else {
for (file in ['build', 'etc', 'js', 'UnitTests']) {
for (file in subdirs) {
sh "ln -s ../${file} ${runDir}/${file}"
}

Expand Down Expand Up @@ -609,7 +603,7 @@ def executeTests(os, edition, mode, engine, portInit, arch, archRuns, archFailed

try {
lock("test-${env.NODE_NAME}-${env.JOB_NAME}-${env.BUILD_ID}-${edition}-${engine}-${lockIndex}") {
setupTestEnvironment(os, logFile, runDir)
setupTestEnvironment(os, edition, logFile, runDir)

try {
timeout(30) {
Expand All @@ -621,14 +615,16 @@ def executeTests(os, edition, mode, engine, portInit, arch, archRuns, archFailed
powershell "cd ${runDir} ; ${command} | Add-Content -PassThru ${logFile}"
}
else {
sh "echo \"Host: `hostname`\" | tee ${logFile}"
sh "echo \"PWD: `pwd`\" | tee -a ${logFile}"
sh "echo \"Date: `date`\" | tee -a ${logFile}"

command = "(cd ${runDir} ; echo 1 > result ; ${command} ; echo \$? > result) 2>&1 | " +
"tee ${logFile} ; exit `cat ${runDir}/result`"
"tee -a ${logFile} ; exit `cat ${runDir}/result`"
echo "executing ${command}"
sh command
}
}

checkCoresAndSave(os, runDir, name, archRuns, archCores)
}
}
catch (exc) {
Expand All @@ -637,6 +633,9 @@ def executeTests(os, edition, mode, engine, portInit, arch, archRuns, archFailed
copyFile(os, logFile, logFileFailed)
throw exc
}
finally {
checkCoresAndSave(os, runDir, name, archRuns, archCores)
}
}
}
catch (exc) {
Expand Down Expand Up @@ -673,62 +672,64 @@ def testCheck(os, edition, mode, engine) {

def testStep(os, edition, mode, engine, testName) {
return {
node(testJenkins[os]) {
stage(testName) {
def archRel = "02_test_${os}_${edition}_${mode}_${engine}"
def archFailedRel = "${archRel}_FAILED"
def archRunsRel = "${archRel}_RUN"
def archCoresRel = "${archRel}_CORES"

def arch = pwd() + "/" + "02_test_${os}_${edition}_${mode}_${engine}"
def archFailed = "${arch}_FAILED"
def archRuns = "${arch}_RUN"
def archCores = "${arch}_CORES"

// clean the current workspace completely
deleteDir()

// create directories for the artifacts
fileOperations([
folderCreateOperation(arch),
folderCreateOperation(archFailed),
folderCreateOperation(archRuns),
folderCreateOperation(archCores)
])

// unstash binaries
unstashBinaries(os, edition)

// find a suitable port
def port = (getStartPort(os) as Integer)
echo "Using start port: ${port}"

// seriously...60 minutes is the super absolute max max max.
// even in the worst situations ArangoDB MUST be able to finish within 60 minutes
// even if the features are green this is completely broken performance wise..
// DO NOT INCREASE!!

timeout(60) {
try {
executeTests(os, edition, mode, engine, port, arch, archRuns, archFailed, archCores)
}
finally {
// step([$class: 'XUnitBuilder',
// thresholds: [[$class: 'FailedThreshold', unstableThreshold: '1']],
// tools: [[$class: 'JUnitType', failIfNotNew: false, pattern: 'out/*.xml']]])

// release the port reservation
if (os == 'linux' || os == 'mac') {
sh "Installation/Pipeline/port.sh --clean ${port}"
}
else if (os == 'windows') {
powershell "remove-item -Force -ErrorAction Ignore C:\\ports\\${port}"
if (testCheck(os, edition, mode, engine)) {
node(testJenkins[os]) {
stage(testName) {
def archRel = "02_test_${os}_${edition}_${mode}_${engine}"
def archFailedRel = "${archRel}_FAILED"
def archRunsRel = "${archRel}_RUN"
def archCoresRel = "${archRel}_CORES"

def arch = pwd() + "/" + "02_test_${os}_${edition}_${mode}_${engine}"
def archFailed = "${arch}_FAILED"
def archRuns = "${arch}_RUN"
def archCores = "${arch}_CORES"

// clean the current workspace completely
deleteDir()

// create directories for the artifacts
fileOperations([
folderCreateOperation(arch),
folderCreateOperation(archFailed),
folderCreateOperation(archRuns),
folderCreateOperation(archCores)
])

// unstash binaries
unstashBinaries(os, edition)

// find a suitable port
def port = (getStartPort(os) as Integer)
echo "Using start port: ${port}"

// seriously...60 minutes is the super absolute max max max.
// even in the worst situations ArangoDB MUST be able to finish within 60 minutes
// even if the features are green this is completely broken performance wise..
// DO NOT INCREASE!!

timeout(60) {
try {
executeTests(os, edition, mode, engine, port, arch, archRuns, archFailed, archCores)
}
finally {
// step([$class: 'XUnitBuilder',
// thresholds: [[$class: 'FailedThreshold', unstableThreshold: '1']],
// tools: [[$class: 'JUnitType', failIfNotNew: false, pattern: 'out/*.xml']]])

// release the port reservation
if (os == 'linux' || os == 'mac') {
sh "Installation/Pipeline/port.sh --clean ${port}"
}
else if (os == 'windows') {
powershell "remove-item -Force -ErrorAction Ignore C:\\ports\\${port}"
}

// archive all artifacts
archiveArtifacts allowEmptyArchive: true,
artifacts: "${archRel}/**, ${archFailedRel}/**, ${archRunsRel}/**, ${archCoresRel}/**",
defaultExcludes: false
// archive all artifacts
archiveArtifacts allowEmptyArchive: true,
artifacts: "${archRel}/**, ${archFailedRel}/**, ${archRunsRel}/**, ${archCoresRel}/**",
defaultExcludes: false
}
}
}
}
Expand All @@ -741,11 +742,8 @@ def testStepParallel(os, edition, modeList) {

for (mode in modeList) {
for (engine in ['mmfiles', 'rocksdb']) {
if (testCheck(os, edition, mode, engine)) {
def name = "test-${os}-${edition}-${mode}-${engine}";

branches[name] = testStep(os, edition, mode, engine, name)
}
def name = "test-${os}-${edition}-${mode}-${engine}";
branches[name] = testStep(os, edition, mode, engine, name)
}
}

Expand Down Expand Up @@ -937,35 +935,36 @@ def buildStepCheck(os, edition) {

def runEdition(os, edition) {
return {
node(buildJenkins[os]) {
stage("build-${os}-${edition}") {
timeout(30) {
checkoutCommunity()
checkCommitMessages()

if (edition == "enterprise") {
checkoutEnterprise()
if (buildStepCheck(os, edition)) {
node(buildJenkins[os]) {
stage("build-${os}-${edition}") {
timeout(30) {
checkoutCommunity()

if (edition == "enterprise") {
checkoutEnterprise()
}

// checkoutResilience()
}

// checkoutResilience()
timeout(90) {
buildEdition(os, edition)
stashBinaries(os, edition)
}
}

timeout(90) {
buildEdition(os, edition)
stashBinaries(os, edition)
// we only need one jslint test per edition
if (os == "linux") {
stage("jslint-${edition}") {
echo "Running jslint for ${edition}"
jslint()
}
}
}

// we only need one jslint test per edition
if (os == "linux") {
stage("jslint-${edition}") {
echo "Running jslint for ${edition}"
jslint()
}
}
testStepParallel(os, edition, ['cluster', 'singleserver'])
}

testStepParallel(os, edition, ['cluster', 'singleserver'])
}
}

Expand All @@ -978,13 +977,12 @@ def runOperatingSystems(osList) {

for (os in osList) {
for (edition in ['community', 'enterprise']) {
if (buildStepCheck(os, edition)) {
branches["build-${os}-${edition}"] = runEdition(os, edition)
}
branches["build-${os}-${edition}"] = runEdition(os, edition)
}
}

parallel branches
}

checkCommitMessages()
runOperatingSystems(['linux', 'mac', 'windows'])

0 comments on commit e62d0a3

Please sign in to comment.