From d8dc24efa4911f8d5fb0a1af5f8248abf6b94ab6 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 08:53:49 +0200 Subject: [PATCH 01/20] Overwrite jenkins params from git commit messages. [ci TESTOPTIONS="-test.run ^TestSimpleSingle$"] --- Jenkinsfile.groovy | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index 5d1402c80..79e830936 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -19,6 +19,19 @@ def notifySlack(String buildStatus = 'STARTED') { slackSend(color: color, channel: '#status-k8s', message: msg) } +def fetchParamsFromGitLog() { + def options = sh(returnStdout: true, script: "git log --reverse master..HEAD | grep -o '\[ci[^\[]*\]' | sed -E 's/\[ci (.*)\]/\1/'").trim().split("\n") + for (opt in options) { + def idx = opt.indexOf('='); + if (idx > 0) { + def key = opt.subString(0, idx); + def value = opt.subString(idx+1); + params[key] = value; + println("Overwriting params.${key} with ${value}"); + } + } +} + def kubeConfigRoot = "/home/jenkins/.kube" def buildTestSteps(String kubeConfigRoot, String kubeconfig) { @@ -32,6 +45,7 @@ def buildTestSteps(String kubeConfigRoot, String kubeconfig) { "IMAGETAG=jenkins-test", "KUBECONFIG=${kubeConfigRoot}/${kubeconfig}", "LONG=${params.LONG ? 1 : 0}", + "TESTOPTIONS=${params.TESTOPTIONS}", ]) { sh "make run-tests" } @@ -68,6 +82,13 @@ pipeline { string(name: 'ENTERPRISEIMAGE', defaultValue: '', description: 'ENTERPRISEIMAGE sets the docker image used for enterprise tests)', ) } stages { + stage("Prepare") { + steps { + script { + fetchParamsFromGitLog() + } + } + } stage('Build') { steps { timestamps { @@ -76,6 +97,7 @@ pipeline { "DOCKERNAMESPACE=${params.DOCKERNAMESPACE}", "IMAGETAG=jenkins-test", "LONG=${params.LONG ? 1 : 0}", + "TESTOPTIONS=${params.TESTOPTIONS}", ]) { sh "make" sh "make run-unit-tests" From ea5c37cb8bfdd8290242c0f1f7f33c875877333d Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 08:58:16 +0200 Subject: [PATCH 02/20] Escaping fix (hopefully) --- Jenkinsfile.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index 79e830936..9d53a339f 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -20,7 +20,7 @@ def notifySlack(String buildStatus = 'STARTED') { } def fetchParamsFromGitLog() { - def options = sh(returnStdout: true, script: "git log --reverse master..HEAD | grep -o '\[ci[^\[]*\]' | sed -E 's/\[ci (.*)\]/\1/'").trim().split("\n") + def options = sh(returnStdout: true, script: "git log --reverse master..HEAD | grep -o \'\\[ci[^\\[]*\\]\' | sed -E \'s/\\[ci (.*)\\]/\\1/\'").trim().split("\n") for (opt in options) { def idx = opt.indexOf('='); if (idx > 0) { From bd089bf308c1b220c3de81d8965e0ad25b750878 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 09:03:16 +0200 Subject: [PATCH 03/20] Fixed git log command --- Jenkinsfile.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index 9d53a339f..f810423d6 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -20,7 +20,7 @@ def notifySlack(String buildStatus = 'STARTED') { } def fetchParamsFromGitLog() { - def options = sh(returnStdout: true, script: "git log --reverse master..HEAD | grep -o \'\\[ci[^\\[]*\\]\' | sed -E \'s/\\[ci (.*)\\]/\\1/\'").trim().split("\n") + def options = sh(returnStdout: true, script: "git log --reverse remotes/origin/master..HEAD | grep -o \'\\[ci[^\\[]*\\]\' | sed -E \'s/\\[ci (.*)\\]/\\1/\'").trim().split("\n") for (opt in options) { def idx = opt.indexOf('='); if (idx > 0) { From 158df395ebe11ea1b52a60ee1062e19699b683e1 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 09:05:50 +0200 Subject: [PATCH 04/20] Debugging groovy --- Jenkinsfile.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index f810423d6..504a7ec62 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -22,7 +22,9 @@ def notifySlack(String buildStatus = 'STARTED') { def fetchParamsFromGitLog() { def options = sh(returnStdout: true, script: "git log --reverse remotes/origin/master..HEAD | grep -o \'\\[ci[^\\[]*\\]\' | sed -E \'s/\\[ci (.*)\\]/\\1/\'").trim().split("\n") for (opt in options) { + println("Processing option ${opt}"); def idx = opt.indexOf('='); + println("Processing option ${opt} with idx=${idx}"); if (idx > 0) { def key = opt.subString(0, idx); def value = opt.subString(idx+1); From 7de70d3307a970f4cfa33a5001b1f78186649470 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 09:06:46 +0200 Subject: [PATCH 05/20] substring --- Jenkinsfile.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index 504a7ec62..978ad0d90 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -26,8 +26,8 @@ def fetchParamsFromGitLog() { def idx = opt.indexOf('='); println("Processing option ${opt} with idx=${idx}"); if (idx > 0) { - def key = opt.subString(0, idx); - def value = opt.subString(idx+1); + def key = opt.substring(0, idx); + def value = opt.substring(idx+1); params[key] = value; println("Overwriting params.${key} with ${value}"); } From 3dcc3019bcd9c281bf6971c17253756a83d8bc85 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 09:12:19 +0200 Subject: [PATCH 06/20] Overwrite in myParams --- Jenkinsfile.groovy | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index 978ad0d90..8242f82b9 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -19,7 +19,17 @@ def notifySlack(String buildStatus = 'STARTED') { slackSend(color: color, channel: '#status-k8s', message: msg) } +def myParams = [:]; + def fetchParamsFromGitLog() { + // Copy configured params + for (entry in params) { + myParams[entry.key] = entry.value; + } + + // Fetch params configured in git commit messages + // Syntax: [ci OPT=value] + // Example: [ci TESTOPTIONS="-test.run ^TestSimpleSingle$"] def options = sh(returnStdout: true, script: "git log --reverse remotes/origin/master..HEAD | grep -o \'\\[ci[^\\[]*\\]\' | sed -E \'s/\\[ci (.*)\\]/\\1/\'").trim().split("\n") for (opt in options) { println("Processing option ${opt}"); @@ -28,8 +38,8 @@ def fetchParamsFromGitLog() { if (idx > 0) { def key = opt.substring(0, idx); def value = opt.substring(idx+1); - params[key] = value; - println("Overwriting params.${key} with ${value}"); + myParams[key] = value; + println("Overwriting myParams.${key} with ${value}"); } } } @@ -41,13 +51,13 @@ def buildTestSteps(String kubeConfigRoot, String kubeconfig) { timestamps { withCredentials([string(credentialsId: 'ENTERPRISEIMAGE', variable: 'DEFAULTENTERPRISEIMAGE')]) { withEnv([ - "DEPLOYMENTNAMESPACE=${params.TESTNAMESPACE}-${env.GIT_COMMIT}", - "DOCKERNAMESPACE=${params.DOCKERNAMESPACE}", - "ENTERPRISEIMAGE=${params.ENTERPRISEIMAGE}", + "DEPLOYMENTNAMESPACE=${myParams.TESTNAMESPACE}-${env.GIT_COMMIT}", + "DOCKERNAMESPACE=${myParams.DOCKERNAMESPACE}", + "ENTERPRISEIMAGE=${myParams.ENTERPRISEIMAGE}", "IMAGETAG=jenkins-test", "KUBECONFIG=${kubeConfigRoot}/${kubeconfig}", - "LONG=${params.LONG ? 1 : 0}", - "TESTOPTIONS=${params.TESTOPTIONS}", + "LONG=${myParams.LONG ? 1 : 0}", + "TESTOPTIONS=${myParams.TESTOPTIONS}", ]) { sh "make run-tests" } @@ -60,8 +70,8 @@ def buildCleanupSteps(String kubeConfigRoot, String kubeconfig) { return { timestamps { withEnv([ - "DEPLOYMENTNAMESPACE=${params.TESTNAMESPACE}-${env.GIT_COMMIT}", - "DOCKERNAMESPACE=${params.DOCKERNAMESPACE}", + "DEPLOYMENTNAMESPACE=${myParams.TESTNAMESPACE}-${env.GIT_COMMIT}", + "DOCKERNAMESPACE=${myParams.DOCKERNAMESPACE}", "KUBECONFIG=${kubeConfigRoot}/${kubeconfig}", ]) { sh "make cleanup-tests" @@ -95,11 +105,11 @@ pipeline { steps { timestamps { withEnv([ - "DEPLOYMENTNAMESPACE=${params.TESTNAMESPACE}-${env.GIT_COMMIT}", - "DOCKERNAMESPACE=${params.DOCKERNAMESPACE}", + "DEPLOYMENTNAMESPACE=${myParams.TESTNAMESPACE}-${env.GIT_COMMIT}", + "DOCKERNAMESPACE=${myParams.DOCKERNAMESPACE}", "IMAGETAG=jenkins-test", - "LONG=${params.LONG ? 1 : 0}", - "TESTOPTIONS=${params.TESTOPTIONS}", + "LONG=${myParams.LONG ? 1 : 0}", + "TESTOPTIONS=${myParams.TESTOPTIONS}", ]) { sh "make" sh "make run-unit-tests" @@ -111,7 +121,7 @@ pipeline { stage('Test') { steps { script { - def configs = "${params.KUBECONFIGS}".split(",") + def configs = "${myParams.KUBECONFIGS}".split(",") def testTasks = [:] for (kubeconfig in configs) { testTasks["${kubeconfig}"] = buildTestSteps(kubeConfigRoot, kubeconfig) @@ -125,7 +135,7 @@ pipeline { post { always { script { - def configs = "${params.KUBECONFIGS}".split(",") + def configs = "${myParams.KUBECONFIGS}".split(",") def cleanupTasks = [:] for (kubeconfig in configs) { cleanupTasks["${kubeconfig}"] = buildCleanupSteps(kubeConfigRoot, kubeconfig) From 082bc1618e23ed637961d5c731b7e48de35003cd Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 09:16:14 +0200 Subject: [PATCH 07/20] Debugging groovy --- Jenkinsfile.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index 8242f82b9..a19b92831 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -24,6 +24,8 @@ def myParams = [:]; def fetchParamsFromGitLog() { // Copy configured params for (entry in params) { + println("Fetching entry from params: ${entry}"); + println("Fetching entry from params: key=${entry.key}, value=${entry.value}"); myParams[entry.key] = entry.value; } From 8244fae064fd835e426390f794fde125d32a011c Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 09:18:34 +0200 Subject: [PATCH 08/20] Debugging groovy --- Jenkinsfile.groovy | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index a19b92831..f65284b9a 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -53,13 +53,13 @@ def buildTestSteps(String kubeConfigRoot, String kubeconfig) { timestamps { withCredentials([string(credentialsId: 'ENTERPRISEIMAGE', variable: 'DEFAULTENTERPRISEIMAGE')]) { withEnv([ - "DEPLOYMENTNAMESPACE=${myParams.TESTNAMESPACE}-${env.GIT_COMMIT}", - "DOCKERNAMESPACE=${myParams.DOCKERNAMESPACE}", - "ENTERPRISEIMAGE=${myParams.ENTERPRISEIMAGE}", + "DEPLOYMENTNAMESPACE=${myParams['TESTNAMESPACE']}-${env.GIT_COMMIT}", + "DOCKERNAMESPACE=${myParams['DOCKERNAMESPACE']}", + "ENTERPRISEIMAGE=${myParams['ENTERPRISEIMAGE']}", "IMAGETAG=jenkins-test", "KUBECONFIG=${kubeConfigRoot}/${kubeconfig}", - "LONG=${myParams.LONG ? 1 : 0}", - "TESTOPTIONS=${myParams.TESTOPTIONS}", + "LONG=${myParams['LONG'] ? 1 : 0}", + "TESTOPTIONS=${myParams['TESTOPTIONS']}", ]) { sh "make run-tests" } @@ -72,8 +72,8 @@ def buildCleanupSteps(String kubeConfigRoot, String kubeconfig) { return { timestamps { withEnv([ - "DEPLOYMENTNAMESPACE=${myParams.TESTNAMESPACE}-${env.GIT_COMMIT}", - "DOCKERNAMESPACE=${myParams.DOCKERNAMESPACE}", + "DEPLOYMENTNAMESPACE=${myParams['TESTNAMESPACE']}-${env.GIT_COMMIT}", + "DOCKERNAMESPACE=${myParams['DOCKERNAMESPACE']}", "KUBECONFIG=${kubeConfigRoot}/${kubeconfig}", ]) { sh "make cleanup-tests" @@ -107,11 +107,11 @@ pipeline { steps { timestamps { withEnv([ - "DEPLOYMENTNAMESPACE=${myParams.TESTNAMESPACE}-${env.GIT_COMMIT}", - "DOCKERNAMESPACE=${myParams.DOCKERNAMESPACE}", + "DEPLOYMENTNAMESPACE=${myParams['TESTNAMESPACE']}-${env.GIT_COMMIT}", + "DOCKERNAMESPACE=${myParams['DOCKERNAMESPACE']}", "IMAGETAG=jenkins-test", - "LONG=${myParams.LONG ? 1 : 0}", - "TESTOPTIONS=${myParams.TESTOPTIONS}", + "LONG=${myParams['LONG'] ? 1 : 0}", + "TESTOPTIONS=${myParams['TESTOPTIONS']}", ]) { sh "make" sh "make run-unit-tests" @@ -123,7 +123,7 @@ pipeline { stage('Test') { steps { script { - def configs = "${myParams.KUBECONFIGS}".split(",") + def configs = "${myParams['KUBECONFIGS']}".split(",") def testTasks = [:] for (kubeconfig in configs) { testTasks["${kubeconfig}"] = buildTestSteps(kubeConfigRoot, kubeconfig) @@ -137,7 +137,7 @@ pipeline { post { always { script { - def configs = "${myParams.KUBECONFIGS}".split(",") + def configs = "${myParams['KUBECONFIGS']}".split(",") def cleanupTasks = [:] for (kubeconfig in configs) { cleanupTasks["${kubeconfig}"] = buildCleanupSteps(kubeConfigRoot, kubeconfig) From c0ac2339dbc86109db19b982c34dac2911bb2ea5 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 09:46:29 +0200 Subject: [PATCH 09/20] Scoping myparams --- Jenkinsfile.groovy | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index f65284b9a..5c8f4d3bf 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -19,9 +19,8 @@ def notifySlack(String buildStatus = 'STARTED') { slackSend(color: color, channel: '#status-k8s', message: msg) } -def myParams = [:]; - def fetchParamsFromGitLog() { + def myParams = [:]; // Copy configured params for (entry in params) { println("Fetching entry from params: ${entry}"); @@ -44,11 +43,12 @@ def fetchParamsFromGitLog() { println("Overwriting myParams.${key} with ${value}"); } } + return myParams; } def kubeConfigRoot = "/home/jenkins/.kube" -def buildTestSteps(String kubeConfigRoot, String kubeconfig) { +def buildTestSteps(Map myParams, String kubeConfigRoot, String kubeconfig) { return { timestamps { withCredentials([string(credentialsId: 'ENTERPRISEIMAGE', variable: 'DEFAULTENTERPRISEIMAGE')]) { @@ -68,7 +68,7 @@ def buildTestSteps(String kubeConfigRoot, String kubeconfig) { } } -def buildCleanupSteps(String kubeConfigRoot, String kubeconfig) { +def buildCleanupSteps(Map myParams, String kubeConfigRoot, String kubeconfig) { return { timestamps { withEnv([ @@ -95,11 +95,12 @@ pipeline { string(name: 'TESTNAMESPACE', defaultValue: 'jenkins', description: 'TESTNAMESPACE sets the kubernetes namespace to ru tests in (this must be short!!)', ) string(name: 'ENTERPRISEIMAGE', defaultValue: '', description: 'ENTERPRISEIMAGE sets the docker image used for enterprise tests)', ) } + def myParams; stages { stage("Prepare") { steps { script { - fetchParamsFromGitLog() + myParams = fetchParamsFromGitLog(); } } } @@ -126,7 +127,7 @@ pipeline { def configs = "${myParams['KUBECONFIGS']}".split(",") def testTasks = [:] for (kubeconfig in configs) { - testTasks["${kubeconfig}"] = buildTestSteps(kubeConfigRoot, kubeconfig) + testTasks["${kubeconfig}"] = buildTestSteps(myParams, kubeConfigRoot, kubeconfig) } parallel testTasks } @@ -140,7 +141,7 @@ pipeline { def configs = "${myParams['KUBECONFIGS']}".split(",") def cleanupTasks = [:] for (kubeconfig in configs) { - cleanupTasks["${kubeconfig}"] = buildCleanupSteps(kubeConfigRoot, kubeconfig) + cleanupTasks["${kubeconfig}"] = buildCleanupSteps(myParams, kubeConfigRoot, kubeconfig) } parallel cleanupTasks } From a650a53ae153a379e513f039104f346868743dd1 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 09:47:12 +0200 Subject: [PATCH 10/20] Scoping myparams --- Jenkinsfile.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index 5c8f4d3bf..d3f9a4583 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -95,7 +95,7 @@ pipeline { string(name: 'TESTNAMESPACE', defaultValue: 'jenkins', description: 'TESTNAMESPACE sets the kubernetes namespace to ru tests in (this must be short!!)', ) string(name: 'ENTERPRISEIMAGE', defaultValue: '', description: 'ENTERPRISEIMAGE sets the docker image used for enterprise tests)', ) } - def myParams; + def myParams = [:]; stages { stage("Prepare") { steps { From 3da7a8c8fd1357565bf651538184c5fa8db64e1d Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 09:52:46 +0200 Subject: [PATCH 11/20] node test --- Jenkinsfile.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index d3f9a4583..5efd18c53 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -82,12 +82,12 @@ def buildCleanupSteps(Map myParams, String kubeConfigRoot, String kubeconfig) { } } -pipeline { +node { options { buildDiscarder(logRotator(daysToKeepStr: '7', numToKeepStr: '10')) lock resource: 'kube-arangodb' } - agent any +// agent any parameters { booleanParam(name: 'LONG', defaultValue: false, description: 'Execute long running tests') string(name: 'DOCKERNAMESPACE', defaultValue: 'arangodb', description: 'DOCKERNAMESPACE sets the docker registry namespace in which the operator docker image will be pushed', ) From 825c9d28f8775afb1746b5f79d05347c402405e9 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 09:56:42 +0200 Subject: [PATCH 12/20] node test --- Jenkinsfile.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index 5efd18c53..a117ce4f7 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -83,10 +83,10 @@ def buildCleanupSteps(Map myParams, String kubeConfigRoot, String kubeconfig) { } node { - options { + /*options { buildDiscarder(logRotator(daysToKeepStr: '7', numToKeepStr: '10')) lock resource: 'kube-arangodb' - } + }*/ // agent any parameters { booleanParam(name: 'LONG', defaultValue: false, description: 'Execute long running tests') From 905ddeb4c757fdd684c256a157150d9eb9e2cc16 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 09:57:31 +0200 Subject: [PATCH 13/20] node test --- Jenkinsfile.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index a117ce4f7..430608cd7 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -96,7 +96,7 @@ node { string(name: 'ENTERPRISEIMAGE', defaultValue: '', description: 'ENTERPRISEIMAGE sets the docker image used for enterprise tests)', ) } def myParams = [:]; - stages { + //stages { stage("Prepare") { steps { script { @@ -133,7 +133,7 @@ node { } } } - } + //} post { always { From 8d86df63dd9897489fa00baeff709714361529a5 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 09:58:49 +0200 Subject: [PATCH 14/20] Testfoo --- Jenkinsfile.groovy | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index 430608cd7..6a2b38716 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -82,12 +82,12 @@ def buildCleanupSteps(Map myParams, String kubeConfigRoot, String kubeconfig) { } } -node { - /*options { +pipeline { + options { buildDiscarder(logRotator(daysToKeepStr: '7', numToKeepStr: '10')) lock resource: 'kube-arangodb' - }*/ -// agent any + } + agent any parameters { booleanParam(name: 'LONG', defaultValue: false, description: 'Execute long running tests') string(name: 'DOCKERNAMESPACE', defaultValue: 'arangodb', description: 'DOCKERNAMESPACE sets the docker registry namespace in which the operator docker image will be pushed', ) @@ -95,8 +95,8 @@ node { string(name: 'TESTNAMESPACE', defaultValue: 'jenkins', description: 'TESTNAMESPACE sets the kubernetes namespace to ru tests in (this must be short!!)', ) string(name: 'ENTERPRISEIMAGE', defaultValue: '', description: 'ENTERPRISEIMAGE sets the docker image used for enterprise tests)', ) } - def myParams = [:]; - //stages { + stages { + def myParams = [:]; stage("Prepare") { steps { script { @@ -133,7 +133,7 @@ node { } } } - //} + } post { always { From fef29752109a6cce1a49d1e0159f5b417f8268c7 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 10:08:21 +0200 Subject: [PATCH 15/20] This is stupid --- Jenkinsfile.groovy | 44 ++++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index 6a2b38716..bff19f038 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -48,6 +48,24 @@ def fetchParamsFromGitLog() { def kubeConfigRoot = "/home/jenkins/.kube" +def buildBuildSteps(Map myParams) { + return { + timestamps { + withEnv([ + "DEPLOYMENTNAMESPACE=${myParams['TESTNAMESPACE']}-${env.GIT_COMMIT}", + "DOCKERNAMESPACE=${myParams['DOCKERNAMESPACE']}", + "IMAGETAG=jenkins-test", + "LONG=${myParams['LONG'] ? 1 : 0}", + "TESTOPTIONS=${myParams['TESTOPTIONS']}", + ]) { + sh "make" + sh "make run-unit-tests" + sh "make docker-test" + } + } + } +} + def buildTestSteps(Map myParams, String kubeConfigRoot, String kubeconfig) { return { timestamps { @@ -96,34 +114,19 @@ pipeline { string(name: 'ENTERPRISEIMAGE', defaultValue: '', description: 'ENTERPRISEIMAGE sets the docker image used for enterprise tests)', ) } stages { - def myParams = [:]; - stage("Prepare") { - steps { - script { - myParams = fetchParamsFromGitLog(); - } - } - } stage('Build') { steps { - timestamps { - withEnv([ - "DEPLOYMENTNAMESPACE=${myParams['TESTNAMESPACE']}-${env.GIT_COMMIT}", - "DOCKERNAMESPACE=${myParams['DOCKERNAMESPACE']}", - "IMAGETAG=jenkins-test", - "LONG=${myParams['LONG'] ? 1 : 0}", - "TESTOPTIONS=${myParams['TESTOPTIONS']}", - ]) { - sh "make" - sh "make run-unit-tests" - sh "make docker-test" - } + script { + def myParams = fetchParamsFromGitLog(); + def buildSteps = buildBuildSteps(myParams); + buildSteps(); } } } stage('Test') { steps { script { + def myParams = fetchParamsFromGitLog(); def configs = "${myParams['KUBECONFIGS']}".split(",") def testTasks = [:] for (kubeconfig in configs) { @@ -138,6 +141,7 @@ pipeline { post { always { script { + def myParams = fetchParamsFromGitLog(); def configs = "${myParams['KUBECONFIGS']}".split(",") def cleanupTasks = [:] for (kubeconfig in configs) { From 1ac163dd8ed0fe2ffa3c49aa3c9b102af8608c3b Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 10:16:11 +0200 Subject: [PATCH 16/20] Fixing groovy --- Jenkinsfile.groovy | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index bff19f038..cd1269936 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -23,8 +23,6 @@ def fetchParamsFromGitLog() { def myParams = [:]; // Copy configured params for (entry in params) { - println("Fetching entry from params: ${entry}"); - println("Fetching entry from params: key=${entry.key}, value=${entry.value}"); myParams[entry.key] = entry.value; } @@ -33,9 +31,7 @@ def fetchParamsFromGitLog() { // Example: [ci TESTOPTIONS="-test.run ^TestSimpleSingle$"] def options = sh(returnStdout: true, script: "git log --reverse remotes/origin/master..HEAD | grep -o \'\\[ci[^\\[]*\\]\' | sed -E \'s/\\[ci (.*)\\]/\\1/\'").trim().split("\n") for (opt in options) { - println("Processing option ${opt}"); def idx = opt.indexOf('='); - println("Processing option ${opt} with idx=${idx}"); if (idx > 0) { def key = opt.substring(0, idx); def value = opt.substring(idx+1); @@ -52,11 +48,11 @@ def buildBuildSteps(Map myParams) { return { timestamps { withEnv([ - "DEPLOYMENTNAMESPACE=${myParams['TESTNAMESPACE']}-${env.GIT_COMMIT}", - "DOCKERNAMESPACE=${myParams['DOCKERNAMESPACE']}", + "DEPLOYMENTNAMESPACE=${myParams.TESTNAMESPACE}-${env.GIT_COMMIT}", + "DOCKERNAMESPACE=${myParams.DOCKERNAMESPACE}", "IMAGETAG=jenkins-test", - "LONG=${myParams['LONG'] ? 1 : 0}", - "TESTOPTIONS=${myParams['TESTOPTIONS']}", + "LONG=${myParams.LONG ? 1 : 0}", + "TESTOPTIONS=${myParams.TESTOPTIONS}", ]) { sh "make" sh "make run-unit-tests" @@ -71,13 +67,13 @@ def buildTestSteps(Map myParams, String kubeConfigRoot, String kubeconfig) { timestamps { withCredentials([string(credentialsId: 'ENTERPRISEIMAGE', variable: 'DEFAULTENTERPRISEIMAGE')]) { withEnv([ - "DEPLOYMENTNAMESPACE=${myParams['TESTNAMESPACE']}-${env.GIT_COMMIT}", - "DOCKERNAMESPACE=${myParams['DOCKERNAMESPACE']}", - "ENTERPRISEIMAGE=${myParams['ENTERPRISEIMAGE']}", + "DEPLOYMENTNAMESPACE=${myParams.TESTNAMESPACE}-${env.GIT_COMMIT}", + "DOCKERNAMESPACE=${myParams.DOCKERNAMESPACE}", + "ENTERPRISEIMAGE=${myParams.ENTERPRISEIMAGE}", "IMAGETAG=jenkins-test", "KUBECONFIG=${kubeConfigRoot}/${kubeconfig}", - "LONG=${myParams['LONG'] ? 1 : 0}", - "TESTOPTIONS=${myParams['TESTOPTIONS']}", + "LONG=${myParams.LONG ? 1 : 0}", + "TESTOPTIONS=${myParams.TESTOPTIONS}", ]) { sh "make run-tests" } @@ -90,8 +86,8 @@ def buildCleanupSteps(Map myParams, String kubeConfigRoot, String kubeconfig) { return { timestamps { withEnv([ - "DEPLOYMENTNAMESPACE=${myParams['TESTNAMESPACE']}-${env.GIT_COMMIT}", - "DOCKERNAMESPACE=${myParams['DOCKERNAMESPACE']}", + "DEPLOYMENTNAMESPACE=${myParams.TESTNAMESPACE}-${env.GIT_COMMIT}", + "DOCKERNAMESPACE=${myParams.DOCKERNAMESPACE}", "KUBECONFIG=${kubeConfigRoot}/${kubeconfig}", ]) { sh "make cleanup-tests" @@ -127,7 +123,7 @@ pipeline { steps { script { def myParams = fetchParamsFromGitLog(); - def configs = "${myParams['KUBECONFIGS']}".split(",") + def configs = "${myParams.KUBECONFIGS}".split(",") def testTasks = [:] for (kubeconfig in configs) { testTasks["${kubeconfig}"] = buildTestSteps(myParams, kubeConfigRoot, kubeconfig) From 9a649964b476739beaee019d86afd3d45daf19c8 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 10:22:42 +0200 Subject: [PATCH 17/20] Remove value quotes --- Jenkinsfile.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index cd1269936..107219910 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -34,7 +34,7 @@ def fetchParamsFromGitLog() { def idx = opt.indexOf('='); if (idx > 0) { def key = opt.substring(0, idx); - def value = opt.substring(idx+1); + def value = opt.substring(idx+1).replaceAll("^\"|\"$", ""); myParams[key] = value; println("Overwriting myParams.${key} with ${value}"); } From efcb6fdf4b813f2696d8ae54479901b6e2365f28 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 10:23:43 +0200 Subject: [PATCH 18/20] Remove value quotes --- Jenkinsfile.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index 107219910..691e63d57 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -34,7 +34,7 @@ def fetchParamsFromGitLog() { def idx = opt.indexOf('='); if (idx > 0) { def key = opt.substring(0, idx); - def value = opt.substring(idx+1).replaceAll("^\"|\"$", ""); + def value = opt.substring(idx+1).replaceAll('^\"|\"$', ''); myParams[key] = value; println("Overwriting myParams.${key} with ${value}"); } From 0070a4457b436bab81700c815f4256c9b9375d34 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 10:29:42 +0200 Subject: [PATCH 19/20] Added TESTOPTIONS [ci TESTOPTIONS=-test.run ^TestSimpleResilientSingle$] --- Jenkinsfile.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index 691e63d57..ebcfa3f61 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -108,6 +108,7 @@ pipeline { string(name: 'KUBECONFIGS', defaultValue: 'kube-ams1,scw-183a3b', description: 'KUBECONFIGS is a comma separated list of Kubernetes configuration files (relative to /home/jenkins/.kube) on which the tests are run', ) string(name: 'TESTNAMESPACE', defaultValue: 'jenkins', description: 'TESTNAMESPACE sets the kubernetes namespace to ru tests in (this must be short!!)', ) string(name: 'ENTERPRISEIMAGE', defaultValue: '', description: 'ENTERPRISEIMAGE sets the docker image used for enterprise tests)', ) + string(name: 'TESTOPTIONS', defaultValue: '', description: 'TESTOPTIONS is used to pass additional test options to the integration test', ) } stages { stage('Build') { From 779772d155bdd124757ea855c9af845c2b8685ee Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 10:31:38 +0200 Subject: [PATCH 20/20] Improve log output --- Jenkinsfile.groovy | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index ebcfa3f61..312305be3 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -36,9 +36,15 @@ def fetchParamsFromGitLog() { def key = opt.substring(0, idx); def value = opt.substring(idx+1).replaceAll('^\"|\"$', ''); myParams[key] = value; - println("Overwriting myParams.${key} with ${value}"); + //println("Overwriting myParams.${key} with ${value}"); } } + + // Show params in log + for (entry in myParams) { + println("Using myParams.${entry.key} with ${entry.value}"); + } + return myParams; }