From c1d9db26571df777eb128e844a7e665843d4e58e Mon Sep 17 00:00:00 2001 From: Marek Markiewka Date: Tue, 20 Apr 2021 10:44:28 +0200 Subject: [PATCH 1/9] first draft of helmrepo implementation Signed-off-by: Marek Markiewka --- .../gitopsbuildlib/deployment/helm/Helm.groovy | 6 ++---- .../deployment/helm/helmrelease/ArgoCDRelease.groovy | 12 ++++++++++-- .../deployment/helm/repotype/HelmRepo.groovy | 2 -- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/helm/Helm.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/helm/Helm.groovy index 13e27f7..69e47bc 100644 --- a/src/com/cloudogu/gitopsbuildlib/deployment/helm/Helm.groovy +++ b/src/com/cloudogu/gitopsbuildlib/deployment/helm/Helm.groovy @@ -43,10 +43,8 @@ class Helm extends Deployment { // since the values are already inline (helmRelease.yaml) we do not need to commit them into the gitops repo script.sh "rm ${stage}/${application}/mergedValues.yaml" - // clean the gitrepo helm chart folder since the helmRelease.yaml ist now created - if (helmConfig.repoType == 'GIT') { - script.sh "rm -rf ${script.env.WORKSPACE}/chart || true" - } + // clean the helm chart folder since the helmRelease.yaml ist now created + script.sh "rm -rf ${script.env.WORKSPACE}/chart || true" } @Override diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDRelease.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDRelease.groovy index 7729d0d..27c8e1a 100644 --- a/src/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDRelease.groovy +++ b/src/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDRelease.groovy @@ -16,19 +16,27 @@ class ArgoCDRelease extends HelmRelease{ if (helmConfig.repoType == 'GIT') { helmRelease = gitRepoRelease(helmConfig, application, valuesFileLocation) } else if (helmConfig.repoType == 'HELM') { - // TODO not yet implemented + helmRelease = helmRepoRelease(helmConfig, application, valuesFileLocation) } return helmRelease } private String gitRepoRelease(Map helmConfig, String application, String valuesFileLocation) { - String helmRelease = "" def chartPath = '' if (helmConfig.containsKey('chartPath')) { chartPath = helmConfig.chartPath } + return createHelmRelease(chartPath as String, application, valuesFileLocation) + } + + private String helmRepoRelease(Map helmConfig, String application, String valuesFileLocation) { + return createHelmRelease(helmConfig.chartName as String, application, valuesFileLocation) + } + + private String createHelmRelease(String chartPath, String application, String valuesFileLocation) { + String helmRelease = "" withHelm { script.dir("${script.env.WORKSPACE}/chart/${chartPath}") { script.sh "helm dep update ." diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy index e8a9639..a666431 100644 --- a/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy +++ b/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy @@ -18,8 +18,6 @@ class HelmRepo extends RepoType{ merge = script.sh returnStdout: true, script: helmScript } - script.sh "rm -rf ${script.env.WORKSPACE}/chart || true" - return merge } } From 325b765d770a5a2fe8158cd0956a7686e7fec713 Mon Sep 17 00:00:00 2001 From: Marek Markiewka Date: Tue, 20 Apr 2021 15:08:33 +0200 Subject: [PATCH 2/9] enabling private helm repos Signed-off-by: Marek Markiewka --- README.md | 2 +- .../deployment/helm/repotype/HelmRepo.groovy | 10 +++++++ .../helm/helmrelease/ArgoCDReleaseTest.groovy | 27 +++++++++++++++---- .../helm/helmrelease/FluxV1ReleaseTest.groovy | 4 +-- 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index bac16a0..5f33cec 100644 --- a/README.md +++ b/README.md @@ -255,7 +255,7 @@ application: 'spring-petclinic' // Name of the application. Used as a ``` ``` -gitopsTool: 'ARGO' // Name of the gitops tool. Currently supporting 'FLUX' (for now only fluxV1) and 'ARGO' (for now supporting only helm charts from git repos) +gitopsTool: 'ARGO' // Name of the gitops tool. Currently supporting 'FLUX' (for now only fluxV1) and 'ARGO' ``` and some optional parameters (below are the defaults) for the configuration of the dependency to the ces-build-lib or the default name for the git branch: diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy index a666431..bce43da 100644 --- a/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy +++ b/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy @@ -10,7 +10,17 @@ class HelmRepo extends RepoType{ String mergeValues(Map helmConfig, String[] valuesFiles) { String merge = "" + if (helmConfig.containsKey('credentialsId') && helmConfig.credentialsId) { + withCredentials([usernamePassword(credentialsId: helmConfig.credentialsId, usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { + mergeValuesFiles() + } + } else { + mergeValuesFiles() + } + withHelm { + script.sh "echo \"$USERNAME\"" + script.sh "echo \"$Password\"" script.sh "helm repo add chartRepo ${helmConfig.repoUrl}" script.sh "helm repo update" script.sh "helm pull chartRepo/${helmConfig.chartName} --version=${helmConfig.version} --untar --untardir=${script.env.WORKSPACE}/chart" diff --git a/test/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDReleaseTest.groovy b/test/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDReleaseTest.groovy index 2b0f4b8..f19497a 100644 --- a/test/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDReleaseTest.groovy +++ b/test/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDReleaseTest.groovy @@ -21,10 +21,10 @@ class ArgoCDReleaseTest { ], 'app', 'namespace', - 'this/is/a/valusfile') + 'this/is/a/valuesfile') assertThat(scriptMock.actualShArgs[0]).isEqualTo('helm dep update .') - assertThat(scriptMock.actualShArgs[1]).isEqualTo('[returnStdout:true, script:helm template app . -f workspace/.configRepoTempDir/this/is/a/valusfile]') + assertThat(scriptMock.actualShArgs[1]).isEqualTo('[returnStdout:true, script:helm template app . -f workspace/.configRepoTempDir/this/is/a/valuesfile]') assertThat(scriptMock.actualDir[0]).isEqualTo('workspace/chart/path') } @@ -38,10 +38,27 @@ class ArgoCDReleaseTest { ], 'app', 'namespace', - 'this/is/a/valusfile') + 'this/is/a/valuesfile') assertThat(scriptMock.actualShArgs[0]).isEqualTo('helm dep update .') - assertThat(scriptMock.actualShArgs[1]).isEqualTo('[returnStdout:true, script:helm template app . -f workspace/.configRepoTempDir/this/is/a/valusfile]') - assertThat(scriptMock.actualDir[0]).isEqualTo('workspace/chart/') + assertThat(scriptMock.actualShArgs[1]).isEqualTo('[returnStdout:true, script:helm template app . -f workspace/.configRepoTempDir/this/is/a/valuesfile]') + assertThat(scriptMock.actualDir[0]).isEqualTo('workspace/chart/chartName') + } + + @Test + void 'correct helm release with helm repo'() { + argoCdReleaseTest.create([ + repoType: 'HELM', + repoUrl: 'url', + chartName: 'chartName', + version: '1.0' + ], + 'app', + 'namespace', + 'this/is/a/valuesfile') + + assertThat(scriptMock.actualShArgs[0]).isEqualTo('helm dep update .') + assertThat(scriptMock.actualShArgs[1]).isEqualTo('[returnStdout:true, script:helm template app . -f workspace/.configRepoTempDir/this/is/a/valuesfile]') + assertThat(scriptMock.actualDir[0]).isEqualTo('workspace/chart/chartName') } } diff --git a/test/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/FluxV1ReleaseTest.groovy b/test/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/FluxV1ReleaseTest.groovy index c0ef668..a7f1f8d 100644 --- a/test/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/FluxV1ReleaseTest.groovy +++ b/test/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/FluxV1ReleaseTest.groovy @@ -20,7 +20,7 @@ class FluxV1ReleaseTest { ], 'app', 'namespace', - 'this/is/a/valusfile') + 'this/is/a/valuesfile') assertThat(output).isEqualTo("""apiVersion: helm.fluxcd.io/v1 kind: HelmRelease @@ -61,7 +61,7 @@ spec: ], 'app', 'namespace', - 'this/is/a/valusfile') + 'this/is/a/valuesfile') assertThat(output).isEqualTo("""apiVersion: helm.fluxcd.io/v1 kind: HelmRelease From 9ce668e87ba8562f749f4b081b659f0c98333f8f Mon Sep 17 00:00:00 2001 From: Marek Markiewka Date: Tue, 20 Apr 2021 16:17:11 +0200 Subject: [PATCH 3/9] fixing tests; helm repo with username and password Signed-off-by: Marek Markiewka --- .../deployment/helm/repotype/HelmRepo.groovy | 22 +++++++++++++------ .../gitopsbuildlib/deployment/HelmTest.groovy | 5 ++--- .../helm/helmrelease/ArgoCDReleaseTest.groovy | 2 +- .../helm/repotype/HelmRepoTest.groovy | 1 - 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy index bce43da..5852863 100644 --- a/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy +++ b/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy @@ -8,20 +8,28 @@ class HelmRepo extends RepoType{ @Override String mergeValues(Map helmConfig, String[] valuesFiles) { - String merge = "" if (helmConfig.containsKey('credentialsId') && helmConfig.credentialsId) { - withCredentials([usernamePassword(credentialsId: helmConfig.credentialsId, usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - mergeValuesFiles() + println "creds exist" + script.withCredentials([ + script.usernamePassword( + credentialsId: helmConfig.credentialsId, + usernameVariable: 'USERNAME', + passwordVariable: 'PASSWORD') + ]) { + String credentialArgs = " --username ${script.USERNAME} --password ${script.PASSWORD}" + return mergeValuesFiles(helmConfig, valuesFiles, credentialArgs) } } else { - mergeValuesFiles() + return mergeValuesFiles(helmConfig, valuesFiles) } + } + + private String mergeValuesFiles(Map helmConfig, String[] valuesFiles, String credentialArgs = "") { + String merge = "" withHelm { - script.sh "echo \"$USERNAME\"" - script.sh "echo \"$Password\"" - script.sh "helm repo add chartRepo ${helmConfig.repoUrl}" + script.sh "helm repo add chartRepo ${helmConfig.repoUrl}${credentialArgs}" script.sh "helm repo update" script.sh "helm pull chartRepo/${helmConfig.chartName} --version=${helmConfig.version} --untar --untardir=${script.env.WORKSPACE}/chart" String helmScript = "helm values ${script.env.WORKSPACE}/chart/${helmConfig.chartName} ${valuesFilesWithParameter(valuesFiles)}" diff --git a/test/com/cloudogu/gitopsbuildlib/deployment/HelmTest.groovy b/test/com/cloudogu/gitopsbuildlib/deployment/HelmTest.groovy index b04ce87..63d167f 100644 --- a/test/com/cloudogu/gitopsbuildlib/deployment/HelmTest.groovy +++ b/test/com/cloudogu/gitopsbuildlib/deployment/HelmTest.groovy @@ -50,7 +50,6 @@ class HelmTest { repoType: 'HELM', repoUrl: 'repoUrl', chartName: 'chartName', - credentials: 'creds', version: '1.0' ] ], @@ -113,8 +112,8 @@ class HelmTest { assertThat(scriptMock.actualShArgs[1]).isEqualTo('helm repo update') assertThat(scriptMock.actualShArgs[2]).isEqualTo('helm pull chartRepo/chartName --version=1.0 --untar --untardir=workspace/chart') assertThat(scriptMock.actualShArgs[3]).isEqualTo('[returnStdout:true, script:helm values workspace/chart/chartName -f workspace/k8s/values-staging.yaml -f workspace/k8s/values-shared.yaml ]') - assertThat(scriptMock.actualShArgs[4]).isEqualTo('rm -rf workspace/chart || true') - assertThat(scriptMock.actualShArgs[5]).isEqualTo('rm staging/testapp/mergedValues.yaml') + assertThat(scriptMock.actualShArgs[4]).isEqualTo('rm staging/testapp/mergedValues.yaml') + assertThat(scriptMock.actualShArgs[5]).isEqualTo('rm -rf workspace/chart || true') assertThat(scriptMock.actualWriteFileArgs[0]).isEqualTo('[file:staging/testapp/mergedValues.yaml, ' + 'text:[helm repo add chartRepo repoUrl, helm repo update, ' + 'helm pull chartRepo/chartName --version=1.0 --untar --untardir=workspace/chart, ' + diff --git a/test/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDReleaseTest.groovy b/test/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDReleaseTest.groovy index f19497a..84dd9da 100644 --- a/test/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDReleaseTest.groovy +++ b/test/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDReleaseTest.groovy @@ -42,7 +42,7 @@ class ArgoCDReleaseTest { assertThat(scriptMock.actualShArgs[0]).isEqualTo('helm dep update .') assertThat(scriptMock.actualShArgs[1]).isEqualTo('[returnStdout:true, script:helm template app . -f workspace/.configRepoTempDir/this/is/a/valuesfile]') - assertThat(scriptMock.actualDir[0]).isEqualTo('workspace/chart/chartName') + assertThat(scriptMock.actualDir[0]).isEqualTo('workspace/chart/') } @Test diff --git a/test/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepoTest.groovy b/test/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepoTest.groovy index 1b4ec30..8f97ec9 100644 --- a/test/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepoTest.groovy +++ b/test/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepoTest.groovy @@ -24,6 +24,5 @@ class HelmRepoTest { assertThat(scriptMock.actualShArgs[1]).isEqualTo('helm repo update') assertThat(scriptMock.actualShArgs[2]).isEqualTo('helm pull chartRepo/chartName --version=1.0 --untar --untardir=workspace/chart') assertThat(scriptMock.actualShArgs[3]).isEqualTo('[returnStdout:true, script:helm values workspace/chart/chartName -f file1 -f file2 ]') - assertThat(scriptMock.actualShArgs[4]).isEqualTo('rm -rf workspace/chart || true') } } From 14568ee2fe99a0931b9f4809e05ccb8e10ec7597 Mon Sep 17 00:00:00 2001 From: Marek Markiewka Date: Wed, 21 Apr 2021 17:22:03 +0200 Subject: [PATCH 4/9] removed println Signed-off-by: Marek Markiewka --- .../gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy index 5852863..266c5db 100644 --- a/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy +++ b/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy @@ -10,7 +10,6 @@ class HelmRepo extends RepoType{ String mergeValues(Map helmConfig, String[] valuesFiles) { if (helmConfig.containsKey('credentialsId') && helmConfig.credentialsId) { - println "creds exist" script.withCredentials([ script.usernamePassword( credentialsId: helmConfig.credentialsId, From e2130c7b0b33af4316d751b0aa70dae487057b1a Mon Sep 17 00:00:00 2001 From: Marek Markiewka Date: Thu, 22 Apr 2021 15:28:50 +0200 Subject: [PATCH 5/9] fixed helm repo folder structure Signed-off-by: Marek Markiewka --- .../gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy index 266c5db..7c70beb 100644 --- a/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy +++ b/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy @@ -30,8 +30,8 @@ class HelmRepo extends RepoType{ withHelm { script.sh "helm repo add chartRepo ${helmConfig.repoUrl}${credentialArgs}" script.sh "helm repo update" - script.sh "helm pull chartRepo/${helmConfig.chartName} --version=${helmConfig.version} --untar --untardir=${script.env.WORKSPACE}/chart" - String helmScript = "helm values ${script.env.WORKSPACE}/chart/${helmConfig.chartName} ${valuesFilesWithParameter(valuesFiles)}" + script.sh "helm pull chartRepo/${helmConfig.chartName} --version=${helmConfig.version} --untar --untardir=chart" + String helmScript = "helm values chart/${helmConfig.chartName} ${valuesFilesWithParameter(valuesFiles)}" merge = script.sh returnStdout: true, script: helmScript } From 3720c26eb76a0294e00d0781e40987145a796c16 Mon Sep 17 00:00:00 2001 From: Marek Markiewka Date: Thu, 22 Apr 2021 16:13:26 +0200 Subject: [PATCH 6/9] plain vs helm folder structure fix Signed-off-by: Marek Markiewka --- .../cloudogu/gitopsbuildlib/deployment/Deployment.groovy | 7 ++++--- .../cloudogu/gitopsbuildlib/deployment/helm/Helm.groovy | 1 + .../deployment/helm/helmrelease/ArgoCDRelease.groovy | 7 +++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/Deployment.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/Deployment.groovy index 8d9190d..8a11608 100644 --- a/src/com/cloudogu/gitopsbuildlib/deployment/Deployment.groovy +++ b/src/com/cloudogu/gitopsbuildlib/deployment/Deployment.groovy @@ -3,6 +3,7 @@ package com.cloudogu.gitopsbuildlib.deployment abstract class Deployment { protected static String getKubectlImage() { 'lachlanevenson/k8s-kubectl:v1.19.3' } + protected String extraResourcesFolder = "" static String getConfigDir() { '.config' } @@ -36,11 +37,11 @@ abstract class Deployment { def sourcePath = gitopsConfig.deployments.sourcePath def application = gitopsConfig.application - script.sh "mkdir -p ${stage}/${application}/extraResources/" + script.sh "mkdir -p ${stage}/${application}/${extraResourcesFolder}" script.sh "mkdir -p ${configDir}/" // copy extra resources like sealed secrets - script.echo "Copying k8s payload from application repo to gitOps Repo: '${sourcePath}/${stage}/*' to '${stage}/${application}/extraResources/'" - script.sh "cp -r ${script.env.WORKSPACE}/${sourcePath}/${stage}/* ${stage}/${application}/extraResources/ || true" + script.echo "Copying k8s payload from application repo to gitOps Repo: '${sourcePath}/${stage}/*' to '${stage}/${application}/${extraResourcesFolder}'" + script.sh "cp -r ${script.env.WORKSPACE}/${sourcePath}/${stage}/* ${stage}/${application}/${extraResourcesFolder} || true" script.sh "cp ${script.env.WORKSPACE}/*.yamllint.yaml ${configDir}/ || true" } diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/helm/Helm.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/helm/Helm.groovy index f08c631..b0722b3 100644 --- a/src/com/cloudogu/gitopsbuildlib/deployment/helm/Helm.groovy +++ b/src/com/cloudogu/gitopsbuildlib/deployment/helm/Helm.groovy @@ -15,6 +15,7 @@ class Helm extends Deployment { Helm(def script, def gitopsConfig) { super(script, gitopsConfig) + this.extraResourcesFolder = "extraResources" if (gitopsConfig.deployments.helm.repoType == 'GIT') { chartRepo = new GitRepo(script) } else if (gitopsConfig.deployments.helm.repoType == 'HELM') { diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDRelease.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDRelease.groovy index f61c51b..747915e 100644 --- a/src/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDRelease.groovy +++ b/src/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDRelease.groovy @@ -40,11 +40,10 @@ class ArgoCDRelease extends HelmRelease{ private String createHelmRelease(String chartPath, String application, String mergedValuesFile) { String helmRelease = "" dockerWrapper.withHelm { - script.dir("chart/${chartPath}") { - String templateScript = "helm template ${application} . -f ${mergedValuesFile}" - helmRelease = script.sh returnStdout: true, script: templateScript - } + String templateScript = "helm template ${application} chart/${chartPath} -f ${mergedValuesFile}" + helmRelease = script.sh returnStdout: true, script: templateScript } + // this line removes all empty lines since helm template creates some and the helm validator will throw an error if there are emtpy lines present helmRelease = helmRelease.replaceAll("(?m)^[ \t]*\r?\n", "") return helmRelease From 644e1ac5b5e3374a9043af80d2eeff75ea92ce8f Mon Sep 17 00:00:00 2001 From: Marek Markiewka Date: Thu, 22 Apr 2021 17:10:29 +0200 Subject: [PATCH 7/9] bugfixes Signed-off-by: Marek Markiewka --- .../gitopsbuildlib/deployment/plain/Plain.groovy | 2 +- .../gitopsbuildlib/deployment/DeploymentTest.groovy | 6 +++--- .../cloudogu/gitopsbuildlib/deployment/HelmTest.groovy | 9 ++++----- .../deployment/helm/helmrelease/ArgoCDReleaseTest.groovy | 8 +++----- .../deployment/helm/repotype/HelmRepoTest.groovy | 4 ++-- 5 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/plain/Plain.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/plain/Plain.groovy index 11729fd..c390727 100644 --- a/src/com/cloudogu/gitopsbuildlib/deployment/plain/Plain.groovy +++ b/src/com/cloudogu/gitopsbuildlib/deployment/plain/Plain.groovy @@ -19,7 +19,7 @@ class Plain extends Deployment{ private updateImage(String stage) { gitopsConfig.deployments.plain.updateImages.each { - def deploymentFilePath = "${stage}/${gitopsConfig.application}/${gitopsConfig.deployments.sourcePath}/${it['filename']}" + def deploymentFilePath = "${stage}/${gitopsConfig.application}/${it['filename']}" def data = script.readYaml file: deploymentFilePath def containers = data.spec.template.spec.containers def containerName = it['containerName'] diff --git a/test/com/cloudogu/gitopsbuildlib/deployment/DeploymentTest.groovy b/test/com/cloudogu/gitopsbuildlib/deployment/DeploymentTest.groovy index f28b974..47923e1 100644 --- a/test/com/cloudogu/gitopsbuildlib/deployment/DeploymentTest.groovy +++ b/test/com/cloudogu/gitopsbuildlib/deployment/DeploymentTest.groovy @@ -56,10 +56,10 @@ class DeploymentTest { @Test void 'creating folders for plain deployment'() { deploymentUnderTest.createFoldersAndCopyK8sResources('staging',) - assertThat(scriptMock.actualEchoArgs[0]).isEqualTo('Copying k8s payload from application repo to gitOps Repo: \'k8s/staging/*\' to \'staging/app/extraResources/\'') - assertThat(scriptMock.actualShArgs[0]).isEqualTo('mkdir -p staging/app/extraResources/') + assertThat(scriptMock.actualEchoArgs[0]).isEqualTo('Copying k8s payload from application repo to gitOps Repo: \'k8s/staging/*\' to \'staging/app/\'') + assertThat(scriptMock.actualShArgs[0]).isEqualTo('mkdir -p staging/app/') assertThat(scriptMock.actualShArgs[1]).isEqualTo('mkdir -p .config/') - assertThat(scriptMock.actualShArgs[2]).isEqualTo('cp -r workspace/k8s/staging/* staging/app/extraResources/ || true') + assertThat(scriptMock.actualShArgs[2]).isEqualTo('cp -r workspace/k8s/staging/* staging/app/ || true') assertThat(scriptMock.actualShArgs[3]).isEqualTo('cp workspace/*.yamllint.yaml .config/ || true') } diff --git a/test/com/cloudogu/gitopsbuildlib/deployment/HelmTest.groovy b/test/com/cloudogu/gitopsbuildlib/deployment/HelmTest.groovy index feca6df..65fca8a 100644 --- a/test/com/cloudogu/gitopsbuildlib/deployment/HelmTest.groovy +++ b/test/com/cloudogu/gitopsbuildlib/deployment/HelmTest.groovy @@ -105,14 +105,13 @@ spec: assertThat(dockerMock.actualImages[0]).contains('ghcr.io/cloudogu/helm:') assertThat(scriptMock.actualShArgs[0]).isEqualTo('helm repo add chartRepo repoUrl') assertThat(scriptMock.actualShArgs[1]).isEqualTo('helm repo update') - assertThat(scriptMock.actualShArgs[2]).isEqualTo('helm pull chartRepo/chartName --version=1.0 --untar --untardir=workspace/chart') - assertThat(scriptMock.actualShArgs[3]).isEqualTo('[returnStdout:true, script:helm values workspace/chart/chartName -f workspace/k8s/values-staging.yaml -f workspace/k8s/values-shared.yaml ]') + assertThat(scriptMock.actualShArgs[2]).isEqualTo('helm pull chartRepo/chartName --version=1.0 --untar --untardir=chart') + assertThat(scriptMock.actualShArgs[3]).isEqualTo('[returnStdout:true, script:helm values chart/chartName -f workspace/k8s/values-staging.yaml -f workspace/k8s/values-shared.yaml ]') assertThat(scriptMock.actualShArgs[4]).isEqualTo('rm staging/testapp/mergedValues.yaml') - assertThat(scriptMock.actualShArgs[5]).isEqualTo('rm -rf workspace/chart || true') assertThat(scriptMock.actualWriteFileArgs[0]).isEqualTo('[file:staging/testapp/mergedValues.yaml, ' + 'text:[helm repo add chartRepo repoUrl, helm repo update, ' + - 'helm pull chartRepo/chartName --version=1.0 --untar --untardir=workspace/chart, ' + - '[returnStdout:true, script:helm values workspace/chart/chartName -f workspace/k8s/values-staging.yaml -f workspace/k8s/values-shared.yaml ]]]') + 'helm pull chartRepo/chartName --version=1.0 --untar --untardir=chart, ' + + '[returnStdout:true, script:helm values chart/chartName -f workspace/k8s/values-staging.yaml -f workspace/k8s/values-shared.yaml ]]]') assertThat(scriptMock.actualWriteFileArgs[1]).isEqualTo('''[file:staging/testapp/applicationRelease.yaml, text:apiVersion: helm.fluxcd.io/v1 kind: HelmRelease metadata: diff --git a/test/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDReleaseTest.groovy b/test/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDReleaseTest.groovy index f4ee718..6825cd4 100644 --- a/test/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDReleaseTest.groovy +++ b/test/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDReleaseTest.groovy @@ -23,7 +23,7 @@ class ArgoCDReleaseTest { 'namespace', 'this/is/a/valuesfile') - assertThat(scriptMock.actualShArgs[0]).isEqualTo('[returnStdout:true, script:helm template app chart/path -f this/is/a/valusfile]') + assertThat(scriptMock.actualShArgs[0]).isEqualTo('[returnStdout:true, script:helm template app chart/path -f this/is/a/valuesfile]') } @Test @@ -38,7 +38,7 @@ class ArgoCDReleaseTest { 'namespace', 'this/is/a/valuesfile') - assertThat(scriptMock.actualShArgs[0]).isEqualTo('[returnStdout:true, script:helm template app chart/ -f this/is/a/valusfile]') + assertThat(scriptMock.actualShArgs[0]).isEqualTo('[returnStdout:true, script:helm template app chart/ -f this/is/a/valuesfile]') } @Test @@ -53,8 +53,6 @@ class ArgoCDReleaseTest { 'namespace', 'this/is/a/valuesfile') - assertThat(scriptMock.actualShArgs[0]).isEqualTo('helm dep update .') - assertThat(scriptMock.actualShArgs[1]).isEqualTo('[returnStdout:true, script:helm template app . -f workspace/.configRepoTempDir/this/is/a/valuesfile]') - assertThat(scriptMock.actualDir[0]).isEqualTo('workspace/chart/chartName') + assertThat(scriptMock.actualShArgs[0]).isEqualTo('[returnStdout:true, script:helm template app chart/chartName -f this/is/a/valuesfile]') } } diff --git a/test/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepoTest.groovy b/test/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepoTest.groovy index 8f97ec9..a2c4e34 100644 --- a/test/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepoTest.groovy +++ b/test/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepoTest.groovy @@ -22,7 +22,7 @@ class HelmRepoTest { assertThat(scriptMock.actualShArgs[0]).isEqualTo('helm repo add chartRepo url') assertThat(scriptMock.actualShArgs[1]).isEqualTo('helm repo update') - assertThat(scriptMock.actualShArgs[2]).isEqualTo('helm pull chartRepo/chartName --version=1.0 --untar --untardir=workspace/chart') - assertThat(scriptMock.actualShArgs[3]).isEqualTo('[returnStdout:true, script:helm values workspace/chart/chartName -f file1 -f file2 ]') + assertThat(scriptMock.actualShArgs[2]).isEqualTo('helm pull chartRepo/chartName --version=1.0 --untar --untardir=chart') + assertThat(scriptMock.actualShArgs[3]).isEqualTo('[returnStdout:true, script:helm values chart/chartName -f file1 -f file2 ]') } } From 0d8da183745e1b6546cd103c6ee83a529f70a615 Mon Sep 17 00:00:00 2001 From: Marek Markiewka Date: Fri, 23 Apr 2021 15:04:51 +0200 Subject: [PATCH 8/9] fixed folder structure; fixed kubeval for missing schemas Signed-off-by: Marek Markiewka --- .../deployment/Deployment.groovy | 3 +- .../deployment/helm/repotype/HelmRepo.groovy | 1 + .../validation/HelmKubeval.groovy | 35 +++++-------------- .../gitopsbuildlib/validation/Kubeval.groovy | 5 +-- .../validation/Validator.groovy | 4 +-- .../gitopsbuildlib/validation/Yamllint.groovy | 2 +- 6 files changed, 15 insertions(+), 35 deletions(-) diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/Deployment.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/Deployment.groovy index 8a11608..2d45070 100644 --- a/src/com/cloudogu/gitopsbuildlib/deployment/Deployment.groovy +++ b/src/com/cloudogu/gitopsbuildlib/deployment/Deployment.groovy @@ -26,10 +26,11 @@ abstract class Deployment { abstract preValidation(String stage) abstract postValidation(String stage) + def validate(String stage) { gitopsConfig.validators.each { validatorConfig -> script.echo "Executing validator ${validatorConfig.key}" - validatorConfig.value.validator.validate(validatorConfig.value.enabled, "${stage}/${gitopsConfig.application}", validatorConfig.value.config, gitopsConfig.deployments) + validatorConfig.value.validator.validate(validatorConfig.value.enabled, "${stage}/${gitopsConfig.application}", validatorConfig.value.config, gitopsConfig) } } diff --git a/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy b/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy index 7c70beb..8733329 100644 --- a/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy +++ b/src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy @@ -30,6 +30,7 @@ class HelmRepo extends RepoType{ withHelm { script.sh "helm repo add chartRepo ${helmConfig.repoUrl}${credentialArgs}" script.sh "helm repo update" + // helm pull also executes helm dependency so we don't need to do it in this step script.sh "helm pull chartRepo/${helmConfig.chartName} --version=${helmConfig.version} --untar --untardir=chart" String helmScript = "helm values chart/${helmConfig.chartName} ${valuesFilesWithParameter(valuesFiles)}" merge = script.sh returnStdout: true, script: helmScript diff --git a/src/com/cloudogu/gitopsbuildlib/validation/HelmKubeval.groovy b/src/com/cloudogu/gitopsbuildlib/validation/HelmKubeval.groovy index 766ed53..1fc2ce8 100644 --- a/src/com/cloudogu/gitopsbuildlib/validation/HelmKubeval.groovy +++ b/src/com/cloudogu/gitopsbuildlib/validation/HelmKubeval.groovy @@ -1,7 +1,5 @@ package com.cloudogu.gitopsbuildlib.validation -import com.cloudogu.gitopsbuildlib.docker.DockerWrapper - class HelmKubeval extends Validator { HelmKubeval(def script) { @@ -11,33 +9,16 @@ class HelmKubeval extends Validator { @Override void validate(String targetDirectory, Map config, Map deployments) { if (deployments.containsKey('helm')) { - if (deployments.helm.repoType == 'GIT') { -// script.dir("${targetDirectory}/chart") { -// def git = (deployments.helm.containsKey('credentialsId')) -// ? script.cesBuildLib.Git.new(script, deployments.helm.credentialsId) -// : script.cesBuildLib.Git.new(script) -// git url: deployments.helm.repoUrl, branch: 'main', changelog: false, poll: false -// -// if(deployments.helm.containsKey('version') && deployments.helm.version) { -// git.checkout(deployments.helm.version) -// } -// } - - def chartPath = '' - if (deployments.helm.containsKey('chartPath')) { - chartPath = deployments.helm.chartPath - } - withDockerImage(config.image) { - script.sh "helm kubeval chart/${chartPath} -v ${config.k8sSchemaVersion}" - } + def chartDir = '' + if (deployments.helm.containsKey('chartPath')) { + chartDir = deployments.helm.chartPath + } else if ( deployments.helm.containsKey('chartName')) { + chartDir = deployments.helm.chartName + } - } else if (deployments.helm.repoType == 'HELM') { - withDockerImage(config.image) { - script.sh "helm repo add chartRepo ${deployments.helm.repoUrl}" - script.sh "helm repo update" - script.sh "helm kubeval chartRepo/${deployments.helm.chartName} --version=${deployments.helm.version} -v ${config.k8sSchemaVersion}" - } + withDockerImage(config.image) { + script.sh "helm kubeval chart/${chartDir} -v ${config.k8sSchemaVersion}" } } } diff --git a/src/com/cloudogu/gitopsbuildlib/validation/Kubeval.groovy b/src/com/cloudogu/gitopsbuildlib/validation/Kubeval.groovy index f8c86aa..1ac36e7 100644 --- a/src/com/cloudogu/gitopsbuildlib/validation/Kubeval.groovy +++ b/src/com/cloudogu/gitopsbuildlib/validation/Kubeval.groovy @@ -1,13 +1,10 @@ package com.cloudogu.gitopsbuildlib.validation -import com.cloudogu.gitopsbuildlib.docker.DockerWrapper - /** * Validates all yaml-resources within the target-directory against the specs of the given k8s version */ class Kubeval extends Validator { - Kubeval(def script) { super(script) } @@ -15,7 +12,7 @@ class Kubeval extends Validator { @Override void validate(String targetDirectory, Map config, Map deployments) { withDockerImage(config.image) { - script.sh "kubeval -d ${targetDirectory} -v ${config.k8sSchemaVersion} --strict" + script.sh "kubeval -d ${targetDirectory} -v ${config.k8sSchemaVersion} --strict --ignore-missing-schemas" } } } diff --git a/src/com/cloudogu/gitopsbuildlib/validation/Validator.groovy b/src/com/cloudogu/gitopsbuildlib/validation/Validator.groovy index 294a610..46019ee 100644 --- a/src/com/cloudogu/gitopsbuildlib/validation/Validator.groovy +++ b/src/com/cloudogu/gitopsbuildlib/validation/Validator.groovy @@ -12,9 +12,9 @@ abstract class Validator { dockerWrapper = new DockerWrapper(script) } - void validate(boolean enabled, String targetDirectory, Map config, Map deployments) { + void validate(boolean enabled, String targetDirectory, Map config, Map gitopsConfig) { if (enabled) { - validate(targetDirectory, config, deployments) + validate(targetDirectory, config, gitopsConfig) } else { script.echo "Skipping validator ${this.getClass().getSimpleName()} because it is configured as enabled=false" } diff --git a/src/com/cloudogu/gitopsbuildlib/validation/Yamllint.groovy b/src/com/cloudogu/gitopsbuildlib/validation/Yamllint.groovy index 8427c77..8d5875f 100644 --- a/src/com/cloudogu/gitopsbuildlib/validation/Yamllint.groovy +++ b/src/com/cloudogu/gitopsbuildlib/validation/Yamllint.groovy @@ -16,7 +16,7 @@ class Yamllint extends Validator { } @Override - void validate(String targetDirectory, Map config, Map deployments) { + void validate(String targetDirectory, Map config, Map gitopsConfig) { withDockerImage(config.image) { script.sh "yamllint " + "${config.profile ? "-d ${config.profile} " : ''}" + From 734fb502eea94ad0b94adfbb10f67dbee09e22e8 Mon Sep 17 00:00:00 2001 From: pmarkiewka Date: Fri, 23 Apr 2021 20:59:54 +0200 Subject: [PATCH 9/9] fixes tests --- test/com/cloudogu/gitopsbuildlib/deployment/PlainTest.groovy | 4 ++-- .../cloudogu/gitopsbuildlib/validation/HelmKubevalTest.groovy | 4 +--- .../com/cloudogu/gitopsbuildlib/validation/KubevalTest.groovy | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/test/com/cloudogu/gitopsbuildlib/deployment/PlainTest.groovy b/test/com/cloudogu/gitopsbuildlib/deployment/PlainTest.groovy index bfbeb9b..94d8009 100644 --- a/test/com/cloudogu/gitopsbuildlib/deployment/PlainTest.groovy +++ b/test/com/cloudogu/gitopsbuildlib/deployment/PlainTest.groovy @@ -27,8 +27,8 @@ class PlainTest { void 'successful update'() { plain.postValidation('staging') - assertThat(scriptMock.actualReadYamlArgs[0]).isEqualTo('[file:staging/testApp/k8s/deployment.yaml]') - assertThat(scriptMock.actualWriteYamlArgs[0]).isEqualTo('[file:staging/testApp/k8s/deployment.yaml, data:[spec:[template:[spec:[containers:[[image:imageNameReplacedTest, name:application]]]]], to:[be:[changed:oldValue]]], overwrite:true]') + assertThat(scriptMock.actualReadYamlArgs[0]).isEqualTo('[file:staging/testApp/deployment.yaml]') + assertThat(scriptMock.actualWriteYamlArgs[0]).isEqualTo('[file:staging/testApp/deployment.yaml, data:[spec:[template:[spec:[containers:[[image:imageNameReplacedTest, name:application]]]]], to:[be:[changed:oldValue]]], overwrite:true]') } } diff --git a/test/com/cloudogu/gitopsbuildlib/validation/HelmKubevalTest.groovy b/test/com/cloudogu/gitopsbuildlib/validation/HelmKubevalTest.groovy index 7af0860..a843808 100644 --- a/test/com/cloudogu/gitopsbuildlib/validation/HelmKubevalTest.groovy +++ b/test/com/cloudogu/gitopsbuildlib/validation/HelmKubevalTest.groovy @@ -51,9 +51,7 @@ class HelmKubevalTest { ] ) assertThat(dockerMock.actualImages[0]).isEqualTo('img') - assertThat(scriptMock.actualShArgs[0]).isEqualTo('helm repo add chartRepo chartRepo') - assertThat(scriptMock.actualShArgs[1]).isEqualTo('helm repo update') - assertThat(scriptMock.actualShArgs[2]).isEqualTo('helm kubeval chartRepo/chart --version=version -v 1.5') + assertThat(scriptMock.actualShArgs[0]).isEqualTo('helm kubeval chart/chart -v 1.5') } @Test diff --git a/test/com/cloudogu/gitopsbuildlib/validation/KubevalTest.groovy b/test/com/cloudogu/gitopsbuildlib/validation/KubevalTest.groovy index 53b5358..686aa05 100644 --- a/test/com/cloudogu/gitopsbuildlib/validation/KubevalTest.groovy +++ b/test/com/cloudogu/gitopsbuildlib/validation/KubevalTest.groovy @@ -24,7 +24,7 @@ class KubevalTest { ) assertThat(dockerMock.actualImages[0]).isEqualTo('img') assertThat(scriptMock.actualShArgs[0]).isEqualTo( - 'kubeval -d target -v 1.5 --strict' + 'kubeval -d target -v 1.5 --strict --ignore-missing-schemas' ) } }