Feature/argocd helm helmrepo

README.md

@@ -255,7 +255,7 @@ application:            'spring-petclinic' // Name of the application. Used as a
 ```
 
 ```
-gitopsTool:            'ARGO' // Name of the gitops tool. Currently supporting 'FLUX' (for now only fluxV1) and 'ARGO' (for now supporting only helm charts from git repos)
+gitopsTool:            'ARGO' // Name of the gitops tool. Currently supporting 'FLUX' (for now only fluxV1) and 'ARGO'
 ```
 
 and some optional parameters (below are the defaults) for the configuration of the dependency to the ces-build-lib or the default name for the git branch:

src/com/cloudogu/gitopsbuildlib/deployment/Deployment.groovy

@@ -3,6 +3,7 @@ package com.cloudogu.gitopsbuildlib.deployment
 abstract class Deployment {
 
     protected static String getKubectlImage() { 'lachlanevenson/k8s-kubectl:v1.19.3' }
+    protected String extraResourcesFolder = ""
 
     static String getConfigDir() { '.config' }
 
@@ -25,22 +26,23 @@ abstract class Deployment {
     abstract preValidation(String stage)
     abstract postValidation(String stage)
 
+
     def validate(String stage) {
         gitopsConfig.validators.each { validatorConfig ->
             script.echo "Executing validator ${validatorConfig.key}"
-            validatorConfig.value.validator.validate(validatorConfig.value.enabled, "${stage}/${gitopsConfig.application}", validatorConfig.value.config, gitopsConfig.deployments)
+            validatorConfig.value.validator.validate(validatorConfig.value.enabled, "${stage}/${gitopsConfig.application}", validatorConfig.value.config, gitopsConfig)
         }
     }
 
     def createFoldersAndCopyK8sResources(String stage) {
         def sourcePath = gitopsConfig.deployments.sourcePath
         def application = gitopsConfig.application
 
-        script.sh "mkdir -p ${stage}/${application}/extraResources/"
+        script.sh "mkdir -p ${stage}/${application}/${extraResourcesFolder}"
         script.sh "mkdir -p ${configDir}/"
         // copy extra resources like sealed secrets
-        script.echo "Copying k8s payload from application repo to gitOps Repo: '${sourcePath}/${stage}/*' to '${stage}/${application}/extraResources/'"
-        script.sh "cp -r ${script.env.WORKSPACE}/${sourcePath}/${stage}/* ${stage}/${application}/extraResources/ || true"
+        script.echo "Copying k8s payload from application repo to gitOps Repo: '${sourcePath}/${stage}/*' to '${stage}/${application}/${extraResourcesFolder}'"
+        script.sh "cp -r ${script.env.WORKSPACE}/${sourcePath}/${stage}/* ${stage}/${application}/${extraResourcesFolder} || true"
         script.sh "cp ${script.env.WORKSPACE}/*.yamllint.yaml ${configDir}/ || true"
     }
 

src/com/cloudogu/gitopsbuildlib/deployment/helm/Helm.groovy

@@ -15,6 +15,7 @@ class Helm extends Deployment {
 
     Helm(def script, def gitopsConfig) {
         super(script, gitopsConfig)
+        this.extraResourcesFolder = "extraResources"
         if (gitopsConfig.deployments.helm.repoType == 'GIT') {
             chartRepo = new GitRepo(script)
         } else if (gitopsConfig.deployments.helm.repoType == 'HELM') {
@@ -49,11 +50,8 @@ class Helm extends Deployment {
     @Override
     def postValidation(String stage) {
         def helmConfig = gitopsConfig.deployments.helm
-
         // clean the gitrepo helm chart folder since the helmRelease.yaml ist now created
-        if (helmConfig.repoType == 'GIT') {
-            script.sh "rm -rf chart || true"
-        }
+        script.sh "rm -rf chart || true"
     }
 
     private void updateYamlValue(String yamlFilePath, Map helmConfig) {

src/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDRelease.groovy

@@ -18,24 +18,33 @@ class ArgoCDRelease extends HelmRelease{
         if (helmConfig.repoType == 'GIT') {
             helmRelease = createResourcesFromGitRepo(helmConfig, application, mergedValuesFile)
         } else if (helmConfig.repoType == 'HELM') {
-            // TODO not yet implemented
+            helmRelease = createResourcesFromHelmRepo(helmConfig, application, mergedValuesFile)
         }
         return helmRelease
     }
 
     private String createResourcesFromGitRepo(Map helmConfig, String application, String mergedValuesFile) {
-        String helmRelease = ""
 
         def chartPath = ''
         if (helmConfig.containsKey('chartPath')) {
             chartPath = helmConfig.chartPath
         }
 
+        return createHelmRelease(chartPath as String, application, mergedValuesFile)
+    }
+
+    private String createResourcesFromHelmRepo(Map helmConfig, String application, String mergedValuesFile) {
+        return createHelmRelease(helmConfig.chartName as String, application, mergedValuesFile)
+    }
+
+    private String createHelmRelease(String chartPath, String application, String mergedValuesFile) {
+        String helmRelease = ""
         dockerWrapper.withHelm {
             String templateScript = "helm template ${application} chart/${chartPath} -f ${mergedValuesFile}"
             helmRelease = script.sh returnStdout: true, script: templateScript
         }
-        // this line removes all empty lines since helm template creates some and the helm kubeval validator will throw an error if there are emtpy lines present
+
+        // this line removes all empty lines since helm template creates some and the helm validator will throw an error if there are emtpy lines present
         helmRelease = helmRelease.replaceAll("(?m)^[ \t]*\r?\n", "")
         return helmRelease
     }

src/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepo.groovy

@@ -8,18 +8,34 @@ class HelmRepo extends RepoType{
 
     @Override
     String mergeValues(Map helmConfig, String[] valuesFiles) {
+
+        if (helmConfig.containsKey('credentialsId') && helmConfig.credentialsId) {
+            script.withCredentials([
+                script.usernamePassword(
+                    credentialsId: helmConfig.credentialsId,
+                    usernameVariable: 'USERNAME',
+                    passwordVariable: 'PASSWORD')
+            ]) {
+                String credentialArgs = " --username ${script.USERNAME} --password ${script.PASSWORD}"
+                return mergeValuesFiles(helmConfig, valuesFiles, credentialArgs)
+            }
+        } else {
+            return mergeValuesFiles(helmConfig, valuesFiles)
+        }
+    }
+
+    private String mergeValuesFiles(Map helmConfig, String[] valuesFiles, String credentialArgs = "") {
         String merge = ""
 
         withHelm {
-            script.sh "helm repo add chartRepo ${helmConfig.repoUrl}"
+            script.sh "helm repo add chartRepo ${helmConfig.repoUrl}${credentialArgs}"
             script.sh "helm repo update"
-            script.sh "helm pull chartRepo/${helmConfig.chartName} --version=${helmConfig.version} --untar --untardir=${script.env.WORKSPACE}/chart"
-            String helmScript = "helm values ${script.env.WORKSPACE}/chart/${helmConfig.chartName} ${valuesFilesWithParameter(valuesFiles)}"
+            // helm pull also executes helm dependency so we don't need to do it in this step
+            script.sh "helm pull chartRepo/${helmConfig.chartName} --version=${helmConfig.version} --untar --untardir=chart"
+            String helmScript = "helm values chart/${helmConfig.chartName} ${valuesFilesWithParameter(valuesFiles)}"
             merge = script.sh returnStdout: true, script: helmScript
         }
 
-        script.sh "rm -rf ${script.env.WORKSPACE}/chart || true"
-
         return merge
     }
 }

src/com/cloudogu/gitopsbuildlib/deployment/plain/Plain.groovy

@@ -19,7 +19,7 @@ class Plain extends Deployment{
 
     private updateImage(String stage) {
         gitopsConfig.deployments.plain.updateImages.each {
-            def deploymentFilePath = "${stage}/${gitopsConfig.application}/${gitopsConfig.deployments.sourcePath}/${it['filename']}"
+            def deploymentFilePath = "${stage}/${gitopsConfig.application}/${it['filename']}"
             def data = script.readYaml file: deploymentFilePath
             def containers = data.spec.template.spec.containers
             def containerName = it['containerName']

src/com/cloudogu/gitopsbuildlib/validation/HelmKubeval.groovy

@@ -1,7 +1,5 @@
 package com.cloudogu.gitopsbuildlib.validation
 
-import com.cloudogu.gitopsbuildlib.docker.DockerWrapper
-
 class HelmKubeval extends Validator {
 
     HelmKubeval(def script) {
@@ -11,33 +9,16 @@ class HelmKubeval extends Validator {
     @Override
     void validate(String targetDirectory, Map config, Map deployments) {
         if (deployments.containsKey('helm')) {
-            if (deployments.helm.repoType == 'GIT') {
-//                script.dir("${targetDirectory}/chart") {
-//                    def git = (deployments.helm.containsKey('credentialsId'))
-//                        ? script.cesBuildLib.Git.new(script, deployments.helm.credentialsId)
-//                        : script.cesBuildLib.Git.new(script)
-//                    git url: deployments.helm.repoUrl, branch: 'main', changelog: false, poll: false
-//
-//                    if(deployments.helm.containsKey('version') && deployments.helm.version) {
-//                        git.checkout(deployments.helm.version)
-//                    }
-//                }
-
-                def chartPath = ''
-                if (deployments.helm.containsKey('chartPath')) {
-                    chartPath = deployments.helm.chartPath
-                }
 
-                withDockerImage(config.image) {
-                    script.sh "helm kubeval chart/${chartPath} -v ${config.k8sSchemaVersion}"
-                }
+            def chartDir = ''
+            if (deployments.helm.containsKey('chartPath')) {
+                chartDir = deployments.helm.chartPath
+            } else if ( deployments.helm.containsKey('chartName')) {
+                chartDir = deployments.helm.chartName
+            }
 
-            } else if (deployments.helm.repoType == 'HELM') {
-                withDockerImage(config.image) {
-                    script.sh "helm repo add chartRepo ${deployments.helm.repoUrl}"
-                    script.sh "helm repo update"
-                    script.sh "helm kubeval chartRepo/${deployments.helm.chartName} --version=${deployments.helm.version} -v ${config.k8sSchemaVersion}"
-                }
+            withDockerImage(config.image) {
+                script.sh "helm kubeval chart/${chartDir} -v ${config.k8sSchemaVersion}"
             }
         }
     }

src/com/cloudogu/gitopsbuildlib/validation/Kubeval.groovy

@@ -1,21 +1,18 @@
 package com.cloudogu.gitopsbuildlib.validation
 
-import com.cloudogu.gitopsbuildlib.docker.DockerWrapper
-
 /**
  * Validates all yaml-resources within the target-directory against the specs of the given k8s version
  */
 class Kubeval extends Validator {
 
-
     Kubeval(def script) {
         super(script)
     }
 
     @Override
     void validate(String targetDirectory, Map config, Map deployments) {
         withDockerImage(config.image) {
-            script.sh "kubeval -d ${targetDirectory} -v ${config.k8sSchemaVersion} --strict"
+            script.sh "kubeval -d ${targetDirectory} -v ${config.k8sSchemaVersion} --strict --ignore-missing-schemas"
         }
     }
 }

src/com/cloudogu/gitopsbuildlib/validation/Validator.groovy

@@ -12,9 +12,9 @@ abstract class Validator {
         dockerWrapper = new DockerWrapper(script)
     }
 
-    void validate(boolean enabled, String targetDirectory, Map config, Map deployments) {
+    void validate(boolean enabled, String targetDirectory, Map config, Map gitopsConfig) {
         if (enabled) {
-            validate(targetDirectory, config, deployments)
+            validate(targetDirectory, config, gitopsConfig)
         } else {
             script.echo "Skipping validator ${this.getClass().getSimpleName()} because it is configured as enabled=false"
         }

src/com/cloudogu/gitopsbuildlib/validation/Yamllint.groovy

@@ -16,7 +16,7 @@ class Yamllint extends Validator {
     }
 
     @Override
-    void validate(String targetDirectory, Map config, Map deployments) {
+    void validate(String targetDirectory, Map config, Map gitopsConfig) {
         withDockerImage(config.image) {
             script.sh "yamllint " +
                 "${config.profile ? "-d ${config.profile} " : ''}" +

test/com/cloudogu/gitopsbuildlib/deployment/DeploymentTest.groovy

@@ -56,10 +56,10 @@ class DeploymentTest {
     @Test
     void 'creating folders for plain deployment'() {
         deploymentUnderTest.createFoldersAndCopyK8sResources('staging',)
-        assertThat(scriptMock.actualEchoArgs[0]).isEqualTo('Copying k8s payload from application repo to gitOps Repo: \'k8s/staging/*\' to \'staging/app/extraResources/\'')
-        assertThat(scriptMock.actualShArgs[0]).isEqualTo('mkdir -p staging/app/extraResources/')
+        assertThat(scriptMock.actualEchoArgs[0]).isEqualTo('Copying k8s payload from application repo to gitOps Repo: \'k8s/staging/*\' to \'staging/app/\'')
+        assertThat(scriptMock.actualShArgs[0]).isEqualTo('mkdir -p staging/app/')
         assertThat(scriptMock.actualShArgs[1]).isEqualTo('mkdir -p .config/')
-        assertThat(scriptMock.actualShArgs[2]).isEqualTo('cp -r workspace/k8s/staging/* staging/app/extraResources/ || true')
+        assertThat(scriptMock.actualShArgs[2]).isEqualTo('cp -r workspace/k8s/staging/* staging/app/ || true')
         assertThat(scriptMock.actualShArgs[3]).isEqualTo('cp workspace/*.yamllint.yaml .config/ || true')
     }
 

test/com/cloudogu/gitopsbuildlib/deployment/HelmTest.groovy

@@ -48,7 +48,6 @@ class HelmTest {
                 repoType: 'HELM',
                 repoUrl: 'repoUrl',
                 chartName: 'chartName',
-                credentials: 'creds',
                 version: '1.0'
             ]
         ],
@@ -106,14 +105,13 @@ spec:
         assertThat(dockerMock.actualImages[0]).contains('ghcr.io/cloudogu/helm:')
         assertThat(scriptMock.actualShArgs[0]).isEqualTo('helm repo add chartRepo repoUrl')
         assertThat(scriptMock.actualShArgs[1]).isEqualTo('helm repo update')
-        assertThat(scriptMock.actualShArgs[2]).isEqualTo('helm pull chartRepo/chartName --version=1.0 --untar --untardir=workspace/chart')
-        assertThat(scriptMock.actualShArgs[3]).isEqualTo('[returnStdout:true, script:helm values workspace/chart/chartName -f workspace/k8s/values-staging.yaml -f workspace/k8s/values-shared.yaml ]')
-        assertThat(scriptMock.actualShArgs[4]).isEqualTo('rm -rf workspace/chart || true')
-        assertThat(scriptMock.actualShArgs[5]).isEqualTo('rm staging/testapp/mergedValues.yaml')
+        assertThat(scriptMock.actualShArgs[2]).isEqualTo('helm pull chartRepo/chartName --version=1.0 --untar --untardir=chart')
+        assertThat(scriptMock.actualShArgs[3]).isEqualTo('[returnStdout:true, script:helm values chart/chartName -f workspace/k8s/values-staging.yaml -f workspace/k8s/values-shared.yaml ]')
+        assertThat(scriptMock.actualShArgs[4]).isEqualTo('rm staging/testapp/mergedValues.yaml')
         assertThat(scriptMock.actualWriteFileArgs[0]).isEqualTo('[file:staging/testapp/mergedValues.yaml, ' +
             'text:[helm repo add chartRepo repoUrl, helm repo update, ' +
-            'helm pull chartRepo/chartName --version=1.0 --untar --untardir=workspace/chart, ' +
-            '[returnStdout:true, script:helm values workspace/chart/chartName -f workspace/k8s/values-staging.yaml -f workspace/k8s/values-shared.yaml ]]]')
+            'helm pull chartRepo/chartName --version=1.0 --untar --untardir=chart, ' +
+            '[returnStdout:true, script:helm values chart/chartName -f workspace/k8s/values-staging.yaml -f workspace/k8s/values-shared.yaml ]]]')
         assertThat(scriptMock.actualWriteFileArgs[1]).isEqualTo('''[file:staging/testapp/applicationRelease.yaml, text:apiVersion: helm.fluxcd.io/v1
 kind: HelmRelease
 metadata:

test/com/cloudogu/gitopsbuildlib/deployment/PlainTest.groovy

@@ -27,8 +27,8 @@ class PlainTest {
     void 'successful update'() {
 
         plain.postValidation('staging')
-        assertThat(scriptMock.actualReadYamlArgs[0]).isEqualTo('[file:staging/testApp/k8s/deployment.yaml]')
-        assertThat(scriptMock.actualWriteYamlArgs[0]).isEqualTo('[file:staging/testApp/k8s/deployment.yaml, data:[spec:[template:[spec:[containers:[[image:imageNameReplacedTest, name:application]]]]], to:[be:[changed:oldValue]]], overwrite:true]')
+        assertThat(scriptMock.actualReadYamlArgs[0]).isEqualTo('[file:staging/testApp/deployment.yaml]')
+        assertThat(scriptMock.actualWriteYamlArgs[0]).isEqualTo('[file:staging/testApp/deployment.yaml, data:[spec:[template:[spec:[containers:[[image:imageNameReplacedTest, name:application]]]]], to:[be:[changed:oldValue]]], overwrite:true]')
 
     }
 }

test/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/ArgoCDReleaseTest.groovy

@@ -21,9 +21,9 @@ class ArgoCDReleaseTest {
         ],
             'app',
             'namespace',
-            'this/is/a/valusfile')
+            'this/is/a/valuesfile')
 
-        assertThat(scriptMock.actualShArgs[0]).isEqualTo('[returnStdout:true, script:helm template app chart/path -f this/is/a/valusfile]')
+        assertThat(scriptMock.actualShArgs[0]).isEqualTo('[returnStdout:true, script:helm template app chart/path -f this/is/a/valuesfile]')
     }
 
     @Test
@@ -36,8 +36,23 @@ class ArgoCDReleaseTest {
         ],
             'app',
             'namespace',
-            'this/is/a/valusfile')
+            'this/is/a/valuesfile')
 
-        assertThat(scriptMock.actualShArgs[0]).isEqualTo('[returnStdout:true, script:helm template app chart/ -f this/is/a/valusfile]')
+        assertThat(scriptMock.actualShArgs[0]).isEqualTo('[returnStdout:true, script:helm template app chart/ -f this/is/a/valuesfile]')
+    }
+
+    @Test
+    void 'correct helm release with helm repo'() {
+        argoCdReleaseTest.create([
+            repoType: 'HELM',
+            repoUrl: 'url',
+            chartName: 'chartName',
+            version: '1.0'
+        ],
+            'app',
+            'namespace',
+            'this/is/a/valuesfile')
+
+        assertThat(scriptMock.actualShArgs[0]).isEqualTo('[returnStdout:true, script:helm template app chart/chartName -f this/is/a/valuesfile]')
     }
 }

test/com/cloudogu/gitopsbuildlib/deployment/helm/helmrelease/FluxV1ReleaseTest.groovy

@@ -20,7 +20,7 @@ class FluxV1ReleaseTest {
         ],
         'app',
         'namespace',
-        'this/is/a/valusfile')
+        'this/is/a/valuesfile')
 
         assertThat(output).isEqualTo("""apiVersion: helm.fluxcd.io/v1
 kind: HelmRelease
@@ -61,7 +61,7 @@ spec:
         ],
             'app',
             'namespace',
-            'this/is/a/valusfile')
+            'this/is/a/valuesfile')
 
         assertThat(output).isEqualTo("""apiVersion: helm.fluxcd.io/v1
 kind: HelmRelease

test/com/cloudogu/gitopsbuildlib/deployment/helm/repotype/HelmRepoTest.groovy

@@ -22,8 +22,7 @@ class HelmRepoTest {
 
         assertThat(scriptMock.actualShArgs[0]).isEqualTo('helm repo add chartRepo url')
         assertThat(scriptMock.actualShArgs[1]).isEqualTo('helm repo update')
-        assertThat(scriptMock.actualShArgs[2]).isEqualTo('helm pull chartRepo/chartName --version=1.0 --untar --untardir=workspace/chart')
-        assertThat(scriptMock.actualShArgs[3]).isEqualTo('[returnStdout:true, script:helm values workspace/chart/chartName -f file1 -f file2 ]')
-        assertThat(scriptMock.actualShArgs[4]).isEqualTo('rm -rf workspace/chart || true')
+        assertThat(scriptMock.actualShArgs[2]).isEqualTo('helm pull chartRepo/chartName --version=1.0 --untar --untardir=chart')
+        assertThat(scriptMock.actualShArgs[3]).isEqualTo('[returnStdout:true, script:helm values chart/chartName -f file1 -f file2 ]')
     }
 }

test/com/cloudogu/gitopsbuildlib/validation/HelmKubevalTest.groovy

@@ -51,9 +51,7 @@ class HelmKubevalTest {
             ]
         )
         assertThat(dockerMock.actualImages[0]).isEqualTo('img')
-        assertThat(scriptMock.actualShArgs[0]).isEqualTo('helm repo add chartRepo chartRepo')
-        assertThat(scriptMock.actualShArgs[1]).isEqualTo('helm repo update')
-        assertThat(scriptMock.actualShArgs[2]).isEqualTo('helm kubeval chartRepo/chart --version=version -v 1.5')
+        assertThat(scriptMock.actualShArgs[0]).isEqualTo('helm kubeval chart/chart -v 1.5')
     }
 
     @Test

test/com/cloudogu/gitopsbuildlib/validation/KubevalTest.groovy

@@ -24,7 +24,7 @@ class KubevalTest {
         )
         assertThat(dockerMock.actualImages[0]).isEqualTo('img')
         assertThat(scriptMock.actualShArgs[0]).isEqualTo(
-            'kubeval -d target -v 1.5 --strict'
+            'kubeval -d target -v 1.5 --strict --ignore-missing-schemas'
         )
     }
 }