Merge pull request #170 from lensesio-dev/feat/OPS-1854-43

Allow not specifying a port in Connect cluster metrics endpoint
lensesio · Oct 11, 2021 · f39aa11 · f39aa11
2 parents a0c17b4 + f676e14
commit f39aa11
Show file tree

Hide file tree

Showing 10 changed files with 368 additions and 122 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -3,7 +3,7 @@
 pipeline {
 
     agent {
-        label 'docker && ephemeral'
+        label 'docker && lightweight'
     }
 
     libraries {
@@ -16,19 +16,29 @@ pipeline {
         copyArtifactPermission('*')
         buildDiscarder(logRotator(numToKeepStr: '30', artifactNumToKeepStr: '10'))
         parallelsAlwaysFailFast()
-        timeout(time: 10, unit: 'MINUTES')
+        timestamps()
+        timeout(time: 30, unit: 'MINUTES')
+    }
+
+    parameters {
+        booleanParam(
+            name: 'DEBUG_FLAG',
+            defaultValue: false,
+            description: 'Enable additional debug output'
+        )
     }
 
     environment {
-        MODE = "${params.FORCE_DEPLOY_TO_PRODUCTION || "${env.BRANCH_NAME}" == 'master' ? 'release' : 'internal'}"
+        TARGET_RELEASE_BRANCH = 'release/4.3'
     }
 
     stages {
         stage('Build Helm Charts') {
             agent {
                 docker {
-                    image "dtzar/helm-kubectl"
-                    args '-e HOME=/tmp -e HELM_HOME=/tmp'
+                    label 'docker && lightweight'
+                    image 'eu.gcr.io/lenses-ci/cicd:helm-with-tools'
+                    args '-e HOME=/tmp'
                     reuseNode true
                 }
             }
@@ -46,6 +56,7 @@ pipeline {
                     if (env.BRANCH_NAME =~ /^release/) {
                         env.BUILD_MODE = 'release'
                     }
+                    jenkinsHelper.info("Build mode: ${env.BUILD_MODE}")
 
                     sh("_cicd/functions.sh setup_helm")
                     sh("_cicd/functions.sh package_all")
@@ -63,68 +74,115 @@ pipeline {
             }
         }
 
-        stage('Upload Helm Chart to public repo') {
+        stage('Upload Helm Chart to public JFrog repo') {
             when {
                 anyOf {
-                    branch 'release/4.3'
+                    branch env.TARGET_RELEASE_BRANCH
                 }
             }
             environment {
                 HELM_REPOSITORY = 'lenses-helm-charts'
-                DOCKER_IMAGE = 'lenses-helm-chart-repo'
                 ARTIFACTORY_URL = 'https://lenses.jfrog.io/artifactory/'
                 ARTIFACTORY_API_KEY = credentials('artifactory-lenses-helm')
-                SSH_HOST = credentials('ssh-host')
             }
             steps {
                 script {
                     docker.image("docker.bintray.io/jfrog/jfrog-cli-go").inside {
                         sh("jfrog rt u build/*.tgz ${HELM_REPOSITORY} --url=${ARTIFACTORY_URL} --apikey=${ARTIFACTORY_API_KEY}")
                     }
+                }
+            }
+        }
 
-                    // Update helm.repo.lenses.io
-                    sshagent (credentials: ['57dab1e7-d47f-4c57-8eef-c107c4bb707a']) {
-                        sh '_cicd/functions.sh clone_site'
+        // We want all public images in private repo too
+        stage('Upload Helm Chart to private JFrog repo') {
+            environment {
+                HELM_REPOSITORY = 'lenses-private-helm-charts'
+                ARTIFACTORY_URL = 'https://lenses.jfrog.io/artifactory/'
+                ARTIFACTORY_API_KEY = credentials('artifactory-lenses-helm')
+            }
+            steps {
+                script {
+                    docker.image("docker.bintray.io/jfrog/jfrog-cli-go").inside {
+                        sh("_cicd/functions.sh publish_all")
                     }
                 }
             }
-            post {
-                always {
-                    jiraSendDeploymentInfo(
-                        site: 'landoop.atlassian.net',
-                        environmentId: "ans-ci-backend-eu-01.landoop.com.",
-                        environmentName: 'helm.repo.lenses.io',
-                        environmentType: 'production'
-                    )
+        }
+
+        stage('Pull Helm Chart repo and build static assets') {
+            agent {
+                docker {
+                    label 'docker && lightweight'
+
+                    image 'eu.gcr.io/lenses-ci/cicd:helm-with-tools'
+                    args '-e HOME=/tmp'
+                    reuseNode true
+                }
+            }
+            environment {
+                SOURCE_HELM_REPO_URL = "${env.BUILD_MODE == 'release' ? 'https://lenses.jfrog.io/artifactory/helm-charts/' : 'https://lenses.jfrog.io/artifactory/lenses-private-helm-repo/'}"
+                TARGET_HELM_REPO_URL = "${env.BUILD_MODE == 'release' ? 'https://helm.repo.lenses.io' : 'https://helm.repo.staging.lenses.io'}"
+            }
+            steps {
+                script {
+                    jenkinsHelper.info("Source helm repo url: ${env.SOURCE_HELM_REPO_URL}")
+                    jenkinsHelper.info("Target helm repo url: ${env.TARGET_HELM_REPO_URL}")
+
+                    // Delete old content
+                    dir('_cicd/image/charts') { deleteDir() }
+
+                    dir('_cicd/image/charts') {
+                        withCredentials([
+                            usernamePassword(
+                                credentialsId: '9ab1ec30-daf0-4311-bc77-0e49cec71a42',
+                                usernameVariable: 'ARTIFACTORY_USER',
+                                passwordVariable: 'ARTIFACTORY_PASSWORD'
+                            )
+                        ]) {
+                            // Build the static site by taking index.yaml/charts from jfrog and static index.html from Google bucket
+                            sh "${WORKSPACE}/_cicd/functions.sh clone_site"
+                        }
+                        archiveArtifacts '*.yaml'
+                    }
                 }
             }
         }
 
-        stage('Upload Helm Chart to private repo') {
+        stage('Upload static assets to production Lenses website') {
             when {
                 anyOf {
-                    branch 'release/4.2'
-                    not { branch 'release/**' }
+                    branch env.TARGET_RELEASE_BRANCH
                 }
             }
             environment {
-                HELM_REPOSITORY = 'lenses-private-helm-charts'
-                ARTIFACTORY_URL = 'https://lenses.jfrog.io/artifactory/'
-                ARTIFACTORY_API_KEY = credentials('artifactory-lenses-helm')
+                SSH_HOST = credentials('ssh-host')
             }
             steps {
                 script {
-                    docker.image("docker.bintray.io/jfrog/jfrog-cli-go").inside {
-                        sh("_cicd/functions.sh publish_all")
+                    dir('_cicd/image/charts') {
+                        // Upload the updated site to helm.repo.lenses.io
+                        sshagent (credentials: ['57dab1e7-d47f-4c57-8eef-c107c4bb707a']) {
+                            sh "${WORKSPACE}/_cicd/functions.sh upload_site"
+                        }
                     }
                 }
             }
+            post {
+                always {
+                    jiraSendDeploymentInfo(
+                        site: 'landoop.atlassian.net',
+                        environmentId: "ans-ci-backend-eu-01.landoop.com.",
+                        environmentName: 'helm.repo.lenses.io',
+                        environmentType: 'production'
+                    )
+                }
+            }
         }
 
         stage('Build Helm repo Docker image') {
             environment {
                 DOCKER_IMAGE = 'eu.gcr.io/lenses-ci/lenses-helm-chart-repo'
-                HELM_REPO_URL = "${env.MODE == 'release' ? 'https://helm.repo.lenses.io' : 'https://helm.repo.staging.lenses.io'}"
             }
             agent {
                 docker {
@@ -141,8 +199,7 @@ pipeline {
 GIT_COMMIT=${env.GIT_COMMIT}
 GIT_BRANCH=${env.GIT_BRANCH}
 """
-                            sh "gcloud builds submit . --config=cloudbuild.yaml --substitutions=_DOCKER_IMAGE='${env.DOCKER_IMAGE}:${env.GIT_COMMIT}',_HELM_REPO_URL='${env.HELM_REPO_URL},_BUILD_INFO=${env.BUILD_INFO}'"
-
+                            sh "gcloud builds submit . --config=cloudbuild.yaml --substitutions=_DOCKER_IMAGE='${env.DOCKER_IMAGE}:${env.GIT_COMMIT}',_BUILD_INFO='${env.BUILD_INFO}'"
                             jenkinsHelper.info("new docker image built and pushed, to use this image: `docker pull ${env.DOCKER_IMAGE}:${env.GIT_COMMIT}`")
                         }
                     }

diff --git a/_cicd/functions.sh b/_cicd/functions.sh
@@ -5,24 +5,101 @@ SCRIPTS_DIR=$(dirname "${BASH_SOURCE[0]}")
 set -o errexit
 set -o nounset
 set -o pipefail
-set -o xtrace
+if [[ "${DEBUG_FLAG:-}" == "true" ]]; then
+    set -o xtrace
+fi
+
+REPO_LABEL="ephemeral"
+
+upload_site() {
+    # local remote_path="/tmp/helm.repo.lenses.io"
+    local remote_path="/mnt/persistent/helm.repo.lenses.io"
 
-clone_site() {
     mkdir -p "${HOME}/.ssh"
     ssh-keyscan -t rsa ${SSH_HOST#*@} >> ~/.ssh/known_hosts
 
     cat <<EOF | ssh ${SSH_HOST}
-rm -rf /mnt/persistent/helm.repo.lenses.io/lenses.jfrog.io
-cd /mnt/persistent/helm.repo.lenses.io
-wget -m https://lenses.jfrog.io/artifactory/helm-charts/
-sed  's|https://lenses.jfrog.io/artifactory/api/helm/helm-charts|https://helm.repo.lenses.io|' -i lenses.jfrog.io/artifactory/helm-charts/index.yaml
-sed  's|https://lenses.jfrog.io/lenses-helm-charts|https://helm.repo.lenses.io|' -i lenses.jfrog.io/artifactory/helm-charts/index.yaml
-mv lenses.jfrog.io/artifactory/helm-charts/* .
-rm -rf lenses.jfrog.io/ index.html
-wget https://raw.githubusercontent.com/lensesio/kafka-helm-charts/gh-pages/index.html
+mkdir -p "${remote_path}"
+EOF
+    # Copy all files to proper path
+    # TODO: Use rsync to update only the changed ones
+    scp -r . "${SSH_HOST}:${remote_path}"
+    cat <<EOF | ssh ${SSH_HOST}
+ls -lsa "${remote_path}"
 EOF
 }
 
+clone_site() {
+    # Get index.yaml
+    local add_args=""
+    if [[ "${BUILD_MODE}" == 'development' ]]; then
+        add_args="--username=${ARTIFACTORY_USER} --password=${ARTIFACTORY_PASSWORD}"
+    fi
+    # shellcheck disable=SC2086
+    helm repo add "${REPO_LABEL}" "${SOURCE_HELM_REPO_URL}" ${add_args}
+    helm repo update
+
+    # Copy index.yaml
+    eval "$(helm env)"
+    cp "${HELM_CACHE_HOME}/repository/${REPO_LABEL}-index.yaml" index.yaml
+    replace_in_index_yaml "https://lenses.jfrog.io/artifactory/api/helm/helm-charts"
+    replace_in_index_yaml "https://lenses.jfrog.io/artifactory/api/helm/lenses-private-helm-repo"
+
+    # Pull all charts // all versions
+    pull_charts
+    echo "=== Total files pulled: $(find .| wc -l)"
+    if [[ "${DEBUG_FLAG:-}" == "true" ]]; then
+        cat index.yaml
+        ls -sla
+    fi
+}
+
+pull_charts() {
+    set +o xtrace
+    local search_args=""
+    if [[ "${BUILD_MODE}" == 'development' ]]; then
+        search_args="--devel"
+    fi
+    CHARTS="$(helm search repo ${REPO_LABEL} ${search_args} | grep "${REPO_LABEL}/" | awk '{print $1}')"
+
+    IFS=$'\n'
+    for chart in $CHARTS; do
+        pull_chart_versions "${chart}"
+        # Use for testing
+        # break
+    done
+
+    if [[ "${DEBUG_FLAG:-}" == "true" ]]; then
+        set -o xtrace
+    fi
+}
+
+pull_chart_versions() {
+    local chart="${1}"
+    local search_args=""
+    if [[ "${BUILD_MODE}" == 'development' ]]; then
+        search_args="--devel"
+    fi
+    CHART_VERSIONS="$(helm search repo "${chart}" --versions ${search_args} | grep "${REPO_LABEL}/" | awk '{print $2}')"
+
+    IFS=$'\n'
+    for version in $CHART_VERSIONS; do
+        echo "Pulling ${chart}:${version}"
+        helm pull "${chart}" --version "${version}" || echo "Pulling ${chart}:${version} FAILED!"
+    done
+}
+
+replace_in_index_yaml() {
+    set -o xtrace
+    # TARGET_HELM_REPO_URL is different depending on if it is a release or not
+    # and is populated in Jenkisfile
+    sed "s|${1}|${TARGET_HELM_REPO_URL}|" -i index.yaml
+
+    if [[ "${DEBUG_FLAG:-}" != "true" ]]; then
+        set +o xtrace
+    fi
+}
+
 setup_helm() {
     echo "=== Helm add plugins"
     eval "$(helm env)"

diff --git a/_cicd/image/Dockerfile b/_cicd/image/Dockerfile
@@ -1,21 +1,11 @@
-FROM alpine:latest AS downloader
-ARG HELM_REPO_URL
-ARG BUILD_INFO
-
-RUN apk --no-cache add --update ca-certificates openssl wget && update-ca-certificates
+FROM caddy:2-alpine
 
-WORKDIR /www
+COPY Caddyfile /etc/caddy/Caddyfile
 
-# Download all charts (tgz files) and index.yaml
-RUN wget -m https://lenses.jfrog.io/artifactory/helm-charts/
-# Replace the absolute url in index.yaml
-RUN sed "s|https://lenses.jfrog.io/artifactory/api/helm/helm-charts|${HELM_REPO_URL}|" -i lenses.jfrog.io/artifactory/helm-charts/index.yaml
-RUN mv lenses.jfrog.io/artifactory/helm-charts/* .
-RUN rm -rf lenses.jfrog.io/ index.html
+WORKDIR /usr/share/caddy/
+COPY ./charts .
+COPY index.html .
 RUN echo "${BUILD_INFO}" > build.info
 
-FROM caddy:2-alpine
-
-COPY Caddyfile /etc/caddy/Caddyfile
-COPY index.html /usr/share/caddy/
-COPY --from=downloader /www /usr/share/caddy/
+EXPOSE 80
+CMD ["caddy", "run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"]
diff --git a/_cicd/image/cloudbuild.yaml b/_cicd/image/cloudbuild.yaml
@@ -1,11 +1,10 @@
 # Build custom connect Docker image with specified SQL processor connector'
 
-# timeout: 300s
+timeout: 300s
 steps:
 - name: 'gcr.io/cloud-builders/docker'
   args:
   - 'build'
-  - '--build-arg=HELM_REPO_URL=${_HELM_REPO_URL}'
   - '--build-arg=BUILD_INFO=${_BUILD_INFO}'
   - '--tag=${_DOCKER_IMAGE}'
   - '--file=Dockerfile'

diff --git a/charts/lenses/Chart.yaml b/charts/lenses/Chart.yaml
@@ -2,5 +2,5 @@ apiVersion: v1
 description: A chart for Lenses
 icon: https://www.lenses.io/images/logos/icon_ellipse2red.png
 name: lenses
-version: 4.3.8
+version: 4.3.9
 appVersion: 4.3.3
diff --git a/charts/lenses/templates/_helper.tpl b/charts/lenses/templates/_helper.tpl
@@ -289,8 +289,10 @@ PLAINTEXT
       {
         {{- if $host.url }}
         url: "{{ $host.url }}"
-        {{- else }}
+        {{- else if $port }}
         url: "{{$protocol}}://{{$host.host}}:{{$port}}"
+        {{- else }}
+        url: "{{$protocol}}://{{$host.host}}"
         {{- end }}
         {{- if $host.metrics -}},
         metrics: {

diff --git a/charts/lenses/tests/deployment.run_test.yaml b/charts/lenses/tests/deployment.run_test.yaml
@@ -104,3 +104,22 @@ tests:
           content:
             name: PAUSE_EXEC
             value: "true"
+
+  - it: should add env vars when set
+    set:
+      lenses:
+        additionalEnv:
+          - name: LENSES_CONNECTORS_INFO
+            value: |
+              {
+                "foo": "bar"
+              }
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LENSES_CONNECTORS_INFO
+            value: |
+              {
+                "foo": "bar"
+              }