Fix the jobs sync from/to bitnami in the GHA's "kubeapps general" wor…

…kflow (#5524) This commit performs mainly the following changes: * Remove the use of unnecessary SSH keys from the jobs sync_from_bitnami and sync_to_bitnami in the GHA kubeapps general workflow. * Make the bash scripts used by the jobs mentioned above, more reliable by making them use the git over SSH protocol when needed for avoiding the interactive request of credentials, instead of relying on the existence of obscure git configurations in the runner that make this process transparent. * Refactor those scripts to try to make them more easily understandable and maintainable, mainly by renaming variables and adding documentation. * Fix a bug in the setup job of the GHA's kubeapps general workflow, that provoked the output variable "triggered_from_fork" to be filled with the wrong value in certain scenarios. * Add some steps to the job mentioned above that show in the output the information about the GitHub event and the PR context that triggered the workflow. Signed-off-by: Jesús Benito Calzada <bjesus@vmware.com>
vmware-tanzu · Oct 20, 2022 · cf45610 · cf45610
1 parent a93f85d
commit cf45610
Show file tree

Hide file tree

Showing 5 changed files with 290 additions and 118 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -275,6 +275,17 @@ common_envars: &common_envars
   OLM_VERSION: << pipeline.parameters.OLM_VERSION >>
   POSTGRESQL_VERSION: << pipeline.parameters.POSTGRESQL_VERSION >>
   RUST_VERSION: << pipeline.parameters.RUST_VERSION >>
+  CI_BOT_USERNAME: << pipeline.parameters.CI_BOT_USERNAME >>
+  CI_BOT_EMAIL: << pipeline.parameters.CI_BOT_EMAIL >>
+  CI_BOT_GPG: << pipeline.parameters.CI_BOT_GPG >>
+  CI_BOT_FORKED_CHARTS_DEPLOYKEY_FILENAME: << pipeline.parameters.CI_BOT_FORKED_CHARTS_DEPLOYKEY_FILENAME >>
+  CHARTS_REPO_ORIGINAL: << pipeline.parameters.CHARTS_REPO_ORIGINAL >>
+  BRANCH_CHARTS_REPO_ORIGINAL: << pipeline.parameters.BRANCH_CHARTS_REPO_ORIGINAL >>
+  CHARTS_REPO_FORKED: << pipeline.parameters.CHARTS_REPO_FORKED >>
+  BRANCH_CHARTS_REPO_FORKED: << pipeline.parameters.BRANCH_CHARTS_REPO_FORKED >>
+  KUBEAPPS_REPO: << pipeline.parameters.KUBEAPPS_REPO >>
+  BRANCH_KUBEAPPS_REPO: << pipeline.parameters.BRANCH_KUBEAPPS_REPO >>
+  README_GENERATOR_REPO: << pipeline.parameters.README_GENERATOR_REPO >>
 
 common_gke_envars: &common_gke_envars
   USE_MULTICLUSTER_OIDC_ENV: "false"
@@ -855,9 +866,10 @@ jobs:
       - <<: *setup_gpg
       - add_ssh_keys:
           fingerprints:
+            # Deployment key uploaded to the kubeapps/kubeapps repository
+            - << pipeline.parameters.CI_BOT_KUBEAPPS_KUBEAPPS_DEPLOYKEY_FINGERPRINT >>
             # Deployment key uploaded to the kubeapps-bot/charts repository
             - << pipeline.parameters.CI_BOT_FORKED_CHARTS_DEPLOYKEY_FINGERPRINT >>
-
       - run:
           # This is a key pair: https://circleci.com/docs/2.0/gh-bb-integration/
           # public key uploaded to GitHub as a deploy key with write permissions in both kubeapps and kubeapps-bot/charts
@@ -866,6 +878,8 @@ jobs:
           command: |
             eval "$(ssh-agent -s)"
             # the name is always "id_rsa_"+fingerprint without ":""
+            # Deployment key uploaded to the kubeapps/kubeapps repository
+            ssh-add ~/.ssh/<< pipeline.parameters.CI_BOT_KUBEAPPS_KUBEAPPS_DEPLOYKEY_FILENAME >>
             # Deployment key uploaded to the kubeapps-bot/charts repository
             ssh-add ~/.ssh/<< pipeline.parameters.CI_BOT_FORKED_CHARTS_DEPLOYKEY_FILENAME >>
       - run:
@@ -874,7 +888,7 @@ jobs:
           # This token is passed as a GITHUB_TOKEN env var via CircleCI
           name: Run the chart_sync script
           command: |
-            ./script/chart_sync.sh << pipeline.parameters.CI_BOT_USERNAME >> << pipeline.parameters.CI_BOT_EMAIL >> << pipeline.parameters.CI_BOT_GPG >> << pipeline.parameters.CHARTS_REPO_ORIGINAL >> << pipeline.parameters.BRANCH_CHARTS_REPO_ORIGINAL >> << pipeline.parameters.CHARTS_REPO_FORKED >> << pipeline.parameters.BRANCH_CHARTS_REPO_FORKED >>
+            ./script/chart_sync.sh $CI_BOT_USERNAME $CI_BOT_EMAIL $CI_BOT_GPG $CI_BOT_FORKED_CHARTS_DEPLOYKEY_FILENAME $CHARTS_REPO_ORIGINAL $BRANCH_CHARTS_REPO_ORIGINAL $CHARTS_REPO_FORKED $BRANCH_CHARTS_REPO_FORKED
   sync_chart_from_bitnami:
     environment:
       <<: *common_envars
@@ -912,7 +926,7 @@ jobs:
           # This token is passed as a GITHUB_TOKEN env var via CircleCI
           name: Run the check_upstream_chart script
           command: |
-            ./script/chart_upstream_checker.sh << pipeline.parameters.CI_BOT_USERNAME >> << pipeline.parameters.CI_BOT_EMAIL >> << pipeline.parameters.CI_BOT_GPG >> << pipeline.parameters.CI_BOT_FORKED_CHARTS_DEPLOYKEY_FILENAME >> << pipeline.parameters.CHARTS_REPO_ORIGINAL >> << pipeline.parameters.BRANCH_CHARTS_REPO_ORIGINAL >> << pipeline.parameters.CHARTS_REPO_FORKED >> << pipeline.parameters.BRANCH_CHARTS_REPO_FORKED >> << pipeline.parameters.KUBEAPPS_REPO >> << pipeline.parameters.BRANCH_KUBEAPPS_REPO >> << pipeline.parameters.README_GENERATOR_REPO >>
+            ./script/chart_upstream_checker.sh $CI_BOT_USERNAME $CI_BOT_EMAIL $CI_BOT_GPG $CI_BOT_FORKED_CHARTS_DEPLOYKEY_FILENAME $CHARTS_REPO_ORIGINAL $BRANCH_CHARTS_REPO_ORIGINAL $CHARTS_REPO_FORKED $BRANCH_CHARTS_REPO_FORKED $KUBEAPPS_REPO $BRANCH_KUBEAPPS_REPO $README_GENERATOR_REPO
   report_srp:
     environment:
       <<: *common_envars

diff --git a/.github/workflows/kubeapps.yaml b/.github/workflows/kubeapps.yaml
@@ -5,8 +5,6 @@ name: Kubeapps general pipeline
 
 on:
   push:
-    branches:
-      - main
   pull_request:
     branches:
       - main
@@ -75,8 +73,19 @@ jobs:
       running_on_tag: ${{ steps.set-outputs.outputs.running_on_tag }}
       triggered_from_fork: ${{ steps.set-outputs.outputs.triggered_from_fork }}
     steps:
+      - name: Show GitHub event
+        env:
+          EVENT_CONTEXT: ${{ toJSON(github.event) }}
+        run: echo $EVENT_CONTEXT | jq
+      - name: Show PR context
+        env:
+          PR_CONTEXT: ${{ toJSON(github.event.pull_request) }}
+        run: echo $EVENT_CONTEXT | jq
       - name: Set outputs
         id: set-outputs
+        env:
+          PR_CONTEXT: ${{ toJSON(github.event.pull_request) }}
+          PR_SOURCE_REPO_NAME: ${{ github.event.pull_request.head.repo.full_name }}
         run: |
           if [[ "${GITHUB_REPOSITORY}" == "${KUBEAPPS_REPO}" ]]; then
             echo "img_prefix=${IMG_PREFIX}" >> $GITHUB_OUTPUT
@@ -87,10 +96,10 @@ jobs:
           fi;
 
           # Check if the workflow is triggered due to a PR from an external fork
-          if [[ "${{ github.event.pull_request.head.repo.full_name }}" == "${GITHUB_REPOSITORY}" ]]; then
-            echo "triggered_from_fork=false" >> $GITHUB_OUTPUT
-          else
+          if [[ ("${PR_CONTEXT}" != "" && "${PR_CONTEXT}" != null) && "${PR_SOURCE_REPO_NAME}" != "${GITHUB_REPOSITORY}" ]]; then
             echo "triggered_from_fork=true" >> $GITHUB_OUTPUT
+          else
+            echo "triggered_from_fork=false" >> $GITHUB_OUTPUT
           fi
 
           if [[ ${GITHUB_REF_TYPE} == "tag" ]]; then
@@ -330,7 +339,7 @@ jobs:
 
   # Push images to docker.io/kubeapps/[image]-ci:[dev-tag]
   push_dev_images:
-    # If the workflow is triggered from a PR from an external fork, secrets won't be available, so we cannot login into dockerhub
+    # If the workflow is triggered from a PR from an external fork, secrets won't be available, so we cannot log into dockerhub
     if: needs.setup.outputs.triggered_from_fork == 'false'
     runs-on: ubuntu-latest
     needs:
@@ -532,21 +541,15 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: "Install CLI tools"
+        env:
+          GPG_KEY_PUBLIC: ${{ secrets.GPG_KEY_PUBLIC }}
+          GPG_KEY_PRIVATE: ${{ secrets.GPG_KEY_PRIVATE }}
         run: |
           source ./script/lib/libcitools.sh
-
+          
           installGithubCLI ${GITHUB_VERSION}
-          installSemver {SEMVER_VERSION}
-          installGPGKey ${{secrets.GPG_KEY_PUBLIC}} ${{secrets.GPG_KEY_PRIVATE}} ${CI_BOT_GPG} ${CI_BOT_EMAIL}
-      - name: "Install SSH key: Kubeapps Deploy Key"
-        uses: shimataro/ssh-key-action@v2
-        with:
-          key: ${{ secrets.SSH_KEY_KUBEAPPS_DEPLOY }}
-          name: ${{ needs.setup.outputs.ssh_key_kubeapps_deploy_filename }}
-          known_hosts: |
-            |1|2YkQ4jjACcc/1rgSBszyeEuKxW4=|hO4GB0XMwQj1gYQDmaS304aU8Tc= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
-          if_key_exists: ignore
-
+          installSemver ${SEMVER_VERSION}
+          installGPGKey ${GPG_KEY_PUBLIC} ${GPG_KEY_PRIVATE} ${CI_BOT_GPG} ${CI_BOT_EMAIL}
       - name: "Install SSH key: Forked Charts Deploy Key"
         uses: shimataro/ssh-key-action@v2
         with:
@@ -556,20 +559,18 @@ jobs:
             |1|2YkQ4jjACcc/1rgSBszyeEuKxW4=|hO4GB0XMwQj1gYQDmaS304aU8Tc= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
           if_key_exists: ignore
       - # This is a key pair
-        # public key uploaded to GitHub as a deploy key with write permissions,
+        # public key uploaded to GitHub as a deployment key with write permissions,
         # private key stored as a secret.
         name: Start ssh-agent and configure the key
         run: |
           eval "$(ssh-agent -s)"
-          # Deployment key uploaded to the vmware-tanzu/kubeapps repository
-          ssh-add ~/.ssh/${SSH_KEY_KUBEAPPS_DEPLOY_FILENAME}
           # Deployment key uploaded to the kubeapps-bot/charts repository
           ssh-add ~/.ssh/${SSH_KEY_FORKED_CHARTS_DEPLOY_FILENAME}
-      -
-        # Assuming there is a personal access token created in GitHub granted with the scopes
+      - # Assuming there is a personal access token created in GitHub granted with the scopes
         # "repo:status", "public_repo" and "read:org"
-        # This token is passed as a GITHUB_TOKEN from CI
         name: Run the check_upstream_chart script
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           ./script/chart_upstream_checker.sh \
               ${CI_BOT_USERNAME} \
@@ -588,17 +589,21 @@ jobs:
   sync_chart_to_bitnami:
     needs:
       - setup
+      - local_e2e_tests
     if: needs.setup.outputs.running_on_tag == 'true'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - name: "Install CLI tools"
+        env:
+          GPG_KEY_PUBLIC: ${{ secrets.GPG_KEY_PUBLIC }}
+          GPG_KEY_PRIVATE: ${{ secrets.GPG_KEY_PRIVATE }}
         run: |
           source ./script/lib/libcitools.sh
-
+          
           installGithubCLI ${GITHUB_VERSION}
-          installSemver {SEMVER_VERSION}
-          installGPGKey ${{secrets.GPG_KEY_PUBLIC}} ${{secrets.GPG_KEY_PRIVATE}} ${CI_BOT_GPG} ${CI_BOT_EMAIL}
+          installSemver ${SEMVER_VERSION}
+          installGPGKey ${GPG_KEY_PUBLIC} ${GPG_KEY_PRIVATE} ${CI_BOT_GPG} ${CI_BOT_EMAIL}
       - name: "Install SSH key: Forked Charts Deploy Key"
         uses: shimataro/ssh-key-action@v2
         with:
@@ -608,22 +613,24 @@ jobs:
             |1|2YkQ4jjACcc/1rgSBszyeEuKxW4=|hO4GB0XMwQj1gYQDmaS304aU8Tc= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==
           if_key_exists: ignore
       - # This is a key pair
-        # public key uploaded to GitHub as a deploy key with write permissions,
+        # public key uploaded to GitHub as a deployment key with write permissions,
         # private key stored as a secret.
         name: Start ssh-agent and configure the key
         run: |
           eval "$(ssh-agent -s)"
           # Deployment key uploaded to the kubeapps-bot/charts repository
           ssh-add ~/.ssh/${SSH_KEY_FORKED_CHARTS_DEPLOY_FILENAME}
-      - # Assuming there is a personal access token created in GitHub granted with the scopes
-        # "repo:status", "public_repo" and "read:org"
-        # This token is passed as a GITHUB_TOKEN from CI
-        name: Run the chart_sync script
+      - name: Run the chart_sync script
+        env:
+          # Assuming there is a personal access token created in GitHub granted with the scopes
+          # "repo:status", "public_repo" and "read:org"
+          GITHUB_TOKEN: ${{ secrets.KUBEAPPS_BOT_GITHUB_TOKEN }}
         run: |
           ./script/chart_sync.sh \
               ${CI_BOT_USERNAME} \
               ${CI_BOT_EMAIL} \
               ${CI_BOT_GPG} \
+              ${SSH_KEY_FORKED_CHARTS_DEPLOY_FILENAME} \
               ${CHARTS_REPO_ORIGINAL} \
               ${BRANCH_CHARTS_REPO_ORIGINAL} \
               ${CHARTS_REPO_FORKED} \

diff --git a/script/chart_sync.sh b/script/chart_sync.sh
@@ -14,23 +14,31 @@ source "$ROOT_DIR/script/chart_sync_utils.sh"
 USERNAME=${1:?Missing git username}
 EMAIL=${2:?Missing git email}
 GPG_KEY=${3:?Missing git gpg key}
-CHARTS_REPO_ORIGINAL=${4:?Missing base chart repository}
-BRANCH_CHARTS_REPO_ORIGINAL=${5:?Missing base chart repository branch}
-CHARTS_REPO_FORKED=${6:?Missing forked chart repository}
-BRANCH_CHARTS_REPO_FORKED=${7:?Missing forked chart repository branch}
-DEV_MODE=${8:-false}
+CHARTS_FORK_SSH_KEY_FILENAME=${4:?Missing forked ssh key filename}
+CHARTS_REPO_UPSTREAM=${5:?Missing chart repository upstream (eg. bitnami/charts)}
+CHARTS_REPO_UPSTREAM_BRANCH=${6:?Missing chart repository upstream\'s  main branch name (eg. main)}
+CHARTS_REPO_FORK=${7:?Missing chart repository fork (eg. kubeapps-bot/charts)}
+CHARTS_REPO_FORK_BRANCH=${8:?Missing chart repository fork\'s main branch name (eg. main)}
+DEV_MODE=${9:-false}
+LOCAL_KUBEAPPS_REPO_PATH=${PROJECT_DIR:?PROJECT_DIR not defined}
 
+info "LOCAL_KUBEAPPS_REPO_PATH: ${LOCAL_KUBEAPPS_REPO_PATH}"
 info "USERNAME: ${USERNAME}"
 info "EMAIL: ${EMAIL}"
 info "GPG_KEY: ${GPG_KEY}"
-info "CHARTS_REPO_ORIGINAL: ${CHARTS_REPO_ORIGINAL}"
-info "BRANCH_CHARTS_REPO_ORIGINAL: ${BRANCH_CHARTS_REPO_ORIGINAL}"
-info "CHARTS_REPO_FORKED: ${CHARTS_REPO_FORKED}"
-info "BRANCH_CHARTS_REPO_FORKED: ${BRANCH_CHARTS_REPO_FORKED}"
+info "CHARTS_FORK_SSH_KEY_FILENAME: ${CHARTS_FORK_SSH_KEY_FILENAME}"
+info "CHARTS_REPO_UPSTREAM: ${CHARTS_REPO_UPSTREAM}"
+info "CHARTS_REPO_UPSTREAM_BRANCH: ${CHARTS_REPO_UPSTREAM_BRANCH}"
+info "CHARTS_REPO_FORK: ${CHARTS_REPO_FORK}"
+info "CHARTS_REPO_FORK_BRANCH: ${CHARTS_REPO_FORK_BRANCH}"
 info "DEV_MODE: ${DEV_MODE}"
 
+if [[ "${DEV_MODE}" == "true" ]]; then
+  set -x
+fi
+
 currentVersion=$(grep -oP '(?<=^version: ).*' <"${KUBEAPPS_CHART_DIR}/Chart.yaml")
-externalVersion=$(curl -s "https://raw.githubusercontent.com/${CHARTS_REPO_ORIGINAL}/${BRANCH_CHARTS_REPO_ORIGINAL}/${CHART_REPO_PATH}/Chart.yaml" | grep -oP '(?<=^version: ).*')
+externalVersion=$(curl -s "https://raw.githubusercontent.com/${CHARTS_REPO_UPSTREAM}/${CHARTS_REPO_UPSTREAM_BRANCH}/${CHART_REPO_PATH}/Chart.yaml" | grep -oP '(?<=^version: ).*')
 semverCompare=$(semver compare "${currentVersion}" "${externalVersion}")
 
 info "currentVersion: ${currentVersion}"
@@ -43,17 +51,15 @@ if [[ ${semverCompare} -gt 0 ]]; then
     CHARTS_FORK_LOCAL_PATH=$(mktemp -u)/charts
     mkdir -p "${CHARTS_FORK_LOCAL_PATH}"
 
-    git clone "https://github.com/${CHARTS_REPO_FORKED}" "${CHARTS_FORK_LOCAL_PATH}" --depth 1 --no-single-branch
-    info "Repo cloned: https://github.com/${CHARTS_REPO_FORKED}"
+    GIT_SSH_COMMAND="ssh -i ~/.ssh/${CHARTS_FORK_SSH_KEY_FILENAME}" git clone "git@github.com:${CHARTS_REPO_FORK}" "${CHARTS_FORK_LOCAL_PATH}" --depth 1 --no-single-branch
     configUser "${CHARTS_FORK_LOCAL_PATH}" "${USERNAME}" "${EMAIL}" "${GPG_KEY}"
-    configUser "${PROJECT_DIR}" "${USERNAME}" "${EMAIL}" "${GPG_KEY}"
-    info "Repos configured"
+    configUser "${LOCAL_KUBEAPPS_REPO_PATH}" "${USERNAME}" "${EMAIL}" "${GPG_KEY}"
 
-    latestVersion=$(latestReleaseTag "${PROJECT_DIR}")
+    latestVersion=$(latestReleaseTag "${LOCAL_KUBEAPPS_REPO_PATH}")
     prBranchName="kubeapps-bump-${currentVersion}"
 
-    updateRepoWithLocalChanges "${CHARTS_FORK_LOCAL_PATH}" "${latestVersion}" "${CHARTS_REPO_ORIGINAL}" "${BRANCH_CHARTS_REPO_ORIGINAL}" "${BRANCH_CHARTS_REPO_FORKED}"
-    commitAndSendExternalPR "${CHARTS_FORK_LOCAL_PATH}" "${prBranchName}" "${currentVersion}" "${CHARTS_REPO_ORIGINAL}" "${BRANCH_CHARTS_REPO_ORIGINAL}" "${DEV_MODE}"
+    updateRepoWithLocalChanges "${CHARTS_FORK_LOCAL_PATH}" "${latestVersion}" "${CHARTS_FORK_SSH_KEY_FILENAME}" "${CHARTS_REPO_UPSTREAM}" "${CHARTS_REPO_UPSTREAM_BRANCH}" "${CHARTS_REPO_FORK_BRANCH}"
+    commitAndSendExternalPR "${CHARTS_FORK_LOCAL_PATH}" "${prBranchName}" "${currentVersion}" "${CHARTS_REPO_UPSTREAM}" "${CHARTS_REPO_UPSTREAM_BRANCH}" "${CHARTS_FORK_SSH_KEY_FILENAME}" "${DEV_MODE}"
 elif [[ ${semverCompare} -lt 0 ]]; then
     echo "Skipping Chart sync. WARNING Current chart version (${currentVersion}) is less than the chart external version (${externalVersion})"
 else