Get rid of outputs.json

.github/workflows/terraform.yml

@@ -18,7 +18,6 @@ jobs:
         tf_root:
           - tests/k3s-docker/terraform
     env:
-      TF_ROOT: ${{ matrix.tf_root }}
       TF_VAR_repo_url: ${{ github.server_url }}/${{ github.actor }}/camptocamp-devops-stack.git
       TF_VAR_target_revision: ${{ github.head_ref }}
     defaults:
@@ -73,11 +72,15 @@ jobs:
           TF_VAR_repo_url: ${{ github.server_url }}/${{ github.repository }}
         run: terraform plan --detailed-exitcode -no-color
 
-      - name: Generate outputs.tf
-        run: terraform-bin output -json > outputs.json
+      - name: Set env
+        run: |
+          terraform-bin output -json kubeconfig | jq -r . > ${{ github.workspace }}/kubeconfig
+          echo "ARGOCD_AUTH_TOKEN=$(terraform-bin output -json argocd_auth_token | jq -r .)" >> $GITHUB_ENV
 
       - name: Wait for App of Apps
         uses: docker://argoproj/argocd:v1.7.12
+        env:
+          KUBECONFIG: ${{ github.workspace }}/kubeconfig
         with:
           entrypoint: ./scripts/wait-for-app-of-apps.sh
 
@@ -127,6 +130,7 @@ jobs:
         run: terraform plan --detailed-exitcode -no-color
 
       - name: Generate outputs.tf
+        # TODO: change this once PR #508 is merged
         run: terraform-bin output -json > outputs.json
 
       - name: Wait for App of Apps
@@ -159,12 +163,26 @@ jobs:
           TF_VAR_target_revision: ${{ github.head_ref }}
         run: terraform plan -no-color -out plan
 
-      - name: Generate outputs.tf
-        run: terraform-bin show -json plan | jq -r '.planned_values.outputs' > outputs.json
+      - name: Set env
+        run: |
+          terraform-bin output -json kubeconfig | jq -r . > ${{ github.workspace }}/kubeconfig
+          echo "ARGOCD_AUTH_TOKEN=$(terraform-bin output -json argocd_auth_token | jq -r .)" >> $GITHUB_ENV
+          echo "REPO_URL=$(terraform-bin output -json repo_url | jq -r .)" >> $GITHUB_ENV
+          echo "TARGET_REVISION=$(terraform-bin output -json target_revision | jq -r .)" >> $GITHUB_ENV
+          # FIXME: find a more robust way to do this
+          terraform-bin show -json plan > plan.json
+          jq -r '.planned_values.outputs.app_of_apps_values.value[0]' plan.json > ${{ github.workspace }}/values0.yaml
+          jq -r '.planned_values.outputs.app_of_apps_values.value[1]' plan.json > ${{ github.workspace }}/values1.yaml
+          jq -r '.planned_values.outputs.app_of_apps_values.value[2]' plan.json > ${{ github.workspace }}/values2.yaml
 
       - name: ArgoCD app diff with ${{ github.ref }}
         id: app-diff
         uses: docker://argoproj/argocd:v1.7.12
+        env:
+          KUBECONFIG: ${{ github.workspace }}/kubeconfig
+          APP_OF_APPS_VALUES_0 : ${{ github.workspace }}/values0.yaml
+          APP_OF_APPS_VALUES_1 : ${{ github.workspace }}/values1.yaml
+          APP_OF_APPS_VALUES_2 : ${{ github.workspace }}/values2.yaml
         with:
           entrypoint: ./scripts/app-diff.sh
 
@@ -176,6 +194,8 @@ jobs:
 
       - name: Wait for App of Apps
         uses: docker://argoproj/argocd:v1.7.12
+        env:
+          KUBECONFIG: ${{ github.workspace }}/kubeconfig
         with:
           entrypoint: ./scripts/wait-for-app-of-apps.sh
 

.gitlab-ci/pipeline.yaml

@@ -28,10 +28,21 @@ mr:terraform:plan:
     - cd "$TF_ROOT"
     - terraform init
     - terraform plan -out plan
-    - terraform show -json plan | jq -r '.planned_values.outputs' > outputs.json
+    - terraform output -json kubeconfig | jq -r . > "$CI_PROJECT_DIR/kubeconfig"
+    - echo "ARGOCD_AUTH_TOKEN=$(terraform output -json argocd_auth_token | jq -r .)" >> "$CI_PROJECT_DIR/.env"
+    - echo "REPO_URL=$(terraform output -json repo_url | jq -r .)" >> "$CI_PROJECT_DIR/.env"
+    - echo "TARGET_REVISION=$(terraform output -json target_revision | jq -r .)" >> "$CI_PROJECT_DIR/.env"
+    - terraform show -json plan > plan.json
+    - jq -r '.planned_values.outputs.app_of_apps_values.value[0]' plan.json > "$CI_PROJECT_DIR/values0.yaml"
+    - jq -r '.planned_values.outputs.app_of_apps_values.value[1]' plan.json > "$CI_PROJECT_DIR/values1.yaml"
+    - jq -r '.planned_values.outputs.app_of_apps_values.value[2]' plan.json > "$CI_PROJECT_DIR/values2.yaml"
   artifacts:
     paths:
-      - "$TF_ROOT/outputs.json"
+      - kubeconfig
+      - .env
+      - values0.yaml
+      - values1.yaml
+      - values2.yaml
     expire_in: 1 hour
     when: always
   rules:
@@ -41,11 +52,17 @@ mr:argocd:diff:
   stage: argocd
   image:
     name: argoproj/argocd:v${ARGOCD_VERSION}
+  variables:
+    KUBECONFIG: "$CI_PROJECT_DIR/kubeconfig"
+    APP_OF_APPS_VALUES_0: "$CI_PROJECT_DIR/values0.yaml"
+    APP_OF_APPS_VALUES_1: "$CI_PROJECT_DIR/values1.yaml"
+    APP_OF_APPS_VALUES_2: "$CI_PROJECT_DIR/values2.yaml"
   before_script:
     - python3 -c "from urllib.request import urlretrieve; urlretrieve('https://amazon-eks.s3.us-west-2.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/aws-iam-authenticator', '/tmp/aws-iam-authenticator')"
     - chmod +x /tmp/aws-iam-authenticator
     - export PATH="/tmp:$PATH"
   script:
+    - set -o allexport && . .env && set +o allexport
     - python3 -c "import urllib.request; print(urllib.request.urlopen('https://raw.githubusercontent.com/camptocamp/camptocamp-devops-stack/v$CAMPTOCAMP_DEVOPS_STACK_VERSION/scripts/app-diff.sh').read().decode())" | bash
   needs:
     - job: mr:terraform:plan
@@ -70,10 +87,12 @@ pb:terraform:apply:
     - terraform init
     - terraform apply --auto-approve
     - terraform plan --detailed-exitcode
-    - terraform output -json > outputs.json
+    - terraform output -json kubeconfig | jq -r . > "$CI_PROJECT_DIR/kubeconfig"
+    - echo "ARGOCD_AUTH_TOKEN=$(terraform output -json argocd_auth_token | jq -r .)" >> "$CI_PROJECT_DIR/.env"
   artifacts:
     paths:
-      - "$TF_ROOT/outputs.json"
+      - kubeconfig
+      - .env
     expire_in: 1 hour
     when: always
   environment:
@@ -86,11 +105,14 @@ pb:argocd:wait-for-app-of-apps:
   stage: argocd
   image:
     name: argoproj/argocd:v${ARGOCD_VERSION}
+  variables:
+    KUBECONFIG: "$CI_PROJECT_DIR/kubeconfig"
   before_script:
     - python3 -c "from urllib.request import urlretrieve; urlretrieve('https://amazon-eks.s3.us-west-2.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/aws-iam-authenticator', '/tmp/aws-iam-authenticator')"
     - chmod +x /tmp/aws-iam-authenticator
     - export PATH="/tmp:$PATH"
   script:
+    - set -o allexport && . .env && set +o allexport
     - python3 -c "import urllib.request; print(urllib.request.urlopen('https://raw.githubusercontent.com/camptocamp/camptocamp-devops-stack/v$CAMPTOCAMP_DEVOPS_STACK_VERSION/scripts/wait-for-app-of-apps.sh').read().decode())" | bash
   needs:
     - job: pb:terraform:apply

scripts/app-diff.sh

@@ -1,25 +1,16 @@
 #!/bin/bash
 
-set -e
-
-TF_ROOT="${TF_ROOT:-terraform}"
-
-KUBECONFIG=$(mktemp /tmp/kubeconfig.XXXXXX)
-export KUBECONFIG
-
-python3 -c "import sys, json; print(json.load(sys.stdin)['kubeconfig']['value'])" < "$TF_ROOT/outputs.json" > "$KUBECONFIG"
-chmod 0600 "$KUBECONFIG"
+# This script requires this environment variables to be set:
+#
+# - KUBECONFIG: pointing to a file containing the Kubernetes context to use as we use port-forwarding
+# - ARGOCD_AUTH_TOKEN: a read-only token that is allowed to perform argocd app list and argocd app diff
+#
+# It also requires 2 files ro be present (TODO: find a better approach for this):
+# - values0.yaml
+# - values1.yaml
+# - values2.yaml
 
-ARGOCD_AUTH_TOKEN=$(python3 -c "import sys, json; print(json.load(sys.stdin)['argocd_auth_token']['value'])" < "$TF_ROOT/outputs.json")
-export ARGOCD_AUTH_TOKEN
-
-REPO_URL=$(python3 -c "import sys, json; print(json.load(sys.stdin)['repo_url']['value'])" < "$TF_ROOT/outputs.json")
-TARGET_REVISION=$(python3 -c "import sys, json; print(json.load(sys.stdin)['target_revision']['value'])" < "$TF_ROOT/outputs.json")
-
-# FIXME: find a more robust way to do this
-APP_OF_APPS_VALUES_0=$(python3 -c "import sys, json; print(json.load(sys.stdin)['app_of_apps_values']['value'][0])" < "$TF_ROOT/outputs.json")
-APP_OF_APPS_VALUES_1=$(python3 -c "import sys, json; print(json.load(sys.stdin)['app_of_apps_values']['value'][1])" < "$TF_ROOT/outputs.json")
-APP_OF_APPS_VALUES_2=$(python3 -c "import sys, json; print(json.load(sys.stdin)['app_of_apps_values']['value'][2])" < "$TF_ROOT/outputs.json")
+set -e
 
 export KUBECTL_EXTERNAL_DIFF="diff -u"
 export ARGOCD_OPTS="--plaintext --port-forward --port-forward-namespace argocd"
@@ -33,9 +24,9 @@ cd - || exit
 
 echo Update app of apps without syncPolicy
 helm -n argocd upgrade app-of-apps camptocamp-devops-stack/argocd/app-of-apps \
-	-f <(echo "$APP_OF_APPS_VALUES_0") \
-	-f <(echo "$APP_OF_APPS_VALUES_1") \
-	-f <(echo "$APP_OF_APPS_VALUES_2") \
+	-f "$APP_OF_APPS_VALUES_0" \
+	-f "$APP_OF_APPS_VALUES_1" \
+	-f "$APP_OF_APPS_VALUES_2" \
 	--set spec.syncPolicy= --wait
 
 echo Waiting for app of apps to sync
@@ -52,5 +43,3 @@ do
 done
 
 helm -n argocd rollback app-of-apps
-
-rm "$KUBECONFIG"

scripts/wait-for-app-of-apps.sh

@@ -1,24 +1,15 @@
 #!/bin/bash
 
-set -e
-
-TF_ROOT="${TF_ROOT:-terraform}"
-
-KUBECONFIG=$(mktemp /tmp/kubeconfig.XXXXXX)
-export KUBECONFIG
+# This script requires this environment variables to be set:
+#
+# - KUBECONFIG: pointing to a file containing the Kubernetes context to use as we use port-forwarding
+# - ARGOCD_AUTH_TOKEN: a read-only token that is allowed to perform argocd app list and argocd app diff
 
-python3 -c "import sys, json; print(json.load(sys.stdin)['kubeconfig']['value'])" < "$TF_ROOT/outputs.json" > "$KUBECONFIG"
-chmod 0600 "$KUBECONFIG"
-
-ARGOCD_AUTH_TOKEN=$(python3 -c "import sys, json; print(json.load(sys.stdin)['argocd_auth_token']['value'])" < "$TF_ROOT/outputs.json")
-export ARGOCD_AUTH_TOKEN
+set -e
 
-ARGOCD_OPTS="--plaintext --port-forward --port-forward-namespace argocd"
-export ARGOCD_OPTS
+export ARGOCD_OPTS="--plaintext --port-forward --port-forward-namespace argocd"
 
 while ! argocd app wait apps --sync --health --timeout 30
 do
 	argocd app list -owide || true
 done
-
-rm "$KUBECONFIG"