Skip to content

(reports) chore: Update .env.example and Dockerfile for improved build and runtime #1179

(reports) chore: Update .env.example and Dockerfile for improved build and runtime

(reports) chore: Update .env.example and Dockerfile for improved build and runtime #1179

name: deploy-frontends-to-staging
on:
pull_request:
branches:
- staging
types:
- closed
env:
REGISTRY_URL: eu.gcr.io
PROJECT_ID: airqo-250220
DEPLOY_BRANCH: staging
jobs:
image-tag:
if: github.event.pull_request.merged == true
name: create image tag
runs-on: ubuntu-latest
outputs:
build_id: ${{ steps.prep.outputs.build_id }} # build id
datetime: ${{ steps.prep.outputs.datetime }} # build date
steps:
- name: generate build ID
id: prep
run: |
sha=${GITHUB_SHA::8}
timestamp=$(date +%s)
datetime=$(date)
echo "build_id=stage-${sha}-${timestamp}" >>$GITHUB_OUTPUT
echo "datetime=${datetime}" >>$GITHUB_OUTPUT
check:
# this job will only run if the PR has been merged
if: github.event.pull_request.merged == true
name: check for changed frontends
outputs:
run_netmanager: ${{ steps.check_files.outputs.run_netmanager }} # netmanager
run_website: ${{ steps.check_files.outputs.run_website }} # website
run_calibrate_app: ${{ steps.check_files.outputs.run_calibrate_app }} # calibrate app
run_analytics_platform: ${{ steps.check_files.outputs.run_analytics_platform }} # analytics platform
run_docs: ${{ steps.check_files.outputs.run_docs }} # docs
run_inventory: ${{ steps.check_files.outputs.run_inventory }} # inventory
run_reports: ${{ steps.check_files.outputs.run_reports }} # reports
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v3
with:
fetch-depth: 2
- name: check modified frontends
id: check_files
run: |
echo "=============== list modified files ==============="
git diff --name-only HEAD^ HEAD
echo "========== check paths of modified files =========="
git diff --name-only HEAD^ HEAD > files.txt
echo "run_netmanager=false" >>$GITHUB_OUTPUT
echo "run_website=false" >>$GITHUB_OUTPUT
echo "run_calibrate_app=false" >>$GITHUB_OUTPUT
echo "run_analytics_platform=false" >>$GITHUB_OUTPUT
echo "run_docs=false" >>$GITHUB_OUTPUT
echo "run_inventory=false" >>$GITHUB_OUTPUT
echo "run_reports=false" >>$GITHUB_OUTPUT
while IFS= read -r file
do
echo $file
if [[ $file == netmanager/* ]]; then
echo "run_netmanager=true" >>$GITHUB_OUTPUT
fi
if [[ $file == website/* ]]; then
echo "run_website=true" >>$GITHUB_OUTPUT
fi
if [[ $file == calibrate/* ]]; then
echo "run_calibrate_app=true" >>$GITHUB_OUTPUT
fi
if [[ $file == platform/* ]]; then
echo "run_analytics_platform=true" >>$GITHUB_OUTPUT
fi
if [[ $file == docs/* ]]; then
echo "run_docs=true" >>$GITHUB_OUTPUT
fi
if [[ $file == inventory/* ]]; then
echo "run_inventory=true" >>$GITHUB_OUTPUT
fi
if [[ $file == reports/* ]]; then
echo "run_reports=true" >>$GITHUB_OUTPUT
fi
if [[ $file == workflow-trigger ]]; then
echo "run_netmanager=true" >>$GITHUB_OUTPUT
echo "run_website=true" >>$GITHUB_OUTPUT
echo "run_calibrate_app=true" >>$GITHUB_OUTPUT
echo "run_analytics_platform=true" >>$GITHUB_OUTPUT
echo "run_docs=true" >>$GITHUB_OUTPUT
echo "run_inventory=true" >>$GITHUB_OUTPUT
echo "run_reports=true" >>$GITHUB_OUTPUT
fi
done < files.txt
### build and push netmanager image ###
netmanager:
name: build-push-netmanager-image
needs: [check, image-tag]
if: needs.check.outputs.run_netmanager == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Google login
uses: google-github-actions/auth@v1
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK(gcloud)
uses: google-github-actions/setup-gcloud@v1
- name: Create .env file
run: gcloud secrets versions access latest --secret="sta-env-netmanager" > netmanager/.env
- name: NPM Setup and Build
uses: actions/setup-node@v3
with:
node-version: "12"
- run: |
cd netmanager/
npm install
CI=false npm run build
- name: Login to GCR
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Build and Push Docker Image
run: |
cd netmanager/
docker build --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-platform-frontend:${{ needs.image-tag.outputs.build_id }} .
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-platform-frontend:${{ needs.image-tag.outputs.build_id }}
docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-platform-frontend:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-platform-frontend:latest
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-platform-frontend:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/wretry.action@v1.2.0 # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/netmanager/values-stage.yaml"
propertyPath: "image.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update netmanger staging image tag to ${{ needs.image-tag.outputs.build_id }}"
createPR: false
- name: Login to K8S
uses: azure/k8s-set-context@v3
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }}
- name: Update the corresponding configmap
run: |
kubectl create configmap --dry-run=client -o yaml \
--from-env-file=netmanager/.env env-platform-staging | kubectl replace -f - -n staging
### deploy website ###
website:
name: build-push-deploy-website
needs: [check, image-tag]
if: needs.check.outputs.run_website == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Login to GCR
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCP_FRONTEND_PROJECT_SA_CREDENTIALS }}
- name: Google login
uses: google-github-actions/auth@v1
with:
credentials_json: "${{ secrets.GCP_FRONTEND_PROJECT_SA_CREDENTIALS }}"
- name: Setup Cloud SDK(gcloud)
uses: "google-github-actions/setup-gcloud@v1"
- name: Create google_application_credentials.json file
id: create-google-application-credentials-file
uses: jsdaniell/create-json@1.1.2
with:
name: "google_application_credentials.json"
json: ${{ secrets.GCP_FRONTEND_PROJECT_SA_CREDENTIALS }}
dir: "website/"
- name: Create app.yaml and .env.yaml files
run: |
cd website/
echo "========== Creating app.yaml file =========="
echo "runtime: python" > app.yaml
echo "env: flex" >> app.yaml
echo "service: staging" >> app.yaml
echo "includes:" >> app.yaml
echo " - .env.yaml" >> app.yaml
echo "========== Creating .env.yaml file =========="
echo "env_variables:" > .env.yaml
echo " SECRET: ${{ secrets.WEBSITE_SECRET }}" >> .env.yaml
echo " CLOUDINARY_KEY: ${{ secrets.WEBSITE_CLOUDINARY_KEY }}" >> .env.yaml
echo " CLOUDINARY_NAME: ${{ secrets.WEBSITE_CLOUDINARY_NAME }}" >> .env.yaml
echo " CLOUDINARY_SECRET: ${{ secrets.WEBSITE_CLOUDINARY_SECRET }}" >> .env.yaml
echo " DATABASE_URI: ${{ secrets.WEBSITE_STAGE_DATABASE_URI }}" >> .env.yaml
echo " GS_BUCKET_NAME: ${{ secrets.WEBSITE_STAGE_GS_BUCKET_NAME }}" >> .env.yaml
echo " REACT_WEB_STATIC_HOST: ${{ secrets.WEBSITE_STAGE_REACT_WEB_STATIC_HOST }}" >> .env.yaml
echo " SECRET_KEY: ${{ secrets.WEBSITE_STAGE_SECRET_KEY }}" >> .env.yaml
echo " DJANGO_ALLOWED_HOSTS: ${{ secrets.WEBSITE_DJANGO_ALLOWED_HOSTS }}" >> .env.yaml
echo " DEBUG: False" >> .env.yaml
echo " GOOGLE_APPLICATION_CREDENTIALS: /app/google_application_credentials.json" >> .env.yaml
- name: Build and Push Docker Image
run: |
cd website/
docker build . \
--build-arg REACT_WEB_STATIC_HOST=${{ secrets.WEBSITE_STAGE_REACT_WEB_STATIC_HOST }} \
--build-arg REACT_NETMANAGER_BASE_URL=${{ secrets.WEBSITE_STAGE_REACT_NETMANAGER_BASE_URL }} \
--build-arg REACT_APP_NETMANAGER_URL=${{ secrets.WEBSITE_STAGE_REACT_NETMANAGER }} \
--build-arg REACT_APP_BASE_AIRQLOUDS_URL=${{ secrets.WEBSITE_STAGE_REACT_APP_BASE_AIRQLOUDS_URL }} \
--build-arg REACT_APP_BASE_NEWSLETTER_URL=${{ secrets.WEBSITE_REACT_APP_BASE_NEWSLETTER_URL }} \
--build-arg REACT_APP_WEBSITE_BASE_URL=${{ secrets.WEBSITE_STAGE_REACT_APP_WEBSITE_BASE_URL }} \
--build-arg REACT_APP_AUTHORIZATION_TOKEN=${{ secrets.WEBSITE_STAGE_REACT_APP_AUTHORIZATION_TOKEN }} \
--build-arg REACT_APP_GEO_LOCATION_URL=${{ secrets.WEBSITE_STAGE_REACT_APP_GEO_LOCATION_URL }} \
--build-arg REACT_APP_OPENCAGE_API_KEY=${{ secrets.REACT_APP_OPENCAGE_API_KEY }} \
--tag ${{ env.REGISTRY_URL }}/${{ secrets.FRONTEND_PROJECT_ID }}/airqo-stage-website:${{ needs.image-tag.outputs.build_id }}
docker push ${{ env.REGISTRY_URL }}/${{ secrets.FRONTEND_PROJECT_ID }}/airqo-stage-website:${{ needs.image-tag.outputs.build_id }}
docker tag ${{ env.REGISTRY_URL }}/${{ secrets.FRONTEND_PROJECT_ID }}/airqo-stage-website:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ secrets.FRONTEND_PROJECT_ID }}/airqo-stage-website:latest
docker push ${{ env.REGISTRY_URL }}/${{ secrets.FRONTEND_PROJECT_ID }}/airqo-stage-website:latest
- name: Deploy to App Engine
run: |-
cd website/
gcloud app deploy \
--image-url=${{ env.REGISTRY_URL }}/${{ secrets.FRONTEND_PROJECT_ID }}/airqo-stage-website:latest \
--project=${{ secrets.FRONTEND_PROJECT_ID }} \
--quiet
- name: Delete Old Versions
run: |-
service="staging"
versions=$(gcloud app versions list --service=$service --sort-by '~LAST_DEPLOYED' --format 'value(VERSION.ID)' | grep -v 'maintenance' | sort -r | tail -n +4)
for version in $versions; do
echo "Deleting version: $version for service: $service"
gcloud app versions delete "$version" --service=$service --quiet
done
### build and push calibrate app image ###
calibrate-app:
name: build-push-calibrate-app-image
needs: [check, image-tag]
if: needs.check.outputs.run_calibrate_app == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Google login
uses: google-github-actions/auth@v1
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK(gcloud)
uses: google-github-actions/setup-gcloud@v1
- name: Create .env file
run: gcloud secrets versions access latest --secret="sta-env-calibrate-app" > calibrate/.env
- name: NPM Setup and Build
uses: actions/setup-node@v3
with:
node-version: "12"
- run: |
cd calibrate/
npm install
CI=false npm run build
- name: Login to GCR
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Build and Push Docker Image
run: |
cd calibrate/
docker build --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-calibrate-app:${{ needs.image-tag.outputs.build_id }} .
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-calibrate-app:${{ needs.image-tag.outputs.build_id }}
docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-calibrate-app:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-calibrate-app:latest
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-calibrate-app:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/wretry.action@v1.2.0 # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/calibrate/values-stage.yaml"
propertyPath: "image.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update calibrate app staging image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to K8S
uses: azure/k8s-set-context@v3
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }}
- name: Update the corresponding configmap
run: |
kubectl create configmap --dry-run=client -o yaml \
--from-env-file=calibrate/.env env-calibrate-staging | kubectl replace -f - -n staging
### build and push analytics platform image ###
analytics-platform:
name: build-push-analytics-platform-image
needs: [check, image-tag]
if: needs.check.outputs.run_analytics_platform == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Google login
uses: google-github-actions/auth@v1
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK(gcloud)
uses: google-github-actions/setup-gcloud@v1
- name: Create .env file
run: gcloud secrets versions access latest --secret="sta-env-next-platform" > platform/.env
- name: Login to GCR
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Build and Push Docker Image
run: |
cd platform/
docker build --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-next-platform:${{ needs.image-tag.outputs.build_id }} .
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-next-platform:${{ needs.image-tag.outputs.build_id }}
docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-next-platform:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-next-platform:latest
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-next-platform:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/wretry.action@v1.2.0 # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/platform/values-stage.yaml"
propertyPath: "image.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update next platform staging image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to K8S
uses: azure/k8s-set-context@v3
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }}
- name: Update the corresponding configmap
run: |
kubectl create configmap --dry-run=client -o yaml \
--from-env-file=platform/.env env-next-platform-staging | kubectl replace -f - -n staging
### build and push docs image ###
docs:
name: build-push-docs-image
needs: [check, image-tag]
if: needs.check.outputs.run_docs == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Login to GCR
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Build and Push Docker Image
run: |
cd docs/
docker build --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-docs:${{ needs.image-tag.outputs.build_id }} .
docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-docs:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-docs:latest
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-docs:latest
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-docs:${{ needs.image-tag.outputs.build_id }}
- name: Update corresponding helm values file(with retry)
uses: Wandalen/wretry.action@v1.2.0 # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/docs/values-stage.yaml"
propertyPath: "image.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update docs staging image tag to ${{ needs.image-tag.outputs.build_id }}"
### build and push inventory image ###
inventory:
name: build-push-inventory-image
needs: [check, image-tag]
if: needs.check.outputs.run_inventory == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Google login
uses: google-github-actions/auth@v1
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK(gcloud)
uses: google-github-actions/setup-gcloud@v1
- name: Create .env file
run: gcloud secrets versions access latest --secret="sta-env-inventory" > inventory/.env
- name: Login to GCR
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Build and Push Docker Image
run: |
cd inventory/
docker build --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-inventory:${{ needs.image-tag.outputs.build_id }} .
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-inventory:${{ needs.image-tag.outputs.build_id }}
docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-inventory:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-inventory:latest
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-inventory:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/wretry.action@v1.2.0 # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/inventory/values-stage.yaml"
propertyPath: "image.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update inventory staging image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to K8S
uses: azure/k8s-set-context@v3
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }}
- name: Update the corresponding configmap
run: |
kubectl create configmap --dry-run=client -o yaml \
--from-env-file=inventory/.env env-inventory-staging | kubectl replace -f - -n staging
### build and push reports image ###
reports:
name: build-push-reports-image
needs: [check, image-tag]
if: needs.check.outputs.run_reports == 'true'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Google login
uses: google-github-actions/auth@v1
with:
credentials_json: ${{ secrets.GCP_SA_CREDENTIALS }}
- name: Setup Cloud SDK(gcloud)
uses: google-github-actions/setup-gcloud@v1
- name: Create .env file
run: gcloud secrets versions access latest --secret="sta-env-reports" > reports/.env
- name: Login to GCR
uses: docker/login-action@v2
with:
registry: ${{ env.REGISTRY_URL }}
username: _json_key
password: ${{ secrets.GCR_CONFIG }}
- name: Build and Push Docker Image
run: |
cd reports/
docker build --tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-reports:${{ needs.image-tag.outputs.build_id }} .
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-reports:${{ needs.image-tag.outputs.build_id }}
docker tag ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-reports:${{ needs.image-tag.outputs.build_id }} ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-reports:latest
docker push ${{ env.REGISTRY_URL }}/${{ env.PROJECT_ID }}/airqo-stage-reports:latest
- name: Update corresponding helm values file(with retry)
uses: Wandalen/wretry.action@v1.2.0 # Retries action on fail
with:
action: fjogeleit/yaml-update-action@main # Action to retry
with: |
valueFile: "k8s/reports/values-stage.yaml"
propertyPath: "image.tag"
value: ${{ needs.image-tag.outputs.build_id }}
branch: ${{ env.DEPLOY_BRANCH }}
token: ${{ secrets.YAML_UPDATER_TOKEN }}
message: "Update reports staging image tag to ${{ needs.image-tag.outputs.build_id }}"
- name: Login to K8S
uses: azure/k8s-set-context@v3
with:
method: kubeconfig
kubeconfig: ${{ secrets.K8S_CONFIG_STAGE }}
- name: Update the corresponding configmap
run: |
kubectl create configmap --dry-run=client -o yaml \
--from-env-file=reports/.env env-reports-staging | kubectl replace -f - -n staging