Staging CI/CD

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,18 +1,20 @@
 # Changes proposed ✍️
-- ...  
-- ### Screenshots (front-end changes only)  
-		- ...  
-
+
+- ...
+- ### Screenshots (front-end changes only)
+      - ...
+
 ## Checklist ✅
+
 - [ ] Label appropriately with `type:feature 🚀`, `type:enhancement ✨`, `type:bug 🐞`, or `type:documentation 📜`.
-- [ ] Tests are passing.  
+- [ ] Tests are passing.
 - [ ] New backend functionality has been unit-tested.
-- [ ] Environment variables have been updated
-  - [ ] Front-end: `frontend/.env.dist`
-  - [ ] Backend: `backend/.env.dist`, `backend/.env.dist.staging`, `backend/.env.dist.staging`.
+- [ ] Environment variables have been updated:
+  - [ ] Local frontend configuration: `frontend/.env.dist.local`, `frontend/.env.dist.composed`.
+  - [ ] Local backend: `backend/.env.dist.local`, `backend/.env.dist.composed`.
   - [ ] [Configuration docs](https://docs.crowd.dev/docs/configuration) have been updated.
-  - [ ] Team members only: update environment variables in Password manager and update the team
-- [ ] API documentation has been updated (if necessary) (see [docs on API documentation](https://docs.crowd.dev/docs/updating-api-documentation)).  
-- [ ] [Quality standards](https://github.com/CrowdDotDev/crowd-github-test-public/blob/main/CONTRIBUTING.md#quality-standards) are met.  
-- [ ] All changes have been tested in a staging site.  
-- [ ] All changes are working locally running crowd.dev's Docker local environment.
+  - [ ] Team members only: update environment variables in override, staging and production env. files and trigger update config script.
+- [ ] API documentation has been updated (if necessary) (see [docs on API documentation](https://docs.crowd.dev/docs/updating-api-documentation)).
+- [ ] [Quality standards](https://github.com/CrowdDotDev/crowd-github-test-public/blob/main/CONTRIBUTING.md#quality-standards) are met.
+- [ ] All changes have been tested in a staging site.
+- [ ] All changes are working locally running crowd.dev's Docker local environment.

.github/actions/build-docker-image/action.yaml

@@ -0,0 +1,34 @@
+name: Crowd docker image builder
+description: Builds and pushes crowd docker images
+
+inputs:
+  image:
+    description: Which docker image to build
+    required: true
+
+outputs:
+  image:
+    description: Image that was built
+    value: crowddotdev/${{ inputs.image }}:${{ steps.version-generator.outputs.IMAGE_VERSION }}
+
+runs:
+  using: composite
+  steps:
+    - name: Get short commit sha hash
+      uses: benjlevesque/short-sha@v2.1
+      id: short-sha
+      with:
+        length: 6
+
+    - run: echo "IMAGE_VERSION=${{ steps.short-sha.outputs.sha }}.${{ github.run_attempt }}" >> $GITHUB_OUTPUT
+      shell: bash
+      id: version-generator
+
+    - name: Login to docker repository
+      shell: bash
+      run: echo ${{ env.DOCKERHUB_PASSWORD }} | docker login -u ${{ env.DOCKERHUB_USERNAME }} --password-stdin
+
+    - name: Build and push docker image
+      shell: bash
+      working-directory: ./scripts
+      run: ./cli build-and-push ${{ inputs.image }} ${{ steps.version-generator.outputs.IMAGE_VERSION }}

.github/actions/deploy-service/action.yaml

@@ -0,0 +1,29 @@
+name: Crowd service deployer
+description: Deploys to crowd.dev kubernetes cloud environments
+
+inputs:
+  service:
+    description: Which cloud service to deploy
+    required: true
+
+  image:
+    description: Which docker image to deploy (full docker registry name with tag)
+    required: true
+
+  cluster:
+    description: To which cloud cluster to deploy
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Initialize staging kubernetes kubectl context
+      shell: bash
+      run: aws eks --region ${{ env.AWS_REGION }} update-kubeconfig --name ${{ inputs.cluster }}
+      env:
+        AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
+
+    - name: Deploy image
+      shell: bash
+      run: kubectl set image deployments/${{ inputs.service }}-dpl ${{ inputs.service }}=${{ inputs.image }}

.github/workflows/staging-deploy-backend.yaml

@@ -0,0 +1,90 @@
+name: Staging Deploy Backend services
+
+on:
+  push:
+    branches:
+      - 'staging/**'
+    paths:
+      - 'backend/**'
+      - '!backend/src/serverless/microservices/python/**'
+
+env:
+  DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+  DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+  CROWD_CLUSTER: ${{ secrets.STAGING_CLUSTER_NAME }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_REGION: ${{ secrets.AWS_REGION }}
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    outputs:
+      image: ${{ steps.image.outputs.IMAGE }}
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/build-docker-image
+        id: image-builder
+        with:
+          image: backend
+
+      - name: Set docker image output
+        id: image
+        run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
+
+  deploy-api:
+    needs: build-and-push
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/deploy-service
+        with:
+          service: api
+          image: ${{ needs.build-and-push.outputs.image }}
+          cluster: ${{ env.CROWD_CLUSTER }}
+
+  deploy-nodejs-worker:
+    needs: build-and-push
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/deploy-service
+        with:
+          service: nodejs-worker
+          image: ${{ needs.build-and-push.outputs.image }}
+          cluster: ${{ env.CROWD_CLUSTER }}
+
+  deploy-job-generator:
+    needs: build-and-push
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/deploy-service
+        with:
+          service: job-generator
+          image: ${{ needs.build-and-push.outputs.image }}
+          cluster: ${{ env.CROWD_CLUSTER }}

.github/workflows/staging-deploy-frontend.yaml

@@ -0,0 +1,93 @@
+name: Staging Deploy Frontend service
+
+on:
+  push:
+    branches:
+      - 'staging/**'
+    paths:
+      - 'frontend/**'
+
+env:
+  DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+  DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+  CROWD_CLUSTER: ${{ secrets.STAGING_CLUSTER_NAME }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_REGION: ${{ secrets.AWS_REGION }}
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    outputs:
+      image: ${{ steps.image.outputs.IMAGE }}
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/build-docker-image
+        id: image-builder
+        with:
+          image: frontend
+
+      - name: Set docker image output
+        id: image
+        run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
+
+  build-and-push-dev:
+    runs-on: ubuntu-latest
+    outputs:
+      image: ${{ steps.image.outputs.IMAGE }}
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/build-docker-image
+        id: image-builder
+        with:
+          image: frontend-dev
+
+      - name: Set docker image output
+        id: image
+        run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
+
+  deploy:
+    needs: build-and-push
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/deploy-service
+        with:
+          service: frontend
+          image: ${{ needs.build-and-push.outputs.image }}
+          cluster: ${{ env.CROWD_CLUSTER }}
+
+  deploy-dev:
+    needs: build-and-push-dev
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/deploy-service
+        with:
+          service: frontend-dev
+          image: ${{ needs.build-and-push-dev.outputs.image }}
+          cluster: ${{ env.CROWD_CLUSTER }}

.github/workflows/staging-deploy-premium-job-generator.yaml

@@ -0,0 +1,55 @@
+name: Staging Deploy Premium Job Generator service
+
+on:
+  push:
+    branches:
+      - 'staging/**'
+    paths:
+      - 'premium/job-generator/**'
+
+env:
+  DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+  DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+  CROWD_CLUSTER: ${{ secrets.STAGING_CLUSTER_NAME }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_REGION: ${{ secrets.AWS_REGION }}
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    outputs:
+      image: ${{ steps.image.outputs.IMAGE }}
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/build-docker-image
+        id: image-builder
+        with:
+          image: premium-job-generator
+
+      - name: Set docker image output
+        id: image
+        run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
+
+  deploy-premium-job-generator:
+    needs: build-and-push
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+
+      - uses: ./.github/actions/deploy-service
+        with:
+          service: premium-job-generator
+          image: ${{ needs.build-and-push.outputs.image }}
+          cluster: ${{ env.CROWD_CLUSTER }}