playground_deploy_examples.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Collect And Deploy Playground Examples

on:
  workflow_dispatch:
env:
  BEAM_ROOT_DIR: "../../"
  SDK_CONFIG: "../../playground/sdks.yaml"
  BEAM_EXAMPLE_CATEGORIES: "../categories.yaml"
  BEAM_VERSION: 2.40.0
  K8S_NAMESPACE: playground-backend
  HELM_APP_NAME: playground-backend
  SUBDIRS: "./learning/katas ./examples ./sdks"
  ORIGIN: PG_EXAMPLES
jobs:
  check_examples:
    name: Check examples
    runs-on: ubuntu-latest
    outputs:
      example_has_changed: ${{ steps.check_has_example.outputs.example_has_changed }}
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v4
        with:
          python-version: '3.8'
      - name: install deps
        run: pip install -r requirements.txt
        working-directory: playground/infrastructure
      - name: get Difference
        id: check_file_changed
        run: |
          set -xeu
          # define the base ref
          BASE_REF=$GITHUB_BASE_REF
          if [ -z "$BASE_REF" ] || [ "$BASE_REF" == "master" ]; then
            BASE_REF=origin/master
          fi
          DIFF=$(git diff --name-only $BASE_REF $GITHUB_SHA -- $SUBDIRS)
          echo "example_diff=$DIFF" >> $GITHUB_OUTPUT
      - name: has Examples
        run: |
          output=$(python3 checker.py ${{ steps.check_file_changed.outputs.example_diff }})
          echo "example_has_changed=$output" >> $GITHUB_OUTPUT
        id: check_has_example
        working-directory: playground/infrastructure
      - name: Print has_example
        run: echo "${{ steps.check_has_example.outputs.example_has_changed }}"
  deploy_examples:
    name: Deploy examples
    runs-on: ubuntu-latest
    needs: [ check_examples ]
    if: needs.check_examples.outputs.example_has_changed == 'True'
    env:
      GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json
      GOOGLE_CLOUD_PROJECT: ${{ secrets.GCP_PLAYGROUND_PROJECT_ID }}
      SA_KEY_CONTENT: ${{ secrets.GCP_PLAYGROUND_SA_KEY }}
      REGION: us-central1
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
      - uses: actions/setup-python@v4
        with:
          python-version: '3.8'
      - uses: actions/setup-java@v3.8.0
        with:
            distribution: 'zulu'
            java-version: '8'
      - name: Install kubectl
        run: |
             curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" &&\
             chmod +x kubectl &&\
             mv kubectl /usr/local/bin/
      - name: Install helm
        run: |
             curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 &&\
             chmod 700 get_helm.sh &&\
             ./get_helm.sh
      - name: Set up Cloud SDK
        uses: google-github-actions/setup-gcloud@v0
        with:
          install_components: 'gke-gcloud-auth-plugin'
      - name: install deps
        run: pip install -r requirements.txt
        working-directory: playground/infrastructure
      - name: Remove default github maven configuration
        # This step is a workaround to avoid a decryption issue
        run: rm ~/.m2/settings.xml
      - name: Setup GCP account
        run: |
             echo "$SA_KEY_CONTENT" | base64 -d > $GOOGLE_APPLICATION_CREDENTIALS
             which gcloud
             gcloud auth activate-service-account --project=$GOOGLE_CLOUD_PROJECT \
              --key-file=$GOOGLE_APPLICATION_CREDENTIALS
      - name: Set Docker Tag
        run: echo "DOCKERTAG=${GITHUB_SHA}" >> $GITHUB_ENV
      - name: Set Docker Tag If Github Tag was trigger
        run: echo "DOCKERTAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
        if: startsWith(github.ref, 'refs/tags/')
      - name: Set Docker Registry env
        run: echo "DOCKER_REGISTRY=$DOCKER_REGISTRY" >> $GITHUB_ENV
        env:
          DOCKER_REGISTRY: ${{ env.REGION }}-docker.pkg.dev/${{ env.GOOGLE_CLOUD_PROJECT }}/playground-repository
  
      - name: Get K8s Config
        run: gcloud container clusters get-credentials --region     us-central1-a playground-examples
      - name: Login to Docker Registry
        run: cat $GOOGLE_APPLICATION_CREDENTIALS | docker login -u _json_key --password-stdin "$PLAYGROUND_REGISTRY_URL"
        env:
          PLAYGROUND_REGISTRY_URL: https://${{ env.REGION }}-docker.pkg.dev
      - name: Build And Push Java Backend
        run: |
             ./gradlew playground:backend:containers:java:dockerTagPush \
             -Pdocker-repository-root="$DOCKER_REGISTRY" \
             -Pbase-image="apache/beam_java8_sdk:$BEAM_VERSION" \
             -Pdocker-tag="$DOCKERTAG"
      - name: Build And Push Go Backend
        run: |
              ./gradlew playground:backend:containers:go:dockerTagPush \
              -Pdocker-repository-root="$DOCKER_REGISTRY" \
              -Pdocker-tag="$DOCKERTAG"
      - name: Build And Push Python Backend
        run: |
              ./gradlew playground:backend:containers:python:dockerTagPush \
              -Pdocker-repository-root="$DOCKER_REGISTRY" \
              -Pdocker-tag="$DOCKERTAG"
      - name: Clean All Build directories
        run: ./gradlew clean
      - name: Install helm chart
        run: |
             kubectl create namespace $K8S_NAMESPACE --dry-run=client -o yaml | kubectl apply -f - &&\
             helm install  --namespace $K8S_NAMESPACE $HELM_APP_NAME . \
               --set global.registry="$DOCKER_REGISTRY" \
               --set global.tag="$DOCKERTAG" &&\
             sleep 120
        working-directory: playground/infrastructure/helm

      - name: Run Python Examples CI
        run: |
             K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-python" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
             echo $K8S_SERVER_ADDRESS
             export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8081"
             python3 ci_cd.py --datastore-project $GOOGLE_CLOUD_PROJECT --step CI --sdk SDK_PYTHON --origin $ORIGIN --subdirs $SUBDIRS
        working-directory: playground/infrastructure

      - name: Run Python Examples CD
        run: |
             K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-python" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
             export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8081"
             python3 ci_cd.py --datastore-project $GOOGLE_CLOUD_PROJECT --step CD --sdk SDK_PYTHON --origin $ORIGIN --subdirs $SUBDIRS
        working-directory: playground/infrastructure

      - name: Run Go Examples CI
        run: |
             K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-go" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
             export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8082"
             python3 ci_cd.py --datastore-project $GOOGLE_CLOUD_PROJECT --step CI --sdk SDK_GO --origin $ORIGIN --subdirs $SUBDIRS
        working-directory: playground/infrastructure

      - name: Run Go Examples CD
        run: |
             K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-go" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
             export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8082"
             python3 ci_cd.py --datastore-project $GOOGLE_CLOUD_PROJECT --step CD --sdk SDK_GO --origin $ORIGIN --subdirs $SUBDIRS
        working-directory: playground/infrastructure

      - name: Run Java Examples CI
        run: |
             K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-java" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
             export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8080"
             python3 ci_cd.py --datastore-project $GOOGLE_CLOUD_PROJECT --step CI --sdk SDK_JAVA --origin $ORIGIN --subdirs $SUBDIRS
        working-directory: playground/infrastructure
      - name: Run Java Examples CD
        run: |
             K8S_SERVER_ADDRESS=$(kubectl get svc -n $K8S_NAMESPACE -l "app=service-java" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}')
             export SERVER_ADDRESS="$K8S_SERVER_ADDRESS:8080"
             python3 ci_cd.py --datastore-project $GOOGLE_CLOUD_PROJECT --step CD --sdk SDK_JAVA --origin $ORIGIN --subdirs $SUBDIRS
        working-directory: playground/infrastructure
      - name: Delete Helm Chart
        if: always()
        run: |
             helm del --namespace $K8S_NAMESPACE $HELM_APP_NAME