ci-model-regression.yml

# The docs:
# - https://www.notion.so/rasa/The-CI-for-model-regression-tests-aa579d5524a544af992f97d132bcc2de
# - https://www.notion.so/rasa/Datadog-Usage-Documentation-422099c9a3a24f5a99d92d904537dd0b
name: CI - Model Regression

on:
  push:
    branches:
      - "[0-9]+.[0-9]+.x"
    tags:
      - "**"
  pull_request:
    types: [opened, synchronize, labeled]

concurrency:
  group: ci-model-regression-${{ github.ref }} # branch or tag name
  cancel-in-progress: true

env:
  GKE_ZONE: us-central1
  GCLOUD_VERSION: "318.0.0"
  DD_PROFILING_ENABLED: false
  TF_FORCE_GPU_ALLOW_GROWTH: true
  NVML_INTERVAL_IN_SEC: 1

jobs:
  read_test_configuration:
    name: Reads tests configuration
    if: ${{ github.repository == 'RasaHQ/rasa' && contains(github.event.pull_request.labels.*.name, 'status:model-regression-tests') }}
    runs-on: ubuntu-22.04
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
      matrix_length: ${{ steps.set-matrix.outputs.matrix_length }}
      configuration_id: ${{ steps.fc_config.outputs.comment-id }}
      dataset_branch: ${{ steps.set-dataset-branch.outputs.dataset_branch }}

    steps:
      - name: Checkout main
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c

      - name: Download gomplate
        run: |-
          sudo curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.9.0/gomplate_linux-amd64
          sudo chmod +x /usr/local/bin/gomplate

      - name: Find a comment with configuration
        uses: tczekajlo/find-comment@16228d0f2100e06ea9bf8c0e7fe7287b7c6b531d
        id: fc_config
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          issue-number: ${{ github.event.number }}
          body-includes: "^/modeltest"

      - run: echo ${{ steps.fc_config.outputs.comment-id }}

      # This step has to happen before the other configuration details are read from
      # the same PR comment, because we need to check out the correct branch to feed the
      # dataset mapping and configs into the 'Read configuration from a PR comment' step
      # which creates the experiments matrix
      - name: Read dataset branch from a PR comment
        if: steps.fc_config.outputs.comment-id != ''
        id: set-dataset-branch
        run: |-
          source <(gomplate -d github=https://api.github.com/repos/${{ github.repository }}/issues/comments/${{ steps.fc_config.outputs.comment-id }} -H 'github=Authorization:token ${{ secrets.GITHUB_TOKEN }}' -f .github/templates/model_regression_test_read_dataset_branch.tmpl)
          echo "dataset_branch=${DATASET_BRANCH}" >> $GITHUB_OUTPUT

      - name: Checkout dataset
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
        with:
          repository: ${{ secrets.DATASET_REPOSITORY }}
          token: ${{ secrets.ML_TEST_SA_PAT }}
          path: "dataset"
          ref: ${{ steps.set-dataset-branch.outputs.dataset_branch }}

      - name: Render help description from template
        id: get_help_description
        run: |
          OUTPUT=$(gomplate -d mapping=./dataset/dataset_config_mapping.json -f .github/templates/model_regression_test_config_comment.tmpl)
          OUTPUT="${OUTPUT//$'\n'/'%0A'}"
          OUTPUT="${OUTPUT//$'\r'/'%0D'}"
          echo "help_description=$OUTPUT" >> $GITHUB_OUTPUT

      - name: Create a comment with help description
        uses: RasaHQ/create-comment@da7b2ec20116674919493bb5894eea70fdaa6486
        with:
          mode: "delete-previous"
          id: comment_help_description
          github-token: ${{ secrets.GITHUB_TOKEN }}
          body: |
            ${{ steps.get_help_description.outputs.help_description }}

      - if: steps.fc_config.outputs.comment-id == ''
        run: echo "::error::Cannot find a comment with the configuration"
        name: Log a warning message if a configuration cannot be found

      - name: Read configuration from a PR comment
        if: steps.fc_config.outputs.comment-id != ''
        id: set-matrix
        run: |-
          matrix=$(gomplate -d mapping=./dataset/dataset_config_mapping.json -d github=https://api.github.com/repos/${{ github.repository }}/issues/comments/${{ steps.fc_config.outputs.comment-id }} -H 'github=Authorization:token ${{ secrets.GITHUB_TOKEN }}' -f .github/templates/model_regression_test_config_to_json.tmpl)

          if [ $? -ne 0 ]; then
            echo "::error::Cannot read config from PR. Please double check your config."
            exit 1
          fi

          matrix_length=$(echo $matrix | jq '.[] | length')
          echo "matrix_length=$matrix_length" >> $GITHUB_OUTPUT
          echo "matrix=$matrix" >> $GITHUB_OUTPUT

      - name: Update the comment with the configuration
        uses: peter-evans/create-or-update-comment@3383acd359705b10cb1eeef05c0e88c056ea4666
        if: steps.fc_config.outputs.comment-id != ''
        with:
          comment-id: ${{ steps.fc_config.outputs.comment-id }}
          body: |
            <!-- comment-id:comment_configuration -->
          reactions: eyes

      - name: Re-create the comment with the configuration
        uses: RasaHQ/create-comment@da7b2ec20116674919493bb5894eea70fdaa6486
        if: steps.fc_config.outputs.comment-id != '' && steps.fc_config.outputs.comment-body != ''
        with:
          mode: "delete-previous"
          id: comment_configuration
          github-token: ${{ secrets.GITHUB_TOKEN }}
          body: ${{ steps.fc_config.outputs.comment-body }}

      - name: Find a comment with configuration - update
        uses: tczekajlo/find-comment@16228d0f2100e06ea9bf8c0e7fe7287b7c6b531d
        id: fc_config_update
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          issue-number: ${{ github.event.number }}
          body-includes: "^/modeltest"

      - name: Add reaction
        uses: peter-evans/create-or-update-comment@3383acd359705b10cb1eeef05c0e88c056ea4666
        if: steps.fc_config_update.outputs.comment-id != ''
        with:
          edit-mode: "replace"
          comment-id: ${{ steps.fc_config_update.outputs.comment-id }}
          reactions: heart, hooray, rocket

      - name: Add a comment that the tests are in progress
        uses: RasaHQ/create-comment@da7b2ec20116674919493bb5894eea70fdaa6486
        if: steps.fc_config_update.outputs.comment-id != ''
        with:
          mode: "delete-previous"
          id: comment_tests_in_progress
          github-token: ${{ secrets.GITHUB_TOKEN }}
          body: |
            The model regression tests have started. It might take a while, please be patient.
            As soon as results are ready you'll see a new comment with the results.

            Used configuration can be found in [the comment.](https://github.com/${{ github.repository }}/pull/${{ github.event.number}}#issuecomment-${{ steps.fc_config_update.outputs.comment-id }})

  deploy_runner_gpu:
    name: Deploy Github Runner - GPU
    needs: read_test_configuration
    runs-on: ubuntu-22.04
    if: ${{ contains(github.event.pull_request.labels.*.name, 'runner:gpu') && github.repository == 'RasaHQ/rasa' && contains(github.event.pull_request.labels.*.name, 'status:model-regression-tests') && needs.read_test_configuration.outputs.configuration_id != '' }}

    steps:
      - name: Checkout
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c

      - name: Download gomplate
        run: |-
          sudo curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.9.0/gomplate_linux-amd64
          sudo chmod +x /usr/local/bin/gomplate

      - name: Get TensorFlow version
        run: |-
          # Read TF version from poetry.lock file
          pip install toml
          TF_VERSION=$(scripts/read_tensorflow_version.sh)
          # Keep the first 3 characters, e.g. we keep 2.3 if TF_VERSION is 2.3.4
          TF_VERSION=${TF_VERSION::3}
          echo "TensorFlow version: $TF_VERSION"
          echo TF_VERSION=$TF_VERSION >> $GITHUB_ENV

      # Use compatible CUDA/cuDNN with the given TF version
      - name: Prepare GitHub runner image tag
        run: |-
          GH_RUNNER_IMAGE_TAG=$(jq -r 'if (.config | any(.TF == "${{ env.TF_VERSION }}" )) then (.config[] | select(.TF == "${{ env.TF_VERSION }}") | .IMAGE_TAG) else .default_image_tag end' .github/configs/tf-cuda.json)
          echo "GitHub runner image tag for TensorFlow ${{ env.TF_VERSION }} is ${GH_RUNNER_IMAGE_TAG}"
          echo GH_RUNNER_IMAGE_TAG=$GH_RUNNER_IMAGE_TAG >> $GITHUB_ENV

          num_max_replicas=3
          matrix_length=${{ needs.read_test_configuration.outputs.matrix_length }}
          if [[ $matrix_length -gt $num_max_replicas ]]; then
            NUM_REPLICAS=$num_max_replicas
          else
            NUM_REPLICAS=$matrix_length
          fi
          echo NUM_REPLICAS=$NUM_REPLICAS >> $GITHUB_ENV

      - name: Send warning if the current TF version does not have CUDA image tags configured
        if: env.GH_RUNNER_IMAGE_TAG == 'latest'
        env:
          TF_CUDA_FILE: ./github/config/tf-cuda.json
        run: |-
          echo "::warning file=${TF_CUDA_FILE},line=3,col=1,endColumn=3::Missing cuda config for tf ${{ env.TF_VERSION }}. If you are not sure how to config CUDA, please reach out to infrastructure."

      - name: Notify slack on tf-cuda config updates
        if: env.GH_RUNNER_IMAGE_TAG == 'latest'
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
        uses: voxmedia/github-action-slack-notify-build@3665186a8c1a022b28a1dbe0954e73aa9081ea9e
        with:
          channel_id: ${{ secrets.SLACK_ALERTS_CHANNEL_ID }}
          status: WARNING
          color: warning

      - name: Render deployment template
        run: |-
          export GH_RUNNER_IMAGE_TAG=${{ env.GH_RUNNER_IMAGE_TAG }}
          export GH_RUNNER_IMAGE=${{ secrets.GH_RUNNER_IMAGE }}
          gomplate -f .github/runner/github-runner-deployment.yaml.tmpl -o runner_deployment.yaml

      # Setup gcloud auth
      - uses: google-github-actions/auth@e8df18b60c5dd38ba618c121b779307266153fbf
        with:
          service_account: ${{ secrets.GKE_RASA_CI_GPU_SA_NAME_RASA_CI_CD }}
          credentials_json: ${{ secrets.GKE_SA_RASA_CI_CD_GPU_RASA_CI_CD }}

      # Get the GKE credentials for the cluster
      - name: Get GKE Cluster Credentials
        uses: google-github-actions/get-gke-credentials@894c221960ab1bc16a69902f29f090638cca753f
        with:
          cluster_name: ${{ secrets.GKE_GPU_CLUSTER_RASA_CI_CD }}
          location: ${{ env.GKE_ZONE }}
          project_id: ${{ secrets.GKE_SA_RASA_CI_GPU_PROJECT_RASA_CI_CD }}

      - name: Deploy Github Runner
        run: |-
          kubectl apply -f runner_deployment.yaml
          kubectl -n github-runner rollout status --timeout=15m deployment/github-runner-$GITHUB_RUN_ID

  model_regression_test_gpu:
    name: Model Regression Tests - GPU
    needs:
      - deploy_runner_gpu
      - read_test_configuration
    env:
      # Determine where CUDA and Nvidia libraries are located. TensorFlow looks for libraries in the given paths
      LD_LIBRARY_PATH: "/usr/local/cuda/extras/CUPTI/lib64:/usr/local/cuda/lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64"
      ACCELERATOR_TYPE: "GPU"
    runs-on: [self-hosted, gpu, "${{ github.run_id }}"]
    strategy:
      # max-parallel: By default, GitHub will maximize the number of jobs run in parallel depending on the available runners on GitHub-hosted virtual machines.
      matrix: ${{fromJson(needs.read_test_configuration.outputs.matrix)}}
      fail-fast: false
    if: ${{ contains(github.event.pull_request.labels.*.name, 'runner:gpu') && github.repository == 'RasaHQ/rasa' && contains(github.event.pull_request.labels.*.name, 'status:model-regression-tests') && needs.read_test_configuration.outputs.configuration_id != '' }}

    steps:
      - name: Checkout
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c

      - name: Checkout dataset
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
        with:
          repository: ${{ secrets.DATASET_REPOSITORY }}
          token: ${{ secrets.ML_TEST_SA_PAT }}
          path: "dataset"
          ref: ${{ needs.read_test_configuration.outputs.dataset_branch }}

      - name: Set env variables
        id: set_dataset_config_vars
        env:
          DATASET_NAME: "${{ matrix.dataset }}"
          CONFIG_NAME: "${{ matrix.config }}"
        run: |-
          # determine extra environment variables
          # - CONFIG
          # - DATASET
          # - IS_EXTERNAL
          # - EXTERNAL_DATASET_REPOSITORY_BRANCH
          # - TRAIN_DIR
          # - TEST_DIR
          # - DOMAIN_FILE
          source <(gomplate -d mapping=./dataset/dataset_config_mapping.json -f .github/templates/configuration_variables.tmpl)

          # Not all configurations are available for all datasets.
          # The job will fail and the workflow continues, if the configuration file doesn't exist
          # for a given dataset

          echo "is_dataset_exists=true" >> $GITHUB_OUTPUT
          echo "is_config_exists=true" >> $GITHUB_OUTPUT
          echo "is_external=${IS_EXTERNAL}" >> $GITHUB_OUTPUT

          # Warn about job if dataset is Hermit and config is BERT + DIET(seq) + ResponseSelector(t2t) or Sparse + BERT + DIET(seq) + ResponseSelector(t2t)
          if [[ "${{ matrix.dataset }}" == "Hermit" && "${{ matrix.config }}" =~ "BERT + DIET(seq) + ResponseSelector(t2t)" ]]; then
            echo "::warning::This ${{ matrix.dataset }} dataset / ${{ matrix.config }} config is currently being skipped on scheduled tests due to OOM associated with the upgrade to TF 2.6. You may see OOM here."
          fi

          if [[ "${IS_EXTERNAL}" == "true" ]]; then
            echo "DATASET_DIR=dataset_external" >> $GITHUB_ENV
          else
            echo "DATASET_DIR=dataset" >> $GITHUB_ENV
            test -d dataset/$DATASET || (echo "::warning::The ${{ matrix.dataset }} dataset doesn't exist. Skipping the job." \
              && echo "is_config_exists=false" >> $GITHUB_OUTPUT && exit 0)
          fi

          # Skip job if a given type is not available for a given dataset
          if [[ -z "${DOMAIN_FILE}" && "${{ matrix.type }}" == "core" ]]; then
            echo "::warning::The ${{ matrix.dataset }} dataset doesn't include core type. Skipping the job." \
              && echo "is_config_exists=false" >> $GITHUB_OUTPUT && exit 0
          fi

          test -f dataset/configs/$CONFIG || (echo "::warning::The ${{ matrix.config }} configuration file doesn't exist. Skipping the job." \
              && echo "is_dataset_exists=false" >> $GITHUB_OUTPUT && exit 0)

          echo "DATASET=${DATASET}" >> $GITHUB_ENV
          echo "CONFIG=${CONFIG}" >> $GITHUB_ENV
          echo "DOMAIN_FILE=${DOMAIN_FILE}" >> $GITHUB_ENV
          echo "EXTERNAL_DATASET_REPOSITORY_BRANCH=${EXTERNAL_DATASET_REPOSITORY_BRANCH}" >> $GITHUB_ENV
          echo "IS_EXTERNAL=${IS_EXTERNAL}" >> $GITHUB_ENV

          if [[ -z "${TRAIN_DIR}" ]]; then
            echo "TRAIN_DIR=train" >> $GITHUB_ENV
          else
            echo "TRAIN_DIR=${TRAIN_DIR}" >> $GITHUB_ENV
          fi

          if [[ -z "${TEST_DIR}" ]]; then
            echo "TEST_DIR=test" >> $GITHUB_ENV
          else
            echo "TEST_DIR=${TEST_DIR}" >> $GITHUB_ENV
          fi

          HOST_NAME=`hostname`
          echo "HOST_NAME=${HOST_NAME}" >> $GITHUB_ENV

      - name: Checkout dataset - external
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
        if: steps.set_dataset_config_vars.outputs.is_external == 'true'
        with:
          repository: ${{ env.DATASET }}
          token: ${{ secrets.ML_TEST_SA_PAT }}
          path: "dataset_external"
          ref: ${{ env.EXTERNAL_DATASET_REPOSITORY_BRANCH }}

      - name: Set dataset commit
        id: set-dataset-commit
        working-directory: ${{ env.DATASET_DIR }}
        run: |
          DATASET_COMMIT=$(git rev-parse HEAD)
          echo $DATASET_COMMIT
          echo "dataset_commit=$DATASET_COMMIT" >> $GITHUB_OUTPUT

      - name: Start Datadog Agent
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        env:
          DATASET_NAME: "${{ matrix.dataset }}"
          CONFIG: "${{ matrix.config }}"
          DATASET_COMMIT: "${{ steps.set-dataset-commit.outputs.dataset_commit }}"
          BRANCH: ${{ github.head_ref }}
          GITHUB_SHA: "${{ github.sha }}"
          PR_ID: "${{ github.event.number }}"
          TYPE: "${{ matrix.type }}"
          DATASET_REPOSITORY_BRANCH: ${{ needs.read_test_configuration.outputs.dataset_branch }}
          INDEX_REPETITION: "${{ matrix.index_repetition }}"
        run: |
          export PR_URL="https://github.com/${GITHUB_REPOSITORY}/pull/${{ github.event.number }}"
          .github/scripts/start_dd_agent.sh "${{ secrets.DD_API_KEY }}" "${{ env.ACCELERATOR_TYPE }}" ${{ env.NVML_INTERVAL_IN_SEC }}

      - name: Set up Python 3.10 🐍
        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        with:
          python-version: '3.10'

      - name: Read Poetry Version 🔢
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        run: |
          echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
        shell: bash

      - name: Install poetry 🦄
        uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        with:
          poetry-version: ${{ env.POETRY_VERSION }}

      - name: Load Poetry Cached Libraries ⬇
        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        with:
          path: ~/.cache/pypoetry/virtualenvs
          key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-3.9-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }}

      - name: Install Dependencies 📦
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        run: |
          make install-full
          poetry run python -m spacy download de_core_news_md

      - name: Install datadog dependencies
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        run: poetry run pip install -U datadog-api-client ddtrace

      - name: Validate that GPUs are working
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        run: |-
          poetry run python .github/scripts/validate_gpus.py

      - name: Download pretrained models 💪
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        run: |-
          poetry run python .github/scripts/download_pretrained.py --config dataset/configs/${CONFIG}

      - name: Run test
        id: run_test
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        env:
          TFHUB_CACHE_DIR: ~/.tfhub_cache/
          OMP_NUM_THREADS: 1
        run: |-
          poetry run rasa --version

          export NOW_TRAIN=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}');
          cd ${{ github.workspace }}

          if [[ "${{ steps.set_dataset_config_vars.outputs.is_external }}" == "true" ]]; then
            export DATASET=.
          fi

          if [[ "${{ matrix.type }}" == "nlu" ]]; then
            poetry run ddtrace-run rasa train nlu --quiet -u ${DATASET_DIR}/${DATASET}/${TRAIN_DIR} -c dataset/configs/${CONFIG} --out ${DATASET_DIR}/models/${DATASET}/${CONFIG}
            echo "train_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT

            export NOW_TEST=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}');
            poetry run ddtrace-run rasa test nlu --quiet -u ${DATASET_DIR}/$DATASET/${TEST_DIR} -m ${DATASET_DIR}/models/$DATASET/$CONFIG --out ${{ github.workspace }}/results/$DATASET/$CONFIG

            echo "test_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TEST") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
            echo "total_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT

          elif [[ "${{ matrix.type }}" == "core" ]]; then
            poetry run ddtrace-run rasa train core --quiet -s ${DATASET_DIR}/$DATASET/$TRAIN_DIR -c dataset/configs/$CONFIG -d ${DATASET_DIR}/${DATASET}/${DOMAIN_FILE}
            echo "train_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT

            export NOW_TEST=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}');
            poetry run ddtrace-run rasa test core -s ${DATASET_DIR}/${DATASET}/${TEST_DIR} --out ${{ github.workspace }}/results/${{ matrix.dataset }}/${CONFIG}

            echo "test_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TEST") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
            echo "total_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
          fi

      - name: Generate a JSON file with a report / Publish results to Datadog
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        env:
          SUMMARY_FILE: "./report.json"
          DATASET_NAME: ${{ matrix.dataset }}
          RESULT_DIR: "${{ github.workspace }}/results"
          CONFIG: ${{ matrix.config }}
          TEST_RUN_TIME: ${{ steps.run_test.outputs.test_run_time }}
          TRAIN_RUN_TIME: ${{ steps.run_test.outputs.train_run_time }}
          TOTAL_RUN_TIME: ${{ steps.run_test.outputs.total_run_time }}
          DATASET_REPOSITORY_BRANCH: ${{ needs.read_test_configuration.outputs.dataset_branch }}
          TYPE: ${{ matrix.type }}
          INDEX_REPETITION: ${{ matrix.index_repetition }}
          DATASET_COMMIT: ${{ steps.set-dataset-commit.outputs.dataset_commit }}
          BRANCH: ${{ github.head_ref }}
          GITHUB_SHA: "${{ github.sha }}"
          PR_ID: "${{ github.event.number }}"
          DD_APP_KEY: ${{ secrets.DD_APP_KEY_PERF_TEST }}
          DD_API_KEY: ${{ secrets.DD_API_KEY }}
          DD_SITE: datadoghq.eu
        run: |-
          export PR_URL="https://github.com/${GITHUB_REPOSITORY}/pull/${{ github.event.number }}"
          poetry run pip install analytics-python
          poetry run python .github/scripts/mr_publish_results.py
          cat $SUMMARY_FILE

      - name: Upload an artifact with the report
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        with:
          name: report-${{ matrix.dataset }}-${{ matrix.config }}-${{ matrix.index_repetition }}
          path: report.json

      - name: Stop Datadog Agent
        if: ${{ always() && steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' }}
        run: |
          sudo service datadog-agent stop

  model_regression_test_cpu:
    name: Model Regression Tests - CPU
    needs:
      - read_test_configuration
    env:
      ACCELERATOR_TYPE: "CPU"
    runs-on: ubuntu-22.04
    strategy:
      max-parallel: 3
      matrix: ${{fromJson(needs.read_test_configuration.outputs.matrix)}}
      fail-fast: false
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'runner:gpu') && github.repository == 'RasaHQ/rasa' && contains(github.event.pull_request.labels.*.name, 'status:model-regression-tests') && needs.read_test_configuration.outputs.configuration_id != '' }}

    steps:
      - name: Checkout
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c

      - name: Checkout dataset
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
        with:
          repository: ${{ secrets.DATASET_REPOSITORY }}
          token: ${{ secrets.ML_TEST_SA_PAT }}
          path: "dataset"
          ref: ${{ needs.read_test_configuration.outputs.dataset_branch }}

      - name: Download gomplate
        run: |-
          sudo curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.9.0/gomplate_linux-amd64
          sudo chmod +x /usr/local/bin/gomplate

      - name: Set env variables
        id: set_dataset_config_vars
        env:
          DATASET_NAME: "${{ matrix.dataset }}"
          CONFIG_NAME: "${{ matrix.config }}"
        run: |-
          # determine extra environment variables
          # - CONFIG
          # - DATASET
          # - IS_EXTERNAL
          # - EXTERNAL_DATASET_REPOSITORY_BRANCH
          # - TRAIN_DIR
          # - TEST_DIR
          # - DOMAIN_FILE
          source <(gomplate -d mapping=./dataset/dataset_config_mapping.json -f .github/templates/configuration_variables.tmpl)

          # Not all configurations are available for all datasets.
          # The job will fail and the workflow continues, if the configuration file doesn't exist
          # for a given dataset

          echo "is_dataset_exists=true" >> $GITHUB_OUTPUT
          echo "is_config_exists=true" >> $GITHUB_OUTPUT
          echo "is_external=${IS_EXTERNAL}" >> $GITHUB_OUTPUT

          if [[ "${IS_EXTERNAL}" == "true" ]]; then
            echo "DATASET_DIR=dataset_external" >> $GITHUB_ENV
          else
            echo "DATASET_DIR=dataset" >> $GITHUB_ENV
            test -d dataset/$DATASET || (echo "::warning::The ${{ matrix.dataset }} dataset doesn't exist. Skipping the job." \
              && echo "is_config_exists=false" >> $GITHUB_OUTPUT && exit 0)
          fi

          # Skip job if a given type is not available for a given dataset
          if [[ -z "${DOMAIN_FILE}" && "${{ matrix.type }}" == "core" ]]; then
            echo "::warning::The ${{ matrix.dataset }} dataset doesn't include core type. Skipping the job." \
              && echo "is_config_exists=false" >> $GITHUB_OUTPUT && exit 0
          fi

          test -f dataset/configs/$CONFIG || (echo "::warning::The ${{ matrix.config }} configuration file doesn't exist. Skipping the job." \
              && echo "is_dataset_exists=false" >> $GITHUB_OUTPUT && exit 0)

          echo "DATASET=${DATASET}" >> $GITHUB_ENV
          echo "CONFIG=${CONFIG}" >> $GITHUB_ENV
          echo "DOMAIN_FILE=${DOMAIN_FILE}" >> $GITHUB_ENV
          echo "EXTERNAL_DATASET_REPOSITORY_BRANCH=${EXTERNAL_DATASET_REPOSITORY_BRANCH}" >> $GITHUB_ENV
          echo "IS_EXTERNAL=${IS_EXTERNAL}" >> $GITHUB_ENV

          if [[ -z "${TRAIN_DIR}" ]]; then
            echo "TRAIN_DIR=train" >> $GITHUB_ENV
          else
            echo "TRAIN_DIR=${TRAIN_DIR}" >> $GITHUB_ENV
          fi

          if [[ -z "${TEST_DIR}" ]]; then
            echo "TEST_DIR=test" >> $GITHUB_ENV
          else
            echo "TEST_DIR=${TEST_DIR}" >> $GITHUB_ENV
          fi

          HOST_NAME=`hostname`
          echo "HOST_NAME=${HOST_NAME}" >> $GITHUB_ENV

      - name: Checkout dataset - external
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
        if: steps.set_dataset_config_vars.outputs.is_external == 'true'
        with:
          repository: ${{ env.DATASET }}
          token: ${{ secrets.ML_TEST_SA_PAT }}
          path: "dataset_external"
          ref: ${{ env.EXTERNAL_DATASET_REPOSITORY_BRANCH }}

      - name: Set dataset commit
        id: set-dataset-commit
        working-directory: ${{ env.DATASET_DIR }}
        run: |
          DATASET_COMMIT=$(git rev-parse HEAD)
          echo $DATASET_COMMIT
          echo "dataset_commit=$DATASET_COMMIT" >> $GITHUB_OUTPUT

      - name: Start Datadog Agent
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        env:
          DATASET_NAME: "${{ matrix.dataset }}"
          CONFIG: "${{ matrix.config }}"
          DATASET_COMMIT: "${{ steps.set-dataset-commit.outputs.dataset_commit }}"
          BRANCH: ${{ github.head_ref }}
          GITHUB_SHA: "${{ github.sha }}"
          PR_ID: "${{ github.event.number }}"
          TYPE: "${{ matrix.type }}"
          DATASET_REPOSITORY_BRANCH: ${{ matrix.dataset_branch }}
          INDEX_REPETITION: "${{ matrix.index_repetition }}"
        run: |
          export PR_URL="https://github.com/${GITHUB_REPOSITORY}/pull/${{ github.event.number }}"
          .github/scripts/start_dd_agent.sh "${{ secrets.DD_API_KEY }}" "${{ env.ACCELERATOR_TYPE }}" ${{ env.NVML_INTERVAL_IN_SEC }}

      - name: Set up Python 3.10 🐍
        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        with:
          python-version: '3.10'

      - name: Read Poetry Version 🔢
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        run: |
          echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV
        shell: bash

      - name: Install poetry 🦄
        uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 # v8
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        with:
          poetry-version: ${{ env.POETRY_VERSION }}

      - name: Load Poetry Cached Libraries ⬇
        uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        with:
          path: ~/.cache/pypoetry/virtualenvs
          key: ${{ runner.os }}-poetry-${{ env.POETRY_VERSION }}-3.9-${{ hashFiles('**/poetry.lock') }}-${{ secrets.POETRY_CACHE_VERSION }}

      - name: Install Dependencies 📦
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        run: |
          make install-full
          poetry run python -m spacy download de_core_news_md

      - name: Install datadog dependencies
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        run: poetry run pip install -U datadog-api-client ddtrace

      - name: CPU run - Validate that no GPUs are available
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        run: |-
          poetry run python .github/scripts/validate_cpu.py

      - name: Download pretrained models 💪
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        run: |-
          poetry run python .github/scripts/download_pretrained.py --config dataset/configs/${CONFIG}

      - name: Run test
        id: run_test
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        env:
          TFHUB_CACHE_DIR: ~/.tfhub_cache/
          OMP_NUM_THREADS: 1
        run: |-
          poetry run rasa --version

          export NOW_TRAIN=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}');
          cd ${{ github.workspace }}

          if [[ "${{ steps.set_dataset_config_vars.outputs.is_external }}" == "true" ]]; then
            export DATASET=.
          fi

          if [[ "${{ matrix.type }}" == "nlu" ]]; then
            poetry run ddtrace-run rasa train nlu --quiet -u ${DATASET_DIR}/${DATASET}/${TRAIN_DIR} -c dataset/configs/${CONFIG} --out ${DATASET_DIR}/models/${DATASET}/${CONFIG}
            echo "train_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT

            export NOW_TEST=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}');
            poetry run ddtrace-run rasa test nlu --quiet -u ${DATASET_DIR}/$DATASET/${TEST_DIR} -m ${DATASET_DIR}/models/$DATASET/$CONFIG --out ${{ github.workspace }}/results/$DATASET/$CONFIG

            echo "test_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TEST") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
            echo "total_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT

          elif [[ "${{ matrix.type }}" == "core" ]]; then
            poetry run ddtrace-run rasa train core --quiet -s ${DATASET_DIR}/$DATASET/$TRAIN_DIR -c dataset/configs/$CONFIG -d ${DATASET_DIR}/${DATASET}/${DOMAIN_FILE}
            echo "train_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT

            export NOW_TEST=$(gomplate -i '{{ (time.Now).Format time.RFC3339}}');
            poetry run ddtrace-run rasa test core -s ${DATASET_DIR}/${DATASET}/${TEST_DIR} --out ${{ github.workspace }}/results/${{ matrix.dataset }}/${CONFIG}

            echo "test_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TEST") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
            echo "total_run_time=$(gomplate -i '{{ $t := time.Parse time.RFC3339 (getenv "NOW_TRAIN") }}{{ (time.Since $t).Round (time.Second 1) }}')" >> $GITHUB_OUTPUT
          fi

      - name: Generate a JSON file with a report / Publish results to Datadog
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        env:
          SUMMARY_FILE: "./report.json"
          DATASET_NAME: ${{ matrix.dataset }}
          RESULT_DIR: "${{ github.workspace }}/results"
          CONFIG: ${{ matrix.config }}
          TEST_RUN_TIME: ${{ steps.run_test.outputs.test_run_time }}
          TRAIN_RUN_TIME: ${{ steps.run_test.outputs.train_run_time }}
          TOTAL_RUN_TIME: ${{ steps.run_test.outputs.total_run_time }}
          DATASET_REPOSITORY_BRANCH: ${{ needs.read_test_configuration.outputs.dataset_branch }}
          TYPE: ${{ matrix.type }}
          INDEX_REPETITION: ${{ matrix.index_repetition }}
          DATASET_COMMIT: ${{ steps.set-dataset-commit.outputs.dataset_commit }}
          BRANCH: ${{ github.head_ref }}
          GITHUB_SHA: "${{ github.sha }}"
          PR_ID: "${{ github.event.number }}"
          DD_APP_KEY: ${{ secrets.DD_APP_KEY_PERF_TEST }}
          DD_API_KEY: ${{ secrets.DD_API_KEY }}
          DD_SITE: datadoghq.eu
        run: |-
          export PR_URL="https://github.com/${GITHUB_REPOSITORY}/pull/${{ github.event.number }}"
          poetry run pip install analytics-python
          poetry run python .github/scripts/mr_publish_results.py
          cat $SUMMARY_FILE

      - name: Upload an artifact with the report
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
        if: steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true'
        with:
          name: report-${{ matrix.dataset }}-${{ matrix.config }}-${{ matrix.index_repetition }}
          path: report.json

      - name: Stop Datadog Agent
        if: ${{ always() && steps.set_dataset_config_vars.outputs.is_dataset_exists == 'true' && steps.set_dataset_config_vars.outputs.is_config_exists == 'true' }}
        run: |
          sudo service datadog-agent stop

  combine_reports:
    name: Combine reports
    runs-on: ubuntu-22.04
    needs:
      - model_regression_test_cpu
      - model_regression_test_gpu
    if: ${{ always() && ((needs.model_regression_test_cpu.result != 'skipped') != (needs.model_regression_test_gpu.result != 'skipped')) }}
    outputs:
      success_status: ${{ steps.set-success-status.outputs.success_status }}

    steps:
      - name: Set success status
        id: set-success-status
        run: |-
          succeeded=${{ needs.model_regression_test_cpu.result == 'success' || needs.model_regression_test_gpu.result == 'success' }}
          if [[ $succeeded == "false" ]]; then
            success_status="Failed"
          elif [[ $succeeded == "true" ]]; then
            success_status="Succeeded"
          else
            success_status="Unknown"
          fi
          echo $success_status
          echo "success_status=$success_status" >> $GITHUB_OUTPUT

      - name: Checkout git repository 🕝
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c

      - name: Set up Python 3.10 🐍
        uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b
        with:
          python-version: '3.10'

      - name: Get reports
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
        with:
          path: reports/

      - name: Display structure of downloaded files
        continue-on-error: true
        run: ls -R
        working-directory: reports/

      - name: Merge all reports
        env:
          SUMMARY_FILE: "./report.json"
          REPORTS_DIR: "reports/"
        run: |
          python .github/scripts/mr_generate_summary.py
          cat $SUMMARY_FILE

      - name: Upload an artifact with the overall report
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
        with:
          name: report.json
          path: ./report.json

  set_job_success_status:
    name: Set job success status
    runs-on: ubuntu-22.04
    needs:
      - combine_reports
    if: ${{ always() && needs.combine_reports.result == 'success' }}
    steps:
      - name: Set return code
        run: |
          success_status=${{ needs.combine_reports.outputs.success_status }}
          echo "Status: $success_status"
          if [[ $success_status == "Succeeded" ]]; then
            exit 0
          else
            exit 1
          fi

  add_comment_results:
    name: Add a comment with the results
    runs-on: ubuntu-22.04
    needs:
      - combine_reports
    if: ${{ always() && needs.combine_reports.result == 'success' }}

    steps:
      - name: Checkout
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c

      - name: Download report from last on-schedule regression test
        run: |
          # Get ID of last on-schedule workflow
          SCHEDULE_ID=$(curl -X GET -s -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -H "Accept: application/vnd.github.v3+json" \
            "https://api.github.com/repos/${{ github.repository }}/actions/workflows" \
            | jq -r '.workflows[] | select(.name == "CI - Model Regression on schedule") | select(.path | test("schedule")) | .id')

          ARTIFACT_URL=$(curl -s -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -H "Accept: application/vnd.github.v3+json" \
            "https://api.github.com/repos/${{ github.repository }}/actions/workflows/${SCHEDULE_ID}/runs?event=schedule&status=completed&branch=main&per_page=1" | jq -r .workflow_runs[0].artifacts_url)

          DOWNLOAD_URL=$(curl -s -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -H "Accept: application/vnd.github.v3+json" "${ARTIFACT_URL}" \
            | jq -r '.artifacts[] | select(.name == "report.json") | .archive_download_url')

          # Download the artifact
          curl -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' -LJO -H "Accept: application/vnd.github.v3+json" $DOWNLOAD_URL

          # Unzip and change name
          unzip report.json.zip && mv report.json report_main.json

      - name: Download the report
        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
        with:
          name: report.json

      - name: Download gomplate
        run: |-
          sudo curl -o /usr/local/bin/gomplate -sSL https://github.com/hairyhenderson/gomplate/releases/download/v3.9.0/gomplate_linux-amd64
          sudo chmod +x /usr/local/bin/gomplate

      - name: Render a comment to add
        id: get_results
        run: |
          OUTPUT="$(gomplate -d data=report.json -d results_main=report_main.json -f .github/templates/model_regression_test_results.tmpl)"
          OUTPUT="${OUTPUT//$'\n'/'%0A'}"
          OUTPUT="${OUTPUT//$'\r'/'%0D'}"
          echo "result=$OUTPUT" >> $GITHUB_OUTPUT

          # Get time of current commit as start time
          TIME_ISO_COMMIT=$(gomplate -d github=https://api.github.com/repos/rasaHQ/rasa/commits/${{ github.sha }} -H 'github=Authorization:token ${{ secrets.GITHUB_TOKEN }}' -i '{{ (ds "github").commit.author.date }}')  # Example "2022-02-17T14:06:38Z"
          TIME_UNIX_COMMIT=$(date -d "${TIME_ISO_COMMIT}" +%s%3N)  # Example: "1645106798"

          # Get current time
          TIME_ISO_NOW=$(gomplate -i '{{ (time.Now).UTC.Format time.RFC3339}}') # Example: "2022-02-17T14:50:54Z%"
          TIME_UNIX_NOW=$(date -d "${TIME_ISO_NOW}" +%s%3N)  # Example: "1645118083"

          echo "from_ts=$TIME_UNIX_COMMIT" >> $GITHUB_OUTPUT
          echo "to_ts=$TIME_UNIX_NOW" >> $GITHUB_OUTPUT

      - name: Publish results as a PR comment
        uses: marocchino/sticky-pull-request-comment@f61b6cf21ef2fcc468f4345cdfcc9bda741d2343 # v2.6.2
        if: ${{ always() }}
        with:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          header: ${{ github.run_id }}
          append: true
          message: |-
            Status of the run: ${{ needs.combine_reports.outputs.success_status }}

            Commit: ${{ github.sha }}, [The full report is available as an artifact.](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})

            [Datadog dashboard link](https://app.datadoghq.eu/dashboard/mf4-2hu-x84?tpl_var_branch_baseline=${{ github.head_ref }}&from_ts=${{ steps.get_results.outputs.from_ts }}&to_ts=${{ steps.get_results.outputs.to_ts }}&live=false)

            ${{ steps.get_results.outputs.result }}

      - name: Remove 'status:model-regression-tests' label
        continue-on-error: true
        uses: buildsville/add-remove-label@6008d7bd99d3baeb7c04033584e68f8ec80b198b # v1.0
        with:
          token: ${{secrets.GITHUB_TOKEN}}
          label: "status:model-regression-tests"
          type: remove

      - name: Remove 'runner:gpu' label
        continue-on-error: true
        uses: buildsville/add-remove-label@6008d7bd99d3baeb7c04033584e68f8ec80b198b # v1.0
        with:
          token: ${{secrets.GITHUB_TOKEN}}
          label: "runner:gpu"
          type: remove

  remove_runner_gpu:
    name: Delete Github Runner - GPU
    needs:
      - deploy_runner_gpu
      - model_regression_test_gpu
    runs-on: ubuntu-22.04
    if: ${{ always() && needs.deploy_runner_gpu.result != 'skipped' && contains(github.event.pull_request.labels.*.name, 'runner:gpu') && contains(github.event.pull_request.labels.*.name, 'status:model-regression-tests') }}

    steps:
      # Setup gcloud auth
      - uses: google-github-actions/auth@e8df18b60c5dd38ba618c121b779307266153fbf
        with:
          service_account: ${{ secrets.GKE_RASA_CI_GPU_SA_NAME_RASA_CI_CD }}
          credentials_json: ${{ secrets.GKE_SA_RASA_CI_CD_GPU_RASA_CI_CD }}

      # Get the GKE credentials for the cluster
      - name: Get GKE Cluster Credentials
        uses: google-github-actions/get-gke-credentials@894c221960ab1bc16a69902f29f090638cca753f
        with:
          cluster_name: ${{ secrets.GKE_GPU_CLUSTER_RASA_CI_CD }}
          location: ${{ env.GKE_ZONE }}
          project_id: ${{ secrets.GKE_SA_RASA_CI_GPU_PROJECT_RASA_CI_CD }}

      - name: Remove Github Runner
        run: kubectl -n github-runner delete deployments github-runner-${GITHUB_RUN_ID} --grace-period=30