automation-ondemand-tests.yml

name: Automation On Demand Tests
on:
  workflow_dispatch:
    inputs:
      chainlinkVersion:
        description: Chainlink image version to use
        required: false
        type: string
      chainlinkImage:
        description: Chainlink image repo to use (Leave empty to build from head/ref)
        required: false
        type: string
      chainlinkVersionUpdate:
        description: Chainlink image version to use initially for upgrade test
        default: latest
        required: true
        type: string
      chainlinkImageUpdate:
        description: Chainlink image repo to use initially for upgrade test
        required: true
        default: public.ecr.aws/chainlink/chainlink
        type: string
      enableChaos:
        description: Check to enable chaos tests
        type: boolean
        default: false
        required: true
      enableReorg:
        description: Check to enable reorg tests
        type: boolean
        default: false
        required: true

env:
  ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ github.sha }}
  CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink

jobs:
  build-chainlink:
    environment: integration
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix:
        image:
          - name: ""
            dockerfile: core/chainlink.Dockerfile
            tag-suffix: ""
          - name: (plugins)
            dockerfile: plugins/chainlink.Dockerfile
            tag-suffix: -plugins
    name: Build Chainlink Image ${{ matrix.image.name }}
    runs-on: ubuntu22.04-16cores-64GB
    steps:
      - name: Collect Metrics
        if: inputs.chainlinkImage == ''
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
        with:
          id: automation-on-demand-build-chainlink
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Build Chainlink Image ${{ matrix.image.name }}
        continue-on-error: true
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          ref: ${{ github.head_ref || github.ref_name }}
      - name: Check if image exists
        if: inputs.chainlinkImage == ''
        id: check-image
        uses: smartcontractkit/chainlink-github-actions/docker/image-exists@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19
        with:
          repository: chainlink
          tag: ${{ github.sha }}${{ matrix.image.tag-suffix }}
          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
      - name: Build Image
        if: steps.check-image.outputs.exists == 'false' && inputs.chainlinkImage == ''
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/build-image@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19
        with:
          cl_repo: smartcontractkit/chainlink
          cl_ref: ${{ github.sha }}
          cl_dockerfile: ${{ matrix.image.dockerfile }}
          push_tag: ${{ env.CHAINLINK_IMAGE }}:${{ github.sha }}${{ matrix.image.tag-suffix }}
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
      - name: Print Chainlink Image Built
        if: inputs.chainlinkImage == ''
        run: |
          echo "### chainlink node image tag used for this test run :link:" >>$GITHUB_STEP_SUMMARY
          echo "\`${GITHUB_SHA}\`" >>$GITHUB_STEP_SUMMARY

  build-test-image:
    environment: integration
    permissions:
      id-token: write
      contents: read
    name: Build Test Image
    runs-on: ubuntu22.04-16cores-64GB
    steps:
      - name: Collect Metrics
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
        with:
          id: automation-on-demand-build-test-image
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Build Test Image
        continue-on-error: true
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          ref: ${{ github.head_ref || github.ref_name }}
      - name: Build Test Image
        if: inputs.enableChaos || inputs.enableReorg
        uses: ./.github/actions/build-test-image
        with:
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}

  automation-on-demand-tests:
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    needs: [build-chainlink, build-test-image]
    env:
      CHAINLINK_COMMIT_SHA: ${{ github.sha }}
      CHAINLINK_ENV_USER: ${{ github.actor }}
      TEST_LOG_LEVEL: info
    strategy:
      fail-fast: false
      matrix:
        tests:
          - name: chaos
            id: chaos
            suite: chaos
            nodes: 15
            os: ubuntu-latest
            enabled: ${{ inputs.enableChaos }}
            pyroscope_env: ci-automation-on-demand-chaos
            network: SIMULATED
            command: -run ^TestAutomationChaos$ ./chaos
          - name: reorg
            id: reorg
            suite: reorg
            nodes: 5
            os: ubuntu-latest
            enabled: ${{ inputs.enableReorg }}
            pyroscope_env: ci-automation-on-demand-reorg
            network: SIMULATED_NONDEV
            command: -run ^TestAutomationReorg$ ./reorg
          - name: upgrade 2.0
            id: upgrade-2.0
            type: upgrade
            suite: smoke
            nodes: 1
            os: ubuntu22.04-8cores-32GB
            enabled: true
            pyroscope_env: ci-automation-on-demand-upgrade
            network: SIMULATED
            command: -run ^TestAutomationNodeUpgrade/registry_2_0 ./smoke
          - name: upgrade 2.1
            id: upgrade-2.1
            type: upgrade
            suite: smoke
            nodes: 5
            os: ubuntu22.04-8cores-32GB
            enabled: true
            pyroscope_env: ci-automation-on-demand-upgrade
            network: SIMULATED
            command: -run ^TestAutomationNodeUpgrade/registry_2_1 ./smoke
          - name: upgrade 2.2
            id: upgrade-2.2
            type: upgrade
            suite: smoke
            nodes: 5
            os: ubuntu22.04-8cores-32GB
            enabled: true
            pyroscope_env: ci-automation-on-demand-upgrade
            network: SIMULATED
            command: -run ^TestAutomationNodeUpgrade/registry_2_2 ./smoke
    runs-on: ${{ matrix.tests.os }}
    name: Automation On Demand ${{ matrix.tests.name }} Test
    steps:
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          ref: ${{ github.head_ref || github.ref_name }}
      - name: Determine build to use
        id: determine-build
        shell: bash
        run: |
          if [[ "${{ inputs.chainlinkImage }}" == "" ]]; then
            echo "image=${{ env.CHAINLINK_IMAGE }}" >>$GITHUB_OUTPUT
            echo "version=${{ github.sha }}" >>$GITHUB_OUTPUT
            echo "upgrade_version=${{ github.sha }}" >>$GITHUB_OUTPUT
            echo "upgrade_image=${{ env.CHAINLINK_IMAGE }}" >>$GITHUB_OUTPUT
          else
            READ_CL_IMAGE=$(jq -r '.inputs.chainlinkImage' $GITHUB_EVENT_PATH)
            echo ::add-mask::$READ_CL_IMAGE
            echo "image=$READ_CL_IMAGE" >>$GITHUB_OUTPUT
            echo "version=${{ inputs.chainlinkVersion }}" >>$GITHUB_OUTPUT
            echo "upgrade_version=${{ inputs.chainlinkVersion }}" >>$GITHUB_OUTPUT
            echo "upgrade_image=$READ_CL_IMAGE" >>$GITHUB_OUTPUT
          fi
          if [[ "${{ matrix.tests.type }}" == "upgrade" ]]; then
            READ_CL_UPGR_IMAGE=$(jq -r '.inputs.chainlinkImageUpdate' $GITHUB_EVENT_PATH)
            echo ::add-mask::$READ_CL_UPGR_IMAGE
            echo "image=$READ_CL_UPGR_IMAGE" >>$GITHUB_OUTPUT
            echo "version=${{ inputs.chainlinkVersionUpdate }}" >>$GITHUB_OUTPUT
          fi
      - name: Setup GAP for Grafana
        uses: smartcontractkit/.github/actions/setup-gap@6c9d62fdad050cfb8b59376ded291f1350705944 # setup-gap@0.2.2
        with:
          # aws inputs
          aws-region: ${{ secrets.AWS_REGION }}
          aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
          api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
          # other inputs
          duplicate-authorization-header: "true"
      - name: Prepare Base64 TOML config
        env:
          SELECTED_NETWORKS: ${{ matrix.tests.network }}
          OLD_IMAGE: ${{ steps.determine-build.outputs.image }}
          OLD_VERSION: ${{ steps.determine-build.outputs.version }}          
          UPGRADE_VERSION: ${{ steps.determine-build.outputs.upgrade_version }}
          UPGRADE_IMAGE: ${{ steps.determine-build.outputs.upgrade_image }}
          PYROSCOPE_SERVER: ${{ matrix.tests.pyroscope_env == '' && '' || !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725
          PYROSCOPE_ENVIRONMENT: ${{ matrix.tests.pyroscope_env }}
          PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
          RUN_ID: ${{ github.run_id }}
          TEST_LOG_COLLECT: "true"
          LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
          LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          LOKI_BASIC_AUTH:  ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          LOGSTREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }}
          GRAFANA_URL: "http://localhost:8080/primary"
          GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
          $GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
        run: |
          echo ::add-mask::$UPGRADE_IMAGE
          echo ::add-mask::$OLD_IMAGE

          # load reusable functions
          source ./.github/scripts/functions.sh
          
          selected_networks=$(convert_to_toml_array "$SELECTED_NETWORKS")          

          if [ -n "$PYROSCOPE_SERVER" ]; then
            pyroscope_enabled=true
          else
            pyroscope_enabled=false
          fi          

          if [ -n "$TEST_LOG_COLLECT" ]; then
          test_log_collect=true
          else
          test_log_collect=false
          fi

          log_targets=$(convert_to_toml_array "$LOGSTREAM_LOG_TARGETS")

          grafana_bearer_token=""
          if [ -n "$GRAFANA_BEARER_TOKEN" ]; then
            grafana_bearer_token="bearer_token_secret=\"$GRAFANA_BEARER_TOKEN\""
          fi

          cat << EOF > config.toml
          [Network]
          selected_networks=$selected_networks
          
          [ChainlinkImage]
          image="$OLD_IMAGE"
          version="$OLD_VERSION"

          [ChainlinkUpgradeImage]
          image="$UPGRADE_IMAGE"
          version="$UPGRADE_VERSION"

          [Logging]
          test_log_collect=$test_log_collect
          run_id="$RUN_ID"

          [Logging.LogStream]
          log_targets=$log_targets

          [Logging.Loki]
          tenant_id="$LOKI_TENANT_ID"
          endpoint="$LOKI_ENDPOINT"
          basic_auth_secret="$LOKI_BASIC_AUTH"

          [Logging.Grafana]
          base_url="$GRAFANA_URL"
          dashboard_url="$GRAFANA_DASHBOARD_URL"
          $grafana_bearer_token

          [Pyroscope]
          enabled=$pyroscope_enabled
          server_url="$PYROSCOPE_SERVER"
          environment="$PYROSCOPE_ENVIRONMENT"
          key_secret="$PYROSCOPE_KEY"          
          EOF
          
          BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0)
          echo ::add-mask::$BASE64_CONFIG_OVERRIDE
          echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV               
      - name: Run Tests
        uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@fc3e0df622521019f50d772726d6bf8dc919dd38 # v2.3.19
        if: ${{ matrix.tests.enabled == true }}
        env:
          TEST_SUITE: ${{ matrix.tests.suite }}
        with:
          test_command_to_run: cd ./integration-tests && go test -timeout 60m -count=1 -json -test.parallel=${{ matrix.tests.nodes }} ${{ matrix.tests.command }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs
          test_download_vendor_packages_command: cd ./integration-tests && go mod download
          cl_repo: ${{ steps.determine-build.outputs.image }}
          cl_image_tag: ${{ steps.determine-build.outputs.version }}
          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
          artifacts_location: ./integration-tests/${{ matrix.tests.suite }}/logs
          publish_check_name: Automation On Demand Results ${{ matrix.tests.name }}
          token: ${{ secrets.GITHUB_TOKEN }}
          go_mod_path: ./integration-tests/go.mod
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
      - name: Upload test log
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
        if: failure()
        with:
          name: test-log-${{ matrix.tests.name }}
          path: /tmp/gotest.log
          retention-days: 7
        continue-on-error: true
      - name: Collect Metrics
        if: always()
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
        with:
          id: automation-on-demand-tests-${{ matrix.tests.id }}
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Automation On Demand ${{ matrix.tests.name }} Test
          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
        continue-on-error: true