deeplake-benchmark.yaml

name: Deep Lake benchmarks

permissions:
  id-token: write
  contents: read
  pull-requests: 'write'

on:
  pull_request:
    branches:
      - "main"
    types:
      - "labeled"

  push:
    branches:
      - "release/**"

# concurrency:
#   group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }}
#   cancel-in-progress: true

jobs:
  remove_label:
    name: Remove run-benchmarks label
    if: github.event_name == 'pull_request' && github.event.label.name == 'run-benchmarks'
    runs-on: ubuntu-latest
    steps:
      - name: Remove run-benchmarks label
        continue-on-error: true
        uses: actions-ecosystem/action-remove-labels@v1.3.0
        with:
          labels: run-benchmarks

  start-ec2:
    name: Start benchmarks ec2
    if: github.event_name == 'push' || (github.event_name == 'pull_request' &&  github.event.label.name == 'run-benchmarks')
    runs-on: ubuntu-latest
    steps:
      - name: checkout
        if: vars.BENCHMARKS_DEFAULT_RUNNERS_TYPE == 'self-hosted-ec2'
        uses: actions/checkout@v3
        with:
          repository: ${{ secrets.BENCHMARKS_REPO }}
          path: "benchmarks"
          ref: "main"
          token: ${{ secrets.BENCHMARK_REPO_TOKEN }}
      - name: checkout
        if: vars.BENCHMARKS_DEFAULT_RUNNERS_TYPE == 'self-hosted-ec2'
        uses: actions/checkout@v3
        with:
          repository: ${{ secrets.BENCHMARKS_REPO }}
          path: "benchmarks"
          ref: "main"
          token: ${{ secrets.BENCHMARK_REPO_TOKEN }}

      - uses: aws-actions/configure-aws-credentials@v1-node16
        if: vars.BENCHMARKS_DEFAULT_RUNNERS_TYPE == 'self-hosted-ec2'
        with:
          role-to-assume: ${{ secrets.BENCHMARKS_AWS_ROLE_ARN }}
          aws-region: us-east-1

      - name: start ec2
        if: vars.BENCHMARKS_DEFAULT_RUNNERS_TYPE == 'self-hosted-ec2'
        working-directory: benchmarks/scripts
        shell: bash
        run: |
          pip install -q boto3
          readarray -t machines_list <<< "$( echo -e 'for i in ${{ vars.BENCHMARK_DEFAULT_MACHINES }}:\n    if "linux" in i:\n        print(i)' | python3)"
          for i in "${machines_list[@]}"
            do
              python3 ec2.py start -t "$i"
            done


  benchmark:
    name: Run Benchmarks
    needs: start-ec2
    if: github.event_name == 'push' || (github.event_name == 'pull_request' &&  github.event.label.name == 'run-benchmarks')
    runs-on: benchmarks_${{ matrix.machine }}
    timeout-minutes: 720
    strategy:
      fail-fast: false
      matrix:
        machine: ${{ fromJSON(vars.BENCHMARK_DEFAULT_MACHINES) }}
    steps:
      - name: Configure AWS credentials
        if: matrix.machine == 'macos_intel'
        uses: aws-actions/configure-aws-credentials@v1-node16
        with:
          role-to-assume: ${{ secrets.BENCHMARKS_AWS_ROLE_ARN }}
          aws-region: us-east-1
          role-duration-seconds: 14400

      - name: pull deeplake
        uses: actions/checkout@v3
        with:
          path: "deeplake"

      - name: pull benchmarks
        uses: actions/checkout@v3
        with:
          repository: ${{ secrets.BENCHMARKS_REPO }}
          path: "benchmarks"
          ref: "main"
          token: ${{ secrets.BENCHMARK_REPO_TOKEN }}

      - name: pull reports
        uses: actions/checkout@v3
        with:
          repository: ${{ secrets.BENCHMARK_REPORT }}
          path: "reports"
          ref: "main"
          token: ${{ secrets.BENCHMARK_REPO_TOKEN }}

      - name: run benchmarks
        shell: bash
        continue-on-error: true
        working-directory: ./benchmarks
        env:
          DEEPLAKE_USER: ${{ secrets.BENCHMARKS_DEEPLAKE_USERNAME}}
          DEEPLAKE_PASS: ${{ secrets.BENCHMARKS_DEEPLAKE_PASSWORD }}
          DEEPLAKE_PATH: '../deeplake'
          REPORTS_PATH: '../reports'
          NODE: ${{ matrix.machine }}
          PR_NUMBER: ${{ github.event.number }}
          BENCHMARKS_THRESHOLD: ${{ vars.BENCHMARKS_THRESHOLD }}
        run: |
          bash benchmarks.sh ${{ matrix.machine }}

      - name: comment pr
        continue-on-error: true
        if: github.event_name == 'pull_request'
        uses: thollander/actions-comment-pull-request@v2
        with:
          filePath: ./benchmarks/report.md
          comment_tag: ${{ matrix.machine }}

      - name: complete
        continue-on-error: true
        shell: bash
        env:
          BENCHMARK_TYPE: 'default'
          MACHINE: ${{ matrix.machine }}
          URL: ${{ secrets.SLACK_URL }}
          PR_NUMBER: ${{ github.event.number }}
        working-directory: ./benchmarks
        run: |
          bash scripts/complete_benchmark.sh
          if [ "{{ vars.BENCHMARKS_DEFAULT_RUNNERS_TYPE }}"" == 'self-hosted-ec2' ]
          then
            self_stop.sh
          fi
      
      - name: stop ec2
        if: vars.BENCHMARKS_DEFAULT_RUNNERS_TYPE == 'self-hosted-ec2' && cancelled()
        shell: bash
        run: self_stop.sh