Devops/benchmarks (#2294)

activeloopai · Apr 18, 2023 · 00abcf9 · 00abcf9
1 parent 8ed820b
commit 00abcf9
Show file tree

Hide file tree

Showing 3 changed files with 146 additions and 32 deletions.
diff --git a/.github/workflows/deeplake-benchmark.yaml b/.github/workflows/deeplake-benchmark.yaml
@@ -32,8 +32,50 @@ jobs:
         with:
           labels: run-benchmarks
 
+  start-ec2:
+    name: Start benchmarks ec2
+    if: github.event_name == 'push' || (github.event_name == 'pull_request' &&  github.event.label.name == 'run-benchmarks')
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        if: vars.BENCHMARKS_DEFAULT_RUNNERS_TYPE == 'self-hosted-ec2'
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ secrets.BENCHMARKS_REPO }}
+          path: "benchmarks"
+          ref: "main"
+          token: ${{ secrets.BENCHMARK_REPO_TOKEN }}
+      - name: checkout
+        if: vars.BENCHMARKS_DEFAULT_RUNNERS_TYPE == 'self-hosted-ec2'
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ secrets.BENCHMARKS_REPO }}
+          path: "benchmarks"
+          ref: "main"
+          token: ${{ secrets.BENCHMARK_REPO_TOKEN }}
+
+      - uses: aws-actions/configure-aws-credentials@v1-node16
+        if: vars.BENCHMARKS_DEFAULT_RUNNERS_TYPE == 'self-hosted-ec2'
+        with:
+          role-to-assume: ${{ secrets.BENCHMARKS_AWS_ROLE_ARN }}
+          aws-region: us-east-1
+
+      - name: start ec2
+        if: vars.BENCHMARKS_DEFAULT_RUNNERS_TYPE == 'self-hosted-ec2'
+        working-directory: benchmarks/scripts
+        shell: bash
+        run: |
+          pip install -q boto3
+          readarray -t machines_list <<< "$( echo -e 'for i in ${{ vars.BENCHMARK_DEFAULT_MACHINES }}:\n    if "linux" in i:\n        print(i)' | python3)"
+          for i in "${machines_list[@]}"
+            do
+              python3 ec2.py start -t "$i"
+            done
+
+
   benchmark:
     name: Run Benchmarks
+    needs: start-ec2
     if: github.event_name == 'push' || (github.event_name == 'pull_request' &&  github.event.label.name == 'run-benchmarks')
     runs-on: benchmarks_${{ matrix.machine }}
     timeout-minutes: 720
@@ -73,6 +115,7 @@ jobs:
 
       - name: run benchmarks
         shell: bash
+        continue-on-error: true
         working-directory: ./benchmarks
         env:
           DEEPLAKE_USER: ${{ secrets.BENCHMARKS_DEEPLAKE_USERNAME}}
@@ -86,18 +129,32 @@ jobs:
           bash benchmarks.sh ${{ matrix.machine }}
 
       - name: comment pr
+        continue-on-error: true
         if: github.event_name == 'pull_request'
         uses: thollander/actions-comment-pull-request@v2
         with:
           filePath: ./benchmarks/report.md
           comment_tag: ${{ matrix.machine }}
 
       - name: complete
+        continue-on-error: true
         shell: bash
         env:
           BENCHMARK_TYPE: 'default'
           MACHINE: ${{ matrix.machine }}
           URL: ${{ secrets.SLACK_URL }}
           PR_NUMBER: ${{ github.event.number }}
         working-directory: ./benchmarks
-        run: bash scripts/complete_benchmark.sh
+        run: |
+          bash scripts/complete_benchmark.sh
+          if [ "{{ vars.BENCHMARKS_DEFAULT_RUNNERS_TYPE }}" == 'self-hosted-ec2' ]
+          then
+            echo bash "$(pwd)/scripts/self_stop_ec2.sh" | at now +2 minutes
+          fi
+      
+      - name: stop ec2
+        if: vars.BENCHMARKS_DEFAULT_RUNNERS_TYPE == 'self-hosted-ec2' && cancelled()
+        working-directory: ./benchmarks
+        shell: bash
+        run: |
+          echo bash "$(pwd)/scripts/self_stop_ec2.sh" | at now +2 minutes
diff --git a/.github/workflows/manual-benchmarks.yaml b/.github/workflows/manual-benchmarks.yaml
@@ -33,6 +33,7 @@ on:
         type: choice
         options:
           - "all machines"
+          - "linux machines"
           - linux_cpu
           - linux_gpu
           - macos_intel
@@ -53,10 +54,48 @@ jobs:
           if [ "${{ github.event.inputs.machine }}" == 'all machines' ]
           then
             echo machine='${{ vars.BENCHMARK_DEFAULT_MACHINES }}' >> $GITHUB_OUTPUT
+          elif [ "${{ github.event.inputs.machine }}" == 'linux machines' ]
+          then
+            readarray -t machines_list <<< "$( echo -e 'for i in ${{ vars.BENCHMARK_DEFAULT_MACHINES }}:\n    if "linux" in i:\n        print(i)' | python3)"
+            machines='['
+            for i in "${machines_list[@]}"
+            do
+              machines+=\"$i\",
+            done
+            machines="$(sed 's/,$/]/' <<< $machines)"
+            echo "machine=$machines" >> $GITHUB_OUTPUT
           else
             echo "machine=[ \"${{ github.event.inputs.machine }}\" ]" >> $GITHUB_OUTPUT
           fi
 
+      - name: checkout
+        if: vars.BENCHMARKS_DEFAULT_RUNNERS_TYPE == 'self-hosted-ec2'
+        uses: actions/checkout@v3
+        with:
+          repository: ${{ secrets.BENCHMARKS_REPO }}
+          path: "benchmarks"
+          ref: "main"
+          token: ${{ secrets.BENCHMARK_REPO_TOKEN }}
+
+      - uses: aws-actions/configure-aws-credentials@v1-node16
+        if: vars.BENCHMARKS_DEFAULT_RUNNERS_TYPE == 'self-hosted-ec2'
+        with:
+          role-to-assume: ${{ secrets.BENCHMARKS_AWS_ROLE_ARN }}
+          aws-region: us-east-1
+
+      - name: start ec2
+        if: vars.BENCHMARKS_DEFAULT_RUNNERS_TYPE == 'self-hosted-ec2'
+        working-directory: benchmarks/scripts
+        shell: bash
+        run: |
+          pip install -q boto3
+          readarray -t machines_list <<< "$( echo -e 'for i in ${{ vars.BENCHMARK_DEFAULT_MACHINES }}:\n    if "linux" in i:\n        print(i)' | python3)"
+          for i in "${machines_list[@]}"
+            do
+              python3 ec2.py start -t "$i"
+            done
+          
+
   default_benchmark:
     needs: set_matrix
     name: Run Default Benchmarks
@@ -99,39 +138,47 @@ jobs:
 
       - name: run benchmarks
         shell: bash
+        continue-on-error: true
         working-directory: ./benchmarks
         env:
           DEEPLAKE_USER: ${{ secrets.BENCHMARKS_DEEPLAKE_USERNAME }}
           DEEPLAKE_PASS: ${{ secrets.BENCHMARKS_DEEPLAKE_PASSWORD }}
           DEEPLAKE_PATH: '../deeplake'
           REPORTS_PATH: '../reports'
           NODE: ${{ matrix.machine }}
-          PR_NUMBER: ${{ github.event.number }}
           BENCHMARKS_THRESHOLD: ${{ vars.BENCHMARKS_THRESHOLD }}
+          BENCHMARK_TYPE: 'default'
+          MACHINE: ${{ matrix.machine }}
+          URL: ${{ secrets.SLACK_URL }}
+          PR_NUMBER: ${{ github.event.number }}
         run: |
           if [ "${{ github.event.inputs.use_config }}" == 'true' ]
           then
             bash benchmarks.sh ${{ github.event.inputs.config }}
           else
             bash benchmarks.sh ${{ matrix.machine }}
           fi
-
-      - name: complete
-        shell: bash
-        env:
-          BENCHMARK_TYPE: 'default'
-          MACHINE: ${{ matrix.machine }}
-          URL: ${{ secrets.SLACK_URL }}
-          PR_NUMBER: ${{ github.event.number }}
+          bash scripts/complete_benchmark.sh
+          if [ "${{ vars.BENCHMARKS_DEFAULT_RUNNERS_TYPE }}" == 'self-hosted-ec2' ]
+          then
+            echo bash "$(pwd)/scripts/self_stop_ec2.sh" | at now +2 minutes
+          fi
+      
+      - name: stop ec2
+        if: vars.BENCHMARKS_DEFAULT_RUNNERS_TYPE == 'self-hosted-ec2' && cancelled()
         working-directory: ./benchmarks
-        run: bash scripts/complete_benchmark.sh
+        shell: bash
+        run: |
+          echo bash "$(pwd)/scripts/self_stop_ec2.sh" | at now +2 minutes
+
 
   start_ec2:
     name: Start benchmarks EC2
     if: github.event.inputs.use_cases == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: checkout
+        if: vars.BENCHMARKS_USE_CASES_RUNNERS_TYPE == 'self-hosted-ec2'
         uses: actions/checkout@v3
         with:
           repository: ${{ secrets.BENCHMARKS_REPO }}
@@ -140,12 +187,14 @@ jobs:
           token: ${{ secrets.BENCHMARK_REPO_TOKEN }}
 
       - uses: aws-actions/configure-aws-credentials@v1-node16
+        if: vars.BENCHMARKS_USE_CASES_RUNNERS_TYPE == 'self-hosted-ec2'
         with:
           role-to-assume: ${{ secrets.BENCHMARKS_AWS_ROLE_ARN }}
           aws-region: us-east-1
-      
+
       - name: start ec2
         working-directory: benchmarks/scripts
+        if: vars.BENCHMARKS_USE_CASES_RUNNERS_TYPE == 'self-hosted-ec2'
         shell: bash
         run: |
           pip install -q boto3
@@ -180,36 +229,33 @@ jobs:
             token: ${{ secrets.BENCHMARK_REPO_TOKEN }}
 
         - name: run benchmarks
+          id: benchmarks
+          continue-on-error: true
           shell: bash
           working-directory: ./benchmarks
           env:
             DEEPLAKE_PATH: '../deeplake'
             REPORTS_PATH: '../reports'
-            URL: ${{ secrets.SLACK_URL }}
             BENCHMARKS_THRESHOLD: ${{ vars.BENCHMARKS_THRESHOLD }}
             BENCHMARKS_DEEPLAKE_TOKEN: ${{ secrets.BENCHMARKS_DEEPLAKE_TOKEN }}
             BENCHMARKS_TRAIN_T5_DATASET: ${{ secrets.BENCHMARKS_TRAIN_T5_DATASET }}
             BENCHMARKS_TRAIN_NANO_DATASET: ${{ secrets.BENCHMARKS_TRAIN_NANO_DATASET }}
             DEEPLAKE_USER: ${{ secrets.BENCHMARKS_DEEPLAKE_USERNAME }}
             DEEPLAKE_PASS: ${{ secrets.BENCHMARKS_DEEPLAKE_PASSWORD }}
             PR_NUMBER: ${{ github.event.number }}
-          run: |
-            bash use-cases.sh
-
-        - name: complete
-          shell: bash
-          env:
-            BENCHMARK_TYPE: 'use-cases'
             URL: ${{ secrets.SLACK_URL }}
-            PR_NUMBER: ${{ github.event.number }}
-          working-directory: ./benchmarks
+            BENCHMARK_TYPE: 'use-cases'
           run: |
+            bash use-cases.sh
             bash scripts/complete_benchmark.sh
+            if [ "${{ vars.BENCHMARKS_USE_CASES_RUNNERS_TYPE }}" == 'self-hosted-ec2' ]
+            then
+              echo bash "$(pwd)/scripts/self_stop_ec2.sh" | at now +2 minutes
+            fi 
 
         - name: stop ec2
+          if: vars.BENCHMARKS_USE_CASES_RUNNERS_TYPE == 'self-hosted-ec2' && cancelled()
+          working-directory: ./benchmarks
           shell: bash
           run: |
-            echo /usr/local/bin/aws \
-              ec2 stop-instances \
-              --instance-ids "$(curl -s http://169.254.169.254/latest/meta-data/instance-id)" \
-              | at now +1 minute
+            echo bash "$(pwd)/scripts/self_stop_ec2.sh" | at now +2 minutes
diff --git a/.github/workflows/use-cases-benchmark.yaml b/.github/workflows/use-cases-benchmark.yaml
@@ -27,6 +27,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout
+        if: vars.BENCHMARKS_USE_CASES_RUNNERS_TYPE == 'self-hosted-ec2'
         uses: actions/checkout@v3
         with:
           repository: ${{ secrets.BENCHMARKS_REPO }}
@@ -35,11 +36,13 @@ jobs:
           token: ${{ secrets.BENCHMARK_REPO_TOKEN }}
 
       - uses: aws-actions/configure-aws-credentials@v1-node16
+        if: vars.BENCHMARKS_USE_CASES_RUNNERS_TYPE == 'self-hosted-ec2'
         with:
           role-to-assume: ${{ secrets.BENCHMARKS_AWS_ROLE_ARN }}
           aws-region: us-east-1
 
       - name: start ec2
+        if: vars.BENCHMARKS_USE_CASES_RUNNERS_TYPE == 'self-hosted-ec2'
         working-directory: benchmarks/scripts
         shell: bash
         run: |
@@ -74,42 +77,50 @@ jobs:
           token: ${{ secrets.BENCHMARK_REPO_TOKEN }}
 
       - name: run benchmarks
+        continue-on-error: true
         shell: bash
         working-directory: ./benchmarks
         env:
           DEEPLAKE_PATH: '../deeplake'
           REPORTS_PATH: '../reports'
-          URL: ${{ secrets.SLACK_URL }}
           BENCHMARKS_THRESHOLD: ${{ vars.BENCHMARKS_THRESHOLD }}
           BENCHMARKS_DEEPLAKE_TOKEN: ${{ secrets.BENCHMARKS_DEEPLAKE_TOKEN }}
           BENCHMARKS_TRAIN_T5_DATASET: ${{ secrets.BENCHMARKS_TRAIN_T5_DATASET }}
           BENCHMARKS_TRAIN_NANO_DATASET: ${{ secrets.BENCHMARKS_TRAIN_NANO_DATASET }}
           DEEPLAKE_USER: ${{ secrets.BENCHMARKS_DEEPLAKE_USERNAME }}
           DEEPLAKE_PASS: ${{ secrets.BENCHMARKS_DEEPLAKE_PASSWORD }}
+          BENCHMARK_TYPE: 'use-cases'
+          URL: ${{ secrets.SLACK_URL }}
           PR_NUMBER: ${{ github.event.number }}
         run: |
           bash use-cases.sh
 
       - name: comment pr
+        continue-on-error: false
         if: github.event_name == 'pull_request'
         uses: thollander/actions-comment-pull-request@v2
         with:
           filePath: ./benchmarks/report.md
           comment_tag: 'report'
 
       - name: complete
+        continue-on-error: true
         shell: bash
         env:
           BENCHMARK_TYPE: 'use-cases'
           URL: ${{ secrets.SLACK_URL }}
           PR_NUMBER: ${{ github.event.number }}
         working-directory: ./benchmarks
-        run: bash scripts/complete_benchmark.sh
+        run: |
+          bash scripts/complete_benchmark.sh
+          if [ "${{ vars.BENCHMARKS_USE_CASES_RUNNERS_TYPE }}" == 'self-hosted-ec2' ]
+          then
+            echo bash "$(pwd)/scripts/self_stop_ec2.sh" | at now +2 minutes
+          fi
 
       - name: stop ec2
+        if: vars.BENCHMARKS_USE_CASES_RUNNERS_TYPE == 'self-hosted-ec2' && cancelled()
+        working-directory: ./benchmarks
         shell: bash
         run: |
-          echo /usr/local/bin/aws \
-            ec2 stop-instances \
-            --instance-ids "$(curl -s http://169.254.169.254/latest/meta-data/instance-id)" \
-            | at now +1 minute
+          echo bash "$(pwd)/scripts/self_stop_ec2.sh" | at now +2 minutes