-
Notifications
You must be signed in to change notification settings - Fork 622
126 lines (112 loc) · 3.81 KB
/
use-cases-benchmark.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
name: Use cases benchmarks
permissions:
id-token: write
contents: read
pull-requests: 'write'
on:
pull_request:
branches:
- "main"
types:
- "labeled"
push:
branches:
- "release/**"
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }}
cancel-in-progress: true
jobs:
start-ec2:
name: Start benchmarks ec2
if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.label.name == 'run-benchmarks')
runs-on: ubuntu-latest
steps:
- name: checkout
if: vars.BENCHMARKS_USE_CASES_RUNNERS_TYPE == 'self-hosted-ec2'
uses: actions/checkout@v3
with:
repository: ${{ secrets.BENCHMARKS_REPO }}
path: "benchmarks"
ref: "main"
token: ${{ secrets.BENCHMARK_REPO_TOKEN }}
- uses: aws-actions/configure-aws-credentials@v1-node16
if: vars.BENCHMARKS_USE_CASES_RUNNERS_TYPE == 'self-hosted-ec2'
with:
role-to-assume: ${{ secrets.BENCHMARKS_AWS_ROLE_ARN }}
aws-region: us-east-1
- name: start ec2
if: vars.BENCHMARKS_USE_CASES_RUNNERS_TYPE == 'self-hosted-ec2'
working-directory: benchmarks/scripts
shell: bash
run: |
pip install -q boto3
python3 ec2.py start -t linux_gpu_a10
benchmark:
name: Run Benchmarks
needs: start-ec2
runs-on: linux_gpu_a10
timeout-minutes: 720
steps:
- name: pull deeplake
uses: actions/checkout@v3
with:
path: "deeplake"
- name: pull benchmarks
uses: actions/checkout@v3
with:
repository: ${{ secrets.BENCHMARKS_REPO }}
path: "benchmarks"
ref: "main"
token: ${{ secrets.BENCHMARK_REPO_TOKEN }}
- name: pull reports
uses: actions/checkout@v3
with:
repository: ${{ secrets.BENCHMARK_REPORT }}
path: "reports"
ref: "main"
token: ${{ secrets.BENCHMARK_REPO_TOKEN }}
- name: run benchmarks
continue-on-error: true
shell: bash
working-directory: ./benchmarks
env:
DEEPLAKE_PATH: '../deeplake'
REPORTS_PATH: '../reports'
BENCHMARKS_THRESHOLD: ${{ vars.BENCHMARKS_THRESHOLD }}
BENCHMARKS_DEEPLAKE_TOKEN: ${{ secrets.BENCHMARKS_DEEPLAKE_TOKEN }}
BENCHMARKS_TRAIN_T5_DATASET: ${{ secrets.BENCHMARKS_TRAIN_T5_DATASET }}
BENCHMARKS_TRAIN_NANO_DATASET: ${{ secrets.BENCHMARKS_TRAIN_NANO_DATASET }}
DEEPLAKE_USER: ${{ secrets.BENCHMARKS_DEEPLAKE_USERNAME }}
DEEPLAKE_PASS: ${{ secrets.BENCHMARKS_DEEPLAKE_PASSWORD }}
BENCHMARK_TYPE: 'use-cases'
URL: ${{ secrets.SLACK_URL }}
PR_NUMBER: ${{ github.event.number }}
run: |
bash use-cases.sh
- name: comment pr
continue-on-error: false
if: github.event_name == 'pull_request'
uses: thollander/actions-comment-pull-request@v2
with:
filePath: ./benchmarks/report.md
comment_tag: 'report'
- name: complete
continue-on-error: true
shell: bash
env:
BENCHMARK_TYPE: 'use-cases'
URL: ${{ secrets.SLACK_URL }}
PR_NUMBER: ${{ github.event.number }}
working-directory: ./benchmarks
run: |
bash scripts/complete_benchmark.sh
if [ "${{ vars.BENCHMARKS_USE_CASES_RUNNERS_TYPE }}" == 'self-hosted-ec2' ]
then
echo bash "$(pwd)/scripts/self_stop_ec2.sh" | at now +2 minutes
fi
- name: stop ec2
if: vars.BENCHMARKS_USE_CASES_RUNNERS_TYPE == 'self-hosted-ec2' && cancelled()
working-directory: ./benchmarks
shell: bash
run: |
echo bash "$(pwd)/scripts/self_stop_ec2.sh" | at now +2 minutes