-
Notifications
You must be signed in to change notification settings - Fork 17
153 lines (117 loc) · 5.32 KB
/
regression-tests.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
name: Regression Test
on:
workflow_dispatch:
jobs:
run-tests:
runs-on: [ubuntu-latest]
timeout-minutes: 20
if: ${{ github.event_name != 'issue_comment' || needs.check-comment-trigger.outputs.is-comment-triggered == 'true' }}
strategy:
matrix:
# These are all the Python versions that we want to regression test with.
python-version: ["3.7", "3.8", "3.9", "3.10"]
name: Run Regressions Tests with Python Version ${{ matrix.python-version }}
steps:
- uses: actions/checkout@v2
- name: Create the logs directory
run: mkdir -p logs
- name: Set filename for the prev ver server's output logs
run: echo "SERVER_LOGS_PREV_FILE=logs/server_logs_prev_ver_${{ matrix.python-version }}" >> $GITHUB_ENV
- name: Set filename for the current ver server's output logs
run: echo "SERVER_LOGS_CURRENT_FILE=logs/server_logs_curr_ver_${{ matrix.python-version }}" >> $GITHUB_ENV
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- name: Set up GOPATH variable
run: echo "GOPATH=$(echo $HOME)" >> $GITHUB_ENV
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.19.1
- name: Install aqueduct-ml
run: pip3 install aqueduct-ml
- name: Start the prev ver server
run: (aqueduct start --verbose > $SERVER_LOGS_PREV_FILE 2>&1 &)
- name: Install Packages needed by the notebooks
run: pip3 install sklearn transformers torch
- name: Wait for server
timeout-minutes: 1
run: while ! echo exit | nc localhost 8080; do sleep 1; done
- name: Fetch the API key
run: echo "API_KEY=$(aqueduct apikey)" >> $GITHUB_ENV
- name: Run Sentiment Notebook on previous release
working-directory: examples
timeout-minutes: 10
run: python3 run_notebook.py --path "sentiment_analysis/Sentiment Model.ipynb" --server_address localhost:8080
- name: Install playwright
run: npm install @playwright/test && npx playwright install && npx playwright install-deps
- name: Take screenshot of old workflow run 1
working-directory: regression_tests
timeout-minutes: 5
run: imgname=old-flow-run-1 npx playwright test tests/screenshot.test.ts
- name: Trigger new workflow run
working-directory: regression_tests
timeout-minutes: 5
run: npx playwright test tests/trigger.test.ts --project=chromium
- name: Take screenshot of old workflow run 2
working-directory: regression_tests
timeout-minutes: 5
run: imgname=old-flow-run-2 npx playwright test tests/screenshot.test.ts
- name: Take checkpoint of workflow
working-directory: regression_tests
timeout-minutes: 5
run: sleep 5 && python3 tests/workflow_comparer.py --server_address localhost:8080 --checkpoint=create --path=checkpoint
# Grabs the pid of the process bound to port 8080 and kills it
- name: Kill the server
run: kill -9 $(lsof -nP -iTCP -sTCP:LISTEN | grep 8080 | awk '{print $2}')
# install_local.py requires ~/.aqueduct to exist.
- name: Update aqueduct with latest code
run: python3 scripts/install_local.py --gobinary --sdk --executor
- name: Start the server again
run: (aqueduct start --verbose > $SERVER_LOGS_CURRENT_FILE 2>&1 &)
- name: Wait for server again
timeout-minutes: 5
run: while ! echo exit | nc localhost 8080; do sleep 1; done
- name: Compare checkpoint of workflow
working-directory: regression_tests
timeout-minutes: 5
run: python3 tests/workflow_comparer.py --server_address localhost:8080 --checkpoint=diff --path=checkpoint
- name: Take screenshot of old workflow run 2
working-directory: regression_tests
timeout-minutes: 5
run: imgname=old-flow-run-2 npx playwright test tests/screenshot.test.ts
- name: Kill the server
run: kill -9 $(lsof -nP -iTCP -sTCP:LISTEN | grep 8080 | awk '{print $2}')
- name: clear the server
run: aqueduct clear
- name: Start the server again
run: (aqueduct start --verbose > $SERVER_LOGS_CURRENT_FILE 2>&1 &)
- name: Wait for server again
timeout-minutes: 5
run: while ! echo exit | nc localhost 8080; do sleep 1; done
- name: Fetch the API key
run: echo "API_KEY=$(aqueduct apikey)" >> $GITHUB_ENV
- name: Run Sentiment Notebook on new release
working-directory: examples
timeout-minutes: 10
run: python3 run_notebook.py --path "sentiment_analysis/Sentiment Model.ipynb" --server_address localhost:8080
- name: Take screenshot of new workflow
working-directory: regression_tests
timeout-minutes: 5
run: imgname=new-flow-run-1 npx playwright test tests/screenshot.test.ts
- uses: actions/upload-artifact@v3
if: always()
with:
name: Server Logs
path: logs/
- uses: actions/upload-artifact@v3
if: always()
with:
name: Executor Logs
path: ~/.aqueduct/server/logs/*
- uses: actions/upload-artifact@v3
if: always()
with:
name: screenshots
path: regression_tests/screenshots