forked from semgrep/semgrep
-
Notifications
You must be signed in to change notification settings - Fork 0
246 lines (238 loc) · 7.96 KB
/
tests.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
name: Tests
on:
workflow_dispatch:
pull_request:
paths-ignore:
- '**.md'
push:
branches:
- develop
paths-ignore:
- '**.md'
jobs:
build-core:
name: Build and Test semgrep-core
runs-on: ubuntu-latest
container: returntocorp/ocaml:alpine
steps:
- name: Pre-checkout fixes
run: |
sudo chmod -R 777 /github
github_cache_dir="/__w"
sudo mkdir -p "$github_cache_dir"
sudo chmod -R 777 "$github_cache_dir"
- name: Checkout
uses: actions/checkout@v2
with:
submodules: true
- name: Post-checkout fixes
run: ./.github/post-checkout
- name: Build semgrep-core and spacegrep
run: ./scripts/install-alpine-semgrep-core
- name: Upload artifacts
uses: actions/upload-artifact@v1
with:
name: ocaml-build-artifacts
path: ocaml-build-artifacts.tgz
- name: Test semgrep-core and spacegrep
run: |
eval $(opam env)
cd semgrep-core
START=`date +%s`
make test
make e2etest
END=`date +%s`
TEST_RUN_TIME=$((END-START))
curl --fail -L -X POST "https://dashboard.semgrep.dev/api/metric/semgrep.core.test-run-time-seconds.num" -d "$TEST_RUN_TIME"
- name: Report Number of Tests Stats
if: github.ref == 'refs/heads/develop'
run: |
./semgrep-core/tests/report_test_metrics.sh
# TODO: move this to a stable host for more reliable results.
#
# It's not clear how to push the stats only when "on the main
# branch". The GitHub Actions documentation is unhelpful. So we
# keep things simple and publish the results every time.
#
- name: Publish match performance
run: |
# This runs a short test suite to track the match performance
# of semgrep-core over time. The results are pushed to the
# dashboard at https://dashboard.semgrep.dev/
#
opam exec -- make -C semgrep-core report-perf
tox-tests:
name: semgrep tests with different python envs
runs-on: ubuntu-latest
needs: [build-core]
strategy:
matrix:
python: [3.6, 3.7, 3.8, 3.9]
steps:
- name: Setup Python
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python }}
- name: Checkout
uses: actions/checkout@v2
- name: Download artifacts
uses: actions/download-artifact@v1
with:
name: ocaml-build-artifacts
- name: Install artifacts
run: |
tar xf ocaml-build-artifacts/ocaml-build-artifacts.tgz
sudo cp ocaml-build-artifacts/bin/* /usr/bin
- name: Test semgrep-core
run: semgrep-core -version
- name: Install Tox and any other packages
run: pip install pipenv==2018.11.26 tox==3.15.0
- name: Run Tox
env:
SEMGREP_SKIP_BIN: true
run: |
cd semgrep
# See tox.ini for pytest command being run
tox -e py # Run tox using the version of Python in `PATH`
build-test:
name: semgrep unit and e2e tests
runs-on: ubuntu-latest
needs: [build-core]
strategy:
matrix:
python: [3.7]
steps:
- name: Checkout
uses: actions/checkout@v2
with:
submodules: true
- name: Setup Python
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python }}
- name: Download artifacts
uses: actions/download-artifact@v1
with:
name: ocaml-build-artifacts
- name: Install artifacts
run: |
tar xf ocaml-build-artifacts/ocaml-build-artifacts.tgz
sudo cp ocaml-build-artifacts/bin/* /usr/bin
- name: Install semgrep
run: |
cd semgrep
export PATH=/github/home/.local/bin:$PATH
pip3 install pipenv
pipenv install --dev
- name: Test semgrep
run: |
cd semgrep
export PATH=/github/home/.local/bin:$PATH
pipenv run pytest --ignore=tests/qa/test_public_repos.py -n auto -v --tb=short --durations=0 tests/
- name: Generate test matrix data
run: |
cd semgrep
pipenv run ../scripts/generate_test_matrix.py --json --directory ../semgrep-core/tests/ --output-file test-matrix.json
- name: Upload artifacts
uses: actions/upload-artifact@v1
with:
name: test-matrix
path: semgrep/test-matrix.json
public-repos-test:
name: run semgrep on public repos
runs-on: ubuntu-latest
needs: [build-core]
strategy:
matrix:
python: [3.7]
steps:
- name: Checkout
uses: actions/checkout@v2
with:
submodules: true
- name: Setup Python
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python }}
- name: Download artifacts
uses: actions/download-artifact@v1
with:
name: ocaml-build-artifacts
- name: Install artifacts
run: |
tar xf ocaml-build-artifacts/ocaml-build-artifacts.tgz
sudo cp ocaml-build-artifacts/bin/* /usr/bin
- name: Install semgrep
run: |
cd semgrep
export PATH=/github/home/.local/bin:$PATH
pip3 install pipenv
pipenv install --dev
- uses: actions/cache@v2
with:
path: ~/.cache/semgrep-cache
key: semgrep-repo-cache-test
- name: create cache dir if not exists
run: mkdir -p ~/.cache/semgrep-cache
- run: touch ~/.cache/semgrep-cache/ok
- name: Test semgrep
run: |
cd semgrep
export PATH=/github/home/.local/bin:$PATH
pipenv run pytest -n auto -v --tb=short --durations=0 tests/qa/test_public_repos.py
env:
GITHUB_REPO_CACHE: ~/.cache/semgrep-cache
# Run each benchmark twice to decrease effect of natural variance
benchmark-test:
name: semgrep benchmark tests
runs-on: ubuntu-latest
needs: [build-core]
strategy:
matrix:
python: [3.7]
steps:
- name: Checkout
uses: actions/checkout@v2
with:
submodules: true
- name: Setup Python
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python }}
- name: Run 0.58.2 Timing Benchmark
run: |
pip3 install semgrep==0.58.2
semgrep --version
python3 -m semgrep --version
export PATH=/github/home/.local/bin:$PATH
python3 perf/run-benchmarks --small-only --std-only --save-to semgrep/baseline_timing1.txt --no-time
cat semgrep/baseline_timing1.txt
python3 perf/run-benchmarks --small-only --std-only --save-to semgrep/baseline_timing2.txt --no-time
cat semgrep/baseline_timing2.txt
pip3 uninstall -y semgrep
- name: Download artifacts
uses: actions/download-artifact@v1
with:
name: ocaml-build-artifacts
- name: Install artifacts
run: |
tar xf ocaml-build-artifacts/ocaml-build-artifacts.tgz
sudo cp ocaml-build-artifacts/bin/* /usr/bin
- name: Install semgrep
run: |
cd semgrep
pip3 install pipenv
pipenv install --dev
- name: Get timing for latest semgrep
run: |
cd semgrep
pipenv run semgrep --version
pipenv run python -m semgrep --version
pipenv run semgrep-core -version
pipenv run python3 ../perf/run-benchmarks --small-only --std-only --save-to timing1.txt
cat timing1.txt
pipenv run python3 ../perf/run-benchmarks --small-only --std-only --save-to timing2.txt
cat timing2.txt
- name: Compare timing infos
run: |
python scripts/compare_perf.py semgrep/baseline_timing1.txt semgrep/baseline_timing2.txt semgrep/timing1.txt semgrep/timing2.txt ${{ secrets.GITHUB_TOKEN }} ${{ github.event.number }}