-
Notifications
You must be signed in to change notification settings - Fork 2
/
justfile
357 lines (296 loc) · 11.2 KB
/
justfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
# We import environment variables from .env
set dotenv-load := true
set shell := ["bash", "-euxc", "-o", "pipefail"]
# If CI and DENY_WARNINGS are set add rust flags
export RUSTFLAGS := if env_var_or_default("CI", "false") == "true" {
if env_var_or_default("DENY_WARNINGS", "true") == "true" {
trim(env_var_or_default("RUSTFLAGS", "") + " -D warnings")
} else {
env_var_or_default("RUSTFLAGS", "")
}
} else {
env_var_or_default("RUSTFLAGS", "")
}
export RUSTDOCFLAGS := if env_var_or_default("CI", "false") == "true" {
if env_var_or_default("DENY_WARNINGS", "true") == "true" {
trim(env_var_or_default("RUSTDOCFLAGS", "") + " -D warnings")
} else {
env_var_or_default("RUSTDOCFLAGS", "")
}
} else {
env_var_or_default("RUSTDOCFLAGS", "")
}
# FIXME we should use `just install**` in `Docker.ci-image` to make sure
# we install it always the same way and then we would not need this here
# but that is a different PR
tool_root := if env_var_or_default("GITHUB_ACTIONS", "false") == "true" {
"/usr/local"
} else {
justfile_directory() + "/.local"
}
# this seems to be case sensitive in some situations
export npm_config_prefix := tool_root
# Runs just --list
default:
@{{just_executable()}} --list
python-deps:
#!/usr/bin/env bash
cd snippet-extractor
pipenv install
# Fetches rust dependencies
rust-deps:
#!/usr/bin/env bash
set -eux -o pipefail
cargo fetch {{ if env_var_or_default("CI", "false") == "true" { "--locked" } else { "" } }}
# Get/Update/Fetch/Install all dependencies
deps: rust-deps
# Formats rust (checks only on CI)
rust-fmt:
#!/usr/bin/env bash
set -eux -o pipefail
cargo +nightly fmt --all -- {{ if env_var_or_default("CI", "false") == "true" { "--check" } else { "" } }};
cargo sort --grouped --workspace {{ if env_var_or_default("CI", "false") == "true" { "--check --check-format" } else { "" } }}
# Formats all code (checks only on CI)
fmt: rust-fmt
# Checks rust code, fails on warnings on CI
rust-check:
cargo clippy --all-targets --locked
# Checks all code, fails if there are any issues on CI
check: rust-check
# Checks if rust documentation can be build without issues
rust-check-doc:
cargo doc --all-features --no-deps --document-private-items --locked
# Builds rust documentation
rust-doc *args:
cargo doc --all-features --document-private-items --locked {{args}}
# Builds all documentation
doc: rust-doc
# Checks if all documentation can be build without issues
check-doc: rust-check-doc
# Builds rust
rust-build:
cargo build --locked
# Builds all code
build: rust-build python-deps
# Tests rust
rust-test: download-assets python-deps
#!/usr/bin/env -S bash -eu -o pipefail
export RUST_BACKTRACE=1
cargo test --lib --bins --tests --locked
cargo test --doc --locked
# Tests all code
test: rust-test
# Cleans up rusts build cache
rust-clean:
cargo clean
install-rust-tools:
cargo sort --help 2>/dev/null 1>&2 || cargo install cargo-sort
# Removes all asset data
remove-assets:
rm -rf ./assets/*
# Removes all local cached dependencies and generated files
clean: rust-clean
# Runs clean and removes assets
clean-fully: clean remove-assets
# Workaround to set env variable CI for all job dependencies
_pre-push: deps fmt check test
# Runs formatting, checks and test steps after deleting generated files.
pre-push $CI="true":
@{{just_executable()}} _pre-push
download-assets *args: python-deps
#!/usr/bin/env -S bash -eu -o pipefail
BASE_DIR="$(pwd)"
cd {{justfile_directory()}}/.github/scripts
PROFILE="{{ env_var_or_default("AWS_PROFILE", "S3BucketsDeveloperAccess-690046978283") }}"
{{ if env_var_or_default("CI", "false") == "false" { "export AWS_PROFILE=\"$PROFILE\"; echo AWS_PROFILE=$AWS_PROFILE;" } else { "" } }}
./download_assets.sh {{args}}
cd "${BASE_DIR}/snippet-extractor"
pipenv run python -c 'import nltk; nltk.download("punkt")'
upload-assets *args:
#!/usr/bin/env -S bash -eu -o pipefail
PROFILE="{{ env_var_or_default("AWS_PROFILE", "S3BucketsDeveloperAccess-690046978283") }}"
{{ if env_var_or_default("CI", "false") == "false" { "export AWS_PROFILE=\"$PROFILE\"; echo AWS_PROFILE=$AWS_PROFILE;" } else { "" } }}
./.github/scripts/prepare_data.sh {{args}} --upload
build-service-args name target="default" features="":
#!/usr/bin/env -S bash -eux -o pipefail
if [[ -z "{{features}}" ]]; then
features=""
else
features="--features {{features}}"
fi
if [[ "{{target}}" == "default" ]]; then
target=""
else
target="--target {{target}}"
fi
echo "--release --bin {{name}} $target $features"
build-service name target="default" features="":
#!/usr/bin/env -S bash -eux -o pipefail
args=$(just build-service-args {{name}} {{target}} {{features}})
if [[ "{{target}}" == "default" ]]; then
cargo build $args
else
cross build $args
fi
web-dev-up:
#!/usr/bin/env -S bash -eu -o pipefail
PROJECT=web-dev
# -gt 1 because of the heading
if [[ "$(docker ps --filter "label=com.docker.compose.project=$PROJECT" | wc -l)" -gt 1 ]]; then
echo "web-dev composition is already running, SKIPPING STARTUP"
exit 0
fi
if [[ "$(ls -l web-api/assets | grep 'assets/xaynia_v0201' | wc -l)" == "0" ]]; then
rm "./web-api/assets" || :
ln -s "./assets/xaynia_v0201" "./web-api/assets"
fi
export HOST_PORT_SCOPE=30
docker-compose -p "$PROJECT" -f "./web-api/compose.db.yml" up --detach --remove-orphans --build
web-dev-down:
#!/usr/bin/env -S bash -eu -o pipefail
docker-compose -p web-dev -f "./web-api/compose.db.yml" down
build-service-image crate_path bin model_dir="" ort_dir="":
#!/usr/bin/env -S bash -eux -o pipefail
docker build -f "{{crate_path}}/Dockerfile" \
--build-arg MODEL_DIR="{{model_dir}}" \
--build-arg ORT_DIR="{{ort_dir}}" \
-t "xayn-{{crate_path}}-{{bin}}" {{justfile_directory()}}
compose-all-build model="xaynia_v0201" ort="ort_v1.15.1":
#!/usr/bin/env -S bash -eux -o pipefail
{{just_executable()}} build-service-image web-api web-api "assets/{{model}}" "assets/{{ort}}"
compose-all-up *args:
#!/usr/bin/env -S bash -eux -o pipefail
PROJECT="compose-all"
# -gt 1 because of the heading
if [[ "$(docker ps --filter "label=com.docker.compose.project=$PROJECT" | wc -l)" -gt 1 ]]; then
echo "compose-all composition is already running, can not continue in this case"
exit 1
fi
export HOST_PORT_SCOPE=40
docker-compose \
-p "$PROJECT" \
-f "./web-api/compose.db.yml" \
-f "./web-api/compose.personalization.yml" \
-f "./web-api/compose.ingestion.yml" \
up \
--detach --remove-orphans --build {{args}}
compose-all-down *args:
#!/usr/bin/env -S bash -eux -o pipefail
docker-compose \
-p "compose-all" \
-f "./web-api/compose.db.yml" \
-f "./web-api/compose.personalization.yml" \
-f "./web-api/compose.ingestion.yml" \
down {{args}}
install-openapi-validator:
#!/usr/bin/env -S bash -eux -o pipefail
npm install -g \
@stoplight/spectral-cli@${SPECTRAL_CLI_VERSION} \
@ibm-cloud/openapi-ruleset@${IBM_OPENAPI_RULESET_VERSION} \
@ibm-cloud/openapi-ruleset-utilities@${IBM_OPENAPI_RULESET_UTILITIES_VERSION} \
validator@${VALIDATOR_VERSION}
validate-openapi:
#!/usr/bin/env -S bash -eux -o pipefail
# We need to call it once per file, if we pass in multiple files it will
# have some bug where it does not report error correctly.
for file in ls web-api/openapi/*.yaml; do
npx spectral lint --verbose -F warn "$file"
done
install-openapi-doc-generator:
#!/usr/bin/env -S bash -eux -o pipefail
npm install -g \
@redocly/cli@${REDOCLY_CLI_VERSION}
install-doc-generator:
#!/usr/bin/env -S bash -eux -o pipefail
cd docs
pipenv install --deploy
install-tools: install-doc-generator install-openapi-doc-generator install-openapi-validator install-rust-tools
generate-docs:
#!/usr/bin/env -S bash -eux -o pipefail
cd docs/
pipenv run sphinx-build -M html source/ build/
npx redocly build-docs ../web-api/openapi/front_office.yaml -o build/html/front_office.html
npx redocly build-docs ../web-api/openapi/back_office.yaml -o build/html/back_office.html
echo "docs.xayn.com" > build/html/CNAME
generate-openapi-docs api:
#!/usr/bin/env -S bash -eu -o pipefail
case "{{api}}" in
front_office|back_office)
;;
*)
echo "Usage: just generate-openapi-docs {front_office|back_office}" >&2
exit 1
;;
esac
npx redocly preview-docs web-api/openapi/{{api}}.yaml
validate-migrations-unchanged cmp_ref:
#!/usr/bin/env -S bash -eu -o pipefail
if ! git rev-list "{{ cmp_ref }}".."{{ cmp_ref }}"; then
git fetch --depth=1 "$(git remote get-url origin)" "{{ cmp_ref }}"
fi
changed_migrations=( $(\
git diff --name-only "{{ cmp_ref }}" | \
grep -E "^web-api/migrations/.*" \
) ) || true
if [ "${#changed_migrations[@]}" -gt 0 ]; then
for migration in "${changed_migrations[@]}"; do
echo "Migrations was changed ${migration}" >&2
done
exit 1
else
echo "OK - migrations unchanged"
fi
print-just-env:
export
mind-benchmark kind:
cargo test --package xayn-web-api --release --lib \
-- --nocapture --include-ignored --exact mind::run_{{kind}}_benchmark
tracing-flamegraph *args:
#!/usr/bin/env -S bash -eu -o pipefail
export XAYN_TEST_FLAME_LOG="${XAYN_TEST_FLAME_LOG:-info}"
cargo test -- {{args}}
for d in ./test-artifacts/*; do
if [[ -e "$d/tracing.folded" && ! -e "$d/tracing.flamegraph.svg" ]]; then
inferno-flamegraph "$d/tracing.folded" > "$d/tracing.flamegraph.svg"
echo "Flamegraph stored at: $d/tracing.flamegraph.svg"
inferno-flamegraph --flamechart "$d/tracing.folded" > "$d/tracing.flamechart.svg"
echo "Flamegraph stored at: $d/tracing.flamechart.svg"
fi
done
perf-flamegraph integration_test_bin:
#!/usr/bin/env -S bash -eu -o pipefail
export CARGO_PROFILE_BENCH_DEBUG=true
OUT_DIR="./test-artifacts/{{integration_test_bin}}"
mkdir -p "$OUT_DIR"
cargo flamegraph -o "$OUT_DIR/flamegraph.svg" --test {{integration_test_bin}}
if [ -e "$OUT_DIR/perf.data" ]; then
mv "$OUT_DIR/perf.data" "$OUT_DIR/perf.data.old"
fi
mv "perf.data" "$OUT_DIR/perf.data"
aws-login:
#!/usr/bin/env bash
PROFILE="{{ env_var_or_default("AWS_PROFILE", "S3BucketsDeveloperAccess-690046978283") }}"
{{ if env_var_or_default("CI", "false") == "false" { "export AWS_PROFILE=\"$PROFILE\"; echo AWS_PROFILE=$AWS_PROFILE;" } else { "" } }}
aws sso login
_test-project-root:
#!/usr/bin/env -S bash -eu -o pipefail
echo -n {{justfile_directory()}}
alias r := rust-test
alias t := test
alias pp := pre-push
# Helpers to make sure you use the right env variables when running any commands.
[no-cd]
run *args:
{{args}}
[no-cd]
pipenv *args:
pipenv {{args}}
[no-cd]
npm *args:
npm {{args}}
[no-cd]
npx *args:
npx {{args}}
[no-cd]
cargo *args:
cargo {{args}}