Skip to content

Commit

Permalink
refactor: simplify Makefile by including .envrc
Browse files Browse the repository at this point in the history
  • Loading branch information
tekumara committed Dec 2, 2023
1 parent a04c7ed commit f324fc0
Show file tree
Hide file tree
Showing 3 changed files with 3 additions and 12 deletions.
2 changes: 1 addition & 1 deletion .envrc
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
export KUBECONFIG=$HOME/.k3d/kubeconfig-prefect.yaml
export KUBECONFIG=${HOME}/.k3d/kubeconfig-prefect.yaml
export PREFECT_API_URL=http://localhost:4200/api
export AWS_ACCESS_KEY_ID=minioadmin
export AWS_SECRET_ACCESS_KEY=minioadmin
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,6 @@ jobs:
- run: make kubes-prefect
- run: make deploy
- run: make run
- name: Setup tmate session
- name: Start tmate session if failure
if: ${{ failure() && (github.event_name == 'workflow_dispatch' || github.run_attempt > 1) }}
uses: mxschmitt/action-tmate@v3
11 changes: 1 addition & 10 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
include *.mk

export KUBECONFIG=$(HOME)/.k3d/kubeconfig-prefect.yaml
include .envrc

## create cluster and install minio and prefect
kubes: cluster kubes-minio kubes-prefect
Expand Down Expand Up @@ -70,15 +69,13 @@ param-flow: $(venv)
$(venv)/bin/python -m flows.param_flow

## run dask flow
dask-flow: export PREFECT_API_URL=http://localhost:4200/api
dask-flow: $(venv)
$(venv)/bin/python -m flows.dask_flow

## run ray flow
ray-flow: export PREFECT_LOCAL_STORAGE_PATH=/tmp/prefect/storage # see https://github.com/PrefectHQ/prefect-ray/issues/26
# PREFECT_API_URL needs to be accessible from the process running the flow and within the ray cluster
# to make this work locally, add 127.0.0.1 prefect-server to /etc/hosts TODO: find a better fix
ray-flow: export PREFECT_API_URL=http://prefect-server:4200/api
ray-flow: $(venv)
$(venv)/bin/python -m flows.ray_flow

Expand All @@ -91,9 +88,6 @@ publish:
docker buildx bake --push

## deploy flows to run on kubernetes
deploy: export PREFECT_API_URL=http://localhost:4200/api
deploy: export AWS_ACCESS_KEY_ID=minioadmin
deploy: export AWS_SECRET_ACCESS_KEY=minioadmin
deploy: $(venv) publish
# use minio as the s3 remote file system & deploy flows via python
set -e && . config/fsspec-env.sh && $(venv)/bin/python -m flows.deploy
Expand All @@ -103,7 +97,6 @@ deploy: $(venv) publish
@echo Visit http://localhost:4200

## run deployments
run: export PREFECT_API_URL=http://localhost:4200/api
run: $(venv)
$(venv)/bin/python -m flows.run

Expand All @@ -125,12 +118,10 @@ kubes-logs-dask:
kubectl logs -l "app=dask" -f --tail=-1

## show flow run logs
logs: export PREFECT_API_URL=http://localhost:4200/api
logs: assert-id
curl -H "Content-Type: application/json" -X POST --data '{"logs":{"flow_run_id":{"any_":["$(id)"]},"level":{"ge_":0}},"sort":"TIMESTAMP_ASC"}' -s "http://localhost:4200/api/logs/filter" | jq -r '.[] | [.timestamp,.level,.message] |@tsv'

## show flow runs
flow-runs: export PREFECT_API_URL=http://localhost:4200/api
flow-runs:
$(venv)/bin/prefect flow-run ls

Expand Down

0 comments on commit f324fc0

Please sign in to comment.