more docker and makefile reorg

.github/workflows/_cd-upload-wheel-to-pypi.yml

@@ -43,7 +43,8 @@ jobs:
 
       - name: Check that wheel can be installed and imported
         run: |
-          make run-command[in-cd-docker] \
+          cd docker
+          make in-cd-docker/run-command \
             COMMAND="(\
               pip install --quiet --user ./pynb_dag_runner/dist/*.whl; \
               python3 -c 'import pynb_dag_runner' \
@@ -88,7 +89,8 @@ jobs:
 
       - name: Uploading package to pypi
         run: |
-          make run-command[in-cd-docker] \
+          cd docker
+          make in-cd-docker/run-command \
             DOCKER_ARGS=" \
               -e TWINE_USERNAME \
               -e TWINE_PASSWORD \

.github/workflows/_ci-run-tests-and-build-python-package.yml

@@ -25,24 +25,27 @@ jobs:
         with:
           persist-credentials: false
 
-      - name: Build docker image
+      - name: Build docker images
         run: |
-          make docker-build-all
+          make build-docker-images
 
       - name: Run unit tests
         run: |
-          make run-command[in-ci-docker] \
+          cd docker
+          make in-ci-docker/run-command \
               RUN_ENVIRONMENT="${{ env.RUN_ENVIRONMENT }}" \
               COMMAND="(cd pynb_dag_runner; make test-pytest )"
 
       - name: Run mypy type checks
         run: |
-          make run-command[in-ci-docker] \
+          cd docker
+          make in-ci-docker/run-command \
               COMMAND="(cd pynb_dag_runner; make test-mypy )" \
 
       - name: Check black formatting
         run: |
-          make run-command[in-ci-docker] \
+          cd docker
+          make in-ci-docker/run-command \
               COMMAND="(cd pynb_dag_runner; make test-black )"
 
       - name: Determining unix epoch time stamp of commit (to version snapshot releases)
@@ -57,7 +60,7 @@ jobs:
 
       - name: Build Python package for the pynb-dag-runner library
         run: |
-          make build[in-ci-docker]
+          make in-ci-docker/build
 
         env:
           PYTHON_PACKAGE_RELEASE_TARGET: ${{ inputs.python-package-release-target }}
@@ -68,4 +71,4 @@ jobs:
           name: pynb-dag-runner-wheel
           path: ${{ github.workspace }}/workspace/pynb_dag_runner/dist/
           if-no-files-found: error
-          retention-days: 5
+          retention-days: 1

.github/workflows/security-scan.yml

@@ -37,7 +37,7 @@ jobs:
         shell: bash
         working-directory: ${{ github.workspace }}
         run: |
-          make docker-build-all
+          make build-docker-images
 
       # ------
       # The below following instructions from here:

docker/.gitignore

@@ -0,0 +1 @@
+.built-docker-images

docker/makefile

@@ -1,31 +1,85 @@
 SHELL := /bin/bash
 
-### Tasks to build Docker images
+DOCKER_IMAGE_NAME       := pynb-dag-runner
+BUILT_DOCKER_LOCK_FILE  := ./.built-docker-images/${DOCKER_IMAGE_NAME}
 
-build-cd-env-docker-image:
-	docker \
-	    build \
+# --- Helper tasks ---
+
+env_%:
+	@# Check that a variable is defined, see stackoverflow.com/a/7367903
+	@if [[ -z "${$*}" ]]; then exit 1; fi
+
+
+# --- Tasks to build Docker images ---
+
+${BUILT_DOCKER_LOCK_FILE}-base: Dockerfile.base makefile
+	docker build \
+	    --no-cache \
 	    --file Dockerfile.base \
-	    --build-arg HOST_UID=$$(id -u) \
-	    --build-arg HOST_GID="$$(id -g)" \
-	    --tag pynb-dag-runner-base \
+	    --build-arg HOST_UID=$(shell id -u) \
+	    --build-arg HOST_GID="$(shell id -g)" \
+	    --tag ${DOCKER_IMAGE_NAME}-base \
 	    .
+	mkdir -p $(shell dirname $@)
+	touch $@
 
-build-ci-env-docker-image:
-	docker \
-	    build \
+${BUILT_DOCKER_LOCK_FILE}-cicd: ${BUILT_DOCKER_LOCK_FILE}-base Dockerfile.cicd makefile
+	docker build \
+	    --no-cache \
 	    --file Dockerfile.cicd \
-	    --tag pynb-dag-runner-cicd \
+	    --tag ${DOCKER_IMAGE_NAME}-cicd \
 	    .
+	mkdir -p $(shell dirname $@)
+	touch $@
 
 JUPYTER_TOKEN:
 	# Create random JUPYTER_TOKEN for Jupyter running in Docker
 	openssl rand -base64 42 > JUPYTER_TOKEN
 
-build-dev-env-docker-image: JUPYTER_TOKEN
-	docker \
-	    build \
+${BUILT_DOCKER_LOCK_FILE}-dev: JUPYTER_TOKEN ${BUILT_DOCKER_LOCK_FILE}-cicd makefile
+	docker build \
+	    --no-cache \
 	    --file Dockerfile.dev \
-	    --build-arg JUPYTER_TOKEN=$$(cat JUPYTER_TOKEN) \
-	    --tag pynb-dag-runner-dev \
+	    --build-arg JUPYTER_TOKEN=$(shell cat JUPYTER_TOKEN) \
+	    --tag ${DOCKER_IMAGE_NAME}-dev \
 	    .
+	mkdir -p $(shell dirname $@)
+	touch $@
+
+build-docker-images: ${BUILT_DOCKER_LOCK_FILE}-dev
+
+
+# --- run command in a docker container ---
+
+run-in-docker: | env_COMMAND env_DOCKER_IMG
+	@# Run bash command(s) in Docker image DOCKER_IMG (=base, cicd or dev)
+	docker run --rm --tty \
+	    ${DOCKER_ARGS} \
+	    --volume $(shell pwd)/../workspace:/home/host_user/workspace \
+	    --workdir /home/host_user/workspace/ \
+	    pynb-dag-runner-${DOCKER_IMG} \
+	    "${COMMAND}"
+
+in-cd-docker/run-command: | env_COMMAND
+	@# This is used to deploy the Python package
+	${MAKE} run-in-docker \
+	    DOCKER_ARGS="${DOCKER_ARGS}" \
+	    COMMAND="${COMMAND}" \
+		DOCKER_IMG="base"
+
+in-dev-docker/run-command: | env_COMMAND
+	@# Note:
+	@#  - dev jobs run without network access
+	@#  - additional DOCKER_ARGS optional
+	${MAKE} run-in-docker \
+	    DOCKER_ARGS="--network none ${DOCKER_ARGS}" \
+	    COMMAND="${COMMAND}" \
+	    DOCKER_IMG="dev"
+
+in-ci-docker/run-command: | env_COMMAND
+	@# Note: ci jobs run without network
+	@# DOCKER_ARGS optional
+	${MAKE} run-in-docker \
+	    DOCKER_ARGS="--network none ${DOCKER_ARGS}" \
+	    COMMAND="${COMMAND}" \
+	    DOCKER_IMG="cicd"

docs/docs/dev/local-dev.md

@@ -2,7 +2,7 @@
 
 ### In VS Code
 - Open the root of the `pynb-dag-runner` repository in VS Code.
-- In the repo root run `make docker-build-all`.
+- In the repo root run `make build-docker-images`.
 - Ensure that the extension "Remote - Containers" (`ms-vscode-remote.remote-containers` published by Microsoft) is installed.
 - Press the `><`-button in the lower left VS Code window corner. Select "Reopen in container".
 - Inside container ensure that the "Python" extension is installed (`ms-python.python` also published by Microsoft) if it is not automatically installed. When installed and enabled, the lower row will show the Python version in use inside the container.
@@ -12,7 +12,29 @@ For more details, see VS Code remote development, [docs<sup><sup><sub>:material-
 
 ### Run tests and build Python wheel file
 
+Common development tasks are found in the root `makefile`.
+
 ```bash
-make docker-build-all
-make [test|build|clean]
+# --- setup ---
+# this will build three Docker images. For cd, ci and local-development.
+make build-docker-images
+
+# --- testing ---
+# run all unit tests in watch mode
+make in-dev-docker/watch-pytest
+
+# run selected unit tests in watch mode.
+make in-dev-docker/watch-pytest PYTEST_FILTER="version"
+
+# Start tmux session to watch
+#  - black formatting
+#  - static type cheks
+#  - and unit tests [optionally filtered as above]
+make in-dev-docker/tmux-watch-all-tests [PYTEST_FILTER="version"]
+
+# --- build package locally ---
+# TODO, see the in-ci-docker/build task in the root makefile
+
+# --- clean up ---
+make clean
 ```

docs/docs/live-demos/mnist-digits-demo-pipeline/index.md

@@ -129,7 +129,7 @@ Internet --> web_static_mlflow
 git clone --recurse-submodules git@github.com:pynb-dag-runner/mnist-digits-demo-pipeline.git
 
 cd mnist-digits-demo-pipeline
-make docker-build-all
+make build-docker-images
 
 make clean
 

docs/makefile

@@ -6,8 +6,8 @@ DOCKER_IMAGE_NAME  := pynb-dag-runner-docs
 docker-build:
 	docker build \
 	    --file Dockerfile \
-	    --build-arg HOST_UID=$$(id -u) \
-	    --build-arg HOST_GID="$$(id -g)" \
+	    --build-arg HOST_UID=$(shell id -u) \
+	    --build-arg HOST_GID="$(shell id -g)" \
 	    --tag ${DOCKER_IMAGE_NAME} \
 	    .
 
@@ -16,7 +16,7 @@ docker-mkdocs-serve:
 	    --tty \
 	    --rm \
 	    -p 4200:4200 \
-	    -v $$(pwd):/home/host_user/docs:ro \
+	    -v $(shell pwd):/home/host_user/docs:ro \
 	    --workdir /home/host_user/docs \
 	    ${DOCKER_IMAGE_NAME} \
 	    "mkdocs serve --dev-addr 0.0.0.0:4200"
@@ -25,7 +25,7 @@ docker-mkdocs-build:
 	docker run \
 	    --tty \
 	    --rm \
-	    -v $$(pwd):/home/host_user/docs \
+	    -v $(shell pwd):/home/host_user/docs \
 	    --workdir /home/host_user/docs \
 	    ${DOCKER_IMAGE_NAME} \
 	    "mkdocs build \

makefile

@@ -7,97 +7,51 @@ env_%:
 	@# Check that a variable is defined, see stackoverflow.com/a/7367903
 	@if [[ -z "$($*)" ]]; then exit 1; fi
 
-# --- docker related tasks ---
-
-docker-build-all:
-	(cd docker; make \
-	    build-cd-env-docker-image \
-	    build-ci-env-docker-image \
-	    build-dev-env-docker-image)
-
-# "dev-{up, down} tasks allow us to manually start/stop the dev-docker container (can be
-# used without VS Code)
-
-dev-up:
-	docker-compose \
-	    -f .devcontainer-docker-compose.yml \
-	    up \
-	    --remove-orphans \
-	    --abort-on-container-exit \
-	    dev-environment
 
-dev-down:
-	docker-compose \
-	    -f .devcontainer-docker-compose.yml \
-	    down \
-	    --remove-orphans
-
-# --- recipes to run commands inside Docker images ---
-
-run-in-docker: | env_COMMAND env_DOCKER_IMG
-	@# Run bash command(s) in Docker image DOCKER_IMG (=cicd or dev)
-	docker run --rm --tty \
-	    $(DOCKER_ARGS) \
-	    --volume $$(pwd)/workspace:/home/host_user/workspace \
-	    --workdir /home/host_user/workspace/ \
-	    pynb-dag-runner-$(DOCKER_IMG) \
-	    "$(COMMAND)"
-
-run-command[in-cd-docker]: | env_COMMAND
-	make run-in-docker \
-	    DOCKER_ARGS="$(DOCKER_ARGS)" \
-	    COMMAND="$(COMMAND)" \
-		DOCKER_IMG="base"
+# --- docker related tasks ---
 
-run-command[in-ci-docker]: | env_COMMAND
-	@# Note: ci jobs run without network
-	@# DOCKER_ARGS optional
-	make run-in-docker \
-	    DOCKER_ARGS="--network none $(DOCKER_ARGS)" \
-	    COMMAND="$(COMMAND)" \
-	    DOCKER_IMG="cicd"
+build-docker-images:
+	@(cd docker; ${MAKE} build-docker-images)
 
-run-command[in-dev-docker]: | env_COMMAND
-	@# Note: ci jobs run without network
-	@# DOCKER_ARGS optional
-	make run-in-docker \
-	    DOCKER_ARGS="--network none $(DOCKER_ARGS)" \
-	    COMMAND="$(COMMAND)" \
-	    DOCKER_IMG="dev"
 
-# --- define dockerized recipes for testing and building pynb-dag-runner package---
+# --- define main dockerized tasks for testing and building pynb-dag-runner Python package---
 
-clean[in-ci-docker]:
-	make COMMAND="(cd pynb_dag_runner; make clean)" run-command[in-ci-docker]
+clean:
+	@(cd workspace/pynb_dag_runner; ${MAKE} clean)
 
-build[in-ci-docker]: | env_GITHUB_SHA env_PYTHON_PACKAGE_RELEASE_TARGET env_LAST_COMMIT_UNIX_EPOCH
-	make run-command[in-ci-docker] \
+in-ci-docker/build: | env_GITHUB_SHA env_PYTHON_PACKAGE_RELEASE_TARGET env_LAST_COMMIT_UNIX_EPOCH
+	# TODO: allow local run
+	cd docker; \
+	${MAKE} in-ci-docker/run-command \
 	    DOCKER_ARGS=" \
 	        -e GITHUB_SHA \
 	        -e PYTHON_PACKAGE_RELEASE_TARGET \
 	        -e LAST_COMMIT_UNIX_EPOCH \
-	        --volume $$(pwd):/repo-root:ro \
+	        --volume $(shell pwd):/repo-root:ro \
 	    " \
 	    COMMAND="( \
 	        cd pynb_dag_runner; \
 	        make build \
 	            README_FILEPATH=/repo-root/README.md; \
 		)"
 
-watch-pytest[in-dev-docker]:
-	@# run pytest in watch mode with ability to filter out specific test(s)
-	make run-command[in-dev-docker] \
+in-dev-docker/watch-pytest:
+	@# run pytest in watch mode. Filter test names with PYTEST_FILTER argument
+	cd docker; \
+	${MAKE} in-dev-docker/run-command \
 	    COMMAND="( \
 	        cd pynb_dag_runner; \
 	        make watch-test-pytest \
-	            PYTEST_FILTER=\"$(PYTEST_FILTER)\" \
+	            PYTEST_FILTER=\"${PYTEST_FILTER}\" \
 	    )"
 
-tmux-watch-all-tests[in-dev-docker]:
-	make run-command[in-dev-docker] \
+in-dev-docker/tmux-watch-all-tests:
+	@# run all tests in tmux/watch mode. Filter unit tests with PYTEST_FILTER argument
+	cd docker; \
+	${MAKE} in-dev-docker/run-command \
 	    DOCKER_ARGS="-i" \
 	    COMMAND="( \
 	        cd pynb_dag_runner; \
 	        make tmux-watch-all-tests \
-	            PYTEST_FILTER=\"$(PYTEST_FILTER)\" \
+	            PYTEST_FILTER=\"${PYTEST_FILTER}\" \
 	    )"

workspace/pynb_dag_runner/.tmux.conf

@@ -4,6 +4,6 @@
 # https://github.com/microsoft/vscode/issues/96058
 set -g mouse on
 
-# Ctrl-B + X + y (to confirm) -> close all panes and exit tmux session
+# Ctrl-B + x -> close all panes and exit tmux session
 # https://maxschmitt.me/posts/tmux-close-all-panes-at-once/
-bind X confirm-before kill-session
+bind x kill-session