move python_cacher stuff into its own dir

ajslater · Jan 25, 2022 · 3d189c7 · 3d189c7
1 parent 5c8dc3d
commit 3d189c7
Show file tree

Hide file tree

Showing 7 changed files with 21 additions and 14 deletions.
diff --git a/builder-base.Dockerfile b/builder-base.Dockerfile
@@ -40,8 +40,8 @@ RUN vendor/shellcheck/install-shellcheck.sh
 # hadolint ignore=DL3022
 COPY $HOST_CACHE_DIR/pip /root/.cache/pip
 COPY $HOST_CACHE_DIR/pypoetry /root/.cache/pypoetry
-COPY ./cache_paths.py ./link_wheels_from_caches.py ./save_py_caches.py ./
-RUN ./link_wheels_from_caches.py
+COPY ./python_cacher ./python_cacher
+RUN ./python_cacher/link_wheels_from_caches.py
 # hadolint ignore=DL3042,DL3059
 RUN pip3 install --find-links=$WHEELS--upgrade pip
 # https://github.com/pyca/cryptography/issues/6673#issuecomment-985943023

diff --git a/builder-final.Dockerfile b/builder-final.Dockerfile
@@ -9,7 +9,7 @@ WORKDIR /app
 # Copy in caches and codex wheel
 COPY $HOST_CACHE_DIR/pip /root/.cache/pip
 COPY $HOST_CACHE_DIR/pypoetry /root/.cache/pypoetry
-RUN ./link_wheels_from_caches.py
+RUN ./python-cacher/link_wheels_from_caches.py
 
 # Install codex
 RUN pip3 install --no-cache-dir --upgrade --find-links=$WHEELS pip

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -39,7 +39,7 @@ services:
     container_name: codex-save-cache
     volumes:
       - ./cache/packages:/app/cache/packages
-    command: ./save_py_caches.py
+    command: ./python_cacher/save_py_caches.py prune
   codex-frontend-lint:
     <<: *codex-dist-builder
     container_name: codex-frontent-lint
@@ -92,7 +92,7 @@ services:
     container_name: codex-save-final-cache
     volumes:
       - ./cache/packages:/app/cache/packages
-    command: ./save_py_caches.py
+    command: ./python_cacher/save_py_caches.py
   codex:
     env_file:
       - .env

diff --git a/docker-version-codex-builder-base.sh b/docker-version-codex-builder-base.sh
@@ -4,21 +4,26 @@ set -euo pipefail
 CODEX_BASE_VERSION=$(./docker-version-codex-base.sh)
 # shellcheck disable=SC2046
 read -ra SHELLCHECK_DEPS <<<$(find vendor/shellcheck -type f \( ! -name "*~" \))
+# shellcheck disable=SC2046
+read -ra PYTHON_CACHER_DEPS <<<$(find python_cacher -type f \( \
+    ! -path "*__pycache__*" \
+    ! -name "*~" \
+    \))
 DEPS=(
     "$0"
     .dockerignore
     builder-base.Dockerfile
     builder-requirements.txt
     docker-build-codex-builder-base.sh
-    link_wheels_from_caches.py
-    save_py_caches.py
+    "${PYTHON_CACHER_DEPS[@]}"
     "${SHELLCHECK_DEPS[@]}"
 )
 DEPS_MD5S=$(md5sum "${DEPS[@]}")
-VERSION=$(echo -e "$CODEX_BASE_VERSION  codex-base-version\n$DEPS_MD5S" |
-    LC_ALL=C sort |
-    md5sum |
-    awk '{print $1}')
+md5sum "${DEPS[@]}"
+VERSION=$(echo -e "$CODEX_BASE_VERSION  codex-base-version\n$DEPS_MD5S" \
+    | LC_ALL=C sort \
+    | md5sum \
+    | awk '{print $1}')
 if [[ ${CIRCLECI:-} ]]; then
     ARCH=$(uname -m)
     VERSION="${VERSION}-$ARCH"

diff --git a/cache_paths.py → python_cacher/cache_paths.py b/cache_paths.py → python_cacher/cache_paths.py
diff --git a/link_wheels_from_caches.py → python_cacher/link_wheels_from_caches.py b/link_wheels_from_caches.py → python_cacher/link_wheels_from_caches.py
diff --git a/save_py_caches.py → python_cacher/save_py_caches.py b/save_py_caches.py → python_cacher/save_py_caches.py
@@ -2,6 +2,7 @@
 """Copy pip & poetry user caches to a volume saved location."""
 import os
 import subprocess
+import sys
 
 from pathlib import Path
 from shutil import copytree
@@ -84,15 +85,16 @@ def save_cache(native_cache_path, arch_cache_path):
     print(f"Copied {native_cache_path} to {dest}")
 
 
-def main():
+def main(args):
     """Prune the poetry cache and then copy poetry & pip cache."""
     arch = run(("uname", "-m"))[0]
     arch_cache_path = CACHE_PATH / arch
     arch_cache_path.mkdir(parents=True, exist_ok=True)
-    prune_poetry_artifacts()
+    if "prune" in args:
+        prune_poetry_artifacts()
     save_cache(POETRY_CACHE_PATH, arch_cache_path)
     save_cache(PIP_CACHE_PATH, arch_cache_path)
 
 
 if __name__ == "__main__":
-    main()
+    main(sys.argv[1:])