Use multi-stage build copying dependencies from separately built image

.github/workflows/continuous-integration.yml

@@ -5,14 +5,10 @@ on:
     branches: [main]
     paths-ignore:
       - 'docs/**'
-    tags:
-      - ci-test-*
   push:
     branches: [main]
     paths-ignore:
       - 'docs/**'
-    tags:
-      - ci-test-*
 
 jobs:
   build:
@@ -21,41 +17,15 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to container registry
-        uses: docker/login-action@v3
-        with:
-          registry: "hub.ncsa.illinois.edu"
-          username: "${{ secrets.NCSA_HARBOR_BLAST_USERNAME }}"
-          password: "${{ secrets.NCSA_HARBOR_BLAST_TOKEN }}"
-
-      - name: Build image
-        id: docker_build
-        uses: docker/build-push-action@v5
-        with:
-          builder: ${{ steps.buildx.outputs.name }}
-          context: ./app
-          file: ./app/Dockerfile
-          tags: hub.ncsa.illinois.edu/blast/blast:ci
-          push: true
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-      - name: Run tests
-        run: |
-          docker pull hub.ncsa.illinois.edu/blast/blast:ci
-          docker tag hub.ncsa.illinois.edu/blast/blast:ci blast:dev
-          touch env/.env.dev
-          docker compose \
-            --profile ci \
-            --project-name blast \
-            -f docker/docker-compose.yml \
-            --env-file env/.env.default \
-            --env-file env/.env.ci \
-            up --exit-code-from app_ci
+      # - name: Run tests
+      #   run: |
+      #     docker compose \
+      #       --project-name blast \
+      #       -f docker/docker-compose.yml \
+      #       --env-file env/.env.default \
+      #       --env-file env/.env.ci \
+      #       --profile ci \
+      #       up --build --exit-code-from app_ci
 
       - uses: codecov/codecov-action@v4
         with:

app/Dockerfile

@@ -1,7 +1,4 @@
-FROM python:3.11 as mc
-
-RUN curl https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc && \
-    chmod +x /usr/local/bin/mc
+FROM hub.ncsa.illinois.edu/blast/deps@sha256:0fe60eb8fee7c93664ee67e2b82fab048311f2bdf574011a402ca656f9e1408d as deps
 
 FROM python:3.11
 ENV PYTHONUNBUFFERED 1
@@ -15,17 +12,13 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive && apt-get install -y --no-
     libhdf5-serial-dev \
     netcdf-bin \
     libnetcdf-dev \
-    rsync \
     && rm -rf /var/lib/apt/list/*
 
-COPY ./requirements.txt /requirements.txt
-RUN pip install -r /requirements.txt --no-cache-dir
-
 COPY ./debug/debug_ipython.py /root/.ipython/profile_default/startup/
 
-RUN mkdir /app
+COPY --from=deps /usr/local/lib/python3.11/site-packages/ /usr/local/lib/python3.11/site-packages/
+COPY --from=deps /usr/local/bin/ /usr/local/bin/
 
+RUN mkdir /app
 COPY . /app
 WORKDIR /app
-
-COPY --from=mc /usr/local/bin/mc /usr/local/bin/mc

app/Dockerfile.deps

@@ -0,0 +1,8 @@
+FROM python:3.11
+
+RUN curl https://dl.min.io/client/mc/release/linux-amd64/mc \
+    -o /usr/local/bin/mc && \
+    chmod +x /usr/local/bin/mc
+
+COPY ./requirements.txt /requirements.txt
+RUN pip install -r /requirements.txt --no-cache-dir

app/entrypoints/blast-data.md5sums

@@ -355,12 +355,6 @@ e85a71766591cd6ca9ccf645425f0f26  fsps/ISOCHRONES/BaSTI/isoc_z0.0020.dat
 ff88aec10ed2a3ef28e9a62a8313d653  fsps/ISOCHRONES/BaSTI/isoc_z0.0100.dat
 2f854c7ccb31e27177053d4f42fe7a43  fsps/ISOCHRONES/BaSTI/isoc_z0.0200.dat
 e1cbb779c76bedbe0747c4e7b386ffa0  fsps/LICENSE
-3967b82a048c48e52467c0934a4cc9ad  sbi_training_sets/y_train_global.pkl
-ed48fe39dfe9c1b212f7998eca6e9688  sbi_training_sets/x_train_local.pkl
-1991a2faa8f9f64f1e47c2b8848f99c0  sbi_training_sets/y_train_local.pkl
-abe43b190eaf588fdc740b52f779e813  sbi_training_sets/hatp_x_y_local.pkl
-eb49b0607f062ed105f25bb1a778c6b9  sbi_training_sets/x_train_global.pkl
-cf3f07f1d7ba424ef02da7716f0729bc  sbi_training_sets/hatp_x_y_global.pkl
 60f7c43d51bccaff82961de3e9f9e789  dustmaps/sfd/SFD_dust_4096_sgp.fits
 6f3a89528bba7d1b7bd55838da86b639  dustmaps/sfd/SFD_dust_4096_ngp.fits
 2574cbe9e1d2d6af3080b0e1240fa63c  sed_output/2010H/2010H_global.h5

app/host/SBI/run_sbi.py

@@ -8,18 +8,21 @@
 from host.models import SEDFittingResult
 from host.prospector import build_model
 from host.prospector import build_obs
+
 # import sys
 # import warnings
 
 # recommend running the full script without the line below first
 # if an error is threw, then uncomment this line
 os.environ["KMP_DUPLICATE_LIB_OK"] = "True"
 import numpy as np
+
 # from numpy.random import normal, uniform
 from scipy.interpolate import interp1d
 
 # torch
 import torch
+
 # import torch.nn as nn
 # import torch.nn.functional as F
 from sbi import utils as Ut
@@ -169,10 +172,10 @@ def main():
 
         obs = {}
         obs["bands"] = filternames
-        obs["mags"] = mags  ##np.array([maggies_to_asinh(p) for p in pobs['maggies']])
-        obs["mags_unc"] = mags_unc  ##2.5/np.log(10)*pobs['maggies_unc']/pobs['maggies']
+        obs["mags"] = mags  # np.array([maggies_to_asinh(p) for p in pobs['maggies']])
+        obs["mags_unc"] = mags_unc  # 2.5/np.log(10)*pobs['maggies_unc']/pobs['maggies']
         obs["redshift"] = pobs["redshift"]
-        ###obs["mags"][np.where(filternames == 'WISE_W3')[0][0]] = np.nan
+        # obs["mags"][np.where(filternames == 'WISE_W3')[0][0]] = np.nan
         # Run SBI++
         # signal.alarm(0)
         # signal.alarm(600)
@@ -204,4 +207,4 @@ def main():
                     file=fout,
                 )
             except Exception as err:
-                print(transient_name, np.mean(chain[:, 1]), mfrac, None, file=fout, err)
+                print(transient_name, np.mean(chain[:, 1]), mfrac, None, err, file=fout)

app/host/ghost.py

@@ -56,8 +56,8 @@ def run_ghost(transient, output_dir=settings.GHOST_OUTPUT_ROOT):
                 dust_path=settings.GHOST_DUST_PATH,
                 model_path=settings.GHOST_PHOTOZ_PATH,
             )
-        except Exception as e:
-            print("warning : photo-z step failed")
+        except Exception as err:
+            print(f"warning : photo-z step failed: {err}")
 
     # clean up after GHOST...
     # dir_list = glob.glob('transients_*/*/*')

env/.env.default

@@ -56,7 +56,7 @@ DATA_ARCHIVE_FILE = ""
 DATA_ARCHIVE_FILE_URL = "https://js2.jetstream-cloud.org:8001/swift/v1/blast-astro-data/blast-data.v1.tar.gz"
 USE_LOCAL_ARCHIVE_FILE = false
 # Set FORCE_INITIALIZATION to remove any stale lock files preventing the initialization from proceeding
-FORCE_INITIALIZATION = false
+FORCE_INITIALIZATION = true
 
 DUSTMAPS_DATA_ROOT = "/data/dustmaps"
 CUTOUT_ROOT = "/data/cutout_cdn"

run/blast.run.sh

@@ -1,4 +1,5 @@
 #!/bin/env bash
+set -e
 
 cd "$(dirname "$(readlink -f "$0")")"/..
 
@@ -7,35 +8,12 @@ cd "$(dirname "$(readlink -f "$0")")"/..
 # fail if the file permissions are not globally readable.
 # chmod -R a+rX app/ 2>/dev/null
 
-# Clear any initialization check files
-docker run --rm -it -v blast_blast-data:/mnt/data blast:dev rm -f /mnt/data/.initializing_db /mnt/data/.initializing_data
-
 if [[ ! -f "env/.env.dev" ]]; then
   touch env/.env.dev
 fi
 
-ENV_FILE="env/.env.dev"
-DOCKER_ARGS=""
-case "$1" in
-  test)
-    DOCKER_ARGS="--exit-code-from app_test"
-    ;;
-  ci)
-    DOCKER_ARGS="--exit-code-from app_ci"
-    ENV_FILE="env/.env.ci"
-    ;;
-  slim_dev)
-    DOCKER_ARGS="--abort-on-container-exit"
-    ;;
-  *)
-    DOCKER_ARGS=""
-    ;;
-esac
+PROFILE=$1
+source run/get_compose_args.sh $PROFILE
 
 set -x
-docker compose \
-  --profile $1 \
-  --project-name blast \
-  -f docker/docker-compose.yml \
-  --env-file env/.env.default --env-file "${ENV_FILE}" \
-  up --build ${DOCKER_ARGS}
+docker compose ${COMPOSE_CONFIG} up --build ${COMPOSE_ARGS}

run/blast.stop.sh

@@ -3,38 +3,21 @@ set -e
 
 cd "$(dirname "$(readlink -f "$0")")"/..
 
+PROFILE=$1
 PURGE_OPTION=$2
 
-DOCKER_ARGS=""
-VOLUMES=""
-case "${PURGE_OPTION}" in
-  "--purge-all")
-    DOCKER_ARGS="--volumes"
-    echo "Purging all data volumes..."
-  ;;
-  "--purge-db")
-    VOLUMES="blast_blast-db blast_django-static"
-    echo "Purging Django database and static file volumes..."
-  ;;
-  "--purge-data")
-    VOLUMES="blast_blast-data"
-    echo "Purging astro data volume..."
-  ;;
-esac
+source run/get_compose_args.sh $PROFILE $PURGE_OPTION
 
 if [[ ! -f "env/.env.dev" ]]; then
   touch env/.env.dev
 fi
 
 set -x
-docker compose \
-  --profile $1 \
-  --project-name blast \
-  -f docker/docker-compose.yml \
-  --env-file env/.env.default --env-file env/.env.dev \
-  down ${DOCKER_ARGS}
+docker compose ${COMPOSE_CONFIG} down ${COMPOSE_ARGS}
 set +x
 
-if [[ "${VOLUMES}x" != "x" ]]; then
-  docker volume rm ${VOLUMES}
+if [[ "${PURGE_VOLUMES}x" != "x" ]]; then
+  set -x
+  docker volume rm ${PURGE_VOLUMES}
+  set +x
 fi

run/get_compose_args.sh

@@ -0,0 +1,61 @@
+#!/bin/env bash
+set -e
+
+PROFILE=$1
+PURGE_OPTION=$2
+
+ENV_FILE="env/.env.dev"
+COMPOSE_ARGS=""
+case "$PROFILE" in
+  "")
+    echo "ERROR: You must specify a profile (e.g. $0 slim_dev)"
+    exit 1
+    ;;
+  test)
+    COMPOSE_ARGS="--exit-code-from app_test"
+    ;;
+  ci)
+    COMPOSE_ARGS="--exit-code-from app_ci"
+    ENV_FILE="env/.env.ci"
+    ;;
+  slim_dev)
+    COMPOSE_ARGS="--abort-on-container-exit"
+    ;;
+  *)
+    COMPOSE_ARGS=""
+    ;;
+esac
+
+COMPOSE_ARGS=""
+PURGE_VOLUMES=""
+case "${PURGE_OPTION}" in
+  "")
+    echo "Retaining all data volumes."
+    ;;
+  "--purge-all")
+    COMPOSE_ARGS="--volumes"
+    echo "Purging all data volumes..."
+  ;;
+  "--purge-db")
+    PURGE_VOLUMES="blast_blast-db blast_django-static"
+    echo "Purging Django database and static file volumes..."
+  ;;
+  "--purge-data")
+    PURGE_VOLUMES="blast_blast-data"
+    echo "Purging astro data volume..."
+  ;;
+  *)
+    echo "ERROR: Invalid purge option."
+    exit 1
+    ;;
+esac
+
+COMPOSE_CONFIG=" --profile $PROFILE"
+COMPOSE_CONFIG="${COMPOSE_CONFIG} --project-name blast"
+COMPOSE_CONFIG="${COMPOSE_CONFIG} -f docker/docker-compose.yml"
+COMPOSE_CONFIG="${COMPOSE_CONFIG} --env-file env/.env.default"
+COMPOSE_CONFIG="${COMPOSE_CONFIG} --env-file ${ENV_FILE}"
+
+export COMPOSE_CONFIG
+export COMPOSE_ARGS
+export PURGE_VOLUMES