From 801fdfddbae6924f69d41c4d8985376e21dede9e Mon Sep 17 00:00:00 2001 From: Roger Hunwicks Date: Sat, 4 Oct 2025 17:06:40 -0500 Subject: [PATCH 1/7] Remove amd64 platform requirement - see HEA-760 --- docker/app/Dockerfile | 2 +- docker/db/Dockerfile | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index cf47a02..61a383b 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=linux/amd64 python:3.12-bookworm as base +FROM python:3.12-bookworm as base # set up apt repositories for postgres installation RUN curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/pgdg.gpg >/dev/null && \ diff --git a/docker/db/Dockerfile b/docker/db/Dockerfile index dbccb48..ea0a4f0 100644 --- a/docker/db/Dockerfile +++ b/docker/db/Dockerfile @@ -1,3 +1,5 @@ -FROM --platform=linux/amd64 postgis/postgis:17-3.5 +# Use a third party multicarch base image for compatibility with both ARM and AMD architectures +# until PostGIS fix https://github.com/postgis/docker-postgis/issues/216 +FROM ghcr.io/baosystems/postgis:17-3.5 COPY create_db.sh /docker-entrypoint-initdb.d/create_db.sh From c32d2df39b0201ed1574e3b9adaf85b7b9e6fbcb Mon Sep 17 00:00:00 2001 From: Roger Hunwicks Date: Sat, 4 Oct 2025 17:07:20 -0500 Subject: [PATCH 2/7] Fix create_db.sh - see HEA-760 --- docker/db/create_db.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/db/create_db.sh b/docker/db/create_db.sh index fd78e59..08ac769 100755 --- a/docker/db/create_db.sh +++ b/docker/db/create_db.sh @@ -1,5 +1,5 @@ #!/bin/sh -psql --set=CLIENT=$CLIENT --set=APP=$APP --set=ENV=$ENV --set=POSTGRES_PASSWORD=$POSTGRES_PASSWORD --set=DAGSTER_PASSWORD=$DAGSTER_PASSWORD --set=CREATE_TEMPLATE=${CREATE_TEMPLATE:-false} -d postgres --echo-all << EOF +psql --set=CLIENT=$CLIENT --set=APP=$APP --set=ENV=$ENV --set=POSTGRES_PASSWORD=$POSTGRES_PASSWORD --set=CREATE_TEMPLATE=${CREATE_TEMPLATE:-false} -d postgres --echo-all << EOF \set DATABASE :CLIENT :APP :ENV \set OWNER :CLIENT :APP :ENV @@ -74,7 +74,7 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA :SCHEMA GRANT SELECT ON TABLES TO :OWNER; \set DAGSTER :CLIENT :APP :ENV -CREATE ROLE :DAGSTER PASSWORD :'DAGSTER_PASSWORD' NOLOGIN CREATEDB NOCREATEROLE NOSUPERUSER; +CREATE ROLE :DAGSTER PASSWORD :'POSTGRES_PASSWORD' NOLOGIN CREATEDB NOCREATEROLE NOSUPERUSER; COMMENT ON ROLE :DAGSTER IS 'Main Dagster pipeline user for :CLIENT :PRJ :ENV'; GRANT :DAGSTER TO :OWNER; GRANT CONNECT, TEMPORARY, CREATE ON DATABASE :DATABASE TO :DAGSTER; From a39debd782d7bb2d070bde94c47456f570db52d0 Mon Sep 17 00:00:00 2001 From: Roger Hunwicks Date: Sat, 4 Oct 2025 17:09:56 -0500 Subject: [PATCH 3/7] Remove Pyrseas - see HEA-370 We don't use Pyrseas at all these days - it is a legacy from before Django had full-featured SQL migrations. --- requirements/test.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements/test.txt b/requirements/test.txt index 692ce0a..5a10e48 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,6 +1,4 @@ -r base.txt beautifulsoup4==4.12.2 coverage==7.2.7 -# Pyrseas==0.10.0 raises "KeyError: ('public', 'spatial_ref_sys')", --schema/--exclude-schema don't fix it. -Pyrseas==0.9.1 safety==3.6.1 From 56c64eb7388c32dc7d8ad2be1dfa080d8105e80b Mon Sep 17 00:00:00 2001 From: Roger Hunwicks Date: Sat, 4 Oct 2025 17:11:56 -0500 Subject: [PATCH 4/7] Remove redundant environment variables - see HEA-370 --- docker-compose.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e75dd65..bc76f3e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -110,14 +110,7 @@ services: MINIO_ENDPOINT_URL: http://minio:9000 SUPPORT_EMAIL_ADDRESS: ${SUPPORT_EMAIL_ADDRESS} DJANGO_MIGRATE: 1 - KILUIGI_INTERMEDIATETARGET_BACKEND_CLASS: ${KILUIGI_INTERMEDIATETARGET_BACKEND_CLASS} - KILUIGI_INTERMEDIATETARGET_ROOT_PATH: ${KILUIGI_INTERMEDIATETARGET_ROOT_PATH} - KILUIGI_FINALTARGET_BACKEND_CLASS: ${KILUIGI_FINALTARGET_BACKEND_CLASS} - KILUIGI_FINALTARGET_ROOT_PATH: ${KILUIGI_FINALTARGET_ROOT_PATH} - KILUIGI_REPORTTARGET_BACKEND_CLASS: ${KILUIGI_REPORTTARGET_BACKEND_CLASS} - KILUIGI_REPORTTARGET_ROOT_PATH: ${KILUIGI_REPORTTARGET_ROOT_PATH} GOOGLE_APPLICATION_CREDENTIALS: ${GOOGLE_APPLICATION_CREDENTIALS} - GOOGLE_ADMIN_EMAIL: ${GOOGLE_ADMIN_EMAIL} command: - --timeout=3600 - --workers=12 From c9da4e5e885d570bee15da631b4479bf8712affc Mon Sep 17 00:00:00 2001 From: Roger Hunwicks Date: Sat, 4 Oct 2025 17:50:13 -0500 Subject: [PATCH 5/7] Enable remote debugging using VSCode - see HEA-760 This changes allows setting LAUNCHER in .env and then using VSCode to attach to the Python process running inside the Docker container(s) --- README.md | 50 +++++++++++++++++++++++++++++ docker-compose.override.yml | 13 ++++++++ docker/app/run_dagster_daemon.sh | 7 ++-- docker/app/run_dagster_webserver.sh | 7 ++-- docker/app/run_django.sh | 5 ++- env.example | 5 +++ requirements/local.txt | 1 + 7 files changed, 83 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 7d78bda..2f29e30 100644 --- a/README.md +++ b/README.md @@ -23,3 +23,53 @@ baseline This produces a .puml file that can be rendered using a PlantUML server, either within your IDE or using a service like http://www.plantuml.com/. + +## Debugging inside Docker Containers + +The `LAUNCHER` environment sets a wrapper program around the Python process +(`gunicorn`, `dagster-daemon`, `dagster-webserver`). This can be used to +enable a debugger inside Docker Containers: + +1. Set `LAUNCHER="python3 -m debugpy --listen 0.0.0.0:5678"` in `.env` +2. Create Launch Configurations in Visual Studio Code like: + +```json + { + "name": "Python: Attach to app (Docker Container)", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder:hea}", + "remoteRoot": "/usr/src/app" + } + ], + "django": true, + "justMyCode": false + }, + { + "name": "Python: Attach to dagster-daemon (Docker Container)", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5680 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder:hea}", + "remoteRoot": "/usr/src/app" + } + ], + "django": true, + "justMyCode": false + } +``` + +3. Start the Docker containers as normal, and then use the Run and Debug +pane in Visual Studio code to launch the configuration that attaches to +the desired server. \ No newline at end of file diff --git a/docker-compose.override.yml b/docker-compose.override.yml index a1ffcc6..dd8e618 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -15,6 +15,7 @@ services: build: target: dev ports: + - "5678:5678" - "8000:8000" - "8888:8888" volumes: @@ -30,6 +31,9 @@ services: - ./env.example:/usr/src/app/.env environment: DJANGO_SETTINGS_MODULE: hea.settings.local + LAUNCHER: ${LAUNCHER} # e.g. "debugpy" or "ddtrace" + # Disable frozen modules warning + PYDEVD_DISABLE_FILE_VALIDATION: 1 # Put .coverage in a writable directory COVERAGE_FILE: log/.coverage command: @@ -41,6 +45,7 @@ services: restart: no ports: - "3000:3000" + - "5679:5678" volumes: - ./:/usr/src/app # Separate volumes for writable directories inside the container @@ -54,9 +59,14 @@ services: - ./env.example:/usr/src/app/.env environment: DJANGO_SETTINGS_MODULE: hea.settings.local + LAUNCHER: ${LAUNCHER} # e.g. "debugpy" or "ddtrace" + # Disable frozen modules warning + PYDEVD_DISABLE_FILE_VALIDATION: 1 dagster_daemon: restart: no + ports: + - "5680:5678" volumes: - ./:/usr/src/app # Separate volumes for writable directories inside the container @@ -70,4 +80,7 @@ services: - ./env.example:/usr/src/app/.env environment: DJANGO_SETTINGS_MODULE: hea.settings.local + LAUNCHER: ${LAUNCHER} # e.g. "debugpy" or "ddtrace" + # Disable frozen modules warning + PYDEVD_DISABLE_FILE_VALIDATION: 1 diff --git a/docker/app/run_dagster_daemon.sh b/docker/app/run_dagster_daemon.sh index 8beac63..87c46b3 100755 --- a/docker/app/run_dagster_daemon.sh +++ b/docker/app/run_dagster_daemon.sh @@ -13,5 +13,8 @@ echo Setting up logs touch log/django.log chown -R django:django log/* -echo Starting Dagster -gosu django dagster-daemon run $* \ No newline at end of file +echo Starting Dagster Daemon +if [ x"$LAUNCHER" != x"" ]; then + echo using ${LAUNCHER} +fi +gosu django ${LAUNCHER} /usr/local/bin/dagster-daemon run $* \ No newline at end of file diff --git a/docker/app/run_dagster_webserver.sh b/docker/app/run_dagster_webserver.sh index e6b55a1..5e27a35 100755 --- a/docker/app/run_dagster_webserver.sh +++ b/docker/app/run_dagster_webserver.sh @@ -13,5 +13,8 @@ echo Setting up logs touch log/django.log chown -R django:django log/* -echo Starting Dagster -gosu django dagster-webserver -h 0.0.0.0 -p 3000 -m pipelines --path-prefix /${DAGSTER_WEBSERVER_PREFIX} $* \ No newline at end of file +echo Starting Dagster Webserver +if [ x"$LAUNCHER" != x"" ]; then + echo using ${LAUNCHER} +fi +gosu django ${LAUNCHER} /usr/local/bin/dagster-webserver -h 0.0.0.0 -p 3000 -m pipelines --path-prefix /${DAGSTER_WEBSERVER_PREFIX} $* \ No newline at end of file diff --git a/docker/app/run_django.sh b/docker/app/run_django.sh index 73112fa..e6f2b48 100755 --- a/docker/app/run_django.sh +++ b/docker/app/run_django.sh @@ -40,7 +40,10 @@ touch log/django_sql.log chown -R django:django log/* echo Starting Gunicorn with DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE} -gosu django gunicorn ${APP}.wsgi:application \ +if [ x"$LAUNCHER" != x"" ]; then + echo using ${LAUNCHER} +fi +gosu django ${LAUNCHER} /usr/local/bin/gunicorn ${APP}.wsgi:application \ --name ${APP}${ENV} \ --config $(dirname $(readlink -f "$0"))/gunicorn_config.py \ $* 2>&1 diff --git a/env.example b/env.example index 680e231..3c4775f 100644 --- a/env.example +++ b/env.example @@ -48,3 +48,8 @@ BSS_METADATA_WORKBOOK='gdrive://Database Design/BSS Metadata' # 15XVXFjbom1sScV BSS_METADATA_STORAGE_OPTIONS='{"token": "service_account", "access": "read_only", "creds": ${GOOGLE_APPLICATION_CREDENTIALS}, "root_file_id": "0AOJ0gJ8sjnO7Uk9PVA"}' BSS_FILES_FOLDER='gdrive://Discovery Folder/Baseline Storage Sheets (BSS)' BSS_FILES_STORAGE_OPTIONS='{"token": "service_account", "access": "read_only", "creds": ${GOOGLE_APPLICATION_CREDENTIALS}, "root_file_id": "0AOJ0gJ8sjnO7Uk9PVA"}' + +# LAUNCHER can be used to configure a wrapper program around the Python process +# For example, to add ddtrace or debugpy +# Use the VSCode debugger as a launcher +# LAUNCHER = "python3 -m debugpy --listen 0.0.0.0:5679" \ No newline at end of file diff --git a/requirements/local.txt b/requirements/local.txt index 4e95b36..8cd2465 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -1,2 +1,3 @@ -r test.txt -r lint.txt +debugpy \ No newline at end of file From 584aca080962f1b3eb71138e58b6cf516b19da6f Mon Sep 17 00:00:00 2001 From: Roger Hunwicks Date: Sun, 12 Oct 2025 11:54:17 -0400 Subject: [PATCH 6/7] Remove dbtoyaml calls from 01-build-then-test - see HEA-760 --- .github/workflows/01-build-then-test.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/01-build-then-test.yml b/.github/workflows/01-build-then-test.yml index a517324..9efe314 100644 --- a/.github/workflows/01-build-then-test.yml +++ b/.github/workflows/01-build-then-test.yml @@ -271,9 +271,6 @@ jobs: docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/log ./ || true docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/coverage.txt ./ || true - # Save the database schema as an artifact - docker compose run --no-deps --rm --entrypoint dbtoyaml app --no-owner --no-privileges test_${PGDATABASE} > schema.yml - diff pyrseas/schema.yaml schema.yml > schema.diff || true - name: "Upload test artifacts" if: success() || failure() uses: actions/upload-artifact@v4 @@ -400,9 +397,6 @@ jobs: # Copy the artifacts out of the Docker container to project directory docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/log ./ || true docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/coverage.txt ./ || true - # Save the database schema as an artifact - docker compose run --no-deps --rm --entrypoint dbtoyaml app --no-owner --no-privileges test_${PGDATABASE} > schema.yml - diff pyrseas/schema.yaml schema.yml > schema.diff || true - name: "Upload test artifacts" if: success() || failure() uses: actions/upload-artifact@v4 @@ -530,9 +524,6 @@ jobs: # Copy the artifacts out of the Docker container to project directory docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/log ./ || true docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/coverage.txt ./ || true - # Save the database schema as an artifact - docker compose run --no-deps --rm --entrypoint dbtoyaml app --no-owner --no-privileges test_${PGDATABASE} > schema.yml - diff pyrseas/schema.yaml schema.yml > schema.diff || true - name: "Upload test artifacts" if: success() || failure() uses: actions/upload-artifact@v4 @@ -661,11 +652,6 @@ jobs: # Copy the artifacts out of the Docker container to project directory docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/log ./ || true docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/coverage.txt ./ || true - # The prod image does not include pyrseas/dbtoyaml. Building a test image to include that - docker compose build app - # Save the database schema as an artifact - docker compose run --no-deps --rm --entrypoint dbtoyaml app --no-owner --no-privileges test_${PGDATABASE} > schema.yml - diff pyrseas/schema.yaml schema.yml > schema.diff || true - name: "Upload test artifacts" if: success() || failure() uses: actions/upload-artifact@v4 From 0bcbd8938cd6f84a3766701217eab7fa41d203dc Mon Sep 17 00:00:00 2001 From: Roger Hunwicks Date: Mon, 13 Oct 2025 21:15:49 -0400 Subject: [PATCH 7/7] Allow GDAL/GEOS library path overrides - see HEA-760 --- hea/settings/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hea/settings/base.py b/hea/settings/base.py index 630b8b1..f27a8f4 100644 --- a/hea/settings/base.py +++ b/hea/settings/base.py @@ -318,3 +318,7 @@ PRIVACY_URL = "https://help.fews.net/fdp/privacy-policy" DISCLAIMER_URL = "https://help.fews.net/fdp/data-and-information-use-and-attribution-policy" + +# Allow GDAL/GEOS library path overrides to be set in the environment, for MacOS. +GDAL_LIBRARY_PATH = env("GDAL_LIBRARY_PATH", default=None) # For example, /opt/homebrew/lib/libgdal.dylib +GEOS_LIBRARY_PATH = env("GEOS_LIBRARY_PATH", default=None) # For example, /opt/homebrew/lib/libgeos_c.dylib