Hea 760/better macos development

.github/workflows/01-build-then-test.yml

@@ -271,9 +271,6 @@ jobs:
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/log ./ || true
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/coverage.txt ./ || true
 
-          # Save the database schema as an artifact
-          docker compose run --no-deps --rm --entrypoint dbtoyaml app --no-owner --no-privileges test_${PGDATABASE} > schema.yml
-          diff pyrseas/schema.yaml schema.yml > schema.diff || true
       - name: "Upload test artifacts"
         if: success() || failure()
         uses: actions/upload-artifact@v4
@@ -400,9 +397,6 @@ jobs:
           # Copy the artifacts out of the Docker container to project directory
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/log ./ || true
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/coverage.txt ./ || true
-          # Save the database schema as an artifact
-          docker compose run --no-deps --rm --entrypoint dbtoyaml app --no-owner --no-privileges test_${PGDATABASE} > schema.yml
-          diff pyrseas/schema.yaml schema.yml > schema.diff || true
       - name: "Upload test artifacts"
         if: success() || failure()
         uses: actions/upload-artifact@v4
@@ -530,9 +524,6 @@ jobs:
           # Copy the artifacts out of the Docker container to project directory
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/log ./ || true
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/coverage.txt ./ || true
-          # Save the database schema as an artifact
-          docker compose run --no-deps --rm --entrypoint dbtoyaml app --no-owner --no-privileges test_${PGDATABASE} > schema.yml
-          diff pyrseas/schema.yaml schema.yml > schema.diff || true
       - name: "Upload test artifacts"
         if: success() || failure()
         uses: actions/upload-artifact@v4
@@ -661,11 +652,6 @@ jobs:
           # Copy the artifacts out of the Docker container to project directory
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/log ./ || true
           docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/coverage.txt ./ || true
-          # The prod image does not include pyrseas/dbtoyaml. Building a test image to include that
-          docker compose build app
-          # Save the database schema as an artifact
-          docker compose run --no-deps --rm --entrypoint dbtoyaml app --no-owner --no-privileges test_${PGDATABASE} > schema.yml
-          diff pyrseas/schema.yaml schema.yml > schema.diff || true
       - name: "Upload test artifacts"
         if: success() || failure()
         uses: actions/upload-artifact@v4

README.md

@@ -23,3 +23,53 @@ baseline
 
 This produces a .puml file that can be rendered using a PlantUML
 server, either within your IDE or using a service like http://www.plantuml.com/.
+
+## Debugging inside Docker Containers
+
+The `LAUNCHER` environment sets a wrapper program around the Python process
+(`gunicorn`, `dagster-daemon`, `dagster-webserver`). This can be used to
+enable a debugger inside Docker Containers:
+
+1. Set `LAUNCHER="python3 -m debugpy --listen 0.0.0.0:5678"` in `.env`
+2. Create Launch Configurations in Visual Studio Code like:
+
+```json
+			{
+				"name": "Python: Attach to app (Docker Container)",
+				"type": "debugpy",
+				"request": "attach",
+				"connect": {
+					"host": "localhost",
+					"port": 5678
+				},
+				"pathMappings": [
+					{
+						"localRoot": "${workspaceFolder:hea}",
+						"remoteRoot": "/usr/src/app"
+					}
+				],
+				"django": true,
+				"justMyCode": false
+			},
+			{
+				"name": "Python: Attach to dagster-daemon (Docker Container)",
+				"type": "debugpy",
+				"request": "attach",
+				"connect": {
+					"host": "localhost",
+					"port": 5680
+				},
+				"pathMappings": [
+					{
+						"localRoot": "${workspaceFolder:hea}",
+						"remoteRoot": "/usr/src/app"
+					}
+				],
+				"django": true,
+				"justMyCode": false
+			}
+```
+
+3. Start the Docker containers as normal, and then use the Run and Debug
+pane in Visual Studio code to launch the configuration that attaches to
+the desired server.

docker-compose.override.yml

@@ -15,6 +15,7 @@ services:
     build:
       target: dev
     ports:
+      - "5678:5678"
       - "8000:8000"
       - "8888:8888"
     volumes:
@@ -30,6 +31,9 @@ services:
       - ./env.example:/usr/src/app/.env
     environment:
       DJANGO_SETTINGS_MODULE: hea.settings.local
+      LAUNCHER: ${LAUNCHER}  # e.g. "debugpy" or "ddtrace"
+      # Disable frozen modules warning
+      PYDEVD_DISABLE_FILE_VALIDATION: 1
       # Put .coverage in a writable directory
       COVERAGE_FILE: log/.coverage
     command:
@@ -41,6 +45,7 @@ services:
     restart: no
     ports:
       - "3000:3000"
+      - "5679:5678"
     volumes:
       - ./:/usr/src/app
       # Separate volumes for writable directories inside the container
@@ -54,9 +59,14 @@ services:
       - ./env.example:/usr/src/app/.env
     environment:
       DJANGO_SETTINGS_MODULE: hea.settings.local
+      LAUNCHER: ${LAUNCHER}  # e.g. "debugpy" or "ddtrace"
+      # Disable frozen modules warning
+      PYDEVD_DISABLE_FILE_VALIDATION: 1
 
   dagster_daemon:
     restart: no
+    ports:
+      - "5680:5678"
     volumes:
       - ./:/usr/src/app
       # Separate volumes for writable directories inside the container
@@ -70,4 +80,7 @@ services:
       - ./env.example:/usr/src/app/.env
     environment:
       DJANGO_SETTINGS_MODULE: hea.settings.local
+      LAUNCHER: ${LAUNCHER}  # e.g. "debugpy" or "ddtrace"
+      # Disable frozen modules warning
+      PYDEVD_DISABLE_FILE_VALIDATION: 1
 

docker-compose.yml

@@ -110,14 +110,7 @@ services:
       MINIO_ENDPOINT_URL: http://minio:9000
       SUPPORT_EMAIL_ADDRESS: ${SUPPORT_EMAIL_ADDRESS}
       DJANGO_MIGRATE: 1
-      KILUIGI_INTERMEDIATETARGET_BACKEND_CLASS: ${KILUIGI_INTERMEDIATETARGET_BACKEND_CLASS}
-      KILUIGI_INTERMEDIATETARGET_ROOT_PATH: ${KILUIGI_INTERMEDIATETARGET_ROOT_PATH}
-      KILUIGI_FINALTARGET_BACKEND_CLASS: ${KILUIGI_FINALTARGET_BACKEND_CLASS}
-      KILUIGI_FINALTARGET_ROOT_PATH: ${KILUIGI_FINALTARGET_ROOT_PATH}
-      KILUIGI_REPORTTARGET_BACKEND_CLASS: ${KILUIGI_REPORTTARGET_BACKEND_CLASS}
-      KILUIGI_REPORTTARGET_ROOT_PATH: ${KILUIGI_REPORTTARGET_ROOT_PATH}      
       GOOGLE_APPLICATION_CREDENTIALS: ${GOOGLE_APPLICATION_CREDENTIALS}
-      GOOGLE_ADMIN_EMAIL: ${GOOGLE_ADMIN_EMAIL}
     command:
       - --timeout=3600
       - --workers=12

docker/app/Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=linux/amd64 python:3.12-bookworm as base
+FROM python:3.12-bookworm as base
 
 # set up apt repositories for postgres installation
 RUN curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/pgdg.gpg >/dev/null && \

docker/app/run_dagster_daemon.sh

@@ -13,5 +13,8 @@ echo Setting up logs
 touch log/django.log
 chown -R django:django log/*
 
-echo Starting Dagster
-gosu django dagster-daemon run $*
+echo Starting Dagster Daemon
+if [ x"$LAUNCHER" != x"" ]; then
+    echo using ${LAUNCHER}
+fi
+gosu django ${LAUNCHER} /usr/local/bin/dagster-daemon run $*

docker/app/run_dagster_webserver.sh

@@ -13,5 +13,8 @@ echo Setting up logs
 touch log/django.log
 chown -R django:django log/*
 
-echo Starting Dagster
-gosu django dagster-webserver -h 0.0.0.0 -p 3000 -m pipelines --path-prefix /${DAGSTER_WEBSERVER_PREFIX} $*
+echo Starting Dagster Webserver
+if [ x"$LAUNCHER" != x"" ]; then
+    echo using ${LAUNCHER}
+fi
+gosu django ${LAUNCHER} /usr/local/bin/dagster-webserver -h 0.0.0.0 -p 3000 -m pipelines --path-prefix /${DAGSTER_WEBSERVER_PREFIX} $*

docker/app/run_django.sh

@@ -40,7 +40,10 @@ touch log/django_sql.log
 chown -R django:django log/*
 
 echo Starting Gunicorn with DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE}
-gosu django  gunicorn ${APP}.wsgi:application \
+if [ x"$LAUNCHER" != x"" ]; then
+    echo using ${LAUNCHER}
+fi
+gosu django ${LAUNCHER} /usr/local/bin/gunicorn ${APP}.wsgi:application \
     --name ${APP}${ENV} \
     --config $(dirname $(readlink -f "$0"))/gunicorn_config.py \
     $* 2>&1

docker/db/Dockerfile

@@ -1,3 +1,5 @@
-FROM --platform=linux/amd64 postgis/postgis:17-3.5
+# Use a third party multicarch base image for compatibility with both ARM and AMD architectures
+# until PostGIS fix https://github.com/postgis/docker-postgis/issues/216
+FROM ghcr.io/baosystems/postgis:17-3.5
 
 COPY create_db.sh /docker-entrypoint-initdb.d/create_db.sh

docker/db/create_db.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-psql --set=CLIENT=$CLIENT --set=APP=$APP --set=ENV=$ENV --set=POSTGRES_PASSWORD=$POSTGRES_PASSWORD --set=DAGSTER_PASSWORD=$DAGSTER_PASSWORD --set=CREATE_TEMPLATE=${CREATE_TEMPLATE:-false} -d postgres --echo-all << EOF
+psql --set=CLIENT=$CLIENT --set=APP=$APP --set=ENV=$ENV --set=POSTGRES_PASSWORD=$POSTGRES_PASSWORD --set=CREATE_TEMPLATE=${CREATE_TEMPLATE:-false} -d postgres --echo-all << EOF
 
 \set DATABASE :CLIENT :APP :ENV
 \set OWNER :CLIENT :APP :ENV
@@ -74,7 +74,7 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA :SCHEMA
   GRANT SELECT ON TABLES TO :OWNER;
 
 \set DAGSTER :CLIENT :APP :ENV
-CREATE ROLE :DAGSTER PASSWORD :'DAGSTER_PASSWORD' NOLOGIN CREATEDB NOCREATEROLE NOSUPERUSER;
+CREATE ROLE :DAGSTER PASSWORD :'POSTGRES_PASSWORD' NOLOGIN CREATEDB NOCREATEROLE NOSUPERUSER;
 COMMENT ON ROLE :DAGSTER IS 'Main Dagster pipeline user for :CLIENT :PRJ :ENV';
 GRANT :DAGSTER TO :OWNER;
 GRANT CONNECT, TEMPORARY, CREATE ON DATABASE :DATABASE TO :DAGSTER;

env.example

@@ -48,3 +48,8 @@ BSS_METADATA_WORKBOOK='gdrive://Database Design/BSS Metadata'  # 15XVXFjbom1sScV
 BSS_METADATA_STORAGE_OPTIONS='{"token": "service_account", "access": "read_only", "creds": ${GOOGLE_APPLICATION_CREDENTIALS}, "root_file_id": "0AOJ0gJ8sjnO7Uk9PVA"}'
 BSS_FILES_FOLDER='gdrive://Discovery Folder/Baseline Storage Sheets (BSS)'
 BSS_FILES_STORAGE_OPTIONS='{"token": "service_account", "access": "read_only", "creds": ${GOOGLE_APPLICATION_CREDENTIALS}, "root_file_id": "0AOJ0gJ8sjnO7Uk9PVA"}'
+
+# LAUNCHER can be used to configure a wrapper program around the Python process
+# For example, to add ddtrace or debugpy
+# Use the VSCode debugger as a launcher
+# LAUNCHER = "python3 -m debugpy --listen 0.0.0.0:5679"

hea/settings/base.py

@@ -318,3 +318,7 @@
 
 PRIVACY_URL = "https://help.fews.net/fdp/privacy-policy"
 DISCLAIMER_URL = "https://help.fews.net/fdp/data-and-information-use-and-attribution-policy"
+
+# Allow GDAL/GEOS library path overrides to be set in the environment, for MacOS.
+GDAL_LIBRARY_PATH = env("GDAL_LIBRARY_PATH", default=None)  # For example, /opt/homebrew/lib/libgdal.dylib
+GEOS_LIBRARY_PATH = env("GEOS_LIBRARY_PATH", default=None)  # For example, /opt/homebrew/lib/libgeos_c.dylib

requirements/local.txt

@@ -1,2 +1,3 @@
 -r test.txt
 -r lint.txt
+debugpy

requirements/test.txt

@@ -1,6 +1,4 @@
 -r base.txt
 beautifulsoup4==4.12.2
 coverage==7.2.7
-# Pyrseas==0.10.0 raises "KeyError: ('public', 'spatial_ref_sys')", --schema/--exclude-schema don't fix it.
-Pyrseas==0.9.1
 safety==3.6.1