diff --git a/.github/workflows/01-build-then-test.yml b/.github/workflows/01-build-then-test.yml index a517324..9efe314 100644 --- a/.github/workflows/01-build-then-test.yml +++ b/.github/workflows/01-build-then-test.yml @@ -271,9 +271,6 @@ jobs: docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/log ./ || true docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/coverage.txt ./ || true - # Save the database schema as an artifact - docker compose run --no-deps --rm --entrypoint dbtoyaml app --no-owner --no-privileges test_${PGDATABASE} > schema.yml - diff pyrseas/schema.yaml schema.yml > schema.diff || true - name: "Upload test artifacts" if: success() || failure() uses: actions/upload-artifact@v4 @@ -400,9 +397,6 @@ jobs: # Copy the artifacts out of the Docker container to project directory docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/log ./ || true docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/coverage.txt ./ || true - # Save the database schema as an artifact - docker compose run --no-deps --rm --entrypoint dbtoyaml app --no-owner --no-privileges test_${PGDATABASE} > schema.yml - diff pyrseas/schema.yaml schema.yml > schema.diff || true - name: "Upload test artifacts" if: success() || failure() uses: actions/upload-artifact@v4 @@ -530,9 +524,6 @@ jobs: # Copy the artifacts out of the Docker container to project directory docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/log ./ || true docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/coverage.txt ./ || true - # Save the database schema as an artifact - docker compose run --no-deps --rm --entrypoint dbtoyaml app --no-owner --no-privileges test_${PGDATABASE} > schema.yml - diff pyrseas/schema.yaml schema.yml > schema.diff || true - name: "Upload test artifacts" if: success() || failure() uses: actions/upload-artifact@v4 @@ -661,11 +652,6 @@ jobs: # Copy the artifacts out of the Docker container to project directory docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/log ./ || true docker cp ci-${APP}-${CI_PIPELINE_ID}-${{ github.job }}:/usr/src/app/coverage.txt ./ || true - # The prod image does not include pyrseas/dbtoyaml. Building a test image to include that - docker compose build app - # Save the database schema as an artifact - docker compose run --no-deps --rm --entrypoint dbtoyaml app --no-owner --no-privileges test_${PGDATABASE} > schema.yml - diff pyrseas/schema.yaml schema.yml > schema.diff || true - name: "Upload test artifacts" if: success() || failure() uses: actions/upload-artifact@v4 diff --git a/README.md b/README.md index 7d78bda..2f29e30 100644 --- a/README.md +++ b/README.md @@ -23,3 +23,53 @@ baseline This produces a .puml file that can be rendered using a PlantUML server, either within your IDE or using a service like http://www.plantuml.com/. + +## Debugging inside Docker Containers + +The `LAUNCHER` environment sets a wrapper program around the Python process +(`gunicorn`, `dagster-daemon`, `dagster-webserver`). This can be used to +enable a debugger inside Docker Containers: + +1. Set `LAUNCHER="python3 -m debugpy --listen 0.0.0.0:5678"` in `.env` +2. Create Launch Configurations in Visual Studio Code like: + +```json + { + "name": "Python: Attach to app (Docker Container)", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder:hea}", + "remoteRoot": "/usr/src/app" + } + ], + "django": true, + "justMyCode": false + }, + { + "name": "Python: Attach to dagster-daemon (Docker Container)", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5680 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder:hea}", + "remoteRoot": "/usr/src/app" + } + ], + "django": true, + "justMyCode": false + } +``` + +3. Start the Docker containers as normal, and then use the Run and Debug +pane in Visual Studio code to launch the configuration that attaches to +the desired server. \ No newline at end of file diff --git a/docker-compose.override.yml b/docker-compose.override.yml index a1ffcc6..dd8e618 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -15,6 +15,7 @@ services: build: target: dev ports: + - "5678:5678" - "8000:8000" - "8888:8888" volumes: @@ -30,6 +31,9 @@ services: - ./env.example:/usr/src/app/.env environment: DJANGO_SETTINGS_MODULE: hea.settings.local + LAUNCHER: ${LAUNCHER} # e.g. "debugpy" or "ddtrace" + # Disable frozen modules warning + PYDEVD_DISABLE_FILE_VALIDATION: 1 # Put .coverage in a writable directory COVERAGE_FILE: log/.coverage command: @@ -41,6 +45,7 @@ services: restart: no ports: - "3000:3000" + - "5679:5678" volumes: - ./:/usr/src/app # Separate volumes for writable directories inside the container @@ -54,9 +59,14 @@ services: - ./env.example:/usr/src/app/.env environment: DJANGO_SETTINGS_MODULE: hea.settings.local + LAUNCHER: ${LAUNCHER} # e.g. "debugpy" or "ddtrace" + # Disable frozen modules warning + PYDEVD_DISABLE_FILE_VALIDATION: 1 dagster_daemon: restart: no + ports: + - "5680:5678" volumes: - ./:/usr/src/app # Separate volumes for writable directories inside the container @@ -70,4 +80,7 @@ services: - ./env.example:/usr/src/app/.env environment: DJANGO_SETTINGS_MODULE: hea.settings.local + LAUNCHER: ${LAUNCHER} # e.g. "debugpy" or "ddtrace" + # Disable frozen modules warning + PYDEVD_DISABLE_FILE_VALIDATION: 1 diff --git a/docker-compose.yml b/docker-compose.yml index e75dd65..bc76f3e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -110,14 +110,7 @@ services: MINIO_ENDPOINT_URL: http://minio:9000 SUPPORT_EMAIL_ADDRESS: ${SUPPORT_EMAIL_ADDRESS} DJANGO_MIGRATE: 1 - KILUIGI_INTERMEDIATETARGET_BACKEND_CLASS: ${KILUIGI_INTERMEDIATETARGET_BACKEND_CLASS} - KILUIGI_INTERMEDIATETARGET_ROOT_PATH: ${KILUIGI_INTERMEDIATETARGET_ROOT_PATH} - KILUIGI_FINALTARGET_BACKEND_CLASS: ${KILUIGI_FINALTARGET_BACKEND_CLASS} - KILUIGI_FINALTARGET_ROOT_PATH: ${KILUIGI_FINALTARGET_ROOT_PATH} - KILUIGI_REPORTTARGET_BACKEND_CLASS: ${KILUIGI_REPORTTARGET_BACKEND_CLASS} - KILUIGI_REPORTTARGET_ROOT_PATH: ${KILUIGI_REPORTTARGET_ROOT_PATH} GOOGLE_APPLICATION_CREDENTIALS: ${GOOGLE_APPLICATION_CREDENTIALS} - GOOGLE_ADMIN_EMAIL: ${GOOGLE_ADMIN_EMAIL} command: - --timeout=3600 - --workers=12 diff --git a/docker/app/Dockerfile b/docker/app/Dockerfile index cf47a02..61a383b 100644 --- a/docker/app/Dockerfile +++ b/docker/app/Dockerfile @@ -1,4 +1,4 @@ -FROM --platform=linux/amd64 python:3.12-bookworm as base +FROM python:3.12-bookworm as base # set up apt repositories for postgres installation RUN curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/pgdg.gpg >/dev/null && \ diff --git a/docker/app/run_dagster_daemon.sh b/docker/app/run_dagster_daemon.sh index 8beac63..87c46b3 100755 --- a/docker/app/run_dagster_daemon.sh +++ b/docker/app/run_dagster_daemon.sh @@ -13,5 +13,8 @@ echo Setting up logs touch log/django.log chown -R django:django log/* -echo Starting Dagster -gosu django dagster-daemon run $* \ No newline at end of file +echo Starting Dagster Daemon +if [ x"$LAUNCHER" != x"" ]; then + echo using ${LAUNCHER} +fi +gosu django ${LAUNCHER} /usr/local/bin/dagster-daemon run $* \ No newline at end of file diff --git a/docker/app/run_dagster_webserver.sh b/docker/app/run_dagster_webserver.sh index e6b55a1..5e27a35 100755 --- a/docker/app/run_dagster_webserver.sh +++ b/docker/app/run_dagster_webserver.sh @@ -13,5 +13,8 @@ echo Setting up logs touch log/django.log chown -R django:django log/* -echo Starting Dagster -gosu django dagster-webserver -h 0.0.0.0 -p 3000 -m pipelines --path-prefix /${DAGSTER_WEBSERVER_PREFIX} $* \ No newline at end of file +echo Starting Dagster Webserver +if [ x"$LAUNCHER" != x"" ]; then + echo using ${LAUNCHER} +fi +gosu django ${LAUNCHER} /usr/local/bin/dagster-webserver -h 0.0.0.0 -p 3000 -m pipelines --path-prefix /${DAGSTER_WEBSERVER_PREFIX} $* \ No newline at end of file diff --git a/docker/app/run_django.sh b/docker/app/run_django.sh index 73112fa..e6f2b48 100755 --- a/docker/app/run_django.sh +++ b/docker/app/run_django.sh @@ -40,7 +40,10 @@ touch log/django_sql.log chown -R django:django log/* echo Starting Gunicorn with DJANGO_SETTINGS_MODULE=${DJANGO_SETTINGS_MODULE} -gosu django gunicorn ${APP}.wsgi:application \ +if [ x"$LAUNCHER" != x"" ]; then + echo using ${LAUNCHER} +fi +gosu django ${LAUNCHER} /usr/local/bin/gunicorn ${APP}.wsgi:application \ --name ${APP}${ENV} \ --config $(dirname $(readlink -f "$0"))/gunicorn_config.py \ $* 2>&1 diff --git a/docker/db/Dockerfile b/docker/db/Dockerfile index dbccb48..ea0a4f0 100644 --- a/docker/db/Dockerfile +++ b/docker/db/Dockerfile @@ -1,3 +1,5 @@ -FROM --platform=linux/amd64 postgis/postgis:17-3.5 +# Use a third party multicarch base image for compatibility with both ARM and AMD architectures +# until PostGIS fix https://github.com/postgis/docker-postgis/issues/216 +FROM ghcr.io/baosystems/postgis:17-3.5 COPY create_db.sh /docker-entrypoint-initdb.d/create_db.sh diff --git a/docker/db/create_db.sh b/docker/db/create_db.sh index fd78e59..08ac769 100755 --- a/docker/db/create_db.sh +++ b/docker/db/create_db.sh @@ -1,5 +1,5 @@ #!/bin/sh -psql --set=CLIENT=$CLIENT --set=APP=$APP --set=ENV=$ENV --set=POSTGRES_PASSWORD=$POSTGRES_PASSWORD --set=DAGSTER_PASSWORD=$DAGSTER_PASSWORD --set=CREATE_TEMPLATE=${CREATE_TEMPLATE:-false} -d postgres --echo-all << EOF +psql --set=CLIENT=$CLIENT --set=APP=$APP --set=ENV=$ENV --set=POSTGRES_PASSWORD=$POSTGRES_PASSWORD --set=CREATE_TEMPLATE=${CREATE_TEMPLATE:-false} -d postgres --echo-all << EOF \set DATABASE :CLIENT :APP :ENV \set OWNER :CLIENT :APP :ENV @@ -74,7 +74,7 @@ ALTER DEFAULT PRIVILEGES IN SCHEMA :SCHEMA GRANT SELECT ON TABLES TO :OWNER; \set DAGSTER :CLIENT :APP :ENV -CREATE ROLE :DAGSTER PASSWORD :'DAGSTER_PASSWORD' NOLOGIN CREATEDB NOCREATEROLE NOSUPERUSER; +CREATE ROLE :DAGSTER PASSWORD :'POSTGRES_PASSWORD' NOLOGIN CREATEDB NOCREATEROLE NOSUPERUSER; COMMENT ON ROLE :DAGSTER IS 'Main Dagster pipeline user for :CLIENT :PRJ :ENV'; GRANT :DAGSTER TO :OWNER; GRANT CONNECT, TEMPORARY, CREATE ON DATABASE :DATABASE TO :DAGSTER; diff --git a/env.example b/env.example index 680e231..3c4775f 100644 --- a/env.example +++ b/env.example @@ -48,3 +48,8 @@ BSS_METADATA_WORKBOOK='gdrive://Database Design/BSS Metadata' # 15XVXFjbom1sScV BSS_METADATA_STORAGE_OPTIONS='{"token": "service_account", "access": "read_only", "creds": ${GOOGLE_APPLICATION_CREDENTIALS}, "root_file_id": "0AOJ0gJ8sjnO7Uk9PVA"}' BSS_FILES_FOLDER='gdrive://Discovery Folder/Baseline Storage Sheets (BSS)' BSS_FILES_STORAGE_OPTIONS='{"token": "service_account", "access": "read_only", "creds": ${GOOGLE_APPLICATION_CREDENTIALS}, "root_file_id": "0AOJ0gJ8sjnO7Uk9PVA"}' + +# LAUNCHER can be used to configure a wrapper program around the Python process +# For example, to add ddtrace or debugpy +# Use the VSCode debugger as a launcher +# LAUNCHER = "python3 -m debugpy --listen 0.0.0.0:5679" \ No newline at end of file diff --git a/hea/settings/base.py b/hea/settings/base.py index 630b8b1..f27a8f4 100644 --- a/hea/settings/base.py +++ b/hea/settings/base.py @@ -318,3 +318,7 @@ PRIVACY_URL = "https://help.fews.net/fdp/privacy-policy" DISCLAIMER_URL = "https://help.fews.net/fdp/data-and-information-use-and-attribution-policy" + +# Allow GDAL/GEOS library path overrides to be set in the environment, for MacOS. +GDAL_LIBRARY_PATH = env("GDAL_LIBRARY_PATH", default=None) # For example, /opt/homebrew/lib/libgdal.dylib +GEOS_LIBRARY_PATH = env("GEOS_LIBRARY_PATH", default=None) # For example, /opt/homebrew/lib/libgeos_c.dylib diff --git a/requirements/local.txt b/requirements/local.txt index 4e95b36..8cd2465 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -1,2 +1,3 @@ -r test.txt -r lint.txt +debugpy \ No newline at end of file diff --git a/requirements/test.txt b/requirements/test.txt index 692ce0a..5a10e48 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,6 +1,4 @@ -r base.txt beautifulsoup4==4.12.2 coverage==7.2.7 -# Pyrseas==0.10.0 raises "KeyError: ('public', 'spatial_ref_sys')", --schema/--exclude-schema don't fix it. -Pyrseas==0.9.1 safety==3.6.1