diff --git a/AGENTS.md b/AGENTS.md index b5a8294..c406aba 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -97,6 +97,13 @@ There is no root `.env` anywhere - not on a target host, not locally. Each app's A vault declares the exact final variable name an app receives directly (e.g. `HTTP_PORT`, not `TRAEFIK_HTTP_PORT`) - there is no automatic prefix-stripping or filtering step anywhere. Variables for one app are never visible to another app, since each app's `.env` is built from that app's own vault(s) only. This allows running docker compose directly from the app folder without any `--env-file` flags while keeping app secrets scoped. +Two variables in every app's `.env` are never vault-sourced - `deploy/deploy.py` computes and writes them itself: + +- `APP_NAME` - just the app's own directory name (`traefik`, `rybbit`, ...). Written by `resolve_app_envs` on the runner, identical for every host a target deploys to, so it lives in the release tree's static `.env` like anything else. +- `DATA_DIR` - the absolute, persistent data path for that app on the *target host* (`{base_path}/apps-data/{app}`). This can't be computed on the runner: `base_path` comes from a target's `path` (default `~/flightdeck`), and `~` only resolves once connected to a specific host (`deploy_to_host` reads `$HOME` over SSH) - and a target's `hosts:` can list more than one, potentially with different home directories. So `DATA_DIR` is appended to each app's already-pushed `.env` on the host itself, inside `deploy_to_host`'s per-app loop, after the release is extracted but before `docker compose` ever runs - never baked into the release tree the way `APP_NAME` is. + +Compose files reference `${DATA_DIR}` directly (`${DATA_DIR}/postgres:/var/lib/postgresql`, or bare `${DATA_DIR}:/letsencrypt` when the whole directory is the mount) - never a relative path like `../../apps-data/${APP_NAME}`. A relative path only resolves correctly when the compose file's distance from the repo root matches its distance from the deployed base path, and those distances *don't* match: on the host, `docker compose` runs from `{base_path}/current/apps/{app}/`, where `current` is a symlink into `releases/{timestamp}/` - one directory hop `..`/`../..` can't see past, since path resolution follows the symlink lexically rather than landing back at `base_path`. Concretely this meant `../../apps-data/${APP_NAME}` from traefik's compose file resolved to `current/apps-data/${APP_NAME}` - physically inside that one release's own directory, not the persistent `apps-data/` sibling of `releases/` - so it would have been silently deleted by the next `keep_releases` rotation. `${DATA_DIR}` sidesteps the whole class of bug by never being a relative path in the first place. + ### Config Templates `docker compose`'s own `${VAR}` interpolation only reaches into `environment:`/`command:` fields inside the compose file itself - it can't populate a mounted config file some image insists on reading from disk (e.g. Codecov Enterprise's settings YAML, which has no env-var-driven config path at all). Config templates exist for exactly that gap: a plain file with `${VAR}`/`$VAR` placeholders, rendered with the app's own decrypted env values before the app ever starts. @@ -198,7 +205,7 @@ Backups are a separate, not-yet-decided piece of tooling (the old `backup.sh` as - Include `../networks.yml` for network definitions - Extend `../common.yml` service definitions (usually `main`) - Include a versioned database/service template if needed, e.g. `../postgres-18.yml`, `../redis-8.yml`, `../mongodb-8.yml` (see "Shared Infrastructure" above for the full list) - - Reference data path: `../../apps-data/${APP_NAME}/` + - Reference data path: `${DATA_DIR}/` - Set the service port explicitly with `expose` and `traefik.http.services.${APP_NAME}.loadbalancer.server.port` 3. Wire it into a target's `apps` mapping and give it a vault declaring the env it needs (see README's "Vaults And Targets") 4. If the app needs a mounted config file with no env-var equivalent, create `{name}.yml.tpl` next to its `docker-compose.yml` (see "Config Templates" above) @@ -223,7 +230,7 @@ services: labels: - "traefik.http.services.${APP_NAME}.loadbalancer.server.port=8080" volumes: - - ../../apps-data/${APP_NAME}/data:/data + - ${DATA_DIR}/data:/data ``` **IMPORTANT: Always use x-environment anchor pattern for environment variables:** @@ -258,7 +265,7 @@ x-environment: &environment # 4. X-VOLUMES (if multiple services share volumes) x-volumes: &volumes - - ../../apps-data/${APP_NAME}/data:/data + - ${DATA_DIR}/data:/data # 5. SERVICES services: @@ -299,7 +306,7 @@ services: # 8. VOLUMES (order: persistent data directories → rendered config files) volumes: - - ../../apps-data/${APP_NAME}/data:/data + - ${DATA_DIR}/data:/data - ./app.yml:/app/config.yml:ro # 9. NETWORKS (inherited from extends, omit this section) @@ -339,7 +346,7 @@ services: 7. **No empty lines in .yml files** - remove all blank lines, keep file compact without any empty line breaks 8. **No trailing spaces** - remove all trailing whitespace 9. **Restart policy** - if used, always `restart: unless-stopped` (not `always`) -10. **Volume paths consistency** - host folder name must match container mount point: `../../apps-data/${APP_NAME}/data:/data` (both are `data`), not `../../apps-data/${APP_NAME}/app-data:/data` or `../../apps-data/${APP_NAME}/library:/data` +10. **Volume paths consistency** - host folder name must match container mount point: `${DATA_DIR}/data:/data` (both are `data`), not `${DATA_DIR}/app-data:/data` or `${DATA_DIR}/library:/data` Example: @@ -361,7 +368,7 @@ services: user: "${PUID}:${PGID}" # not UID/GID - those are shell-reserved environment: *environment volumes: - - ../../apps-data/${APP_NAME}/data:/data + - ${DATA_DIR}/data:/data ``` ## CI/CD diff --git a/README.md b/README.md index 75d5a29..05a29bd 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ services: - "traefik.http.services.${APP_NAME}.loadbalancer.server.port=8080" environment: *environment volumes: - - ../../apps-data/${APP_NAME}/data:/data + - ${DATA_DIR}/data:/data ``` **Important**: Always use the `x-environment` anchor pattern for environment variables. This ensures consistency and reduces duplication. diff --git a/apps/beszel/docker-compose.yml b/apps/beszel/docker-compose.yml index 762e647..728d306 100644 --- a/apps/beszel/docker-compose.yml +++ b/apps/beszel/docker-compose.yml @@ -11,7 +11,7 @@ services: labels: - "traefik.http.services.${APP_NAME}.loadbalancer.server.port=8090" volumes: - - ../../apps-data/${APP_NAME}/beszel_data:/beszel_data + - ${DATA_DIR}/beszel_data:/beszel_data healthcheck: test: ["CMD", "/beszel", "health", "--url", "http://localhost:8090"] start_period: 30s diff --git a/apps/clickhouse-25.4.yml b/apps/clickhouse-25.4.yml index e652e64..ad0b0e4 100644 --- a/apps/clickhouse-25.4.yml +++ b/apps/clickhouse-25.4.yml @@ -5,8 +5,8 @@ services: environment: CLICKHOUSE_DB: ${APP_NAME} volumes: - - ../apps-data/${APP_NAME}/clickhouse/data:/var/lib/clickhouse - - ../apps-data/${APP_NAME}/clickhouse/logs:/var/log/clickhouse-server + - ${DATA_DIR}/clickhouse/data:/var/lib/clickhouse + - ${DATA_DIR}/clickhouse/logs:/var/log/clickhouse-server configs: - source: clickhouse_network target: /etc/clickhouse-server/config.d/network.xml diff --git a/apps/clickhouse-26.5.yml b/apps/clickhouse-26.5.yml index 99d7d2f..5da0d74 100644 --- a/apps/clickhouse-26.5.yml +++ b/apps/clickhouse-26.5.yml @@ -8,8 +8,8 @@ services: CLICKHOUSE_PASSWORD: ${DATABASE_PASSWORD} CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 volumes: - - ../apps-data/${APP_NAME}/clickhouse/data:/var/lib/clickhouse - - ../apps-data/${APP_NAME}/clickhouse/logs:/var/log/clickhouse-server + - ${DATA_DIR}/clickhouse/data:/var/lib/clickhouse + - ${DATA_DIR}/clickhouse/logs:/var/log/clickhouse-server configs: - source: clickhouse_network target: /etc/clickhouse-server/config.d/network.xml diff --git a/apps/codecov/docker-compose.yml b/apps/codecov/docker-compose.yml index 60b9aa4..8bf09e2 100644 --- a/apps/codecov/docker-compose.yml +++ b/apps/codecov/docker-compose.yml @@ -60,4 +60,4 @@ services: <<: [*vault-env, *internal-env] volumes: - ./codecov.yml:/config/codecov.yml - - ../../apps-data/${APP_NAME}/archive:/archive + - ${DATA_DIR}/archive:/archive diff --git a/apps/databasus/docker-compose.yml b/apps/databasus/docker-compose.yml index 365ea0f..3508236 100644 --- a/apps/databasus/docker-compose.yml +++ b/apps/databasus/docker-compose.yml @@ -11,7 +11,7 @@ services: labels: - "traefik.http.services.${APP_NAME}.loadbalancer.server.port=4005" volumes: - - ../../apps-data/${APP_NAME}/data:/databasus-data + - ${DATA_DIR}/data:/databasus-data networks: - internal - traefik diff --git a/apps/gatus/docker-compose.yml b/apps/gatus/docker-compose.yml index cf75b9f..6b7ca1e 100644 --- a/apps/gatus/docker-compose.yml +++ b/apps/gatus/docker-compose.yml @@ -23,7 +23,7 @@ services: environment: <<: [*vault-env, *internal-env] volumes: - - ../../apps-data/${APP_NAME}/config:/config + - ${DATA_DIR}/config:/config - ./global.yml:/config/yml/global.yml depends_on: - postgres diff --git a/apps/glitchtip/docker-compose.yml b/apps/glitchtip/docker-compose.yml index d4fff11..d3c3ec7 100644 --- a/apps/glitchtip/docker-compose.yml +++ b/apps/glitchtip/docker-compose.yml @@ -27,7 +27,7 @@ services: environment: <<: [*vault-env, *internal-env] volumes: - - ../../apps-data/${APP_NAME}/uploads:/code/uploads + - ${DATA_DIR}/uploads:/code/uploads depends_on: - postgres - redis @@ -40,7 +40,7 @@ services: environment: <<: [*vault-env, *internal-env] volumes: - - ../../apps-data/${APP_NAME}/uploads:/code/uploads + - ${DATA_DIR}/uploads:/code/uploads depends_on: - postgres - redis diff --git a/apps/homepage/docker-compose.yml b/apps/homepage/docker-compose.yml index fddfcc7..4ac6501 100644 --- a/apps/homepage/docker-compose.yml +++ b/apps/homepage/docker-compose.yml @@ -14,4 +14,4 @@ services: - "traefik.http.services.${APP_NAME}.loadbalancer.server.port=3000" environment: *vault-env volumes: - - ../../apps-data/${APP_NAME}/config:/app/config + - ${DATA_DIR}/config:/app/config diff --git a/apps/mongodb-8.yml b/apps/mongodb-8.yml index 69b1c6e..1dc1816 100644 --- a/apps/mongodb-8.yml +++ b/apps/mongodb-8.yml @@ -8,7 +8,7 @@ services: MONGODB_PASSWORD: ${DATABASE_PASSWORD} MONGODB_DATABASE: ${APP_NAME} volumes: - - ../apps-data/${APP_NAME}/mongodb:/bitnami/mongodb + - ${DATA_DIR}/mongodb:/bitnami/mongodb healthcheck: test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')"] start_period: 30s diff --git a/apps/mysql-8.yml b/apps/mysql-8.yml index 6c8e75e..e1e13ce 100644 --- a/apps/mysql-8.yml +++ b/apps/mysql-8.yml @@ -8,7 +8,7 @@ services: MYSQL_PASSWORD: ${DATABASE_PASSWORD} MYSQL_ROOT_PASSWORD: ${DATABASE_PASSWORD} volumes: - - ../apps-data/${APP_NAME}/mysql:/var/lib/mysql + - ${DATA_DIR}/mysql:/var/lib/mysql healthcheck: test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] start_period: 30s diff --git a/apps/paradedb-17.yml b/apps/paradedb-17.yml index 544a17d..1cf351c 100644 --- a/apps/paradedb-17.yml +++ b/apps/paradedb-17.yml @@ -8,7 +8,7 @@ services: POSTGRES_PASSWORD: ${DATABASE_PASSWORD} POSTGRES_USER: ${APP_NAME} volumes: - - ../apps-data/${APP_NAME}/paradedb:/var/lib/postgresql/data + - ${DATA_DIR}/paradedb:/var/lib/postgresql/data healthcheck: test: ["CMD", "pg_isready", "-d", "${APP_NAME}", "-U", "${APP_NAME}"] start_period: 30s diff --git a/apps/pgvector-17.yml b/apps/pgvector-17.yml index 7629fd0..fd486d3 100644 --- a/apps/pgvector-17.yml +++ b/apps/pgvector-17.yml @@ -8,7 +8,7 @@ services: POSTGRES_PASSWORD: ${DATABASE_PASSWORD} POSTGRES_USER: ${APP_NAME} volumes: - - ../apps-data/${APP_NAME}/pgvector:/var/lib/postgresql/data + - ${DATA_DIR}/pgvector:/var/lib/postgresql/data healthcheck: test: ["CMD", "pg_isready", "-d", "${APP_NAME}", "-U", "${APP_NAME}"] start_period: 30s diff --git a/apps/postgres-17.yml b/apps/postgres-17.yml index 2ea9b0c..e54305d 100644 --- a/apps/postgres-17.yml +++ b/apps/postgres-17.yml @@ -8,7 +8,7 @@ services: POSTGRES_PASSWORD: ${DATABASE_PASSWORD} POSTGRES_USER: ${APP_NAME} volumes: - - ../apps-data/${APP_NAME}/postgres:/var/lib/postgresql/data + - ${DATA_DIR}/postgres:/var/lib/postgresql/data healthcheck: test: ["CMD", "pg_isready", "-d", "${APP_NAME}", "-U", "${APP_NAME}"] start_period: 30s diff --git a/apps/postgres-18.yml b/apps/postgres-18.yml index e8988de..9170443 100644 --- a/apps/postgres-18.yml +++ b/apps/postgres-18.yml @@ -8,7 +8,7 @@ services: POSTGRES_PASSWORD: ${DATABASE_PASSWORD} POSTGRES_USER: ${APP_NAME} volumes: - - ../apps-data/${APP_NAME}/postgres:/var/lib/postgresql + - ${DATA_DIR}/postgres:/var/lib/postgresql healthcheck: test: ["CMD", "pg_isready", "-d", "${APP_NAME}", "-U", "${APP_NAME}"] start_period: 30s diff --git a/apps/redis-7.yml b/apps/redis-7.yml index af20331..3be1602 100644 --- a/apps/redis-7.yml +++ b/apps/redis-7.yml @@ -3,7 +3,7 @@ services: image: redis:7 restart: unless-stopped volumes: - - ../apps-data/${APP_NAME}/redis:/data + - ${DATA_DIR}/redis:/data healthcheck: test: ["CMD", "redis-cli", "ping"] start_period: 30s diff --git a/apps/redis-8.yml b/apps/redis-8.yml index 4cc66c0..4c14743 100644 --- a/apps/redis-8.yml +++ b/apps/redis-8.yml @@ -3,7 +3,7 @@ services: image: redis:8 restart: unless-stopped volumes: - - ../apps-data/${APP_NAME}/redis:/data + - ${DATA_DIR}/redis:/data healthcheck: test: ["CMD", "redis-cli", "ping"] start_period: 30s diff --git a/apps/timescale-17.yml b/apps/timescale-17.yml index 48e90fe..0c20e5f 100644 --- a/apps/timescale-17.yml +++ b/apps/timescale-17.yml @@ -7,7 +7,7 @@ services: POSTGRES_USER: ${APP_NAME} POSTGRES_PASSWORD: ${DATABASE_PASSWORD} volumes: - - ../apps-data/${APP_NAME}/timescale:/var/lib/postgresql/data + - ${DATA_DIR}/timescale:/var/lib/postgresql/data healthcheck: test: ["CMD", "pg_isready", "-d", "${APP_NAME}", "-U", "${APP_NAME}"] start_period: 30s diff --git a/apps/traefik/docker-compose.yml b/apps/traefik/docker-compose.yml index 4bd12bf..66d38bd 100644 --- a/apps/traefik/docker-compose.yml +++ b/apps/traefik/docker-compose.yml @@ -32,7 +32,7 @@ services: volumes: - /etc/localtime:/etc/localtime:ro - /var/run/docker.sock:/var/run/docker.sock:ro - - ../../apps-data/${APP_NAME}:/letsencrypt + - ${DATA_DIR}:/letsencrypt networks: - traefik security_opt: diff --git a/apps/twofauth/docker-compose.yml b/apps/twofauth/docker-compose.yml index 3cf8e9e..d434342 100644 --- a/apps/twofauth/docker-compose.yml +++ b/apps/twofauth/docker-compose.yml @@ -50,4 +50,4 @@ services: environment: <<: [*vault-env, *internal-env] volumes: - - ../../apps-data/${APP_NAME}/2fauth:/2fauth + - ${DATA_DIR}/2fauth:/2fauth diff --git a/deploy/deploy.py b/deploy/deploy.py index 2d4a0ac..504b1e9 100644 --- a/deploy/deploy.py +++ b/deploy/deploy.py @@ -80,7 +80,7 @@ def resolve_app_envs(config, work_dir, release_dir, age_key_file): ] check_env_collisions(paths) - plaintext = "".join(decrypt_env(path, age_key_file) for path in paths) + plaintext = f"APP_NAME={app}\n" + "".join(decrypt_env(path, age_key_file) for path in paths) app_env_path = release_dir / "apps" / app / ".env" app_env_path.write_text(plaintext) app_env_path.chmod(0o600) @@ -147,7 +147,10 @@ def deploy_to_host(host, archive_path, apps, networks, config): bootstrap_host(connection, base_path, networks) push_release(connection, archive_path, release_path) for app in apps: - connection.run(f"mkdir -p {shlex.quote(f'{base_path}/apps-data/{app}')}", hide=True) + data_dir = f"{base_path}/apps-data/{app}" + connection.run(f"mkdir -p {shlex.quote(data_dir)}", hide=True) + env_path = f"{release_path}/apps/{app}/.env" + connection.run(f"echo {shlex.quote(f'DATA_DIR={data_dir}')} >> {shlex.quote(env_path)}", hide=True) connection.run(f"ln -sfn {shlex.quote(release_path)} {shlex.quote(current_path)}", hide=True) diff --git a/deploy/tests/test_deploy.py b/deploy/tests/test_deploy.py index b67f62a..37cc3a3 100644 --- a/deploy/tests/test_deploy.py +++ b/deploy/tests/test_deploy.py @@ -192,7 +192,7 @@ def test_writes_decrypted_env_and_renders_configs(self): deploy.resolve_app_envs(config, work_dir / "work", release_dir, work_dir / "key.txt") env_path = release_dir / "apps" / "codecov" / ".env" - self.assertEqual(env_path.read_text(), "ADMIN_MAIL=a@example.com\n") + self.assertEqual(env_path.read_text(), "APP_NAME=codecov\nADMIN_MAIL=a@example.com\n") self.assertEqual(oct(env_path.stat().st_mode)[-3:], "600") rendered_path = release_dir / "apps" / "codecov" / "codecov.yml" @@ -336,6 +336,10 @@ def test_full_sequence(self): self.assertIn("tar -xzf", joined) self.assertIn("mkdir -p /home/deploy/flightdeck/apps-data/traefik", joined) self.assertIn("mkdir -p /home/deploy/flightdeck/apps-data/rybbit", joined) + self.assertIn("DATA_DIR=/home/deploy/flightdeck/apps-data/traefik", joined) + self.assertIn("DATA_DIR=/home/deploy/flightdeck/apps-data/rybbit", joined) + self.assertIn("apps/traefik/.env", joined) + self.assertIn("apps/rybbit/.env", joined) self.assertIn("ln -sfn", joined) self.assertIn("apps/rybbit && docker compose pull && docker compose up -d --remove-orphans", joined) self.assertIn("apps/traefik && docker compose pull && docker compose up -d --remove-orphans", joined)