diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5850c598b..7bf890c5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -187,6 +187,24 @@ jobs: - run: npm ci - run: npm run type-check + frontend-typed-router-determinism: + name: Frontend Typed Router Determinism + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v6 + with: + node-version: 22 + cache: 'npm' + cache-dependency-path: frontend/package-lock.json + - run: npm ci + - name: Setup dummy backend .env + run: cp ../.env.precommit ../.env + - run: npm run check:typed-router:build + frontend-unit-tests: name: Frontend Unit Tests runs-on: ubuntu-latest diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index e585fdbcd..375020e3d 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,7 +2,7 @@ name: Close stale PRs on: schedule: - - cron: "0 9 * * 1" + - cron: '0 9 * * 1' jobs: stale: @@ -10,7 +10,7 @@ jobs: steps: - uses: actions/stale@v10 with: - stale-pr-message: "This PR has been inactive for 30 days. It will be closed in 7 days if there is no further activity." + stale-pr-message: 'This PR has been inactive for 30 days. It will be closed in 7 days if there is no further activity.' days-before-pr-stale: 30 days-before-pr-close: 7 days-before-issue-stale: -1 diff --git a/.gitignore b/.gitignore index e158803ed..0d5c84a69 100644 --- a/.gitignore +++ b/.gitignore @@ -142,6 +142,7 @@ backups/ # Environments .env +.env.tmp.* .venv env/ venv/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3cefeb721..ab9ea1506 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -165,9 +165,16 @@ repos: files: '^frontend/.*\.(js|ts|vue|json|md|yml|css)$' pass_filenames: false + - id: frontend-workflow-format + name: Check GitHub workflow formatting + entry: bash -c 'cd frontend && npm run check:workflow-format' + language: system + files: '^\.github/workflows/.*\.yml$' + pass_filenames: false + - id: codesight-requirements - name: Generate requirements.txt for codesight - entry: bash -c 'set -euo pipefail; echo "# autogenerated from poetry.lock via poetry export" > requirements.txt && poetry export --without-hashes --without-urls --all-groups --all-extras -f requirements.txt >> requirements.txt && git add requirements.txt' + name: Check requirements.txt for codesight + entry: bash scripts/check_requirements.sh language: system pass_filenames: false always_run: true diff --git a/apps/workflow/tests/test_xero_instance_templates.py b/apps/workflow/tests/test_xero_instance_templates.py index f9d0982fb..4c821bfb2 100644 --- a/apps/workflow/tests/test_xero_instance_templates.py +++ b/apps/workflow/tests/test_xero_instance_templates.py @@ -1,4 +1,6 @@ import json +import subprocess +import tempfile from pathlib import Path from django.test import SimpleTestCase @@ -11,12 +13,19 @@ REPO_ROOT / "scripts" / "server" / "templates" / "xero-apps.json.template" ) INSTANCE_SCRIPT = REPO_ROOT / "scripts" / "server" / "instance.sh" +DEPLOY_SCRIPT = REPO_ROOT / "scripts" / "server" / "deploy.sh" +COMMON_SCRIPT = REPO_ROOT / "scripts" / "server" / "common.sh" +SERVER_SETUP_SCRIPT = REPO_ROOT / "scripts" / "server" / "server-setup.sh" +SERVER_README = REPO_ROOT / "scripts" / "server" / "README.md" +PRODUCTION_SETUP_DOC = REPO_ROOT / "docs" / "instance-setup-production.md" +DEMO_SETUP_DOC = REPO_ROOT / "docs" / "instance-setup-demo.md" class XeroInstanceTemplateTests(SimpleTestCase): def test_credentials_template_includes_xero_oauth_env_vars(self): content = CREDENTIALS_TEMPLATE.read_text() + self.assertIn("XERO_DEFAULT_USER_ID=", content) self.assertIn("XERO_CLIENT_ID=", content) self.assertIn("XERO_CLIENT_SECRET=", content) self.assertIn("XERO_WEBHOOK_KEY=", content) @@ -72,3 +81,192 @@ def test_instance_script_requires_and_loads_xero_app_fixture(self): self.assertNotIn( 'rm -f "$INSTANCE_DIR/apps/workflow/fixtures/xero_apps.json"', content ) + + def test_instance_script_requires_xero_default_user_id(self) -> None: + content = INSTANCE_SCRIPT.read_text() + + self.assertIn('[[ -z "${XERO_DEFAULT_USER_ID:-}" ]]', content) + self.assertIn('MISSING+=("XERO_DEFAULT_USER_ID")', content) + self.assertNotIn("UNCONFIGURED_XERO_DEFAULT_USER_ID", content) + + def test_instance_script_exposes_reconfigure_as_convergent_command(self) -> None: + content = INSTANCE_SCRIPT.read_text() + + self.assertIn("instance.sh reconfigure ", content) + self.assertIn("do_reconfigure()", content) + self.assertIn("do_configure false reconfigure", content) + self.assertIn("reconfigure) do_reconfigure", content) + + def test_instance_script_rerenders_env_preserving_generated_values(self) -> None: + content = INSTANCE_SCRIPT.read_text() + + self.assertIn("render_instance_env()", content) + self.assertIn( + 'db_password="$(read_env_value "$env_file" DB_PASSWORD)"', content + ) + self.assertIn( + 'test_db_password="$(read_env_value "$env_file" TEST_DB_PASSWORD)"', + content, + ) + self.assertIn('secret_key="$(read_env_value "$env_file" SECRET_KEY)"', content) + self.assertIn( + 'bearer_secret="$(read_env_value "$env_file" BEARER_SECRET)"', + content, + ) + self.assertIn('tmp_env="$(mktemp "$instance_dir/.env.tmp.XXXXXX")"', content) + self.assertIn('mv "$tmp_env" "$env_file"', content) + self.assertNotIn(".env already exists — skipping", content) + self.assertIn( + 'DB_PASSWORD="$(read_env_value "$INSTANCE_DIR/.env" DB_PASSWORD)"', + content, + ) + self.assertNotIn('DB_PASSWORD="$(. "$INSTANCE_DIR/.env"', content) + + def test_instance_script_only_seeds_missing_db_config(self) -> None: + content = INSTANCE_SCRIPT.read_text() + + self.assertIn( + "AIProvider already configured; skipping ai_providers.json load", content + ) + self.assertIn("if AIProvider.objects.exists()", content) + self.assertIn( + "XeroApp already configured; skipping xero_apps.json load", content + ) + self.assertIn("if XeroApp.objects.exists()", content) + self.assertNotIn( + "python manage.py loaddata apps/workflow/fixtures/ai_providers.json", + content, + ) + + def test_instance_script_rejects_seed_for_existing_checkout(self) -> None: + content = INSTANCE_SCRIPT.read_text() + + self.assertIn('[[ -d "$INSTANCE_DIR/.git" && "$SEED" == "true" ]]', content) + self.assertIn("--seed is only valid when creating a new instance", content) + + def test_credentials_file_stays_root_owned_before_root_source(self) -> None: + common_content = COMMON_SCRIPT.read_text() + instance_content = INSTANCE_SCRIPT.read_text() + deploy_content = DEPLOY_SCRIPT.read_text() + server_setup_content = SERVER_SETUP_SCRIPT.read_text() + + self.assertIn("require_root_owned_credentials_file()", common_content) + self.assertIn("stat -c '%u:%g:%a' \"$creds_file\"", common_content) + self.assertIn('"0:0:600"', common_content) + self.assertIn('[[ -L "$creds_file" ]]', common_content) + self.assertIn("ensure_config_dir()", common_content) + self.assertIn("stat -c '%u:%g:%a' \"$config_dir\"", common_content) + self.assertIn('"0:0:755"', common_content) + self.assertIn('[[ -L "$CONFIG_DIR" ]]', common_content) + self.assertIn('[[ -L "$config_dir" ]]', common_content) + + self.assertIn('chown root:root "$CREDS_FILE"', instance_content) + self.assertIn( + 'require_root_owned_credentials_file "$creds_file"', + instance_content, + ) + self.assertIn( + 'require_root_owned_credentials_file "$CREDS_FILE"', + instance_content, + ) + self.assertNotIn( + 'chown "$INSTANCE_USER:$INSTANCE_USER" "$CREDS_FILE"', + instance_content, + ) + + self.assertIn( + 'require_root_owned_credentials_file "$creds_file"', + deploy_content, + ) + self.assertIn("chown root:root /opt/docketworks/config", server_setup_content) + self.assertIn("chmod 755 /opt/docketworks/config", server_setup_content) + + def test_node_major_parsing_accepts_patch_versions(self) -> None: + common_content = COMMON_SCRIPT.read_text() + deploy_content = DEPLOY_SCRIPT.read_text() + server_setup_content = SERVER_SETUP_SCRIPT.read_text() + + self.assertIn("node_major_from_nvmrc()", common_content) + self.assertIn("node_major_from_nvmrc()", server_setup_content) + self.assertIn( + "sed -nE 's/^[[:space:]]*v?([0-9]+).*/\\1/p'", + common_content, + ) + self.assertIn( + 'REQUIRED_NODE_MAJOR="$(node_major_from_nvmrc ' + '"$LOCAL_REPO/frontend/.nvmrc")', + deploy_content, + ) + self.assertIn( + 'REQUIRED_NODE_MAJOR="$(node_major_from_nvmrc ' + '"$LOCAL_REPO/frontend/.nvmrc")', + server_setup_content, + ) + self.assertNotIn("tr -d 'v[:space:]'", deploy_content) + self.assertNotIn("tr -d 'v[:space:]'", server_setup_content) + + for nvmrc_value in ["18", "v18", "18.2.0", "v18.2.0", " v18.2.0"]: + with tempfile.NamedTemporaryFile("w", encoding="utf-8") as nvmrc: + nvmrc.write(nvmrc_value) + nvmrc.flush() + + result = subprocess.run( + [ + "bash", + "-c", + 'source "$1"; node_major_from_nvmrc "$2"', + "_", + str(COMMON_SCRIPT), + nvmrc.name, + ], + check=True, + capture_output=True, + text=True, + ) + + self.assertEqual(result.stdout.strip(), "18") + + def test_instance_mediafiles_are_owned_for_app_writes_and_nginx_reads( + self, + ) -> None: + content = INSTANCE_SCRIPT.read_text() + + self.assertIn( + 'chown "$INSTANCE_USER:www-data" "$INSTANCE_DIR/mediafiles"', + content, + ) + self.assertIn('chmod 750 "$INSTANCE_DIR/mediafiles"', content) + + def test_xero_default_user_id_docs_match_required_create_time_workflow( + self, + ) -> None: + docs = "\n".join( + [ + CREDENTIALS_TEMPLATE.read_text(), + SERVER_README.read_text(), + PRODUCTION_SETUP_DOC.read_text(), + DEMO_SETUP_DOC.read_text(), + ] + ) + + self.assertIn("XERO_DEFAULT_USER_ID must be present", docs) + self.assertIn("required before `instance.sh create`", docs) + self.assertNotIn("leave blank for now", docs) + self.assertNotIn("Create the instance first", docs) + self.assertNotIn("copy that UUID", docs) + self.assertNotIn("Copy the relevant user ID into credentials.env", docs) + self.assertNotIn("then run `instance.sh reconfigure`", docs) + + def test_deploy_restores_typed_router_after_drift_detection(self) -> None: + content = DEPLOY_SCRIPT.read_text() + + self.assertIn( + "server generated a different frontend/src/typed-router.d.ts", + content, + ) + self.assertIn( + 'git -C "$instance_dir" restore --source=HEAD -- ' + "frontend/src/typed-router.d.ts", + content, + ) + self.assertIn('FAILED_INSTANCES+=("$instance")', content) diff --git a/docs/client_onboarding.md b/docs/client_onboarding.md index d073aaa98..7b257308d 100644 --- a/docs/client_onboarding.md +++ b/docs/client_onboarding.md @@ -99,7 +99,7 @@ The client needs a Xero subscription. DocketWorks handles jobs and delegates inv 6. Create the app, copy **Client ID** and **Client Secret** 7. Under Webhooks, create a subscription, copy the **Webhook Key** -These go into the instance's `credentials.env`. +These go into the instance's root-owned `credentials.env`. --- @@ -216,7 +216,7 @@ Follow `uat_setup.md` (Part C) or the production deployment process. ```bash # UAT sudo scripts/server/instance.sh prepare-config -# Fill credentials.env with Xero values +sudoedit /opt/docketworks/config/-.credentials.env sudo scripts/server/instance.sh create ``` @@ -298,10 +298,10 @@ Upload the company logo and wide/letterhead logo via Admin > Settings > Company | Information | Destination | |------------|-------------| -| Xero Client ID / Secret / Webhook Key | `credentials.env` | -| GCP service account JSON key path | `.env` (`GCP_CREDENTIALS`) | -| Google Maps API key | `.env` or `shared.env` (`GOOGLE_MAPS_API_KEY`) | -| Email SMTP credentials | `.env` or `shared.env` | +| Xero Client ID / Secret / Webhook Key | root-owned `credentials.env` | +| GCP service account JSON key path | root-owned `credentials.env` (`GCP_CREDENTIALS`) | +| Google Maps API key | `shared.env` (`GOOGLE_MAPS_API_KEY`) | +| Email SMTP credentials | root-owned `credentials.env` | | Supplier credentials (Steel & Tube) | `.env` | | Company details, rates, markups, hours | CompanyDefaults (Admin > Settings) | | Google Drive folder IDs | CompanyDefaults (Admin > Settings) | diff --git a/docs/instance-setup-demo.md b/docs/instance-setup-demo.md index 178811e02..c78ba5b9e 100644 --- a/docs/instance-setup-demo.md +++ b/docs/instance-setup-demo.md @@ -18,11 +18,22 @@ Onboard a prospect for a paid trial of DocketWorks. Uses dummy staff but the pro sudo scripts/server/instance.sh prepare-config uat ``` -Fill in `/opt/docketworks/instances/-uat/credentials.env`: +Edit the root-owned credentials file: + +```bash +sudoedit /opt/docketworks/config/-uat.credentials.env +``` + +Fill in: +- XERO_DEFAULT_USER_ID — the existing Xero Demo Company login/user ID that will own time entries - GCP_CREDENTIALS — shared dev service account key - EMAIL credentials -(The Xero Client ID, Client Secret, and Webhook Key for the **Xero Demo Company** app go into the `xero_apps.json` fixture in Step 3.5, not `credentials.env`.) +XERO_DEFAULT_USER_ID must be present before `instance.sh create` runs. + +Also fill in the Xero Client ID, Client Secret, Webhook Key, and Redirect URI +for the **Xero Demo Company** app. `instance.sh create` uses these values to +render and load the initial XeroApp fixture. ## Step 2: Create Instance @@ -47,23 +58,8 @@ scripts/server/dw-run.sh -uat python manage.py loaddata apps/workflow/fi scripts/server/dw-run.sh -uat python scripts/restore_checks/check_company_defaults.py ``` -## Step 3.5: Load Xero App Credentials +## Step 3.5: Check Xero App Credentials -Copy the example fixture and fill in the **Xero Demo Company** app's Client ID, Client Secret, Redirect URI, and Webhook Key. Set `label` to something identifiable like `-uat xero`. - -```bash -# instance.sh creates the checkout directly at /opt/docketworks/instances// -# (no /docketworks suffix) and the OS user as dw__ (underscores — -# matches the DB role; see scripts/server/common.sh:instance_user). -INSTANCE_DIR=/opt/docketworks/instances/-uat -sudo -u dw__uat cp \ - $INSTANCE_DIR/apps/workflow/fixtures/xero_apps.json.example \ - $INSTANCE_DIR/apps/workflow/fixtures/xero_apps.json -sudo -u dw__uat $EDITOR $INSTANCE_DIR/apps/workflow/fixtures/xero_apps.json -scripts/server/dw-run.sh -uat python manage.py loaddata apps/workflow/fixtures/xero_apps.json -``` - -**Check:** ```bash scripts/server/dw-run.sh -uat python scripts/restore_checks/check_xero_app.py ``` diff --git a/docs/instance-setup-production.md b/docs/instance-setup-production.md index 5bab5ab26..9f347a20f 100644 --- a/docs/instance-setup-production.md +++ b/docs/instance-setup-production.md @@ -14,12 +14,22 @@ Set up a production instance for a client connecting to their real Xero organisa sudo scripts/server/instance.sh prepare-config prod ``` -Fill in `/opt/docketworks/instances/-prod/credentials.env`: -- XERO_DEFAULT_USER_ID (leave blank for now — set after Step 7) +Edit the root-owned credentials file: + +```bash +sudoedit /opt/docketworks/config/-prod.credentials.env +``` + +Fill in: +- XERO_DEFAULT_USER_ID — the existing Xero login/user ID that will own time entries - GCP_CREDENTIALS path (from Phase 3a of client_onboarding.md) - EMAIL_HOST_USER + EMAIL_HOST_PASSWORD -(The Xero Client ID, Client Secret, and Webhook Key go into the `xero_apps.json` fixture in Step 2.5, not `credentials.env`.) +XERO_DEFAULT_USER_ID must be present before `instance.sh create` runs. + +Also fill in the Xero Client ID, Client Secret, Webhook Key, and Redirect URI +from the client's Xero app. `instance.sh create` uses these values to render +and load the initial XeroApp fixture. ## Step 2: Create Instance @@ -27,27 +37,12 @@ Fill in `/opt/docketworks/instances/-prod/credentials.env`: sudo scripts/server/instance.sh create prod ``` -Creates: OS user, database, .env, code clone, frontend build, migrations, admin user, gunicorn service, nginx config. +Creates: OS user, database, .env, code clone, frontend build, migrations, admin user, systemd services (gunicorn + celery), nightly backup timer, and nginx config. **Check:** `https://-prod.docketworks.site` shows login page. -## Step 2.5: Load Xero App Credentials - -Copy the example fixture and fill in the client's prod Xero app's Client ID, Client Secret, Redirect URI, and Webhook Key (all from Phase 2b of client_onboarding.md). Set `label` to `-prod xero`. - -```bash -# instance.sh creates the checkout directly at /opt/docketworks/instances// -# (no /docketworks suffix) and the OS user as dw__ (underscores — -# matches the DB role; see scripts/server/common.sh:instance_user). -INSTANCE_DIR=/opt/docketworks/instances/-prod -sudo -u dw__prod cp \ - $INSTANCE_DIR/apps/workflow/fixtures/xero_apps.json.example \ - $INSTANCE_DIR/apps/workflow/fixtures/xero_apps.json -sudo -u dw__prod $EDITOR $INSTANCE_DIR/apps/workflow/fixtures/xero_apps.json -scripts/server/dw-run.sh -prod python manage.py loaddata apps/workflow/fixtures/xero_apps.json -``` +## Step 2.5: Check Xero App Credentials -**Check:** ```bash scripts/server/dw-run.sh -prod python scripts/restore_checks/check_xero_app.py ``` @@ -115,15 +110,6 @@ scripts/server/dw-run.sh -prod python manage.py xero --import-staff This creates Staff records from Xero Payroll employees with wage rates and working hours. All imported staff get `password_needs_reset=True`. -After import, find the default Xero user ID for timesheets: -```bash -scripts/server/dw-run.sh -prod python manage.py xero --users -``` -Copy the relevant user ID into credentials.env as XERO_DEFAULT_USER_ID, then re-run create to update .env: -```bash -sudo scripts/server/instance.sh create prod -``` - ## Step 8: Create Shop Jobs ```bash diff --git a/docs/restore-prod-to-nonprod.md b/docs/restore-prod-to-nonprod.md index 3c4b48c45..035842b22 100644 --- a/docs/restore-prod-to-nonprod.md +++ b/docs/restore-prod-to-nonprod.md @@ -400,4 +400,6 @@ gunzip -c "$LATEST" | PGPASSWORD="$DB_PASSWORD" psql \ ## First-time setup (existing instances only) -New instances pick up the scrub DB automatically via `scripts/server/instance.sh`. Existing instances provisioned before this change need a one-off `instance.sh create` re-run (idempotent — adds the scrub DB, skips anything that already exists). +New instances pick up the scrub DB automatically via `scripts/server/instance.sh`. +Existing instances provisioned before this change need one `instance.sh reconfigure` +run. diff --git a/docs/server_setup.md b/docs/server_setup.md index 223c2210a..1019e76fa 100644 --- a/docs/server_setup.md +++ b/docs/server_setup.md @@ -117,12 +117,15 @@ Certs auto-renew via `certbot renew` using the same Dreamhost DNS hooks. # Step 1: scaffold credentials file sudo scripts/server/instance.sh prepare-config -# Step 2: fill in the credentials -sudo nano /opt/docketworks/instances/-/credentials.env +# Step 2: fill in the root-owned credentials +sudoedit /opt/docketworks/config/-.credentials.env # Step 3: create the instance sudo scripts/server/instance.sh create +# Re-run after root-owned credential/config edits +sudo scripts/server/instance.sh reconfigure + # Or with demo fixtures: sudo scripts/server/instance.sh create --seed ``` @@ -290,11 +293,7 @@ For a prospect trying DocketWorks with their own Xero: ### Deploy (update to latest code) -```bash -sudo scripts/server/deploy.sh -``` - -This pulls latest code, installs dependencies, runs migrations, rebuilds frontend, and restarts Gunicorn. +See [updating.md](updating.md) — the deploy runbook (the `deploy.sh` command, and when to also run `instance.sh reconfigure`). ### Backups @@ -309,6 +308,10 @@ put the folder ID in `BACKUP_GDRIVE_ROOT_FOLDER_ID` in `/opt/docketworks/config/.credentials.env`; create/deploy writes `/opt/docketworks/config/rclone/.conf`. +The credentials file is a root-owned operator input (`root:root`, mode 600). +Edit it with `sudoedit`; do not hand ownership to the instance user, because +root-run orchestration sources the file. + Smoke test: ```bash @@ -420,17 +423,7 @@ ssh-keygen -t ed25519 -C "github-actions-uat" -f uat_deploy_key -N "" **Step 1 (automatic):** On push to `main`, `deploy-uat.yml` SSHes into the server as `docketworks` and pulls the latest code into `/opt/docketworks/repo`. This only updates the shared repo — no instances are touched. -**Step 2 (manual):** When ready to deploy to instances, SSH into the server and run: - -```bash -# Deploy all instances -sudo ./scripts/server/deploy.sh --all - -# Or a single instance -sudo ./scripts/server/deploy.sh -``` - -This updates shared Python/Node deps, then for each instance: builds frontend, runs migrate, restarts Gunicorn. +**Step 2 (manual):** When ready to deploy to instances, follow the deploy runbook in [updating.md](updating.md). ### Install log diff --git a/docs/updating.md b/docs/updating.md index 94946d962..a5065b645 100644 --- a/docs/updating.md +++ b/docs/updating.md @@ -28,23 +28,25 @@ If you're running the application locally for development: ## Server Environment (Multi-Tenant) -See [server_setup.md](server_setup.md) for full architecture details. +**This section is the deploy runbook.** PR merged to `main`? SSH into the server and run: -Deployment is a two-step process: +```bash +# One client +sudo ./scripts/server/deploy.sh - -1. **Merge a PR to `main`** — GitHub Actions (`deploy-uat.yml`) automatically pulls the latest code into `/opt/docketworks/repo` on the server. +# Or all instances +sudo ./scripts/server/deploy.sh --all +``` -2. **Deploy to instances** — SSH into the server and run: +That's it for a normal code release. `deploy.sh` pulls `main` itself, then for each instance takes a pre-deploy DB backup, runs migrations, rebuilds the frontend, and restarts its services — you don't run anything per service. - ```bash - # All instances - sudo ./scripts/server/deploy.sh --all +**Only if the release changed per-instance config** that `deploy.sh` does not re-render — a new `.env` variable, or a change to the gunicorn systemd unit — also run, once per instance: - # Single instance - sudo ./scripts/server/deploy.sh - ``` +```bash +sudo ./scripts/server/instance.sh reconfigure +``` - This updates shared Python/Node deps, then for each instance: builds frontend, runs migrate, restarts Gunicorn. +For architecture, see [server_setup.md](server_setup.md); for the exact internal deploy sequence, see [scripts/server/README.md](../scripts/server/README.md). ## Troubleshooting diff --git a/frontend/package.json b/frontend/package.json index 6926259b0..fba0e36db 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -13,8 +13,10 @@ "format": "prettier --write src/", "gen:api": "node ./scripts/gen-api.js", "check:api-contract": "node ./scripts/check-api-contract-boundary.js", + "check:workflow-format": "prettier --config .prettierrc.json --check ../.github/workflows/*.yml", "gen:typed-router": "tsx ./scripts/generate-typed-router.ts", "check:typed-router": "tsx ./scripts/generate-typed-router.ts --check", + "check:typed-router:build": "npm run check:typed-router && npm run build && git diff --exit-code -- src/typed-router.d.ts", "update-schema": "npx prettier --write schema.yml && npm run gen:api", "test": "vitest", "test:unit": "vitest run", diff --git a/requirements.txt b/requirements.txt index 099895cb6..e177db180 100644 --- a/requirements.txt +++ b/requirements.txt @@ -76,7 +76,7 @@ google-api-python-client-stubs==1.36.0 ; python_version >= "3.12" and python_ver google-api-python-client==2.195.0 ; python_version >= "3.12" and python_version < "4.0" google-auth-httplib2==0.3.1 ; python_version >= "3.12" and python_version < "4.0" google-auth-stubs==0.3.0 ; python_version >= "3.12" and python_version < "4.0" -google-auth==2.54.0 ; python_version >= "3.12" and python_version < "4.0" +google-auth==2.53.0 ; python_version >= "3.12" and python_version < "4.0" google-genai==1.74.0 ; python_version >= "3.12" and python_version < "4.0" google-generativeai==0.8.6 ; python_version >= "3.12" and python_version < "4.0" googleapis-common-protos==1.74.0 ; python_version >= "3.12" and python_version < "4.0" @@ -86,7 +86,7 @@ grpcio==1.80.0 ; python_version >= "3.12" and python_version < "4.0" gunicorn==26.0.0 ; python_version >= "3.12" and python_version < "4.0" h11==0.16.0 ; python_version >= "3.12" and python_version < "4.0" hf-xet==1.4.3 ; python_version >= "3.12" and python_version < "3.14" and (platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "arm64" or platform_machine == "aarch64") -holidays==0.94 ; python_version >= "3.12" and python_version < "4.0" +holidays==0.98 ; python_version >= "3.12" and python_version < "4.0" httpcore==1.0.9 ; python_version >= "3.12" and python_version < "4.0" httplib2==0.31.2 ; python_version >= "3.12" and python_version < "4.0" httpx-sse==0.4.3 ; python_version >= "3.12" and python_version < "4.0" @@ -94,7 +94,7 @@ httpx==0.28.1 ; python_version >= "3.12" and python_version < "4.0" huggingface-hub==1.13.0 ; python_version >= "3.12" and python_version < "3.14" identify==2.6.19 ; python_version >= "3.12" and python_version < "4.0" idna==3.18 ; python_version >= "3.12" and python_version < "4.0" -importlib-metadata==9.0.0 ; python_version >= "3.12" and python_version < "4.0" +importlib-metadata==8.7.1 ; python_version >= "3.12" and python_version < "4.0" inflection==0.5.1 ; python_version >= "3.12" and python_version < "4.0" iniconfig==2.3.0 ; python_version >= "3.12" and python_version < "4.0" isort==8.0.1 ; python_version >= "3.12" and python_version < "4.0" @@ -127,7 +127,7 @@ numpy==2.4.4 ; python_version >= "3.12" and python_version < "4.0" openai==2.24.0 ; python_version >= "3.12" and python_version < "3.14" openpyxl==3.1.5 ; python_version >= "3.12" and python_version < "4.0" opentelemetry-api==1.39.1 ; python_version >= "3.12" and python_version < "4.0" -opentelemetry-semantic-conventions==0.63b1 ; python_version >= "3.12" and python_version < "4.0" +opentelemetry-semantic-conventions==0.60b1 ; python_version >= "3.12" and python_version < "4.0" outcome==1.3.0.post0 ; python_version >= "3.12" and python_version < "4.0" packaging==26.2 ; python_version >= "3.12" and python_version < "4.0" pandas-stubs==3.0.3.260530 ; python_version >= "3.12" and python_version < "4.0" @@ -163,7 +163,7 @@ pylint-plugin-utils==0.9.0 ; python_version >= "3.12" and python_version < "4.0" pylint==4.0.5 ; python_version >= "3.12" and python_version < "4.0" pyngrok==7.5.1 ; python_version >= "3.12" and python_version < "4.0" pyparsing==3.3.2 ; python_version >= "3.12" and python_version < "4.0" -pypdf==6.12.0 ; python_version >= "3.12" and python_version < "4.0" +pypdf==6.10.2 ; python_version >= "3.12" and python_version < "4.0" pypdfium2==5.7.1 ; python_version >= "3.12" and python_version < "4.0" pyproject-api==1.10.0 ; python_version >= "3.12" and python_version < "4.0" pysocks==1.7.1 ; python_version >= "3.12" and python_version < "4.0" @@ -182,7 +182,7 @@ pytz==2026.2 ; python_version >= "3.12" and python_version < "4.0" pywin32==311 ; python_version >= "3.12" and python_version < "4.0" and (platform_system == "Windows" or sys_platform == "win32") pyyaml==6.0.3 ; python_version >= "3.12" and python_version < "4.0" rapidfuzz==3.14.5 ; python_version >= "3.12" and python_version < "4.0" -redis==8.0.0 ; python_version >= "3.12" and python_version < "4.0" +redis==7.4.0 ; python_version >= "3.12" and python_version < "4.0" referencing==0.37.0 ; python_version >= "3.12" and python_version < "4.0" regex==2026.4.4 ; python_version >= "3.12" and python_version < "3.14" reportlab==4.5.0 ; python_version >= "3.12" and python_version < "4.0" diff --git a/scripts/check_requirements.sh b/scripts/check_requirements.sh new file mode 100755 index 000000000..ad36cbdcf --- /dev/null +++ b/scripts/check_requirements.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Check requirements.txt is the deterministic export of poetry.lock. +set -euo pipefail + +repo_root="$(git rev-parse --show-toplevel)" +cd "$repo_root" + +tmp="$(mktemp)" +trap 'rm -f "$tmp"' EXIT + +{ + echo "# autogenerated from poetry.lock via poetry export" + poetry export --without-hashes --without-urls --all-groups --all-extras -f requirements.txt +} > "$tmp" + +if cmp -s "$tmp" requirements.txt; then + exit 0 +fi + +echo "ERROR: requirements.txt is stale for poetry.lock." >&2 +echo "Run: scripts/generate_requirements.sh" >&2 +echo "" >&2 +echo "Poetry version:" >&2 +poetry --version >&2 || true +echo "" >&2 +echo "Diff:" >&2 +diff -u requirements.txt "$tmp" >&2 || true +exit 1 diff --git a/scripts/generate_requirements.sh b/scripts/generate_requirements.sh new file mode 100755 index 000000000..bac24bc9e --- /dev/null +++ b/scripts/generate_requirements.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Regenerate requirements.txt from poetry.lock for Codesight. +set -euo pipefail + +repo_root="$(git rev-parse --show-toplevel)" +cd "$repo_root" + +tmp="$(mktemp)" +trap 'rm -f "$tmp"' EXIT + +{ + echo "# autogenerated from poetry.lock via poetry export" + poetry export --without-hashes --without-urls --all-groups --all-extras -f requirements.txt +} > "$tmp" + +if ! cmp -s "$tmp" requirements.txt; then + mv "$tmp" requirements.txt + trap - EXIT +fi diff --git a/scripts/pre-push b/scripts/pre-push index 5d24b2b94..2e14dc815 100755 --- a/scripts/pre-push +++ b/scripts/pre-push @@ -3,9 +3,12 @@ set -e bash "$(git rev-parse --show-toplevel)/scripts/check_mypy.sh" +bash "$(git rev-parse --show-toplevel)/scripts/check_requirements.sh" cd "$(git rev-parse --show-toplevel)/frontend" npm run check:typed-router npm run test:unit npm run lint npm run type-check +npm run check:typed-router:build +npm run check:workflow-format diff --git a/scripts/server/README.md b/scripts/server/README.md index 0340ae162..5bc8e1ae0 100644 --- a/scripts/server/README.md +++ b/scripts/server/README.md @@ -58,11 +58,14 @@ Two-step process: # Step 1: creates the credentials file from template sudo ./scripts/server/instance.sh prepare-config mycompany uat -# Fill out the credentials file (see "Xero Setup" below) -sudo vi /opt/docketworks/config/mycompany-uat.credentials.env +# Fill out the root-owned credentials file (see "Xero Setup" below) +sudoedit /opt/docketworks/config/mycompany-uat.credentials.env # Step 2: reads credentials, creates everything sudo ./scripts/server/instance.sh create mycompany uat + +# Re-run after root-owned credential/config edits +sudo ./scripts/server/instance.sh reconfigure mycompany uat ``` Add `--seed` to load demo fixture data: @@ -79,38 +82,34 @@ The credentials file needs: ``` XERO_DEFAULT_USER_ID= +XERO_CLIENT_ID= +XERO_CLIENT_SECRET= +XERO_WEBHOOK_KEY= +XERO_REDIRECT_URI= GCP_CREDENTIALS= EMAIL_HOST_USER= EMAIL_HOST_PASSWORD= ``` -Xero client_id, client_secret, and webhook_key live on the XeroApp model -(loaded from `apps/workflow/fixtures/xero_apps.json` or set via the Xero -Apps admin UI), not in the credentials file. +Xero client_id, client_secret, webhook_key, and redirect URI are also required +in the credentials file. `instance.sh create` renders them into the XeroApp +bootstrap fixture and only loads that fixture when no XeroApp exists yet. How to get them: 1. **Create a Xero app** at https://developer.xero.com/app/manage 2. **Set redirect URI** to `https://.docketworks.site/api/xero/oauth/callback/` -3. **Copy Client ID, Client Secret, and webhook signing key** into either - `apps/workflow/fixtures/xero_apps.json` (copy from `.example` first) - or paste them via Admin → Xero Apps after deploy. -4. **XERO_DEFAULT_USER_ID:** Create the instance first (it will work without Xero initially), create a Staff member in the app's admin, then copy that staff member's UUID into the credentials file and re-run create +3. **Copy Client ID, Client Secret, and webhook signing key** into the instance credentials file. +4. **XERO_DEFAULT_USER_ID:** Use the existing Xero login/user ID that will own time entries. This value is required before `instance.sh create`; do not leave it blank for a first create. 5. **GCP_CREDENTIALS:** Path to a GCP service account JSON key file. Each instance needs its own service account to isolate tenant data. The key file is copied into the instance directory during creation. 6. **BACKUP_GDRIVE_ROOT_FOLDER_ID:** Optional Google Drive folder ID for the backup parent. Share that folder with the service account. Backups upload under `dw_backups//` from that root. 7. **EMAIL_HOST_USER + EMAIL_HOST_PASSWORD:** Gmail address and app password for this instance's outgoing email (password resets, notifications). Generate an app password at Google Account → Security → App passwords. ## Deploying Updates -```bash -# Single instance -sudo ./scripts/server/deploy.sh mycompany-uat - -# All instances -sudo ./scripts/server/deploy.sh --all -``` +Operator runbook (the commands to run): [docs/updating.md](../../docs/updating.md). -What deploy does, in order: +What `deploy.sh` does, in order: 1. Pull latest code from GitHub (into the shared local repo). 2. Run `server-setup.sh` to converge host-level deps. Cheap when nothing's missing; lands new system deps automatically when a future PR adds them. 3. Update shared Python/Node deps. @@ -156,7 +155,7 @@ Shows each instance's name, status (running/stopped/no service), git branch, and ├── package.json # Shared node_modules ├── certbot-hooks/ # Dreamhost DNS challenge scripts ├── config/ -│ ├── .credentials.env # Xero + GCP + email secrets (survives destroy) +│ ├── .credentials.env # root-owned operator input (survives destroy) │ └── rclone/.conf # Per-instance backup upload config └── instances/ └── / # = git checkout (always on main) @@ -174,7 +173,7 @@ Shows each instance's name, status (running/stopped/no service), git branch, and ### How Env Vars Flow ``` -config/.credentials.env (user fills Xero + GCP + email values) +config/.credentials.env (root-owned operator input: Xero + GCP + email) ↓ instance.sh reads + validates ↓ @@ -191,6 +190,8 @@ gunicorn systemd service loads .env via EnvironmentFile= - **Shared user** `docketworks` owns the venv, repo, and shared.env - **Per-instance user** `dw-` runs gunicorn, owns the instance directory +- **Credentials input** in `/opt/docketworks/config` is `root:root` mode 600 + because `instance.sh` and `deploy.sh` source it during root-run orchestration - Instance dirs are `dw-:www-data` mode 750 — Nginx (www-data) can read static files, other instance users cannot access - `.env` files are mode 600, owner-only — even www-data can't read secrets - Each instance has its own PostgreSQL database and user @@ -201,7 +202,7 @@ gunicorn systemd service loads .env via EnvironmentFile= |------|-------------| | `common.sh` | Shared constants: domain, paths, directories | | `server-setup.sh` | Host-level convergence (packages, venv, SSL, shared config). Runs every deploy — see "Server Setup". | -| `instance.sh` | Prepare config, create, destroy, or list instances | +| `instance.sh` | Prepare config, create/reconfigure, destroy, or list instances | | `deploy.sh` | Pull updates and redeploy one or all instances | | `dw-run.sh` | Run a command in an instance's environment | | `certbot-dreamhost-auth.sh` | Certbot DNS-01 auth hook (adds TXT record via Dreamhost API) | diff --git a/scripts/server/common.sh b/scripts/server/common.sh index 40880c39f..3e2cc0868 100755 --- a/scripts/server/common.sh +++ b/scripts/server/common.sh @@ -44,6 +44,63 @@ instance_rclone_config() { echo "$RCLONE_CONFIG_DIR/$instance.conf" } +node_major_from_nvmrc() { + local nvmrc_file="$1" + local major + major="$(sed -nE 's/^[[:space:]]*v?([0-9]+).*/\1/p' "$nvmrc_file" | head -n 1)" + if [[ -z "$major" ]]; then + echo "ERROR: Could not parse Node major from $nvmrc_file" >&2 + exit 1 + fi + printf "%s\n" "$major" +} + +ensure_config_dir() { + if [[ -L "$CONFIG_DIR" ]]; then + echo "ERROR: Credentials directory must not be a symlink: $CONFIG_DIR" >&2 + exit 1 + fi + mkdir -p "$CONFIG_DIR" + chown root:root "$CONFIG_DIR" + chmod 755 "$CONFIG_DIR" +} + +require_root_owned_credentials_file() { + local creds_file="$1" + local config_dir + config_dir="$(dirname "$creds_file")" + + if [[ ! -d "$config_dir" ]]; then + echo "ERROR: Credentials directory not found: $config_dir" >&2 + exit 1 + fi + if [[ -L "$config_dir" ]]; then + echo "ERROR: Credentials directory must not be a symlink: $config_dir" >&2 + exit 1 + fi + if [[ "$(stat -c '%u:%g:%a' "$config_dir")" != "0:0:755" ]]; then + echo "ERROR: Credentials directory must be root:root mode 755: $config_dir" >&2 + echo " Fix after auditing contents:" >&2 + echo " sudo chown root:root $config_dir && sudo chmod 755 $config_dir" >&2 + exit 1 + fi + + if [[ ! -f "$creds_file" ]]; then + echo "ERROR: Credentials file not found: $creds_file" >&2 + exit 1 + fi + if [[ -L "$creds_file" ]]; then + echo "ERROR: Credentials file must not be a symlink: $creds_file" >&2 + exit 1 + fi + if [[ "$(stat -c '%u:%g:%a' "$creds_file")" != "0:0:600" ]]; then + echo "ERROR: Credentials file must be root:root mode 600: $creds_file" >&2 + echo " Fix after auditing contents:" >&2 + echo " sudo chown root:root $creds_file && sudo chmod 600 $creds_file" >&2 + exit 1 + fi +} + write_instance_rclone_config() { local instance="$1" local instance_user="$2" diff --git a/scripts/server/deploy.sh b/scripts/server/deploy.sh index aa9ed6aec..41d7b72d0 100755 --- a/scripts/server/deploy.sh +++ b/scripts/server/deploy.sh @@ -203,11 +203,18 @@ sudo -u docketworks bash -c " # --- Update shared node_modules --- log "Updating shared node_modules..." +REQUIRED_NODE_MAJOR="$(node_major_from_nvmrc "$LOCAL_REPO/frontend/.nvmrc")" sudo -u docketworks bash -c " cp '$LOCAL_REPO/frontend/package.json' '$BASE_DIR/package.json' cp '$LOCAL_REPO/frontend/package-lock.json' '$BASE_DIR/package-lock.json' + REQUIRED_NODE_MAJOR='$REQUIRED_NODE_MAJOR' + CURRENT_NODE_MAJOR=\$(node --version | sed -E 's/^v([0-9]+).*/\1/') + if [[ \"\$CURRENT_NODE_MAJOR\" != \"\$REQUIRED_NODE_MAJOR\" ]]; then + echo \"ERROR: Node major \$CURRENT_NODE_MAJOR does not match frontend/.nvmrc (\$REQUIRED_NODE_MAJOR)\" >&2 + exit 1 + fi cd '$BASE_DIR' - npm install + npm ci --include=dev " # --- Per-instance: build, migrate, restart --- @@ -224,9 +231,18 @@ for instance in "${TARGETS[@]}"; do log " Building frontend..." sudo -u "$inst_user" bash -c " cd '$instance_dir/frontend' + npm run check:typed-router npm run build npm run manual:build " + if ! sudo -u "$inst_user" git -C "$instance_dir" diff --quiet --ignore-submodules HEAD -- frontend/src/typed-router.d.ts; then + log " ERROR: server generated a different frontend/src/typed-router.d.ts for $instance" + log " Investigate with: sudo -u $inst_user git -C $instance_dir diff -- frontend/src/typed-router.d.ts" + log " Restoring frontend/src/typed-router.d.ts so the checkout is not left dirty" + sudo -u "$inst_user" git -C "$instance_dir" restore --source=HEAD -- frontend/src/typed-router.d.ts + FAILED_INSTANCES+=("$instance") + continue + fi # Collect static files + run migrations log " Running migrate..." @@ -286,6 +302,7 @@ for instance in "${TARGETS[@]}"; do backup_root_folder_id="" creds_file="$CONFIG_DIR/$instance.credentials.env" if [[ -f "$creds_file" ]]; then + require_root_owned_credentials_file "$creds_file" backup_root_folder_id="$( bash -c 'set -a; source "$1"; printf "%s" "${BACKUP_GDRIVE_ROOT_FOLDER_ID:-}"' _ "$creds_file" )" diff --git a/scripts/server/instance.sh b/scripts/server/instance.sh index 7d311098d..d2cf706df 100755 --- a/scripts/server/instance.sh +++ b/scripts/server/instance.sh @@ -4,6 +4,7 @@ set -euo pipefail # Manage docketworks instances. # Usage: instance.sh prepare-config # instance.sh create [--seed] [--fqdn ] [--no-start] +# instance.sh reconfigure [--fqdn ] [--no-start] # instance.sh destroy # instance.sh list # @@ -74,9 +75,10 @@ do_prepare_config() { exit 0 fi - mkdir -p "$CONFIG_DIR" + ensure_config_dir sed "s|__INSTANCE__|$INSTANCE|g" "$TEMPLATE_DIR/credentials-instance.template" \ > "$CREDS_FILE" + chown root:root "$CREDS_FILE" chmod 600 "$CREDS_FILE" echo "" @@ -91,48 +93,59 @@ do_prepare_config() { echo "============================================================" } -# ============================================================ -# create -# ============================================================ -do_create() { - parse_client_env "$@" - shift 2 +read_env_value() { + local env_file="$1" + local var_name="$2" + local line value - local SEED=false - local CUSTOM_FQDN="" - local NO_START=false - local parsed - if ! parsed=$(getopt -o '' --long seed,fqdn:,no-start -n "$(basename "$0") create" -- "$@"); then - echo "Usage: $(basename "$0") create [--seed] [--fqdn ] [--no-start]" >&2 - exit 1 + if [[ ! -f "$env_file" ]]; then + printf "" + return fi - eval set -- "$parsed" - while true; do - case "$1" in - --seed) SEED=true; shift ;; - --fqdn) CUSTOM_FQDN="$2"; shift 2 ;; - --no-start) NO_START=true; shift ;; - --) shift; break ;; - esac - done - if [[ $# -gt 0 ]]; then - echo "ERROR: Unexpected arguments to 'create': $*" >&2 + if [[ ! "$var_name" =~ ^[A-Z0-9_]+$ ]]; then + echo "ERROR: Invalid env var name requested: $var_name" >&2 exit 1 fi - # --- Read instance credentials file --- - local CREDS_FILE="$CONFIG_DIR/$INSTANCE.credentials.env" - if [[ ! -f "$CREDS_FILE" ]]; then - echo "ERROR: No credentials file found at $CREDS_FILE" + line="$(grep -m1 -E "^${var_name}=" "$env_file" || true)" + if [[ -z "$line" ]]; then + printf "" + return + fi + + value="${line#*=}" + if [[ "$value" == \"*\" && "$value" == *\" ]]; then + value="${value:1:${#value}-2}" + elif [[ "$value" == \'*\' && "$value" == *\' ]]; then + value="${value:1:${#value}-2}" + fi + printf "%s" "$value" +} + +generate_secret() { + python3 -c 'import secrets; print(secrets.token_urlsafe(50))' +} + +generate_password() { + openssl rand -base64 24 | tr -d '/+=' | head -c 32 +} + +require_instance_credentials() { + local creds_file="$1" + + if [[ ! -f "$creds_file" ]]; then + echo "ERROR: No credentials file found at $creds_file" echo "" echo "Run prepare-config first:" echo " sudo $0 prepare-config $CLIENT $ENV" exit 1 fi - # Safe: edited only by the sysadmin who already has root access to run this script + require_root_owned_credentials_file "$creds_file" + + # Safe only after the root-owned/mode guard above: source executes shell. set -a - source "$CREDS_FILE" + source "$creds_file" set +a local MISSING=() @@ -155,20 +168,196 @@ do_create() { [[ -z "${XERO_REDIRECT_URI:-}" ]] && MISSING+=("XERO_REDIRECT_URI") if [[ ${#MISSING[@]} -gt 0 ]]; then - echo "ERROR: Missing required values in $CREDS_FILE:" + echo "ERROR: Missing required values in $creds_file:" for var in "${MISSING[@]}"; do echo " - $var" done exit 1 fi - # Validate GCP credentials file exists at the path provided if [[ ! -f "$GCP_CREDENTIALS" ]]; then echo "ERROR: GCP_CREDENTIALS file not found: $GCP_CREDENTIALS" - echo " Provide a valid path to a GCP service account JSON key in $CREDS_FILE" + echo " Provide a valid path to a GCP service account JSON key in $creds_file" + exit 1 + fi +} + +render_instance_env() { + local instance_dir="$1" + local instance_user="$2" + local db_name="$3" + local db_user="$4" + local scrub_db_name="$5" + local test_db_user="$6" + local fqdn="$7" + + local env_file="$instance_dir/.env" + local db_password test_db_password secret_key bearer_secret + db_password="$(read_env_value "$env_file" DB_PASSWORD)" + test_db_password="$(read_env_value "$env_file" TEST_DB_PASSWORD)" + secret_key="$(read_env_value "$env_file" SECRET_KEY)" + bearer_secret="$(read_env_value "$env_file" BEARER_SECRET)" + + [[ -n "$db_password" ]] || db_password="$(generate_password)" + [[ -n "$test_db_password" ]] || test_db_password="$(generate_password)" + [[ -n "$secret_key" ]] || secret_key="$(generate_secret)" + [[ -n "$bearer_secret" ]] || bearer_secret="$(generate_secret)" + + local ESC_XERO_DEFAULT_USER_ID + ESC_XERO_DEFAULT_USER_ID="$(sed_escape "$XERO_DEFAULT_USER_ID")" + local ESC_EMAIL_HOST_USER ESC_EMAIL_HOST_PASSWORD ESC_DJANGO_ADMINS ESC_EMAIL_BCC + ESC_EMAIL_HOST_USER="$(sed_escape "$EMAIL_HOST_USER")" + ESC_EMAIL_HOST_PASSWORD="$(sed_escape "$EMAIL_HOST_PASSWORD")" + ESC_DJANGO_ADMINS="$(sed_escape "$DJANGO_ADMINS")" + ESC_EMAIL_BCC="$(sed_escape "$EMAIL_BCC")" + local gcp_dest="$instance_dir/gcp-credentials.json" + local tmp_env + tmp_env="$(mktemp "$instance_dir/.env.tmp.XXXXXX")" + + sed \ + -e "s|__INSTANCE__|$INSTANCE|g" \ + -e "s|__DOMAIN__|$DOMAIN|g" \ + -e "s|__FQDN__|$fqdn|g" \ + -e "s|__DB_NAME__|$db_name|g" \ + -e "s|__DB_USER__|$db_user|g" \ + -e "s|__DB_PASSWORD__|$db_password|g" \ + -e "s|__SCRUB_DB_NAME__|$scrub_db_name|g" \ + -e "s|__TEST_DB_USER__|$test_db_user|g" \ + -e "s|__TEST_DB_PASSWORD__|$test_db_password|g" \ + -e "s|__SECRET_KEY__|$secret_key|g" \ + -e "s|__BEARER_SECRET__|$bearer_secret|g" \ + -e "s|__XERO_DEFAULT_USER_ID__|$ESC_XERO_DEFAULT_USER_ID|g" \ + -e "s|__GCP_CREDENTIALS_PATH__|$gcp_dest|g" \ + -e "s|__EMAIL_HOST_USER__|$ESC_EMAIL_HOST_USER|g" \ + -e "s|__EMAIL_HOST_PASSWORD__|$ESC_EMAIL_HOST_PASSWORD|g" \ + -e "s|__DJANGO_ADMINS__|$ESC_DJANGO_ADMINS|g" \ + -e "s|__EMAIL_BCC__|$ESC_EMAIL_BCC|g" \ + "$TEMPLATE_DIR/env-instance.template" > "$tmp_env" + + local shared_env="$BASE_DIR/shared.env" + if [[ ! -f "$shared_env" ]]; then + rm -f "$tmp_env" + echo "ERROR: $shared_env not found. Run server-setup.sh first." + exit 1 + fi + echo "" >> "$tmp_env" + grep '^GOOGLE_MAPS_API_KEY=' "$shared_env" >> "$tmp_env" + chown "$instance_user:$instance_user" "$tmp_env" + chmod 600 "$tmp_env" + mv "$tmp_env" "$env_file" +} + +render_frontend_env() { + local instance_dir="$1" + local instance_user="$2" + local frontend_env="$instance_dir/frontend/.env" + + log "Rendering frontend/.env from template..." + local ESC_E2E_TEST_USERNAME ESC_E2E_TEST_PASSWORD ESC_XERO_USERNAME ESC_XERO_PASSWORD + ESC_E2E_TEST_USERNAME="$(sed_escape "$E2E_TEST_USERNAME")" + ESC_E2E_TEST_PASSWORD="$(sed_escape "$E2E_TEST_PASSWORD")" + ESC_XERO_USERNAME="$(sed_escape "$XERO_USERNAME")" + ESC_XERO_PASSWORD="$(sed_escape "$XERO_PASSWORD")" + sed \ + -e "s|__INSTANCE__|$INSTANCE|g" \ + -e "s|__CLIENT__|$CLIENT|g" \ + -e "s|__DOMAIN__|$DOMAIN|g" \ + -e "s|__E2E_TEST_USERNAME__|$ESC_E2E_TEST_USERNAME|g" \ + -e "s|__E2E_TEST_PASSWORD__|$ESC_E2E_TEST_PASSWORD|g" \ + -e "s|__XERO_USERNAME__|$ESC_XERO_USERNAME|g" \ + -e "s|__XERO_PASSWORD__|$ESC_XERO_PASSWORD|g" \ + "$TEMPLATE_DIR/frontend-env-instance.template" > "$frontend_env" + chown "$instance_user:$instance_user" "$frontend_env" + chmod 600 "$frontend_env" +} + +render_ai_providers_fixture() { + local instance_dir="$1" + local instance_user="$2" + + log "Generating AI providers fixture..." + local ESC_ANTHROPIC_API_KEY ESC_GEMINI_API_KEY ESC_MISTRAL_API_KEY + ESC_ANTHROPIC_API_KEY="$(sed_escape "$ANTHROPIC_API_KEY")" + ESC_GEMINI_API_KEY="$(sed_escape "$GEMINI_API_KEY")" + ESC_MISTRAL_API_KEY="$(sed_escape "$MISTRAL_API_KEY")" + sed \ + -e "s|__ANTHROPIC_API_KEY__|$ESC_ANTHROPIC_API_KEY|g" \ + -e "s|__GEMINI_API_KEY__|$ESC_GEMINI_API_KEY|g" \ + -e "s|__MISTRAL_API_KEY__|$ESC_MISTRAL_API_KEY|g" \ + "$TEMPLATE_DIR/ai-providers.json.template" \ + > "$instance_dir/apps/workflow/fixtures/ai_providers.json" + chown "$instance_user:$instance_user" \ + "$instance_dir/apps/workflow/fixtures/ai_providers.json" + chmod 600 "$instance_dir/apps/workflow/fixtures/ai_providers.json" +} + +render_xero_apps_fixture() { + local instance_dir="$1" + local instance_user="$2" + + log "Generating Xero apps fixture..." + local ESC_XERO_CLIENT_ID ESC_XERO_CLIENT_SECRET ESC_XERO_WEBHOOK_KEY ESC_XERO_REDIRECT_URI + ESC_XERO_CLIENT_ID="$(sed_escape "$XERO_CLIENT_ID")" + ESC_XERO_CLIENT_SECRET="$(sed_escape "$XERO_CLIENT_SECRET")" + ESC_XERO_WEBHOOK_KEY="$(sed_escape "$XERO_WEBHOOK_KEY")" + ESC_XERO_REDIRECT_URI="$(sed_escape "$XERO_REDIRECT_URI")" + sed \ + -e "s|__INSTANCE__|$INSTANCE|g" \ + -e "s|__XERO_CLIENT_ID__|$ESC_XERO_CLIENT_ID|g" \ + -e "s|__XERO_CLIENT_SECRET__|$ESC_XERO_CLIENT_SECRET|g" \ + -e "s|__XERO_WEBHOOK_KEY__|$ESC_XERO_WEBHOOK_KEY|g" \ + -e "s|__XERO_REDIRECT_URI__|$ESC_XERO_REDIRECT_URI|g" \ + "$TEMPLATE_DIR/xero-apps.json.template" \ + > "$instance_dir/apps/workflow/fixtures/xero_apps.json" + chown "$instance_user:$instance_user" \ + "$instance_dir/apps/workflow/fixtures/xero_apps.json" + chmod 600 "$instance_dir/apps/workflow/fixtures/xero_apps.json" +} + +# ============================================================ +# create / reconfigure +# ============================================================ +do_configure() { + local allow_seed="$1" + local command_name="$2" + shift 2 + + parse_client_env "$@" + shift 2 + + local SEED=false + local CUSTOM_FQDN="" + local NO_START=false + local parsed + local long_opts="fqdn:,no-start" + if [[ "$allow_seed" == "true" ]]; then + long_opts="seed,$long_opts" + fi + if ! parsed=$(getopt -o '' --long "$long_opts" -n "$(basename "$0") $command_name" -- "$@"); then + if [[ "$allow_seed" == "true" ]]; then + echo "Usage: $(basename "$0") $command_name [--seed] [--fqdn ] [--no-start]" >&2 + else + echo "Usage: $(basename "$0") $command_name [--fqdn ] [--no-start]" >&2 + fi + exit 1 + fi + eval set -- "$parsed" + while true; do + case "$1" in + --seed) SEED=true; shift ;; + --fqdn) CUSTOM_FQDN="$2"; shift 2 ;; + --no-start) NO_START=true; shift ;; + --) shift; break ;; + esac + done + if [[ $# -gt 0 ]]; then + echo "ERROR: Unexpected arguments to '$command_name': $*" >&2 exit 1 fi + local CREDS_FILE="$CONFIG_DIR/$INSTANCE.credentials.env" + require_instance_credentials "$CREDS_FILE" + local INSTANCE_DIR="$INSTANCES_DIR/$INSTANCE" local INSTANCE_USER INSTANCE_USER="$(instance_user "$INSTANCE")" @@ -177,9 +366,26 @@ do_create() { local SCRUB_DB_NAME="dw_${CLIENT}_${ENV}_scrub" local TEST_DB_USER="dw_${CLIENT}_${ENV}_test" local TEST_DB_NAME="$TEST_DB_USER" + local IS_EXISTING=false + local NEEDS_APP_BOOTSTRAP=false + if [[ -d "$INSTANCE_DIR/.git" || -f "$INSTANCE_DIR/.env" ]]; then + IS_EXISTING=true + fi + if [[ ! -d "$INSTANCE_DIR/.git" ]]; then + NEEDS_APP_BOOTSTRAP=true + fi + if [[ -d "$INSTANCE_DIR/.git" && "$SEED" == "true" ]]; then + echo "ERROR: --seed is only valid when creating a new instance." >&2 + echo " Existing instance: $INSTANCE_DIR" >&2 + exit 1 + fi log "==========================================" - log "Creating docketworks instance: $INSTANCE" + if [[ "$IS_EXISTING" == "true" ]]; then + log "Reconfiguring docketworks instance: $INSTANCE" + else + log "Creating docketworks instance: $INSTANCE" + fi log " Client: $CLIENT" log " Env: $ENV" log " Directory: $INSTANCE_DIR (= git checkout)" @@ -215,35 +421,40 @@ do_create() { log "WARNING: setquota not found — install quota package: sudo apt install quota" fi - # --- Create instance directory structure --- - # Instance dir is 750 with group www-data so nginx can traverse to - # mediafiles, frontend/dist, and gunicorn.sock. - # .env and logs stay owner-only (dw__:dw__, 600/700). - # Instance users have NO supplementary groups, so dw_acme_uat cannot - # traverse dw_msm_uat's dir (not owner, not in www-data). - # Derive FQDN and cert domain local FQDN CERT_DOMAIN if [[ -n "$CUSTOM_FQDN" ]]; then FQDN="$CUSTOM_FQDN" CERT_DOMAIN="$CUSTOM_FQDN" + elif [[ "$IS_EXISTING" == "true" && -f "$INSTANCE_DIR/.fqdn" ]]; then + FQDN="$(cat "$INSTANCE_DIR/.fqdn")" + if [[ "$FQDN" == *".$DOMAIN" ]]; then + CERT_DOMAIN="$DOMAIN" + else + CERT_DOMAIN="$FQDN" + fi else FQDN="${INSTANCE}.${DOMAIN}" CERT_DOMAIN="$DOMAIN" fi - log "Creating instance directory structure..." - mkdir -p "$INSTANCE_DIR"/{logs,mediafiles,dropbox} - chown -R "$INSTANCE_USER:www-data" "$INSTANCE_DIR" + log "Ensuring instance directory structure..." + mkdir -p "$INSTANCE_DIR"/{logs,mediafiles,dropbox,phone-recordings,session-replays} + chown "$INSTANCE_USER:www-data" "$INSTANCE_DIR" chmod 750 "$INSTANCE_DIR" + chown "$INSTANCE_USER:$INSTANCE_USER" "$INSTANCE_DIR/logs" "$INSTANCE_DIR/dropbox" chmod 700 "$INSTANCE_DIR/logs" chmod 700 "$INSTANCE_DIR/dropbox" - # Lock down credentials file — not needed by www-data - chmod 600 "$CREDS_FILE" - chown "$INSTANCE_USER:$INSTANCE_USER" "$CREDS_FILE" - # Copy GCP service account key into instance dir with restricted perms + chown "$INSTANCE_USER:www-data" "$INSTANCE_DIR/mediafiles" + chmod 750 "$INSTANCE_DIR/mediafiles" + chown "$INSTANCE_USER:$INSTANCE_USER" \ + "$INSTANCE_DIR/phone-recordings" \ + "$INSTANCE_DIR/session-replays" + chmod 700 "$INSTANCE_DIR/phone-recordings" "$INSTANCE_DIR/session-replays" + require_root_owned_credentials_file "$CREDS_FILE" cp "$GCP_CREDENTIALS" "$INSTANCE_DIR/gcp-credentials.json" chown "$INSTANCE_USER:$INSTANCE_USER" "$INSTANCE_DIR/gcp-credentials.json" chmod 600 "$INSTANCE_DIR/gcp-credentials.json" + log "Writing rclone config for $INSTANCE to $(instance_rclone_config "$INSTANCE")..." write_instance_rclone_config \ "$INSTANCE" \ @@ -251,11 +462,8 @@ do_create() { "${BACKUP_GDRIVE_ROOT_FOLDER_ID:-}" echo "$FQDN" > "$INSTANCE_DIR/.fqdn" chown "$INSTANCE_USER:$INSTANCE_USER" "$INSTANCE_DIR/.fqdn" - # Symlink shared venv into instance dir so the user can `source ~/.venv/bin/activate` ln -sfn "$SHARED_VENV" "$INSTANCE_DIR/.venv" - # Create .bash_profile that activates venv and loads .env. - # .bash_profile (not .bashrc) because SSH login shells read .bash_profile. - # The home dir IS the git checkout, so no cd needed. + cat > "$INSTANCE_DIR/.bash_profile" <<'BASH_PROFILE' source ~/.venv/bin/activate set -a; source ~/.env; set +a @@ -263,69 +471,19 @@ BASH_PROFILE chown "$INSTANCE_USER:$INSTANCE_USER" "$INSTANCE_DIR/.bash_profile" chmod 644 "$INSTANCE_DIR/.bash_profile" - # --- Generate .env (skip if it already exists) --- - if [[ -f "$INSTANCE_DIR/.env" ]]; then - log ".env already exists — skipping (credentials preserved)." - else - local DB_PASSWORD TEST_DB_PASSWORD SECRET_KEY BEARER_SECRET - DB_PASSWORD="$(openssl rand -base64 24 | tr -d '/+=' | head -c 32)" - TEST_DB_PASSWORD="$(openssl rand -base64 24 | tr -d '/+=' | head -c 32)" - SECRET_KEY="$(python3 -c 'import secrets; print(secrets.token_urlsafe(50))')" - BEARER_SECRET="$(python3 -c 'import secrets; print(secrets.token_urlsafe(50))')" - - log "Generating .env from template..." - # Escape sed-special chars in values that come from human-edited credentials.env - local ESC_XERO_DEFAULT_USER_ID - ESC_XERO_DEFAULT_USER_ID="$(sed_escape "$XERO_DEFAULT_USER_ID")" - local ESC_EMAIL_HOST_USER ESC_EMAIL_HOST_PASSWORD ESC_DJANGO_ADMINS ESC_EMAIL_BCC - ESC_EMAIL_HOST_USER="$(sed_escape "$EMAIL_HOST_USER")" - ESC_EMAIL_HOST_PASSWORD="$(sed_escape "$EMAIL_HOST_PASSWORD")" - ESC_DJANGO_ADMINS="$(sed_escape "$DJANGO_ADMINS")" - ESC_EMAIL_BCC="$(sed_escape "$EMAIL_BCC")" - local GCP_DEST="$INSTANCE_DIR/gcp-credentials.json" - sed \ - -e "s|__INSTANCE__|$INSTANCE|g" \ - -e "s|__DOMAIN__|$DOMAIN|g" \ - -e "s|__FQDN__|$FQDN|g" \ - -e "s|__DB_NAME__|$DB_NAME|g" \ - -e "s|__DB_USER__|$DB_USER|g" \ - -e "s|__DB_PASSWORD__|$DB_PASSWORD|g" \ - -e "s|__SCRUB_DB_NAME__|$SCRUB_DB_NAME|g" \ - -e "s|__TEST_DB_USER__|$TEST_DB_USER|g" \ - -e "s|__TEST_DB_PASSWORD__|$TEST_DB_PASSWORD|g" \ - -e "s|__SECRET_KEY__|$SECRET_KEY|g" \ - -e "s|__BEARER_SECRET__|$BEARER_SECRET|g" \ - -e "s|__XERO_DEFAULT_USER_ID__|$ESC_XERO_DEFAULT_USER_ID|g" \ - -e "s|__GCP_CREDENTIALS_PATH__|$GCP_DEST|g" \ - -e "s|__EMAIL_HOST_USER__|$ESC_EMAIL_HOST_USER|g" \ - -e "s|__EMAIL_HOST_PASSWORD__|$ESC_EMAIL_HOST_PASSWORD|g" \ - -e "s|__DJANGO_ADMINS__|$ESC_DJANGO_ADMINS|g" \ - -e "s|__EMAIL_BCC__|$ESC_EMAIL_BCC|g" \ - "$TEMPLATE_DIR/env-instance.template" > "$INSTANCE_DIR/.env" - - # Append shared config: Google credentials (base-setup must have run first) - local SHARED_ENV="$BASE_DIR/shared.env" - if [[ ! -f "$SHARED_ENV" ]]; then - echo "ERROR: $SHARED_ENV not found. Run server-setup.sh first." - exit 1 - fi - echo "" >> "$INSTANCE_DIR/.env" - grep '^GOOGLE_MAPS_API_KEY=' "$SHARED_ENV" >> "$INSTANCE_DIR/.env" - log " Appended Google Maps API key from $SHARED_ENV" - chown "$INSTANCE_USER:$INSTANCE_USER" "$INSTANCE_DIR/.env" - chmod 600 "$INSTANCE_DIR/.env" - fi - - # --- Ensure databases and DB users exist (always, even if .env was preserved) --- - # Two roles per tenant, each owning only its own DB(s): - # $DB_USER → $DB_NAME (app), $SCRUB_DB_NAME (backport scrubber) - # $TEST_DB_USER → $TEST_DB_NAME (pytest) - # Test role has no CREATEDB; the test DB is pre-provisioned here so pytest - # never needs cluster-level privileges. Keeping it separate from $DB_USER - # means a misconfigured pytest run cannot reach the app DB. + log "Rendering .env from template (preserving generated secrets)..." + render_instance_env \ + "$INSTANCE_DIR" \ + "$INSTANCE_USER" \ + "$DB_NAME" \ + "$DB_USER" \ + "$SCRUB_DB_NAME" \ + "$TEST_DB_USER" \ + "$FQDN" + local DB_PASSWORD TEST_DB_PASSWORD - DB_PASSWORD="$(. "$INSTANCE_DIR/.env" && echo "$DB_PASSWORD")" - TEST_DB_PASSWORD="$(. "$INSTANCE_DIR/.env" && echo "$TEST_DB_PASSWORD")" + DB_PASSWORD="$(read_env_value "$INSTANCE_DIR/.env" DB_PASSWORD)" + TEST_DB_PASSWORD="$(read_env_value "$INSTANCE_DIR/.env" TEST_DB_PASSWORD)" if [[ -z "$TEST_DB_PASSWORD" ]]; then echo "ERROR: TEST_DB_PASSWORD missing from $INSTANCE_DIR/.env" >&2 echo " This instance was created before per-tenant test roles were added." >&2 @@ -363,13 +521,9 @@ WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '$TEST_DB_NAME')\gexec GRANT ALL PRIVILEGES ON DATABASE "$TEST_DB_NAME" TO "$TEST_DB_USER"; EOSQL - # --- Clone repo directly into instance dir (instance dir = git checkout) --- if [[ -d "$INSTANCE_DIR/.git" ]]; then - log "Code already cloned — pulling latest on main..." + log "Code already cloned — ensuring origin points at local repo." sudo -u "$INSTANCE_USER" git -C "$INSTANCE_DIR" remote set-url origin "$LOCAL_REPO" - sudo -u "$INSTANCE_USER" git -C "$INSTANCE_DIR" fetch origin - sudo -u "$INSTANCE_USER" git -C "$INSTANCE_DIR" checkout main - sudo -u "$INSTANCE_USER" git -C "$INSTANCE_DIR" pull --ff-only else log "Initialising codebase in $INSTANCE_DIR from local repo (branch: main)..." sudo -u "$INSTANCE_USER" git -C "$INSTANCE_DIR" init -b main @@ -378,112 +532,41 @@ EOSQL sudo -u "$INSTANCE_USER" git -C "$INSTANCE_DIR" checkout -f main fi - # --- Generate frontend/.env (skip if it already exists) --- - local FRONTEND_ENV="$INSTANCE_DIR/frontend/.env" - if [[ -f "$FRONTEND_ENV" ]]; then - log "frontend/.env already exists — skipping (credentials preserved)." - else - log "Generating frontend/.env from template..." - local ESC_E2E_TEST_USERNAME ESC_E2E_TEST_PASSWORD ESC_XERO_USERNAME ESC_XERO_PASSWORD - ESC_E2E_TEST_USERNAME="$(sed_escape "$E2E_TEST_USERNAME")" - ESC_E2E_TEST_PASSWORD="$(sed_escape "$E2E_TEST_PASSWORD")" - ESC_XERO_USERNAME="$(sed_escape "$XERO_USERNAME")" - ESC_XERO_PASSWORD="$(sed_escape "$XERO_PASSWORD")" - sed \ - -e "s|__INSTANCE__|$INSTANCE|g" \ - -e "s|__CLIENT__|$CLIENT|g" \ - -e "s|__DOMAIN__|$DOMAIN|g" \ - -e "s|__E2E_TEST_USERNAME__|$ESC_E2E_TEST_USERNAME|g" \ - -e "s|__E2E_TEST_PASSWORD__|$ESC_E2E_TEST_PASSWORD|g" \ - -e "s|__XERO_USERNAME__|$ESC_XERO_USERNAME|g" \ - -e "s|__XERO_PASSWORD__|$ESC_XERO_PASSWORD|g" \ - "$TEMPLATE_DIR/frontend-env-instance.template" > "$FRONTEND_ENV" - chown "$INSTANCE_USER:$INSTANCE_USER" "$FRONTEND_ENV" - chmod 600 "$FRONTEND_ENV" - fi - - # --- Build frontend --- - log "Building frontend for instance $INSTANCE..." - sudo -u "$INSTANCE_USER" bash -c " - cd '$INSTANCE_DIR/frontend' - npm run build - npm run manual:build - " - - # --- Generate AI providers fixture from template --- - log "Generating AI providers fixture..." - local ESC_ANTHROPIC_API_KEY ESC_GEMINI_API_KEY ESC_MISTRAL_API_KEY - ESC_ANTHROPIC_API_KEY="$(sed_escape "$ANTHROPIC_API_KEY")" - ESC_GEMINI_API_KEY="$(sed_escape "$GEMINI_API_KEY")" - ESC_MISTRAL_API_KEY="$(sed_escape "$MISTRAL_API_KEY")" - sed \ - -e "s|__ANTHROPIC_API_KEY__|$ESC_ANTHROPIC_API_KEY|g" \ - -e "s|__GEMINI_API_KEY__|$ESC_GEMINI_API_KEY|g" \ - -e "s|__MISTRAL_API_KEY__|$ESC_MISTRAL_API_KEY|g" \ - "$TEMPLATE_DIR/ai-providers.json.template" \ - > "$INSTANCE_DIR/apps/workflow/fixtures/ai_providers.json" - chown "$INSTANCE_USER:$INSTANCE_USER" "$INSTANCE_DIR/apps/workflow/fixtures/ai_providers.json" - chmod 600 "$INSTANCE_DIR/apps/workflow/fixtures/ai_providers.json" + render_frontend_env "$INSTANCE_DIR" "$INSTANCE_USER" - # --- Run Django commands as instance user --- - log "Running Django migrate..." - "$SCRIPT_DIR/dw-run.sh" "$INSTANCE" python manage.py migrate --no-input + if [[ "$NEEDS_APP_BOOTSTRAP" == "true" ]]; then + log "Building frontend for instance $INSTANCE..." + sudo -u "$INSTANCE_USER" bash -c " + cd '$INSTANCE_DIR/frontend' + npm run build + npm run manual:build + " - # --- Load AI providers --- - log "Loading AI providers..." - "$SCRIPT_DIR/dw-run.sh" "$INSTANCE" python manage.py loaddata apps/workflow/fixtures/ai_providers.json + log "Running Django migrate..." + "$SCRIPT_DIR/dw-run.sh" "$INSTANCE" python manage.py migrate --no-input + fi - # Remove the fixture after loading — keys live in the DB now, and the - # file is a loaddata time-bomb (restore-prod-to-nonprod runs loaddata on - # it, which would overwrite real DB keys with whatever is on disk). + render_ai_providers_fixture "$INSTANCE_DIR" "$INSTANCE_USER" + log "Loading AI providers..." + "$SCRIPT_DIR/dw-run.sh" "$INSTANCE" python manage.py shell -c \ + 'from django.core.management import call_command; from apps.workflow.models import AIProvider; print("AIProvider already configured; skipping ai_providers.json load") if AIProvider.objects.exists() else call_command("loaddata", "apps/workflow/fixtures/ai_providers.json")' rm -f "$INSTANCE_DIR/apps/workflow/fixtures/ai_providers.json" - # --- Generate Xero apps fixture from template --- - log "Generating Xero apps fixture..." - local ESC_XERO_CLIENT_ID ESC_XERO_CLIENT_SECRET ESC_XERO_WEBHOOK_KEY ESC_XERO_REDIRECT_URI - ESC_XERO_CLIENT_ID="$(sed_escape "$XERO_CLIENT_ID")" - ESC_XERO_CLIENT_SECRET="$(sed_escape "$XERO_CLIENT_SECRET")" - ESC_XERO_WEBHOOK_KEY="$(sed_escape "$XERO_WEBHOOK_KEY")" - ESC_XERO_REDIRECT_URI="$(sed_escape "$XERO_REDIRECT_URI")" - sed \ - -e "s|__INSTANCE__|$INSTANCE|g" \ - -e "s|__XERO_CLIENT_ID__|$ESC_XERO_CLIENT_ID|g" \ - -e "s|__XERO_CLIENT_SECRET__|$ESC_XERO_CLIENT_SECRET|g" \ - -e "s|__XERO_WEBHOOK_KEY__|$ESC_XERO_WEBHOOK_KEY|g" \ - -e "s|__XERO_REDIRECT_URI__|$ESC_XERO_REDIRECT_URI|g" \ - "$TEMPLATE_DIR/xero-apps.json.template" \ - > "$INSTANCE_DIR/apps/workflow/fixtures/xero_apps.json" - chown "$INSTANCE_USER:$INSTANCE_USER" "$INSTANCE_DIR/apps/workflow/fixtures/xero_apps.json" - chmod 600 "$INSTANCE_DIR/apps/workflow/fixtures/xero_apps.json" - - # --- Load Xero apps --- + render_xero_apps_fixture "$INSTANCE_DIR" "$INSTANCE_USER" log "Loading Xero apps..." "$SCRIPT_DIR/dw-run.sh" "$INSTANCE" python manage.py shell -c \ 'from django.core.management import call_command; from apps.workflow.models import XeroApp; print("XeroApp already configured; skipping xero_apps.json load") if XeroApp.objects.exists() else call_command("loaddata", "apps/workflow/fixtures/xero_apps.json")' - # Intentionally NOT deleted after load: restore-prod-to-nonprod runs - # loaddata against this file (workflow_xeroapp is excluded from the - # prod dump, so the table is empty after restore and needs reseeding). - # The file holds only the static OAuth app credentials; runtime token - # state is never written back to disk, so re-loaddata at restore time - # is safe. - - # --- Create initial admin user --- - log "Creating initial admin user..." - "$SCRIPT_DIR/dw-run.sh" "$INSTANCE" python scripts/setup_dev_logins.py - - # --- Optionally seed data --- - if [[ "$SEED" == "true" ]]; then - log "Loading demo fixtures..." - "$SCRIPT_DIR/dw-run.sh" "$INSTANCE" python manage.py loaddata demo_fixtures - fi - - # --- DR-mode marker --- - # If --no-start was passed, drop a marker file BEFORE the systemd installs - # below so the marker check skips enable/restart for celery-beat+celery-worker - # in this run, and so subsequent deploy.sh runs also leave them alone. The - # unit files themselves are still rendered — "go live" later is just - # `rm .dr-mode && systemctl enable --now celery-beat-* celery-worker-*`. + if [[ "$NEEDS_APP_BOOTSTRAP" == "true" ]]; then + log "Creating initial admin user..." + "$SCRIPT_DIR/dw-run.sh" "$INSTANCE" python scripts/setup_dev_logins.py + + if [[ "$SEED" == "true" ]]; then + log "Loading demo fixtures..." + "$SCRIPT_DIR/dw-run.sh" "$INSTANCE" python manage.py loaddata demo_fixtures + fi + fi + if [[ "$NO_START" == "true" ]]; then log "DR mode: writing $INSTANCE_DIR/.dr-mode (celery-beat+celery-worker will not be auto-started)" touch "$INSTANCE_DIR/.dr-mode" @@ -491,7 +574,6 @@ EOSQL chmod 644 "$INSTANCE_DIR/.dr-mode" fi - # --- Install systemd service --- log "Installing systemd service gunicorn-$INSTANCE..." sed \ -e "s|__INSTANCE__|$INSTANCE|g" \ @@ -538,19 +620,12 @@ EOSQL systemctl restart "celery-worker-$INSTANCE" fi - # --- Install backup timer --- - # Backups run nightly as the instance user, using the per-instance - # rclone config generated above. log "Installing backup timer backup-db-$INSTANCE..." render_backup_units "$INSTANCE" "$INSTANCE_USER" "$TEMPLATE_DIR" systemctl daemon-reload systemctl enable --now "backup-db-$INSTANCE.timer" log " Enabled nightly backup timer backup-db-$INSTANCE.timer" - # --- Install sudoers drop-in --- - # Lets the instance user restart its own units without a password. - # Render to a temp file, validate with visudo, then install atomically — - # a malformed file in /etc/sudoers.d locks out sudo entirely. log "Installing sudoers drop-in for $INSTANCE_USER..." local SUDOERS_TMP SUDOERS_TMP="$(mktemp)" @@ -563,7 +638,6 @@ EOSQL install -m 0440 -o root -g root "$SUDOERS_TMP" "/etc/sudoers.d/$INSTANCE_USER" rm -f "$SUDOERS_TMP" - # --- Install Nginx server block --- log "Installing Nginx config for $FQDN..." sed \ -e "s|__INSTANCE__|$INSTANCE|g" \ @@ -581,9 +655,12 @@ EOSQL log " After DNS cutover: sudo certbot --nginx -d $FQDN" fi - # --- Summary --- log "==========================================" - log "Instance '$INSTANCE' created successfully" + if [[ "$IS_EXISTING" == "true" ]]; then + log "Instance '$INSTANCE' reconfigured successfully" + else + log "Instance '$INSTANCE' created successfully" + fi log " URL: https://$FQDN" log " Directory: $INSTANCE_DIR (= git checkout)" log " User: $INSTANCE_USER" @@ -596,6 +673,14 @@ EOSQL echo " Instance is live at: https://$FQDN" } +do_create() { + do_configure true create "$@" +} + +do_reconfigure() { + do_configure false reconfigure "$@" +} + # ============================================================ # destroy # ============================================================ @@ -770,9 +855,10 @@ do_list() { # main # ============================================================ if [[ $# -lt 1 ]]; then - echo "Usage: $0 {prepare-config|create|destroy|list} [args...]" + echo "Usage: $0 {prepare-config|create|reconfigure|destroy|list} [args...]" echo " prepare-config — scaffold credentials file" - echo " create [--seed]" + echo " create [--seed] [--fqdn ] [--no-start]" + echo " reconfigure [--fqdn ] [--no-start]" echo " destroy " echo " list" exit 1 @@ -788,7 +874,8 @@ fi case "$COMMAND" in prepare-config) do_prepare_config "$@" ;; create) do_create "$@" ;; + reconfigure) do_reconfigure "$@" ;; destroy) do_destroy "$@" ;; list) do_list ;; - *) echo "Unknown command: $COMMAND"; echo "Usage: $0 {prepare-config|create|destroy|list}"; exit 1 ;; + *) echo "Unknown command: $COMMAND"; echo "Usage: $0 {prepare-config|create|reconfigure|destroy|list}"; exit 1 ;; esac diff --git a/scripts/server/server-setup.sh b/scripts/server/server-setup.sh index fa3cb7ff9..c4f6bd2ad 100755 --- a/scripts/server/server-setup.sh +++ b/scripts/server/server-setup.sh @@ -34,6 +34,17 @@ log_version() { log " Installed: $name $version" } +node_major_from_nvmrc() { + local nvmrc_file="$1" + local major + major="$(sed -nE 's/^[[:space:]]*v?([0-9]+).*/\1/p' "$nvmrc_file" | head -n 1)" + if [[ -z "$major" ]]; then + echo "ERROR: Could not parse Node major from $nvmrc_file" >&2 + exit 1 + fi + printf "%s\n" "$major" +} + # --- Pre-flight checks --- if [[ $EUID -ne 0 ]]; then @@ -434,6 +445,9 @@ chown docketworks:docketworks /opt/docketworks # of its own home and gunicorn/celery/dw-run all fail with EACCES. chmod 755 /opt/docketworks chown -R docketworks:docketworks /opt/docketworks/.local +mkdir -p /opt/docketworks/config +chown root:root /opt/docketworks/config +chmod 755 /opt/docketworks/config # --- Install Dreamhost API key for certbot hooks --- @@ -693,11 +707,18 @@ log " Shared Python dependencies installed." # --- Install shared node_modules --- log "Installing shared node_modules..." +REQUIRED_NODE_MAJOR="$(node_major_from_nvmrc "$LOCAL_REPO/frontend/.nvmrc")" sudo -u docketworks bash -c " cp '$LOCAL_REPO/frontend/package.json' '/opt/docketworks/package.json' cp '$LOCAL_REPO/frontend/package-lock.json' '/opt/docketworks/package-lock.json' + REQUIRED_NODE_MAJOR='$REQUIRED_NODE_MAJOR' + CURRENT_NODE_MAJOR=\$(node --version | sed -E 's/^v([0-9]+).*/\1/') + if [[ \"\$CURRENT_NODE_MAJOR\" != \"\$REQUIRED_NODE_MAJOR\" ]]; then + echo \"ERROR: Node major \$CURRENT_NODE_MAJOR does not match frontend/.nvmrc (\$REQUIRED_NODE_MAJOR)\" >&2 + exit 1 + fi cd /opt/docketworks - npm install + npm ci --include=dev " log " Shared node_modules installed." diff --git a/scripts/server/templates/credentials-instance.template b/scripts/server/templates/credentials-instance.template index a31662050..0118ef2b6 100644 --- a/scripts/server/templates/credentials-instance.template +++ b/scripts/server/templates/credentials-instance.template @@ -13,9 +13,10 @@ # 3. Go to the Webhooks section → "Add webhook" # - Delivery URL: https://__INSTANCE__.docketworks.site/api/xero/webhook/ # - Copy the "Webhook signing key" into XERO_WEBHOOK_KEY below. -# 4. XERO_DEFAULT_USER_ID — after completing OAuth in docketworks, fetch from -# the Xero API: GET /projects.xro/2.0/projectsusers -# Use the userId of the account that will own time entries. +# 4. XERO_DEFAULT_USER_ID — the Xero login/user ID that will own time +# entries. This must exist before instance.sh create runs. +# Use the userId from the Xero Projects users API: +# GET /projects.xro/2.0/projectsusers # (Temporary workaround — will be replaced by per-staff mapping.) XERO_DEFAULT_USER_ID=