From fa8711f3998bbca520a34f5b3b04e357b03ad1e8 Mon Sep 17 00:00:00 2001 From: Igor Zalutski Date: Thu, 27 Nov 2025 16:27:34 +0200 Subject: [PATCH 01/10] Bring back docs on local development --- docs/ce/local-development/backend.mdx | 61 ++++++++++++++++++++++++ docs/ce/local-development/github-app.mdx | 33 +++++++++++++ docs/ce/local-development/overview.mdx | 39 +++++++++++++++ docs/ce/local-development/statesman.mdx | 54 +++++++++++++++++++++ docs/ce/local-development/ui.mdx | 52 ++++++++++++++++++++ docs/docs.json | 10 ++++ 6 files changed, 249 insertions(+) create mode 100644 docs/ce/local-development/backend.mdx create mode 100644 docs/ce/local-development/github-app.mdx create mode 100644 docs/ce/local-development/overview.mdx create mode 100644 docs/ce/local-development/statesman.mdx create mode 100644 docs/ce/local-development/ui.mdx diff --git a/docs/ce/local-development/backend.mdx b/docs/ce/local-development/backend.mdx new file mode 100644 index 000000000..69c53a445 --- /dev/null +++ b/docs/ce/local-development/backend.mdx @@ -0,0 +1,61 @@ +--- +title: Orchestrator local setup +--- + +The backend serves orchestration APIs, GitHub app endpoints, and internal APIs the UI relies on. + +## Quick start + +1. Create `backend/.env` with the essentials (adjust DB URL/ports): + ```bash + DATABASE_URL=postgres://postgres:root@localhost:5432/postgres?sslmode=disable + DIGGER_INTERNAL_SECRET=orchestrator-secret # must match UI ORCHESTRATOR_BACKEND_SECRET + DIGGER_ENABLE_INTERNAL_ENDPOINTS=true # enables /_internal/* + DIGGER_ENABLE_API_ENDPOINTS=true # enables /api/* + HOSTNAME=http://localhost:3000 # used for GitHub app callbacks + ``` + Optional but useful: + - `GITHUB_APP_ID`, `GITHUB_APP_PEM`, `GITHUB_APP_WEBHOOK_SECRET` if you already have an app. + - `GITHUB_ORG` if you want `/github/setup` to target an org. +2. Start the service (from `backend/`): + ```bash + set -a; source .env; set +a + go run main.go # or: make start + ``` + Default port: `3000`. + +## Make the UI happy + +- The UI calls `/api/*` and `/github/*` with `Authorization: Bearer $DIGGER_INTERNAL_SECRET` and `DIGGER_ORG_ID`/`DIGGER_USER_ID` headers. +- You must upsert the WorkOS org + user the UI is authenticated as: + ```bash + SECRET=$DIGGER_INTERNAL_SECRET + ORG_ID=org_xxx # WorkOS org id + ORG_NAME=my-org # slug shown in backend + ADMIN_EMAIL=you@example.com + USER_ID=user_xxx # WorkOS user id + USER_EMAIL=$ADMIN_EMAIL + + # org + curl -s -X POST http://localhost:3000/_internal/api/upsert_org \ + -H "Authorization: Bearer $SECRET" \ + -H "Content-Type: application/json" \ + -d '{"org_name":"'"$ORG_NAME"'","external_source":"workos","external_id":"'"$ORG_ID"'","admin_email":"'"$ADMIN_EMAIL"'"}' + + # user + curl -s -X POST http://localhost:3000/_internal/api/create_user \ + -H "Authorization: Bearer $SECRET" \ + -H "Content-Type: application/json" \ + -d '{"external_source":"workos","external_id":"'"$USER_ID"'","email":"'"$USER_EMAIL"'","external_org_id":"'"$ORG_ID"'"}' + ``` + +## GitHub app integration + +- For a quick install link, set `ORCHESTRATOR_GITHUB_APP_URL` in `ui/.env.local` to your app's install URL (`https://github.com/apps//installations/new`). +- To create a new app via the backend, open `http://localhost:3000/github/setup` (requires `HOSTNAME` set to a reachable URL for callbacks). + +## Troubleshooting + +- **404 on /api/repos**: ensure `DIGGER_ENABLE_API_ENDPOINTS=true` and the org/user above are created. +- **401/403**: verify `Authorization` header uses `DIGGER_INTERNAL_SECRET`. +- **GitHub connect 404**: set `ORCHESTRATOR_GITHUB_APP_URL` as described. diff --git a/docs/ce/local-development/github-app.mdx b/docs/ce/local-development/github-app.mdx new file mode 100644 index 000000000..98e79c8aa --- /dev/null +++ b/docs/ce/local-development/github-app.mdx @@ -0,0 +1,33 @@ +--- +title: GitHub App settings for local dev +--- + +Use these settings when connecting a GitHub App to your local stack (tunneled via the UI domain). + +## Required URLs + +- **Callback URL** (OAuth/web): `https:///orchestrator/github/callback` +- **Webhook URL**: `https:///orchestrator/github/webhook` +- **Setup URL (optional)**: `https:///dashboard/onboarding?step=github` + +> The UI forwards these to the backend. Ensure `ORCHESTRATOR_BACKEND_URL`/`SECRET` are set in UI and the backend is reachable from the UI host. + +## Permissions & events (recommended) + +- Permissions: `contents:read`, `pull_requests:write`, `issues:write`, `statuses:write`, `checks:write`, `metadata:read`, `administration:read`, `workflows:write`, `repository_hooks:write`, `members:read`. +- Events: `issue_comment`, `pull_request`, `pull_request_review`, `pull_request_review_comment`, `push`, `check_run`, `status`. + +## Install URL + +After creating the app, use its install URL (e.g., `https://github.com/apps//installations/new`) as `ORCHESTRATOR_GITHUB_APP_URL` in `ui/.env.local`. This drives the "Connect with GitHub" button. + +## Creating an app via the backend wizard + +- Open `http://localhost:3000/github/setup` (or the backend's public URL) to generate a manifest and create the app in GitHub. Needed envs on backend: `HOSTNAME` set to a reachable URL, and optional `GITHUB_ORG` if you want to scope to an org. +- Once created, copy the install URL into `ORCHESTRATOR_GITHUB_APP_URL` and restart the UI. + +## Troubleshooting + +- **404 on connect**: `ORCHESTRATOR_GITHUB_APP_URL` not set or points to a non-existent path. +- **Callbacks fail**: UI not exposed publicly; tunnel the UI port and update callback/webhook URLs to that domain. +- **Backend rejects /api/github/link**: ensure `DIGGER_ENABLE_API_ENDPOINTS=true` and `DIGGER_INTERNAL_SECRET` matches the UI `ORCHESTRATOR_BACKEND_SECRET`. diff --git a/docs/ce/local-development/overview.mdx b/docs/ce/local-development/overview.mdx new file mode 100644 index 000000000..3b945e17b --- /dev/null +++ b/docs/ce/local-development/overview.mdx @@ -0,0 +1,39 @@ +--- +title: Local development overview +--- + +This section explains how to run the three core services locally: + +- **Backend** (`backend/`, port 3000 by default) – orchestrator + REST APIs for repos/orgs/jobs. +- **Statesman** (`taco/cmd/statesman`, port 8080) – state storage API and Terraform Cloud-compatible endpoints. +- **UI** (`ui/`, port 3030) – TanStack Start frontend that talks to both services and WorkOS. When tunneling (e.g., ngrok), expose the UI host; WorkOS and GitHub callbacks should point to the UI domain. + +## Prerequisites + +- Go toolchain for backend + statesman, Node 18+ for UI (`pnpm` or `npm`). +- A WorkOS project with User Management enabled and at least one organization + member (needed for UI auth and org IDs). +- Optional: GitHub App for repo onboarding (the backend can help you create one via `/github/setup`). + +## Shared secrets and ports + +- Pick two secrets and reuse them across components: + - `ORCHESTRATOR_BACKEND_SECRET` ≡ `DIGGER_INTERNAL_SECRET` (backend) ≡ UI env. + - `STATESMAN_BACKEND_WEBHOOK_SECRET` ≡ `OPENTACO_ENABLE_INTERNAL_ENDPOINTS` (statesman) ≡ UI env. +- Default ports: backend `3000`, statesman `8080`, UI `3030`. + +## High-level workflow + +1) **Start backend** with internal + API endpoints enabled (so UI can call `/api/*` and `/github/*`). +2) **Start statesman** with internal endpoints enabled; use memory storage for quick start. +3) **Configure UI** `.env.local` with URLs + secrets + WorkOS creds; run `pnpm dev --host --port 3030`. +4) **Sync org/user** into backend and statesman (WorkOS org id and user id/email) via the provided curl snippets in each page. +5) (Optional) **GitHub App**: set `ORCHESTRATOR_GITHUB_APP_URL` to your install URL or `http://localhost:3000/github/setup` to create one via the backend. Use the UI domain for app callback/webhook URLs (see GitHub App settings page). + +## Troubleshooting cheatsheet + +- **Backend /api/* returns 404**: `DIGGER_ENABLE_API_ENDPOINTS` not `true` or org not upserted. +- **Statesman 403**: webhook secret mismatch. **Statesman 404/500 resolving org**: org not synced (missing `external_org_id`). +- **UI WorkOS auth succeeds but org is empty**: add membership in WorkOS and resync org/user to services. +- **GitHub connect opens 404**: set `ORCHESTRATOR_GITHUB_APP_URL` to a valid install/setup URL. + +Continue with the per-service pages for commands and env examples. diff --git a/docs/ce/local-development/statesman.mdx b/docs/ce/local-development/statesman.mdx new file mode 100644 index 000000000..ac426b1e3 --- /dev/null +++ b/docs/ce/local-development/statesman.mdx @@ -0,0 +1,54 @@ +--- +title: Statesman local setup +--- + +Statesman serves state storage + Terraform Cloud-compatible APIs. The UI uses its internal endpoints, so enable webhook auth and sync your org/user. + +## Quick start + +1. Set env vars: + ```bash + OPENTACO_ENABLE_INTERNAL_ENDPOINTS=statesman-secret # must match UI STATESMAN_BACKEND_WEBHOOK_SECRET + OPENTACO_AUTH_DISABLE=true # skips OIDC for local + OPENTACO_STORAGE=memory # default; uses SQLite query backend automatically + # Optional: OPENTACO_SECRET_KEY for signed URLs; OPENTACO_PORT=8080 + ``` +2. Run the service (from repo root): + ```bash + cd taco + go run cmd/statesman/main.go -storage memory -auth-disable # or ./statesman with the same flags + ``` + Default port: `8080`. + +## Sync org and user (required for UI) + +Statesman resolves orgs by `external_org_id` (your WorkOS org id). If it cannot resolve, `/internal/api/units` will 500. + +```bash +SECRET=$OPENTACO_ENABLE_INTERNAL_ENDPOINTS +ORG_ID=org_xxx # WorkOS org id +ORG_NAME=digger-org # slug to store +ORG_DISPLAY="Digger Org" +USER_ID=user_xxx # WorkOS user id +USER_EMAIL=you@example.com + +# create/sync org +curl -s -X POST http://localhost:8080/internal/api/orgs/sync \ + -H "Authorization: Bearer $SECRET" \ + -H "X-User-ID: $USER_ID" -H "X-Email: $USER_EMAIL" \ + -H "Content-Type: application/json" \ + -d '{"name":"'"$ORG_NAME"'","display_name":"'"$ORG_DISPLAY"'","external_org_id":"'"$ORG_ID"'"}' + +# ensure user exists in that org +curl -s -X POST http://localhost:8080/internal/api/users \ + -H "Authorization: Bearer $SECRET" \ + -H "X-Org-ID: '$ORG_ID'" -H "X-User-ID: $USER_ID" -H "X-Email: $USER_EMAIL" \ + -H "Content-Type: application/json" \ + -d '{"subject":"'"$USER_ID"'","email":"'"$USER_EMAIL"'"}' +``` + +## Troubleshooting + +- **403**: webhook secret mismatch (`OPENTACO_ENABLE_INTERNAL_ENDPOINTS` vs UI `STATESMAN_BACKEND_WEBHOOK_SECRET`). +- **404/500 resolving org**: org not synced; rerun the `orgs/sync` call above. +- **SQLite quirks**: defaults to SQLite in-process; no config needed for local. For Postgres/MySQL, set `TACO_QUERY_BACKEND` and related envs (see `docs/ce/state-management/query-backend`). diff --git a/docs/ce/local-development/ui.mdx b/docs/ce/local-development/ui.mdx new file mode 100644 index 000000000..efb83022a --- /dev/null +++ b/docs/ce/local-development/ui.mdx @@ -0,0 +1,52 @@ +--- +title: UI local setup +--- + +The UI is a TanStack Start app that authenticates via WorkOS and calls both backend and statesman. It also acts as the public gateway when tunneling (e.g., ngrok): expose the UI port, and point external callbacks to the UI domain. + +## Quick start + +1. Copy `.env.example` to `.env.local` in `ui/` and fill the essentials: + ```bash + # URLs + PUBLIC_URL=http://localhost:3030 # replace host with your public tunnel when exposing UI + ALLOWED_HOSTS=localhost,127.0.0.1 # include your public tunnel host here + + # WorkOS (User Management) + WORKOS_CLIENT_ID= + WORKOS_API_KEY= + WORKOS_COOKIE_PASSWORD=<32+ random chars> + WORKOS_REDIRECT_URI=http://localhost:3030/api/auth/callback # replace host with your public tunnel; must match WorkOS config + WORKOS_WEBHOOK_SECRET= + + # Backend + ORCHESTRATOR_BACKEND_URL=http://localhost:3000 + ORCHESTRATOR_BACKEND_SECRET=orchestrator-secret # matches backend DIGGER_INTERNAL_SECRET + ORCHESTRATOR_GITHUB_APP_URL=http://localhost:3000/github/setup # or your app install URL + + # Statesman + STATESMAN_BACKEND_URL=http://localhost:8080 + STATESMAN_BACKEND_WEBHOOK_SECRET=statesman-secret # matches OPENTACO_ENABLE_INTERNAL_ENDPOINTS + + # Optional + DRIFT_REPORTING_BACKEND_URL=http://localhost:3004 + DRIFT_REPORTING_BACKEND_WEBHOOK_SECRET=... + POSTHOG_KEY=... + ``` + In WorkOS, add `http://localhost:3030/api/auth/callback` as a redirect. +2. Install deps and run: + ```bash + cd ui + pnpm install # or npm install + pnpm dev --host --port 3030 + ``` + Open `http://localhost:3030` (or your tunnel URL) and sign in with a WorkOS user that belongs to at least one org. Ensure the WorkOS redirect URI matches the public URL you configured. +3. Ensure backend + statesman were started and the same secrets are in place (see [Backend](./backend) and [Statesman](./statesman)). +4. Sync the WorkOS org/user to both services using the curl snippets on those pages (required for repos/units to load). + +## Common errors + +- **NotFound/Forbidden listing units**: statesman org/user not synced or webhook secret mismatch. +- **404 on repos or GitHub connect**: backend missing org/user, `DIGGER_ENABLE_API_ENDPOINTS` not set, or `ORCHESTRATOR_GITHUB_APP_URL` points to a non-existent path. +- **WorkOS login succeeds but dashboard redirects to / or errors**: the signed-in user has no WorkOS org membership; add to an org and resync to services. +- **WorkOS redirect blocked**: public URL not whitelisted; add your tunnel host to `ALLOWED_HOSTS` and to the WorkOS redirect URI list. diff --git a/docs/docs.json b/docs/docs.json index 3ecc2bbdf..2a220bb0e 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -165,6 +165,16 @@ "ce/azure-specific/azure-devops-locking-connection-methods" ] }, + { + "group": "Local Development", + "pages": [ + "ce/local-development/overview", + "ce/local-development/backend", + "ce/local-development/statesman", + "ce/local-development/ui", + "ce/local-development/github-app" + ] + }, { "group": "Contributing", "pages": [ From a302d72848d9d8b26351ea707b392d1b23ae6610 Mon Sep 17 00:00:00 2001 From: Igor Zalutski Date: Thu, 27 Nov 2025 17:02:25 +0200 Subject: [PATCH 02/10] Improve local dev docs --- docs/ce/local-development/statesman.mdx | 29 ++++++++++++++++++++++++- taco/cmd/statesman/sync-org.sh | 28 ++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100755 taco/cmd/statesman/sync-org.sh diff --git a/docs/ce/local-development/statesman.mdx b/docs/ce/local-development/statesman.mdx index ac426b1e3..32e7d58ba 100644 --- a/docs/ce/local-development/statesman.mdx +++ b/docs/ce/local-development/statesman.mdx @@ -4,6 +4,22 @@ title: Statesman local setup Statesman serves state storage + Terraform Cloud-compatible APIs. The UI uses its internal endpoints, so enable webhook auth and sync your org/user. +## First-time setup (SQLite migrations) + +If using SQLite with persistence, run migrations before starting: + +```bash +cd taco + +# If you get checksum errors, regenerate the hash first: +atlas migrate hash --dir file://migrations/sqlite + +# Apply migrations (adjust path to match your OPENTACO_SQLITE_DB_PATH) +atlas migrate apply --dir file://migrations/sqlite --url "sqlite://cmd/statesman/data/taco.db" +``` + +If you don't have Atlas installed: `make atlas-install` + ## Quick start 1. Set env vars: @@ -24,6 +40,15 @@ Statesman serves state storage + Terraform Cloud-compatible APIs. The UI uses it Statesman resolves orgs by `external_org_id` (your WorkOS org id). If it cannot resolve, `/internal/api/units` will 500. +A helper script is available at `taco/cmd/statesman/sync-org.sh` - edit the values before running: + +```bash +chmod +x taco/cmd/statesman/sync-org.sh +./taco/cmd/statesman/sync-org.sh +``` + +Or run manually: + ```bash SECRET=$OPENTACO_ENABLE_INTERNAL_ENDPOINTS ORG_ID=org_xxx # WorkOS org id @@ -49,6 +74,8 @@ curl -s -X POST http://localhost:8080/internal/api/users \ ## Troubleshooting +- **"no such table: organizations"**: Run migrations first (see First-time setup above). +- **Atlas checksum mismatch**: Run `atlas migrate hash --dir file://migrations/sqlite` then retry apply. - **403**: webhook secret mismatch (`OPENTACO_ENABLE_INTERNAL_ENDPOINTS` vs UI `STATESMAN_BACKEND_WEBHOOK_SECRET`). -- **404/500 resolving org**: org not synced; rerun the `orgs/sync` call above. +- **404/500 resolving org**: org not synced; rerun the sync script or `orgs/sync` call above. - **SQLite quirks**: defaults to SQLite in-process; no config needed for local. For Postgres/MySQL, set `TACO_QUERY_BACKEND` and related envs (see `docs/ce/state-management/query-backend`). diff --git a/taco/cmd/statesman/sync-org.sh b/taco/cmd/statesman/sync-org.sh new file mode 100755 index 000000000..7b1264044 --- /dev/null +++ b/taco/cmd/statesman/sync-org.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +SECRET=topsecret # must match OPENTACO_ENABLE_INTERNAL_ENDPOINTS +ORG_ID=org_xxx # your WorkOS org id +ORG_NAME=my-org # internal org name/slug +ORG_DISPLAY="My Org" # display name +USER_ID=user_xxx # your WorkOS user id +USER_EMAIL=you@example.com # your email + +# create/sync org +curl -s -X POST http://localhost:8080/internal/api/orgs/sync \ + -H "Authorization: Bearer $SECRET" \ + -H "X-User-ID: $USER_ID" -H "X-Email: $USER_EMAIL" \ + -H "Content-Type: application/json" \ + -d '{"name":"'"$ORG_NAME"'","display_name":"'"$ORG_DISPLAY"'","external_org_id":"'"$ORG_ID"'"}' + +echo "" +echo "Org synced. Now syncing user..." + +# ensure user exists in that org +curl -s -X POST http://localhost:8080/internal/api/users \ + -H "Authorization: Bearer $SECRET" \ + -H "X-Org-ID: $ORG_ID" -H "X-User-ID: $USER_ID" -H "X-Email: $USER_EMAIL" \ + -H "Content-Type: application/json" \ + -d '{"subject":"'"$USER_ID"'","email":"'"$USER_EMAIL"'"}' + +echo "" +echo "Done!" From 499d00fed8351e232d6eae595ca0ddd2dbd93942 Mon Sep 17 00:00:00 2001 From: Igor Zalutski Date: Thu, 27 Nov 2025 17:38:43 +0200 Subject: [PATCH 03/10] Agent task for moving from callbacks to webhooks --- agent-tasks/webhook-based-repo-management.md | 605 +++++++++++++++++++ 1 file changed, 605 insertions(+) create mode 100644 agent-tasks/webhook-based-repo-management.md diff --git a/agent-tasks/webhook-based-repo-management.md b/agent-tasks/webhook-based-repo-management.md new file mode 100644 index 000000000..1bd7b5e59 --- /dev/null +++ b/agent-tasks/webhook-based-repo-management.md @@ -0,0 +1,605 @@ +# Migrate Repository Management from Callback to Webhooks + +- Target: `backend/controllers/github*.go`, `backend/models/storage.go` +- Related: `ee/backend/controllers/github.go` (app manifest), `libs/ci/github/github_utils.go` + +## Goal + +Move repository synchronization entirely to webhook-driven updates. The OAuth callback should only handle organization/installation linking, not repository CRUD. Repositories should be created, soft-deleted, and restored exclusively via webhook events (`installation`, `installation_repositories`). + +## Non-Goals + +- Changing how other webhook events (push, PR, comments) are handled. +- Modifying the `GithubAppInstallation` model (per-repo installation records). +- Changing authentication or GitHub client provider logic. + +## Current State Summary + +### Callback Flow (`backend/controllers/github_callback.go`) + +The `GithubAppCallbackPage()` handler currently: +1. Receives `installation_id`, `code`, and `state` (appId) from GitHub. +2. Validates the callback via `validateGithubCallback()` using the OAuth code. +3. Creates or fetches `GithubAppInstallationLink` (org ↔ installation mapping). +4. **Problem**: Fetches all repos via `github.ListGithubRepos(client)` and syncs them: + - Soft-deletes ALL existing repos for the org (`models.DB.GormDB.Delete(ExistingRepos, "organisation_id=?", orgId)`). + - Marks all `GithubAppInstallation` records as deleted. + - Re-creates repos one by one via `createOrGetDiggerRepoForGithubRepo()`. +5. **Problem**: Fails if `code` parameter is missing (GitHub doesn't always send it on re-auth). + +### Webhook Flow (`backend/controllers/github.go`) + +The `GithubAppWebHook()` handler processes: +- `InstallationEvent` with action `deleted` → calls `handleInstallationDeletedEvent()`. +- `PushEvent`, `IssueCommentEvent`, `PullRequestEvent` → existing handlers. + +**Missing**: No handler for `InstallationRepositoriesEvent` (fired when repos are added/removed from an existing installation). + +### Existing Handlers + +**`handleInstallationDeletedEvent()`** (`github_installation.go:9-49`): +- Makes `GithubAppInstallationLink` inactive. +- Calls `GithubRepoRemoved()` for each repo in the event payload. +- **Does NOT soft-delete `Repo` records** — only updates `GithubAppInstallation.Status`. + +### Database Models + +**`Repo`** (`backend/models/orgs.go:33-48`): +- Uses `gorm.Model` with built-in `DeletedAt` for soft-delete. +- Fields: `Name`, `RepoFullName`, `GithubAppInstallationId`, `GithubAppId`, `DefaultBranch`, `CloneUrl`. + +**`GithubAppInstallation`** (`backend/models/github.go:40-48`): +- Per-repo record linking `GithubInstallationId` + `Repo` (full name) + `GithubAppId`. +- `Status`: `GithubAppInstallActive` (1) or `GithubAppInstallDeleted` (2). + +**`GithubAppInstallationLink`** (`backend/models/github.go:57-64`): +- Links `GithubInstallationId` ↔ `OrganisationId`. +- `Status`: `GithubAppInstallationLinkActive` (1) or `GithubAppInstallationLinkInactive` (2). + +**`Project`** (`backend/models/orgs.go`): +- Linked to repos via `RepoFullName`. +- Also uses `gorm.Model` with soft-delete support. +- **Must be soft-deleted alongside repos** on removal/uninstall. + +### Helper Functions + +**`createOrGetDiggerRepoForGithubRepo()`** (`github_helpers.go:914-976`): +- Looks up existing repo (including soft-deleted via `Unscoped()`). +- If found and deleted: restores by clearing `DeletedAt`. +- If not found: creates new repo. + +**`GithubRepoAdded()`** (`storage.go:460-489`): +- Creates or reactivates `GithubAppInstallation` record. + +**`GithubRepoRemoved()`** (`storage.go:491-512`): +- Sets `GithubAppInstallation.Status` to `GithubAppInstallDeleted`. + +### GitHub App Manifest (`ee/backend/controllers/github.go:61-73`) + +Currently subscribed events: +``` +check_run, create, delete, issue_comment, issues, status, +pull_request_review_thread, pull_request_review_comment, +pull_request_review, pull_request, push +``` + +**Missing**: `installation_repositories` event not subscribed. + +## Acceptance Criteria + +1. **Callback resilience**: `GithubAppCallbackPage()` succeeds even when `code` is missing. +2. **No repo sync in callback**: Callback only creates/updates org and installation link; no repo listing or CRUD. +3. **Webhook handles app install**: New `installation` event with action `created`/`unsuspended`/`new_permissions_accepted` creates repos from `event.Repositories`. +4. **Webhook handles app uninstall**: Existing `installation` event with action `deleted` soft-deletes all repos **and their projects** for the installation. +5. **Webhook handles scope changes**: New `installation_repositories` event handler: + - Action `added`: Creates or restores repos from `event.RepositoriesAdded`. + - Action `removed`: Soft-deletes repos **and their projects** from `event.RepositoriesRemoved`. +6. **App manifest updated**: `installation_repositories` event added to default events list. +7. **Soft-delete for Repos and Projects**: Both `Repo` and `Project` records are soft-deleted on removal/uninstall. +8. **Retry logic for race conditions**: Webhook handlers retry fetching installation link using `github.com/sethvargo/go-retry` since webhook may arrive before callback creates the link. +9. **Manual resync endpoint**: Admin API endpoint to resync repos for an existing installation. + +## Risks & Mitigations + +| Risk | Mitigation | +|------|------------| +| Existing installations miss initial repos | Provide manual resync endpoint (`POST /api/github/resync`) | +| Webhook delivery failures | GitHub retries webhooks; handlers are idempotent | +| Race conditions between callback and webhook | Webhook handlers retry with exponential backoff using `go-retry` | +| Missing `code` breaks existing flow | Make `code` optional; skip OAuth validation if missing | +| Projects orphaned when repos deleted | Soft-delete projects alongside repos in same transaction | + +## Implementation Plan + +### Commit 1: Add go-retry dependency + +**Action**: Add `github.com/sethvargo/go-retry` to `backend/go.mod`. + +**Files**: `backend/go.mod`, `backend/go.sum` + +**Changes**: +```go +require ( + github.com/sethvargo/go-retry v0.3.0 +) +``` + +**Verify**: `go mod tidy` succeeds. + +### Commit 2: Make callback resilient to missing `code` + +**Action**: Modify `GithubAppCallbackPage()` to not fail when `code` is missing. GitHub omits `code` on some re-authorization flows. + +**Files**: `backend/controllers/github_callback.go` + +**Changes**: +- Check if `code` exists before validation. +- If missing, skip `validateGithubCallback()` but continue with installation link creation. +- Log info when code is missing ("repos will sync via webhook"). +- Populate `vcsOwner` from OAuth validation only when code is present. + +**Verify**: Callback succeeds with and without `code` parameter. + +### Commit 3: Remove repo sync from callback + +**Action**: Strip all repo-fetching and repo-creation logic from `GithubAppCallbackPage()`. The callback should only: +1. Parse `installation_id` and optional `code`/`state`. +2. Validate callback (if code present). +3. Create or fetch `GithubAppInstallationLink`. +4. Return success page. + +**Files**: `backend/controllers/github_callback.go` + +**Changes**: +- Remove `github.ListGithubRepos(client)` call. +- Remove soft-delete of existing repos. +- Remove soft-delete of `GithubAppInstallation` records. +- Remove repo creation loop. +- Keep org creation and link creation logic. +- Remove unused imports (`strings`, `github`, `utils`). + +**Verify**: Callback completes without touching repos table. + +### Commit 4: Add soft-delete helpers for repos and projects + +**Action**: Add database helper functions to soft-delete repos and their associated projects. + +**Files**: `backend/models/storage.go` + +**Changes**: +- Add `SoftDeleteRepoAndProjects(orgId uint, repoFullName string) error`: + - Use transaction to soft-delete projects first, then repo. + - Query by `organisation_id` and `repo_full_name`. +- Add `SoftDeleteReposAndProjectsByInstallation(orgId uint, installationId int64) error`: + - Fetch all repos for the org with matching `github_app_installation_id`. + - Call `SoftDeleteRepoAndProjects` for each. + +**Verify**: Unit tests for both functions pass. + +### Commit 5: Add helper functions for repo upsert/remove + +**Action**: Create reusable helper functions for webhook handlers. + +**Files**: `backend/controllers/github_installation.go` + +**Changes**: +- Add `getAccountDetails(account *github.User) (login string, accountId int64)`: + - Safely extract login and ID with nil checks. +- Add `fetchRepoIdentifiers(ctx, client, repo, installationId) (fullName, owner, name, defaultBranch, cloneURL, error)`: + - Extract identifiers from webhook payload. + - If `defaultBranch` or `cloneURL` missing, fetch from GitHub API. +- Add `upsertRepo(ctx, client, repo, installationId, appId, accountLogin, accountId) error`: + - Call `fetchRepoIdentifiers`. + - Call `GithubRepoAdded`. + - Call `createOrGetDiggerRepoForGithubRepo`. +- Add `removeRepo(ctx, repo, installationId, appId, orgId) error`: + - Call `GithubRepoRemoved`. + - Call `SoftDeleteRepoAndProjects`. + +**Verify**: Helper functions compile and are ready for use. + +### Commit 6: Add installation upsert handler with retry + +**Action**: Handle `InstallationEvent` with actions `created`, `unsuspended`, `new_permissions_accepted` to sync repos. + +**Files**: `backend/controllers/github.go`, `backend/controllers/github_installation.go` + +**Changes**: +- In `GithubAppWebHook()` switch case for `InstallationEvent`: + - Add handling for `created`, `unsuspended`, `new_permissions_accepted` actions. + - Call new `handleInstallationUpsertEvent()`. +- Create `handleInstallationUpsertEvent(ctx, gh, event, appId) error`: + - **Retry logic**: Use `retry.Do` with `retry.WithMaxRetries(5, retry.NewConstant(2*time.Second))` to fetch installation link. + - If link not found after retries, return error (webhook will be retried by GitHub). + - Soft-delete existing `GithubAppInstallation` records for the installation. + - Soft-delete existing repos and projects for the installation. + - For each repo in `event.Repositories`, call `upsertRepo`. + +**Verify**: Installing app creates repos via webhook; handles race condition gracefully. + +### Commit 7: Update installation deleted handler + +**Action**: Modify `handleInstallationDeletedEvent()` to soft-delete repos and projects, not just `GithubAppInstallation` records. + +**Files**: `backend/controllers/github_installation.go` + +**Changes**: +- Mark all `GithubAppInstallation` records as deleted for the installation. +- Call `SoftDeleteReposAndProjectsByInstallation` to soft-delete all repos and projects. +- Keep existing logic for individual repo removal from payload. + +**Verify**: Uninstalling app soft-deletes repos and projects. + +### Commit 8: Add installation_repositories event handler with retry + +**Action**: Handle `InstallationRepositoriesEvent` for incremental repo changes (scope modifications). + +**Files**: `backend/controllers/github.go`, `backend/controllers/github_installation.go` + +**Changes**: +- In `GithubAppWebHook()`: + - Add new case for `*github.InstallationRepositoriesEvent`. + - Log action, added count, removed count. + - Call new `handleInstallationRepositoriesEvent()`. +- Create `handleInstallationRepositoriesEvent(ctx, gh, event, appId) error`: + - **Retry logic**: Use same retry pattern as upsert handler. + - For `RepositoriesAdded`: call `upsertRepo` for each. + - For `RepositoriesRemoved`: call `removeRepo` for each. + - Collect errors but continue processing; return aggregated error at end. + +**Verify**: Adding/removing repos from app scope creates/soft-deletes repos and projects. + +### Commit 9: Add manual resync API endpoint + +**Action**: Add admin API endpoint to resync repos for an existing installation. + +**Files**: `backend/controllers/github_api.go`, `backend/bootstrap/main.go` + +**Changes**: +- Add `ResyncGithubInstallationApi(c *gin.Context)`: + - Accept `installation_id` in JSON body. + - Fetch installation link; return 404 if not found. + - Fetch a `GithubAppInstallation` record to get `appId`. + - Create GitHub client and call `ListGithubRepos`. + - Build synthetic `InstallationEvent` and call `handleInstallationUpsertEvent`. + - Return success with repo count. +- Register route: `githubApiGroup.POST("/resync", controllers.ResyncGithubInstallationApi)`. + +**Verify**: `POST /api/github/resync` resyncs repos for an installation. + +### Commit 10: Update GitHub App manifest with new event + +**Action**: Add `installation_repositories` to the default events list in app manifest. + +**Files**: `ee/backend/controllers/github.go` + +**Changes**: +- Add `"installation_repositories"` to the `Events` slice. + +**Verify**: New app installations subscribe to `installation_repositories` webhook. + +### Commit 11: Add unit tests + +**Action**: Add tests for new handlers and storage functions. + +**Files**: `backend/models/storage_test.go`, `backend/controllers/github_installation_test.go` (new) + +**Changes**: +- Test `SoftDeleteRepoAndProjects`: verify repo and projects are soft-deleted. +- Test `SoftDeleteReposAndProjectsByInstallation`: verify only repos for specific installation are deleted. +- Test webhook handlers with mocked events. + +**Verify**: All tests pass. + +## Code Sketches + +### Retry pattern for installation link lookup + +```go +import ( + "context" + "errors" + "time" + "github.com/sethvargo/go-retry" +) + +// In handleInstallationUpsertEvent and handleInstallationRepositoriesEvent: +var link *models.GithubAppInstallationLink +backoff := retry.WithMaxRetries(5, retry.NewConstant(2*time.Second)) +err := retry.Do(ctx, backoff, func(ctx context.Context) error { + var dbErr error + link, dbErr = models.DB.GetGithubInstallationLinkForInstallationId(installationId) + if dbErr != nil { + return dbErr // permanent error, stop retrying + } + if link == nil { + return retry.RetryableError(errors.New("installation link not found")) + } + return nil +}) +if err != nil { + slog.Error("Installation link not found after retries", "installationId", installationId, "error", err) + return fmt.Errorf("installation link not found for installation %d after retries: %w", installationId, err) +} +``` + +### SoftDeleteRepoAndProjects + +```go +func (db *Database) SoftDeleteRepoAndProjects(orgId uint, repoFullName string) error { + return db.GormDB.Transaction(func(tx *gorm.DB) error { + // Soft-delete projects first + if err := tx.Where("organisation_id = ? AND repo_full_name = ?", orgId, repoFullName).Delete(&Project{}).Error; err != nil { + slog.Error("failed to soft delete projects for repo", "orgId", orgId, "repoFullName", repoFullName, "error", err) + return err + } + // Soft-delete repo + if err := tx.Where("organisation_id = ? AND repo_full_name = ?", orgId, repoFullName).Delete(&Repo{}).Error; err != nil { + slog.Error("failed to soft delete repo", "orgId", orgId, "repoFullName", repoFullName, "error", err) + return err + } + return nil + }) +} +``` + +### SoftDeleteReposAndProjectsByInstallation + +```go +func (db *Database) SoftDeleteReposAndProjectsByInstallation(orgId uint, installationId int64) error { + var repos []Repo + if err := db.GormDB.Where("organisation_id = ? AND github_app_installation_id = ?", orgId, installationId).Find(&repos).Error; err != nil { + slog.Error("failed to fetch repos for soft delete", "orgId", orgId, "installationId", installationId, "error", err) + return err + } + + for _, repo := range repos { + if err := db.SoftDeleteRepoAndProjects(orgId, repo.RepoFullName); err != nil { + return err + } + } + + return nil +} +``` + +### handleInstallationUpsertEvent + +```go +func handleInstallationUpsertEvent(ctx context.Context, gh utils.GithubClientProvider, installation *github.InstallationEvent, appId int64) error { + installationId := installation.Installation.GetID() + appIdFromPayload := appId + if installation.Installation.AppID != nil { + appIdFromPayload = installation.Installation.GetAppID() + } + + accountLogin, accountId := getAccountDetails(installation.Installation.Account) + + // Retry fetching the link since webhook may arrive before OAuth callback creates it + var link *models.GithubAppInstallationLink + backoff := retry.WithMaxRetries(5, retry.NewConstant(2*time.Second)) + err := retry.Do(ctx, backoff, func(ctx context.Context) error { + var dbErr error + link, dbErr = models.DB.GetGithubInstallationLinkForInstallationId(installationId) + if dbErr != nil { + return dbErr // permanent error, stop retrying + } + if link == nil { + return retry.RetryableError(errors.New("installation link not found")) + } + return nil + }) + if err != nil { + slog.Error("Installation link not found after retries", "installationId", installationId, "error", err) + return fmt.Errorf("installation link not found for installation %d after retries: %w", installationId, err) + } + + repoList := installation.Repositories + if len(repoList) == 0 { + slog.Warn("No repositories found to sync for installation", "installationId", installationId) + return nil + } + + slog.Info("Syncing repositories for installation", + "installationId", installationId, + "appId", appIdFromPayload, + "repoCount", len(repoList), + ) + + // Mark existing installations as deleted before resync + if err := models.DB.GormDB.Model(&models.GithubAppInstallation{}).Where("github_installation_id = ?", installationId).Update("status", models.GithubAppInstallDeleted).Error; err != nil { + slog.Error("Error marking installations deleted prior to resync", "installationId", installationId, "error", err) + return err + } + + // Soft-delete existing repos and projects + if err := models.DB.SoftDeleteReposAndProjectsByInstallation(link.OrganisationId, installationId); err != nil { + slog.Error("Error soft deleting existing repos/projects prior to resync", "installationId", installationId, "orgId", link.OrganisationId, "error", err) + return err + } + + ghClient, _, err := gh.Get(appIdFromPayload, installationId) + if err != nil { + slog.Error("Error creating GitHub client for repo sync", "installationId", installationId, "error", err) + return err + } + + for _, repo := range repoList { + if err := upsertRepo(ctx, ghClient, repo, installationId, appIdFromPayload, accountLogin, accountId); err != nil { + return err + } + } + + slog.Info("Successfully synced repositories for installation", "installationId", installationId) + return nil +} +``` + +### handleInstallationRepositoriesEvent + +```go +func handleInstallationRepositoriesEvent(ctx context.Context, gh utils.GithubClientProvider, event *github.InstallationRepositoriesEvent, appId int64) error { + installationId := event.Installation.GetID() + appIdFromPayload := appId + if event.Installation.AppID != nil { + appIdFromPayload = event.Installation.GetAppID() + } + + accountLogin, accountId := getAccountDetails(event.Installation.Account) + + // Retry fetching the link since webhook may arrive before OAuth callback creates it + var link *models.GithubAppInstallationLink + backoff := retry.WithMaxRetries(5, retry.NewConstant(2*time.Second)) + err := retry.Do(ctx, backoff, func(ctx context.Context) error { + var dbErr error + link, dbErr = models.DB.GetGithubInstallationLinkForInstallationId(installationId) + if dbErr != nil { + return dbErr // permanent error, stop retrying + } + if link == nil { + return retry.RetryableError(errors.New("installation link not found")) + } + return nil + }) + if err != nil { + slog.Error("Installation link not found after retries", "installationId", installationId, "error", err) + return fmt.Errorf("installation link not found for installation %d after retries: %w", installationId, err) + } + + client, _, err := gh.Get(appIdFromPayload, installationId) + if err != nil { + slog.Error("Error creating GitHub client for installation_repositories event", "installationId", installationId, "error", err) + return err + } + + var errs []error + for _, repo := range event.RepositoriesAdded { + if err := upsertRepo(ctx, client, repo, installationId, appIdFromPayload, accountLogin, accountId); err != nil { + errs = append(errs, err) + } + } + + for _, repo := range event.RepositoriesRemoved { + if err := removeRepo(ctx, repo, installationId, appIdFromPayload, link.OrganisationId); err != nil { + errs = append(errs, err) + } + } + + slog.Info("Handled installation_repositories event", + "installationId", installationId, + "addedCount", len(event.RepositoriesAdded), + "removedCount", len(event.RepositoriesRemoved), + ) + if len(errs) > 0 { + return fmt.Errorf("one or more errors during installation_repositories handling: %v", errs) + } + return nil +} +``` + +### ResyncGithubInstallationApi + +```go +func ResyncGithubInstallationApi(c *gin.Context) { + type ResyncInstallationRequest struct { + InstallationId string `json:"installation_id"` + } + + var request ResyncInstallationRequest + if err := c.BindJSON(&request); err != nil { + slog.Error("Error binding JSON for resync", "error", err) + c.JSON(http.StatusBadRequest, gin.H{"status": "Invalid request format"}) + return + } + + installationId, err := strconv.ParseInt(request.InstallationId, 10, 64) + if err != nil { + slog.Error("Failed to convert InstallationId to int64", "installationId", request.InstallationId, "error", err) + c.JSON(http.StatusBadRequest, gin.H{"status": "installationID should be a valid integer"}) + return + } + + link, err := models.DB.GetGithubAppInstallationLink(installationId) + if err != nil { + slog.Error("Could not get installation link for resync", "installationId", installationId, "error", err) + c.JSON(http.StatusInternalServerError, gin.H{"status": "Could not get installation link"}) + return + } + if link == nil { + slog.Warn("Installation link not found for resync", "installationId", installationId) + c.JSON(http.StatusNotFound, gin.H{"status": "Installation link not found"}) + return + } + + // Get appId from an existing installation record + var installationRecord models.GithubAppInstallation + if err := models.DB.GormDB.Where("github_installation_id = ?", installationId).Order("updated_at desc").First(&installationRecord).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + slog.Warn("No installation records found for resync", "installationId", installationId) + c.JSON(http.StatusNotFound, gin.H{"status": "No installation records found"}) + return + } + slog.Error("Failed to fetch installation record for resync", "installationId", installationId, "error", err) + c.JSON(http.StatusInternalServerError, gin.H{"status": "Could not fetch installation records"}) + return + } + + appId := installationRecord.GithubAppId + ghProvider := utils.DiggerGithubRealClientProvider{} + + client, _, err := ghProvider.Get(appId, installationId) + if err != nil { + slog.Error("Failed to create GitHub client for resync", "installationId", installationId, "appId", appId, "error", err) + c.JSON(http.StatusInternalServerError, gin.H{"status": "Failed to create GitHub client"}) + return + } + + repos, err := ci_github.ListGithubRepos(client) + if err != nil { + slog.Error("Failed to list repos for resync", "installationId", installationId, "error", err) + c.JSON(http.StatusInternalServerError, gin.H{"status": "Failed to list repos for resync"}) + return + } + + // Build synthetic InstallationEvent and call upsert handler + installationPayload := &github.Installation{ + ID: github.Int64(installationId), + AppID: github.Int64(appId), + } + resyncEvent := &github.InstallationEvent{ + Installation: installationPayload, + Repositories: repos, + } + + if err := handleInstallationUpsertEvent(c.Request.Context(), ghProvider, resyncEvent, appId); err != nil { + slog.Error("Resync failed", "installationId", installationId, "error", err) + c.JSON(http.StatusInternalServerError, gin.H{"status": "Resync failed"}) + return + } + + slog.Info("Resync completed", "installationId", installationId, "repoCount", len(repos)) + c.JSON(http.StatusOK, gin.H{"status": "Resync completed", "repoCount": len(repos)}) +} +``` + +## Verification Steps + +1. **Unit tests**: Add tests for new handlers with mocked events. + - Test `SoftDeleteRepoAndProjects`: verify repo and projects are soft-deleted. + - Test `SoftDeleteReposAndProjectsByInstallation`: verify only repos for specific installation are deleted, others untouched. +2. **Integration test**: + - Install app → verify repos created via webhook. + - Remove repo from scope → verify repo and projects soft-deleted. + - Add repo back → verify restored. + - Uninstall app → verify all repos and projects soft-deleted. +3. **Callback test**: Hit callback endpoint without `code` → should succeed. +4. **Race condition test**: Trigger webhook before callback completes → verify retry succeeds. +5. **Resync test**: Call `POST /api/github/resync` → verify repos are synced. +6. **Manual verification**: Use GitHub App settings to add/remove repos and observe DB changes. + +## Follow-ups (Later; not in this plan) + +- Consider adding `webhook_synced_at` timestamp to `Repo` for debugging. +- Rate limiting for webhook processing if needed. +- Metrics/observability for sync success/failure per installation. +- Document migration path for existing installations in user-facing docs. From 947fe3b0c9b4652063eccf6f57cb2b2f4f50872d Mon Sep 17 00:00:00 2001 From: Igor Zalutski Date: Thu, 27 Nov 2025 18:19:56 +0200 Subject: [PATCH 04/10] Move from callback to webhooks, again --- backend/bootstrap/main.go | 1 + backend/controllers/github.go | 18 ++ backend/controllers/github_api.go | 84 ++++++ backend/controllers/github_callback.go | 205 +++----------- backend/controllers/github_installation.go | 295 +++++++++++++++++++-- backend/go.mod | 1 + backend/go.sum | 2 + backend/models/storage.go | 34 +++ backend/models/storage_test.go | 87 ++++++ ee/backend/controllers/github.go | 1 + go.work.sum | 1 + 11 files changed, 549 insertions(+), 180 deletions(-) diff --git a/backend/bootstrap/main.go b/backend/bootstrap/main.go index 99ce5c9b3..aed837454 100644 --- a/backend/bootstrap/main.go +++ b/backend/bootstrap/main.go @@ -242,6 +242,7 @@ func Bootstrap(templates embed.FS, diggerController controllers.DiggerController githubApiGroup := apiGroup.Group("/github") githubApiGroup.POST("/link", controllers.LinkGithubInstallationToOrgApi) + githubApiGroup.POST("/resync", controllers.ResyncGithubInstallationApi) vcsApiGroup := apiGroup.Group("/connections") vcsApiGroup.GET("/:id", controllers.GetVCSConnection) diff --git a/backend/controllers/github.go b/backend/controllers/github.go index fe35ad39a..1c1649dc1 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -72,6 +72,24 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) { c.String(http.StatusAccepted, "Failed to handle webhook event.") return } + } else if *event.Action == "created" || *event.Action == "unsuspended" || *event.Action == "new_permissions_accepted" { + if err := handleInstallationUpsertEvent(c.Request.Context(), gh, event, appId64); err != nil { + slog.Error("Failed to handle installation upsert event", "error", err) + c.String(http.StatusAccepted, "Failed to handle webhook event.") + return + } + } + case *github.InstallationRepositoriesEvent: + slog.Info("Processing InstallationRepositoriesEvent", + "action", event.GetAction(), + "installationId", event.Installation.GetID(), + "added", len(event.RepositoriesAdded), + "removed", len(event.RepositoriesRemoved), + ) + if err := handleInstallationRepositoriesEvent(c.Request.Context(), gh, event, appId64); err != nil { + slog.Error("Failed to handle installation repositories event", "error", err) + c.String(http.StatusAccepted, "Failed to handle webhook event.") + return } case *github.PushEvent: slog.Info("Processing PushEvent", diff --git a/backend/controllers/github_api.go b/backend/controllers/github_api.go index ea58ea056..c59383bf8 100644 --- a/backend/controllers/github_api.go +++ b/backend/controllers/github_api.go @@ -8,7 +8,10 @@ import ( "github.com/diggerhq/digger/backend/middleware" "github.com/diggerhq/digger/backend/models" + "github.com/diggerhq/digger/backend/utils" + ci_github "github.com/diggerhq/digger/libs/ci/github" "github.com/gin-gonic/gin" + "github.com/google/go-github/v61/github" "gorm.io/gorm" ) @@ -85,3 +88,84 @@ func LinkGithubInstallationToOrgApi(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"status": "Successfully created Github installation link"}) return } + +func ResyncGithubInstallationApi(c *gin.Context) { + type ResyncInstallationRequest struct { + InstallationId string `json:"installation_id"` + } + + var request ResyncInstallationRequest + if err := c.BindJSON(&request); err != nil { + slog.Error("Error binding JSON for resync", "error", err) + c.JSON(http.StatusBadRequest, gin.H{"status": "Invalid request format"}) + return + } + + installationId, err := strconv.ParseInt(request.InstallationId, 10, 64) + if err != nil { + slog.Error("Failed to convert InstallationId to int64", "installationId", request.InstallationId, "error", err) + c.JSON(http.StatusBadRequest, gin.H{"status": "installationID should be a valid integer"}) + return + } + + link, err := models.DB.GetGithubAppInstallationLink(installationId) + if err != nil { + slog.Error("Could not get installation link for resync", "installationId", installationId, "error", err) + c.JSON(http.StatusInternalServerError, gin.H{"status": "Could not get installation link"}) + return + } + if link == nil { + slog.Warn("Installation link not found for resync", "installationId", installationId) + c.JSON(http.StatusNotFound, gin.H{"status": "Installation link not found"}) + return + } + + // Get appId from an existing installation record + var installationRecord models.GithubAppInstallation + if err := models.DB.GormDB.Where("github_installation_id = ?", installationId).Order("updated_at desc").First(&installationRecord).Error; err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + slog.Warn("No installation records found for resync", "installationId", installationId) + c.JSON(http.StatusNotFound, gin.H{"status": "No installation records found"}) + return + } + slog.Error("Failed to fetch installation record for resync", "installationId", installationId, "error", err) + c.JSON(http.StatusInternalServerError, gin.H{"status": "Could not fetch installation records"}) + return + } + + appId := installationRecord.GithubAppId + ghProvider := utils.DiggerGithubRealClientProvider{} + + client, _, err := ghProvider.Get(appId, installationId) + if err != nil { + slog.Error("Failed to create GitHub client for resync", "installationId", installationId, "appId", appId, "error", err) + c.JSON(http.StatusInternalServerError, gin.H{"status": "Failed to create GitHub client"}) + return + } + + repos, err := ci_github.ListGithubRepos(client) + if err != nil { + slog.Error("Failed to list repos for resync", "installationId", installationId, "error", err) + c.JSON(http.StatusInternalServerError, gin.H{"status": "Failed to list repos for resync"}) + return + } + + // Build synthetic InstallationEvent and call upsert handler + installationPayload := &github.Installation{ + ID: github.Int64(installationId), + AppID: github.Int64(appId), + } + resyncEvent := &github.InstallationEvent{ + Installation: installationPayload, + Repositories: repos, + } + + if err := handleInstallationUpsertEvent(c.Request.Context(), ghProvider, resyncEvent, appId); err != nil { + slog.Error("Resync failed", "installationId", installationId, "error", err) + c.JSON(http.StatusInternalServerError, gin.H{"status": "Resync failed"}) + return + } + + slog.Info("Resync completed", "installationId", installationId, "repoCount", len(repos)) + c.JSON(http.StatusOK, gin.H{"status": "Resync completed", "repoCount": len(repos)}) +} diff --git a/backend/controllers/github_callback.go b/backend/controllers/github_callback.go index 7b6d34ee3..4337ce5a0 100644 --- a/backend/controllers/github_callback.go +++ b/backend/controllers/github_callback.go @@ -5,12 +5,9 @@ import ( "log/slog" "net/http" "strconv" - "strings" "github.com/diggerhq/digger/backend/models" "github.com/diggerhq/digger/backend/segment" - "github.com/diggerhq/digger/backend/utils" - "github.com/diggerhq/digger/libs/ci/github" "github.com/gin-gonic/gin" "github.com/google/uuid" ) @@ -28,29 +25,17 @@ func (d DiggerController) GithubAppCallbackPage(c *gin.Context) { c.String(http.StatusBadRequest, "installation_id parameter for github app is empty") return } - //setupAction := c.Request.URL.Query()["setup_action"][0] + + // Code parameter is optional - GitHub doesn't always send it on re-authorization flows codeParams, codeExists := c.Request.URL.Query()["code"] - if !codeExists || len(codeParams) == 0 { - slog.Error("There was no code in the url query parameters") - c.String(http.StatusBadRequest, "could not find the code query parameter for github app") - return - } - code := codeParams[0] - if len(code) < 1 { - slog.Error("Code parameter is empty") - c.String(http.StatusBadRequest, "code parameter for github app is empty") - return + code := "" + if codeExists && len(codeParams) > 0 && len(codeParams[0]) > 0 { + code = codeParams[0] } - appId := c.Request.URL.Query().Get("state") - slog.Info("Processing GitHub app callback", "installationId", installationId, "appId", appId) + appId := c.Request.URL.Query().Get("state") - clientId, clientSecret, _, _, err := d.GithubClientProvider.FetchCredentials(appId) - if err != nil { - slog.Error("Could not fetch credentials for GitHub app", "appId", appId, "error", err) - c.String(http.StatusInternalServerError, "could not find credentials for github app") - return - } + slog.Info("Processing GitHub app callback", "installationId", installationId, "appId", appId, "hasCode", code != "") installationId64, err := strconv.ParseInt(installationId, 10, 64) if err != nil { @@ -62,38 +47,47 @@ func (d DiggerController) GithubAppCallbackPage(c *gin.Context) { return } - slog.Debug("Validating GitHub callback", "installationId", installationId64, "clientId", clientId) + // vcsOwner is used for analytics; we'll populate it if we can validate via OAuth + var vcsOwner string - result, installation, err := validateGithubCallback(d.GithubClientProvider, clientId, clientSecret, code, installationId64) - if !result { - slog.Error("Failed to validate installation ID", - "installationId", installationId64, - "error", err, - ) - c.String(http.StatusInternalServerError, "Failed to validate installation_id.") - return - } + // If we have a code parameter, validate the callback via OAuth + // This provides additional security by confirming the user authorized the installation + if code != "" { + clientId, clientSecret, _, _, err := d.GithubClientProvider.FetchCredentials(appId) + if err != nil { + slog.Error("Could not fetch credentials for GitHub app", "appId", appId, "error", err) + c.String(http.StatusInternalServerError, "could not find credentials for github app") + return + } - // TODO: Lookup org in GithubAppInstallation by installationID if found use that installationID otherwise - // create a new org for this installationID - // retrieve org for current orgID - installationIdInt64, err := strconv.ParseInt(installationId, 10, 64) - if err != nil { - slog.Error("Failed to parse installation ID as int64", - "installationId", installationId, - "error", err, + slog.Debug("Validating GitHub callback", "installationId", installationId64, "clientId", clientId) + + result, installation, err := validateGithubCallback(d.GithubClientProvider, clientId, clientSecret, code, installationId64) + if !result { + slog.Error("Failed to validate installation ID", + "installationId", installationId64, + "error", err, + ) + c.String(http.StatusInternalServerError, "Failed to validate installation_id.") + return + } + + if installation != nil && installation.Account != nil && installation.Account.Login != nil { + vcsOwner = *installation.Account.Login + } + } else { + slog.Info("No code parameter provided, skipping OAuth validation (repos will sync via webhook)", + "installationId", installationId64, ) - c.JSON(http.StatusInternalServerError, gin.H{"error": "installationId could not be parsed"}) - return } - slog.Debug("Looking up GitHub app installation link", "installationId", installationIdInt64) + slog.Debug("Looking up GitHub app installation link", "installationId", installationId64) var link *models.GithubAppInstallationLink - link, err = models.DB.GetGithubAppInstallationLink(installationIdInt64) + link, err = models.DB.GetGithubAppInstallationLink(installationId64) if err != nil { slog.Error("Error getting GitHub app installation link", - "installationId", installationIdInt64, + "installationId", installationId64, "error", err, ) c.JSON(http.StatusInternalServerError, gin.H{"error": "error getting github app link"}) @@ -153,14 +147,10 @@ func (d DiggerController) GithubAppCallbackPage(c *gin.Context) { org := link.Organisation orgId := link.OrganisationId - var vcsOwner string = "" - if installation.Account.Login != nil { - vcsOwner = *installation.Account.Login - } - // we have multiple repos here, we don't really want to send an track event for each repo, so we just send the vcs owner + // Track the installation event for analytics segment.Track(*org, vcsOwner, "", "github", "vcs_repo_installed", map[string]string{}) - // create a github installation link (org ID matched to installation ID) + // Ensure the installation link exists (idempotent operation) _, err = models.DB.CreateGithubInstallationLink(org, installationId64) if err != nil { slog.Error("Error creating GitHub installation link", @@ -172,120 +162,9 @@ func (d DiggerController) GithubAppCallbackPage(c *gin.Context) { return } - slog.Debug("Getting GitHub client", - "appId", *installation.AppID, - "installationId", installationId64, - ) - - client, _, err := d.GithubClientProvider.Get(*installation.AppID, installationId64) - if err != nil { - slog.Error("Error retrieving GitHub client", - "appId", *installation.AppID, - "installationId", installationId64, - "error", err, - ) - c.JSON(http.StatusInternalServerError, gin.H{"error": "Error fetching organisation"}) - return - } - - // we get repos accessible to this installation - slog.Debug("Listing repositories for installation", "installationId", installationId64) - - repos, err := github.ListGithubRepos(client) - if err != nil { - slog.Error("Failed to list existing repositories", - "installationId", installationId64, - "error", err, - ) - c.String(http.StatusInternalServerError, "Failed to list existing repos: %v", err) - return - } - - // resets all existing installations (soft delete) - slog.Debug("Resetting existing GitHub installations", - "installationId", installationId, - ) - - var AppInstallation models.GithubAppInstallation - err = models.DB.GormDB.Model(&AppInstallation).Where("github_installation_id=?", installationId).Update("status", models.GithubAppInstallDeleted).Error - if err != nil { - slog.Error("Failed to update GitHub installations", - "installationId", installationId, - "error", err, - ) - c.String(http.StatusInternalServerError, "Failed to update github installations: %v", err) - return - } - - // reset all existing repos (soft delete) - slog.Debug("Soft deleting existing repositories", - "orgId", orgId, - ) - - var ExistingRepos []models.Repo - err = models.DB.GormDB.Delete(ExistingRepos, "organisation_id=?", orgId).Error - if err != nil { - slog.Error("Could not delete repositories", - "orgId", orgId, - "error", err, - ) - c.String(http.StatusInternalServerError, "could not delete repos: %v", err) - return - } - - // here we mark repos that are available one by one - slog.Info("Adding repositories to organization", - "orgId", orgId, - "repoCount", len(repos), - ) - - for i, repo := range repos { - repoFullName := *repo.FullName - repoOwner := strings.Split(*repo.FullName, "/")[0] - repoName := *repo.Name - repoUrl := fmt.Sprintf("https://%v/%v", utils.GetGithubHostname(), repoFullName) - - slog.Debug("Processing repository", - "index", i+1, - "repoFullName", repoFullName, - "repoOwner", repoOwner, - "repoName", repoName, - ) - - _, err := models.DB.GithubRepoAdded( - installationId64, - *installation.AppID, - *installation.Account.Login, - *installation.Account.ID, - repoFullName, - ) - if err != nil { - slog.Error("Error recording GitHub repository", - "repoFullName", repoFullName, - "error", err, - ) - c.String(http.StatusInternalServerError, "github repos added error: %v", err) - return - } - - cloneUrl := *repo.CloneURL - defaultBranch := *repo.DefaultBranch - - _, _, err = createOrGetDiggerRepoForGithubRepo(repoFullName, repoOwner, repoName, repoUrl, installationId64, *installation.AppID, defaultBranch, cloneUrl) - if err != nil { - slog.Error("Error creating or getting Digger repo", - "repoFullName", repoFullName, - "error", err, - ) - c.String(http.StatusInternalServerError, "createOrGetDiggerRepoForGithubRepo error: %v", err) - return - } - } - - slog.Info("GitHub app callback processed successfully", + slog.Info("GitHub app callback processed", "installationId", installationId64, "orgId", orgId, - "repoCount", len(repos), ) c.HTML(http.StatusOK, "github_success.tmpl", gin.H{}) diff --git a/backend/controllers/github_installation.go b/backend/controllers/github_installation.go index 1db95b13a..9642a40fc 100644 --- a/backend/controllers/github_installation.go +++ b/backend/controllers/github_installation.go @@ -1,13 +1,141 @@ package controllers import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + "time" + "github.com/diggerhq/digger/backend/models" + "github.com/diggerhq/digger/backend/utils" "github.com/google/go-github/v61/github" - "log/slog" + "github.com/sethvargo/go-retry" ) +// getAccountDetails safely extracts login and account ID from a GitHub user. +func getAccountDetails(account *github.User) (string, int64) { + if account == nil { + return "", 0 + } + return account.GetLogin(), int64(account.GetID()) +} + +// fetchRepoIdentifiers returns repo identifiers and fills missing branch/clone URL by calling GitHub if needed. +func fetchRepoIdentifiers(ctx context.Context, client *github.Client, repo *github.Repository, installationId int64) (repoFullName, owner, name, defaultBranch, cloneURL string, err error) { + repoFullName = repo.GetFullName() + if repo.Owner != nil { + owner = repo.Owner.GetLogin() + } + name = repo.GetName() + defaultBranch = repo.GetDefaultBranch() + cloneURL = repo.GetCloneURL() + + // If owner is missing but we have fullName, parse it from there + // This handles webhook payloads that don't include the full Owner object + if owner == "" && repoFullName != "" { + parts := strings.Split(repoFullName, "/") + if len(parts) == 2 { + owner = parts[0] + if name == "" { + name = parts[1] + } + } + } + + if repoFullName == "" && owner != "" && name != "" { + repoFullName = fmt.Sprintf("%s/%s", owner, name) + } + + // If we don't have full repo details (defaultBranch or cloneURL), fetch them from GitHub + if (defaultBranch == "" || cloneURL == "") && owner != "" && name != "" { + repoDetails, _, fetchErr := client.Repositories.Get(ctx, owner, name) + if fetchErr != nil { + slog.Error("Error fetching repo details", + "installationId", installationId, + "repoOwner", owner, + "repoName", name, + "error", fetchErr) + return repoFullName, owner, name, defaultBranch, cloneURL, fetchErr + } + if defaultBranch == "" { + defaultBranch = repoDetails.GetDefaultBranch() + } + if cloneURL == "" { + cloneURL = repoDetails.GetCloneURL() + } + } + + return repoFullName, owner, name, defaultBranch, cloneURL, nil +} + +// upsertRepo creates or restores a repo and its GithubAppInstallation record. +func upsertRepo(ctx context.Context, ghClient *github.Client, repo *github.Repository, installationId int64, appId int64, accountLogin string, accountId int64) error { + repoFullName, owner, name, defaultBranch, cloneURL, err := fetchRepoIdentifiers(ctx, ghClient, repo, installationId) + if err != nil { + return err + } + if repoFullName == "" || owner == "" || name == "" { + slog.Warn("Skipping repo with missing identifiers", + "installationId", installationId, + "repoFullName", repoFullName, + "owner", owner, + "name", name, + ) + return nil + } + + if _, err := models.DB.GithubRepoAdded(installationId, appId, accountLogin, accountId, repoFullName); err != nil { + slog.Error("Error recording GitHub repository", + "installationId", installationId, + "repoFullName", repoFullName, + "error", err) + return err + } + + repoUrl := fmt.Sprintf("https://%s/%s", utils.GetGithubHostname(), repoFullName) + if _, _, err := createOrGetDiggerRepoForGithubRepo(repoFullName, owner, name, repoUrl, installationId, appId, defaultBranch, cloneURL); err != nil { + slog.Error("Error creating or getting Digger repo", + "installationId", installationId, + "repoFullName", repoFullName, + "error", err) + return err + } + + return nil +} + +// removeRepo marks a repo as removed and soft-deletes it along with its projects. +func removeRepo(ctx context.Context, repo *github.Repository, installationId int64, appId int64, orgId uint) error { + repoFullName := repo.GetFullName() + if repoFullName == "" { + slog.Warn("Skipping repo removal with empty full name", "installationId", installationId) + return nil + } + + if _, err := models.DB.GithubRepoRemoved(installationId, appId, repoFullName); err != nil { + slog.Error("Error marking GitHub repo removed", + "installationId", installationId, + "repoFullName", repoFullName, + "error", err) + return err + } + + if err := models.DB.SoftDeleteRepoAndProjects(orgId, repoFullName); err != nil { + slog.Error("Error soft deleting repo and projects on remove", + "installationId", installationId, + "repoFullName", repoFullName, + "orgId", orgId, + "error", err) + return err + } + + return nil +} + func handleInstallationDeletedEvent(installation *github.InstallationEvent, appId int64) error { - installationId := *installation.Installation.ID + installationId := installation.Installation.GetID() slog.Info("Handling installation deleted event", "installationId", installationId, @@ -20,26 +148,31 @@ func handleInstallationDeletedEvent(installation *github.InstallationEvent, appI return err } - _, err = models.DB.MakeGithubAppInstallationLinkInactive(link) - if err != nil { + if link == nil { + slog.Error("Installation link not found for deletion", "installationId", installationId) + return nil + } + + if _, err = models.DB.MakeGithubAppInstallationLinkInactive(link); err != nil { slog.Error("Error making installation link inactive", "installationId", installationId, "error", err) return err } - for _, repo := range installation.Repositories { - repoFullName := *repo.FullName - slog.Info("Removing installation for repo", - "installationId", installationId, - "repoFullName", repoFullName, - ) + // Mark all GithubAppInstallation records as deleted + if err := models.DB.GormDB.Model(&models.GithubAppInstallation{}).Where("github_installation_id = ?", installationId).Update("status", models.GithubAppInstallDeleted).Error; err != nil { + slog.Error("Error marking installations deleted", "installationId", installationId, "error", err) + return err + } - _, err := models.DB.GithubRepoRemoved(installationId, appId, repoFullName) - if err != nil { - slog.Error("Error removing GitHub repo", - "installationId", installationId, - "repoFullName", repoFullName, - "error", err, - ) + // Soft-delete all repos and projects for this installation + if err := models.DB.SoftDeleteReposAndProjectsByInstallation(link.OrganisationId, installationId); err != nil { + slog.Error("Error soft deleting repos/projects for installation", "installationId", installationId, "orgId", link.OrganisationId, "error", err) + return err + } + + // Also process individual repos from the payload (for consistency) + for _, repo := range installation.Repositories { + if err := removeRepo(context.Background(), repo, installationId, appId, link.OrganisationId); err != nil { return err } } @@ -47,3 +180,131 @@ func handleInstallationDeletedEvent(installation *github.InstallationEvent, appI slog.Info("Successfully handled installation deleted event", "installationId", installationId) return nil } + +// handleInstallationUpsertEvent handles installation created/unsuspended/new_permissions_accepted events. +func handleInstallationUpsertEvent(ctx context.Context, gh utils.GithubClientProvider, installation *github.InstallationEvent, appId int64) error { + installationId := installation.Installation.GetID() + appIdFromPayload := appId + if installation.Installation.AppID != nil { + appIdFromPayload = installation.Installation.GetAppID() + } + + accountLogin, accountId := getAccountDetails(installation.Installation.Account) + + // Retry fetching the link since webhook may arrive before OAuth callback creates it + var link *models.GithubAppInstallationLink + backoff := retry.WithMaxRetries(5, retry.NewConstant(2*time.Second)) + err := retry.Do(ctx, backoff, func(ctx context.Context) error { + var dbErr error + link, dbErr = models.DB.GetGithubInstallationLinkForInstallationId(installationId) + if dbErr != nil { + return dbErr // permanent error, stop retrying + } + if link == nil { + return retry.RetryableError(errors.New("installation link not found")) + } + return nil + }) + if err != nil { + slog.Error("Installation link not found after retries", "installationId", installationId, "error", err) + return fmt.Errorf("installation link not found for installation %d after retries: %w", installationId, err) + } + + repoList := installation.Repositories + if len(repoList) == 0 { + slog.Warn("No repositories found to sync for installation", "installationId", installationId) + return nil + } + + slog.Info("Syncing repositories for installation", + "installationId", installationId, + "appId", appIdFromPayload, + "repoCount", len(repoList), + ) + + // Mark existing installations as deleted before resync + if err := models.DB.GormDB.Model(&models.GithubAppInstallation{}).Where("github_installation_id = ?", installationId).Update("status", models.GithubAppInstallDeleted).Error; err != nil { + slog.Error("Error marking installations deleted prior to resync", "installationId", installationId, "error", err) + return err + } + + // Soft-delete existing repos and projects + if err := models.DB.SoftDeleteReposAndProjectsByInstallation(link.OrganisationId, installationId); err != nil { + slog.Error("Error soft deleting existing repos/projects prior to resync", "installationId", installationId, "orgId", link.OrganisationId, "error", err) + return err + } + + ghClient, _, err := gh.Get(appIdFromPayload, installationId) + if err != nil { + slog.Error("Error creating GitHub client for repo sync", "installationId", installationId, "error", err) + return err + } + + for _, repo := range repoList { + if err := upsertRepo(ctx, ghClient, repo, installationId, appIdFromPayload, accountLogin, accountId); err != nil { + return err + } + } + + slog.Info("Successfully synced repositories for installation", "installationId", installationId) + return nil +} + +// handleInstallationRepositoriesEvent handles incremental repo changes (added/removed from installation scope). +func handleInstallationRepositoriesEvent(ctx context.Context, gh utils.GithubClientProvider, event *github.InstallationRepositoriesEvent, appId int64) error { + installationId := event.Installation.GetID() + appIdFromPayload := appId + if event.Installation.AppID != nil { + appIdFromPayload = event.Installation.GetAppID() + } + + accountLogin, accountId := getAccountDetails(event.Installation.Account) + + // Retry fetching the link since webhook may arrive before OAuth callback creates it + var link *models.GithubAppInstallationLink + backoff := retry.WithMaxRetries(5, retry.NewConstant(2*time.Second)) + err := retry.Do(ctx, backoff, func(ctx context.Context) error { + var dbErr error + link, dbErr = models.DB.GetGithubInstallationLinkForInstallationId(installationId) + if dbErr != nil { + return dbErr // permanent error, stop retrying + } + if link == nil { + return retry.RetryableError(errors.New("installation link not found")) + } + return nil + }) + if err != nil { + slog.Error("Installation link not found after retries", "installationId", installationId, "error", err) + return fmt.Errorf("installation link not found for installation %d after retries: %w", installationId, err) + } + + client, _, err := gh.Get(appIdFromPayload, installationId) + if err != nil { + slog.Error("Error creating GitHub client for installation_repositories event", "installationId", installationId, "error", err) + return err + } + + var errs []error + for _, repo := range event.RepositoriesAdded { + if err := upsertRepo(ctx, client, repo, installationId, appIdFromPayload, accountLogin, accountId); err != nil { + errs = append(errs, err) + } + } + + for _, repo := range event.RepositoriesRemoved { + if err := removeRepo(ctx, repo, installationId, appIdFromPayload, link.OrganisationId); err != nil { + errs = append(errs, err) + } + } + + slog.Info("Handled installation_repositories event", + "installationId", installationId, + "addedCount", len(event.RepositoriesAdded), + "removedCount", len(event.RepositoriesRemoved), + ) + if len(errs) > 0 { + return fmt.Errorf("one or more errors during installation_repositories handling: %v", errs) + } + return nil +} diff --git a/backend/go.mod b/backend/go.mod index d25779eab..41f632b9c 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -52,6 +52,7 @@ require ( require ( github.com/diggerhq/digger/libs v0.0.0-00010101000000-000000000000 + github.com/sethvargo/go-retry v0.3.0 gorm.io/datatypes v1.2.7 ) diff --git a/backend/go.sum b/backend/go.sum index 7b0b22dad..a7cd77248 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -1801,6 +1801,8 @@ github.com/segmentio/backo-go v1.0.0/go.mod h1:kJ9mm9YmoWSkk+oQ+5Cj8DEoRCX2JT6As github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= +github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= diff --git a/backend/models/storage.go b/backend/models/storage.go index cc2e70ddb..1423c7b1d 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -511,6 +511,40 @@ func (db *Database) GithubRepoRemoved(installationId int64, appId int64, repoFul return item, nil } +// SoftDeleteRepoAndProjects soft deletes a repo and all its projects for the given org and repo full name. +func (db *Database) SoftDeleteRepoAndProjects(orgId uint, repoFullName string) error { + return db.GormDB.Transaction(func(tx *gorm.DB) error { + // Soft-delete projects first + if err := tx.Where("organisation_id = ? AND repo_full_name = ?", orgId, repoFullName).Delete(&Project{}).Error; err != nil { + slog.Error("failed to soft delete projects for repo", "orgId", orgId, "repoFullName", repoFullName, "error", err) + return err + } + // Soft-delete repo + if err := tx.Where("organisation_id = ? AND repo_full_name = ?", orgId, repoFullName).Delete(&Repo{}).Error; err != nil { + slog.Error("failed to soft delete repo", "orgId", orgId, "repoFullName", repoFullName, "error", err) + return err + } + return nil + }) +} + +// SoftDeleteReposAndProjectsByInstallation soft deletes all repos and projects for a specific installation in an org. +func (db *Database) SoftDeleteReposAndProjectsByInstallation(orgId uint, installationId int64) error { + var repos []Repo + if err := db.GormDB.Where("organisation_id = ? AND github_app_installation_id = ?", orgId, installationId).Find(&repos).Error; err != nil { + slog.Error("failed to fetch repos for soft delete", "orgId", orgId, "installationId", installationId, "error", err) + return err + } + + for _, repo := range repos { + if err := db.SoftDeleteRepoAndProjects(orgId, repo.RepoFullName); err != nil { + return err + } + } + + return nil +} + func (db *Database) GetGithubAppInstallationByOrgAndRepo(orgId any, repo string, status GithubAppInstallStatus) (*GithubAppInstallation, error) { link, err := db.GetGithubInstallationLinkForOrg(orgId) if err != nil { diff --git a/backend/models/storage_test.go b/backend/models/storage_test.go index f4aca9e0e..19498a2b5 100644 --- a/backend/models/storage_test.go +++ b/backend/models/storage_test.go @@ -125,6 +125,93 @@ func TestGithubRepoRemoved(t *testing.T) { assert.Equal(t, GithubAppInstallDeleted, i.Status) } +func TestSoftDeleteRepoAndProjects(t *testing.T) { + teardownSuite, db, org := setupSuite(t) + defer teardownSuite(t) + + installationId := int64(1) + appId := int64(1) + repoFullName := "test/test" + + repo, err := db.CreateRepo("test-test", repoFullName, "test", "test", "", org, "", installationId, appId, "main", "") + assert.NoError(t, err) + assert.NotNil(t, repo) + + project := Project{ + Name: "proj", + OrganisationID: org.ID, + Organisation: org, + RepoFullName: repoFullName, + Status: ProjectActive, + } + err = db.GormDB.Create(&project).Error + assert.NoError(t, err) + + err = db.SoftDeleteRepoAndProjects(org.ID, repoFullName) + assert.NoError(t, err) + + // Verify repo is soft-deleted + var repoRecord Repo + err = db.GormDB.Unscoped().Where("id = ?", repo.ID).First(&repoRecord).Error + assert.NoError(t, err) + assert.True(t, repoRecord.DeletedAt.Valid) + + // Verify project is soft-deleted + var projectRecord Project + err = db.GormDB.Unscoped().Where("id = ?", project.ID).First(&projectRecord).Error + assert.NoError(t, err) + assert.True(t, projectRecord.DeletedAt.Valid) +} + +func TestSoftDeleteReposAndProjectsByInstallation(t *testing.T) { + teardownSuite, db, org := setupSuite(t) + defer teardownSuite(t) + + appId := int64(1) + installA := int64(1) + installB := int64(2) + + repoA, err := db.CreateRepo("org-repo-a", "org/repo-a", "org", "repo-a", "", org, "", installA, appId, "main", "") + assert.NoError(t, err) + repoB, err := db.CreateRepo("org-repo-b", "org/repo-b", "org", "repo-b", "", org, "", installB, appId, "main", "") + assert.NoError(t, err) + + projectA := Project{ + Name: "proj-a", + OrganisationID: org.ID, + Organisation: org, + RepoFullName: repoA.RepoFullName, + Status: ProjectActive, + } + projectB := Project{ + Name: "proj-b", + OrganisationID: org.ID, + Organisation: org, + RepoFullName: repoB.RepoFullName, + Status: ProjectActive, + } + assert.NoError(t, db.GormDB.Create(&projectA).Error) + assert.NoError(t, db.GormDB.Create(&projectB).Error) + + // Soft-delete only repos for installA + err = db.SoftDeleteReposAndProjectsByInstallation(org.ID, installA) + assert.NoError(t, err) + + // Verify repoA is soft-deleted, repoB is not + var repoARecord, repoBRecord Repo + assert.NoError(t, db.GormDB.Unscoped().Where("id = ?", repoA.ID).First(&repoARecord).Error) + assert.NoError(t, db.GormDB.Unscoped().Where("id = ?", repoB.ID).First(&repoBRecord).Error) + assert.True(t, repoARecord.DeletedAt.Valid) + assert.False(t, repoBRecord.DeletedAt.Valid) + + // Verify projectA is soft-deleted, projectB is not + var projectARecord, projectBRecord Project + assert.NoError(t, db.GormDB.Unscoped().Where("id = ?", projectA.ID).First(&projectARecord).Error) + assert.NoError(t, db.GormDB.Unscoped().Where("id = ?", projectB.ID).First(&projectBRecord).Error) + assert.True(t, projectARecord.DeletedAt.Valid) + assert.False(t, projectBRecord.DeletedAt.Valid) +} + func TestGetDiggerJobsForBatchPreloadsSummary(t *testing.T) { teardownSuite, _, _ := setupSuite(t) defer teardownSuite(t) diff --git a/ee/backend/controllers/github.go b/ee/backend/controllers/github.go index 355a6f67b..d9c2225fb 100644 --- a/ee/backend/controllers/github.go +++ b/ee/backend/controllers/github.go @@ -62,6 +62,7 @@ func GithubAppConnections(c *gin.Context) { "check_run", "create", "delete", + "installation_repositories", "issue_comment", "issues", "status", diff --git a/go.work.sum b/go.work.sum index 870706a74..e52f35dbc 100644 --- a/go.work.sum +++ b/go.work.sum @@ -2062,6 +2062,7 @@ github.com/segmentio/conf v1.2.0 h1:5OT9+6OyVHLsFLsiJa/2KlqiA1m7mpdUBlkB/qYTMts= github.com/segmentio/conf v1.2.0/go.mod h1:Y3B9O/PqqWqjyxyWWseyj/quPEtMu1zDp/kVbSWWaB0= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/shirou/gopsutil/v3 v3.23.2 h1:PAWSuiAszn7IhPMBtXsbSCafej7PqUOvY6YywlQUExU= github.com/shirou/gopsutil/v3 v3.23.2/go.mod h1:gv0aQw33GLo3pG8SiWKiQrbDzbRY1K80RyZJ7V4Th1M= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= From b66a6f2babf3b66b55be6d15139fa753d51f6272 Mon Sep 17 00:00:00 2001 From: Igor Zalutski Date: Fri, 28 Nov 2025 18:40:53 +0200 Subject: [PATCH 05/10] Make repos webhook async; dont comment erorr when installed to all repos --- backend/controllers/github.go | 41 +++++++++-------- backend/controllers/github_comment.go | 2 +- backend/controllers/github_helpers.go | 4 +- backend/controllers/github_pull_request.go | 22 ++++++++- backend/utils/github.go | 53 ++++++++++++++++++++++ libs/digger_config/digger_config.go | 5 +- 6 files changed, 104 insertions(+), 23 deletions(-) diff --git a/backend/controllers/github.go b/backend/controllers/github.go index 1c1649dc1..f1a9941b0 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -65,20 +65,21 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) { "installationId", *event.Installation.ID, ) - if *event.Action == "deleted" { - err := handleInstallationDeletedEvent(event, appId64) - if err != nil { - slog.Error("Failed to handle installation deleted event", "error", err) - c.String(http.StatusAccepted, "Failed to handle webhook event.") - return - } - } else if *event.Action == "created" || *event.Action == "unsuspended" || *event.Action == "new_permissions_accepted" { - if err := handleInstallationUpsertEvent(c.Request.Context(), gh, event, appId64); err != nil { - slog.Error("Failed to handle installation upsert event", "error", err) - c.String(http.StatusAccepted, "Failed to handle webhook event.") - return + // Run in goroutine to avoid webhook timeouts for large installations + go func(ctx context.Context) { + defer logging.InheritRequestLogger(ctx)() + if *event.Action == "deleted" { + if err := handleInstallationDeletedEvent(event, appId64); err != nil { + slog.Error("Failed to handle installation deleted event", "error", err) + } + } else if *event.Action == "created" || *event.Action == "unsuspended" || *event.Action == "new_permissions_accepted" { + // Use background context so work continues after HTTP response + if err := handleInstallationUpsertEvent(context.Background(), gh, event, appId64); err != nil { + slog.Error("Failed to handle installation upsert event", "error", err) + } } - } + }(c.Request.Context()) + case *github.InstallationRepositoriesEvent: slog.Info("Processing InstallationRepositoriesEvent", "action", event.GetAction(), @@ -86,11 +87,15 @@ func (d DiggerController) GithubAppWebHook(c *gin.Context) { "added", len(event.RepositoriesAdded), "removed", len(event.RepositoriesRemoved), ) - if err := handleInstallationRepositoriesEvent(c.Request.Context(), gh, event, appId64); err != nil { - slog.Error("Failed to handle installation repositories event", "error", err) - c.String(http.StatusAccepted, "Failed to handle webhook event.") - return - } + + // Run in goroutine to avoid webhook timeouts for large installations + go func(ctx context.Context) { + defer logging.InheritRequestLogger(ctx)() + // Use background context so work continues after HTTP response + if err := handleInstallationRepositoriesEvent(context.Background(), gh, event, appId64); err != nil { + slog.Error("Failed to handle installation repositories event", "error", err) + } + }(c.Request.Context()) case *github.PushEvent: slog.Info("Processing PushEvent", "repo", *event.Repo.FullName, diff --git a/backend/controllers/github_comment.go b/backend/controllers/github_comment.go index 5bd23bbf7..92c7d664f 100644 --- a/backend/controllers/github_comment.go +++ b/backend/controllers/github_comment.go @@ -162,7 +162,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu } diggerYmlStr, ghService, config, projectsGraph, prSourceBranch, commitSha, changedFiles, err := getDiggerConfigForPR(gh, orgId, prLabelsStr, installationId, repoFullName, repoOwner, repoName, cloneURL, issueNumber) - if err != nil { + if err != nil { slog.Error("Error getting Digger config for PR", "issueNumber", issueNumber, "repoFullName", repoFullName, diff --git a/backend/controllers/github_helpers.go b/backend/controllers/github_helpers.go index 0f3906310..630081ded 100644 --- a/backend/controllers/github_helpers.go +++ b/backend/controllers/github_helpers.go @@ -831,7 +831,7 @@ func getDiggerConfigForPR(gh utils.GithubClientProvider, orgId uint, prLabels [] "branch", prBranch, "error", err, ) - return "", nil, nil, nil, nil, nil, nil, fmt.Errorf("error loading digger.yml: %v", err) + return "", nil, nil, nil, nil, nil, nil, fmt.Errorf("error loading digger.yml: %w", err) } return diggerYmlStr, ghService, config, dependencyGraph, &prBranch, &prCommitSha, changedFiles, nil @@ -894,7 +894,7 @@ func GetDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6 "branch", branch, "error", err, ) - return "", nil, nil, nil, fmt.Errorf("error cloning and loading config %v", err) + return "", nil, nil, nil, fmt.Errorf("error cloning and loading config: %w", err) } projectCount := 0 diff --git a/backend/controllers/github_pull_request.go b/backend/controllers/github_pull_request.go index a934725b3..a3661dfce 100644 --- a/backend/controllers/github_pull_request.go +++ b/backend/controllers/github_pull_request.go @@ -136,7 +136,27 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR ) return nil } - + + // Check if the error is due to missing digger config and the app is installed for all repos + if errors.Is(err, digger_config.ErrDiggerConfigNotFound) { + slog.Debug("Digger config not found, checking if app is installed for all repos", + "prNumber", prNumber, + "repoFullName", repoFullName, + ) + isAllRepos, checkErr := utils.IsAllReposInstallation(appId, installationId) + if checkErr != nil { + slog.Warn("Failed to check if installation is for all repos", + "error", checkErr, + ) + } else if isAllRepos { + slog.Info("Digger config not found but GitHub App is installed for all repos, skipping error comment", + "prNumber", prNumber, + "repoFullName", repoFullName, + ) + return nil + } + } + slog.Error("Error getting Digger config for PR", "prNumber", prNumber, "repoFullName", repoFullName, diff --git a/backend/utils/github.go b/backend/utils/github.go index d00cff511..a8671d061 100644 --- a/backend/utils/github.go +++ b/backend/utils/github.go @@ -249,6 +249,59 @@ func GetGithubHostname() string { return githubHostname } +// IsAllReposInstallation checks if the GitHub App installation is configured to access all repositories +// (as opposed to a selected subset). Returns true if installation is for "all" repos. +// Note: This requires app-level JWT authentication, not installation token authentication. +func IsAllReposInstallation(appId int64, installationId int64) (bool, error) { + githubAppPrivateKey := "" + githubAppPrivateKeyB64 := os.Getenv("GITHUB_APP_PRIVATE_KEY_BASE64") + if githubAppPrivateKeyB64 != "" { + decodedBytes, err := base64.StdEncoding.DecodeString(githubAppPrivateKeyB64) + if err != nil { + slog.Error("Failed to decode GITHUB_APP_PRIVATE_KEY_BASE64", "error", err) + return false, fmt.Errorf("error decoding private key: %v", err) + } + githubAppPrivateKey = string(decodedBytes) + } else { + githubAppPrivateKey = os.Getenv("GITHUB_APP_PRIVATE_KEY") + if githubAppPrivateKey == "" { + return false, fmt.Errorf("missing GitHub app private key") + } + } + + // Use app-level transport (JWT) instead of installation token + atr, err := ghinstallation.NewAppsTransport(net.DefaultTransport, appId, []byte(githubAppPrivateKey)) + if err != nil { + slog.Error("Failed to create GitHub app transport", + "appId", appId, + "error", err, + ) + return false, fmt.Errorf("error creating app transport: %v", err) + } + + client := github.NewClient(&net.Client{Transport: atr}) + + installation, _, err := client.Apps.GetInstallation(context.Background(), installationId) + if err != nil { + slog.Error("Failed to get GitHub installation details", + "installationId", installationId, + "error", err, + ) + return false, fmt.Errorf("error getting installation details: %v", err) + } + + repositorySelection := installation.GetRepositorySelection() + isAllRepos := repositorySelection == "all" + + slog.Debug("Checked installation repository selection", + "installationId", installationId, + "repositorySelection", repositorySelection, + "isAllRepos", isAllRepos, + ) + + return isAllRepos, nil +} + func GetWorkflowIdAndUrlFromDiggerJobId(client *github.Client, repoOwner string, repoName string, diggerJobID string) (int64, string, error) { slog.Debug("Looking for workflow for job", "diggerJobId", diggerJobID, diff --git a/libs/digger_config/digger_config.go b/libs/digger_config/digger_config.go index e42c79016..5e2b8d19c 100644 --- a/libs/digger_config/digger_config.go +++ b/libs/digger_config/digger_config.go @@ -18,6 +18,9 @@ import ( "gopkg.in/yaml.v3" ) +// ErrDiggerConfigNotFound is returned when neither digger.yml nor digger.yaml exists in the repository +var ErrDiggerConfigNotFound = errors.New("digger config file not found") + type DirWalker interface { GetDirs(workingDir string, config DiggerConfigYaml) ([]string, error) } @@ -39,7 +42,7 @@ func ReadDiggerYmlFileContents(dir string) (string, error) { slog.Error("could not read digger config file", "error", err, "dir", dir) - return "", fmt.Errorf("could not read the file both digger.yml and digger.yaml are missing: %v", err) + return "", fmt.Errorf("%w: both digger.yml and digger.yaml are missing: %v", ErrDiggerConfigNotFound, err) } } diggerYmlStr := string(diggerYmlBytes) From 1dd6d88b7c4b372a20734b9f7f12abb2a0a770fd Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 2 Dec 2025 10:55:07 -0800 Subject: [PATCH 06/10] remove unecessary check --- backend/controllers/github_pull_request.go | 11 ----------- go.work.sum | 5 ++++- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/backend/controllers/github_pull_request.go b/backend/controllers/github_pull_request.go index e456893b8..6fa30b9f6 100644 --- a/backend/controllers/github_pull_request.go +++ b/backend/controllers/github_pull_request.go @@ -9,7 +9,6 @@ import ( "runtime/debug" "slices" "strconv" - "strings" "github.com/diggerhq/digger/backend/ci_backends" config2 "github.com/diggerhq/digger/backend/config" @@ -159,16 +158,6 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR } - // Silently skip repos without digger.yml - this is expected for org-wide installations - if strings.Contains(err.Error(), "could not find digger.yml") || - strings.Contains(err.Error(), "could not find digger.yaml") { - slog.Info("No Digger config found, skipping repo", - "prNumber", prNumber, - "repoFullName", repoFullName, - ) - return nil - } - slog.Error("Error getting Digger config for PR", "prNumber", prNumber, "repoFullName", repoFullName, diff --git a/go.work.sum b/go.work.sum index 9a906912f..e9d5d226e 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1454,6 +1454,7 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8 github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633 h1:H2pdYOb3KQ1/YsqVWoWNLQO+fusocsw354rqGTZtAgw= @@ -1622,6 +1623,7 @@ github.com/google/go-pkcs11 v0.3.0 h1:PVRnTgtArZ3QQqTGtbtjtnIkzl2iY2kt24yqbrf7td github.com/google/go-pkcs11 v0.3.0/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= @@ -2031,6 +2033,7 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= @@ -2139,6 +2142,7 @@ github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4 h1:BN/Nyn2nWMo github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/zerolog v1.15.0 h1:uPRuwkWF4J6fGsJ2R0Gn2jB1EQiav9k3S6CSdygQJXY= @@ -2179,7 +2183,6 @@ github.com/segmentio/conf v1.2.0 h1:5OT9+6OyVHLsFLsiJa/2KlqiA1m7mpdUBlkB/qYTMts= github.com/segmentio/conf v1.2.0/go.mod h1:Y3B9O/PqqWqjyxyWWseyj/quPEtMu1zDp/kVbSWWaB0= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= -github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/shirou/gopsutil/v3 v3.23.2 h1:PAWSuiAszn7IhPMBtXsbSCafej7PqUOvY6YywlQUExU= github.com/shirou/gopsutil/v3 v3.23.2/go.mod h1:gv0aQw33GLo3pG8SiWKiQrbDzbRY1K80RyZJ7V4Th1M= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= From 88424cbfa64a1c853fb8d8a3d597547fafcff883 Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 2 Dec 2025 11:21:35 -0800 Subject: [PATCH 07/10] unnecessary comments --- libs/ci/github/github.go | 50 +--------------------------------------- 1 file changed, 1 insertion(+), 49 deletions(-) diff --git a/libs/ci/github/github.go b/libs/ci/github/github.go index cfa95c4f7..26d8388ca 100644 --- a/libs/ci/github/github.go +++ b/libs/ci/github/github.go @@ -490,55 +490,7 @@ func (svc GithubService) UpdateCheckRun(checkRunId string, options GithubCheckRu opts.Conclusion = github.String(*conclusion) } - checkRun, resp, err := client.Checks.UpdateCheckRun(ctx, owner, repoName, checkRunIdInt64, opts) - - // Log rate limit information - if resp != nil { - limit := resp.Header.Get("X-RateLimit-Limit") - remaining := resp.Header.Get("X-RateLimit-Remaining") - reset := resp.Header.Get("X-RateLimit-Reset") - - if limit != "" && remaining != "" { - limitInt, _ := strconv.Atoi(limit) - remainingInt, _ := strconv.Atoi(remaining) - - // Calculate percentage remaining - var percentRemaining float64 - if limitInt > 0 { - percentRemaining = (float64(remainingInt) / float64(limitInt)) * 100 - } - - // Log based on severity - if remainingInt == 0 { - slog.Error("GitHub API rate limit EXHAUSTED", - "operation", "UpdateCheckRun", - "checkRunId", checkRunId, - "limit", limit, - "remaining", remaining, - "reset", reset, - "owner", owner, - "repo", repoName) - } else if percentRemaining < 20 { - slog.Warn("GitHub API rate limit getting LOW", - "operation", "UpdateCheckRun", - "checkRunId", checkRunId, - "limit", limit, - "remaining", remaining, - "percentRemaining", fmt.Sprintf("%.1f%%", percentRemaining), - "reset", reset, - "owner", owner, - "repo", repoName) - } else { - slog.Debug("GitHub API rate limit status", - "operation", "UpdateCheckRun", - "checkRunId", checkRunId, - "limit", limit, - "remaining", remaining, - "percentRemaining", fmt.Sprintf("%.1f%%", percentRemaining)) - } - } - } - + checkRun, _, err := client.Checks.UpdateCheckRun(ctx, owner, repoName, checkRunIdInt64, opts) if err != nil { slog.Error("Failed to update check run", "inputCheckRunId", checkRunId, From 72a98fbce9bcf049f5f70bc18c392be378163790 Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 2 Dec 2025 11:53:03 -0800 Subject: [PATCH 08/10] remove commented out code, ensure code parameter handled gracefully --- backend/controllers/github_callback.go | 4 ++++ backend/controllers/projects.go | 2 -- backend/models/scheduler.go | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/backend/controllers/github_callback.go b/backend/controllers/github_callback.go index 4337ce5a0..6855fedd3 100644 --- a/backend/controllers/github_callback.go +++ b/backend/controllers/github_callback.go @@ -31,6 +31,10 @@ func (d DiggerController) GithubAppCallbackPage(c *gin.Context) { code := "" if codeExists && len(codeParams) > 0 && len(codeParams[0]) > 0 { code = codeParams[0] + } else { + slog.Debug("No code parameter found, probably a setup update, going to return success since we are relying on webhooks now") + c.HTML(http.StatusOK, "github_success.tmpl", gin.H{}) + return } appId := c.Request.URL.Query().Get("state") diff --git a/backend/controllers/projects.go b/backend/controllers/projects.go index fe5bcb745..6ebd2695a 100644 --- a/backend/controllers/projects.go +++ b/backend/controllers/projects.go @@ -1026,7 +1026,6 @@ func (d DiggerController) SetJobStatusForProject(c *gin.Context) { c.JSON(http.StatusInternalServerError, gin.H{"error": "Error getting refreshed batch"}) return } - //err = UpdateCheckStatusForBatch(d.GithubClientProvider, refreshedBatch) slog.Debug("Attempting to update GitHub Check Run for batch", "batchId", batch.ID, "checkRunId", refreshedBatch.CheckRunId, @@ -1056,7 +1055,6 @@ func (d DiggerController) SetJobStatusForProject(c *gin.Context) { c.JSON(http.StatusInternalServerError, gin.H{"error": "Error getting refreshed job"}) return } - //err = UpdateCommitStatusForJob(d.GithubClientProvider, refreshedJob) slog.Debug("Attempting to update GitHub Check Run for job", "jobId", jobId, "checkRunId", refreshedJob.CheckRunId, diff --git a/backend/models/scheduler.go b/backend/models/scheduler.go index 87f193dba..4af04fc25 100644 --- a/backend/models/scheduler.go +++ b/backend/models/scheduler.go @@ -258,7 +258,7 @@ func GetCheckRunConclusionForJob(job *DiggerJob) (string, error) { case orchestrator_scheduler.DiggerJobFailed: return "failure", nil } - slog.Error("Unknown job status in GetCheckRunConclusionForJob - this will cause GitHub API 422 error", + slog.Error("Unknown job status in GetCheckRunConclusionForJob - this will cause GitHub API 422 error", "jobId", job.DiggerJobID, "jobStatus", job.Status, "jobStatusInt", int(job.Status), From a18156dcb54119c236dd7ae2f8f897ebc2d9522d Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 2 Dec 2025 12:12:38 -0800 Subject: [PATCH 09/10] fix error --- backend/models/scheduler.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/models/scheduler.go b/backend/models/scheduler.go index 4af04fc25..f92cd22f4 100644 --- a/backend/models/scheduler.go +++ b/backend/models/scheduler.go @@ -258,10 +258,10 @@ func GetCheckRunConclusionForJob(job *DiggerJob) (string, error) { case orchestrator_scheduler.DiggerJobFailed: return "failure", nil } - slog.Error("Unknown job status in GetCheckRunConclusionForJob - this will cause GitHub API 422 error", - "jobId", job.DiggerJobID, - "jobStatus", job.Status, - "jobStatusInt", int(job.Status), - "validStatuses", []string{"created", "triggered", "started", "queued_for_run", "succeeded", "failed"}) + slog.Error("Unknown job status in GetCheckRunConclusionForJob - this will cause GitHub API 422 error", + "jobId", job.DiggerJobID, + "jobStatus", job.Status, + "jobStatusInt", int(job.Status), + "validStatuses", []string{"created", "triggered", "started", "queued_for_run", "succeeded", "failed"}) return "", fmt.Errorf("unknown job status: %v", job.Status) } From d5c292cfce9e5f546b258091979abe2daad7dc4e Mon Sep 17 00:00:00 2001 From: motatoes Date: Tue, 2 Dec 2025 12:20:05 -0800 Subject: [PATCH 10/10] ensure no unnecessary ai summaries generated --- backend/controllers/projects_helpers.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/backend/controllers/projects_helpers.go b/backend/controllers/projects_helpers.go index 2c44a7809..3870d3190 100644 --- a/backend/controllers/projects_helpers.go +++ b/backend/controllers/projects_helpers.go @@ -210,9 +210,12 @@ func UpdateCheckRunForBatch(gh utils.GithubClientProvider, batch *models.DiggerB return fmt.Errorf("error generating realtime comment message: %v", err) } - summary, err := GenerateChecksSummaryForBatch(batch) - if err != nil { - slog.Warn("Error generating checks summary for batch", "batchId", batch.ID, "error", err) + var summary = "" + if batch.Status == orchestrator_scheduler.BatchJobSucceeded || batch.Status == orchestrator_scheduler.BatchJobFailed { + summary, err = GenerateChecksSummaryForBatch(batch) + if err != nil { + slog.Warn("Error generating checks summary for batch", "batchId", batch.ID, "error", err) + } } if isPlanBatch { @@ -397,11 +400,15 @@ func UpdateCheckRunForJob(gh utils.GithubClientProvider, job *models.DiggerJob) "```\n" - summary, err := GenerateChecksSummaryForJob(job) - if err != nil { - slog.Warn("Error generating checks summary for batch", "batchId", batch.ID, "error", err) + var summary = "" + if job.Status == orchestrator_scheduler.DiggerJobSucceeded || job.Status == orchestrator_scheduler.DiggerJobFailed { + summary, err = GenerateChecksSummaryForJob(job) + if err != nil { + slog.Warn("Error generating checks summary for batch", "batchId", batch.ID, "error", err) + } } + slog.Debug("Updating PR status for job", "jobId", job.DiggerJobID, "status", status, "conclusion", conclusion) if isPlan { title := fmt.Sprintf("%v to create %v to update %v to delete", job.DiggerJobSummary.ResourcesCreated, job.DiggerJobSummary.ResourcesUpdated, job.DiggerJobSummary.ResourcesDeleted)