From 2c27e13f7966886de438ac33d6001d176619658a Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sat, 25 Jul 2026 03:28:02 -0700 Subject: [PATCH] fix(selfhost): replace Sentry with PostHog for error tracking (#8287) Removes src/selfhost/sentry.ts and every real call site now that PostHog error tracking is a full replacement, not a parallel-run sink (2026-07-25 epic #8286 correction). PostHog already mirrored Sentry's capture/monitor/ structured-log-forwarding behavior field-for-field, so this is mostly subtraction: strip the Sentry imports, move the one shared constant (operational tag keys) into redaction-scrub.ts, and drop @sentry/node + @sentry/opentelemetry from the root package.json (the hosted Worker's own separate @sentry/cloudflare + @sentry/hono integration is untouched). Also wires self-host's OpenTelemetry exporter to default its OTLP trace endpoint at PostHog's own distributed-tracing product (beta, plain OTLP/ HTTP) when POSTHOG_API_KEY is set and no explicit OTEL_EXPORTER_OTLP_* override is given -- this replaces the bespoke Sentry span-processor/ sampler bridge with a strictly simpler generic-OTLP path, so removing the bridge is not a capability loss. Updates the self-hosting-operations docs and every affected drift-guard test accordingly. Deliberately out of scope, left for follow-up PRs: the hosted ORB Worker's own Sentry middleware (src/api/routes.ts), and the Grafana Sentry-issues dashboard + release pipeline's @sentry/cli source-map upload (neither is coupled to this error-tracking code path). --- .../content/docs/self-hosting-docs-audit.mdx | 2 +- .../content/docs/self-hosting-operations.mdx | 183 +- .../src/lib/selfhost-docs-audit.ts | 8 +- .../src/lib/selfhost-env-reference.ts | 43 +- package-lock.json | 2 - package.json | 2 - scripts/branding-drift-baseline.json | 1 - src/env.d.ts | 14 +- src/queue/ai-review-orchestration.ts | 48 +- src/queue/processors.ts | 32 +- src/review/active-review-reconciliation.ts | 4 +- src/selfhost/host-pressure.ts | 2 +- src/selfhost/monitored-work.ts | 24 +- src/selfhost/orb-collector.ts | 4 +- src/selfhost/otel.ts | 142 +- src/selfhost/pg-queue.ts | 50 +- src/selfhost/posthog.ts | 39 +- src/selfhost/redaction-scrub.ts | 33 + src/selfhost/sentry.ts | 644 ------- src/selfhost/sqlite-queue.ts | 51 +- src/selfhost/tracing.ts | 4 +- src/server.ts | 94 +- src/services/agent-action-executor.ts | 16 +- test/unit/agent-action-executor.test.ts | 16 +- test/unit/ai-review-advisory.test.ts | 8 +- .../docs-selfhost-audit-checklist.test.ts | 6 +- ...ocs-selfhost-posthog-observability.test.ts | 9 +- ...docs-selfhost-sentry-observability.test.ts | 57 - test/unit/queue-2.test.ts | 4 +- test/unit/queue-3.test.ts | 1 - test/unit/queue-4.test.ts | 4 +- test/unit/queue-5.test.ts | 1 - test/unit/queue-lifecycle-guards.test.ts | 1 - test/unit/queue.test.ts | 1 - .../unit/self-host-ops-rename-residue.test.ts | 11 - test/unit/selfhost-monitored-work.test.ts | 16 +- test/unit/selfhost-otel.test.ts | 253 +-- test/unit/selfhost-pg-queue.test.ts | 16 +- test/unit/selfhost-sentry.test.ts | 1473 ----------------- test/unit/selfhost-sqlite-queue.test.ts | 16 +- 40 files changed, 372 insertions(+), 2963 deletions(-) delete mode 100644 src/selfhost/sentry.ts delete mode 100644 test/unit/docs-selfhost-sentry-observability.test.ts delete mode 100644 test/unit/selfhost-sentry.test.ts diff --git a/apps/loopover-ui/content/docs/self-hosting-docs-audit.mdx b/apps/loopover-ui/content/docs/self-hosting-docs-audit.mdx index feb51302e0..e0cd614116 100644 --- a/apps/loopover-ui/content/docs/self-hosting-docs-audit.mdx +++ b/apps/loopover-ui/content/docs/self-hosting-docs-audit.mdx @@ -80,7 +80,7 @@ When you change runtime behavior, update the docs page **and** extend the drift - **Core stack (default):** loopover + Redis + SQLite on the mounted data volume; `SELFHOST_DEPLOYMENT_MODE=dry-run` in `.env.selfhost.example`. - **Optional profiles:** Postgres, REES sidecar, observability (Prometheus, Grafana, OTEL, Tempo, Loki), and backup — each documented on its concern page; none are required for a first healthy instance. -- **Sentry:** opt-in and off by default — set an operator-owned `SENTRY_DSN` or mount `SENTRY_DSN_FILE`; see [Operations](/docs/self-hosting-operations). +- **PostHog error tracking:** opt-in and off by default, replaces Sentry — set an operator-owned `POSTHOG_API_KEY`; see [Operations](/docs/self-hosting-operations). - **AI / RAG / REES:** off in the starter env until webhook delivery and deterministic review are verified; each has explicit enablement switches. - **Operator-owned paths:** `loopover-config/`, `loopover-data`, and secrets via `.env` or `*_FILE` mounts — never baked into images. diff --git a/apps/loopover-ui/content/docs/self-hosting-operations.mdx b/apps/loopover-ui/content/docs/self-hosting-operations.mdx index 8df82316fd..786e9e14b3 100644 --- a/apps/loopover-ui/content/docs/self-hosting-operations.mdx +++ b/apps/loopover-ui/content/docs/self-hosting-operations.mdx @@ -59,8 +59,8 @@ connection pressure, lock waits, long transactions, deadlocks, database/table gr tuples, autovacuum activity, and backup freshness. Backup freshness appears when the `backup` profile is active. -When OpenTelemetry and Sentry are enabled, job audit logs and Sentry events include -trace_id/span_id fields so an operator can jump from a failed job or issue to the matching +When OpenTelemetry and PostHog error tracking are enabled, job audit logs and PostHog events +include trace_id/span_id fields so an operator can jump from a failed job or issue to the matching trace in Grafana or Tempo. -## Enabling Sentry (your own DSN) +## Enabling PostHog error tracking (replaces Sentry) -Sentry is **opt-in and off by default**. Leave `SENTRY_DSN` unset -for a complete no-op with negligible overhead — no events leave your box. When you want -error tracking, point the runtime at a project you control in your own Sentry organization. - - - -Official release images bake `LOOPOVER_VERSION` as the default release id; -override with `SENTRY_RELEASE` when you tag custom builds. Mount secrets with -`SENTRY_DSN_FILE` instead of inline env when you prefer a file-backed DSN. After -changing Sentry env, restart the `loopover` service — there is no hot reload. - - - Community self-hosters should send events only to their own DSN. The shipped stack never phones - home to a maintainer-owned project unless you configure one. - - -## Enabling PostHog error tracking (parallel run alongside Sentry) - -PostHog error tracking is **opt-in and off by default**, and runs **in parallel with Sentry -above** — both sinks are active simultaneously when both are configured; enabling one does not -disable the other. Leave `POSTHOG_API_KEY` unset for a complete no-op with negligible overhead. -The same key also activates MCP tool-call telemetry (`src/mcp/telemetry.ts`) — one PostHog -project covers both surfaces. +PostHog error tracking is **opt-in and off by default** and **replaces Sentry** entirely for this +surface — there is no Sentry integration left to run alongside it. Leave `POSTHOG_API_KEY` unset +for a complete no-op with negligible overhead — no events leave your box. The same key also +activates MCP tool-call telemetry (`src/mcp/telemetry.ts`) — one PostHog project covers both +surfaces. -Every captured event is scrubbed by the same redaction primitives Sentry's `beforeSend` uses -(`src/selfhost/redaction-scrub.ts`, shared by both sinks so there is one security-critical -implementation, not two independently-drifting copies): secret-shaped keys/values, JWTs, local -filesystem paths, and installation ids (hashed, never raw) are all redacted before an event -leaves the box. +Official release images bake `LOOPOVER_VERSION` as the default release id; override with +`POSTHOG_RELEASE` when you tag custom builds. Mount secrets with a `POSTHOG_API_KEY_FILE` +instead of inline env when you prefer a file-backed key. After changing PostHog env, restart the +`loopover` service — there is no hot reload. -PostHog's own [exception autocapture](https://posthog.com/docs/error-tracking/installation/node) -is enabled, so genuinely uncaught exceptions and unhandled rejections are captured automatically -— mirroring Sentry's own default global-handler posture — in addition to the explicit capture -call sites mirrored throughout the codebase. + + Community self-hosters should send events only to their own PostHog project. The shipped stack + never phones home to a maintainer-owned project unless you configure one. + -There is no PostHog equivalent to Sentry Cron Monitors' missed-check-in alerting, so the five -recurring jobs (`scheduled-loop`, `orb-export`, `orb-relay-drain`, `orb-relay-register`, -`queue-dead-letter-revive`) instead emit a plain heartbeat event, `orb_monitor_heartbeat` -(properties: `monitor`, `status: "ok" | "error"`, `duration_ms`), on every run. Configure a -PostHog insight alert on a no-data condition over that event (filtered to a given monitor name) -to reproduce the same "silent death is an alert" property Sentry Crons provides. +Every captured event is scrubbed by the same redaction primitives (`src/selfhost/redaction-scrub.ts`): +secret-shaped keys/values, JWTs, local filesystem paths, and installation ids (hashed, never raw) +are all redacted before an event leaves the box. -After changing PostHog env, restart the `loopover` service — there is no hot reload. +PostHog's own [exception autocapture](https://posthog.com/docs/error-tracking/installation/node) +is enabled, so genuinely uncaught exceptions and unhandled rejections are captured automatically, +in addition to the explicit capture call sites throughout the codebase. ## Browser Sentry (operator UI) The operator UI (`apps/loopover-ui`) has its own, separate client-side Sentry integration for route errors, unhandled browser exceptions, and failed app-level resource -loads — independent of the backend's `SENTRY_DSN` above. +loads — independent of the backend's PostHog error tracking above (not yet migrated; see +epic #8286). **Opt-in and off by default**: leave `VITE_SENTRY_DSN` unset for a complete no-op — the SDK is never even fetched by the browser. Session Replay is not enabled. @@ -628,24 +598,26 @@ low-cardinality set: `route` (pathname only), `release`, whenever `apps/loopover-ui` changes on `main`. -## Sentry context taxonomy +## PostHog event properties -Self-host Sentry events carry a small, scrubbed taxonomy so operators can filter by +Self-host PostHog events carry a small, scrubbed set of properties so operators can filter by subsystem without opening raw payloads. Structured error logs forwarded from -`console.error` use the JSON `event` slug as the issue type; direct -captures use a `kind` or review `operation` tag instead. +`console.error` use the JSON `event` slug as the captured error's name; direct +captures use a `kind` or review `operation` property instead. Unlike Sentry's separate +tags/contexts/extra namespaces, every property below lives in one flat `properties` bag on the +captured event. -Cron monitor slugs follow -`loopover-selfhost-{environment}-{loop}` (for example -`loopover-selfhost-production-scheduled-loop`). Pair monitor alerts with queue -depth, dead-job counts, and the matching structured log event. - -## Sentry alert classes and runbook +## PostHog alert classes and runbook -Tune Sentry alert rules for **persistent failure classes**, not one-off +Tune PostHog insight alerts for **persistent failure classes**, not one-off fail-open noise. The table below lists actionable signals, what they usually mean, and the first checks an operator should run. Prometheus/Grafana alerts in the observability profile -cover the same failure modes from a metrics angle — use both when Sentry is enabled. +cover the same failure modes from a metrics angle — use both when PostHog is enabled.
- + @@ -753,11 +720,12 @@ cover the same failure modes from a metrics angle — use both when Sentry is en
Alert classSentry signalPostHog signal Threshold guidance First response
Scheduled monitor misses - Sentry monitor alert on scheduled-loop, orb-export, or other - wrapped loops + PostHog insight alert on a no-data condition over orb_monitor_heartbeat, + filtered to scheduled-loop, orb-export, or other wrapped loops - Use Sentry's built-in monitor failure thresholds (2 consecutive misses on most loops) + Alert when a monitor's heartbeat is missing for roughly 2 expected intervals (there is no + built-in failure threshold like Sentry Crons had — configure the no-data window yourself) Process may still be alive but cron work stopped — check selfhost_cron_error, @@ -769,25 +737,45 @@ cover the same failure modes from a metrics angle — use both when Sentry is en
-## Sentry server name +## PostHog server name -`SENTRY_SERVER_NAME` sets a clean, human name for this instance in Sentry (for +`POSTHOG_SERVER_NAME` sets a clean, human name for this instance in PostHog (for example `loopover-us-east`). Unset defaults to the OS hostname — never the public-origin URL. Set this explicitly if you run more than one instance and want to tell -their Sentry events apart at a glance instead of matching container hostnames. +their PostHog events apart at a glance instead of matching container hostnames. -## Sentry tracing +## Distributed tracing -Leave `SENTRY_TRACES_SAMPLE_RATE` unset or blank to disable trace export, or set -a positive sample rate such as `0.05` to send sampled review spans to Sentry. The -custom OpenTelemetry provider installs Sentry hooks for review-stage spans carrying repo, -PR, operation, outcome, and hashed installation tags. +Self-host's OpenTelemetry provider is standard OTLP/HTTP — set +`OTEL_TRACES_EXPORTER=otlp` to enable trace export at all. Where the traces go depends on +what else is configured: -## Sentry cron monitors + -When `SENTRY_DSN` is set, the self-host runtime emits Sentry monitor check-ins -for the recurring loops where silent stoppage matters most. Leaving `SENTRY_DSN` -unset keeps monitor reporting off. +Review-stage spans carry repo, PR, operation, outcome, and hashed installation attributes. +`OTEL_TRACES_SAMPLER`/`OTEL_TRACES_SAMPLER_ARG` control sampling the same way any standard OTel +SDK does (default: `parentbased_always_on`). + +## PostHog monitor heartbeats + +When `POSTHOG_API_KEY` is set, the self-host runtime emits a PostHog heartbeat event, +`orb_monitor_heartbeat` (properties: `monitor`, `status: "ok" | "error"`, `duration_ms`), for the +recurring loops where silent stoppage matters most, on every run. Leaving `POSTHOG_API_KEY` +unset keeps monitor reporting off. Unlike Sentry Cron Monitors, PostHog has no built-in missed-check-in +alerting — configure a PostHog insight alert on a no-data condition over that event (filtered to +a given monitor name) to get the same "silent death is an alert" coverage. -Monitor loop slugs (the `{loop}` segment in the slug) are +The `monitor` property on each heartbeat event is the loop's own name: `scheduled-loop`, `orb-export`, `orb-relay-drain`, -`orb-relay-register`, and `queue-dead-letter-revive`. A missed monitor +`orb-relay-register`, and `queue-dead-letter-revive`. A missed heartbeat means the process may still be alive but the recurring work is not checking in on schedule. Pair the monitor with queue depth, dead-job counts, and the structured error log for the same subsystem. ## Grafana Sentry data source (in-Grafana issue visualization) + + Self-host no longer sends new events to Sentry (error tracking now goes through PostHog, above) + -- this panel only remains useful for an operator's own PRE-EXISTING Sentry issue history in an + org they still separately maintain. It is pending removal (tracked under epic #8286). + + Query recent Sentry issues, top issues by event count, and error-volume trend directly in Grafana — no more switching tabs to check Sentry, and errors line up in time with the rest of the stack's metrics/logs/traces. This is read-only visualization; alert routing to Sentry/Discord/Slack is a separate, unrelated concern covered above. - - The `SENTRY_DSN` above authenticates event **ingestion** (sending errors to Sentry), not the - read/query API this data source needs. You need a separate **Sentry Internal Integration token**: - Sentry → Settings → Developer Settings → Custom Integrations → New Internal Integration (requires - an Admin/Manager/Owner role in Sentry), with **Read** access on the **Project**, **Issue & - Event**, and **Organization** resource scopes. + + This data source needs a separate **Sentry Internal Integration token**, not an event-ingestion + DSN: Sentry → Settings → Developer Settings → Custom Integrations → New Internal Integration + (requires an Admin/Manager/Owner role in Sentry), with **Read** access on the **Project**, + **Issue & Event**, and **Organization** resource scopes. The [grafana-sentry-datasource](https://grafana.com/grafana/plugins/grafana-sentry-datasource/) diff --git a/apps/loopover-ui/src/lib/selfhost-docs-audit.ts b/apps/loopover-ui/src/lib/selfhost-docs-audit.ts index 01849371bd..fdd2a4ca2c 100644 --- a/apps/loopover-ui/src/lib/selfhost-docs-audit.ts +++ b/apps/loopover-ui/src/lib/selfhost-docs-audit.ts @@ -179,11 +179,11 @@ export const SELFHOST_SOURCE_OF_TRUTH_ROWS: readonly SelfHostSourceOfTruthRow[] driftGuard: "docs-selfhost-release-checklist-event-names.test.ts", }, { - topic: "Sentry (opt-in, operator-owned DSN)", - runtimeSources: ["src/selfhost/sentry.ts", "docker-compose.yml"], + topic: "PostHog error tracking (opt-in, replaces Sentry)", + runtimeSources: ["src/selfhost/posthog.ts", "docker-compose.yml"], docsPath: "/docs/self-hosting-operations", - driftGuard: "docs-selfhost-sentry-observability.test.ts", - notes: "Sentry is off by default until SENTRY_DSN or SENTRY_DSN_FILE is set.", + driftGuard: "docs-selfhost-posthog-observability.test.ts", + notes: "PostHog is off by default until POSTHOG_API_KEY is set.", }, { topic: "OTEL metrics/traces and Grafana dashboards", diff --git a/apps/loopover-ui/src/lib/selfhost-env-reference.ts b/apps/loopover-ui/src/lib/selfhost-env-reference.ts index f4701460cf..cff7ff3ed9 100644 --- a/apps/loopover-ui/src/lib/selfhost-env-reference.ts +++ b/apps/loopover-ui/src/lib/selfhost-env-reference.ts @@ -429,13 +429,21 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ name: "PORT", firstReference: "src/server.ts", }, + { + name: "POSTHOG_API_KEY", + firstReference: "src/selfhost/otel.ts", + }, { name: "POSTHOG_ENVIRONMENT", - firstReference: "src/server.ts", + firstReference: "src/selfhost/otel.ts", + }, + { + name: "POSTHOG_HOST", + firstReference: "src/selfhost/otel.ts", }, { name: "POSTHOG_RELEASE", - firstReference: "src/selfhost/posthog.ts", + firstReference: "src/selfhost/otel.ts", }, { name: "PUBLIC_API_ORIGIN", @@ -525,26 +533,6 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ name: "SELFHOST_SETUP_TOKEN", firstReference: "src/selfhost/preflight.ts", }, - { - name: "SENTRY_DSN", - firstReference: "src/selfhost/sentry.ts", - }, - { - name: "SENTRY_ENVIRONMENT", - firstReference: "src/selfhost/otel.ts", - }, - { - name: "SENTRY_RELEASE", - firstReference: "src/selfhost/otel.ts", - }, - { - name: "SENTRY_SERVER_NAME", - firstReference: "src/selfhost/sentry.ts", - }, - { - name: "SENTRY_TRACES_SAMPLE_RATE", - firstReference: "src/selfhost/sentry.ts", - }, { name: "SETUP_OUTPUT_PATH", firstReference: "src/server.ts", @@ -664,8 +652,10 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `PGPOOL_MAX` | `src/selfhost/queue-common.ts` |", "| `PGVECTOR_ENABLED` | `src/server.ts` |", "| `PORT` | `src/server.ts` |", - "| `POSTHOG_ENVIRONMENT` | `src/server.ts` |", - "| `POSTHOG_RELEASE` | `src/selfhost/posthog.ts` |", + "| `POSTHOG_API_KEY` | `src/selfhost/otel.ts` |", + "| `POSTHOG_ENVIRONMENT` | `src/selfhost/otel.ts` |", + "| `POSTHOG_HOST` | `src/selfhost/otel.ts` |", + "| `POSTHOG_RELEASE` | `src/selfhost/otel.ts` |", "| `PUBLIC_API_ORIGIN` | `src/selfhost/preflight.ts` |", "| `PUBLIC_ORIGIN_ACKNOWLEDGED` | `src/server.ts` |", "| `PUBLIC_SITE_ORIGIN` | `src/server.ts` |", @@ -688,11 +678,6 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `SELFHOST_BUNDLE_ALL` | `scripts/build-selfhost.ts` |", "| `SELFHOST_SERVICE` | `scripts/smoke-observability-traces.ts` |", "| `SELFHOST_SETUP_TOKEN` | `src/selfhost/preflight.ts` |", - "| `SENTRY_DSN` | `src/selfhost/sentry.ts` |", - "| `SENTRY_ENVIRONMENT` | `src/selfhost/otel.ts` |", - "| `SENTRY_RELEASE` | `src/selfhost/otel.ts` |", - "| `SENTRY_SERVER_NAME` | `src/selfhost/sentry.ts` |", - "| `SENTRY_TRACES_SAMPLE_RATE` | `src/selfhost/sentry.ts` |", "| `SETUP_OUTPUT_PATH` | `src/server.ts` |", "| `SLACK_WEBHOOK_URL` | `src/services/notify-discord.ts` |", ].join("\n"); diff --git a/package-lock.json b/package-lock.json index b378b3f12f..66a3504e98 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,8 +25,6 @@ "@opentelemetry/sdk-trace-node": "^2.9.0", "@sentry/cloudflare": "^10.63.0", "@sentry/hono": "^10.63.0", - "@sentry/node": "^10.63.0", - "@sentry/opentelemetry": "^10.63.0", "agents": "^0.17.3", "aws4fetch": "^1.0.20", "drizzle-orm": "^0.45.2", diff --git a/package.json b/package.json index b14e80c9f4..68e80bf7c4 100644 --- a/package.json +++ b/package.json @@ -127,8 +127,6 @@ "@opentelemetry/sdk-trace-node": "^2.9.0", "@sentry/cloudflare": "^10.63.0", "@sentry/hono": "^10.63.0", - "@sentry/node": "^10.63.0", - "@sentry/opentelemetry": "^10.63.0", "agents": "^0.17.3", "aws4fetch": "^1.0.20", "drizzle-orm": "^0.45.2", diff --git a/scripts/branding-drift-baseline.json b/scripts/branding-drift-baseline.json index b161c01de0..1b116393fc 100644 --- a/scripts/branding-drift-baseline.json +++ b/scripts/branding-drift-baseline.json @@ -29,7 +29,6 @@ "src/selfhost/health.ts": 3, "src/selfhost/monitored-work.ts": 1, "src/selfhost/orb-collector.ts": 1, - "src/selfhost/sentry.ts": 2, "src/services/ai-review.ts": 4, "src/services/ai-slop.ts": 1, "src/services/ai-summaries.ts": 1, diff --git a/src/env.d.ts b/src/env.d.ts index 9895969df0..bfb4531726 100644 --- a/src/env.d.ts +++ b/src/env.d.ts @@ -281,17 +281,6 @@ declare global { * Defaults to 60 when unset. This is on top of PagerDuty's own `dedup_key` coalescing (which prevents * duplicate *incidents*, not duplicate *pages* for a still-open one). */ PAGERDUTY_COOLDOWN_MINUTES?: string; - /** Sentry noise control (#5119): the minimum severity (`info` < `warning` < `error` < `critical`) that - * reaches Sentry from captureError/captureReviewFailure/forwardStructuredLogToSentry (src/selfhost/sentry.ts), - * for any repo not present in SENTRY_REPO_MIN_SEVERITY (a JSON `{repoFullName: severity}` map, same - * deliberately-untyped pattern as PAGERDUTY_REPO_MIN_SEVERITY). Defaults to `error` when unset — matches - * every capture path's pre-#5119 behavior exactly, so an operator who never touches these vars sees no - * change. Lower a specific repo's threshold via the map (e.g. to `info`) for full visibility while - * actively debugging it, without raising Sentry noise everywhere else. Shares one severity-threshold - * resolver with PAGERDUTY_MIN_SEVERITY (src/services/severity-threshold.ts) — not a parallel concept. */ - SENTRY_MIN_SEVERITY?: string; - /** Per-repo override map for SENTRY_MIN_SEVERITY — see its doc comment for the shape and precedence. */ - SENTRY_REPO_MIN_SEVERITY?: string; PRODUCT_USAGE_HASH_SALT?: string; /** Server-to-server API bearer token — bypasses per-repo write checks (src/auth/security.ts). */ LOOPOVER_API_TOKEN?: string; @@ -656,8 +645,7 @@ declare global { * POSTHOG_API_KEY activates. Unset ⇒ the US-cloud default (https://us.i.posthog.com). Only meaningful * alongside POSTHOG_API_KEY; ignored when unconfigured. Self-host error tracking also reads its own * process.env-only vars (POSTHOG_MIN_SEVERITY, POSTHOG_REPO_MIN_SEVERITY, POSTHOG_ENVIRONMENT, - * POSTHOG_SERVER_NAME, POSTHOG_RELEASE) — not declared here, mirroring SENTRY_MIN_SEVERITY/ - * SENTRY_REPO_MIN_SEVERITY's identical self-host-only precedent (src/selfhost/sentry.ts). */ + * POSTHOG_SERVER_NAME, POSTHOG_RELEASE) — not declared here, self-host-exclusive config. */ POSTHOG_HOST?: string; } } diff --git a/src/queue/ai-review-orchestration.ts b/src/queue/ai-review-orchestration.ts index d9cf92248a..364d9d54cc 100644 --- a/src/queue/ai-review-orchestration.ts +++ b/src/queue/ai-review-orchestration.ts @@ -58,7 +58,6 @@ import { resolveEnrichmentLinkedIssue, resolveEnrichmentLinkedIssueNumbers, } from "../review/enrichment-wire"; -import { captureReviewFailure } from "../selfhost/sentry"; import { capturePostHogReviewFailure } from "../selfhost/posthog"; import { isReputationEnabled, shouldSkipAiForReputation } from "../review/reputation-wire"; import { isConvergenceRepoAllowed } from "../review/cutover-gate"; @@ -805,24 +804,8 @@ export async function runAiReviewForAdvisory( action: "The gate is held for a human reviewer rather than passed automatically; it re-evaluates on the next update.", }); - // A review that could not be produced is a real failure the maintainer must SEE — surface it to Sentry as an + // A review that could not be produced is a real failure the maintainer must SEE — surface it to PostHog as an // ERROR (this also covers the INCOHERENT_DIFF bail, which parses to a missing opinion → inconclusive). (#1468) - captureReviewFailure(new Error("AI review inconclusive — no usable verdict for the PR head"), { - kind: "review", - reason: "ai_review_inconclusive", - installationId: args.installationId, - owner: args.repoFullName.split("/")[0], - repo: args.repoFullName, - pr: args.pr.number, - head_sha: args.advisory.headSha, - ai_review_mode: args.settings.aiReviewMode, - reviewer_count: result.reviewerCount, - public_notes: hasPublicReviewAssessment(result.advisoryNotes), - // Compact strings, not the raw objects -- Sentry's normalizeDepth flattens nested entries to "[Object]" - // and destroys the per-attempt detail (LOOPOVER-2B); see formatReviewDiagnosticsForCapture. - /* v8 ignore next -- current review runner always supplies diagnostics for completed AI attempts. */ - review_diagnostics: formatReviewDiagnosticsForCapture(result.reviewDiagnostics ?? []), - }, "ai_review_inconclusive"); capturePostHogReviewFailure(new Error("AI review inconclusive — no usable verdict for the PR head"), { kind: "review", reason: "ai_review_inconclusive", @@ -893,28 +876,6 @@ export async function runAiReviewForAdvisory( }; findings.push(unavailableFinding); args.advisory.findings.push(unavailableFinding); - captureReviewFailure( - new Error("AI review did not produce public notes for the PR head"), - { - kind: "review", - reason: "ai_review_public_summary_missing", - installationId: args.installationId, - owner: args.repoFullName.split("/")[0], - repo: args.repoFullName, - pr: args.pr.number, - head_sha: args.advisory.headSha, - ai_review_mode: args.settings.aiReviewMode, - reviewer_count: result.reviewerCount, - // Same "[Object]" flattening hazard as the inconclusive capture above (LOOPOVER-2B). - /* v8 ignore next -- current review runner always supplies diagnostics for completed AI attempts. */ - review_diagnostics: formatReviewDiagnosticsForCapture(result.reviewDiagnostics ?? []), - configured_reviewers: - env.AI_REVIEW_PLAN?.reviewers?.map((reviewer) => reviewer.model) ?? - null, - combine: env.AI_REVIEW_PLAN?.combine ?? null, - }, - "ai_review_public_summary_missing", - ); capturePostHogReviewFailure( new Error("AI review did not produce public notes for the PR head"), { @@ -958,13 +919,6 @@ export async function runAiReviewForAdvisory( // error is a genuinely caught exception here (unlike the two captures above, which construct their own // Error to report a known condition) -- named to mirror the structured log's own "event" field just above, // not the exception's native class, so every unexpected review crash groups under one readable title. - captureReviewFailure(error, { - kind: "review", - installationId: args.installationId, - repo: args.repoFullName, - pr: args.pr.number, - head_sha: args.advisory.headSha, - }, "ai_review_failed"); capturePostHogReviewFailure(error, { kind: "review", installationId: args.installationId, diff --git a/src/queue/processors.ts b/src/queue/processors.ts index e4e3a3ec73..d40788c4b9 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -584,7 +584,6 @@ import { commitE2eTestToPrBranch } from "../github/e2e-test-commit"; import { shouldApplyRepoCultureProfile } from "../review/repo-culture-profile-wire"; import { applyReviewMemorySuppression, getCachedReviewSuppressions, invalidateReviewSuppressionCache, shouldApplyReviewMemory } from "../review/review-memory-wire"; import { isEnrichmentEnabled } from "../review/enrichment-wire"; -import { captureReviewFailure } from "../selfhost/sentry"; import { capturePostHogReviewFailure } from "../selfhost/posthog"; import { setReviewPipelineSpanOutcome, @@ -1119,7 +1118,7 @@ async function isRegateRepairExhausted(env: Env, repoFullName: string, pr: Pick< }); // level:"error" is deliberate, not a code failure: this line only fires once the cap above already // stopped the wasteful repair loop, so its OWN existence is the operator-visible signal (via the - // structured log → Sentry forwarder, forwardStructuredLogToSentry) that a PR kept failing repair for the + // structured log → PostHog forwarder, forwardStructuredLogToPostHog) that a PR kept failing repair for the // same head SHA — the same "surface an anomaly at error level" convention selfhost_ai_provider_failed / // selfhost_ai_providers_exhausted already use in src/selfhost/ai.ts. console.error( @@ -3904,15 +3903,15 @@ async function prReadyForReview( }).catch(() => undefined); // level:"error" is deliberate, not a code failure: this line only fires once the guard above already // stopped the wasteful re-review, so its OWN existence is the operator-visible signal (via the structured - // log → Sentry forwarder, forwardStructuredLogToSentry) that a PR's CI has been permanently stuck long + // log → PostHog forwarder, forwardStructuredLogToPostHog) that a PR's CI has been permanently stuck long // enough to need a human — the same "surface an anomaly at error level" convention selfhost_ai_provider_ // failed / selfhost_ai_providers_exhausted already use in src/selfhost/ai.ts. Rate-limited to once per // (repo, pr, headSha) per day (#4998) — the defer above still runs on every evaluation; only the log is - // coalesced, so one permanently-stuck PR doesn't flood Sentry with hundreds of copies of the same signal. + // coalesced, so one permanently-stuck PR doesn't flood PostHog with hundreds of copies of the same signal. if (!(await ciStuckRepeatLogCoalesced(env, repoFullName, pr.number, pr.headSha))) { // LOOPOVER-2F: for a release-automation PR the "stuck" state is EXPECTED (see // isReleaseAutomationHeadRef), so the once-a-day signal drops to warn — below the - // Sentry forwarder's error threshold, still in Workers Logs — while a real PR keeps + // PostHog forwarder's error threshold, still in Workers Logs — while a real PR keeps // paging at error. Sink matches the stamped level per #7806 (console.warn for warn, // console.error for error), which is why the whole call is picked, not just the level. const releaseAutomation = isReleaseAutomationHeadRef(pr.headRef); @@ -5707,7 +5706,7 @@ async function maybeHandleRepositoryRenamedWebhookEvent( * (LOOPOVER_REPO_CONFIG_DIR/{owner}__{repo}/...), which the app can only READ (the mount is read-only) * and which private-config.ts derives from the repo's CURRENT name. If one existed under the old name but * not the new one, the operator's gate/autonomy/review policy for this repo just silently reverted to - * global defaults -- loud enough to reach Sentry (level:"error"), not a routine info line, because the + * global defaults -- loud enough to reach PostHog (level:"error"), not a routine info line, because the * failure mode is exactly "reviews quietly stop matching what the operator configured," not a crash. * A cloud deployment (no local reader registered) always resolves both sides false, so this never fires there. */ @@ -9159,16 +9158,7 @@ async function maybePublishPrPublicSurface( }, }); // The advisory ran but NOTHING reached the PR (revoked token / perms removed / GitHub 5xx). For an - // advisory-only bot this is the worst failure — escalate to Sentry at error level, not just the audit ledger. - captureReviewFailure(new Error("PR public-surface publish failed — review produced output but nothing was posted to the PR"), { - kind: "publish", - installationId, - owner: repoFullName.split("/")[0], - repo: repoFullName, - pr: pr.number, - head_sha: advisory.headSha, - failedOutputs: failedOutputs.map((failure) => failure.output), - }, "pr_public_surface_publish_failed"); + // advisory-only bot this is the worst failure — escalate to PostHog at error level, not just the audit ledger. capturePostHogReviewFailure(new Error("PR public-surface publish failed — review produced output but nothing was posted to the PR"), { kind: "publish", installationId, @@ -10367,16 +10357,6 @@ async function maybePublishPrPublicSurface( aiReviewMode: settings.aiReviewMode, }, }).catch(() => undefined); - captureReviewFailure(new Error(message), { - kind: "review", - reason: "ai_review_public_summary_missing", - installationId, - repo: repoFullName, - pr: pr.number, - head_sha: advisory.headSha, - reviewer_count: aiReview?.reviewerCount ?? 0, - public_notes: hasPublicReviewAssessment(aiReview?.notes), - }, "ai_review_public_summary_missing"); capturePostHogReviewFailure(new Error(message), { kind: "review", reason: "ai_review_public_summary_missing", diff --git a/src/review/active-review-reconciliation.ts b/src/review/active-review-reconciliation.ts index c23543f672..98447bd7e7 100644 --- a/src/review/active-review-reconciliation.ts +++ b/src/review/active-review-reconciliation.ts @@ -115,8 +115,8 @@ export async function runActiveReviewReconciliation(env: Env, nowMs: number = Da // warn, not error (LOOPOVER-2K): a successful terminalization is this feature WORKING, not an anomaly -- // logging it at error level turned a backlog drain into 547 Sentry error events in a day (one per healed // row, message-fingerprint-collapsed into a single escalating issue). warn stays visible in stdout/ - // Workers Logs but sits below the default SENTRY_MIN_SEVERITY of "error" (selfhost/sentry.ts's - // resolveSentryMinSeverity), so Sentry only sees it when an operator has explicitly lowered the + // Workers Logs but sits below the default POSTHOG_MIN_SEVERITY of "error" (selfhost/posthog.ts's + // resolvePostHogMinSeverity), so PostHog only sees it when an operator has explicitly lowered the // threshold. The row_error/scan-error paths below stay at error: a FAILED heal is still an anomaly. console.warn( JSON.stringify({ diff --git a/src/selfhost/host-pressure.ts b/src/selfhost/host-pressure.ts index ae86f3de18..7e8c0af100 100644 --- a/src/selfhost/host-pressure.ts +++ b/src/selfhost/host-pressure.ts @@ -1,7 +1,7 @@ // Optional host-CPU-pressure hint for maintenance-job admission (see maintenance-admission.ts). Node-only -- // `node:os`'s loadavg() has no meaningful signal on Cloudflare Workers -- this module is imported ONLY by the // self-host Node queue backends (sqlite-queue.ts / pg-queue.ts), never by src/index.ts's Worker bundle, so a -// static `node:os` import here is safe (mirrors the existing `hostname` import in selfhost/sentry.ts). +// static `node:os` import here is safe (mirrors the existing `hostname` import in selfhost/posthog.ts). import { cpus, loadavg } from "node:os"; /** The 1-minute load average normalized per logical core, so the SAME threshold means the same thing on a diff --git a/src/selfhost/monitored-work.ts b/src/selfhost/monitored-work.ts index 856f9c756f..4a0dbcd6db 100644 --- a/src/selfhost/monitored-work.ts +++ b/src/selfhost/monitored-work.ts @@ -1,22 +1,8 @@ import type { EnqueueWebhookResult } from "../github/webhook"; import { ORB_RELAY_REGISTER_UNHEALTHY_FAILURE_STREAK, type OrbRelayRegistrationState } from "../orb/broker-client"; import { incr } from "./metrics"; -import { withSentryMonitor } from "./sentry"; import { withPostHogMonitor } from "./posthog"; -/** Run both monitor wrappers around one callback: Sentry's structured check-in + PostHog's heartbeat event, - * in parallel (#8287) -- neither observes the other's outcome, each independently no-ops when its own sink - * is unconfigured. The callback itself only runs ONCE; withPostHogMonitor wraps a thunk that replays whatever - * withSentryMonitor already produced/threw, so a failure is captured by both sinks without executing the - * underlying work twice. */ -async function withBothMonitors( - name: Parameters[0] & Parameters[0], - context: Record | undefined, - callback: () => Promise, -): Promise { - return withPostHogMonitor(name, context, () => withSentryMonitor(name, context, callback)); -} - export type OrbRelayEvent = { deliveryId: string; eventName: string; @@ -73,7 +59,7 @@ export async function runScheduledLoopWithMonitor( cron: string, scheduled: () => T | Promise, ): Promise { - return withBothMonitors( + return withPostHogMonitor( "scheduled-loop", { jobType: "scheduled-loop", cron }, () => Promise.resolve(scheduled()), @@ -84,7 +70,7 @@ export async function runOrbExportWithMonitor( exportBatch: () => Promise, log: (line: string) => void = console.log, ): Promise { - await withBothMonitors("orb-export", { jobType: "orb-export" }, async () => { + await withPostHogMonitor("orb-export", { jobType: "orb-export" }, async () => { const exported = await exportBatch(); if (exported > 0) log(JSON.stringify({ event: "selfhost_orb_export", exported })); @@ -105,7 +91,7 @@ export async function drainOrbRelayWithMonitor(args: { log?: (line: string) => void; nowMs?: number; }): Promise { - await withBothMonitors( + await withPostHogMonitor( "orb-relay-drain", { jobType: "orb-relay-drain", pendingAckCount: args.state.pendingAck.length }, async () => { @@ -250,7 +236,7 @@ export function isOrbRelayRegistrationAlerting(args: { * one-shot boot-time call never recovers from a transient broker outage without a process restart. Called on * a timer (state persists across calls), it observes + logs only the calls that actually attempted the * network request (`registered` / `failed`) — `already_registered` / `backoff` / `skipped` are silent no-ops - * so a healthy or intentionally-idle container does not spam logs/Sentry every tick. `drainState` is the + * so a healthy or intentionally-idle container does not spam logs/PostHog every tick. `drainState` is the * pull-mode drain loop's shared state (omitted/undefined in push mode, where there is no drain loop) -- its * `lastDrainAtMs` feeds the no-progress-window half of {@link isOrbRelayRegistrationAlerting}. */ export async function registerOrbRelayWithMonitor(args: { @@ -261,7 +247,7 @@ export async function registerOrbRelayWithMonitor(args: { log?: (line: string) => void; nowMs?: number; }): Promise { - await withBothMonitors("orb-relay-register", { jobType: "orb-relay-register" }, async () => { + await withPostHogMonitor("orb-relay-register", { jobType: "orb-relay-register" }, async () => { const result = await args.register(args.env, args.state); if (result.status === "skipped" || result.status === "already_registered" || result.status === "backoff") return; const mode = args.env.ORB_RELAY_MODE === "pull" ? "pull" : "push"; diff --git a/src/selfhost/orb-collector.ts b/src/selfhost/orb-collector.ts index b0086f9be5..e8c42f7774 100644 --- a/src/selfhost/orb-collector.ts +++ b/src/selfhost/orb-collector.ts @@ -236,8 +236,8 @@ export async function exportOrbBatch(db: D1Database, batchSize = 200, fetchFn: t } } catch (error) { // Was silent beyond the counter -- this catch also swallows the error rather than rethrowing, so - // withSentryMonitor's own capture (wrapping this call in monitored-work.ts) never sees it either; a hung/ - // failed export tick was previously indistinguishable from "nothing new to export" in both Sentry and Loki. + // withPostHogMonitor's own capture (wrapping this call in monitored-work.ts) never sees it either; a hung/ + // failed export tick was previously indistinguishable from "nothing new to export" in both PostHog and Loki. incr("loopover_orb_export_errors_total"); console.error(JSON.stringify({ level: "error", event: "orb_export_failed", message: String(error).slice(0, 200) })); return 0; diff --git a/src/selfhost/otel.ts b/src/selfhost/otel.ts index 9d89af9488..ee6be9cd08 100644 --- a/src/selfhost/otel.ts +++ b/src/selfhost/otel.ts @@ -1,6 +1,6 @@ import { AsyncLocalStorage } from "node:async_hooks"; -import type { Attributes, Context, ContextManager, TextMapPropagator, Tracer } from "@opentelemetry/api"; -import type { ReadableSpan, Sampler, Span, SpanProcessor } from "@opentelemetry/sdk-trace-base"; +import type { Attributes, Context, Tracer } from "@opentelemetry/api"; +import type { SpanProcessor } from "@opentelemetry/sdk-trace-base"; type OtelApi = typeof import("@opentelemetry/api"); type OtelSdk = typeof import("@opentelemetry/sdk-trace-node"); @@ -10,13 +10,6 @@ type OtelProvider = { shutdown(): Promise; }; type SpanOptions = { parentTraceParent?: string | undefined }; -export type OpenTelemetryBridge = { - sampler?: Sampler; - spanProcessor?: SpanProcessor; - propagator?: TextMapPropagator; - contextManager?: ContextManager; - validate?: () => void; -}; export type OtelTraceIds = { trace_id: string; span_id: string }; export type OtelTraceLogFields = { trace_id: string; span_id?: string }; @@ -41,13 +34,42 @@ function traceExporterEnabled(env: NodeJS.ProcessEnv): boolean { return exporters?.split(",").map((part) => part.trim().toLowerCase()).includes("otlp") === true; } +const DEFAULT_POSTHOG_OTEL_HOST = "https://us.i.posthog.com"; + +/** PostHog's distributed-tracing product (beta: https://posthog.com/docs/distributed-tracing) is a plain + * OTLP/HTTP receiver -- PostHog's own docs describe pointing an existing OTel exporter at it as the whole + * integration, not a proprietary SDK. Used only as a FALLBACK target/auth pair when the operator hasn't + * explicitly configured their own OTEL_EXPORTER_OTLP_(TRACES_)ENDPOINT (e.g. their own collector) -- an + * explicit endpoint always wins. Reuses the SAME POSTHOG_API_KEY/POSTHOG_HOST self-host's own error + * tracking (posthog.ts) already reads off process.env (read directly here, not imported, to avoid a + * posthog.ts<->otel.ts import cycle -- posthog.ts already imports currentOtelTraceIds from this file). + * Per PostHog's docs, this must be the PROJECT token (phc_...), the same one POSTHOG_API_KEY already holds + * for error capture -- never a personal API key. */ +function resolvePostHogOtelTraceTarget(env: NodeJS.ProcessEnv): { endpoint: string; headers: Record } | undefined { + const apiKey = nonBlank(env.POSTHOG_API_KEY); + if (!apiKey) return undefined; + const host = (nonBlank(env.POSTHOG_HOST) ?? DEFAULT_POSTHOG_OTEL_HOST).replace(/\/+$/, ""); + return { endpoint: `${host}/i/v1/traces`, headers: { Authorization: `Bearer ${apiKey}` } }; +} + export function resolveOtelTraceEndpoint(env: NodeJS.ProcessEnv): string | undefined { const explicit = nonBlank(env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT); if (explicit) return explicit; const base = nonBlank(env.OTEL_EXPORTER_OTLP_ENDPOINT); - if (!base) return undefined; - const trimmed = base.replace(/\/+$/, ""); - return trimmed.endsWith("/v1/traces") ? trimmed : `${trimmed}/v1/traces`; + if (base) { + const trimmed = base.replace(/\/+$/, ""); + return trimmed.endsWith("/v1/traces") ? trimmed : `${trimmed}/v1/traces`; + } + return resolvePostHogOtelTraceTarget(env)?.endpoint; +} + +/** Extra headers to send with the OTLP trace exporter, beyond what the SDK's own native + * OTEL_EXPORTER_OTLP_(TRACES_)HEADERS env var reading already provides -- only non-empty when + * {@link resolveOtelTraceEndpoint} resolved via the PostHog fallback (an explicit operator-configured + * endpoint relies entirely on the SDK's native header handling, never overridden here). */ +function resolveOtelTraceHeaders(env: NodeJS.ProcessEnv): Record | undefined { + if (nonBlank(env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) || nonBlank(env.OTEL_EXPORTER_OTLP_ENDPOINT)) return undefined; + return resolvePostHogOtelTraceTarget(env)?.headers; } export function openTelemetryTraceExportEnabled(env: NodeJS.ProcessEnv): boolean { @@ -57,9 +79,9 @@ export function openTelemetryTraceExportEnabled(env: NodeJS.ProcessEnv): boolean function serviceAttributes(env: NodeJS.ProcessEnv): Attributes { const attrs: Attributes = { "service.name": nonBlank(env.OTEL_SERVICE_NAME) ?? "loopover-selfhost", - "deployment.environment.name": nonBlank(env.OTEL_SERVICE_ENVIRONMENT) ?? nonBlank(env.SENTRY_ENVIRONMENT) ?? "selfhost", + "deployment.environment.name": nonBlank(env.OTEL_SERVICE_ENVIRONMENT) ?? nonBlank(env.POSTHOG_ENVIRONMENT) ?? "selfhost", }; - const version = nonBlank(env.LOOPOVER_VERSION) ?? nonBlank(env.SENTRY_RELEASE); + const version = nonBlank(env.LOOPOVER_VERSION) ?? nonBlank(env.POSTHOG_RELEASE); if (version) attrs["service.version"] = version; return attrs; } @@ -80,48 +102,6 @@ function samplerFromEnv(env: NodeJS.ProcessEnv, sdk: OtelSdk) { return new sdk.ParentBasedSampler({ root: new sdk.AlwaysOnSampler() }); } -function processorSamplingContext( - parentContext: Context, - spanDecisions: WeakMap, -): Context { - const api = Otel; - const parentSpan = api?.trace.getSpan(parentContext); - if (!api || !parentSpan) return parentContext; - const sampled = spanDecisions.get(parentSpan); - if (sampled === undefined) return parentContext; - return api.trace.setSpanContext(parentContext, { - ...parentSpan.spanContext(), - traceFlags: sampled ? api.TraceFlags.SAMPLED : api.TraceFlags.NONE, - }); -} - -function sampledSpanProcessor(spanProcessor: SpanProcessor, sampler: Sampler, sampledDecision: number): SpanProcessor { - const spanDecisions = new WeakMap(); - return { - forceFlush: () => spanProcessor.forceFlush(), - shutdown: () => spanProcessor.shutdown(), - onStart(span: Span, parentContext: Context) { - const samplingContext = processorSamplingContext(parentContext, spanDecisions); - const decision = sampler.shouldSample( - samplingContext, - span.spanContext().traceId, - span.name, - span.kind, - span.attributes, - span.links, - ).decision; - const sampled = decision === sampledDecision; - spanDecisions.set(span, sampled); - if (!sampled) return; - spanProcessor.onStart(span, parentContext); - }, - onEnd(span: ReadableSpan) { - if (!spanDecisions.get(span)) return; - spanProcessor.onEnd(span); - }, - }; -} - export function otelSafeAttributes(input: Record | undefined): Attributes { const out: Attributes = {}; if (!input) return out; @@ -241,55 +221,29 @@ export function selfHostHttpResponseAttributes(status: number): Record { +/** Initialize self-host OTEL traces. A no-op unless OTEL_TRACES_EXPORTER includes "otlp" AND a trace + * endpoint resolves (explicit OTEL_EXPORTER_OTLP_(TRACES_)ENDPOINT, or a PostHog-derived fallback -- + * {@link resolveOtelTraceEndpoint}). PostHog's distributed-tracing product is plain OTLP/HTTP, so it needs + * no dedicated bridge/sampler-reconciliation machinery the way Sentry's proprietary tracing SDK once did + * (removed alongside sentry.ts) -- exactly one exporter, one sampler, straight from the OTel SDK. */ +export async function initOpenTelemetry(env: NodeJS.ProcessEnv): Promise { const endpoint = resolveOtelTraceEndpoint(env); - const otlpEnabled = Boolean(endpoint && traceExporterEnabled(env)); - if (!otlpEnabled && !bridge?.spanProcessor) return false; + if (!endpoint || !traceExporterEnabled(env)) return false; if (active) return true; const [api, sdk, exporterNs, resources] = await Promise.all([ import("@opentelemetry/api"), import("@opentelemetry/sdk-trace-node"), - otlpEnabled ? import("@opentelemetry/exporter-trace-otlp-http") : Promise.resolve(undefined), + import("@opentelemetry/exporter-trace-otlp-http"), import("@opentelemetry/resources"), ]); - const otelSampler = samplerFromEnv(env, sdk); - const independentBridgeSampler = Boolean(otlpEnabled && bridge?.spanProcessor && bridge.sampler); - const spanProcessors: SpanProcessor[] = []; - if (otlpEnabled && endpoint && exporterNs) { - const exporter = new exporterNs.OTLPTraceExporter({ url: endpoint }); - const otlpSpanProcessor = new sdk.BatchSpanProcessor(exporter); - spanProcessors.push( - independentBridgeSampler - ? sampledSpanProcessor(otlpSpanProcessor, otelSampler, sdk.SamplingDecision.RECORD_AND_SAMPLED) - : otlpSpanProcessor, - ); - } - if (bridge?.spanProcessor) { - const bridgeSpanProcessor = independentBridgeSampler && bridge.sampler - ? sampledSpanProcessor(bridge.spanProcessor, bridge.sampler, sdk.SamplingDecision.RECORD_AND_SAMPLED) - : bridge.spanProcessor; - spanProcessors.push(bridgeSpanProcessor); - } - const sampler = independentBridgeSampler - ? new sdk.AlwaysOnSampler() - : otlpEnabled - ? otelSampler - : bridge?.sampler ?? otelSampler; + const headers = resolveOtelTraceHeaders(env); + const exporter = new exporterNs.OTLPTraceExporter({ url: endpoint, ...(headers ? { headers } : {}) }); + const spanProcessors: SpanProcessor[] = [new sdk.BatchSpanProcessor(exporter)]; const nextProvider = new sdk.NodeTracerProvider({ resource: resources.resourceFromAttributes(serviceAttributes(env)), - sampler, + sampler: samplerFromEnv(env, sdk), spanProcessors, }); - if (bridge?.propagator || bridge?.contextManager) - nextProvider.register({ - ...(bridge.propagator ? { propagator: bridge.propagator } : {}), - ...(bridge.contextManager ? { contextManager: bridge.contextManager } : {}), - }); - bridge?.validate?.(); provider = nextProvider; Otel = api; tracer = nextProvider.getTracer("loopover-selfhost"); diff --git a/src/selfhost/pg-queue.ts b/src/selfhost/pg-queue.ts index f925ef2a6f..15299da657 100644 --- a/src/selfhost/pg-queue.ts +++ b/src/selfhost/pg-queue.ts @@ -8,7 +8,6 @@ import { logAudit, extractPayloadType, extractPayloadContext } from "./audit"; import { incr } from "./metrics"; import { withReviewSpan } from "./tracing"; import { withOtelSpan } from "./otel"; -import { captureError, withSentryMonitor } from "./sentry"; import { capturePostHogError, withPostHogMonitor } from "./posthog"; import { consumingRetryDelayMs, @@ -621,27 +620,22 @@ export function createPgQueue( /** Wraps reviveDeadLetterJobs() for the setInterval callback below, which has no rejection handler of its * own -- a transient pool/driver/metric failure here would otherwise surface as an unhandled promise - * rejection and can terminate the process (fatal when SENTRY_DSN is unset, since server.ts only installs - * the handler when Sentry is configured), exactly the failure mode pump()'s own try/catch above guards - * against for the main poll loop. A failed revive tick just waits for the next interval, same as a failed - * poll tick waits for the next poll. + * rejection and can terminate the process (fatal when POSTHOG_API_KEY is unset, since initPostHog's + * enableExceptionAutocapture only installs a handler when PostHog is configured), exactly the failure mode + * pump()'s own try/catch above guards against for the main poll loop. A failed revive tick just waits for + * the next interval, same as a failed poll tick waits for the next poll. * - * Also wrapped in a Sentry cron monitor (#1824): dead-letter revival stopping SILENTLY (the timer never - * fires again, e.g. after an unexpected process-level disruption) is worse than one throwing tick -- a - * crashed tick self-reports via captureError below, but a stopped one reports nothing at all without a - * monitor watching for the missed check-in. withSentryMonitor rethrows on failure so its own capture - * fires; the outer try/catch (this function's actual job) still guards the setInterval callback. */ + * Also wrapped in a PostHog monitor heartbeat (#1824): dead-letter revival stopping SILENTLY (the timer + * never fires again, e.g. after an unexpected process-level disruption) is worse than one throwing tick -- + * a crashed tick self-reports via capturePostHogError below, but a stopped one reports nothing at all + * without a monitor watching for the missed heartbeat. withPostHogMonitor rethrows on failure so its own + * capture fires; the outer try/catch (this function's actual job) still guards the setInterval callback. */ async function reviveDeadLetterJobsSafely(): Promise { try { await withPostHogMonitor( "queue-dead-letter-revive", { jobType: "queue-dead-letter-revive" }, - () => - withSentryMonitor( - "queue-dead-letter-revive", - { jobType: "queue-dead-letter-revive" }, - reviveDeadLetterJobs, - ), + reviveDeadLetterJobs, ); } catch (error) { console.error( @@ -651,7 +645,6 @@ export function createPgQueue( error: errorMessageWithCause(error), }), ); - captureError(error, { kind: "queue_dead_letter_revive_crashed" }, "queue_dead_letter_revive_crashed"); capturePostHogError(error, { kind: "queue_dead_letter_revive_crashed" }, "queue_dead_letter_revive_crashed"); } } @@ -778,7 +771,6 @@ export function createPgQueue( error: errorMessageWithCause(error), }), ); - captureError(error, { kind: "queue_foreground_liveness_release_crashed" }, "queue_foreground_liveness_release_crashed"); capturePostHogError(error, { kind: "queue_foreground_liveness_release_crashed" }, "queue_foreground_liveness_release_crashed"); } } @@ -1104,12 +1096,6 @@ export function createPgQueue( timeout_ms: processingTimeoutMs, }), ); - captureError(new Error("self-host queue processing lease expired"), { - kind: "job_recovered", - reason: "processing_timeout", - recovered, - timeoutMs: processingTimeoutMs, - }, "processing_timeout"); capturePostHogError(new Error("self-host queue processing lease expired"), { kind: "job_recovered", reason: "processing_timeout", @@ -1139,11 +1125,6 @@ export function createPgQueue( attempts: Number(job.attempts) + 1, error: "unparseable payload", }); - captureError(new Error("unparseable queue payload"), { - kind: "job_dead", - reason: "unparseable_payload", - jobId: job.id, - }, "unparseable_payload"); capturePostHogError(new Error("unparseable queue payload"), { kind: "job_dead", reason: "unparseable_payload", @@ -1453,13 +1434,6 @@ export function createPgQueue( attempts, error: errMsg, }, jobTraceParent); - captureError(error, { - kind: "job_dead", - reason: "max_retries_exhausted", - jobType: extractPayloadType(job.payload), - jobId: job.id, - attempts, - }, "job_dead"); capturePostHogError(error, { kind: "job_dead", reason: "max_retries_exhausted", @@ -1505,7 +1479,8 @@ export function createPgQueue( // claimNext()/reclaimExpiredProcessingJobs() run OUTSIDE processOne's own try/finally, so a raw pool // failure (a dropped connection, a lock timeout) lands here. Every `void pump()` call site (kickOne/kickAll) // is fire-and-forget, so an uncaught rejection here would surface as an unhandled promise rejection — fatal - // when SENTRY_DSN is unset (server.ts only installs the handler when Sentry is configured) (#2498). + // when POSTHOG_API_KEY is unset (initPostHog's enableExceptionAutocapture only installs a handler when + // PostHog is configured) (#2498). console.error( JSON.stringify({ level: "error", @@ -1513,7 +1488,6 @@ export function createPgQueue( error: errorMessageWithCause(error), }), ); - captureError(error, { kind: "queue_pump_crashed" }, "queue_pump_crashed"); capturePostHogError(error, { kind: "queue_pump_crashed" }, "queue_pump_crashed"); } finally { active--; diff --git a/src/selfhost/posthog.ts b/src/selfhost/posthog.ts index 5edad7b25f..3683f60ae0 100644 --- a/src/selfhost/posthog.ts +++ b/src/selfhost/posthog.ts @@ -1,33 +1,33 @@ -// Self-host PostHog error tracking (#8287, epic #8286). Opt-in: a complete NO-OP unless POSTHOG_API_KEY is set, -// mirroring sentry.ts's env-gated posture exactly. posthog-node is dynamically imported inside initPostHog() -// so it never enters the Worker bundle when unconfigured, same discipline as @sentry/node in sentry.ts. +// Self-host PostHog error tracking (#1468, epic #8286). Opt-in: a complete NO-OP unless POSTHOG_API_KEY is +// set. posthog-node is dynamically imported inside initPostHog() so it never enters the Worker bundle when +// unconfigured. // -// PARALLEL-RUN (epic principle, confirmed 2026-07-25): this sink runs alongside sentry.ts, not instead of it. -// Both are active simultaneously when both are configured; Sentry is only removed once the gated decommission -// issue (#8298) says so, per-issue across all six Phase-1 surfaces at once -- never removed early just because -// this surface's PostHog sink works. +// REPLACES Sentry entirely (2026-07-25 epic correction on #8286: full replacement, not a parallel-run -- +// src/selfhost/sentry.ts, @sentry/node, and @sentry/opentelemetry are gone from this surface). This file was +// originally built as a parallel sink alongside sentry.ts (#8287); its shape reflects that heritage (mirrors +// what sentry.ts used to do, field for field) even though sentry.ts itself no longer exists. // -// Reuses the SAME pure redaction primitives sentry.ts uses (./redaction-scrub) -- one security-critical scrub -// implementation, two provider-specific event-shape orchestrators. PostHog's event shape (event name + a flat -// `properties` bag) is materially simpler than Sentry's (request/contexts/extra/tags/breadcrumbs/exception), -// so this file's own orchestrator (scrubPostHogEvent) is correspondingly smaller than sentry.ts's scrubEvent. +// Reuses the pure redaction primitives originally extracted out of sentry.ts into ./redaction-scrub (#8287) -- +// PostHog's event shape (event name + a flat `properties` bag) is materially simpler than Sentry's own +// request/contexts/extra/tags/breadcrumbs/exception shape was, so this file's own orchestrator +// (scrubPostHogEvent) stays a straightforward single-bag walk. // // Env-var decision (#8287's own deliverable): POSTHOG_API_KEY/POSTHOG_HOST are the SAME vars #6235's MCP // telemetry (src/mcp/telemetry.ts) already reads off the typed Cloudflare Env -- one project key activates // both surfaces, the MCP tool-call allowlist (#6228) is untouched. Everything else here (POSTHOG_MIN_SEVERITY, // POSTHOG_REPO_MIN_SEVERITY, POSTHOG_ENVIRONMENT, POSTHOG_SERVER_NAME, POSTHOG_RELEASE) is self-host-only, -// read off real process.env exactly like sentry.ts's own SENTRY_MIN_SEVERITY/SENTRY_REPO_MIN_SEVERITY/etc -- -// never added to src/env.d.ts's typed Env, matching that file's precedent for self-host-exclusive config. +// read off real process.env, never added to src/env.d.ts's typed Env, matching that file's precedent for +// self-host-exclusive config. import { hostname } from "node:os"; import { currentOtelTraceIds, } from "./otel"; import { meetsSeverityThreshold, resolveSeverityThreshold, type LoopoverSeverity } from "../services/severity-threshold"; -import { SENTRY_OPERATIONAL_TAG_KEYS } from "./sentry"; import { hashedInstallationContext, loadNodeHasher, nonBlank, + OPERATIONAL_TAG_KEYS, REDACTED, resetRedactionScrubForTest, scrubRecord, @@ -72,9 +72,8 @@ function contextRepoFullName(properties: Record | undefined): s } /** Resolve the minimum severity for `repoFullName`: POSTHOG_REPO_MIN_SEVERITY (a JSON `{repoFullName: severity}` - * map) wins, else the global POSTHOG_MIN_SEVERITY, else `"error"`. Separate knobs from Sentry's - * SENTRY_MIN_SEVERITY/SENTRY_REPO_MIN_SEVERITY on purpose: an operator running both sinks in parallel may - * legitimately want PostHog quieter/noisier than Sentry while comparing the two during the parallel-run window. */ + * map) wins, else the global POSTHOG_MIN_SEVERITY, else `"error"` -- the quietest safe default, matching the + * behavior every capture path below already had (error/fatal-only) before this resolver existed. */ function resolvePostHogMinSeverity(repoFullName: string): LoopoverSeverity { const processEnv = (globalThis as unknown as { process?: { env?: Record } }).process?.env ?? {}; return resolveSeverityThreshold(processEnv as unknown as Env, repoFullName, "POSTHOG_MIN_SEVERITY", "POSTHOG_REPO_MIN_SEVERITY"); @@ -107,8 +106,8 @@ export function scrubPostHogEvent(event: PostHogEventMessage | null): PostHogEve } /** Build the properties bag for a captured error/log: hashes any installation id, then tags the shared - * operational key allowlist (reused from sentry.ts -- the same fields are exactly as safe to surface as a - * PostHog property as they are as a Sentry tag) alongside whatever else the caller passed. */ + * operational key allowlist (./redaction-scrub's OPERATIONAL_TAG_KEYS) alongside whatever else the caller + * passed. */ function operationalProperties(context: Record | undefined): Record { const safeContext = context ? hashedInstallationContext(context) : {}; const normalized: Record = @@ -116,7 +115,7 @@ function operationalProperties(context: Record | undefined): Re ? { ...safeContext, repo: safeContext.repository } : { ...safeContext }; const properties: Record = {}; - for (const key of SENTRY_OPERATIONAL_TAG_KEYS) { + for (const key of OPERATIONAL_TAG_KEYS) { const value = normalized[key]; if (typeof value === "string" || typeof value === "number") properties[key] = value; } diff --git a/src/selfhost/redaction-scrub.ts b/src/selfhost/redaction-scrub.ts index 101c4f4677..b154455ad5 100644 --- a/src/selfhost/redaction-scrub.ts +++ b/src/selfhost/redaction-scrub.ts @@ -38,6 +38,39 @@ export const PRIVATE_TEXT = export const PUBLIC_UNSAFE_SCRUB = new RegExp(String.raw`\b(${PUBLIC_UNSAFE_TERMS})\b`, "gi"); export const REDACTED = "[redacted]"; +/** Low-cardinality operational fields safe to surface as an indexed tag/property on a captured error -- + * formerly sentry.ts's SENTRY_OPERATIONAL_TAG_KEYS, moved here (2026-07-25 Sentry removal) since posthog.ts + * is now this repo's only consumer. */ +export const OPERATIONAL_TAG_KEYS = [ + "repo", + "repository", + "owner", + "installation_id_hash", + "pull", + "pullNumber", + "pr", + "head_sha", + "project", + "kind", + "subsystem", + "job_type", + "jobType", + "reason", + "result", + "deliveryId", + "provider", + "model", + "effort", + "timeoutMs", + "trace_id", + "span_id", + "operation", + "agent", + "decision_outcome", + "event", + "monitor", +] as const; + export function nonBlank(value: string | undefined): string | undefined { const trimmed = value?.trim(); return trimmed ? trimmed : undefined; diff --git a/src/selfhost/sentry.ts b/src/selfhost/sentry.ts deleted file mode 100644 index ee6f0b7154..0000000000 --- a/src/selfhost/sentry.ts +++ /dev/null @@ -1,644 +0,0 @@ -// Self-host-only error tracking (#1468). Opt-in: a complete NO-OP when SENTRY_DSN is unset, mirroring the -// env-gated, dynamically-imported selfhost-integration pattern (Redis/Qdrant/embed-provider in server.ts). -// @sentry/node is NEVER imported at module top level — it loads lazily inside initSentry(), so it never enters -// the Worker bundle (src/index.ts) and cloudflare:* stubbing stays clean. All helpers are safe to call when off. -// -// The pure secret/private-text redaction primitives (SECRET_KEY, scrubString, scrubRecord, installation-id -// hashing, ...) live in ./redaction-scrub (#8287) so the PostHog sink (./posthog.ts) reuses the identical -// security-critical detection instead of a hand-duplicated second copy. Only this file's own Sentry-event- -// SHAPE orchestration (scrubEvent walking request/contexts/extra/tags/breadcrumbs) stays here. -import { hostname } from "node:os"; -import { - currentOtelTraceIds, - openTelemetryTraceExportEnabled, - type OpenTelemetryBridge, -} from "./otel"; -import { queueDeadLetterReviveIntervalMs } from "./queue-common"; -import { meetsSeverityThreshold, resolveSeverityThreshold, type LoopoverSeverity } from "../services/severity-threshold"; -import { - hashedInstallationContext, - installationIdHash, - isInstallationIdKey, - loadNodeHasher, - nonBlank, - REDACTED, - resetRedactionScrubForTest, - scrubQueryString, - scrubRecord, - scrubString, - scrubStringField, - scrubUrl, - SECRET_KEY, - shouldRedactKey, -} from "./redaction-scrub"; - -type SentryNs = typeof import("@sentry/node"); -type SentryClient = NonNullable>; -type SentryMonitorConfig = NonNullable[1]>; -export type SentryMonitorName = "scheduled-loop" | "orb-export" | "orb-relay-drain" | "orb-relay-register" | "queue-dead-letter-revive"; -export const SENTRY_MONITOR_NAMES: readonly SentryMonitorName[] = ["scheduled-loop", "orb-export", "orb-relay-drain", "orb-relay-register", "queue-dead-letter-revive"]; -export const SENTRY_OPERATIONAL_SUBSYSTEMS = { webhook: "GitHub webhook ingest and enqueue", queue: "Job claim, process, dead-letter revival, and pump loops", github: "GitHub App token minting and broker calls", ai: "AI provider attempts, rate limits, and close-breaker engagement", gate: "Gate verdict and check-run publish", publish: "PR comment and public-surface publish", scheduled: "Maintenance tick, regate sweeps, and cron fan-out", backup: "Backup profile runs and freshness advisories", relay: "Orb relay register/drain and broker export loops" } as const; -export const SENTRY_OPERATIONAL_TAG_KEYS = ["repo", "repository", "owner", "installation_id_hash", "pull", "pullNumber", "pr", "head_sha", "project", "kind", "subsystem", "job_type", "jobType", "reason", "result", "deliveryId", "provider", "model", "effort", "timeoutMs", "trace_id", "span_id", "operation", "agent", "decision_outcome", "event", "monitor"] as const; -type SentryScope = { - setContext(name: string, context: Record): void; - setTag(key: string, value: string): void; -}; -let Sentry: SentryNs | undefined; -let sentryClient: SentryClient | undefined; -let sentryTraceSampleRate: number | undefined; -let active = false; -let sentryEnvironment = "production"; - -const ALLOWED_CONTEXTS = new Set([ - "loopover", - "review", - "log", - "sentry_monitor", - "otel", - "trace", - "runtime", - "os", -]); - -const SENTRY_MONITORS: Record SentryMonitorConfig) }> = { - "scheduled-loop": { - slug: "scheduled-loop", - config: { - schedule: { type: "interval", value: 2, unit: "minute" }, - checkinMargin: 3, - maxRuntime: 2, - failureIssueThreshold: 2, - recoveryThreshold: 1, - }, - }, - "orb-export": { - slug: "orb-export", - config: { - schedule: { type: "interval", value: 1, unit: "hour" }, - checkinMargin: 10, - maxRuntime: 10, - failureIssueThreshold: 2, - recoveryThreshold: 1, - }, - }, - "orb-relay-drain": { - slug: "orb-relay-drain", - config: { - schedule: { type: "interval", value: 1, unit: "minute" }, - // #6685-followup-2 (LOOPOVER-12, 1600+ "missed check-in" occurrences over 2+ weeks, still recurring): - // checkinMargin: 2 was too tight for a self-host container that gets fully recreated on every redeploy, - // not just restarted in place -- both leading in-process hypotheses were investigated and ruled out with - // real production evidence: the reentrancy-guard overlap-skip added in #instrument-drain-skip (see - // withOrbRelayDrainReentrancyGuard's own header comment) never fired once in 14 days of live coverage - // (loopover_orb_relay_drain_skipped_total stayed at 0), and the genuine in-process timeout errors - // (LOOPOVER-1Y/1Q, kind: orb_relay_drain) total only ~13 occurrences -- nowhere near enough to explain - // 1600+ missed check-ins. What DOES correlate: this container's own StartedAt resets on every redeploy - // (unlike its sidecars', which stay up for days), and boot involves an image pull + up to a 30s Postgres- - // readiness retry loop (waitForPostgres, server.ts) + app init BEFORE this drain loop's first tick can - // even register -- routinely exceeding the old 2-minute margin on a fleet that redeploys multiple times a - // day. 6 minutes gives a normal single redeploy cycle room to complete without a false "outage" issue, - // while a genuinely stuck/crash-looping instance still alerts within ~3 x (1+6) = ~21 minutes - // (failureIssueThreshold below is unchanged -- only the margin, which is what the evidence points at). - checkinMargin: 6, - maxRuntime: 1, - failureIssueThreshold: 3, - recoveryThreshold: 1, - }, - }, - "orb-relay-register": { - slug: "orb-relay-register", - config: { - schedule: { type: "interval", value: 1, unit: "minute" }, - checkinMargin: 2, - maxRuntime: 1, - failureIssueThreshold: 3, - recoveryThreshold: 1, - }, - }, - // Derived from the LIVE QUEUE_DEAD_LETTER_REVIVE_INTERVAL_MS override (default 30min, see queue-common.ts) - // rather than hard-coded: a static 30min schedule would report false missed check-ins for any operator who - // configures an interval longer than the schedule + margin window, even though the job is running exactly on - // its own configured cadence. Silent stoppage here means dead jobs never retry again without manual - // intervention (#1824), so the monitor must track whatever interval is actually in effect. - "queue-dead-letter-revive": { - slug: "queue-dead-letter-revive", - config: () => { - const intervalMinutes = Math.max(1, Math.round(queueDeadLetterReviveIntervalMs() / 60_000)); - return { - schedule: { type: "interval", value: intervalMinutes, unit: "minute" }, - checkinMargin: Math.max(5, Math.ceil(intervalMinutes / 3)), - maxRuntime: 5, - failureIssueThreshold: 2, - recoveryThreshold: 1, - }; - }, - }, -}; - -function slugPart(value: string | undefined): string { - const slug = nonBlank(value) - ?.toLowerCase() - .replace(/[^a-z0-9-]+/g, "-") - .replace(/^-+|-+$/g, "") - .slice(0, 48); - return slug || "production"; -} - -export function resolveSentryMonitorSlug( - name: SentryMonitorName, - environment = sentryEnvironment, -): string { - return `loopover-selfhost-${slugPart(environment)}-${SENTRY_MONITORS[name].slug}`; -} - -function safeMonitorContext( - name: SentryMonitorName, - monitorSlug: string, - context: Record | undefined, -): Record { - const safe: Record = { monitor: name, monitorSlug }; - if (!context) return safe; - for (const [key, value] of Object.entries(context)) { - if (SECRET_KEY.test(key) || value === null || value === undefined) continue; - if (typeof value === "string") - safe[key] = value.length > 160 ? `${value.slice(0, 157)}...` : value; - else if (typeof value === "number" && Number.isFinite(value)) safe[key] = value; - else if (typeof value === "boolean") safe[key] = value; - } - return safe; -} - -function setOtelTraceScope(scope: SentryScope): void { - const trace = currentOtelTraceIds(); - if (!trace) return; - scope.setTag("trace_id", trace.trace_id); - scope.setTag("span_id", trace.span_id); - scope.setContext("otel", { ...trace }); -} - -/** Resolve the Sentry release id from explicit override first, then the image-baked self-host version - * (LOOPOVER_VERSION). */ -export function resolveSentryRelease( - env: NodeJS.ProcessEnv, -): string | undefined { - return nonBlank(env.SENTRY_RELEASE) ?? nonBlank(env.LOOPOVER_VERSION); -} - -export function resolveSentryTracesSampleRate( - env: NodeJS.ProcessEnv, -): number | undefined { - const raw = nonBlank(env.SENTRY_TRACES_SAMPLE_RATE); - if (!raw) return undefined; - const parsed = Number(raw); - if (!Number.isFinite(parsed) || parsed <= 0) return undefined; - return Math.min(parsed, 1); -} - -/** beforeSend scrubber — redact anything token/secret-like before an event leaves the box (privacy boundary). */ -export function scrubEvent(event: T): T | null { - try { - const e = event as { - request?: Record; - contexts?: Record; - extra?: Record; - tags?: Record; - breadcrumbs?: Array>; - exception?: unknown; - logentry?: unknown; - message?: unknown; - spans?: unknown; - transaction?: unknown; - user?: unknown; - }; - scrubRequest(e.request); - scrubAllowedContexts(e.contexts); - scrubRecord(e.extra, 0); - scrubRecord(e.tags, 0); - scrubRecord(e.exception, 0); - scrubRecord(e.logentry, 0); - scrubRecord(e.spans, 0); - delete e.user; - if (typeof e.message === "string") e.message = scrubString(e.message); - if (typeof e.transaction === "string") e.transaction = scrubString(e.transaction); - if (Array.isArray(e.breadcrumbs)) { - for (const breadcrumb of e.breadcrumbs) scrubRecord(breadcrumb, 0); - } - } catch { - return null; - } - return event; -} - -function tagHashedInstallation(scope: SentryScope, context: Record): void { - const hash = installationIdHash(context.installation_id ?? context.installationId); - if (hash) scope.setTag("installation_id_hash", hash); -} - -function applyOperationalTags(scope: SentryScope, context: Record): void { const normalized: Record = typeof context.repository === "string" && context.repo === undefined ? { ...context, repo: context.repository } : { ...context }; tagHashedInstallation(scope, normalized); for (const key of SENTRY_OPERATIONAL_TAG_KEYS) { const tagValue = normalized[key]; if (typeof tagValue === "string" || typeof tagValue === "number") scope.setTag(key, String(tagValue)); } } - -function scrubRequest(request: Record | undefined): void { - if (!request) return; - scrubRecord(request.headers, 0); - for (const key of ["url", "query_string", "queryString", "query"] as const) { - const value = request[key]; - if (typeof value === "string") request[key] = scrubStringField(key, value); - else if (value && typeof value === "object") scrubRecord(value, 0); - } - for (const key of ["body", "data", "payload", "cookies"] as const) { - if (key in request) delete request[key]; - } -} - -function scrubAllowedContexts(contexts: Record | undefined): void { - if (!contexts) return; - for (const key of Object.keys(contexts)) { - if (!ALLOWED_CONTEXTS.has(key)) { - delete contexts[key]; - continue; - } - scrubRecord(contexts[key], 0); - } -} - -/** Initialize Sentry from the environment. Returns false (and stays a no-op) when SENTRY_DSN is unset. */ -export async function initSentry(env: NodeJS.ProcessEnv): Promise { - if (!env.SENTRY_DSN) return false; - await loadNodeHasher(); - Sentry = await import("@sentry/node"); - const release = resolveSentryRelease(env); - sentryTraceSampleRate = resolveSentryTracesSampleRate(env); - const useCustomOpenTelemetry = - sentryTraceSampleRate !== undefined || openTelemetryTraceExportEnabled(env); - sentryEnvironment = nonBlank(env.SENTRY_ENVIRONMENT) ?? "production"; - sentryClient = Sentry.init({ - dsn: env.SENTRY_DSN, - environment: sentryEnvironment, - ...(release ? { release } : {}), - ...(sentryTraceSampleRate !== undefined - ? { tracesSampleRate: sentryTraceSampleRate } - : {}), - ...(useCustomOpenTelemetry ? { skipOpenTelemetrySetup: true } : {}), - // Identify this instance by a CLEAN, configurable name, not the public-origin URL. An operator sets - // SENTRY_SERVER_NAME (e.g. "loopover-us-east"); unset falls back to the OS hostname. - serverName: nonBlank(env.SENTRY_SERVER_NAME) ?? hostname(), - beforeSend: (e) => scrubEvent(e), - beforeSendTransaction: (e) => scrubEvent(e), - }); - active = true; - return true; -} - -export async function buildSentryOpenTelemetryBridge(): Promise { - if (!active || !Sentry || !sentryClient) return undefined; - const SentryOtel = await import("@sentry/opentelemetry"); - const exportSentrySpans = sentryTraceSampleRate !== undefined; - return { - ...(exportSentrySpans ? { sampler: new SentryOtel.SentrySampler(sentryClient) } : {}), - propagator: new SentryOtel.SentryPropagator(), - contextManager: new Sentry.SentryContextManager(), - ...(exportSentrySpans ? { spanProcessor: new SentryOtel.SentrySpanProcessor() } : {}), - validate: () => { - Sentry?.validateOpenTelemetrySetup?.(); - }, - }; -} - -/** The repo a capture's context belongs to, for per-repo severity-threshold lookup (#5119) -- mirrors - * applyOperationalTags's own `repo`-over-`repository` normalization. `""` (never `undefined`) so - * {@link resolveSentryMinSeverity} always has a lookup key: a non-repo-scoped capture's (empty) per-repo map - * lookup simply misses and falls through to the global threshold, which is the correct behavior. */ -function contextRepoFullName(context: Record | undefined): string { - if (!context) return ""; - const repo = typeof context.repo === "string" ? context.repo : typeof context.repository === "string" ? context.repository : undefined; - return repo ?? ""; -} - -/** Resolve the minimum severity Sentry capture for `repoFullName`: SENTRY_REPO_MIN_SEVERITY (a JSON - * `{repoFullName: severity}` map) wins, else the global SENTRY_MIN_SEVERITY, else `"error"` -- the quietest - * safe default, matching today's de facto behavior (every capture path below was already error/fatal-only - * before this resolver existed). Reads the real Node `process.env` directly: captureError/ - * captureReviewFailure/forwardStructuredLogToSentry take no `env` parameter (initSentry's own env argument is - * not retained), so this is the only env self-host functions in this file can reach at capture time. */ -function resolveSentryMinSeverity(repoFullName: string): LoopoverSeverity { - const processEnv = (globalThis as unknown as { process?: { env?: Record } }).process?.env ?? {}; - return resolveSeverityThreshold(processEnv as unknown as Env, repoFullName, "SENTRY_MIN_SEVERITY", "SENTRY_REPO_MIN_SEVERITY"); -} - -/** Map a structured log's own `level` field (Sentry-native `debug`/`info`/`warning`/`warn`/`error`/`fatal`) onto - * the shared 4-tier {@link LoopoverSeverity} taxonomy for threshold comparison. `debug` folds into `info` (the - * taxonomy has no separate debug tier, matching PagerDutySeverity's shape for consistency -- #5119). A level - * that ISN'T one of these recognized severity words (e.g. `"audit"` -- a log CATEGORY, not a severity grade) - * is treated as the quietest tier (`info`), never promoted to `error` -- matching this function's pre-#5119 - * behavior of silently skipping anything that wasn't literally `error`/`fatal`. */ -function normalizeLoopoverSeverity(level: string): LoopoverSeverity { - const lower = level.toLowerCase(); - if (lower === "critical" || lower === "fatal") return "critical"; - if (lower === "error") return "error"; - if (lower === "warning" || lower === "warn") return "warning"; - return "info"; -} - -/** Name a captured Error before capture so its Sentry issue title reads "eventName: message" instead of the - * generic "Error: message" (or a caught exception's own class name, e.g. "HttpError: ..."). Mirrors - * forwardStructuredLogToSentry's `errorEvent.name = event` below, but never mutates the caught value: some - * runtime errors (notably DOMException from AbortSignal.timeout/fetch) expose a read-only `name` in strict mode. */ -function namedCaptureError(error: unknown, eventName?: string): Error { - const err = error instanceof Error ? error : new Error(String(error)); - if (!eventName) return err; - const namedError = new Error(err.message, { cause: err }); - namedError.name = eventName; - Object.defineProperty(namedError, "stack", { - value: err.stack, - configurable: true, - writable: true, - }); - return namedError; -} - -/** Capture an error with optional structured context. No-op when Sentry is off OR the repo's resolved severity - * threshold (#5119) is above `error` (the fixed grade every call here represents) -- suppressed from Sentry, - * still visible in Workers Logs/stdout via the console call that led here. `eventName`, when given, becomes - * the Sentry issue title's prefix (see {@link namedCaptureError}) AND the grouping fingerprint (#5010) -- - * Sentry's default stack-trace-based grouping fragments the SAME logical failure into separate issues whenever - * it is captured from more than one call site (e.g. two different functions each constructing the identical - * `new Error("...")` message), which is exactly what happened to GITTENSORY-5/10 and GITTENSORY-C/W before this. - * Mirrors forwardStructuredLogToSentry's identical `scope.setFingerprint(["loopover-log", event, ev?])` - * discipline (the `ev` sub-field, when present, further splits one broad `event` slug shared by several call - * sites into separate issues per actual failure mode -- see that function's own comment). */ -export function captureError( - error: unknown, - context?: Record, - eventName?: string, -): void { - if (!active || !Sentry) return; - if (!meetsSeverityThreshold("error", resolveSentryMinSeverity(contextRepoFullName(context)))) return; - Sentry.withScope((scope) => { - setOtelTraceScope(scope); - if (context) { const safeContext = hashedInstallationContext(context); scope.setContext("loopover", safeContext); applyOperationalTags(scope, safeContext); } - if (eventName) scope.setFingerprint(["loopover-error", eventName]); - Sentry!.captureException(namedCaptureError(error, eventName)); - }); -} - -/** Capture a failed review at ERROR level, tagged by repo/PR/SHA for triage. A review that cannot be produced is a - * real failure the maintainer must SEE — not a warning that hides in the noise. No-op when off OR the repo's - * resolved severity threshold (#5119) is above `error` (this always captures at error grade). `eventName`, when - * given, becomes the Sentry issue title's prefix AND the grouping fingerprint -- see {@link captureError}'s - * identical discipline and #5010. */ -export function captureReviewFailure( - error: unknown, - context?: Record, - eventName?: string, -): void { - if (!active || !Sentry) return; - if (!meetsSeverityThreshold("error", resolveSentryMinSeverity(contextRepoFullName(context)))) return; - Sentry.withScope((scope) => { - scope.setLevel("error"); - setOtelTraceScope(scope); - if (context) { - const safeContext = hashedInstallationContext(context); - scope.setContext("review", safeContext); - applyOperationalTags(scope, safeContext); - } - if (eventName) scope.setFingerprint(["loopover-review-failure", eventName]); - Sentry!.captureException(namedCaptureError(error, eventName)); - }); -} - -/** A SHORT location suffix — " (repo#pr)" — for a no-message error title, so the issue list shows WHERE without - * dumping every scalar field (which made titles unreadably long, e.g. trailing a full deliveryId). The complete - * field set is still indexed as Sentry tags + kept in the "log" context. Empty when the log carries no repo. */ -function logLocation(obj: Record): string { - const repo = - typeof obj.repository === "string" - ? obj.repository - : typeof obj.repo === "string" - ? obj.repo - : undefined; - if (!repo) return ""; - // The standard pullNumber locates the PR in the title; other pr aliases stay in the tags/context (not the title). - const pr = obj.pullNumber; - return typeof pr === "number" ? ` (${repo}#${pr})` : ` (${repo})`; -} - -/** When a log carries no message/error, summarize its SALIENT scalar fields (project, counts, precisions, …) into the - * Sentry value so a field-only log — e.g. close_breaker_engaged{project,closePrecision,floor} or closehold_backlog - * {count,projects} — shows real data instead of "(no message)". Skips meta + the location keys logLocation already - * used + long blobs (IDs/bodies stay in the indexed tags + the "log" context); caps to a few fields so the title - * stays readable. This is the STRUCTURAL fix for field-only error logs (current + future), not per-log message-adding. */ -const SUMMARY_SKIP_KEYS = new Set([ - "level", - "event", - "ts", - "time", - "timestamp", - "msg", - "ev", - "message", - "error", - "repo", - "repository", - "installationId", - "installation_id", - "installation_id_hash", - "pullNumber", - "deliveryId", - "trace_id", - "span_id", -]); -function redactSummaryValue(value: unknown, depth = 0): unknown { - if (!value || typeof value !== "object") return value; - if (depth >= 6) return "[redacted]"; - if (Array.isArray(value)) - return value.map((item) => redactSummaryValue(item, depth + 1)); - return Object.fromEntries( - Object.entries(value as Record).map(([key, nested]) => [ - key, - SECRET_KEY.test(key) - ? "[redacted]" - : redactSummaryValue(nested, depth + 1), - ]), - ); -} - -function summarizeLogFields(obj: Record): string { - return Object.entries(obj) - .filter( - ([k, v]) => !SUMMARY_SKIP_KEYS.has(k) && !SECRET_KEY.test(k) && v !== null, - ) - .map( - ([k, v]) => - `${k}=${typeof v === "object" ? JSON.stringify(redactSummaryValue(v)) : String(v)}`, - ) - .filter((part) => part.length <= 90) // a long blob (id/body) belongs in the context, not the title - .slice(0, 5) // a few salient fields, not a dump - .join(", "); -} - -/** Forward a structured console line to Sentry when its level meets the repo's resolved severity threshold - * (#5119, default `error` — matches this function's pre-#5119 hardcoded error/fatal-only behavior byte for - * byte). The engine logs operational failures (orb_broker_unavailable, gate-check errors, relay drops, …) as - * JSON strings, often via console.error. No-op when Sentry is off, the line isn't a JSON object string, or it - * carries no level at all (and isn't from the error sink) — a log with no severity signal is a data-completeness - * gap, not a below-threshold decision, so it is always skipped regardless of any repo's configured threshold. - * An operator can lower a specific repo's threshold (SENTRY_REPO_MIN_SEVERITY) to `warning` or `info` to see - * routine logs from that repo while actively debugging it, without raising Sentry noise everywhere else. */ -export function forwardStructuredLogToSentry(line: unknown, fromErrorSink = false): void { - if (!active || !Sentry) return; - if (typeof line !== "string" || line.charCodeAt(0) !== 123 /* "{" */) return; - let obj: Record; - try { - // A "{"-prefixed string that parses is always an object (else JSON.parse throws → caught below). - obj = JSON.parse(line) as Record; - } catch { - return; // not JSON — an ordinary log line - } - const safeObj = hashedInstallationContext(obj); - // A console.error sink is error-level by DEFAULT even when the JSON omits an explicit level (many engine error - // logs do) — that's how those errors reach Sentry instead of printing to stderr and vanishing. An EXPLICIT level - // always wins over the error-sink default. - const explicitLevel = typeof obj.level === "string" ? obj.level : undefined; - const level = explicitLevel ?? (fromErrorSink ? "error" : undefined); - if (!level) return; // no severity signal at all — never forwarded, independent of any threshold - const loopoverSeverity = normalizeLoopoverSeverity(level); - if (!meetsSeverityThreshold(loopoverSeverity, resolveSentryMinSeverity(contextRepoFullName(safeObj)))) return; - // Sentry's own native level string (setLevel below) — critical maps back to "fatal", its Sentry-native spelling. - const severity = loopoverSeverity === "critical" ? "fatal" : loopoverSeverity === "warning" ? "warning" : loopoverSeverity === "info" ? "info" : "error"; - const event = typeof obj.event === "string" ? obj.event : undefined; - // Many call sites share one broad `event` slug (e.g. every RAG failure mode logs `event: - // "review_context_fetch_failed"`) and rely on a finer-grained `ev` field to distinguish WHICH failure it - // actually was (rag_upsert_error vs. rag_retrieve_error vs. ...). Fold `ev` into both the title and the - // fingerprint below when present, so genuinely different failures never collapse into one misleading issue - // bucket that mixes their causes together (confirmed in the wild: GITTENSORY-D's own event history mixes a - // context-length-overflow case with an unrelated Postgres NUL-byte-rejection case). - const subEvent = typeof obj.ev === "string" ? obj.ev : undefined; - // Lead the Sentry title with the real failure detail (message → error), not just the event slug, so an operator - // sees WHAT broke straight from the issue list instead of having to open the context blob. - const detail = typeof obj.message === "string" ? obj.message : typeof obj.error === "string" ? obj.error : undefined; - // Forward as a synthetic EXCEPTION, NOT captureMessage. captureMessage leaves the exception value empty, which - // Sentry's issue UI renders as "(No error message)". An exception gives the issue a real `type: value`: - // name (type) = the event slug (e.g. check_run_post_denied) - // message (value) = the failure detail (message/error) → else the PR location → else a pointer to the context - // So the issue list always shows a legible "event: detail", never a bare slug or "(No error message)". The - // fingerprint (by event) still groups recurrences, so the synthetic stack doesn't fragment grouping. (#1468) - // value = the real detail (message/error) → else the PR location + a summary of salient fields (so a field-only log - // like close_breaker_engaged shows "project=x, closePrecision=0.6, floor=0.8") → else a context pointer. - const value = - detail ?? - ([logLocation(safeObj).trim(), summarizeLogFields(safeObj)] - .filter(Boolean) - .join(" ") || "(no message — see the log context)"); - const errorEvent = new Error(value); - errorEvent.name = event ? (subEvent ? `${event}/${subEvent}` : event) : "LoopOverLog"; - // This exception is synthetic: it was minted from a console line, never thrown at the failing code. Strip the - // wrapper stack so Sentry does not attribute forwarded operational issues to this forwarding helper. - errorEvent.stack = `${errorEvent.name}: ${value}`; - Sentry.withScope((scope) => { - scope.setLevel(severity); - setOtelTraceScope(scope); - scope.setContext("log", safeObj); - if (event) safeObj.event = event; - applyOperationalTags(scope, safeObj); - // Group recurrences of ONE failure into a single issue (by event + ev, not the variable detail in the value). - if (event) scope.setFingerprint(["loopover-log", event, ...(subEvent ? [subEvent] : [])]); - // Sentry uses event.transaction as the issue culprit fallback when the stack has no frames; point it at the - // operational event slug rather than the forwarding helper. - if (event) - scope.addEventProcessor((sentryEvent) => { - sentryEvent.transaction = event; - return sentryEvent; - }); - Sentry!.captureException(errorEvent); - }); -} - -/** Wrap recurring self-host work with Sentry cron check-ins. No-op when Sentry is disabled. */ -export async function withSentryMonitor( - name: SentryMonitorName, - context: Record | undefined, - callback: () => Promise, -): Promise { - if (!active || !Sentry) return callback(); - const monitorSlug = resolveSentryMonitorSlug(name); - const configOrResolver = SENTRY_MONITORS[name].config; - const resolvedConfig = typeof configOrResolver === "function" ? configOrResolver() : configOrResolver; - const checkInId = Sentry.captureCheckIn({ monitorSlug, status: "in_progress" }, resolvedConfig); - const startedAt = Date.now(); - try { - const result = await callback(); - Sentry.captureCheckIn({ - monitorSlug, - status: "ok", - checkInId, - duration: (Date.now() - startedAt) / 1000, - }); - return result; - } catch (error) { - Sentry.captureCheckIn({ - monitorSlug, - status: "error", - checkInId, - duration: (Date.now() - startedAt) / 1000, - }); - Sentry.withScope((scope) => { - scope.setLevel("error"); - setOtelTraceScope(scope); - const monitorContext = safeMonitorContext(name, monitorSlug, context); - scope.setContext("sentry_monitor", monitorContext); - applyOperationalTags(scope, { ...monitorContext, monitor: monitorSlug, kind: `sentry_monitor_${name}`, subsystem: "scheduled" }); - scope.setFingerprint(["loopover-sentry-monitor", name]); - Sentry!.captureException(error instanceof Error ? error : new Error(String(error))); - }); - throw error; - } -} - -/** Flush buffered events before exit. No-op when off. */ -export async function flushSentry(timeoutMs = 2000): Promise { - if (!active || !Sentry) return; - await Sentry.flush(timeoutMs).catch(() => undefined); -} - -/** Test-only: reset module state between cases. */ -export function resetSentryForTest(): void { - Sentry = undefined; - sentryClient = undefined; - sentryTraceSampleRate = undefined; - active = false; - sentryEnvironment = "production"; - resetRedactionScrubForTest(); -} - -interface StructuredLogConsole { - log: (...args: unknown[]) => void; - error: (...args: unknown[]) => void; -} - -/** Install central structured-log forwarding for both stdout and stderr sinks used by self-host. */ -export function installStructuredLogForwarding( - target: StructuredLogConsole = console, -): void { - const baseConsoleLog = target.log.bind(target); - const baseConsoleError = target.error.bind(target); - let forwardingToSentry = false; - const forward = (line: unknown, fromErrorSink: boolean): void => { - if (forwardingToSentry) return; - forwardingToSentry = true; - try { - forwardStructuredLogToSentry(line, fromErrorSink); - } finally { - forwardingToSentry = false; - } - }; - // stdout (console.log): forward only an EXPLICIT level:error/fatal. stderr (console.error): forward as error by - // default (an explicit level still wins) — so EVERY console.error structured log reaches Sentry, not just the - // ones that happened to include a level field. - target.log = (...args: unknown[]): void => { - baseConsoleLog(...args); - forward(args[0], false); - }; - target.error = (...args: unknown[]): void => { - baseConsoleError(...args); - forward(args[0], true); - }; -} diff --git a/src/selfhost/sqlite-queue.ts b/src/selfhost/sqlite-queue.ts index 1ca737647c..29df3ee602 100644 --- a/src/selfhost/sqlite-queue.ts +++ b/src/selfhost/sqlite-queue.ts @@ -9,7 +9,6 @@ import { logAudit, extractPayloadType, extractPayloadContext } from "./audit"; import { incr } from "./metrics"; import { withReviewSpan } from "./tracing"; import { withOtelSpan } from "./otel"; -import { captureError, withSentryMonitor } from "./sentry"; import { capturePostHogError, withPostHogMonitor } from "./posthog"; import { consumingRetryDelayMs, @@ -300,27 +299,23 @@ export function createSqliteQueue( /** Wraps reviveDeadLetterJobs() for the setInterval callback below, which has no error handler of its own -- * a transient driver/metric failure here would otherwise surface as an uncaught exception and can terminate - * the process (fatal when SENTRY_DSN is unset, since server.ts only installs the handler when Sentry is - * configured), exactly the failure mode pump()'s own try/catch above guards against for the main poll loop. - * A failed revive tick just waits for the next interval, same as a failed poll tick waits for the next poll. + * the process (fatal when POSTHOG_API_KEY is unset, since initPostHog's enableExceptionAutocapture only + * installs a handler when PostHog is configured), exactly the failure mode pump()'s own try/catch above + * guards against for the main poll loop. A failed revive tick just waits for the next interval, same as a + * failed poll tick waits for the next poll. * - * Also wrapped in a Sentry cron monitor (#1824): dead-letter revival stopping SILENTLY (the timer never fires - * again) is worse than one throwing tick -- a crashed tick self-reports via captureError below, but a stopped - * one reports nothing without a monitor watching for the missed check-in. withSentryMonitor rethrows on - * failure so its own capture fires; the outer try/catch (this function's actual job) still guards the - * setInterval callback. Async now (setInterval tolerates a Promise-returning callback the same as the - * synchronous one it replaces -- see the call site). */ + * Also wrapped in a PostHog monitor heartbeat (#1824): dead-letter revival stopping SILENTLY (the timer + * never fires again) is worse than one throwing tick -- a crashed tick self-reports via capturePostHogError + * below, but a stopped one reports nothing without a monitor watching for the missed heartbeat. + * withPostHogMonitor rethrows on failure so its own capture fires; the outer try/catch (this function's + * actual job) still guards the setInterval callback. Async now (setInterval tolerates a Promise-returning + * callback the same as the synchronous one it replaces -- see the call site). */ async function reviveDeadLetterJobsSafely(): Promise { try { await withPostHogMonitor( "queue-dead-letter-revive", { jobType: "queue-dead-letter-revive" }, - () => - withSentryMonitor( - "queue-dead-letter-revive", - { jobType: "queue-dead-letter-revive" }, - () => Promise.resolve(reviveDeadLetterJobs()), - ), + () => Promise.resolve(reviveDeadLetterJobs()), ); } catch (error) { console.error( @@ -330,7 +325,6 @@ export function createSqliteQueue( error: errorMessageWithCause(error), }), ); - captureError(error, { kind: "queue_dead_letter_revive_crashed" }, "queue_dead_letter_revive_crashed"); capturePostHogError(error, { kind: "queue_dead_letter_revive_crashed" }, "queue_dead_letter_revive_crashed"); } } @@ -452,7 +446,6 @@ export function createSqliteQueue( error: errorMessageWithCause(error), }), ); - captureError(error, { kind: "queue_foreground_liveness_release_crashed" }, "queue_foreground_liveness_release_crashed"); capturePostHogError(error, { kind: "queue_foreground_liveness_release_crashed" }, "queue_foreground_liveness_release_crashed"); } } @@ -833,12 +826,6 @@ export function createSqliteQueue( timeout_ms: processingTimeoutMs, }), ); - captureError(new Error("self-host queue processing lease expired"), { - kind: "job_recovered", - reason: "processing_timeout", - recovered, - timeoutMs: processingTimeoutMs, - }, "processing_timeout"); capturePostHogError(new Error("self-host queue processing lease expired"), { kind: "job_recovered", reason: "processing_timeout", @@ -868,11 +855,6 @@ export function createSqliteQueue( attempts: job.attempts + 1, error: "unparseable payload", }); - captureError(new Error("unparseable queue payload"), { - kind: "job_dead", - reason: "unparseable_payload", - jobId: job.id, - }, "unparseable_payload"); capturePostHogError(new Error("unparseable queue payload"), { kind: "job_dead", reason: "unparseable_payload", @@ -1130,13 +1112,6 @@ export function createSqliteQueue( attempts, error: errMsg, }, jobTraceParent); - captureError(error, { - kind: "job_dead", - reason: "max_retries_exhausted", - jobType: extractPayloadType(job.payload), - jobId: job.id, - attempts, - }, "job_dead"); capturePostHogError(error, { kind: "job_dead", reason: "max_retries_exhausted", @@ -1185,7 +1160,8 @@ export function createSqliteQueue( // claimNext()/reclaimExpiredProcessingJobs() run OUTSIDE processOne's own try/finally, so a raw driver // failure (e.g. a transient SQLite error) lands here. Every `void pump()` call site (kickOne/kickAll) is // fire-and-forget, so an uncaught rejection here would surface as an unhandled promise rejection — fatal - // when SENTRY_DSN is unset (server.ts only installs the handler when Sentry is configured) (#2498). + // when POSTHOG_API_KEY is unset (initPostHog's enableExceptionAutocapture only installs a handler when + // PostHog is configured) (#2498). console.error( JSON.stringify({ level: "error", @@ -1193,7 +1169,6 @@ export function createSqliteQueue( error: errorMessageWithCause(error), }), ); - captureError(error, { kind: "queue_pump_crashed" }, "queue_pump_crashed"); capturePostHogError(error, { kind: "queue_pump_crashed" }, "queue_pump_crashed"); } finally { active--; diff --git a/src/selfhost/tracing.ts b/src/selfhost/tracing.ts index cd0498d791..bdba618605 100644 --- a/src/selfhost/tracing.ts +++ b/src/selfhost/tracing.ts @@ -1,5 +1,5 @@ -// Shared review-pipeline span wrapper (#1734). Opens ONE OpenTelemetry boundary; Sentry receives the same span via -// the SentrySpanProcessor bridge when configured, avoiding duplicate direct Sentry spans. +// Shared review-pipeline span wrapper (#1734). Opens ONE OpenTelemetry boundary, exported over OTLP when +// configured (see ./otel's resolveOtelTraceEndpoint, including its PostHog-derived default). import { withOtelSpan } from "./otel"; export async function withReviewSpan( diff --git a/src/server.ts b/src/server.ts index 38a77e48a9..326ece02d7 100644 --- a/src/server.ts +++ b/src/server.ts @@ -84,13 +84,6 @@ import { setConfigAdminFunctions } from "./mcp/private-config-admin-registry"; import { setRedeployTrigger } from "./mcp/redeploy-companion-registry"; import { triggerRedeploy } from "./selfhost/redeploy-companion-client"; import { assertSelfHostPreflight } from "./selfhost/preflight"; -import { - buildSentryOpenTelemetryBridge, - captureError, - flushSentry, - initSentry, - installStructuredLogForwarding, -} from "./selfhost/sentry"; import { capturePostHogError, flushPostHog, @@ -108,7 +101,6 @@ import { import { currentOtelTraceParent, initOpenTelemetry, - openTelemetryTraceExportEnabled, selfHostHttpRequestAttributes, selfHostHttpResponseAttributes, setCurrentOtelSpanAttributes, @@ -325,55 +317,33 @@ async function main(): Promise { loadFileSecrets(); /* v8 ignore next -- importing this entrypoint starts the Node server; pure validation is covered in selfhost-preflight tests. */ assertSelfHostPreflight(process.env); - // Error tracking (#1468): opt-in via SENTRY_DSN — a complete no-op when unset. When on, capture uncaught crashes - // + unhandled rejections (flush before exit for the fatal case); per-subsystem captures (queue dead-letter, - // review failures) are wired at their sites. + // Error tracking (#1468, epic #8286): opt-in via POSTHOG_API_KEY -- the same var #6235's MCP telemetry + // already reads -- a complete no-op when unset. REPLACES the old Sentry sink entirely (2026-07-25 epic + // correction: full replacement, not a parallel-run). enableExceptionAutocapture (set inside initPostHog) + // already installs its own uncaughtException/unhandledRejection handlers per PostHog's own documented + // Node.js setup, so no manual process.on wiring is needed here for the crash case; only structured-log + // forwarding needs an explicit install. // // #6325 follow-up: initialized HERE, before every boot-time advisory below (emptyConfigDirAdvisory / // sqliteBackupAdvisory / publicOriginReachabilityAdvisory all "warn LOUDLY" via console.error, which - // installStructuredLogForwarding — wired below — is what actually forwards it to Sentry: only console.error - // and level:error/fatal console.log lines are ever forwarded, never console.warn). Originally this ran - // AFTER emptyConfigDirAdvisory's own check, so that ONE advisory's console.error was still silently - // unreachable by Sentry even after #6325's console.warn->console.error fix landed for the other two: the - // forwarding hook simply didn't exist yet at that point in the boot sequence. Kept immediately after - // loadFileSecrets()/assertSelfHostPreflight() specifically — a self-host SENTRY_DSN is commonly supplied via - // a mounted secret file loadFileSecrets() reads into process.env, and preflight is a fatal-exit gate that - // should run before anything else regardless of Sentry's own state. - /* v8 ignore start -- importing this entrypoint starts the Node server; Sentry/OTEL init behavior is covered in selfhost tests. */ - const sentryEnabled = await initSentry(process.env); - if (sentryEnabled) { - console.log( - JSON.stringify({ - event: "selfhost_sentry", - environment: process.env.SENTRY_ENVIRONMENT ?? "production", - }), - ); - process.on("uncaughtException", (error) => { - captureError(error, { kind: "uncaughtException" }, "uncaughtException"); - console.error(error); - void flushSentry().finally(() => process.exit(1)); - }); - process.on("unhandledRejection", (reason) => { - captureError(reason, { kind: "unhandledRejection" }, "unhandledRejection"); - console.error(reason); - }); - // Central error forwarding (#1468): operational failures are structured JSON logs emitted through stdout and - // stderr. Wrap both sinks so every level:"error"/"fatal" line surfaces as a Sentry issue WITHOUT per-site wiring. - installStructuredLogForwarding(); - } - // PostHog error tracking (#8287, epic #8286): opt-in via POSTHOG_API_KEY -- the same var #6235's MCP telemetry - // already reads. PARALLEL-RUN: active alongside Sentry above, not instead of it, until the gated decommission - // issue (#8298) says otherwise. enableExceptionAutocapture (set inside initPostHog) already installs its own - // uncaughtException/unhandledRejection handlers per PostHog's own documented Node.js setup, so -- unlike - // Sentry's explicit process.on wiring above, added before that capability existed -- no manual mirror is - // needed here for the crash case; only structured-log forwarding needs an explicit install, same as Sentry. + // installPostHogStructuredLogForwarding — wired below — is what actually forwards it to PostHog: only + // console.error and level:error/fatal console.log lines are ever forwarded, never console.warn). Kept + // immediately after loadFileSecrets()/assertSelfHostPreflight() specifically — a self-host POSTHOG_API_KEY + // is commonly supplied via a mounted secret file loadFileSecrets() reads into process.env, and preflight is + // a fatal-exit gate that should run before anything else regardless of PostHog's own state. + /* v8 ignore start -- importing this entrypoint starts the Node server; PostHog/OTEL init behavior is covered in selfhost tests. */ const posthogEnabled = await initPostHog(process.env); if (posthogEnabled) { console.log(JSON.stringify({ event: "selfhost_posthog", environment: process.env.POSTHOG_ENVIRONMENT ?? "production" })); installPostHogStructuredLogForwarding(); } - if (await initOpenTelemetry(process.env, sentryEnabled ? await buildSentryOpenTelemetryBridge() : undefined)) - console.log(JSON.stringify({ event: "selfhost_otel", traces: openTelemetryTraceExportEnabled(process.env) ? "otlp" : "sentry" })); + // PostHog's distributed-tracing product (beta: https://posthog.com/docs/distributed-tracing) is plain + // OTLP/HTTP, so initOpenTelemetry needs no PostHog-specific bridge argument -- it already defaults its OTLP + // trace endpoint to PostHog when POSTHOG_API_KEY is set and no explicit OTEL_EXPORTER_OTLP_* override is + // given (see resolveOtelTraceEndpoint in ./selfhost/otel). An operator still opts in via + // OTEL_TRACES_EXPORTER=otlp -- this never turns tracing on just because POSTHOG_API_KEY happens to be set. + if (await initOpenTelemetry(process.env)) + console.log(JSON.stringify({ event: "selfhost_otel", traces: "otlp" })); /* v8 ignore stop */ const startedAt = Date.now(); // This entrypoint IS the self-host runtime by definition (the cloud worker never imports server.ts), so the @@ -446,10 +416,10 @@ async function main(): Promise { // Config-drift advisory: warn LOUDLY (not just the log line above) when the mount resolves but is empty -- // see emptyConfigDirAdvisory's own doc comment for the incident this guards against. const configDirAdvisory = emptyConfigDirAdvisory(configDirOpts); - // #6325 follow-up: console.error, not console.warn -- installStructuredLogForwarding (initSentry, now above - // this check) only intercepts console.log (level:error/fatal only) and console.error (always forwarded); - // console.warn is never wrapped at all. `level: "warn"` in the payload still maps this to Sentry's own - // "warning" severity (see forwardStructuredLogToSentry), not an "error". + // #6325 follow-up: console.error, not console.warn -- installPostHogStructuredLogForwarding (initPostHog, + // now above this check) only intercepts console.log (level:error/fatal only) and console.error (always + // forwarded); console.warn is never wrapped at all. `level: "warn"` in the payload still maps this to + // PostHog's own "warning" severity (see forwardStructuredLogToPostHog), not an "error". if (configDirAdvisory) console.error( JSON.stringify({ @@ -514,11 +484,11 @@ async function main(): Promise { backupAcknowledged: process.env.BACKUP_ACKNOWLEDGED === "true", }; const backupAdvisory = sqliteBackupAdvisory(sqliteBackupOpts); - // #6325: console.error, not console.warn -- installStructuredLogForwarding (initSentry, above) only + // #6325: console.error, not console.warn -- installPostHogStructuredLogForwarding (initPostHog, above) only // intercepts console.log (level:error/fatal only) and console.error (always forwarded); console.warn is - // NEVER wrapped at all, so this "warn LOUDLY" advisory was silently unreachable by Sentry regardless of - // whether Sentry was configured. `level: "warn"` in the payload still maps this to Sentry's own "warning" - // severity (see forwardStructuredLogToSentry), not an "error" -- only the CONSOLE METHOD used to reach the + // NEVER wrapped at all, so this "warn LOUDLY" advisory was silently unreachable by PostHog regardless of + // whether PostHog was configured. `level: "warn"` in the payload still maps this to PostHog's own "warning" + // severity (see forwardStructuredLogToPostHog), not an "error" -- only the CONSOLE METHOD used to reach the // forwarder changes here, not the reported severity. if (backupAdvisory) console.error( @@ -1139,7 +1109,7 @@ async function main(): Promise { }, () => { console.log(JSON.stringify({ event: "selfhost_listening", port })); - // Probe REES shared secret at startup so mismatches appear in logs/Sentry before + // Probe REES shared secret at startup so mismatches appear in logs/PostHog before // any PR triggers a review (fire-and-forget; never blocks server startup). probeReesSecretAtStartup(env); }, @@ -1240,7 +1210,6 @@ async function main(): Promise { register: registerOrbRelayTargetWithRetry, ...(relayDrainState ? { drainState: relayDrainState } : {}), }).catch((error) => { - captureError(error, { kind: "orb_relay_register" }, "orb_relay_register"); capturePostHogError(error, { kind: "orb_relay_register" }, "orb_relay_register"); }); void attemptOrbRelayRegistration(); @@ -1268,7 +1237,6 @@ async function main(): Promise { /* v8 ignore start -- self-host entrypoint timer; probe logic itself is unit-tested in d1-size-probe.test.ts. */ const runD1Probe = () => runD1SizeProbe(d1ProbeEnv).catch((error) => { - captureError(error, { kind: "d1_size_probe" }, "d1_size_probe"); capturePostHogError(error, { kind: "d1_size_probe" }, "d1_size_probe"); }); void runD1Probe(); @@ -1297,7 +1265,6 @@ async function main(): Promise { }), ); void drainRelay().catch((error) => { - captureError(error, { kind: "orb_relay_drain" }, "orb_relay_drain"); capturePostHogError(error, { kind: "orb_relay_drain" }, "orb_relay_drain"); }); // 30s matches broker-client's request timeout so a slow/degraded broker's in-flight drain has fully @@ -1305,7 +1272,6 @@ async function main(): Promise { setInterval( () => void drainRelay().catch((error) => { - captureError(error, { kind: "orb_relay_drain" }, "orb_relay_drain"); capturePostHogError(error, { kind: "orb_relay_drain" }, "orb_relay_drain"); }), 30_000, @@ -1324,7 +1290,6 @@ async function main(): Promise { await backend.shutdown(); /* v8 ignore next -- graceful process signal path is not imported in unit tests; shutdown helper is covered. */ await shutdownOpenTelemetry(); - await flushSentry(); await shutdownPostHog(); process.exit(0); }; @@ -1333,9 +1298,8 @@ async function main(): Promise { } main().catch((error) => { - captureError(error, { kind: "boot" }, "boot"); capturePostHogError(error, { kind: "boot" }, "boot"); console.error(error); /* v8 ignore next -- boot failure exits the process; shutdown helper is covered independently. */ - void Promise.all([shutdownOpenTelemetry(), flushSentry(), flushPostHog()]).finally(() => process.exit(1)); + void Promise.all([shutdownOpenTelemetry(), flushPostHog()]).finally(() => process.exit(1)); }); diff --git a/src/services/agent-action-executor.ts b/src/services/agent-action-executor.ts index 5ea34b6d3d..4c41d7909b 100644 --- a/src/services/agent-action-executor.ts +++ b/src/services/agent-action-executor.ts @@ -40,7 +40,6 @@ import { } from "../settings/moderation-rules"; import { incr } from "../selfhost/metrics"; import { shouldWaitForOlderSiblings } from "../review/merge-train"; -import { captureError } from "../selfhost/sentry"; import { capturePostHogError } from "../selfhost/posthog"; import { claimContributorCapLock, releaseContributorCapLock } from "../queue/transient-locks"; @@ -631,15 +630,14 @@ export async function executeAgentMaintenanceActions(env: Env, ctx: AgentActionE // LOOPOVER-24 (regressed shape): a 422 "There are no new commits on the base branch." means the head // was already up to date when update-branch fired -- the readiness check acted on a stale/cached // mergeable_state read. Nothing went wrong and nothing is stuck: the caller falls through to reviewing - // the current head exactly as in the conflict case above. Audit-only; never a Sentry page. + // the current head exactly as in the conflict case above. Audit-only; never a PostHog page. } else { // Non-merge action classes have no retry loop -- a single failure here is already this pass's terminal // outcome (the planner may re-attempt on the next sweep if the underlying condition clears itself), so // it is captured immediately rather than only on eventual exhaustion. Mirrors handleMergeFailure's own // terminal-hold capture below and the "a real failure the maintainer must see" convention already used - // for review-pass failures (selfhost/sentry.ts's captureReviewFailure, queue/processors.ts). Previously - // this class of failure was audit-log-only, invisible without a manual audit_events query. - captureError(error, { kind: "agent_action_execution_failed", repo: ctx.repoFullName, pr: ctx.pullNumber, installationId: ctx.installationId, actionClass: action.actionClass }, "agent_action_execution_failed"); + // for review-pass failures (selfhost/posthog.ts's capturePostHogReviewFailure, queue/processors.ts). + // Previously this class of failure was audit-log-only, invisible without a manual audit_events query. capturePostHogError(error, { kind: "agent_action_execution_failed", repo: ctx.repoFullName, pr: ctx.pullNumber, installationId: ctx.installationId, actionClass: action.actionClass }, "agent_action_execution_failed"); } // #2265: a permission-looking 403 on a PR-write mutation can mean the LOCAL installations.permissions @@ -942,7 +940,6 @@ export async function executeIssueMaintenanceActions(env: Env, ctx: IssueActionE await audit("error", errorMessage(error)); // Mirrors executeAgentMaintenanceActions's non-merge capture below -- issue-side label/close has no retry // loop either, so a single failure here is already this pass's terminal outcome. - captureError(error, { kind: "agent_issue_action_execution_failed", repo: ctx.repoFullName, issue: ctx.issueNumber, installationId: ctx.installationId, actionClass: action.actionClass }, "agent_issue_action_execution_failed"); capturePostHogError(error, { kind: "agent_issue_action_execution_failed", repo: ctx.repoFullName, issue: ctx.issueNumber, installationId: ctx.installationId, actionClass: action.actionClass }, "agent_issue_action_execution_failed"); } } @@ -973,13 +970,12 @@ async function handleMergeFailure(env: Env, ctx: AgentActionExecutionContext, er if (!terminal) return; await markPullRequestMergeBlocked(env, ctx.repoFullName, ctx.pullNumber, headSha, reason); // A merge held for a human is the terminal outcome of this whole retry sequence -- exactly the "a real - // failure the maintainer must see" case captureReviewFailure already covers for an exhausted AI review pass. - // Fires once per hold (not per retry attempt), so a transient failure that resolves within MERGE_RETRY_CAP - // never reaches Sentry at all. + // failure the maintainer must see" case capturePostHogReviewFailure already covers for an exhausted AI + // review pass. Fires once per hold (not per retry attempt), so a transient failure that resolves within + // MERGE_RETRY_CAP never reaches PostHog at all. // Named "agent_merge_blocked" (not the caught exception's own class, e.g. "HttpError") so every terminal // merge hold groups under one readable title regardless of which HTTP status caused it -- the specific // status/reason stays in the message and the "review" context object either way. - captureError(error, { kind: "agent_merge_blocked", repo: ctx.repoFullName, pr: ctx.pullNumber, installationId: ctx.installationId, reason: reason.slice(0, 280) }, "agent_merge_blocked"); capturePostHogError(error, { kind: "agent_merge_blocked", repo: ctx.repoFullName, pr: ctx.pullNumber, installationId: ctx.installationId, reason: reason.slice(0, 280) }, "agent_merge_blocked"); await recordAuditEvent(env, { eventType: "agent.action.merge_blocked", diff --git a/test/unit/agent-action-executor.test.ts b/test/unit/agent-action-executor.test.ts index 7b289879b8..882c2d0f83 100644 --- a/test/unit/agent-action-executor.test.ts +++ b/test/unit/agent-action-executor.test.ts @@ -76,7 +76,7 @@ import { STRUCTURED_CLOSE_REASONS_MAX_COUNT } from "../../src/settings/agent-exe import { AGENT_LABEL_PENDING_CLOSURE } from "../../src/review/linked-issue-hard-rules"; import { clearProcessLocalGlobalAgentFrozenCacheForTest, getGlobalContributorBlacklist, isGlobalAgentFrozen, setGlobalAgentFrozen, upsertGlobalModerationConfig, upsertPullRequestFile, upsertPullRequestFromGitHub } from "../../src/db/repositories"; import * as repositoriesModule from "../../src/db/repositories"; -import * as sentryModule from "../../src/selfhost/sentry"; +import * as posthogModule from "../../src/selfhost/posthog"; import { renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; import { createTestEnv } from "../helpers/d1"; import { MODERATION_VIOLATION_EVENT_TYPE } from "../../src/settings/moderation-rules"; @@ -1381,7 +1381,7 @@ describe("executeAgentMaintenanceActions (#778 gate stack)", () => { it("records a failed mutation as error rather than swallowing it", async () => { const env = createTestEnv({}); vi.mocked(mergePullRequest).mockRejectedValueOnce(new Error("Pull Request is not mergeable")); - const captureSpy = vi.spyOn(sentryModule, "captureError"); + const captureSpy = vi.spyOn(posthogModule, "capturePostHogError"); const outcomes = await executeAgentMaintenanceActions(env, ctx(), [merge]); expect(outcomes[0]?.outcome).toBe("error"); expect(outcomes[0]?.detail).toMatch(/not mergeable/i); @@ -1396,7 +1396,7 @@ describe("executeAgentMaintenanceActions (#778 gate stack)", () => { const env = createTestEnv({}); await upsertPullRequestFromGitHub(env, "owner/repo", { number: 7, title: "PR", state: "open", user: { login: "c" }, head: { sha: "sha7" }, labels: [], body: "" }); vi.mocked(mergePullRequest).mockRejectedValueOnce(Object.assign(new Error("Resource not accessible by integration"), { status: 403 })); - const captureSpy = vi.spyOn(sentryModule, "captureError"); + const captureSpy = vi.spyOn(posthogModule, "capturePostHogError"); const outcomes = await executeAgentMaintenanceActions(env, ctx(), [merge]); @@ -1420,7 +1420,7 @@ describe("executeAgentMaintenanceActions (#778 gate stack)", () => { it("opportunistically refreshes installation health when a PR-write mutation fails with a 403 (#2265)", async () => { const env = createTestEnv({}); vi.mocked(closePullRequest).mockRejectedValueOnce(Object.assign(new Error("Resource not accessible by integration"), { status: 403 })); - const captureSpy = vi.spyOn(sentryModule, "captureError"); + const captureSpy = vi.spyOn(posthogModule, "capturePostHogError"); const outcomes = await executeAgentMaintenanceActions(env, ctx(), [close]); expect(outcomes[0]?.outcome).toBe("error"); expect(refreshInstallationHealthForInstallation).toHaveBeenCalledTimes(1); @@ -1459,7 +1459,7 @@ describe("executeAgentMaintenanceActions (#778 gate stack)", () => { it("REGRESSION (LOOPOVER-24): a merge-conflict update_branch failure does not page Sentry", async () => { const env = createTestEnv({}); vi.mocked(updatePullRequestBranch).mockRejectedValueOnce(new Error("merge conflict between base and head")); - const captureSpy = vi.spyOn(sentryModule, "captureError"); + const captureSpy = vi.spyOn(posthogModule, "capturePostHogError"); const outcomes = await executeAgentMaintenanceActions(env, ctx(), [updateBranch]); @@ -1479,7 +1479,7 @@ describe("executeAgentMaintenanceActions (#778 gate stack)", () => { vi.mocked(updatePullRequestBranch).mockRejectedValueOnce( Object.assign(new Error("There are no new commits on the base branch. - https://docs.github.com/rest/pulls/pulls#update-a-pull-request-branch"), { status: 422 }), ); - const captureSpy = vi.spyOn(sentryModule, "captureError"); + const captureSpy = vi.spyOn(posthogModule, "capturePostHogError"); const outcomes = await executeAgentMaintenanceActions(env, ctx(), [updateBranch]); @@ -1492,7 +1492,7 @@ describe("executeAgentMaintenanceActions (#778 gate stack)", () => { it("a non-conflict update_branch failure still pages Sentry (#agent_action_execution_failed unchanged)", async () => { const env = createTestEnv({}); vi.mocked(updatePullRequestBranch).mockRejectedValueOnce(new Error("network timeout")); - const captureSpy = vi.spyOn(sentryModule, "captureError"); + const captureSpy = vi.spyOn(posthogModule, "capturePostHogError"); await executeAgentMaintenanceActions(env, ctx(), [updateBranch]); @@ -1956,7 +1956,7 @@ describe("executeIssueMaintenanceActions (#2270 issue-side actuation)", () => { it("records a failed mutation as error rather than swallowing it", async () => { const env = createTestEnv({}); vi.mocked(closeIssue).mockRejectedValueOnce(new Error("github 500")); - const captureSpy = vi.spyOn(sentryModule, "captureError"); + const captureSpy = vi.spyOn(posthogModule, "capturePostHogError"); const outcomes = await executeIssueMaintenanceActions(env, issueCtx(), [issueClose]); expect(outcomes[0]?.outcome).toBe("error"); expect((await auditFor(env, "close"))?.outcome).toBe("error"); diff --git a/test/unit/ai-review-advisory.test.ts b/test/unit/ai-review-advisory.test.ts index abd32c0bea..de611d8b46 100644 --- a/test/unit/ai-review-advisory.test.ts +++ b/test/unit/ai-review-advisory.test.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { buildAiReviewDiff, claimAiReviewLock, runAiReviewForAdvisory, shouldStartAiReviewForAdvisory } from "../../src/queue/processors"; import { resolveAiReviewableAuthor } from "../../src/queue/ai-review-orchestration"; import { BEST_REVIEW_MODELS, INCOHERENT_DIFF_ASSESSMENT } from "../../src/services/ai-review"; -import * as sentryModule from "../../src/selfhost/sentry"; +import * as posthogModule from "../../src/selfhost/posthog"; import { upsertRepositoryAiKey } from "../../src/db/repositories"; import type { Advisory, PullRequestFileRecord, RepositorySettings } from "../../src/types"; import { createTestEnv } from "../helpers/d1"; @@ -432,7 +432,7 @@ describe("runAiReviewForAdvisory", () => { // The first slot parses; the second slot's primary AND its reliable fallback fail → no consensus possible. const run = (async (model: string) => ({ response: model === BEST_REVIEW_MODELS[0] ? notesOnlyJson() : "garbage" })) as unknown as () => Promise; const env = aiEnv(run); - const captureSpy = vi.spyOn(sentryModule, "captureReviewFailure"); + const captureSpy = vi.spyOn(posthogModule, "capturePostHogReviewFailure"); const result = await runAiReviewForAdvisory(env, { mode: "live", settings: { aiReviewMode: "block" } as RepositorySettings, @@ -473,7 +473,7 @@ describe("runAiReviewForAdvisory", () => { // opinion parses to null → the combiner yields `inconclusive`, the same review-failure path as a missing opinion. const incoherent = JSON.stringify({ assessment: INCOHERENT_DIFF_ASSESSMENT, blockers: [], nits: [], suggestions: [] }); const env = aiEnv((async () => ({ response: incoherent })) as unknown as () => Promise); - const captureSpy = vi.spyOn(sentryModule, "captureReviewFailure"); + const captureSpy = vi.spyOn(posthogModule, "capturePostHogReviewFailure"); await runAiReviewForAdvisory(env, { mode: "live", settings: { aiReviewMode: "block" } as RepositorySettings, @@ -725,7 +725,7 @@ describe("runAiReviewForAdvisory", () => { it("holds for manual review when the AI provider produces no public notes", async () => { const adv = advisory(); - const captureSpy = vi.spyOn(sentryModule, "captureReviewFailure"); + const captureSpy = vi.spyOn(posthogModule, "capturePostHogReviewFailure"); const result = await runAiReviewForAdvisory(aiEnv(async () => ({ response: "" })), { mode: "live", settings: { aiReviewMode: "advisory" } as RepositorySettings, diff --git a/test/unit/docs-selfhost-audit-checklist.test.ts b/test/unit/docs-selfhost-audit-checklist.test.ts index 3b95c7f9c7..fad88bbe9f 100644 --- a/test/unit/docs-selfhost-audit-checklist.test.ts +++ b/test/unit/docs-selfhost-audit-checklist.test.ts @@ -73,10 +73,10 @@ describe("self-host docs accuracy audit (#1829)", () => { ); }); - it("documents Sentry as opt-in with an operator-owned DSN", () => { + it("documents PostHog error tracking as opt-in with an operator-owned API key", () => { expect(auditSource).toMatch(/opt-in and off by default/i); - expect(auditSource).toContain("SENTRY_DSN"); - expect(auditSource).toContain("SENTRY_DSN_FILE"); + expect(auditSource).toContain("POSTHOG_API_KEY"); + expect(auditSource).toContain("replaces Sentry"); }); it("records the loose-docs consolidation plan with canonical website links", () => { diff --git a/test/unit/docs-selfhost-posthog-observability.test.ts b/test/unit/docs-selfhost-posthog-observability.test.ts index d002c35143..1fbac42c1f 100644 --- a/test/unit/docs-selfhost-posthog-observability.test.ts +++ b/test/unit/docs-selfhost-posthog-observability.test.ts @@ -3,18 +3,19 @@ import { describe, expect, it } from "vitest"; import { POSTHOG_MONITOR_HEARTBEAT_EVENT } from "../../src/selfhost/posthog"; -// Drift guard (#8287): self-host PostHog docs must stay aligned with the exported monitor-heartbeat event -// name, mirroring docs-selfhost-sentry-observability.test.ts's identical discipline for Sentry. +// Drift guard (#8287, #1468 -- 2026-07-25 Sentry removal): self-host PostHog docs must stay aligned with the +// exported monitor-heartbeat event name. Sentry's own docs test (docs-selfhost-sentry-observability.test.ts) +// was deleted alongside src/selfhost/sentry.ts -- PostHog is this surface's only error-tracking sink now. const OPERATIONS = "apps/loopover-ui/content/docs/self-hosting-operations.mdx"; const operations = readFileSync(OPERATIONS, "utf8"); describe("self-host PostHog observability docs (#8287)", () => { - it("documents enabling PostHog as opt-in and parallel-run alongside Sentry", () => { + it("documents enabling PostHog as opt-in and REPLACING Sentry", () => { expect(operations).toContain("Enabling PostHog error tracking"); expect(operations).toContain("POSTHOG_API_KEY"); expect(operations).toContain("opt-in and off by default"); - expect(operations).toContain("parallel with Sentry"); + expect(operations).toContain("replaces Sentry"); }); it("documents the shared redaction module both sinks use", () => { diff --git a/test/unit/docs-selfhost-sentry-observability.test.ts b/test/unit/docs-selfhost-sentry-observability.test.ts deleted file mode 100644 index bb89b25bdb..0000000000 --- a/test/unit/docs-selfhost-sentry-observability.test.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { readFileSync } from "node:fs"; -import { describe, expect, it } from "vitest"; - -import { - SENTRY_MONITOR_NAMES, - SENTRY_OPERATIONAL_SUBSYSTEMS, - SENTRY_OPERATIONAL_TAG_KEYS, -} from "../../src/selfhost/sentry"; - -// Drift guard (#1824): self-host Sentry docs must stay aligned with exported taxonomy and runbook signals. - -const OPERATIONS = "apps/loopover-ui/content/docs/self-hosting-operations.mdx"; -const operations = readFileSync(OPERATIONS, "utf8"); - -describe("self-host Sentry observability docs (#1824)", () => { - it("documents enabling Sentry with an operator-owned DSN", () => { - expect(operations).toContain("Enabling Sentry (your own DSN)"); - expect(operations).toContain("SENTRY_DSN"); - expect(operations).toContain("opt-in and off by default"); - expect(operations).toContain("SENTRY_DSN_FILE"); - }); - - it("documents context taxonomy and cron monitor slugs", () => { - expect(operations).toContain("Sentry context taxonomy"); - expect(operations).toContain("installation_id_hash"); - for (const monitor of SENTRY_MONITOR_NAMES) { - expect(operations).toContain(monitor); - } - for (const subsystem of Object.keys(SENTRY_OPERATIONAL_SUBSYSTEMS)) { - expect(operations).toContain(subsystem); - } - for (const tag of ["kind", "subsystem", "jobType", "operation"]) { - expect(SENTRY_OPERATIONAL_TAG_KEYS).toContain(tag); - expect(operations).toContain(tag); - } - expect(operations).toContain("repository"); - }); - - it("documents alert classes and runbook first-response checks", () => { - expect(operations).toContain("Sentry alert classes and runbook"); - expect(operations).toContain("selfhost_job_dead"); - expect(operations).toContain("check_run_post_denied"); - expect(operations).toContain("close_breaker_engaged"); - expect(operations).toContain("orb_broker_unavailable"); - expect(operations).toContain("selfhost_backup_advisory"); - expect(operations).toContain("scheduled-loop"); - }); - - it("documents the in-Grafana Sentry data source (#5369) as a separate credential from SENTRY_DSN", () => { - expect(operations).toContain("Grafana Sentry data source"); - expect(operations).toContain("SENTRY_API_TOKEN"); - expect(operations).toContain("SENTRY_ORG_SLUG"); - expect(operations).toContain("setup-sentry-datasource.sh"); - expect(operations).toContain("Internal Integration"); - expect(operations).toMatch(/SENTRY_DSN.*NOT reusable|not reusable.*SENTRY_DSN/i); - }); -}); diff --git a/test/unit/queue-2.test.ts b/test/unit/queue-2.test.ts index 00c41d7569..54a3aaac35 100644 --- a/test/unit/queue-2.test.ts +++ b/test/unit/queue-2.test.ts @@ -8,7 +8,7 @@ import * as rateLimitModule from "../../src/github/rate-limit"; import * as repositoriesModule from "../../src/db/repositories"; import * as reviewEffortModule from "../../src/review/review-effort"; import * as repositorySettingsModule from "../../src/settings/repository-settings"; -import * as sentryModule from "../../src/selfhost/sentry"; +import * as posthogModule from "../../src/selfhost/posthog"; import { renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; import { jobCoalesceKey } from "../../src/selfhost/queue-common"; import { @@ -1448,7 +1448,7 @@ describe("queue processors", () => { await upsertOfficialMinerDetection(env, "contributor", { status: "confirmed", snapshot: queueMinerSnapshot("contributor") }, 60_000); await upsertPullRequestFromGitHub(env, "JSONbored/gittensory", { number: 49, title: "Clean PR", state: "open", user: { login: "contributor" }, head: { sha: "a49" }, labels: [], body: "Closes #1" }); const commentBodies: string[] = []; - const captureSpy = vi.spyOn(sentryModule, "captureReviewFailure"); + const captureSpy = vi.spyOn(posthogModule, "capturePostHogReviewFailure"); vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { const url = input.toString(); const method = init?.method ?? "GET"; diff --git a/test/unit/queue-3.test.ts b/test/unit/queue-3.test.ts index e14f045622..e33a752ddb 100644 --- a/test/unit/queue-3.test.ts +++ b/test/unit/queue-3.test.ts @@ -8,7 +8,6 @@ import * as rateLimitModule from "../../src/github/rate-limit"; import * as repositoriesModule from "../../src/db/repositories"; import * as reviewEffortModule from "../../src/review/review-effort"; import * as repositorySettingsModule from "../../src/settings/repository-settings"; -import * as sentryModule from "../../src/selfhost/sentry"; import { renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; import { jobCoalesceKey } from "../../src/selfhost/queue-common"; import { diff --git a/test/unit/queue-4.test.ts b/test/unit/queue-4.test.ts index ab981067cc..a9d94eb0ea 100644 --- a/test/unit/queue-4.test.ts +++ b/test/unit/queue-4.test.ts @@ -8,7 +8,7 @@ import * as rateLimitModule from "../../src/github/rate-limit"; import * as repositoriesModule from "../../src/db/repositories"; import * as reviewEffortModule from "../../src/review/review-effort"; import * as repositorySettingsModule from "../../src/settings/repository-settings"; -import * as sentryModule from "../../src/selfhost/sentry"; +import * as posthogModule from "../../src/selfhost/posthog"; import { renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; import { jobCoalesceKey } from "../../src/selfhost/queue-common"; import { @@ -6307,7 +6307,7 @@ describe("queue processors", () => { if (url.includes("/commits/context500/check-runs")) return new Response("GitHub check API failed", { status: 500 }); return new Response("not found", { status: 404 }); }); - const captureSpy = vi.spyOn(sentryModule, "captureReviewFailure"); + const captureSpy = vi.spyOn(posthogModule, "capturePostHogReviewFailure"); await expect( processJob(env, { diff --git a/test/unit/queue-5.test.ts b/test/unit/queue-5.test.ts index 1c530f9c65..dafc96976e 100644 --- a/test/unit/queue-5.test.ts +++ b/test/unit/queue-5.test.ts @@ -10,7 +10,6 @@ import * as rateLimitModule from "../../src/github/rate-limit"; import * as repositoriesModule from "../../src/db/repositories"; import * as reviewEffortModule from "../../src/review/review-effort"; import * as repositorySettingsModule from "../../src/settings/repository-settings"; -import * as sentryModule from "../../src/selfhost/sentry"; import { renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; import { jobCoalesceKey } from "../../src/selfhost/queue-common"; import { diff --git a/test/unit/queue-lifecycle-guards.test.ts b/test/unit/queue-lifecycle-guards.test.ts index c37d6f609d..2d5fba9084 100644 --- a/test/unit/queue-lifecycle-guards.test.ts +++ b/test/unit/queue-lifecycle-guards.test.ts @@ -8,7 +8,6 @@ import * as rateLimitModule from "../../src/github/rate-limit"; import * as repositoriesModule from "../../src/db/repositories"; import * as reviewEffortModule from "../../src/review/review-effort"; import * as repositorySettingsModule from "../../src/settings/repository-settings"; -import * as sentryModule from "../../src/selfhost/sentry"; import { renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; import { jobCoalesceKey } from "../../src/selfhost/queue-common"; import { diff --git a/test/unit/queue.test.ts b/test/unit/queue.test.ts index 2e0191b25a..f6c0b152bc 100644 --- a/test/unit/queue.test.ts +++ b/test/unit/queue.test.ts @@ -9,7 +9,6 @@ import * as rateLimitModule from "../../src/github/rate-limit"; import * as repositoriesModule from "../../src/db/repositories"; import * as reviewEffortModule from "../../src/review/review-effort"; import * as repositorySettingsModule from "../../src/settings/repository-settings"; -import * as sentryModule from "../../src/selfhost/sentry"; import { counterValue, renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; import { REQUIRED_CONTEXTS_UNRESOLVED_METRIC } from "../../src/queue/ci-resolution"; import { jobCoalesceKey } from "../../src/selfhost/queue-common"; diff --git a/test/unit/self-host-ops-rename-residue.test.ts b/test/unit/self-host-ops-rename-residue.test.ts index afae5da92b..fc46ce26f5 100644 --- a/test/unit/self-host-ops-rename-residue.test.ts +++ b/test/unit/self-host-ops-rename-residue.test.ts @@ -24,17 +24,6 @@ describe("self-host ops docs rename residue (#5937)", () => { expect(doc).toContain("loopover-docker-prune"); }); - it("documents the real loopover-selfhost-{environment}-{loop} cron monitor slug, not the pre-rename prefix", () => { - // resolveSentryMonitorSlug (src/selfhost/sentry.ts) has emitted "loopover-selfhost-..." slugs since the - // Sentry monitor-naming rebrand; this doc paragraph was left describing the pre-rename "gittensory-selfhost-..." - // pattern. Unlike the SENTRY_RELEASE default (scripts/deploy-selfhost-prebuilt.sh, deliberately still - // "gittensory-selfhost@..." per test/unit/selfhost-sentry-release.test.ts), the monitor slug has no such - // pinned exception -- it's a plain doc/code mismatch. - const doc = readFileSync(SELF_HOSTING_OPS_DOC, "utf8"); - expect(doc).not.toContain("gittensory-selfhost-{environment}-{loop}"); - expect(doc).toContain("loopover-selfhost-{environment}-{loop}"); - }); - it("documents the real loopover-selfhost OTEL_SERVICE_NAME default, not the pre-rename value", () => { // src/selfhost/otel.ts falls back to "loopover-selfhost" when OTEL_SERVICE_NAME is unset; this doc line // still described the pre-rename default. diff --git a/test/unit/selfhost-monitored-work.test.ts b/test/unit/selfhost-monitored-work.test.ts index 9ee7da6fb5..992eb02f62 100644 --- a/test/unit/selfhost-monitored-work.test.ts +++ b/test/unit/selfhost-monitored-work.test.ts @@ -1,14 +1,14 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ - withSentryMonitor: vi.fn( + withPostHogMonitor: vi.fn( async (_name: string, _context: Record, callback: () => Promise) => callback(), ), })); -vi.mock("../../src/selfhost/sentry", () => ({ - withSentryMonitor: mocks.withSentryMonitor, +vi.mock("../../src/selfhost/posthog", () => ({ + withPostHogMonitor: mocks.withPostHogMonitor, })); import { @@ -30,14 +30,14 @@ beforeEach(() => { }); describe("self-host monitored recurring work", () => { - it("runs the scheduled loop through the Sentry monitor with cron context", async () => { + it("runs the scheduled loop through the PostHog monitor with cron context", async () => { const scheduled = vi.fn().mockResolvedValue("done"); await expect(runScheduledLoopWithMonitor("*/2 * * * *", scheduled)).resolves.toBe( "done", ); - expect(mocks.withSentryMonitor).toHaveBeenCalledWith( + expect(mocks.withPostHogMonitor).toHaveBeenCalledWith( "scheduled-loop", { jobType: "scheduled-loop", cron: "*/2 * * * *" }, expect.any(Function), @@ -50,7 +50,7 @@ describe("self-host monitored recurring work", () => { const log = vi.fn(); await runOrbExportWithMonitor(exportBatch, log); - expect(mocks.withSentryMonitor).toHaveBeenLastCalledWith( + expect(mocks.withPostHogMonitor).toHaveBeenLastCalledWith( "orb-export", { jobType: "orb-export" }, expect.any(Function), @@ -117,7 +117,7 @@ describe("self-host monitored recurring work", () => { log, }); - expect(mocks.withSentryMonitor).toHaveBeenCalledWith( + expect(mocks.withPostHogMonitor).toHaveBeenCalledWith( "orb-relay-drain", { jobType: "orb-relay-drain", pendingAckCount: 1 }, expect.any(Function), @@ -427,7 +427,7 @@ describe("self-host monitored recurring work", () => { await registerOrbRelayWithMonitor({ env: { ORB_RELAY_MODE: "push" }, state, register, log }); - expect(mocks.withSentryMonitor).toHaveBeenCalledWith( + expect(mocks.withPostHogMonitor).toHaveBeenCalledWith( "orb-relay-register", { jobType: "orb-relay-register" }, expect.any(Function), diff --git a/test/unit/selfhost-otel.test.ts b/test/unit/selfhost-otel.test.ts index e26515f5a6..42d66d1034 100644 --- a/test/unit/selfhost-otel.test.ts +++ b/test/unit/selfhost-otel.test.ts @@ -1,5 +1,4 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { ROOT_CONTEXT } from "@opentelemetry/api"; const otelMocks = vi.hoisted(() => { const exportedSpans: any[] = []; @@ -93,7 +92,7 @@ describe("self-host OpenTelemetry", () => { OTEL_TRACES_EXPORTER: "console,otlp", OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4318/", OTEL_SERVICE_NAME: "loopover-test", - SENTRY_ENVIRONMENT: "selfhost-test", + POSTHOG_ENVIRONMENT: "selfhost-test", LOOPOVER_VERSION: "loopover-selfhost@test", }), ), @@ -306,7 +305,7 @@ describe("self-host OpenTelemetry", () => { await initOpenTelemetry(env({ OTEL_TRACES_EXPORTER: "otlp", OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector", - SENTRY_RELEASE: "custom-release", + POSTHOG_RELEASE: "custom-release", })); await expect(withOtelSpan("plain-failure", undefined, async () => { throw "plain boom"; @@ -353,228 +352,50 @@ describe("self-host OpenTelemetry", () => { expect(otelMocks.exportedSpans.map((span) => span.name)).toContain("ratio-defaults-on"); }); - it("can export custom spans through a Sentry bridge without requiring OTLP export", async () => { - const sentryEndedSpans: any[] = []; - const sentryProcessor = { - onStart: vi.fn(), - onEnd: vi.fn((span: unknown) => sentryEndedSpans.push(span)), - forceFlush: vi.fn(async () => undefined), - shutdown: vi.fn(async () => undefined), - }; - const validate = vi.fn(); - const propagator = { - inject: vi.fn(), - extract: vi.fn((context) => context), - fields: vi.fn(() => []), - }; - let contextManager: any; - contextManager = { - active: vi.fn(() => ROOT_CONTEXT), - with: vi.fn((context, fn, thisArg, ...args) => fn.apply(thisArg, args)), - bind: vi.fn((_context, target) => target), - enable: vi.fn(() => contextManager), - disable: vi.fn(() => contextManager), - }; + it("defaults the OTLP trace endpoint and auth header to PostHog when POSTHOG_API_KEY is set and no explicit OTEL endpoint is configured", async () => { + expect(resolveOtelTraceEndpoint(env({ POSTHOG_API_KEY: "phc_test" }))).toBe("https://us.i.posthog.com/i/v1/traces"); + expect(resolveOtelTraceEndpoint(env({ POSTHOG_API_KEY: "phc_test", POSTHOG_HOST: "https://eu.i.posthog.com/" }))).toBe( + "https://eu.i.posthog.com/i/v1/traces", + ); + // An explicit operator-configured endpoint always wins over the PostHog fallback. + expect( + resolveOtelTraceEndpoint(env({ POSTHOG_API_KEY: "phc_test", OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector:4318" })), + ).toBe("http://collector:4318/v1/traces"); - expect(await initOpenTelemetry(env({}), { - spanProcessor: sentryProcessor, - propagator, - contextManager, - validate, - })).toBe(true); - await withOtelSpan("selfhost.review.gate", { "loopover.operation": "gate_decision" }, () => undefined); + expect( + await initOpenTelemetry(env({ OTEL_TRACES_EXPORTER: "otlp", POSTHOG_API_KEY: "phc_test" })), + ).toBe(true); + await withOtelSpan("selfhost.queue.job", {}, () => undefined); await flushOpenTelemetry(); - - expect(otelMocks.OTLPTraceExporter).not.toHaveBeenCalled(); - expect(validate).toHaveBeenCalledTimes(1); - expect(contextManager.enable).toHaveBeenCalledTimes(1); - expect(sentryProcessor.onEnd).toHaveBeenCalledTimes(1); - expect(sentryEndedSpans[0].name).toBe("selfhost.review.gate"); - expect(sentryEndedSpans[0].attributes).toMatchObject({ - "loopover.operation": "gate_decision", + expect(otelMocks.exporterInstances[0]?.options).toEqual({ + url: "https://us.i.posthog.com/i/v1/traces", + headers: { Authorization: "Bearer phc_test" }, }); await resetOpenTelemetryForTest(); + otelMocks.exporterInstances.length = 0; - expect(await initOpenTelemetry(env({}), { spanProcessor: sentryProcessor, propagator })).toBe(true); - await resetOpenTelemetryForTest(); - - expect(await initOpenTelemetry(env({}), { spanProcessor: sentryProcessor, contextManager })).toBe(true); - expect(contextManager.enable).toHaveBeenCalledTimes(2); - - await resetOpenTelemetryForTest(); - otelMocks.exportedSpans.length = 0; - - expect(await initOpenTelemetry(env({ - OTEL_TRACES_EXPORTER: "otlp", - OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector", - OTEL_TRACES_SAMPLER: "always_on", - }), { propagator, contextManager })).toBe(true); - await withOtelSpan("otlp-only-with-sentry-context", undefined, () => undefined); - await flushOpenTelemetry(); - expect(otelMocks.exportedSpans.map((span) => span.name)).toContain("otlp-only-with-sentry-context"); - expect(contextManager.enable).toHaveBeenCalledTimes(3); - - await resetOpenTelemetryForTest(); - otelMocks.exportedSpans.length = 0; - sentryEndedSpans.length = 0; - sentryProcessor.onStart.mockClear(); - sentryProcessor.onEnd.mockClear(); - sentryProcessor.forceFlush.mockClear(); - sentryProcessor.shutdown.mockClear(); - - expect(await initOpenTelemetry(env({ - OTEL_TRACES_EXPORTER: "otlp", - OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector", - OTEL_TRACES_SAMPLER: "always_on", - }), { - spanProcessor: sentryProcessor, - propagator, - contextManager, - })).toBe(true); - await withOtelSpan("otlp-and-sentry-no-sampler", undefined, () => undefined); - await flushOpenTelemetry(); - expect(otelMocks.exportedSpans.map((span) => span.name)).toContain("otlp-and-sentry-no-sampler"); - expect(sentryEndedSpans.map((span) => span.name)).toContain("otlp-and-sentry-no-sampler"); - expect(sentryProcessor.forceFlush).toHaveBeenCalledTimes(1); - expect(contextManager.enable).toHaveBeenCalledTimes(4); - - await resetOpenTelemetryForTest(); - otelMocks.exportedSpans.length = 0; - sentryEndedSpans.length = 0; - sentryProcessor.onStart.mockClear(); - sentryProcessor.onEnd.mockClear(); - sentryProcessor.forceFlush.mockClear(); - sentryProcessor.shutdown.mockClear(); - const dropAllBridgeSampler = { - shouldSample: vi.fn((..._args: any[]) => ({ decision: 0 })), - toString: () => "drop-all-bridge-sampler", - }; - - expect(await initOpenTelemetry(env({ - OTEL_TRACES_EXPORTER: "otlp", - OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector", - OTEL_TRACES_SAMPLER: "always_on", - }), { - sampler: dropAllBridgeSampler as any, - spanProcessor: sentryProcessor, - propagator, - contextManager, - })).toBe(true); - await withOtelSpan("otlp-keeps-env-sampler", undefined, () => undefined); - await flushOpenTelemetry(); - expect(otelMocks.exportedSpans.map((span) => span.name)).toContain("otlp-keeps-env-sampler"); - expect(dropAllBridgeSampler.shouldSample).toHaveBeenCalledTimes(1); - expect(dropAllBridgeSampler.shouldSample.mock.calls[0]?.[2]).toBe("otlp-keeps-env-sampler"); - expect(sentryProcessor.onStart).not.toHaveBeenCalled(); - expect(sentryProcessor.onEnd).not.toHaveBeenCalled(); - expect(sentryProcessor.forceFlush).toHaveBeenCalledTimes(1); - expect(contextManager.enable).toHaveBeenCalledTimes(5); - - await resetOpenTelemetryForTest(); - otelMocks.exportedSpans.length = 0; - sentryEndedSpans.length = 0; - sentryProcessor.onStart.mockClear(); - sentryProcessor.onEnd.mockClear(); - sentryProcessor.forceFlush.mockClear(); - sentryProcessor.shutdown.mockClear(); - const sampleBridgeSampler = { - shouldSample: vi.fn((..._args: any[]) => ({ decision: 2 })), - toString: () => "sample-bridge-sampler", - }; - - expect(await initOpenTelemetry(env({ - OTEL_TRACES_EXPORTER: "otlp", - OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector", - OTEL_TRACES_SAMPLER: "always_on", - }), { - sampler: sampleBridgeSampler as any, - spanProcessor: sentryProcessor, - propagator, - contextManager, - })).toBe(true); - await withOtelSpan("otlp-and-sentry-sampled", undefined, () => undefined); - await flushOpenTelemetry(); - expect(otelMocks.exportedSpans.map((span) => span.name)).toContain("otlp-and-sentry-sampled"); - expect(sampleBridgeSampler.shouldSample).toHaveBeenCalledTimes(1); - expect(sentryProcessor.onStart).toHaveBeenCalledTimes(1); - expect(sentryEndedSpans.map((span) => span.name)).toContain("otlp-and-sentry-sampled"); - expect(sentryProcessor.forceFlush).toHaveBeenCalledTimes(1); - expect(contextManager.enable).toHaveBeenCalledTimes(6); - - await resetOpenTelemetryForTest(); - otelMocks.exportedSpans.length = 0; - sentryEndedSpans.length = 0; - sentryProcessor.onStart.mockClear(); - sentryProcessor.onEnd.mockClear(); - sentryProcessor.forceFlush.mockClear(); - sentryProcessor.shutdown.mockClear(); - const sampleSentryWhenOtelDropsSampler = { - shouldSample: vi.fn((..._args: any[]) => ({ decision: 2 })), - toString: () => "sample-sentry-when-otel-drops", - }; - - expect(await initOpenTelemetry(env({ - OTEL_TRACES_EXPORTER: "otlp", - OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector", - OTEL_TRACES_SAMPLER: "always_off", - }), { - sampler: sampleSentryWhenOtelDropsSampler as any, - spanProcessor: sentryProcessor, - propagator, - contextManager, - })).toBe(true); - await withOtelSpan("sentry-keeps-span-when-otel-drops", undefined, () => undefined); - await flushOpenTelemetry(); - expect(otelMocks.exportedSpans.map((span) => span.name)).not.toContain("sentry-keeps-span-when-otel-drops"); - expect(sampleSentryWhenOtelDropsSampler.shouldSample).toHaveBeenCalledTimes(1); - expect(sentryProcessor.onStart).toHaveBeenCalledTimes(1); - expect(sentryEndedSpans.map((span) => span.name)).toContain("sentry-keeps-span-when-otel-drops"); - expect(sentryProcessor.forceFlush).toHaveBeenCalledTimes(1); - expect(contextManager.enable).toHaveBeenCalledTimes(7); + // An explicit endpoint relies entirely on the SDK's own native header handling -- the PostHog auth header + // is never injected once an explicit endpoint is configured, even if POSTHOG_API_KEY also happens to be set. + expect( + await initOpenTelemetry(env({ + OTEL_TRACES_EXPORTER: "otlp", + OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector:4318", + POSTHOG_API_KEY: "phc_test", + })), + ).toBe(true); + expect(otelMocks.exporterInstances[0]?.options).toEqual({ url: "http://collector:4318/v1/traces" }); }); - it("keeps parent-based OTLP sampling isolated when Sentry samples nested spans", async () => { - const sentryEndedSpans: any[] = []; - const sentryProcessor = { - onStart: vi.fn(), - onEnd: vi.fn((span: unknown) => sentryEndedSpans.push(span)), - forceFlush: vi.fn(async () => undefined), - shutdown: vi.fn(async () => undefined), - }; - const sampleSentrySampler = { - shouldSample: vi.fn((..._args: any[]) => ({ decision: 2 })), - toString: () => "sample-sentry", - }; - - expect(await initOpenTelemetry(env({ - OTEL_TRACES_EXPORTER: "otlp", - OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector", - OTEL_TRACES_SAMPLER: "parentbased_always_off", - }), { - sampler: sampleSentrySampler as any, - spanProcessor: sentryProcessor, - })).toBe(true); - await withOtelSpan("sentry-only-parent", undefined, async () => { - await withOtelSpan("sentry-only-child", undefined, () => undefined); - }); - await withOtelSpan( - "remote-parent-not-sampled", - undefined, - () => undefined, - { parentTraceParent: "00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-00" }, - ); - await flushOpenTelemetry(); + it("stays a no-op when POSTHOG_API_KEY is unset and no explicit OTEL endpoint is configured, even with OTEL_TRACES_EXPORTER=otlp", async () => { + expect(await initOpenTelemetry(env({ OTEL_TRACES_EXPORTER: "otlp" }))).toBe(false); + expect(otelMocks.OTLPTraceExporter).not.toHaveBeenCalled(); + }); - expect(otelMocks.exportedSpans.map((span) => span.name)).toEqual([]); - expect(sentryEndedSpans.map((span) => span.name)).toEqual([ - "sentry-only-child", - "sentry-only-parent", - "remote-parent-not-sampled", - ]); - expect(sampleSentrySampler.shouldSample).toHaveBeenCalledTimes(3); - expect(sentryProcessor.onStart).toHaveBeenCalledTimes(3); - expect(sentryProcessor.forceFlush).toHaveBeenCalledTimes(1); + it("never enables tracing just because POSTHOG_API_KEY is set, without OTEL_TRACES_EXPORTER=otlp", async () => { + expect(openTelemetryTraceExportEnabled(env({ POSTHOG_API_KEY: "phc_test" }))).toBe(false); + expect(await initOpenTelemetry(env({ POSTHOG_API_KEY: "phc_test" }))).toBe(false); + expect(otelMocks.OTLPTraceExporter).not.toHaveBeenCalled(); }); it("adds hashed tenant and decision attributes to review pipeline spans", async () => { diff --git a/test/unit/selfhost-pg-queue.test.ts b/test/unit/selfhost-pg-queue.test.ts index ae13dbd42f..80778ade6d 100644 --- a/test/unit/selfhost-pg-queue.test.ts +++ b/test/unit/selfhost-pg-queue.test.ts @@ -7,7 +7,7 @@ import { queueSnapshotFromBinding } from "../../src/selfhost/queue-common"; import { renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; import { RetryableJobError } from "../../src/queue/retryable"; import { hostLoadAvg1PerCore } from "../../src/selfhost/host-pressure"; -import * as sentryModule from "../../src/selfhost/sentry"; +import * as posthogModule from "../../src/selfhost/posthog"; import type { JobMessage } from "../../src/types"; // Real host CPU load is nondeterministic (and can legitimately spike on a busy CI runner), so every @@ -2189,12 +2189,12 @@ describe("createPgQueue (durable #977)", () => { } }); - // (#1824): dead-letter revival stopping SILENTLY is worse than one throwing tick -- a Sentry cron monitor - // now wraps every tick so a stopped timer shows up as a missed check-in, not silence. - it("wraps each revive tick in the queue-dead-letter-revive Sentry monitor", async () => { + // (#1824): dead-letter revival stopping SILENTLY is worse than one throwing tick -- a PostHog monitor + // now wraps every tick so a stopped timer shows up as a missed heartbeat, not silence. + it("wraps each revive tick in the queue-dead-letter-revive PostHog monitor", async () => { process.env.QUEUE_DEAD_LETTER_REVIVE_INTERVAL_MS = "1000"; vi.useFakeTimers(); - const monitorSpy = vi.spyOn(sentryModule, "withSentryMonitor"); + const monitorSpy = vi.spyOn(posthogModule, "withPostHogMonitor"); try { const m = makePool(); // default mock returns { rows: [], rowCount: 0 } for the dead-letter SELECT -- a no-op tick const q = createPgQueue(m.pool, async () => undefined, { maxRetries: 1 }); @@ -2214,13 +2214,13 @@ describe("createPgQueue (durable #977)", () => { } }); - // The monitor rethrows on failure (withSentryMonitor's own contract) -- confirms that rethrow is still caught + // The monitor rethrows on failure (withPostHogMonitor's own contract) -- confirms that rethrow is still caught // by reviveDeadLetterJobsSafely's own try/catch, so a crashing tick behaves exactly as it did before the // monitor was added: logged + captured, never an unhandled rejection. - it("still catches a revive crash after adding the Sentry monitor wrapper (no regression on #2581)", async () => { + it("still catches a revive crash after adding the PostHog monitor wrapper (no regression on #2581)", async () => { process.env.QUEUE_DEAD_LETTER_REVIVE_INTERVAL_MS = "1000"; vi.useFakeTimers(); - const monitorSpy = vi.spyOn(sentryModule, "withSentryMonitor"); + const monitorSpy = vi.spyOn(posthogModule, "withPostHogMonitor"); try { const fn = vi.fn().mockImplementation(async (sql: unknown) => { if (String(sql).includes("WHERE status='dead' AND attempts<$1")) throw new Error("connection terminated unexpectedly"); diff --git a/test/unit/selfhost-sentry.test.ts b/test/unit/selfhost-sentry.test.ts deleted file mode 100644 index 57542172eb..0000000000 --- a/test/unit/selfhost-sentry.test.ts +++ /dev/null @@ -1,1473 +0,0 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; -import { hostname } from "node:os"; - -// Mock @sentry/node so the dynamic import inside initSentry() resolves to spies. Hoisted so vi.mock can see it. -const mocks = vi.hoisted(() => { - const scope = { - setContext: vi.fn(), - setLevel: vi.fn(), - setTag: vi.fn(), - setFingerprint: vi.fn(), - addEventProcessor: vi.fn(), - }; - const client = { id: "sentry-client" }; - return { - client, - scope, - init: vi.fn(() => client), - withScope: vi.fn((cb: (s: typeof scope) => void) => cb(scope)), - captureException: vi.fn(), - captureMessage: vi.fn(), - captureCheckIn: vi.fn((checkIn: { checkInId?: string }) => checkIn.checkInId ?? "check-in-id"), - flush: vi.fn().mockResolvedValue(true), - SentryContextManager: vi.fn(function (this: { kind: string }) { - this.kind = "context-manager"; - }), - validateOpenTelemetrySetup: vi.fn(), - }; -}); -const sentryOtelMocks = vi.hoisted(() => ({ - SentrySampler: vi.fn(function (this: { client: unknown }, client: unknown) { - this.client = client; - }), - SentryPropagator: vi.fn(function (this: { kind: string }) { - this.kind = "propagator"; - }), - SentrySpanProcessor: vi.fn(function (this: { kind: string }) { - this.kind = "span-processor"; - }), -})); -const otelMocks = vi.hoisted(() => ({ - currentOtelTraceIds: vi.fn(), -})); -vi.mock("@sentry/node", () => ({ - init: mocks.init, - withScope: mocks.withScope, - captureException: mocks.captureException, - captureMessage: mocks.captureMessage, - captureCheckIn: mocks.captureCheckIn, - flush: mocks.flush, - SentryContextManager: mocks.SentryContextManager, - validateOpenTelemetrySetup: mocks.validateOpenTelemetrySetup, -})); -vi.mock("@sentry/opentelemetry", () => ({ - SentrySampler: sentryOtelMocks.SentrySampler, - SentryPropagator: sentryOtelMocks.SentryPropagator, - SentrySpanProcessor: sentryOtelMocks.SentrySpanProcessor, -})); -vi.mock("../../src/selfhost/otel", () => ({ - currentOtelTraceIds: otelMocks.currentOtelTraceIds, - openTelemetryTraceExportEnabled: (env: NodeJS.ProcessEnv) => - Boolean(env.OTEL_TRACES_EXPORTER?.includes("otlp") && env.OTEL_EXPORTER_OTLP_ENDPOINT), -})); - -import { - buildSentryOpenTelemetryBridge, - initSentry, - captureError, - captureReviewFailure, - flushSentry, - forwardStructuredLogToSentry, - installStructuredLogForwarding, - resolveSentryRelease, - resolveSentryTracesSampleRate, - resolveSentryMonitorSlug, - scrubEvent, - resetSentryForTest, - withSentryMonitor, - SENTRY_MONITOR_NAMES, - SENTRY_OPERATIONAL_SUBSYSTEMS, - SENTRY_OPERATIONAL_TAG_KEYS, -} from "../../src/selfhost/sentry"; - -beforeEach(() => { - resetSentryForTest(); - vi.clearAllMocks(); - otelMocks.currentOtelTraceIds.mockReturnValue(undefined); -}); - -// The structured-log forwarder captures a synthetic Error via captureException (name = event slug, message = the -// value) so issues show a real "type: value", never "(No error message)". This reads back the last captured Error. -const lastCapturedError = (): Error => - mocks.captureException.mock.calls.at(-1)?.[0] as Error; -const scrubbedEvent = (event: T): T => { - const scrubbed = scrubEvent(event); - expect(scrubbed).not.toBeNull(); - return scrubbed as T; -}; -const lastInitOptions = (): any => - (mocks.init.mock.calls.at(-1) as unknown[] | undefined)?.[0] as any; -const fakeClassicAccessToken = (): string => `${"github" + "_pat_"}${"a".repeat(24)}`; -const fakeQueryTokenKey = (): string => "github" + "_token"; - -describe("scrubEvent — redact secrets before an event leaves the box", () => { - it("redacts secret-keyed fields in headers/contexts/extra, recurses, and leaves safe fields", () => { - const ev = scrubbedEvent({ - request: { headers: { authorization: "Bearer abc", "x-trace": "ok" } }, - contexts: { - loopover: { - jobId: "j1", - apiKey: "shh", - nested: { secretToken: "deep" }, - }, - }, - extra: { note: "fine" }, - }) as any; - expect(ev.request.headers.authorization).toBe("[redacted]"); - expect(ev.request.headers["x-trace"]).toBe("ok"); - expect(ev.contexts.loopover.apiKey).toBe("[redacted]"); - expect(ev.contexts.loopover.jobId).toBe("j1"); - expect(ev.contexts.loopover.nested.secretToken).toBe("[redacted]"); - expect(ev.extra.note).toBe("fine"); - }); - - it("is safe when headers/contexts/extra are absent (the !obj branch)", () => { - expect(() => scrubEvent({})).not.toThrow(); - expect(scrubEvent({})).toEqual({}); - }); - - it("stops at the depth guard without infinite recursion, still redacting shallow secrets", () => { - let deep: any = { secretToken: "x" }; - for (let i = 0; i < 8; i++) deep = { a: deep }; - let deepArray: any = { secretToken: "x" }; - for (let i = 0; i < 7; i++) deepArray = [deepArray]; - const ev = scrubbedEvent({ - extra: { token: "shallow", deep, deepArray }, - }) as any; - let deepCursor = ev.extra.deep; - for (let i = 0; i < 5; i++) deepCursor = deepCursor.a; - let arrayCursor = ev.extra.deepArray; - for (let i = 0; i < 5; i++) arrayCursor = arrayCursor[0]; - expect(ev.extra.token).toBe("[redacted]"); - expect(deepCursor.a).toBe("[redacted]"); - expect(arrayCursor[0]).toBe("[redacted]"); - }); - - it("drops request bodies, denies unknown contexts, and scrubs PR/private payload fields (#1000)", () => { - const fakeToken = fakeClassicAccessToken(); - const ev = scrubbedEvent({ - request: { - headers: { authorization: `Bearer ${"a".repeat(16)}`, "x-trace": "ok" }, - data: { prompt: "review this diff" }, - body: "raw request body", - cookies: { session: "abc" }, - }, - contexts: { - loopover: { - safeReason: "provider unavailable", - pullRequestTitle: "PR title with private rubric", - reviewText: "raw review body", - repoConfig: "private repo config", - nested: { apiKey: "provider secret" }, - }, - mystery: { repoConfig: "should not leave" }, - runtime: { name: "node" }, - }, - extra: { - diff: "@@ raw diff", - note: `wallet raw score /home/alice/project ${fakeToken}`, - attempts: 2, - nil: null, - values: ["hotkey", { apiKey: "nested" }, 3, null], - }, - tags: { repo: "owner/repo", authToken: "token" }, - }) as any; - - expect(ev.request.data).toBeUndefined(); - expect(ev.request.body).toBeUndefined(); - expect(ev.request.cookies).toBeUndefined(); - expect(ev.request.headers.authorization).toBe("[redacted]"); - expect(ev.request.headers["x-trace"]).toBe("ok"); - expect(ev.contexts.mystery).toBeUndefined(); - expect(ev.contexts.runtime.name).toBe("node"); - expect(ev.contexts.loopover.pullRequestTitle).toBe("[redacted]"); - expect(ev.contexts.loopover.reviewText).toBe("[redacted]"); - expect(ev.contexts.loopover.repoConfig).toBe("[redacted]"); - expect(ev.contexts.loopover.nested.apiKey).toBe("[redacted]"); - expect(ev.extra.diff).toBe("[redacted]"); - expect(ev.extra.note).not.toContain(fakeToken); - expect(ev.extra.note).not.toMatch(/wallet|raw score|\/home\/alice/i); - expect(ev.extra.note).toContain(""); - expect(ev.extra.attempts).toBe(2); - expect(ev.extra.nil).toBeNull(); - expect(ev.extra.values).toEqual([ - "private context", - { apiKey: "[redacted]" }, - 3, - null, - ]); - expect(ev.tags.repo).toBe("owner/repo"); - expect(ev.tags.authToken).toBe("[redacted]"); - }); - - it("preserves contexts.review repo/pr while redacting review text fields (#1824, PR #1881 regression)", () => { - const ev = scrubbedEvent({ - contexts: { - review: { - repo: "owner/repo", - pr: 7, - head_sha: "abc123", - operation: "gate_decision", - reviewText: "private review body", - reviewBody: "also private", - commentBody: "private comment", - comment_text: "private comment text", - }, - }, - }) as any; - - expect(ev.contexts.review.repo).toBe("owner/repo"); - expect(ev.contexts.review.pr).toBe(7); - expect(ev.contexts.review.head_sha).toBe("abc123"); - expect(ev.contexts.review.operation).toBe("gate_decision"); - expect(ev.contexts.review.reviewText).toBe("[redacted]"); - expect(ev.contexts.review.reviewBody).toBe("[redacted]"); - expect(ev.contexts.review.commentBody).toBe("[redacted]"); - expect(ev.contexts.review.comment_text).toBe("[redacted]"); - }); - - it("scrubs request URL/query fields and deletes top-level user data", () => { - const queryTokenKey = fakeQueryTokenKey(); - const ev = scrubbedEvent({ - request: { - url: `https://self.host/review?${queryTokenKey}=abc123&repo=owner%2Frepo`, - query_string: `${queryTokenKey}=abc123&path=/home/alice/project&safe=ok`, - query: { [queryTokenKey]: "abc123", safe: "ok" }, - }, - user: { id: "123", email: "person@example.com" }, - }) as any; - - const url = new URL(ev.request.url); - const query = new URLSearchParams(ev.request.query_string); - expect(url.searchParams.get(queryTokenKey)).toBe("[redacted]"); - expect(url.searchParams.get("repo")).toBe("owner/repo"); - expect(query.get(queryTokenKey)).toBe("[redacted]"); - expect(query.get("path")).toBe(""); - expect(query.get("safe")).toBe("ok"); - expect(ev.request.query[queryTokenKey]).toBe("[redacted]"); - expect(ev.request.query.safe).toBe("ok"); - expect(ev.user).toBeUndefined(); - }); - - it("scrubs breadcrumbs, exception metadata, messages, and transaction names", () => { - const ev = scrubbedEvent({ - message: "gate prompt leaked with Bearer abcdefghijklmnop", - transaction: "review /Users/alice/private", - breadcrumbs: [ - { - message: "prompt mentions hotkey", - data: { responseBody: "raw provider body", safe: "kept" }, - }, - ], - exception: { - values: [ - { - value: "codex failed with eyJaaaaaaaa.bbbbbbbb.cccccccc", - stacktrace: { - frames: [ - { - filename: "/tmp/repo/file.ts", - vars: { token: "abc", safe: "value" }, - }, - ], - }, - }, - ], - }, - }) as any; - - expect(ev.message).not.toMatch(/gate prompt|Bearer abc/i); - expect(ev.transaction).toContain(""); - expect(ev.breadcrumbs[0].message).not.toMatch(/hotkey/i); - expect(ev.breadcrumbs[0].data.responseBody).toBe("[redacted]"); - expect(ev.breadcrumbs[0].data.safe).toBe("kept"); - expect(ev.exception.values[0].value).not.toMatch(/eyJaaaaaaaa/i); - expect(ev.exception.values[0].stacktrace.frames[0].filename).toContain(""); - expect(ev.exception.values[0].stacktrace.frames[0].vars.token).toBe("[redacted]"); - expect(ev.exception.values[0].stacktrace.frames[0].vars.safe).toBe("value"); - }); - - // Regression (#1825): the Orb broker's enrollment id/secret (createOpaqueToken("orbenr"/"orbsec"), - // src/orb/broker.ts) are bare opaque tokens with no "secret"/"token"-NAMED field for the key-based redaction - // to catch when a broker error message quotes one directly (e.g. an error string embedding the failed - // Authorization value) — the VALUE-based SECRET_VALUE pattern must recognize the orbenr_/orbsec_ shape too. - it("redacts a bare Orb enrollment id/secret value from an exception message (#1825)", () => { - const fakeEnrollId = `orbenr_${"c".repeat(64)}`; - const fakeSecret = `orbsec_${"d".repeat(64)}`; - const ev = scrubbedEvent({ - exception: { - values: [{ value: `Orb broker rejected enrollment ${fakeEnrollId} using secret ${fakeSecret}` }], - }, - }) as any; - expect(ev.exception.values[0].value).not.toContain(fakeEnrollId); - expect(ev.exception.values[0].value).not.toContain(fakeSecret); - expect(ev.exception.values[0].value).toBe("Orb broker rejected enrollment [redacted] using secret [redacted]"); - }); - - it("scrubs transaction span descriptions and data before sending transaction events", () => { - const queryTokenKey = fakeQueryTokenKey(); - const ev = scrubbedEvent({ - spans: [ - { - description: `GET /hooks?${queryTokenKey}=abc123&safe=ok`, - data: { - callbackUrl: `https://self.host/callback?${queryTokenKey}=abc123&safe=ok`, - relativeUrl: `/callback?${queryTokenKey}=abc123&safe=ok`, - noQueryUrl: "https://self.host/callback", - query_string: `${queryTokenKey}=abc123&path=/home/alice/project`, - prompt: "raw prompt", - }, - }, - ], - }) as any; - - const callbackUrl = new URL(ev.spans[0].data.callbackUrl); - expect(ev.spans[0].description).not.toContain("abc123"); - expect(callbackUrl.searchParams.get(queryTokenKey)).toBe("[redacted]"); - expect(callbackUrl.searchParams.get("safe")).toBe("ok"); - expect(ev.spans[0].data.relativeUrl).toContain( - `${queryTokenKey}=%5Bredacted%5D`, - ); - expect(ev.spans[0].data.noQueryUrl).toBe("https://self.host/callback"); - expect(new URLSearchParams(ev.spans[0].data.query_string).get("path")).toBe( - "", - ); - expect(ev.spans[0].data.prompt).toBe("[redacted]"); - }); - - it("drops the event when scrubbing itself fails instead of sending it unscrubbed", () => { - const event = { - get request() { - throw new Error("getter failed"); - }, - }; - - expect(scrubEvent(event)).toBeNull(); - }); - - it("drops raw installation ids before hashing is available and uses a stable hash after init", async () => { - const noHasherEvent = scrubbedEvent({ - extra: { installationId: 143010787 }, - }) as any; - expect(noHasherEvent.extra.installationId).toBeUndefined(); - expect(noHasherEvent.extra.installation_id_hash).toBeUndefined(); - - const invalidEvent = scrubbedEvent({ - extra: { installation_id: "not-an-installation" }, - }) as any; - expect(invalidEvent.extra.installation_id).toBeUndefined(); - expect(invalidEvent.extra.installation_id_hash).toBeUndefined(); - - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - const ev = scrubbedEvent({ - tags: { installationId: 143010787, repo: "owner/repo" }, - contexts: { log: { installation_id: "143010787" } }, - }) as any; - - expect(ev.tags.installationId).toBeUndefined(); - expect(ev.tags.installation_id_hash).toBe("68b9c2136087c5ca"); - expect(ev.contexts.log.installation_id).toBeUndefined(); - expect(ev.contexts.log.installation_id_hash).toBe("68b9c2136087c5ca"); - }); -}); - -describe("disabled when SENTRY_DSN is unset (modular opt-out → complete no-op)", () => { - it("initSentry returns false; capture/flush are safe no-ops and never touch the SDK", async () => { - expect(await initSentry({} as unknown as NodeJS.ProcessEnv)).toBe(false); - captureError(new Error("x"), { a: 1 }); - captureReviewFailure(new Error("y"), { repo: "o/r" }); - await expect( - withSentryMonitor( - "scheduled-loop", - { jobType: "scheduled-loop" }, - async () => "ok", - ), - ).resolves.toBe("ok"); - await flushSentry(); - expect(mocks.init).not.toHaveBeenCalled(); - expect(mocks.captureException).not.toHaveBeenCalled(); - expect(mocks.captureCheckIn).not.toHaveBeenCalled(); - expect(mocks.flush).not.toHaveBeenCalled(); - }); -}); - -describe("enabled when SENTRY_DSN is set", () => { - it("resolves the Sentry release from explicit env, then the baked image version, ignoring blanks", () => { - expect( - resolveSentryRelease({ - SENTRY_RELEASE: " custom-release ", - LOOPOVER_VERSION: "gittensory-selfhost@0.1.0", - } as unknown as NodeJS.ProcessEnv), - ).toBe("custom-release"); - expect( - resolveSentryRelease({ - SENTRY_RELEASE: " ", - LOOPOVER_VERSION: " gittensory-selfhost@0.1.0 ", - } as unknown as NodeJS.ProcessEnv), - ).toBe("gittensory-selfhost@0.1.0"); - expect(resolveSentryRelease({} as unknown as NodeJS.ProcessEnv)).toBeUndefined(); - }); - - it("resolves a positive Sentry trace sample rate and treats unset/zero/invalid as disabled", () => { - expect(resolveSentryTracesSampleRate({} as unknown as NodeJS.ProcessEnv)).toBeUndefined(); - expect(resolveSentryTracesSampleRate({ SENTRY_TRACES_SAMPLE_RATE: " " } as unknown as NodeJS.ProcessEnv)).toBeUndefined(); - expect(resolveSentryTracesSampleRate({ SENTRY_TRACES_SAMPLE_RATE: "0" } as unknown as NodeJS.ProcessEnv)).toBeUndefined(); - expect(resolveSentryTracesSampleRate({ SENTRY_TRACES_SAMPLE_RATE: "-1" } as unknown as NodeJS.ProcessEnv)).toBeUndefined(); - expect(resolveSentryTracesSampleRate({ SENTRY_TRACES_SAMPLE_RATE: "nope" } as unknown as NodeJS.ProcessEnv)).toBeUndefined(); - expect(resolveSentryTracesSampleRate({ SENTRY_TRACES_SAMPLE_RATE: "0.25" } as unknown as NodeJS.ProcessEnv)).toBe(0.25); - expect(resolveSentryTracesSampleRate({ SENTRY_TRACES_SAMPLE_RATE: "9" } as unknown as NodeJS.ProcessEnv)).toBe(1); - }); - - it("returns true and wires init with defaults (?? right-hand branches) + the scrubber as beforeSend", async () => { - expect( - await initSentry({ - SENTRY_DSN: "https://k@o.ingest/1", - } as unknown as NodeJS.ProcessEnv), - ).toBe(true); - expect(mocks.init).toHaveBeenCalledTimes(1); - const opts = lastInitOptions(); - expect(opts.environment).toBe("production"); - expect(opts.release).toBeUndefined(); - expect(opts.tracesSampleRate).toBeUndefined(); - expect(opts.skipOpenTelemetrySetup).toBeUndefined(); - expect( - opts.beforeSend({ extra: { sessionToken: "s" } }).extra.sessionToken, - ).toBe("[redacted]"); - expect( - opts.beforeSendTransaction({ - contexts: { unknown: { token: "s" }, trace: { op: "job" } }, - }).contexts, - ).toEqual({ trace: { op: "job" } }); - }); - - it("honors explicit env (?? left-hand branches)", async () => { - await initSentry({ - SENTRY_DSN: "d", - SENTRY_ENVIRONMENT: "staging", - SENTRY_RELEASE: "v9", - SENTRY_TRACES_SAMPLE_RATE: "0.5", - SENTRY_SERVER_NAME: "gittensory-us-east", - } as unknown as NodeJS.ProcessEnv); - const opts = lastInitOptions(); - expect(opts.environment).toBe("staging"); - expect(opts.release).toBe("v9"); - expect(opts.tracesSampleRate).toBe(0.5); - expect(opts.skipOpenTelemetrySetup).toBe(true); - expect(opts.serverName).toBe("gittensory-us-east"); - }); - - it("defaults serverName to the OS hostname (not the API-origin URL) when SENTRY_SERVER_NAME is unset/blank", async () => { - await initSentry({ SENTRY_DSN: "d", SENTRY_SERVER_NAME: " ", PUBLIC_API_ORIGIN: "https://self.host" } as unknown as NodeJS.ProcessEnv); - expect(lastInitOptions().serverName).toBe(hostname()); - }); - - it("uses the custom OpenTelemetry setup when OTLP traces are enabled even if Sentry trace export is off", async () => { - await initSentry({ - SENTRY_DSN: "d", - OTEL_TRACES_EXPORTER: "otlp", - OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4318", - } as unknown as NodeJS.ProcessEnv); - const opts = lastInitOptions(); - expect(opts.tracesSampleRate).toBeUndefined(); - expect(opts.skipOpenTelemetrySetup).toBe(true); - }); - - it("builds the Sentry OpenTelemetry bridge, adding the span processor only when Sentry traces are sampled", async () => { - await expect(buildSentryOpenTelemetryBridge()).resolves.toBeUndefined(); - - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - let bridge = await buildSentryOpenTelemetryBridge(); - expect(sentryOtelMocks.SentrySampler).not.toHaveBeenCalled(); - expect(sentryOtelMocks.SentryPropagator).toHaveBeenCalledTimes(1); - expect(mocks.SentryContextManager).toHaveBeenCalledTimes(1); - expect(sentryOtelMocks.SentrySpanProcessor).not.toHaveBeenCalled(); - bridge?.validate?.(); - expect(mocks.validateOpenTelemetrySetup).toHaveBeenCalledTimes(1); - - resetSentryForTest(); - vi.clearAllMocks(); - await initSentry({ - SENTRY_DSN: "d", - SENTRY_TRACES_SAMPLE_RATE: "0.5", - } as unknown as NodeJS.ProcessEnv); - bridge = await buildSentryOpenTelemetryBridge(); - expect(bridge?.sampler).toBeInstanceOf(sentryOtelMocks.SentrySampler); - expect(bridge?.spanProcessor).toBeInstanceOf(sentryOtelMocks.SentrySpanProcessor); - }); - - it("uses the image-baked version as the release fallback and ignores blank overrides", async () => { - expect( - resolveSentryRelease({ - SENTRY_RELEASE: " ", - LOOPOVER_VERSION: " gittensory-selfhost@0.1.0 ", - } as unknown as NodeJS.ProcessEnv), - ).toBe("gittensory-selfhost@0.1.0"); - - await initSentry({ - SENTRY_DSN: "d", - SENTRY_RELEASE: "", - LOOPOVER_VERSION: "gittensory-selfhost@0.1.0", - } as unknown as NodeJS.ProcessEnv); - expect(lastInitOptions().release).toBe( - "gittensory-selfhost@0.1.0", - ); - }); - - it("prefers an explicit nonblank SENTRY_RELEASE over LOOPOVER_VERSION", () => { - expect( - resolveSentryRelease({ - SENTRY_RELEASE: "custom@sha", - LOOPOVER_VERSION: "gittensory-selfhost@0.1.0", - } as unknown as NodeJS.ProcessEnv), - ).toBe("custom@sha"); - }); - - it("captureError sends with context, tags operational fields, and without context skips setContext", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - captureError(new Error("boom"), { kind: "job_dead" }); - expect(mocks.scope.setContext).toHaveBeenCalledWith("loopover", { - kind: "job_dead", - }); - expect(mocks.scope.setTag).toHaveBeenCalledWith("kind", "job_dead"); - expect(mocks.captureException).toHaveBeenCalledTimes(1); - mocks.scope.setContext.mockClear(); - mocks.scope.setTag.mockClear(); - captureError(new Error("invalid install"), { - kind: "job_dead", - installation_id: "not-an-installation", - }); - expect(mocks.scope.setContext).toHaveBeenCalledWith("loopover", { - kind: "job_dead", - }); - expect(mocks.scope.setTag).toHaveBeenCalledWith("kind", "job_dead"); - mocks.scope.setContext.mockClear(); - captureError("plain string with no context"); - expect(mocks.scope.setContext).not.toHaveBeenCalled(); - expect(mocks.captureException).toHaveBeenCalledTimes(3); - }); - - it("captureReviewFailure sets error level + repo/PR/SHA tags, skipping null/undefined, and works without context", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - captureReviewFailure(new Error("rev"), { - repo: "o/r", - pr: 7, - head_sha: "abc", - installationId: 1, - operation: "gate_decision", - decision_outcome: "failure", - owner: null, - }); - expect(mocks.scope.setLevel).toHaveBeenCalledWith("error"); - expect(mocks.scope.setTag).toHaveBeenCalledWith("repo", "o/r"); - expect(mocks.scope.setTag).toHaveBeenCalledWith("pr", "7"); - expect(mocks.scope.setTag).toHaveBeenCalledWith("head_sha", "abc"); - expect(mocks.scope.setTag).toHaveBeenCalledWith("installation_id_hash", "21ab41515eeee762"); - expect(mocks.scope.setTag).toHaveBeenCalledWith("operation", "gate_decision"); - expect(mocks.scope.setTag).toHaveBeenCalledWith("decision_outcome", "failure"); - expect(mocks.scope.setContext).toHaveBeenCalledWith("review", expect.objectContaining({ - installation_id_hash: "21ab41515eeee762", - })); - expect(mocks.scope.setContext).not.toHaveBeenCalledWith("review", expect.objectContaining({ - installationId: 1, - })); - expect(mocks.scope.setTag).not.toHaveBeenCalledWith( - "owner", - expect.anything(), - ); - captureReviewFailure("string failure, no context"); - expect(mocks.captureException).toHaveBeenCalledTimes(2); - }); - - it("captureError with an eventName renames the captured Error so the Sentry title isn't the generic 'Error', and groups it by that same name (#5010)", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - captureError(new Error("self-host queue processing lease expired"), { kind: "job_dead" }, "processing_timeout"); - expect(lastCapturedError().name).toBe("processing_timeout"); - expect(lastCapturedError().message).toBe("self-host queue processing lease expired"); - expect(mocks.scope.setFingerprint).toHaveBeenCalledWith(["loopover-error", "processing_timeout"]); - }); - - it("captureError without an eventName leaves a caught exception's own name untouched, and never overrides Sentry's default grouping", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - class HttpError extends Error { - constructor(message: string) { - super(message); - this.name = "HttpError"; - } - } - captureError(new HttpError("merge already in progress"), { kind: "agent_merge_blocked" }); - expect(lastCapturedError().name).toBe("HttpError"); - expect(mocks.scope.setFingerprint).not.toHaveBeenCalled(); - }); - - it("captureReviewFailure with an eventName renames the captured Error and groups it the same way captureError does (#5010) -- this is what consolidates the same failure captured from two different call sites (GITTENSORY-5/10, GITTENSORY-C/W) into one issue", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - captureReviewFailure(new Error("AI review inconclusive — no usable verdict for the PR head"), { repo: "o/r" }, "ai_review_inconclusive"); - expect(lastCapturedError().name).toBe("ai_review_inconclusive"); - expect(mocks.scope.setFingerprint).toHaveBeenCalledWith(["loopover-review-failure", "ai_review_inconclusive"]); - }); - - it("captureReviewFailure without an eventName never overrides Sentry's default grouping", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - captureReviewFailure(new Error("rev"), { repo: "o/r" }); - expect(mocks.scope.setFingerprint).not.toHaveBeenCalled(); - }); - - it("captureError/captureReviewFailure with an eventName still names a non-Error value's synthesized Error", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - captureError("plain string failure", undefined, "boot"); - expect(lastCapturedError().name).toBe("boot"); - expect(lastCapturedError().message).toBe("plain string failure"); - captureReviewFailure("plain string review failure", undefined, "ai_review_failed"); - expect(lastCapturedError().name).toBe("ai_review_failed"); - }); - - it("captureError/captureReviewFailure with an eventName never mutate caught errors that reject name writes", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - - const timeoutError = new DOMException("signal timed out", "TimeoutError"); - captureError(timeoutError, { kind: "fetch_timeout" }, "github_fetch_failed"); - - expect(timeoutError.name).toBe("TimeoutError"); - expect(lastCapturedError()).not.toBe(timeoutError); - expect(lastCapturedError().name).toBe("github_fetch_failed"); - expect(lastCapturedError().message).toBe("signal timed out"); - expect(lastCapturedError().cause).toBe(timeoutError); - - const frozenError = Object.freeze(new Error("review failed")); - captureReviewFailure(frozenError, { repo: "o/r" }, "ai_review_failed"); - - expect(frozenError.name).toBe("Error"); - expect(lastCapturedError()).not.toBe(frozenError); - expect(lastCapturedError().name).toBe("ai_review_failed"); - expect(lastCapturedError().message).toBe("review failed"); - expect(lastCapturedError().cause).toBe(frozenError); - }); - - it("adds active OTEL trace ids to captured Sentry events", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - otelMocks.currentOtelTraceIds.mockReturnValue({ - trace_id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - span_id: "bbbbbbbbbbbbbbbb", - }); - - captureError(new Error("boom"), { kind: "job_dead" }); - expect(mocks.scope.setTag).toHaveBeenCalledWith("trace_id", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - expect(mocks.scope.setTag).toHaveBeenCalledWith("span_id", "bbbbbbbbbbbbbbbb"); - expect(mocks.scope.setContext).toHaveBeenCalledWith("otel", { - trace_id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - span_id: "bbbbbbbbbbbbbbbb", - }); - - mocks.scope.setTag.mockClear(); - mocks.scope.setContext.mockClear(); - captureReviewFailure(new Error("review"), { repo: "o/r", pr: 9 }); - expect(mocks.scope.setTag).toHaveBeenCalledWith("trace_id", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - expect(mocks.scope.setTag).toHaveBeenCalledWith("span_id", "bbbbbbbbbbbbbbbb"); - expect(mocks.scope.setContext).toHaveBeenCalledWith("otel", { - trace_id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - span_id: "bbbbbbbbbbbbbbbb", - }); - }); - - it("flushSentry delegates to Sentry.flush with the timeout", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - await flushSentry(123); - expect(mocks.flush).toHaveBeenCalledWith(123); - }); - - it("flushSentry swallows a flush rejection (never breaks shutdown)", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - mocks.flush.mockRejectedValueOnce(new Error("network")); - await expect(flushSentry()).resolves.toBeUndefined(); - }); - - it("builds stable environment-aware monitor slugs", () => { - expect(resolveSentryMonitorSlug("scheduled-loop", "Prod East/1")).toBe( - "loopover-selfhost-prod-east-1-scheduled-loop", - ); - expect(resolveSentryMonitorSlug("orb-export", " !!! ")).toBe( - "loopover-selfhost-production-orb-export", - ); - expect(resolveSentryMonitorSlug("orb-relay-drain", "x".repeat(60))).toBe( - `loopover-selfhost-${"x".repeat(48)}-orb-relay-drain`, - ); - expect(resolveSentryMonitorSlug("queue-dead-letter-revive", "prod")).toBe( - "loopover-selfhost-prod-queue-dead-letter-revive", - ); - }); - - it("records dead-letter-revive check-ins on the 30-minute schedule (#1824)", async () => { - await initSentry({ - SENTRY_DSN: "d", - SENTRY_ENVIRONMENT: "prod", - } as unknown as NodeJS.ProcessEnv); - - await expect( - withSentryMonitor( - "queue-dead-letter-revive", - { jobType: "queue-dead-letter-revive" }, - async () => 3, - ), - ).resolves.toBe(3); - - expect(mocks.captureCheckIn).toHaveBeenNthCalledWith( - 1, - { monitorSlug: "loopover-selfhost-prod-queue-dead-letter-revive", status: "in_progress" }, - expect.objectContaining({ - schedule: { type: "interval", value: 30, unit: "minute" }, - checkinMargin: 10, - maxRuntime: 5, - failureIssueThreshold: 2, - recoveryThreshold: 1, - }), - ); - expect(mocks.captureCheckIn).toHaveBeenNthCalledWith( - 2, - expect.objectContaining({ - monitorSlug: "loopover-selfhost-prod-queue-dead-letter-revive", - status: "ok", - checkInId: "check-in-id", - duration: expect.any(Number), - }), - ); - expect(mocks.captureException).not.toHaveBeenCalled(); - }); - - it("derives the dead-letter-revive schedule from an operator's configured interval override (#1824 regression)", async () => { - process.env.QUEUE_DEAD_LETTER_REVIVE_INTERVAL_MS = String(90 * 60_000); - try { - await initSentry({ - SENTRY_DSN: "d", - SENTRY_ENVIRONMENT: "prod", - } as unknown as NodeJS.ProcessEnv); - - await withSentryMonitor("queue-dead-letter-revive", { jobType: "queue-dead-letter-revive" }, async () => 1); - - expect(mocks.captureCheckIn).toHaveBeenNthCalledWith( - 1, - { monitorSlug: "loopover-selfhost-prod-queue-dead-letter-revive", status: "in_progress" }, - expect.objectContaining({ - schedule: { type: "interval", value: 90, unit: "minute" }, - checkinMargin: 30, - }), - ); - } finally { - delete process.env.QUEUE_DEAD_LETTER_REVIVE_INTERVAL_MS; - } - }); - - it("floors an operator's dead-letter-revive interval override to at least 1 minute for the monitor schedule", async () => { - process.env.QUEUE_DEAD_LETTER_REVIVE_INTERVAL_MS = "1000"; - try { - await initSentry({ - SENTRY_DSN: "d", - SENTRY_ENVIRONMENT: "prod", - } as unknown as NodeJS.ProcessEnv); - - await withSentryMonitor("queue-dead-letter-revive", { jobType: "queue-dead-letter-revive" }, async () => 1); - - expect(mocks.captureCheckIn).toHaveBeenNthCalledWith( - 1, - { monitorSlug: "loopover-selfhost-prod-queue-dead-letter-revive", status: "in_progress" }, - expect.objectContaining({ - schedule: { type: "interval", value: 1, unit: "minute" }, - checkinMargin: 5, - }), - ); - } finally { - delete process.env.QUEUE_DEAD_LETTER_REVIVE_INTERVAL_MS; - } - }); - - it("records successful Sentry cron monitor check-ins with the configured schedule", async () => { - await initSentry({ - SENTRY_DSN: "d", - SENTRY_ENVIRONMENT: "Self Host", - } as unknown as NodeJS.ProcessEnv); - - await expect( - withSentryMonitor( - "scheduled-loop", - { jobType: "scheduled-loop" }, - async () => "ok", - ), - ).resolves.toBe("ok"); - - expect(mocks.captureCheckIn).toHaveBeenNthCalledWith( - 1, - { monitorSlug: "loopover-selfhost-self-host-scheduled-loop", status: "in_progress" }, - expect.objectContaining({ - schedule: { type: "interval", value: 2, unit: "minute" }, - checkinMargin: 3, - maxRuntime: 2, - failureIssueThreshold: 2, - recoveryThreshold: 1, - }), - ); - expect(mocks.captureCheckIn).toHaveBeenNthCalledWith( - 2, - expect.objectContaining({ - monitorSlug: "loopover-selfhost-self-host-scheduled-loop", - status: "ok", - checkInId: "check-in-id", - duration: expect.any(Number), - }), - ); - expect(mocks.captureException).not.toHaveBeenCalled(); - }); - - it("gives orb-relay-drain a 6-minute checkin margin so a normal redeploy cycle doesn't false-positive as a missed check-in (LOOPOVER-12 regression)", async () => { - await initSentry({ - SENTRY_DSN: "d", - SENTRY_ENVIRONMENT: "prod", - } as unknown as NodeJS.ProcessEnv); - - await expect( - withSentryMonitor("orb-relay-drain", { jobType: "orb-relay-drain" }, async () => "ok"), - ).resolves.toBe("ok"); - - expect(mocks.captureCheckIn).toHaveBeenNthCalledWith( - 1, - { monitorSlug: "loopover-selfhost-prod-orb-relay-drain", status: "in_progress" }, - expect.objectContaining({ - schedule: { type: "interval", value: 1, unit: "minute" }, - checkinMargin: 6, - maxRuntime: 1, - failureIssueThreshold: 3, - recoveryThreshold: 1, - }), - ); - expect(mocks.captureException).not.toHaveBeenCalled(); - }); - - it("records failed Sentry cron monitor check-ins with sanitized context", async () => { - await initSentry({ - SENTRY_DSN: "d", - SENTRY_ENVIRONMENT: "prod", - } as unknown as NodeJS.ProcessEnv); - const longText = "x".repeat(200); - - await expect( - withSentryMonitor( - "orb-export", - { - jobType: "orb-export", - repo: "JSONbored/gittensory", - exported: 7, - dryRun: false, - token: "secret", - privateKey: "key", - badNumber: Number.NaN, - nested: { ignored: true }, - empty: null, - missing: undefined, - longText, - }, - async () => { - throw new Error("export failed"); - }, - ), - ).rejects.toThrow("export failed"); - - expect(mocks.captureCheckIn).toHaveBeenNthCalledWith( - 2, - expect.objectContaining({ - monitorSlug: "loopover-selfhost-prod-orb-export", - status: "error", - checkInId: "check-in-id", - duration: expect.any(Number), - }), - ); - expect(mocks.scope.setLevel).toHaveBeenCalledWith("error"); - expect(mocks.scope.setTag).toHaveBeenCalledWith( - "monitor", - "loopover-selfhost-prod-orb-export", - ); - expect(mocks.scope.setTag).toHaveBeenCalledWith("jobType", "orb-export"); - expect(mocks.scope.setTag).toHaveBeenCalledWith( - "kind", - "sentry_monitor_orb-export", - ); - expect(mocks.scope.setTag).toHaveBeenCalledWith("subsystem", "scheduled"); - expect(mocks.scope.setFingerprint).toHaveBeenCalledWith([ - "loopover-sentry-monitor", - "orb-export", - ]); - expect(mocks.scope.setContext).toHaveBeenCalledWith("sentry_monitor", { - monitor: "orb-export", - monitorSlug: "loopover-selfhost-prod-orb-export", - jobType: "orb-export", - repo: "JSONbored/gittensory", - exported: 7, - dryRun: false, - longText: `${"x".repeat(157)}...`, - }); - expect(JSON.stringify(mocks.scope.setContext.mock.calls)).not.toContain("secret"); - expect(JSON.stringify(mocks.scope.setContext.mock.calls)).not.toContain("key"); - expect(mocks.captureException).toHaveBeenCalledWith(expect.any(Error)); - }); - - it("records monitor failures without context and normalizes non-Error throws", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - - await expect( - withSentryMonitor("orb-relay-drain", undefined, async () => { - throw "relay failed"; - }), - ).rejects.toBe("relay failed"); - - expect(mocks.scope.setContext).toHaveBeenCalledWith("sentry_monitor", { - monitor: "orb-relay-drain", - monitorSlug: "loopover-selfhost-production-orb-relay-drain", - }); - expect((mocks.captureException.mock.calls.at(-1)?.[0] as Error).message).toBe( - "relay failed", - ); - }); -}); - -describe("forwardStructuredLogToSentry — central console.log → Sentry error forwarding (#1468)", () => { - it("is a no-op when Sentry is off", () => { - forwardStructuredLogToSentry( - JSON.stringify({ level: "error", event: "x" }), - ); - expect(mocks.captureException).not.toHaveBeenCalled(); - }); - - it("ignores non-strings, non-JSON-object strings, and unparseable JSON when enabled", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry(42); // not a string - forwardStructuredLogToSentry({ level: "error" }); // not a string - forwardStructuredLogToSentry("plain log line"); // doesn't start with "{" - forwardStructuredLogToSentry(""); // empty string (charCodeAt(0) is NaN) - forwardStructuredLogToSentry("{not valid json"); // throws → caught - expect(mocks.captureException).not.toHaveBeenCalled(); - }); - - it("skips routine (non-error) structured logs", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry( - JSON.stringify({ level: "audit", event: "job_complete" }), - ); - forwardStructuredLogToSentry( - JSON.stringify({ event: "regate_sweep_throttled" }), - ); // no level - expect(mocks.captureException).not.toHaveBeenCalled(); - }); - - it("titles a no-message error log with event + a SHORT (repo#pr) location, not a field dump", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry( - JSON.stringify({ - level: "error", - event: "gate_check_permission_missing", - repository: "JSONbored/awesome-claude", - pullNumber: 4240, - deliveryId: "regate-sweep:JSONbored/awesome-claude#4240", - }), - ); - expect(mocks.scope.setLevel).toHaveBeenCalledWith("error"); - expect(mocks.scope.setTag).toHaveBeenCalledWith( - "event", - "gate_check_permission_missing", - ); - // No message/error → captureException with value = the PR location (a real value, NOT "(No error message)"); - // the long deliveryId stays in the tags/context only. - expect(lastCapturedError().name).toBe("gate_check_permission_missing"); - expect(lastCapturedError().message).toBe("(JSONbored/awesome-claude#4240)"); - }); - - it("leads the title with the real error detail + indexes filterable hashed tenant tags + fingerprints by event (#observability)", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry( - JSON.stringify({ - level: "error", - event: "orb_broker_unavailable", - error: "The operation was aborted due to timeout", - repo: "JSONbored/gittensory", - installationId: 143010787, - }), - ); - // The issue carries the actual failure as the exception VALUE (no hunting through the context blob). - expect(lastCapturedError().name).toBe("orb_broker_unavailable"); - expect(lastCapturedError().message).toBe("The operation was aborted due to timeout"); - // The present log dimensions become filterable tags. - expect(mocks.scope.setTag).toHaveBeenCalledWith("repo", "JSONbored/gittensory"); - expect(mocks.scope.setTag).toHaveBeenCalledWith("installation_id_hash", "68b9c2136087c5ca"); - expect(mocks.scope.setTag).not.toHaveBeenCalledWith("installationId", expect.anything()); - expect(mocks.scope.setContext).toHaveBeenCalledWith("log", expect.objectContaining({ - installation_id_hash: "68b9c2136087c5ca", - })); - expect(mocks.scope.setContext).not.toHaveBeenCalledWith("log", expect.objectContaining({ - installationId: 143010787, - })); - // Recurrences of one failure group into a single issue by event. - expect(mocks.scope.setFingerprint).toHaveBeenCalledWith(["loopover-log", "orb_broker_unavailable"]); - }); - - it("REGRESSION (GITTENSORY-D): further splits the fingerprint + title by the `ev` sub-field, so several call sites sharing one broad `event` slug don't collapse into one misleading issue", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry( - JSON.stringify({ - level: "error", - event: "review_context_fetch_failed", - ev: "rag_upsert_error", - message: "invalid byte sequence for encoding \"UTF8\": 0x00", - }), - ); - expect(lastCapturedError().name).toBe("review_context_fetch_failed/rag_upsert_error"); - expect(mocks.scope.setFingerprint).toHaveBeenCalledWith([ - "loopover-log", - "review_context_fetch_failed", - "rag_upsert_error", - ]); - }); - - it("falls back to the plain event-only fingerprint + title when `ev` is absent (unchanged behavior)", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry( - JSON.stringify({ level: "error", event: "review_context_fetch_failed", message: "no sub-event here" }), - ); - expect(lastCapturedError().name).toBe("review_context_fetch_failed"); - expect(mocks.scope.setFingerprint).toHaveBeenCalledWith(["loopover-log", "review_context_fetch_failed"]); - }); - - it("ignores a non-string `ev` field (treats it the same as absent)", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry( - JSON.stringify({ level: "error", event: "review_context_fetch_failed", ev: 42, message: "m" }), - ); - expect(lastCapturedError().name).toBe("review_context_fetch_failed"); - expect(mocks.scope.setFingerprint).toHaveBeenCalledWith(["loopover-log", "review_context_fetch_failed"]); - }); - - it("strips the synthetic wrapper stack so the issue culprit is not forwardStructuredLogToSentry", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry( - JSON.stringify({ level: "error", event: "orb_broker_unavailable", message: "timeout" }), - ); - - const stack = lastCapturedError().stack ?? ""; - expect(stack).not.toMatch(/\n\s+at /); - expect(stack).toBe("orb_broker_unavailable: timeout"); - }); - - it("sets the issue culprit transaction to the event slug, and skips it when there is no event", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry( - JSON.stringify({ level: "error", event: "orb_broker_unavailable", message: "timeout" }), - ); - - const processor = mocks.scope.addEventProcessor.mock.calls.at(-1)?.[0] as ( - event: Record, - ) => Record; - expect(processor({})).toEqual({ transaction: "orb_broker_unavailable" }); - - vi.clearAllMocks(); - forwardStructuredLogToSentry(JSON.stringify({ level: "error", message: "timeout" })); - expect(mocks.scope.addEventProcessor).not.toHaveBeenCalled(); - }); - - it("indexes self-host AI provider dimensions as Sentry tags", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry( - JSON.stringify({ - level: "error", - event: "selfhost_ai_provider_failed", - provider: "codex", - model: "gpt-5.5", - effort: "high", - timeoutMs: 240000, - error: "subscription_cli_timeout", - }), - ); - expect(lastCapturedError().name).toBe("selfhost_ai_provider_failed"); - expect(lastCapturedError().message).toBe("subscription_cli_timeout"); - expect(mocks.scope.setTag).toHaveBeenCalledWith("provider", "codex"); - expect(mocks.scope.setTag).toHaveBeenCalledWith("model", "gpt-5.5"); - expect(mocks.scope.setTag).toHaveBeenCalledWith("effort", "high"); - expect(mocks.scope.setTag).toHaveBeenCalledWith("timeoutMs", "240000"); - }); - - it("normalizes repository log alias to repo tag (#1881 nit)", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry( - JSON.stringify({ - level: "error", - event: "check_run_post_denied", - repository: "owner/repo", - pullNumber: 3, - message: "permission denied", - }), - ); - expect(mocks.scope.setTag).toHaveBeenCalledWith("repo", "owner/repo"); - expect(mocks.scope.setTag).toHaveBeenCalledWith("repository", "owner/repo"); - }); - - it("indexes trace ids already present on structured error logs", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry( - JSON.stringify({ - level: "error", - event: "selfhost_job_dead", - trace_id: "cccccccccccccccccccccccccccccccc", - span_id: "dddddddddddddddd", - }), - ); - expect(mocks.scope.setTag).toHaveBeenCalledWith("trace_id", "cccccccccccccccccccccccccccccccc"); - expect(mocks.scope.setTag).toHaveBeenCalledWith("span_id", "dddddddddddddddd"); - }); - - it("forwards a level:fatal log titled by message (no event ⇒ no tag)", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry( - JSON.stringify({ level: "fatal", message: "boom" }), - ); - expect(mocks.scope.setLevel).toHaveBeenCalledWith("fatal"); - expect(mocks.scope.setTag).not.toHaveBeenCalled(); - expect(lastCapturedError().name).toBe("LoopOverLog"); - expect(lastCapturedError().message).toBe("boom"); - }); - - it("summarizes salient fields when neither event nor message is present", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry(JSON.stringify({ level: "error", code: 500 })); - expect(lastCapturedError().name).toBe("LoopOverLog"); - expect(lastCapturedError().message).toBe("code=500"); - }); - - it("uses a bare event title when a no-message error log has no repo to locate it", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry( - JSON.stringify({ level: "error", event: "relay_drained_error" }), - ); - expect(lastCapturedError().name).toBe("relay_drained_error"); - expect(lastCapturedError().message).toBe("(no message — see the log context)"); - }); - - it("summarizes salient fields (count/projects) alongside the repo location", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry( - JSON.stringify({ - level: "error", - event: "closehold_backlog", - repo: "JSONbored/gittensory", - count: 2, - projects: ["a", "b"], - }), - ); - // The repo locates it AND its salient fields are summarized, so the issue shows real data, not "(no message)". - expect(lastCapturedError().name).toBe("closehold_backlog"); - expect(lastCapturedError().message).toBe( - '(JSONbored/gittensory) count=2, projects=["a","b"]', - ); - }); - - it("summarizes a field-only error log (close_breaker_engaged), skipping nulls + long blobs", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry( - JSON.stringify({ - level: "error", - event: "close_breaker_engaged", - project: "JSONbored/gittensory", - closePrecision: 0.6, - floor: 0.8, - extra: null, - note: "x".repeat(100), - }), - ); - // project/closePrecision/floor are summarized; the null `extra` and the 100-char `note` are skipped. - expect(lastCapturedError().name).toBe("close_breaker_engaged"); - expect(lastCapturedError().message).toBe( - "project=JSONbored/gittensory, closePrecision=0.6, floor=0.8", - ); - }); - - it("does not promote secret-keyed scalar fields into no-message titles (regression)", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry( - JSON.stringify({ - level: "error", - event: "attacker_controlled_error", - token: "gts_SUPER_SECRET_TOKEN_12345", - apiKey: "shh", - repository: "owner/repo", - pullNumber: 7, - project: "safe-project", - }), - ); - - expect(lastCapturedError().name).toBe("attacker_controlled_error"); - expect(lastCapturedError().message).toBe( - "(owner/repo#7) project=safe-project", - ); - expect(lastCapturedError().message).not.toContain( - "gts_SUPER_SECRET_TOKEN_12345", - ); - expect(lastCapturedError().message).not.toContain("shh"); - }); - - it("redacts nested secret-keyed values before summarizing object fields", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry( - JSON.stringify({ - level: "error", - event: "provider_metadata_failed", - provider: { name: "github", token: "nested-secret" }, - attempts: [ - { name: "first", privateKey: "nested-key" }, - "retry", - ], - }), - ); - - expect(lastCapturedError().name).toBe("provider_metadata_failed"); - expect(lastCapturedError().message).toBe( - 'provider={"name":"github","token":"[redacted]"}, attempts=[{"name":"first","privateKey":"[redacted]"},"retry"]', - ); - expect(lastCapturedError().message).not.toContain("nested-secret"); - expect(lastCapturedError().message).not.toContain("nested-key"); - }); - - it("redacts deeply nested summary objects instead of serializing past the depth cap", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry( - JSON.stringify({ - level: "error", - event: "deep_provider_metadata_failed", - meta: { a: { b: { c: { d: { e: { f: { token: "deep-secret" } } } } } } }, - }), - ); - - expect(lastCapturedError().name).toBe("deep_provider_metadata_failed"); - expect(lastCapturedError().message).toBe( - 'meta={"a":{"b":{"c":{"d":{"e":{"f":"[redacted]"}}}}}}', - ); - expect(lastCapturedError().message).not.toContain("deep-secret"); - }); -}); - -describe("severity-threshold gating (#5119) — captureError/captureReviewFailure/forwardStructuredLogToSentry", () => { - const clearThresholdEnv = () => { - delete process.env.SENTRY_MIN_SEVERITY; - delete process.env.SENTRY_REPO_MIN_SEVERITY; - }; - afterEach(clearThresholdEnv); - - it("captureError: default threshold (error) changes nothing — an error-grade capture still fires", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - captureError(new Error("boom"), { repo: "acme/widgets" }); - expect(mocks.captureException).toHaveBeenCalledTimes(1); - }); - - it("captureError: a repo threshold above error (critical) suppresses the capture", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - process.env.SENTRY_REPO_MIN_SEVERITY = JSON.stringify({ "acme/widgets": "critical" }); - captureError(new Error("boom"), { repo: "acme/widgets" }); - expect(mocks.captureException).not.toHaveBeenCalled(); - }); - - it("captureError: a repo threshold above error does not suppress a DIFFERENT repo's capture", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - process.env.SENTRY_REPO_MIN_SEVERITY = JSON.stringify({ "acme/widgets": "critical" }); - captureError(new Error("boom"), { repo: "other/repo" }); - expect(mocks.captureException).toHaveBeenCalledTimes(1); - }); - - it("captureError: the global threshold applies when context carries no repo", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - process.env.SENTRY_MIN_SEVERITY = "critical"; - captureError(new Error("boom")); - expect(mocks.captureException).not.toHaveBeenCalled(); - }); - - it("captureError: reads `repository` when `repo` is absent from context", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - process.env.SENTRY_REPO_MIN_SEVERITY = JSON.stringify({ "acme/widgets": "critical" }); - captureError(new Error("boom"), { repository: "acme/widgets" }); - expect(mocks.captureException).not.toHaveBeenCalled(); - }); - - it("captureReviewFailure: default threshold (error) changes nothing — still fires", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - captureReviewFailure(new Error("rev"), { repo: "acme/widgets" }); - expect(mocks.captureException).toHaveBeenCalledTimes(1); - }); - - it("captureReviewFailure: a repo threshold above error (critical) suppresses the capture", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - process.env.SENTRY_REPO_MIN_SEVERITY = JSON.stringify({ "acme/widgets": "critical" }); - captureReviewFailure(new Error("rev"), { repo: "acme/widgets" }); - expect(mocks.captureException).not.toHaveBeenCalled(); - }); - - it("forwardStructuredLogToSentry: default threshold (error) still forwards error/fatal and still skips warning/info — byte-identical to pre-#5119 behavior", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - forwardStructuredLogToSentry(JSON.stringify({ level: "error", event: "x", repo: "acme/widgets" })); - forwardStructuredLogToSentry(JSON.stringify({ level: "fatal", event: "y", repo: "acme/widgets" })); - forwardStructuredLogToSentry(JSON.stringify({ level: "warning", event: "z", repo: "acme/widgets" })); - forwardStructuredLogToSentry(JSON.stringify({ level: "info", event: "w", repo: "acme/widgets" })); - expect(mocks.captureException).toHaveBeenCalledTimes(2); - }); - - it("forwardStructuredLogToSentry: lowering a repo's threshold to info surfaces its warning/info-grade logs (#5119's core deliverable)", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - process.env.SENTRY_REPO_MIN_SEVERITY = JSON.stringify({ "acme/widgets": "info" }); - forwardStructuredLogToSentry(JSON.stringify({ level: "warning", event: "z", repo: "acme/widgets" })); - expect(mocks.captureException).toHaveBeenCalledTimes(1); - expect(mocks.scope.setLevel).toHaveBeenLastCalledWith("warning"); - forwardStructuredLogToSentry(JSON.stringify({ level: "info", event: "w", repo: "acme/widgets" })); - expect(mocks.captureException).toHaveBeenCalledTimes(2); - expect(mocks.scope.setLevel).toHaveBeenLastCalledWith("info"); - }); - - it("forwardStructuredLogToSentry: lowering ONE repo's threshold does not affect a different repo's default gating", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - process.env.SENTRY_REPO_MIN_SEVERITY = JSON.stringify({ "acme/widgets": "info" }); - forwardStructuredLogToSentry(JSON.stringify({ level: "warning", event: "z", repo: "other/repo" })); - expect(mocks.captureException).not.toHaveBeenCalled(); - }); - - it("forwardStructuredLogToSentry: an unrecognized level (a log CATEGORY like \"audit\", not a severity) never promotes to error grade even with a lowered threshold", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - process.env.SENTRY_MIN_SEVERITY = "info"; - forwardStructuredLogToSentry(JSON.stringify({ level: "audit", event: "job_complete", repo: "acme/widgets" })); - expect(mocks.captureException).toHaveBeenCalledTimes(1); - expect(mocks.scope.setLevel).toHaveBeenLastCalledWith("info"); - }); - - it("forwardStructuredLogToSentry: raising the global threshold to critical suppresses ordinary error-grade logs but still lets fatal through", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - process.env.SENTRY_MIN_SEVERITY = "critical"; - forwardStructuredLogToSentry(JSON.stringify({ level: "error", event: "x" })); - expect(mocks.captureException).not.toHaveBeenCalled(); - forwardStructuredLogToSentry(JSON.stringify({ level: "fatal", event: "y" })); - expect(mocks.captureException).toHaveBeenCalledTimes(1); - }); -}); - -describe("installStructuredLogForwarding — central console sink instrumentation (#1468)", () => { - const makeConsole = () => { - const base = { log: vi.fn(), error: vi.fn() }; - return { target: { ...base }, base }; - }; - - it("forwards structured level:error logs emitted through console.error (regression)", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - const { target, base } = makeConsole(); - - installStructuredLogForwarding(target); - target.error( - JSON.stringify({ - level: "error", - event: "orb_broker_unavailable", - installationId: 1, - }), - ); - - expect(lastCapturedError().name).toBe("orb_broker_unavailable"); - expect(lastCapturedError().message).toBe("(no message — see the log context)"); - expect(mocks.scope.setTag).toHaveBeenCalledWith("installation_id_hash", "21ab41515eeee762"); - expect(base.error).toHaveBeenCalledTimes(1); - }); - - it("keeps forwarding structured level:error logs emitted through console.log", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - const { target, base } = makeConsole(); - - installStructuredLogForwarding(target); - target.log(JSON.stringify({ level: "error", event: "gate_check_failed" })); - - expect(lastCapturedError().name).toBe("gate_check_failed"); - expect(lastCapturedError().message).toBe("(no message — see the log context)"); - expect(base.log).toHaveBeenCalledTimes(1); - }); - - it("forwards a NO-LEVEL structured log emitted through console.error (the error sink defaults to error)", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - const { target } = makeConsole(); - installStructuredLogForwarding(target); - // No `level` field — previously dropped on the floor; now console.error forwards it as error (short location). - target.error( - JSON.stringify({ event: "selfhost_ai_provider_failed", repo: "o/r" }), - ); - expect(lastCapturedError().name).toBe("selfhost_ai_provider_failed"); - expect(lastCapturedError().message).toBe("(o/r)"); - }); - - it("does NOT forward a no-level log through console.log (stdout is not error by default)", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - const { target } = makeConsole(); - installStructuredLogForwarding(target); - target.log(JSON.stringify({ event: "job_complete" })); - expect(mocks.captureException).not.toHaveBeenCalled(); - }); - - it("keeps skipping an EXPLICIT level:warn through console.error (explicit level wins over the sink default)", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - const { target } = makeConsole(); - installStructuredLogForwarding(target); - target.error( - JSON.stringify({ level: "warn", event: "orb_broker_degraded" }), - ); - expect(mocks.captureException).not.toHaveBeenCalled(); - }); - - it("does not recursively forward if the Sentry path logs while forwarding", async () => { - await initSentry({ SENTRY_DSN: "d" } as unknown as NodeJS.ProcessEnv); - const { target, base } = makeConsole(); - installStructuredLogForwarding(target); - mocks.captureException.mockImplementationOnce(() => { - target.error(JSON.stringify({ level: "error", event: "recursive" })); - }); - - target.error(JSON.stringify({ level: "error", event: "outer" })); - - expect(mocks.captureException).toHaveBeenCalledTimes(1); - expect(lastCapturedError().name).toBe("outer"); - expect(base.error).toHaveBeenCalledTimes(2); - }); -}); - -describe("Sentry operational taxonomy exports (#1824)", () => { - it("exports monitor names aligned with wrapped cron loops", () => { - expect(SENTRY_MONITOR_NAMES).toEqual([ - "scheduled-loop", - "orb-export", - "orb-relay-drain", - "orb-relay-register", - "queue-dead-letter-revive", - ]); - }); - - it("exports subsystem taxonomy keys for operator docs", () => { - expect(Object.keys(SENTRY_OPERATIONAL_SUBSYSTEMS).sort()).toEqual([ - "ai", - "backup", - "gate", - "github", - "publish", - "queue", - "relay", - "scheduled", - "webhook", - ]); - }); - - it("keeps operational tag keys low-cardinality and review-aware", () => { - expect(SENTRY_OPERATIONAL_TAG_KEYS).toEqual( - expect.arrayContaining([ - "kind", - "subsystem", - "jobType", - "repo", - "pr", - "head_sha", - "operation", - "event", - "monitor", - ]), - ); - expect(SENTRY_OPERATIONAL_TAG_KEYS).not.toContain("installationId"); - }); -}); diff --git a/test/unit/selfhost-sqlite-queue.test.ts b/test/unit/selfhost-sqlite-queue.test.ts index 64df58c635..4fd3ccc4dd 100644 --- a/test/unit/selfhost-sqlite-queue.test.ts +++ b/test/unit/selfhost-sqlite-queue.test.ts @@ -6,7 +6,7 @@ import { jobCoalesceKey, queueSnapshotFromBinding } from "../../src/selfhost/que import { renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; import { RetryableJobError } from "../../src/queue/retryable"; import { hostLoadAvg1PerCore } from "../../src/selfhost/host-pressure"; -import * as sentryModule from "../../src/selfhost/sentry"; +import * as posthogModule from "../../src/selfhost/posthog"; import type { JobMessage } from "../../src/types"; // Real host CPU load is nondeterministic (and can legitimately spike on a busy CI runner), so every @@ -2156,12 +2156,12 @@ describe("createSqliteQueue (durable #980)", () => { await q.stop(); }); - // (#1824): dead-letter revival stopping SILENTLY is worse than one throwing tick -- a Sentry cron monitor - // now wraps every tick so a stopped timer shows up as a missed check-in, not silence. - it("wraps each revive tick in the queue-dead-letter-revive Sentry monitor", async () => { + // (#1824): dead-letter revival stopping SILENTLY is worse than one throwing tick -- a PostHog monitor + // now wraps every tick so a stopped timer shows up as a missed heartbeat, not silence. + it("wraps each revive tick in the queue-dead-letter-revive PostHog monitor", async () => { process.env.QUEUE_DEAD_LETTER_REVIVE_INTERVAL_MS = "1000"; vi.useFakeTimers(); - const monitorSpy = vi.spyOn(sentryModule, "withSentryMonitor"); + const monitorSpy = vi.spyOn(posthogModule, "withPostHogMonitor"); try { const driver = makeDriver(); const q = createSqliteQueue(driver, async () => undefined, { maxRetries: 1 }); @@ -2180,13 +2180,13 @@ describe("createSqliteQueue (durable #980)", () => { } }); - // The monitor rethrows on failure (withSentryMonitor's own contract) -- confirms that rethrow is still caught + // The monitor rethrows on failure (withPostHogMonitor's own contract) -- confirms that rethrow is still caught // by reviveDeadLetterJobsSafely's own try/catch, so a crashing tick behaves exactly as it did before the // monitor was added: logged + captured, never an uncaught exception. - it("still catches a revive crash after adding the Sentry monitor wrapper (no regression on #2581)", async () => { + it("still catches a revive crash after adding the PostHog monitor wrapper (no regression on #2581)", async () => { process.env.QUEUE_DEAD_LETTER_REVIVE_INTERVAL_MS = "1000"; vi.useFakeTimers(); - const monitorSpy = vi.spyOn(sentryModule, "withSentryMonitor"); + const monitorSpy = vi.spyOn(posthogModule, "withPostHogMonitor"); try { const driver = makeDriver(); const realQuery = driver.query.bind(driver);