diff --git a/platform/env.example b/platform/env.example index af5877e..bed9da6 100644 --- a/platform/env.example +++ b/platform/env.example @@ -48,6 +48,15 @@ NEXT_PUBLIC_DEFAULT_LOGO_URL=/logo.svg # a one-shot script before changing it in production. INTEGRATIONS_ENCRYPTION_KEY= +# Shared secret for the daily Datadog sync cron job. Vercel Cron sends this +# automatically as `Authorization: Bearer $CRON_SECRET` to the cron route; +# in local dev you can pass it via `x-cron-secret: ` to hit +# /api/cron/sync-integrations manually. Generate with: +# openssl rand -base64 32 +# If empty in development, the cron route is unauthenticated (production +# deployments still reject unauthenticated requests). +CRON_SECRET= + # --- Resend (required for transactional emails) --- # Get an API key at https://resend.com/api-keys RESEND_API_KEY=re_... diff --git a/platform/lib/integrations/datadog/client.ts b/platform/lib/integrations/datadog/client.ts index bae6f44..8851196 100644 --- a/platform/lib/integrations/datadog/client.ts +++ b/platform/lib/integrations/datadog/client.ts @@ -1,11 +1,10 @@ /** - * Datadog API client for the integration's connect flow. + * Datadog API client for the DORA Metrics v2 read endpoints. * - * Only the bits slice 2 needs land here: credential validation against - * the DORA Metrics v2 read endpoints. Slice 3 will extend this module - * with the actual sync calls (deployments / failures pagination). - * - * See docs/PLAN-datadog.md §9 for the request shape and probe findings. + * Covers credential validation (slice 2) and the list-with-filters + * fetchers used by the daily sync (slice 3). Pagination semantics are + * documented in docs/PLAN-datadog.md §9.5 — there is no cursor mechanism + * on these endpoints, the only way forward is to shrink the `to` window. */ export type DatadogSite = @@ -134,3 +133,174 @@ function stripMillis(d: Date): string { // sites. The Z form is the safest documented shape. return d.toISOString().replace(/\.\d{3}Z$/, "Z"); } + +export function toDatadogTimestamp(d: Date): string { + return stripMillis(d); +} + +export interface DoraListFilters { + /** ISO 8601 (seconds precision, trailing Z). */ + from: string; + /** ISO 8601 (seconds precision, trailing Z). */ + to: string; + /** Datadog query language. Use "*" for unfiltered. */ + query: string; + /** + * Page size. Probed ceiling is at least 500; we cap at 100 for v1 to + * keep response latency predictable. The §9.5 pagination loop relies + * on `len(events) < limit` as the terminal condition. + */ + limit: number; +} + +export interface DoraDeploymentCommit { + sha: string; + timestamp?: string; + author?: { + email?: string; + canonical_email?: string; + is_bot?: boolean; + }; + message?: string; + html_url?: string; + change_lead_time?: number; + time_to_deploy?: number; +} + +export interface DoraDeployment { + type: "dora_deployment"; + id: string; + attributes: { + git?: { commit_sha?: string; repository_id?: string }; + commits?: DoraDeploymentCommit[]; + pull_requests?: Array<{ + created_at?: string; + merged_at?: string; + is_fully_automated?: boolean; + }>; + service?: string; + env?: string; + team?: string; + version?: string; + /** TRI-STATE: true | false | null (null = pending evaluation). */ + change_failure?: boolean | null; + deployment_type?: string; + source?: string; + started_at: string; + finished_at?: string; + duration?: number; + created_at?: string; + number_of_commits?: number; + number_of_pull_requests?: number; + /** Present only when change_failure === true. */ + recovery_time_sec?: number; + remediation?: { id?: string; type?: string }; + }; +} + +export interface DoraFailure { + type: "dora_failure"; + id: string; + attributes: { + service?: string[]; + env?: string[]; + team?: string[]; + name?: string; + /** "Normal" | "High" | "Urgent" observed; free-form. */ + severity?: string; + started_at: string; + finished_at?: string; + created_at?: string; + /** Seconds. */ + time_to_restore?: number; + source?: string; + }; +} + +export class DatadogApiError extends Error { + constructor( + message: string, + public readonly status: number, + public readonly detail?: string, + ) { + super(message); + this.name = "DatadogApiError"; + } +} + +async function postDoraList( + creds: DatadogCredentials, + endpoint: "deployments" | "failures", + requestType: "dora_deployments_list_request" | "dora_failures_list_request", + filters: DoraListFilters, +): Promise { + const url = `https://api.${creds.site}/api/v2/dora/${endpoint}`; + const body = { + data: { + type: requestType, + attributes: { + from: filters.from, + to: filters.to, + query: filters.query, + limit: filters.limit, + }, + }, + }; + + const response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + "DD-API-KEY": creds.apiKey, + "DD-APPLICATION-KEY": creds.appKey, + }, + body: JSON.stringify(body), + // Each page fetch must complete inside the per-org sync budget. + signal: AbortSignal.timeout(30_000), + }); + + if (!response.ok) { + let detail: string | undefined; + try { + const payload = (await response.json()) as { + errors?: Array<{ detail?: string; title?: string }>; + }; + const first = payload.errors?.[0]; + detail = first?.detail ?? first?.title; + } catch { + detail = undefined; + } + throw new DatadogApiError( + `Datadog ${endpoint} list failed with HTTP ${response.status}`, + response.status, + detail, + ); + } + + const payload = (await response.json()) as { data?: T[] }; + return payload.data ?? []; +} + +export function listDeployments( + creds: DatadogCredentials, + filters: DoraListFilters, +): Promise { + return postDoraList( + creds, + "deployments", + "dora_deployments_list_request", + filters, + ); +} + +export function listFailures( + creds: DatadogCredentials, + filters: DoraListFilters, +): Promise { + return postDoraList( + creds, + "failures", + "dora_failures_list_request", + filters, + ); +} diff --git a/platform/lib/integrations/datadog/sync.ts b/platform/lib/integrations/datadog/sync.ts new file mode 100644 index 0000000..90c6839 --- /dev/null +++ b/platform/lib/integrations/datadog/sync.ts @@ -0,0 +1,462 @@ +/** + * Datadog daily sync. + * + * Pulls DORA deployment and failure events into `external_deployments` + * (+ `external_deployment_commits`) and `external_incidents`. Idempotent + * by `(provider, provider_event_id)` so repeat runs are safe. + * + * Pagination uses the time-slicing strategy documented in §9.5 of + * docs/PLAN-datadog.md — there is no cursor mechanism on the DORA v2 + * list endpoints. + */ + +import type { SupabaseClient } from "@supabase/supabase-js"; + +import { + listDeployments, + listFailures, + normalizeSite, + toDatadogTimestamp, + type DatadogCredentials, + type DatadogSite, + type DoraDeployment, + type DoraFailure, +} from "./client"; + +import { logger } from "@/lib/debug"; +import { decryptCredentials } from "@/lib/encryption"; + +const DEFAULT_BACKFILL_DAYS = 30; +const PAGE_LIMIT = 100; +const MAX_PAGES_PER_ENDPOINT = 200; +const PROVIDER = "datadog" as const; + +export interface SyncOptions { + /** Override the backfill window for a first sync. Defaults to 30 days. */ + backfillDays?: number; + /** Inject a clock for testing. Defaults to `Date.now()`. */ + now?: () => Date; +} + +export interface SyncResult { + organizationId: string; + deploymentsUpserted: number; + commitsUpserted: number; + failuresUpserted: number; + /** Deployments persisted where DD's repository_id didn't resolve to a tracked Iris repo. */ + unmatchedDeployments: number; + /** ISO 8601 window the sync covered. */ + from: string; + to: string; +} + +interface RepoLookup { + /** Normalized slug → repositories.id */ + byNormalizedSlug: Map; +} + +/** + * Sync a single org's Datadog integration. Updates `last_sync_at` on + * success and `last_error` on failure; never throws — callers (cron + * route) inspect the returned result or query the table afterward. + */ +export async function syncOrganization( + supabase: SupabaseClient, + organizationId: string, + opts: SyncOptions = {}, +): Promise { + const now = (opts.now ?? (() => new Date()))(); + const backfillDays = opts.backfillDays ?? DEFAULT_BACKFILL_DAYS; + + try { + const { data: integration, error: loadErr } = await supabase + .from("org_integrations") + .select("id, credentials_encrypted, last_sync_at, status, config") + .eq("organization_id", organizationId) + .eq("provider", PROVIDER) + .maybeSingle(); + + if (loadErr) throw new Error(`load integration: ${loadErr.message}`); + if (!integration) throw new Error("integration row not found"); + if (integration.status !== "active") { + throw new Error(`integration status is ${integration.status}`); + } + if (!integration.credentials_encrypted) { + throw new Error("integration has no credentials (disconnected?)"); + } + + const credsPayload = await decryptCredentials<{ + apiKey: string; + appKey: string; + site: string; + }>(integration.credentials_encrypted); + + const creds: DatadogCredentials = { + apiKey: credsPayload.apiKey, + appKey: credsPayload.appKey, + site: normalizeSite(credsPayload.site) as DatadogSite, + }; + + const repoLookup = await loadRepoLookup(supabase, organizationId); + + const fromDate = integration.last_sync_at + ? new Date(integration.last_sync_at) + : new Date(now.getTime() - backfillDays * 24 * 60 * 60 * 1000); + const from = toDatadogTimestamp(fromDate); + const to = toDatadogTimestamp(now); + + const deployments = await fetchAllDeployments(creds, from, to); + const failures = await fetchAllFailures(creds, from, to); + + const { deploymentsUpserted, commitsUpserted, unmatchedDeployments } = + await persistDeployments( + supabase, + organizationId, + deployments, + repoLookup, + ); + const failuresUpserted = await persistFailures( + supabase, + organizationId, + failures, + ); + + await supabase + .from("org_integrations") + .update({ + last_sync_at: now.toISOString(), + last_error: null, + status: "active", + }) + .eq("organization_id", organizationId) + .eq("provider", PROVIDER); + + return { + organizationId, + deploymentsUpserted, + commitsUpserted, + failuresUpserted, + unmatchedDeployments, + from, + to, + }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + logger.error("datadog sync failed", { organizationId, error: message }); + await supabase + .from("org_integrations") + .update({ + last_error: truncate(message, 1000), + status: "error", + }) + .eq("organization_id", organizationId) + .eq("provider", PROVIDER); + return { organizationId, error: message }; + } +} + +async function loadRepoLookup( + supabase: SupabaseClient, + organizationId: string, +): Promise { + const { data, error } = await supabase + .from("repositories") + .select("id, remote_url, name") + .eq("organization_id", organizationId); + + if (error) throw new Error(`load repos: ${error.message}`); + + const byNormalizedSlug = new Map(); + for (const row of data ?? []) { + const slug = normalizeRepoSlug(row.remote_url); + if (slug) byNormalizedSlug.set(slug, row.id); + } + return { byNormalizedSlug }; +} + +/** + * Normalize a git remote URL or Datadog slug into "host/path" form + * (lowercased, no scheme, no `.git`, no trailing slash). Returns `null` + * for empty input. + */ +export function normalizeRepoSlug( + input: string | null | undefined, +): string | null { + if (!input) return null; + let s = input.trim().toLowerCase(); + if (!s) return null; + // git@github.com:org/repo.git → github.com/org/repo + s = s.replace(/^git@([^:]+):/, "$1/"); + // ssh://git@host/org/repo or https://host/org/repo → host/org/repo + s = s.replace(/^[a-z]+:\/\//, ""); + s = s.replace(/^git@/, ""); + s = s.replace(/^www\./, ""); + s = s.replace(/\.git$/, ""); + s = s.replace(/\/+$/, ""); + return s || null; +} + +async function fetchAllDeployments( + creds: DatadogCredentials, + from: string, + to: string, +): Promise { + const events: DoraDeployment[] = []; + let toTs = to; + for (let page = 0; page < MAX_PAGES_PER_ENDPOINT; page++) { + const batch = await listDeployments(creds, { + from, + to: toTs, + query: "*", + limit: PAGE_LIMIT, + }); + if (batch.length === 0) break; + events.push(...batch); + if (batch.length < PAGE_LIMIT) break; + + const nextTs = oldestStartedAt(batch, (e) => e.attributes.started_at); + if (!nextTs || nextTs === toTs) { + // §9.5 anti-spin guard: page is full and the boundary didn't move + // (sub-second co-occurrence on every event). Step back 1 ms. + toTs = toDatadogTimestamp(new Date(parseIso(toTs).getTime() - 1)); + } else { + toTs = nextTs; + } + } + return events; +} + +async function fetchAllFailures( + creds: DatadogCredentials, + from: string, + to: string, +): Promise { + const events: DoraFailure[] = []; + let toTs = to; + for (let page = 0; page < MAX_PAGES_PER_ENDPOINT; page++) { + const batch = await listFailures(creds, { + from, + to: toTs, + query: "*", + limit: PAGE_LIMIT, + }); + if (batch.length === 0) break; + events.push(...batch); + if (batch.length < PAGE_LIMIT) break; + + const nextTs = oldestStartedAt(batch, (e) => e.attributes.started_at); + if (!nextTs || nextTs === toTs) { + toTs = toDatadogTimestamp(new Date(parseIso(toTs).getTime() - 1)); + } else { + toTs = nextTs; + } + } + return events; +} + +function oldestStartedAt(batch: T[], pick: (e: T) => string): string | null { + let minMs = Infinity; + for (const e of batch) { + const ms = parseIso(pick(e)).getTime(); + if (ms < minMs) minMs = ms; + } + if (!isFinite(minMs)) return null; + return toDatadogTimestamp(new Date(minMs)); +} + +function parseIso(iso: string): Date { + return new Date(iso); +} + +async function persistDeployments( + supabase: SupabaseClient, + organizationId: string, + deployments: DoraDeployment[], + repoLookup: RepoLookup, +): Promise<{ + deploymentsUpserted: number; + commitsUpserted: number; + unmatchedDeployments: number; +}> { + if (deployments.length === 0) { + return { + deploymentsUpserted: 0, + commitsUpserted: 0, + unmatchedDeployments: 0, + }; + } + + let unmatched = 0; + const rows = deployments.map((event) => { + const ddSlug = event.attributes.git?.repository_id ?? null; + const repositoryId = matchRepo(ddSlug, repoLookup); + if (ddSlug && !repositoryId) unmatched++; + + return { + organization_id: organizationId, + provider: PROVIDER, + provider_event_id: event.id, + repository_id: repositoryId, + dd_repository_id: ddSlug, + service: event.attributes.service ?? null, + env: event.attributes.env ?? null, + team: event.attributes.team ?? null, + version: event.attributes.version ?? null, + commit_sha: event.attributes.git?.commit_sha ?? null, + change_failure: + event.attributes.change_failure === undefined + ? null + : event.attributes.change_failure, + deployment_type: event.attributes.deployment_type ?? null, + source: event.attributes.source ?? null, + started_at: event.attributes.started_at, + finished_at: event.attributes.finished_at ?? null, + duration_seconds: secondsFromDuration(event.attributes.duration), + number_of_commits: event.attributes.number_of_commits ?? null, + number_of_pull_requests: event.attributes.number_of_pull_requests ?? null, + recovery_time_sec: event.attributes.recovery_time_sec ?? null, + remediation_type: event.attributes.remediation?.type ?? null, + remediation_id: event.attributes.remediation?.id ?? null, + raw: event, + }; + }); + + const { data: upserted, error } = await supabase + .from("external_deployments") + .upsert(rows, { onConflict: "provider,provider_event_id" }) + .select("id, provider_event_id"); + + if (error) throw new Error(`upsert deployments: ${error.message}`); + if (!upserted) throw new Error("upsert deployments returned no rows"); + + const idByEventId = new Map(); + for (const row of upserted) { + idByEventId.set(row.provider_event_id, row.id); + } + + const commitsUpserted = await persistDeploymentCommits( + supabase, + deployments, + idByEventId, + ); + + return { + deploymentsUpserted: upserted.length, + commitsUpserted, + unmatchedDeployments: unmatched, + }; +} + +async function persistDeploymentCommits( + supabase: SupabaseClient, + deployments: DoraDeployment[], + idByEventId: Map, +): Promise { + const commitRows: Array<{ + deployment_id: string; + commit_sha: string; + commit_timestamp: string | null; + author_email: string | null; + author_canonical_email: string | null; + is_bot: boolean | null; + change_lead_time: number | null; + time_to_deploy: number | null; + }> = []; + + for (const event of deployments) { + const deploymentId = idByEventId.get(event.id); + if (!deploymentId) continue; + const commits = event.attributes.commits ?? []; + for (const c of commits) { + if (!c.sha) continue; + commitRows.push({ + deployment_id: deploymentId, + commit_sha: c.sha, + commit_timestamp: normalizeNullableIso(c.timestamp), + author_email: c.author?.email ?? null, + author_canonical_email: c.author?.canonical_email ?? null, + is_bot: c.author?.is_bot ?? null, + change_lead_time: c.change_lead_time ?? null, + time_to_deploy: c.time_to_deploy ?? null, + }); + } + } + + if (commitRows.length === 0) return 0; + + const { error } = await supabase + .from("external_deployment_commits") + .upsert(commitRows, { + onConflict: "deployment_id,commit_sha", + ignoreDuplicates: false, + }); + + if (error) throw new Error(`upsert deployment commits: ${error.message}`); + return commitRows.length; +} + +async function persistFailures( + supabase: SupabaseClient, + organizationId: string, + failures: DoraFailure[], +): Promise { + if (failures.length === 0) return 0; + + const rows = failures.map((event) => ({ + organization_id: organizationId, + provider: PROVIDER, + provider_event_id: event.id, + service: event.attributes.service ?? null, + env: event.attributes.env ?? null, + team: event.attributes.team ?? null, + name: event.attributes.name ?? null, + severity: event.attributes.severity ?? null, + started_at: event.attributes.started_at, + finished_at: event.attributes.finished_at ?? null, + time_to_restore_seconds: event.attributes.time_to_restore ?? null, + source: event.attributes.source ?? null, + raw: event, + })); + + const { data, error } = await supabase + .from("external_incidents") + .upsert(rows, { onConflict: "provider,provider_event_id" }) + .select("id"); + + if (error) throw new Error(`upsert failures: ${error.message}`); + return data?.length ?? 0; +} + +function matchRepo(slug: string | null, lookup: RepoLookup): string | null { + const normalized = normalizeRepoSlug(slug); + if (!normalized) return null; + return lookup.byNormalizedSlug.get(normalized) ?? null; +} + +function secondsFromDuration(duration: number | undefined): number | null { + if (typeof duration !== "number" || !Number.isFinite(duration)) return null; + // Datadog returns duration in nanoseconds for some endpoints; the DORA + // event payload reports seconds in the probed responses. Treat as + // seconds and clamp to an int. + return Math.round(duration); +} + +function normalizeNullableIso(iso: string | undefined): string | null { + if (!iso) return null; + // Datadog occasionally returns the zero value "0001-01-01T00:00:00Z" + // on fields it doesn't have data for (observed in pull_requests[]); + // treat anything before 2000 as missing. + const d = new Date(iso); + if (Number.isNaN(d.getTime()) || d.getUTCFullYear() < 2000) return null; + return d.toISOString(); +} + +function truncate(s: string, max: number): string { + return s.length <= max ? s : s.slice(0, max - 1) + "…"; +} + +export const __testing = { + normalizeRepoSlug, + oldestStartedAt, + normalizeNullableIso, +}; diff --git a/platform/lib/translations.ts b/platform/lib/translations.ts index c686a45..ce316d3 100644 --- a/platform/lib/translations.ts +++ b/platform/lib/translations.ts @@ -785,6 +785,9 @@ export const translations = { lastSyncAt: "Last sync", connectedAt: "Connected at", neverSynced: "Never synced yet", + unmatchedDeployments: "Unmatched deployments", + unmatchedDeploymentsHint: + "Deployments whose Datadog repository slug didn't resolve to a tracked Iris repo. Verify the Iris repo's remote URL matches the Datadog repository_id.", }, disconnectDialog: { title: "Disconnect Datadog?", @@ -2045,6 +2048,9 @@ export const translations = { lastSyncAt: "Última sincronização", connectedAt: "Conectada em", neverSynced: "Ainda não sincronizada", + unmatchedDeployments: "Deploys não vinculados", + unmatchedDeploymentsHint: + "Deploys cujo slug de repositório do Datadog não correspondeu a um repositório rastreado pelo Iris. Verifique se a URL remota do repositório no Iris bate com o repository_id do Datadog.", }, disconnectDialog: { title: "Desconectar o Datadog?", diff --git a/platform/src/app/[tenant]/settings/integrations/[provider]/page.tsx b/platform/src/app/[tenant]/settings/integrations/[provider]/page.tsx index 5b80ff1..2c7dfba 100644 --- a/platform/src/app/[tenant]/settings/integrations/[provider]/page.tsx +++ b/platform/src/app/[tenant]/settings/integrations/[provider]/page.tsx @@ -71,6 +71,13 @@ export default async function IntegrationProviderPage({ site?: string; apiKeyMask?: string; }; + const { count: unmatchedCount } = await supabaseAdmin + .from("external_deployments") + .select("id", { count: "exact", head: true }) + .eq("organization_id", org.id) + .eq("provider", "datadog") + .is("repository_id", null); + initial = { status: data.status as "active" | "error" | "disconnected", site: config.site ?? null, @@ -79,6 +86,7 @@ export default async function IntegrationProviderPage({ lastError: data.last_error, createdAt: data.created_at, updatedAt: data.updated_at, + unmatchedDeploymentsCount: unmatchedCount ?? 0, }; } } diff --git a/platform/src/app/api/cron/sync-integrations/route.ts b/platform/src/app/api/cron/sync-integrations/route.ts new file mode 100644 index 0000000..e711365 --- /dev/null +++ b/platform/src/app/api/cron/sync-integrations/route.ts @@ -0,0 +1,103 @@ +import { NextRequest, NextResponse } from "next/server"; + +import { logger } from "@/lib/debug"; +import { syncOrganization } from "@/lib/integrations/datadog/sync"; +import { supabaseAdmin } from "@/lib/supabase"; + +// The cron loops sequentially across active integrations; allow it to +// run for the full Fluid Compute default. Per-org sync still has its +// own per-request HTTP timeouts inside the Datadog client. +export const maxDuration = 300; +export const dynamic = "force-dynamic"; + +interface PerOrgOutcome { + organizationId: string; + ok: boolean; + deploymentsUpserted?: number; + commitsUpserted?: number; + failuresUpserted?: number; + unmatchedDeployments?: number; + error?: string; +} + +export async function GET(request: NextRequest) { + if (!isAuthorized(request)) { + return NextResponse.json({ message: "Unauthorized" }, { status: 401 }); + } + + const { data: integrations, error } = await supabaseAdmin + .from("org_integrations") + .select("organization_id, provider") + .eq("status", "active"); + + if (error) { + logger.error("cron list integrations failed", { error: error.message }); + return NextResponse.json( + { message: "Failed to list integrations" }, + { status: 500 }, + ); + } + + const outcomes: PerOrgOutcome[] = []; + for (const integration of integrations ?? []) { + if (integration.provider !== "datadog") continue; + + const result = await syncOrganization( + supabaseAdmin, + integration.organization_id, + ); + + if ("error" in result) { + outcomes.push({ + organizationId: integration.organization_id, + ok: false, + error: result.error, + }); + } else { + outcomes.push({ + organizationId: integration.organization_id, + ok: true, + deploymentsUpserted: result.deploymentsUpserted, + commitsUpserted: result.commitsUpserted, + failuresUpserted: result.failuresUpserted, + unmatchedDeployments: result.unmatchedDeployments, + }); + } + } + + const succeeded = outcomes.filter((o) => o.ok).length; + const failed = outcomes.length - succeeded; + logger.info("cron sync-integrations finished", { + total: outcomes.length, + succeeded, + failed, + }); + + return NextResponse.json({ + total: outcomes.length, + succeeded, + failed, + outcomes, + }); +} + +/** + * Vercel Cron sends `Authorization: Bearer $CRON_SECRET` automatically + * when the env var is configured on the project. We accept both that + * header and an explicit `x-cron-secret` header so the route can be + * triggered manually from local dev with `curl`. + * + * Returns true when CRON_SECRET isn't configured AND we're running in + * a non-production environment — keeps `npm run dev` ergonomic while + * still blocking unauthenticated access in production. + */ +function isAuthorized(request: NextRequest): boolean { + const expected = process.env.CRON_SECRET; + if (!expected) { + return process.env.NODE_ENV !== "production"; + } + const bearer = request.headers.get("authorization"); + if (bearer === `Bearer ${expected}`) return true; + if (request.headers.get("x-cron-secret") === expected) return true; + return false; +} diff --git a/platform/src/components/integrations/datadog-connect-form.tsx b/platform/src/components/integrations/datadog-connect-form.tsx index 5109e5c..30abbb5 100644 --- a/platform/src/components/integrations/datadog-connect-form.tsx +++ b/platform/src/components/integrations/datadog-connect-form.tsx @@ -56,6 +56,7 @@ export type DatadogIntegrationStatus = lastError: string | null; createdAt: string; updatedAt: string; + unmatchedDeploymentsCount: number; }; interface Props { @@ -107,6 +108,7 @@ export function DatadogConnectForm({ organizationId, initial }: Props) { lastError: null, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), + unmatchedDeploymentsCount: 0, }); router.refresh(); } catch (err) { @@ -190,6 +192,25 @@ export function DatadogConnectForm({ organizationId, initial }: Props) {
{new Date(state.createdAt).toLocaleString()}
+ {state.unmatchedDeploymentsCount > 0 && ( +
+
+ {t( + "settings.integrations.datadog.fields.unmatchedDeployments", + )} +
+
+ + {state.unmatchedDeploymentsCount} + + + {t( + "settings.integrations.datadog.fields.unmatchedDeploymentsHint", + )} + +
+
+ )} {state.lastError && ( diff --git a/platform/supabase/migrations/015_external_deployments.sql b/platform/supabase/migrations/015_external_deployments.sql new file mode 100644 index 0000000..1f051ba --- /dev/null +++ b/platform/supabase/migrations/015_external_deployments.sql @@ -0,0 +1,49 @@ +-- 015_external_deployments.sql +-- Slice 3 of the Datadog integration (#15). Persists raw DORA deployment +-- events pulled by the daily sync from POST /api/v2/dora/deployments. +-- Schema matches the probed payload (see docs/PLAN-datadog.md §9.2). + +CREATE TABLE external_deployments ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + organization_id UUID NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, + provider integration_provider NOT NULL, + -- Datadog's event id (e.g. "43vkaZNgiso"). Idempotency key for upsert. + provider_event_id TEXT NOT NULL, + -- Matched Iris repo when DD's repository_id slug resolves to a tracked + -- repo; NULL otherwise. Match logic normalizes both sides. + repository_id UUID REFERENCES repositories(id) ON DELETE SET NULL, + -- Raw DD slug "github.com//", stored for debuggability when + -- the match fails. + dd_repository_id TEXT, + service TEXT, + env TEXT, + team TEXT, + version TEXT, + commit_sha TEXT, + -- Tri-state: TRUE | FALSE | NULL. NULL = Datadog hasn't evaluated yet. + -- CFR denominator must exclude NULL rows; surface as "pending" in UI. + change_failure BOOLEAN, + deployment_type TEXT, + source TEXT, + started_at TIMESTAMPTZ NOT NULL, + finished_at TIMESTAMPTZ, + duration_seconds INTEGER, + number_of_commits INTEGER, + number_of_pull_requests INTEGER, + -- Present only when change_failure = TRUE. + recovery_time_sec INTEGER, + remediation_type TEXT, + remediation_id TEXT, + raw JSONB NOT NULL, + fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (provider, provider_event_id) +); + +CREATE INDEX idx_external_deployments_org_started + ON external_deployments(organization_id, started_at DESC); +CREATE INDEX idx_external_deployments_repo_started + ON external_deployments(repository_id, started_at DESC) + WHERE repository_id IS NOT NULL; +CREATE INDEX idx_external_deployments_commit_sha + ON external_deployments(commit_sha) + WHERE commit_sha IS NOT NULL; diff --git a/platform/supabase/migrations/016_external_deployment_commits.sql b/platform/supabase/migrations/016_external_deployment_commits.sql new file mode 100644 index 0000000..eed6a13 --- /dev/null +++ b/platform/supabase/migrations/016_external_deployment_commits.sql @@ -0,0 +1,23 @@ +-- 016_external_deployment_commits.sql +-- Slice 3 of the Datadog integration (#15). Per-commit detail unpacked +-- from attributes.commits[] on each DORA deployment event. This is the +-- join key for the AI-vs-human CFR correlation (PR 5): join on +-- commit_sha against the engine's commit_origin classifier output. + +CREATE TABLE external_deployment_commits ( + deployment_id UUID NOT NULL REFERENCES external_deployments(id) ON DELETE CASCADE, + commit_sha TEXT NOT NULL, + commit_timestamp TIMESTAMPTZ, + author_email TEXT, + -- Datadog's canonicalized form (lowercased, alias-resolved); preferred + -- for cross-referencing against Iris's commit_origin author field. + author_canonical_email TEXT, + is_bot BOOLEAN, + -- Both in seconds. Per Datadog's DORA event payload. + change_lead_time INTEGER, + time_to_deploy INTEGER, + PRIMARY KEY (deployment_id, commit_sha) +); + +CREATE INDEX idx_external_deployment_commits_sha + ON external_deployment_commits(commit_sha); diff --git a/platform/supabase/migrations/017_external_incidents.sql b/platform/supabase/migrations/017_external_incidents.sql new file mode 100644 index 0000000..d2a0069 --- /dev/null +++ b/platform/supabase/migrations/017_external_incidents.sql @@ -0,0 +1,33 @@ +-- 017_external_incidents.sql +-- Slice 3 of the Datadog integration (#15). Persists raw DORA failure +-- events pulled by the daily sync from POST /api/v2/dora/failures. +-- service / env / team are arrays because that's how DD returns them +-- (see docs/PLAN-datadog.md §9.2). + +CREATE TABLE external_incidents ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + organization_id UUID NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, + provider integration_provider NOT NULL, + -- Datadog's event id. Idempotency key for upsert. + provider_event_id TEXT NOT NULL, + service TEXT[], + env TEXT[], + team TEXT[], + -- DD's incident name, e.g. "RIO-978 | Pedidos sendo processados sem cobrança". + name TEXT, + -- Free-form ("Normal" | "High" | "Urgent" observed; not normalized). + severity TEXT, + started_at TIMESTAMPTZ NOT NULL, + finished_at TIMESTAMPTZ, + time_to_restore_seconds INTEGER, + -- "api" in all observed events (customer pushes from their RIO workflow). + source TEXT, + raw JSONB NOT NULL, + fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (provider, provider_event_id) +); + +CREATE INDEX idx_external_incidents_org_started + ON external_incidents(organization_id, started_at DESC); +CREATE INDEX idx_external_incidents_service_gin + ON external_incidents USING GIN (service); diff --git a/platform/tests/datadog-sync.test.ts b/platform/tests/datadog-sync.test.ts new file mode 100644 index 0000000..f2e4e28 --- /dev/null +++ b/platform/tests/datadog-sync.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; + +import { __testing } from "@/lib/integrations/datadog/sync"; + +const { normalizeRepoSlug, oldestStartedAt, normalizeNullableIso } = __testing; + +describe("normalizeRepoSlug", () => { + it("returns null for empty input", () => { + expect(normalizeRepoSlug(null)).toBeNull(); + expect(normalizeRepoSlug(undefined)).toBeNull(); + expect(normalizeRepoSlug("")).toBeNull(); + expect(normalizeRepoSlug(" ")).toBeNull(); + }); + + it("normalizes a Datadog slug verbatim", () => { + expect(normalizeRepoSlug("github.com/rocketbus/search")).toBe( + "github.com/rocketbus/search", + ); + }); + + it("strips https scheme and .git suffix", () => { + expect(normalizeRepoSlug("https://github.com/RocketBus/search.git")).toBe( + "github.com/rocketbus/search", + ); + }); + + it("converts ssh git@host:org/repo to host/org/repo", () => { + expect(normalizeRepoSlug("git@github.com:RocketBus/search.git")).toBe( + "github.com/rocketbus/search", + ); + }); + + it("handles ssh:// scheme", () => { + expect(normalizeRepoSlug("ssh://git@github.com/RocketBus/search.git")).toBe( + "github.com/rocketbus/search", + ); + }); + + it("matches DD slug to https remote URL after normalization", () => { + const dd = normalizeRepoSlug("github.com/rocketbus/search"); + const remote = normalizeRepoSlug("https://github.com/RocketBus/search.git"); + expect(dd).toEqual(remote); + }); + + it("strips trailing slashes and www", () => { + expect(normalizeRepoSlug("https://www.github.com/Org/Repo/")).toBe( + "github.com/org/repo", + ); + }); +}); + +describe("oldestStartedAt", () => { + it("returns the minimum timestamp in ISO 8601 (seconds precision, Z)", () => { + const batch = [ + { attributes: { started_at: "2026-03-05T10:00:00Z" } }, + { attributes: { started_at: "2026-03-05T09:00:00Z" } }, + { attributes: { started_at: "2026-03-05T11:00:00Z" } }, + ]; + expect(oldestStartedAt(batch, (e) => e.attributes.started_at)).toBe( + "2026-03-05T09:00:00Z", + ); + }); + + it("returns null for empty batch", () => { + expect(oldestStartedAt([], () => "")).toBeNull(); + }); +}); + +describe("normalizeNullableIso", () => { + it("returns null for the Datadog zero-value timestamp", () => { + expect(normalizeNullableIso("0001-01-01T00:00:00Z")).toBeNull(); + }); + + it("returns null for missing input", () => { + expect(normalizeNullableIso(undefined)).toBeNull(); + }); + + it("passes through real ISO timestamps", () => { + expect(normalizeNullableIso("2026-03-05T12:34:56Z")).toBe( + "2026-03-05T12:34:56.000Z", + ); + }); + + it("returns null for invalid input", () => { + expect(normalizeNullableIso("not-a-date")).toBeNull(); + }); +}); diff --git a/platform/vercel.json b/platform/vercel.json index 9ee38e5..75a325a 100644 --- a/platform/vercel.json +++ b/platform/vercel.json @@ -1,5 +1,11 @@ { "$schema": "https://openapi.vercel.sh/vercel.json", "framework": "nextjs", - "regions": ["gru1"] + "regions": ["gru1"], + "crons": [ + { + "path": "/api/cron/sync-integrations", + "schedule": "0 4 * * *" + } + ] }