From e989aa263a5fc086cf074e5be76c0b331e551bca Mon Sep 17 00:00:00 2001 From: William Hill Date: Sun, 3 May 2026 14:42:05 -0400 Subject: [PATCH 1/2] feat(upload): validation report UI, diff vs prior, CSV export (#110) Extends POST /api/admin/upload/commit with full row-level errors (capped), comparison to the last upload of the same schema, and optional JSONB validation_report on upload_history (migration + 42703 fallback). Completes a first slice of epic #124 AASCU follow-ups: human-readable report after upload, downloadable CSV, baseline for richer coercion/dedup later. Co-authored-by: Cursor --- .../app/admin/upload/page.tsx | 16 +- .../app/api/admin/upload/commit/route.ts | 109 ++++++++++- .../components/upload/upload-summary.tsx | 180 ++++++++++++++++-- .../upload-validation-report.test.ts | 47 +++++ .../lib/upload-validation-report.ts | 115 +++++++++++ ...40000_upload_history_validation_report.sql | 7 + 6 files changed, 436 insertions(+), 38 deletions(-) create mode 100644 codebenders-dashboard/lib/__tests__/upload-validation-report.test.ts create mode 100644 codebenders-dashboard/lib/upload-validation-report.ts create mode 100644 supabase/migrations/20260503140000_upload_history_validation_report.sql diff --git a/codebenders-dashboard/app/admin/upload/page.tsx b/codebenders-dashboard/app/admin/upload/page.tsx index be3870f..7e3b37c 100644 --- a/codebenders-dashboard/app/admin/upload/page.tsx +++ b/codebenders-dashboard/app/admin/upload/page.tsx @@ -9,6 +9,7 @@ import { UploadSummary } from "@/components/upload/upload-summary" import { Button } from "@/components/ui/button" import { AlertCircle, CheckCircle, Loader2 } from "lucide-react" import { SCHEMAS, CONFIDENT_THRESHOLD, type ColumnMapping } from "@/lib/upload-schemas" +import type { UploadCommitApiResponse } from "@/lib/upload-validation-report" type Step = "upload" | "preview" | "complete" @@ -24,13 +25,6 @@ interface PreviewData { errors: string[] } -interface CommitResult { - inserted: number - skipped: number - errors: Array<{ row: number; message: string }> - uploadId: number -} - interface HistoryEntry { id: number filename: string @@ -48,7 +42,7 @@ export default function UploadPage() { const [columns, setColumns] = useState([]) const [selectedSchema, setSelectedSchema] = useState(null) const [showSchemaOverride, setShowSchemaOverride] = useState(false) - const [commitResult, setCommitResult] = useState(null) + const [commitResult, setCommitResult] = useState(null) const [loading, setLoading] = useState(false) const [error, setError] = useState(null) const [recentUploads, setRecentUploads] = useState([]) @@ -123,7 +117,7 @@ export default function UploadPage() { return } - const data: CommitResult = await res.json() + const data: UploadCommitApiResponse = await res.json() setCommitResult(data) setStep("complete") } catch (err) { @@ -368,9 +362,7 @@ export default function UploadPage() { router.push("/admin/upload/history")} /> diff --git a/codebenders-dashboard/app/api/admin/upload/commit/route.ts b/codebenders-dashboard/app/api/admin/upload/commit/route.ts index b3feedb..9313655 100644 --- a/codebenders-dashboard/app/api/admin/upload/commit/route.ts +++ b/codebenders-dashboard/app/api/admin/upload/commit/route.ts @@ -2,8 +2,15 @@ import { NextRequest, NextResponse } from "next/server" import { parseFileBuffer, getFileType, validateFileSize } from "@/lib/upload-parser" import { SCHEMAS, type UploadSchema, type ColumnMapping } from "@/lib/upload-schemas" import { getPool } from "@/lib/db" +import { + computeUploadDiff, + type PreviousUploadSnapshot, + type UploadCommitApiResponse, +} from "@/lib/upload-validation-report" const BATCH_SIZE = 500 +const MAX_ERRORS_IN_REPORT = 5000 +const MAX_ERRORS_IN_RESPONSE = 3000 export async function POST(request: NextRequest) { const userId = request.headers.get("x-user-id") ?? "" @@ -56,18 +63,102 @@ export async function POST(request: NextRequest) { ? "partial" : "success" - const { rows: historyRows } = await pool.query( - `INSERT INTO upload_history (user_id, user_email, filename, file_type, rows_inserted, rows_skipped, error_count, status) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING id`, - [userId, userEmail, file.name, schemaId, result.inserted, result.skipped, result.errors.length, status] + const prevRes = await pool.query<{ + id: string + filename: string + rows_inserted: number + rows_skipped: number + error_count: number + uploaded_at: Date + }>( + `SELECT id, filename, rows_inserted, rows_skipped, error_count, uploaded_at + FROM upload_history + WHERE file_type = $1 + ORDER BY uploaded_at DESC + LIMIT 1`, + [schemaId] ) - return NextResponse.json({ + const previousUpload: PreviousUploadSnapshot | null = prevRes.rows[0] + ? { + id: Number(prevRes.rows[0].id), + filename: prevRes.rows[0].filename, + rowsInserted: prevRes.rows[0].rows_inserted, + rowsSkipped: prevRes.rows[0].rows_skipped, + errorCount: prevRes.rows[0].error_count, + uploadedAt: prevRes.rows[0].uploaded_at.toISOString(), + } + : null + + const reportGeneratedAt = new Date().toISOString() + const diff = computeUploadDiff( + { + inserted: result.inserted, + skipped: result.skipped, + errorCount: result.errors.length, + }, + previousUpload + ) + + const errorsForStorage = result.errors.slice(0, MAX_ERRORS_IN_REPORT) + const reportJson = { + version: 1 as const, + schemaId, + totalRowsInFile: rows.length, inserted: result.inserted, skipped: result.skipped, - errors: result.errors.slice(0, 50), - uploadId: historyRows[0].id, - }) + errors: errorsForStorage, + errorsTotal: result.errors.length, + errorsTruncated: result.errors.length > errorsForStorage.length, + previousUpload, + diff, + generatedAt: reportGeneratedAt, + } + + const baseInsertParams = [ + userId, + userEmail, + file.name, + schemaId, + result.inserted, + result.skipped, + result.errors.length, + status, + ] as const + + let historyId: number + try { + const insertedRow = await pool.query( + `INSERT INTO upload_history (user_id, user_email, filename, file_type, rows_inserted, rows_skipped, error_count, status, validation_report) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb) RETURNING id`, + [...baseInsertParams, JSON.stringify(reportJson)] + ) + historyId = Number(insertedRow.rows[0].id) + } catch (err: unknown) { + const code = err && typeof err === "object" && "code" in err ? String((err as { code: string }).code) : "" + if (code !== "42703") throw err + const insertedRow = await pool.query( + `INSERT INTO upload_history (user_id, user_email, filename, file_type, rows_inserted, rows_skipped, error_count, status) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING id`, + [...baseInsertParams] + ) + historyId = Number(insertedRow.rows[0].id) + } + + const responseBody: UploadCommitApiResponse = { + inserted: result.inserted, + skipped: result.skipped, + errors: result.errors.slice(0, MAX_ERRORS_IN_RESPONSE), + errorsTotal: result.errors.length, + errorsTruncated: result.errors.length > MAX_ERRORS_IN_RESPONSE, + uploadId: historyId, + totalRowsInFile: rows.length, + previousUpload, + diff, + reportGeneratedAt, + } + + return NextResponse.json(responseBody) } catch (err) { console.error("Upload commit error:", err) return NextResponse.json( @@ -176,7 +267,7 @@ async function upsertRows( const result = await pool.query(batchSql, batchParams) inserted += result.rowCount ?? 0 - } catch (err) { + } catch { // If batch fails, fall back to per-row to identify the bad row(s) for (let j = 0; j < batch.length; j++) { const row = batch[j] diff --git a/codebenders-dashboard/components/upload/upload-summary.tsx b/codebenders-dashboard/components/upload/upload-summary.tsx index c60b354..48ae53d 100644 --- a/codebenders-dashboard/components/upload/upload-summary.tsx +++ b/codebenders-dashboard/components/upload/upload-summary.tsx @@ -1,14 +1,25 @@ "use client" +import { useCallback, useMemo } from "react" import { Button } from "@/components/ui/button" -import { CheckCircle } from "lucide-react" +import { AlertTriangle, CheckCircle, Download } from "lucide-react" +import { + serializeUploadValidationReportCsv, + type UploadCommitApiResponse, +} from "@/lib/upload-validation-report" + +function reportFilenameDate(iso: string): string { + const d = new Date(iso) + const y = d.getFullYear() + const m = String(d.getMonth() + 1).padStart(2, "0") + const day = String(d.getDate()).padStart(2, "0") + return `${y}-${m}-${day}` +} interface UploadSummaryProps { filename: string schemaLabel: string - inserted: number - skipped: number - errorCount: number + report: UploadCommitApiResponse onUploadAnother: () => void onViewHistory: () => void } @@ -16,45 +27,180 @@ interface UploadSummaryProps { export function UploadSummary({ filename, schemaLabel, - inserted, - skipped, - errorCount, + report, onUploadAnother, onViewHistory, }: UploadSummaryProps) { + const { inserted, skipped, errors, errorsTotal, errorsTruncated, diff } = report + + const headlineOk = inserted > 0 && errorsTotal === 0 && skipped === 0 + const headlinePartial = inserted > 0 && (errorsTotal > 0 || skipped > 0) + const headlineFailed = inserted === 0 && errorsTotal > 0 + + const downloadCsv = useCallback(() => { + const csv = serializeUploadValidationReportCsv({ + filename, + schemaLabel, + uploadId: report.uploadId, + totalRowsInFile: report.totalRowsInFile, + inserted, + skipped, + errorCount: errorsTotal, + errors, + diff, + reportGeneratedAt: report.reportGeneratedAt, + }) + const blob = new Blob([csv], { type: "text/csv;charset=utf-8" }) + const url = URL.createObjectURL(blob) + const a = document.createElement("a") + a.href = url + a.download = `upload-validation-report_${report.uploadId}_${reportFilenameDate(report.reportGeneratedAt)}.csv` + document.body.appendChild(a) + a.click() + document.body.removeChild(a) + URL.revokeObjectURL(url) + }, [diff, errors, filename, inserted, report, schemaLabel, skipped, errorsTotal]) + + const statusClass = useMemo(() => { + if (headlineFailed) return "bg-red-50 border-red-200" + if (headlinePartial) return "bg-amber-50 border-amber-200" + return "bg-green-50 border-green-200" + }, [headlineFailed, headlinePartial]) + + const iconClass = headlineFailed ? "text-red-600" : headlinePartial ? "text-amber-600" : "text-green-600" + return ( -
-
- -

Upload Complete

+
+
+ {headlineFailed ? ( + + ) : ( + + )} +

+ {headlineFailed ? "Upload did not insert rows" : "Upload complete"} +

{filename} — {schemaLabel}

+

+ Validation report #{report.uploadId} · {report.totalRowsInFile.toLocaleString()} rows in file +

- {inserted} + {inserted}
inserted
- {skipped} + {skipped}
skipped
- {errorCount} + {errorsTotal}
- errors + row issues
+ {diff ? ( +
+

Compared to previous upload (same data type)

+

+ Prior: {report.previousUpload?.filename} ·{" "} + {report.previousUpload?.rowsInserted?.toLocaleString() ?? "—"} inserted ·{" "} + {new Date(report.previousUpload?.uploadedAt ?? "").toLocaleString()} +

+
    +
  • + Δ Inserted:{" "} + {diff.rowsInsertedDelta >= 0 ? "+" : ""} + {diff.rowsInsertedDelta.toLocaleString()} +
    • +
  • + Δ Skipped:{" "} + {diff.rowsSkippedDelta >= 0 ? "+" : ""} + {diff.rowsSkippedDelta.toLocaleString()} +
    • +
  • + Δ Row-level issues:{" "} + {diff.errorCountDelta >= 0 ? "+" : ""} + {diff.errorCountDelta.toLocaleString()} +
    • + {diff.percentInsertedChange != null ? ( +
  • + Change vs prior inserted:{" "} + {diff.percentInsertedChange}% +
    • + ) : null} +
+ {diff.anomalyLargeSwing ? ( +

+ + Inserted row count moved by 50% or more vs the last upload of this type — confirm cohort or + file scope before relying on aggregates. +

+ ) : null} +
+ ) : ( +

+ No prior upload of this data type in history — baseline established for future diffs. +

+ )} + +
+
+

Row-level messages

+ +
+ {errorsTruncated ? ( +

+ Showing first {errors.length} of {errorsTotal} messages in the UI and export slice. Full counts + are in the database report when validation_report is + enabled. +

+ ) : null} + {errors.length === 0 ? ( +

+ {headlineOk + ? "No row-level validation messages — clean run." + : "No per-row messages returned; see inserted vs skipped counts above."} +

+ ) : ( +
+ + + + + + + + + + {errors.map((e, i) => ( + + + + + + ))} + +
RowColumnMessage
{e.row}{e.column ?? "—"}{e.message}
+
+ )} +
+
diff --git a/codebenders-dashboard/lib/__tests__/upload-validation-report.test.ts b/codebenders-dashboard/lib/__tests__/upload-validation-report.test.ts new file mode 100644 index 0000000..47e7c15 --- /dev/null +++ b/codebenders-dashboard/lib/__tests__/upload-validation-report.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest" +import { computeUploadDiff, serializeUploadValidationReportCsv } from "@/lib/upload-validation-report" + +describe("upload validation report (#110)", () => { + it("computeUploadDiff returns null without a baseline", () => { + expect( + computeUploadDiff({ inserted: 10, skipped: 1, errorCount: 2 }, null) + ).toBeNull() + }) + + it("computeUploadDiff flags large swings in inserted rows", () => { + const diff = computeUploadDiff( + { inserted: 100, skipped: 0, errorCount: 0 }, + { + id: 1, + filename: "prior.csv", + rowsInserted: 40, + rowsSkipped: 0, + errorCount: 0, + uploadedAt: "2026-05-01T12:00:00.000Z", + } + ) + expect(diff).not.toBeNull() + expect(diff!.rowsInsertedDelta).toBe(60) + expect(diff!.percentInsertedChange).toBe(150) + expect(diff!.anomalyLargeSwing).toBe(true) + }) + + it("serializeUploadValidationReportCsv includes summary and error rows", () => { + const csv = serializeUploadValidationReportCsv({ + filename: "t.csv", + schemaLabel: "Student", + uploadId: 42, + totalRowsInFile: 100, + inserted: 90, + skipped: 10, + errorCount: 1, + errors: [{ row: 5, column: "gpa", message: "invalid" }], + diff: null, + reportGeneratedAt: "2026-05-03T12:00:00.000Z", + }) + expect(csv).toContain("# Upload validation report") + expect(csv).toContain("# Upload ID,42") + expect(csv).toContain("row,column,message") + expect(csv).toContain("5,gpa,invalid") + }) +}) diff --git a/codebenders-dashboard/lib/upload-validation-report.ts b/codebenders-dashboard/lib/upload-validation-report.ts new file mode 100644 index 0000000..ada9b91 --- /dev/null +++ b/codebenders-dashboard/lib/upload-validation-report.ts @@ -0,0 +1,115 @@ +/** Types and CSV serialization for upload validation reports (#110). */ + +/** Response shape from POST /api/admin/upload/commit (validation report). */ +export interface UploadCommitApiResponse { + inserted: number + skipped: number + errors: UploadRowError[] + errorsTotal: number + errorsTruncated: boolean + uploadId: number + totalRowsInFile: number + previousUpload: PreviousUploadSnapshot | null + diff: UploadValidationDiff | null + reportGeneratedAt: string +} + +export interface UploadRowError { + row: number + column?: string + message: string +} + +export interface PreviousUploadSnapshot { + id: number + filename: string + rowsInserted: number + rowsSkipped: number + errorCount: number + uploadedAt: string +} + +export interface UploadValidationDiff { + rowsInsertedDelta: number + rowsSkippedDelta: number + errorCountDelta: number + /** Percent change vs previous rows_inserted; null if no meaningful baseline. */ + percentInsertedChange: number | null + /** True when percent change magnitude exceeds threshold (e.g. cohort size swing). */ + anomalyLargeSwing: boolean +} + +const LARGE_SWING_PCT = 50 + +export function computeUploadDiff( + current: { inserted: number; skipped: number; errorCount: number }, + previous: PreviousUploadSnapshot | null +): UploadValidationDiff | null { + if (!previous) return null + const rowsInsertedDelta = current.inserted - previous.rowsInserted + const rowsSkippedDelta = current.skipped - previous.rowsSkipped + const errorCountDelta = current.errorCount - previous.errorCount + const base = previous.rowsInserted + const percentInsertedChange = + base > 0 ? Math.round(((current.inserted - base) / base) * 1000) / 10 : null + const anomalyLargeSwing = + percentInsertedChange !== null && Math.abs(percentInsertedChange) >= LARGE_SWING_PCT + + return { + rowsInsertedDelta, + rowsSkippedDelta, + errorCountDelta, + percentInsertedChange, + anomalyLargeSwing, + } +} + +function escapeCsvCell(v: string | number): string { + const s = String(v) + if (/[",\n\r]/.test(s)) return `"${s.replace(/"/g, '""')}"` + return s +} + +export function serializeUploadValidationReportCsv(params: { + filename: string + schemaLabel: string + uploadId: number + totalRowsInFile: number + inserted: number + skipped: number + errorCount: number + errors: UploadRowError[] + diff: UploadValidationDiff | null + reportGeneratedAt: string +}): string { + const lines: string[] = [] + lines.push("# Upload validation report") + lines.push(`# Generated,${escapeCsvCell(params.reportGeneratedAt)}`) + lines.push(`# Upload ID,${params.uploadId}`) + lines.push(`# File,${escapeCsvCell(params.filename)}`) + lines.push(`# Schema,${escapeCsvCell(params.schemaLabel)}`) + lines.push(`# Rows in file,${params.totalRowsInFile}`) + lines.push(`# Inserted,${params.inserted}`) + lines.push(`# Skipped,${params.skipped}`) + lines.push(`# Row-level issues (total),${params.errorCount}`) + lines.push(`# Row-level issues (rows in this file),${params.errors.length}`) + if (params.diff) { + lines.push( + `# Delta vs previous (same data type) — inserted,${params.diff.rowsInsertedDelta}` + ) + lines.push(`# Delta — skipped,${params.diff.rowsSkippedDelta}`) + lines.push(`# Delta — error count,${params.diff.errorCountDelta}`) + lines.push( + `# Pct change inserted (prev baseline),${params.diff.percentInsertedChange ?? ""}` + ) + lines.push(`# Large swing flag,${params.diff.anomalyLargeSwing}`) + } + lines.push("") + lines.push("row,column,message") + for (const e of params.errors) { + lines.push( + [e.row, e.column ?? "", e.message].map((c) => escapeCsvCell(c)).join(",") + ) + } + return lines.join("\n") +} diff --git a/supabase/migrations/20260503140000_upload_history_validation_report.sql b/supabase/migrations/20260503140000_upload_history_validation_report.sql new file mode 100644 index 0000000..3cd8644 --- /dev/null +++ b/supabase/migrations/20260503140000_upload_history_validation_report.sql @@ -0,0 +1,7 @@ +-- Issue #110 / epic #124: persist upload validation report for audit trail (AASCU convening follow-up). + +ALTER TABLE public.upload_history + ADD COLUMN IF NOT EXISTS validation_report JSONB; + +COMMENT ON COLUMN public.upload_history.validation_report IS + 'Row-level validation summary, diff vs prior upload of same schema, and metadata.'; From 90da28656085a8785af7df790c62bbcefcf9c3c5 Mon Sep 17 00:00:00 2001 From: William Hill Date: Sun, 3 May 2026 14:52:04 -0400 Subject: [PATCH 2/2] refactor(upload): simplify validation report commit path and UI (#110) Extract insertUploadHistoryRow, pg 42703 guard, uploadStatus helper; centralize swing threshold constant; trim upload summary and step JSX. Co-authored-by: Cursor --- .../app/admin/upload/page.tsx | 16 +- .../app/api/admin/upload/commit/route.ts | 144 +++++++++++------- .../components/upload/upload-summary.tsx | 41 ++--- .../lib/upload-validation-report.ts | 29 +++- 4 files changed, 149 insertions(+), 81 deletions(-) diff --git a/codebenders-dashboard/app/admin/upload/page.tsx b/codebenders-dashboard/app/admin/upload/page.tsx index 7e3b37c..4083d25 100644 --- a/codebenders-dashboard/app/admin/upload/page.tsx +++ b/codebenders-dashboard/app/admin/upload/page.tsx @@ -13,6 +13,12 @@ import type { UploadCommitApiResponse } from "@/lib/upload-validation-report" type Step = "upload" | "preview" | "complete" +function stepIndicatorClass(i: number, stepIndex: number): string { + if (i < stepIndex) return "text-green-600 line-through" + if (i === stepIndex) return "font-bold text-purple-700 bg-purple-50 px-2 py-0.5 rounded-full" + return "text-muted-foreground" +} + interface PreviewData { detectedSchema: string | null detectedSchemaLabel: string | null @@ -162,15 +168,7 @@ export default function UploadPage() { {stepLabels.map((label, i) => ( {i > 0 && →} - + {i < stepIndex ? `${label} ✓` : `${i + 1}. ${label}`} diff --git a/codebenders-dashboard/app/api/admin/upload/commit/route.ts b/codebenders-dashboard/app/api/admin/upload/commit/route.ts index 9313655..70bde05 100644 --- a/codebenders-dashboard/app/api/admin/upload/commit/route.ts +++ b/codebenders-dashboard/app/api/admin/upload/commit/route.ts @@ -6,12 +6,86 @@ import { computeUploadDiff, type PreviousUploadSnapshot, type UploadCommitApiResponse, + type UploadCurrentMetrics, + type UploadHistoryStoredReport, + type UploadRowError, } from "@/lib/upload-validation-report" const BATCH_SIZE = 500 const MAX_ERRORS_IN_REPORT = 5000 const MAX_ERRORS_IN_RESPONSE = 3000 +type UploadHistoryRow = { + id: string + filename: string + rows_inserted: number + rows_skipped: number + error_count: number + uploaded_at: Date +} + +function pgErrorCode(err: unknown): string | undefined { + if (err && typeof err === "object" && "code" in err) { + return String((err as { code: unknown }).code) + } + return undefined +} + +function isUndefinedColumnPgError(err: unknown): boolean { + return pgErrorCode(err) === "42703" +} + +function previousUploadFromRow(row: UploadHistoryRow): PreviousUploadSnapshot { + return { + id: Number(row.id), + filename: row.filename, + rowsInserted: row.rows_inserted, + rowsSkipped: row.rows_skipped, + errorCount: row.error_count, + uploadedAt: row.uploaded_at.toISOString(), + } +} + +function uploadStatus(errorsCount: number, inserted: number): "failed" | "partial" | "success" { + if (errorsCount > 0 && inserted === 0) return "failed" + if (errorsCount > 0) return "partial" + return "success" +} + +async function insertUploadHistoryRow(params: { + pool: ReturnType + baseInsertParams: readonly [ + string, + string, + string, + string, + number, + number, + number, + "failed" | "partial" | "success", + ] + reportJson: UploadHistoryStoredReport +}): Promise { + const { pool, baseInsertParams, reportJson } = params + try { + const insertedRow = await pool.query( + `INSERT INTO upload_history (user_id, user_email, filename, file_type, rows_inserted, rows_skipped, error_count, status, validation_report) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb) RETURNING id`, + [...baseInsertParams, JSON.stringify(reportJson)] + ) + return Number(insertedRow.rows[0].id) + } catch (err: unknown) { + if (!isUndefinedColumnPgError(err)) throw err + } + + const insertedRow = await pool.query( + `INSERT INTO upload_history (user_id, user_email, filename, file_type, rows_inserted, rows_skipped, error_count, status) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING id`, + [...baseInsertParams] + ) + return Number(insertedRow.rows[0].id) +} + export async function POST(request: NextRequest) { const userId = request.headers.get("x-user-id") ?? "" const userEmail = request.headers.get("x-user-email") ?? "" @@ -56,21 +130,9 @@ export async function POST(request: NextRequest) { const result = await upsertRows(rows, columnMapping, schema) const pool = getPool() - const status = - result.errors.length > 0 && result.inserted === 0 - ? "failed" - : result.errors.length > 0 - ? "partial" - : "success" - - const prevRes = await pool.query<{ - id: string - filename: string - rows_inserted: number - rows_skipped: number - error_count: number - uploaded_at: Date - }>( + const status = uploadStatus(result.errors.length, result.inserted) + + const prevRes = await pool.query( `SELECT id, filename, rows_inserted, rows_skipped, error_count, uploaded_at FROM upload_history WHERE file_type = $1 @@ -80,29 +142,20 @@ export async function POST(request: NextRequest) { ) const previousUpload: PreviousUploadSnapshot | null = prevRes.rows[0] - ? { - id: Number(prevRes.rows[0].id), - filename: prevRes.rows[0].filename, - rowsInserted: prevRes.rows[0].rows_inserted, - rowsSkipped: prevRes.rows[0].rows_skipped, - errorCount: prevRes.rows[0].error_count, - uploadedAt: prevRes.rows[0].uploaded_at.toISOString(), - } + ? previousUploadFromRow(prevRes.rows[0]) : null const reportGeneratedAt = new Date().toISOString() - const diff = computeUploadDiff( - { - inserted: result.inserted, - skipped: result.skipped, - errorCount: result.errors.length, - }, - previousUpload - ) + const currentMetrics: UploadCurrentMetrics = { + inserted: result.inserted, + skipped: result.skipped, + errorCount: result.errors.length, + } + const diff = computeUploadDiff(currentMetrics, previousUpload) const errorsForStorage = result.errors.slice(0, MAX_ERRORS_IN_REPORT) - const reportJson = { - version: 1 as const, + const reportJson: UploadHistoryStoredReport = { + version: 1, schemaId, totalRowsInFile: rows.length, inserted: result.inserted, @@ -126,24 +179,11 @@ export async function POST(request: NextRequest) { status, ] as const - let historyId: number - try { - const insertedRow = await pool.query( - `INSERT INTO upload_history (user_id, user_email, filename, file_type, rows_inserted, rows_skipped, error_count, status, validation_report) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb) RETURNING id`, - [...baseInsertParams, JSON.stringify(reportJson)] - ) - historyId = Number(insertedRow.rows[0].id) - } catch (err: unknown) { - const code = err && typeof err === "object" && "code" in err ? String((err as { code: string }).code) : "" - if (code !== "42703") throw err - const insertedRow = await pool.query( - `INSERT INTO upload_history (user_id, user_email, filename, file_type, rows_inserted, rows_skipped, error_count, status) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING id`, - [...baseInsertParams] - ) - historyId = Number(insertedRow.rows[0].id) - } + const historyId = await insertUploadHistoryRow({ + pool, + baseInsertParams, + reportJson, + }) const responseBody: UploadCommitApiResponse = { inserted: result.inserted, @@ -171,7 +211,7 @@ export async function POST(request: NextRequest) { interface UpsertResult { inserted: number skipped: number - errors: Array<{ row: number; column?: string; message: string }> + errors: UploadRowError[] } async function upsertRows( diff --git a/codebenders-dashboard/components/upload/upload-summary.tsx b/codebenders-dashboard/components/upload/upload-summary.tsx index 48ae53d..a8af228 100644 --- a/codebenders-dashboard/components/upload/upload-summary.tsx +++ b/codebenders-dashboard/components/upload/upload-summary.tsx @@ -5,6 +5,7 @@ import { Button } from "@/components/ui/button" import { AlertTriangle, CheckCircle, Download } from "lucide-react" import { serializeUploadValidationReportCsv, + UPLOAD_INSERTED_SWING_THRESHOLD_PCT, type UploadCommitApiResponse, } from "@/lib/upload-validation-report" @@ -16,6 +17,17 @@ function reportFilenameDate(iso: string): string { return `${y}-${m}-${day}` } +function signedIntLabel(n: number): string { + const sign = n >= 0 ? "+" : "" + return `${sign}${n.toLocaleString()}` +} + +function summaryIconClass(headlineFailed: boolean, headlinePartial: boolean): string { + if (headlineFailed) return "text-red-600" + if (headlinePartial) return "text-amber-600" + return "text-green-600" +} + interface UploadSummaryProps { filename: string schemaLabel: string @@ -43,11 +55,11 @@ export function UploadSummary({ schemaLabel, uploadId: report.uploadId, totalRowsInFile: report.totalRowsInFile, - inserted, - skipped, - errorCount: errorsTotal, - errors, - diff, + inserted: report.inserted, + skipped: report.skipped, + errorCount: report.errorsTotal, + errors: report.errors, + diff: report.diff, reportGeneratedAt: report.reportGeneratedAt, }) const blob = new Blob([csv], { type: "text/csv;charset=utf-8" }) @@ -59,7 +71,7 @@ export function UploadSummary({ a.click() document.body.removeChild(a) URL.revokeObjectURL(url) - }, [diff, errors, filename, inserted, report, schemaLabel, skipped, errorsTotal]) + }, [filename, schemaLabel, report]) const statusClass = useMemo(() => { if (headlineFailed) return "bg-red-50 border-red-200" @@ -67,7 +79,7 @@ export function UploadSummary({ return "bg-green-50 border-green-200" }, [headlineFailed, headlinePartial]) - const iconClass = headlineFailed ? "text-red-600" : headlinePartial ? "text-amber-600" : "text-green-600" + const iconClass = summaryIconClass(headlineFailed, headlinePartial) return (
@@ -115,19 +127,14 @@ export function UploadSummary({

  • - Δ Inserted:{" "} - {diff.rowsInsertedDelta >= 0 ? "+" : ""} - {diff.rowsInsertedDelta.toLocaleString()} + Δ Inserted: {signedIntLabel(diff.rowsInsertedDelta)}
  • - Δ Skipped:{" "} - {diff.rowsSkippedDelta >= 0 ? "+" : ""} - {diff.rowsSkippedDelta.toLocaleString()} + Δ Skipped: {signedIntLabel(diff.rowsSkippedDelta)}
  • Δ Row-level issues:{" "} - {diff.errorCountDelta >= 0 ? "+" : ""} - {diff.errorCountDelta.toLocaleString()} + {signedIntLabel(diff.errorCountDelta)}
    • {diff.percentInsertedChange != null ? (
  • @@ -139,8 +146,8 @@ export function UploadSummary({ {diff.anomalyLargeSwing ? (

    - Inserted row count moved by 50% or more vs the last upload of this type — confirm cohort or - file scope before relying on aggregates. + Inserted row count moved by {UPLOAD_INSERTED_SWING_THRESHOLD_PCT}% or more vs the last upload + of this type — confirm cohort or file scope before relying on aggregates.

    ) : null}
diff --git a/codebenders-dashboard/lib/upload-validation-report.ts b/codebenders-dashboard/lib/upload-validation-report.ts index ada9b91..f969a1b 100644 --- a/codebenders-dashboard/lib/upload-validation-report.ts +++ b/codebenders-dashboard/lib/upload-validation-report.ts @@ -39,10 +39,32 @@ export interface UploadValidationDiff { anomalyLargeSwing: boolean } -const LARGE_SWING_PCT = 50 +/** Magnitude threshold (percent points) for `anomalyLargeSwing` vs prior inserted rows. */ +export const UPLOAD_INSERTED_SWING_THRESHOLD_PCT = 50 + +export interface UploadCurrentMetrics { + inserted: number + skipped: number + errorCount: number +} + +/** Stored on `upload_history.validation_report` (JSONB). */ +export interface UploadHistoryStoredReport { + version: 1 + schemaId: string + totalRowsInFile: number + inserted: number + skipped: number + errors: UploadRowError[] + errorsTotal: number + errorsTruncated: boolean + previousUpload: PreviousUploadSnapshot | null + diff: UploadValidationDiff | null + generatedAt: string +} export function computeUploadDiff( - current: { inserted: number; skipped: number; errorCount: number }, + current: UploadCurrentMetrics, previous: PreviousUploadSnapshot | null ): UploadValidationDiff | null { if (!previous) return null @@ -53,7 +75,8 @@ export function computeUploadDiff( const percentInsertedChange = base > 0 ? Math.round(((current.inserted - base) / base) * 1000) / 10 : null const anomalyLargeSwing = - percentInsertedChange !== null && Math.abs(percentInsertedChange) >= LARGE_SWING_PCT + percentInsertedChange !== null && + Math.abs(percentInsertedChange) >= UPLOAD_INSERTED_SWING_THRESHOLD_PCT return { rowsInsertedDelta,