Agent citations + multi-turn conversations

CHANGELOG.md

@@ -4,6 +4,29 @@ All notable changes to DocMind are logged here, phase by phase. This is the publ
 
 Format loosely follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
+## Agent Citations + Multi-Turn Conversations — 2026-07-26
+
+### Added
+
+#### Multi-Turn Agent Conversations
+- **`Conversation` + `Message` Prisma models** — new `MessageRole` enum (`user`/`assistant`); `Message.citations` stores citation data for assistant turns produced via the short-circuit path below.
+- **`ConversationsModule`** (`ConversationsService`, `ConversationsController`) — `GET /v1/conversations` (list current user's conversations), `GET /v1/conversations/:id/messages` (full history), ownership-checked via `assertOwnership()`.
+- **`trimHistory()` util** — hybrid history-window strategy: hard ceiling of the last 20 messages, then a further ~3000-token budget trim within that window, always preserving at least the most recent turn even if it alone exceeds the budget.
+- **`AgentChatDto.conversationId`** (optional) — omit to start a new conversation; the new id is returned via a `conversation_started` SSE event before the answer stream begins.
+- **`AgentService.run()`** now loads and seeds trimmed prior history into the graph's initial `messages` state, and persists the user query immediately plus the final assistant answer once produced. Nothing is persisted for turns that pause on an `external_write` confirmation (no final answer yet).
+- Scope: multi-turn ships on `/v1/agent/chat` only; `/v1/chat/stream` remains single-turn.
+
+#### Agent Citations
+- **Citation short-circuit** — when a dispatched tool (`query_documents`) returns `{answer, citations}`, `AgentService` now emits a `citations` SSE event and streams that answer directly instead of routing back through `modelTurn` for re-synthesis. This guarantees `[N]` markers in the answer text stay aligned with the emitted citation array, and closes a previously dead `citations` event type in `agent-sse.types.ts` that the frontend already knew how to render.
+
+#### Frontend
+- **`useChatStream`** rewritten around a `messages: ChatMessage[]` thread and a `conversationId` ref, replacing the old single-answer `content`/`citations` state.
+- **`/chat` page** rewritten as a scrollable message thread (`MessageBubble` per turn) with a "New conversation" control (`startNewConversation()`), instead of rendering only the latest Q&A pair.
+
+### Tests
+- `agent.service.spec.ts` — citation short-circuit (single `generate()` call, correct citation payload), history seeding into the first `generate()` call, message persistence across the cited-answer/normal/proposal-pause paths.
+- `history.util.spec.ts` — hard-ceiling cap, token-budget trim, and the always-keep-latest-turn safeguard.
+
 ## JWT Auth — 2026-07-24
 
 ### Added
@@ -337,4 +360,4 @@ Format loosely follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 - Starter branding renamed `jsstack` → `docmind` across package names, Docker Compose services, env files, and CI registry paths.
 - Postgres image `postgres:16-alpine` → `pgvector/pgvector:pg16`; added missing `migrate` service.
 - Port assignments set to non-default values (backend 4500, frontend 3400, Postgres 5349, Redis 6399) to avoid VPS conflicts.
-- CI/CD split into `ci` / `build` / `deploy` workflows with EC2 deployment via WireGuard SSH.
+- CI/CD split into `ci` / `build` / `deploy` workflows with EC2 deployment via WireGuard SSH.

backend/Dockerfile

@@ -53,7 +53,7 @@ USER nestjs
 EXPOSE 4500
 
 HEALTHCHECK --interval=10s --timeout=5s --start-period=30s --retries=5 \
-  CMD node -e "require('http').get('http://localhost:4500/health', r => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"
+  CMD node -e "require('http').get('http://127.0.0.1:4500/health', r => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"
 
 ENTRYPOINT ["dumb-init", "--"]
 # Running from the workspace directory structure

backend/prisma.config.ts

@@ -1,5 +1,7 @@
 import { defineConfig } from 'prisma/config';
+import dotenv from 'dotenv';
 
+dotenv.config();
 export default defineConfig({
   schema: 'prisma/schema.prisma',
   migrations: {

backend/prisma/migrations/20260727045518_add_conversations/migration.sql

@@ -0,0 +1,108 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `content_tsv` on the `chunks` table. All the data in the column will be lost.
+
+*/
+-- CreateEnum
+CREATE TYPE "MessageRole" AS ENUM ('user', 'assistant');
+
+-- DropForeignKey
+ALTER TABLE "chunks" DROP CONSTRAINT "chunks_documentId_fkey";
+
+-- DropForeignKey
+ALTER TABLE "notes" DROP CONSTRAINT "notes_userId_fkey";
+
+-- DropForeignKey
+ALTER TABLE "query_traces" DROP CONSTRAINT "query_traces_userId_fkey";
+
+-- DropForeignKey
+ALTER TABLE "tasks" DROP CONSTRAINT "tasks_userId_fkey";
+
+-- DropForeignKey
+ALTER TABLE "tool_call_audits" DROP CONSTRAINT "tool_call_audits_userId_fkey";
+
+-- DropIndex
+DROP INDEX "chunks_content_tsv_idx";
+
+-- DropIndex
+DROP INDEX "idx_chunks_embedding_hnsw";
+
+-- AlterTable
+ALTER TABLE "chunks" DROP COLUMN "content_tsv",
+ALTER COLUMN "id" DROP DEFAULT;
+
+-- AlterTable
+ALTER TABLE "documents" ALTER COLUMN "id" DROP DEFAULT,
+ALTER COLUMN "updatedAt" DROP DEFAULT;
+
+-- AlterTable
+ALTER TABLE "notes" ALTER COLUMN "id" DROP DEFAULT,
+ALTER COLUMN "updatedAt" DROP DEFAULT;
+
+-- AlterTable
+ALTER TABLE "query_traces" ALTER COLUMN "id" DROP DEFAULT,
+ALTER COLUMN "toolCallAuditIds" DROP DEFAULT;
+
+-- AlterTable
+ALTER TABLE "tasks" ALTER COLUMN "id" DROP DEFAULT,
+ALTER COLUMN "updatedAt" DROP DEFAULT;
+
+-- AlterTable
+ALTER TABLE "tool_call_audits" ALTER COLUMN "id" DROP DEFAULT;
+
+-- AlterTable
+ALTER TABLE "users" ALTER COLUMN "id" DROP DEFAULT,
+ALTER COLUMN "updatedAt" DROP DEFAULT;
+
+-- CreateTable
+CREATE TABLE "conversations" (
+    "id" TEXT NOT NULL,
+    "userId" TEXT NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "conversations_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "messages" (
+    "id" TEXT NOT NULL,
+    "conversationId" TEXT NOT NULL,
+    "role" "MessageRole" NOT NULL,
+    "content" TEXT NOT NULL,
+    "citations" JSONB,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "messages_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "conversations_userId_updatedAt_idx" ON "conversations"("userId", "updatedAt");
+
+-- CreateIndex
+CREATE INDEX "messages_conversationId_createdAt_idx" ON "messages"("conversationId", "createdAt");
+
+-- AddForeignKey
+ALTER TABLE "chunks" ADD CONSTRAINT "chunks_documentId_fkey" FOREIGN KEY ("documentId") REFERENCES "documents"("id") ON DELETE RESTRICT ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "notes" ADD CONSTRAINT "notes_userId_fkey" FOREIGN KEY ("userId") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "tasks" ADD CONSTRAINT "tasks_userId_fkey" FOREIGN KEY ("userId") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "tool_call_audits" ADD CONSTRAINT "tool_call_audits_userId_fkey" FOREIGN KEY ("userId") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "query_traces" ADD CONSTRAINT "query_traces_userId_fkey" FOREIGN KEY ("userId") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "conversations" ADD CONSTRAINT "conversations_userId_fkey" FOREIGN KEY ("userId") REFERENCES "users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "messages" ADD CONSTRAINT "messages_conversationId_fkey" FOREIGN KEY ("conversationId") REFERENCES "conversations"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- RenameIndex
+ALTER INDEX "idx_chunks_documentId" RENAME TO "chunks_documentId_idx";

backend/prisma/migrations/20260727060000_restore_keyword_and_vector_indexes/migration.sql

@@ -0,0 +1,29 @@
+-- Defensive restore migration.
+--
+-- Root cause: content_tsv (0007_keyword_search) and idx_chunks_embedding_hnsw
+-- (0003_chunk_schema) are hand-written SQL objects that Prisma cannot express
+-- in schema.prisma (a GENERATED ALWAYS AS STORED column, and a vector HNSW
+-- index respectively). Neither was declared as an Unsupported() field/index,
+-- so a subsequent `prisma migrate dev` run diffed the live DB against
+-- schema.prisma, saw both as "not in schema", and generated DROP statements
+-- for them as part of an unrelated migration (add_conversations).
+--
+-- This migration is fully idempotent (IF NOT EXISTS / IF EXISTS guards) so
+-- it's safe to run whether or not the drop actually happened on a given
+-- database. schema.prisma now declares content_tsv as Unsupported("tsvector")
+-- to prevent this from recurring for the column; the two indexes below have
+-- no equivalent protection in Prisma and must stay hand-maintained — see the
+-- comments in schema.prisma next to the Chunk model.
+
+-- Restore the generated tsvector column for full-text search on chunks.content
+ALTER TABLE "chunks"
+  ADD COLUMN IF NOT EXISTS "content_tsv" tsvector
+    GENERATED ALWAYS AS (to_tsvector('english', content)) STORED;
+
+-- Restore the GIN index backing keyword search
+CREATE INDEX IF NOT EXISTS "chunks_content_tsv_idx"
+  ON "chunks" USING GIN ("content_tsv");
+
+-- Restore the HNSW index backing vector similarity search
+CREATE INDEX IF NOT EXISTS "idx_chunks_embedding_hnsw" ON "chunks"
+  USING hnsw ("embedding" vector_cosine_ops);

backend/prisma/migrations/migration_lock.toml

@@ -1,3 +1,3 @@
 # Please do not edit this file manually
-# It should be added in your version-control system (i.e. Git)
+# It should be added in your version-control system (e.g., Git)
 provider = "postgresql"

backend/prisma/schema.prisma

@@ -60,6 +60,16 @@ model Chunk {
   chunkIndex        Int
   embeddingProvider String   @default("gemini-embedding-001")
   embedding         Unsupported("vector(768)")
+  // Generated column created by hand-written SQL (0007_keyword_search) —
+  // Prisma can't create GENERATED ALWAYS AS columns itself. Declaring it
+  // here as Unsupported only tells Prisma "this column exists, don't try
+  // to drop it" on future `migrate dev` diffs. It does NOT protect the
+  // chunks_content_tsv_idx GIN index, or idx_chunks_embedding_hnsw above —
+  // Prisma cannot represent indexes on Unsupported-typed columns at all.
+  // Always run `prisma migrate dev --create-only` for schema changes and
+  // inspect the generated SQL for an unexpected DROP INDEX/DROP COLUMN on
+  // either of those two objects before applying.
+  contentTsv        Unsupported("tsvector")? @map("content_tsv")
   createdAt         DateTime @default(now())
 
   document Document @relation(fields: [documentId], references: [id])
@@ -86,6 +96,7 @@ model User {
   tasks         Task[]
   toolCallAudits ToolCallAudit[]
   queryTraces   QueryTrace[]
+  conversations Conversation[]
 
   @@map("users")
 }
@@ -152,3 +163,43 @@ model QueryTrace {
 
   @@map("query_traces")
 }
+
+// ── Multi-turn agent chat history ───────────────────────────────────
+// Scoped to /v1/agent/chat only (see plan doc). Tool-call/tool-result
+// scaffolding is intentionally NOT persisted here — only user turns and
+// the final assistant answer, so replayed history stays clean. Per-turn
+// tool-call detail is already captured separately in ToolCallAudit.
+
+enum MessageRole {
+  user
+  assistant
+}
+
+model Conversation {
+  id        String   @id @default(uuid())
+  userId    String
+  createdAt DateTime @default(now())
+  updatedAt DateTime @updatedAt
+
+  user     User      @relation(fields: [userId], references: [id], onDelete: Cascade)
+  messages Message[]
+
+  @@index([userId, updatedAt])
+  @@map("conversations")
+}
+
+model Message {
+  id             String      @id @default(uuid())
+  conversationId String
+  role           MessageRole
+  content        String
+  // Present only on assistant messages produced by a citation-bearing
+  // tool result (see agent.service.ts finalAnswerOverride short-circuit).
+  citations      Json?
+  createdAt      DateTime    @default(now())
+
+  conversation Conversation @relation(fields: [conversationId], references: [id], onDelete: Cascade)
+
+  @@index([conversationId, createdAt])
+  @@map("messages")
+}

backend/src/app.module.ts

@@ -14,6 +14,7 @@ import { RetrievalModule } from './modules/retrieval/retrieval.module';
 import { QueryModule } from './modules/query/query.module';
 import { ToolsModule } from './modules/tools/tools.module';
 import { AgentModule } from './modules/agent/agent.module';
+import { ConversationsModule } from './modules/conversations/conversations.module';
 import { NotesModule } from './modules/notes/notes.module';
 import { TasksModule } from './modules/tasks/tasks.module';
 import { TraceModule } from './modules/trace/trace.module';
@@ -55,6 +56,7 @@ const configValidationSchema = Joi.object({
     QueryModule,
     ToolsModule,
     AgentModule,
+    ConversationsModule,
     NotesModule,
     TasksModule,
     TraceModule,

backend/src/modules/agent/agent-sse.types.ts

@@ -1,6 +1,7 @@
 import type { ToolProposal } from '../tools/tool-proposal.type';
 
 export type AgentSseEvent =
+  | { type: 'conversation_started'; data: { conversationId: string } }
   | { type: 'token'; data: string }
   | { type: 'citations'; data: unknown[] }
   | { type: 'done'; data: string }

backend/src/modules/agent/agent.controller.ts

@@ -27,6 +27,7 @@ import { Observable, Subject } from 'rxjs';
 import type Redis from 'ioredis';
 import { REDIS_CLIENT } from '../../redis/redis.module';
 import { ToolRegistryService } from '../tools/tool-registry.service';
+import { ConversationsService } from '../conversations/conversations.service';
 import { AgentService } from './agent.service';
 import {
   CurrentUser,
@@ -44,6 +45,16 @@ export class AgentChatDto {
   @MinLength(1)
   @MaxLength(5000)
   query!: string;
+
+  @ApiProperty({
+    required: false,
+    description:
+      'Existing conversation to continue. Omit to start a new conversation ' +
+      '— the new id is returned via a conversation_started SSE event.',
+  })
+  @IsOptional()
+  @IsUUID()
+  conversationId?: string;
 }
 
 export class ConfirmDto {
@@ -71,6 +82,7 @@ export class AgentController {
   constructor(
     private readonly agentService: AgentService,
     private readonly toolRegistry: ToolRegistryService,
+    private readonly conversations: ConversationsService,
     @Optional() @Inject(REDIS_CLIENT) private readonly redis: Redis | null,
   ) {}
 
@@ -86,18 +98,40 @@ export class AgentController {
   ): Observable<SseMessage> {
     const subject = new Subject<SseMessage>();
 
-    void this.agentService
-      .run(dto.query, user.sub, (event: AgentSseEvent) => {
-        subject.next({ data: JSON.stringify(event) });
-        if (event.type === 'done' || event.type === 'error') {
-          subject.complete();
+    void (async () => {
+      try {
+        let conversationId = dto.conversationId;
+
+        if (conversationId) {
+          await this.conversations.assertOwnership(user.sub, conversationId);
+        } else {
+          const conversation = await this.conversations.create(user.sub);
+          conversationId = conversation.id;
+          subject.next({
+            data: JSON.stringify({
+              type: 'conversation_started',
+              data: { conversationId },
+            }),
+          });
         }
-      })
-      .catch((err: unknown) => {
+
+        await this.agentService.run(
+          dto.query,
+          user.sub,
+          conversationId,
+          (event: AgentSseEvent) => {
+            subject.next({ data: JSON.stringify(event) });
+            if (event.type === 'done' || event.type === 'error') {
+              subject.complete();
+            }
+          },
+        );
+      } catch (err) {
         const msg = err instanceof Error ? err.message : 'Agent error';
         subject.next({ data: JSON.stringify({ type: 'error', data: msg }) });
         subject.complete();
-      });
+      }
+    })();
 
     return subject.asObservable();
   }

backend/src/modules/agent/agent.module.ts

@@ -2,11 +2,12 @@ import { Module } from '@nestjs/common';
 import { PrismaModule } from '../../prisma/prisma.module';
 import { ProvidersModule } from '../providers/providers.module';
 import { ToolsModule } from '../tools/tools.module';
+import { ConversationsModule } from '../conversations/conversations.module';
 import { AgentController } from './agent.controller';
 import { AgentService } from './agent.service';
 
 @Module({
-  imports: [ToolsModule, ProvidersModule, PrismaModule],
+  imports: [ToolsModule, ProvidersModule, PrismaModule, ConversationsModule],
   controllers: [AgentController],
   providers: [AgentService],
   exports: [AgentService],