WIP: RAG chat cookbook with interstitial text

api/content-markdown.ts

@@ -2,13 +2,14 @@ import { existsSync, readFileSync } from "fs";
 import { resolve } from "path";
 import { hasMarkdownSlug } from "../src/lib/content-markdown";
 import { recipes, templates } from "../src/lib/recipes/recipes";
+import {
+  buildTemplateMarkdownDocument,
+  collectTemplateRecipeIds,
+  parseTemplateMarkdown,
+} from "../src/lib/template-markdown";
 
 export type MarkdownSection = "docs" | "recipes" | "solutions" | "templates";
 
-function recipeMarkdownPath(recipeId: string): string {
-  return `content/recipes/${recipeId}.md`;
-}
-
 function validateSlug(slug: string): void {
   if (!slug || slug.trim() === "") {
     throw new Error("Missing slug");
@@ -92,42 +93,46 @@ function readTemplateMarkdown(rootDir: string, slug: string): string {
     throw new Error(`Template page not found: "${slug}"`);
   }
 
-  const lines: string[] = [
-    "---",
-    `title: "${template.name.replace(/"/g, '\\"')}"`,
-    `url: /resources/${template.id}`,
-    `summary: "${template.description.replace(/"/g, '\\"')}"`,
-    "---",
-    "",
-    `# ${template.name}`,
-    "",
-    template.description,
-    "",
-  ];
-
-  for (const recipeId of template.recipeIds) {
-    const recipe = recipes.find((entry) => entry.id === recipeId);
-    if (!recipe) {
-      throw new Error(`Recipe not found: "${recipeId}"`);
-    }
-    if (!hasMarkdownSlug(rootDir, "recipes", recipeId)) {
-      throw new Error(`Recipe page not found: "${recipeId}"`);
-    }
-
-    const recipePath = recipeMarkdownPath(recipeId);
-    const absoluteRecipePath = resolve(rootDir, recipePath);
-    const recipeContent = readIfExists(absoluteRecipePath);
-    if (!recipeContent) {
+  if (template.id === "rag-chat-app-template") {
+    const cookbookPath = resolve(
+      rootDir,
+      "content",
+      "cookbooks",
+      "rag-chat-app-template.md",
+    );
+    const cookbookContent = readIfExists(cookbookPath);
+    if (!cookbookContent) {
       throw new Error(
-        `Recipe markdown missing for "${recipeId}" at ${recipePath}`,
+        `Cookbook markdown not found at content/cookbooks/rag-chat-app-template.md`,
       );
     }
+    const blocks = parseTemplateMarkdown(cookbookContent);
+    const rawBySlug = Object.fromEntries(
+      collectTemplateRecipeIds(blocks).map((recipeId) => {
+        const recipe = recipes.find((entry) => entry.id === recipeId);
+        if (!recipe) {
+          throw new Error(`Recipe not found: "${recipeId}"`);
+        }
+
+        return [recipeId, readRecipeMarkdown(rootDir, recipeId)];
+      }),
+    );
 
-    lines.push(recipeContent.trim());
-    lines.push("");
+    return buildTemplateMarkdownDocument(template, rawBySlug, blocks);
   }
 
-  return lines.join("\n");
+  const rawBySlug = Object.fromEntries(
+    template.recipeIds.map((recipeId) => {
+      const recipe = recipes.find((entry) => entry.id === recipeId);
+      if (!recipe) {
+        throw new Error(`Recipe not found: "${recipeId}"`);
+      }
+
+      return [recipeId, readRecipeMarkdown(rootDir, recipeId)];
+    }),
+  );
+
+  return buildTemplateMarkdownDocument(template, rawBySlug);
 }
 
 export function getDetailMarkdown(

content/recipes/ai-chat-model-serving.md

@@ -28,7 +28,7 @@ If you run `npm run dev` before deploying, your user creates schemas that the de
 ### 2. Install AI SDK packages
 
 ```bash
-npm install ai@6 @ai-sdk/react@3 @ai-sdk/openai @databricks/sdk-experimental
+npm install ai@6 @ai-sdk/react@3 @ai-sdk/openai@3 @databricks/sdk-experimental
 ```
 
 > **Version note**: This recipe uses AI SDK v6 APIs (`TextStreamChatTransport`, `sendMessage({ text })`, transport-based `useChat`). Tested with `ai@6.1`, `@ai-sdk/react@3.1`, and `@ai-sdk/openai@3.x`.

content/recipes/embeddings-generation.md

@@ -0,0 +1,65 @@
+## Generate Embeddings with AI Gateway
+
+Generate text embeddings from a Databricks AI Gateway endpoint using the Databricks SDK.
+
+### 1. Find an embedding endpoint
+
+```bash
+databricks serving-endpoints list --profile <PROFILE>
+```
+
+Common embedding endpoints: `databricks-gte-large-en` (1024d), `databricks-bge-large-en` (1024d).
+
+### 2. Configure environment
+
+`.env`:
+
+```bash
+DATABRICKS_EMBEDDING_ENDPOINT=databricks-gte-large-en
+```
+
+`app.yaml`:
+
+```yaml
+env:
+  - name: DATABRICKS_EMBEDDING_ENDPOINT
+    value: "databricks-gte-large-en"
+```
+
+### 3. Embedding helper
+
+Create `server/lib/embeddings.ts`:
+
+`server/lib/embeddings.ts`:
+
+```typescript
+import { getWorkspaceClient } from "@databricks/appkit";
+
+const workspaceClient = getWorkspaceClient({});
+
+export async function generateEmbedding(text: string): Promise<number[]> {
+  const endpoint =
+    process.env.DATABRICKS_EMBEDDING_ENDPOINT || "databricks-gte-large-en";
+  const result = await workspaceClient.servingEndpoints.query({
+    name: endpoint,
+    input: text,
+  });
+  return result.data![0].embedding!;
+}
+```
+
+No additional dependencies — uses `@databricks/appkit` already in your project.
+
+### 4. Verify
+
+```bash
+databricks serving-endpoints query <embedding-endpoint> \
+  --json '{"input": "Hello, world!"}' \
+  --profile <PROFILE>
+```
+
+Response includes a `data` array with `embedding` (float array).
+
+#### References
+
+- [Query embedding models](https://docs.databricks.com/aws/en/machine-learning/model-serving/query-embedding-models)

content/recipes/lakebase-chat-persistence.md

@@ -42,11 +42,75 @@ In `server/server.ts`, keep `autoStart: false` and run schema setup before `appk
 
 ### 3. Add persistence helpers
 
-Create `server/lib/chat-store.ts` and use parameterized queries:
+Create `server/lib/chat-store.ts` with table setup and parameterized query helpers:
 
 > **Getting userId**: In deployed Databricks Apps, use `req.header("x-forwarded-email")` from the request headers. For local development, use a hardcoded test user ID.
 
+`server/lib/chat-store.ts`:
+
 ```typescript
+import type { Application } from "express";
+
+interface AppKitWithLakebase {
+  lakebase: {
+    query(
+      text: string,
+      params?: unknown[],
+    ): Promise<{ rows: Record<string, unknown>[] }>;
+  };
+  server: {
+    extend(fn: (app: Application) => void): void;
+  };
+}
+
+export async function setupChatTables(appkit: AppKitWithLakebase) {
+  await appkit.lakebase.query("CREATE SCHEMA IF NOT EXISTS chat");
+  await appkit.lakebase.query(`
+    CREATE TABLE IF NOT EXISTS chat.chats (
+      id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+      user_id TEXT NOT NULL,
+      title TEXT NOT NULL,
+      created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+      updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+    )
+  `);
+  await appkit.lakebase.query(`
+    CREATE TABLE IF NOT EXISTS chat.messages (
+      id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+      chat_id UUID NOT NULL REFERENCES chat.chats(id) ON DELETE CASCADE,
+      role TEXT NOT NULL CHECK (role IN ('system', 'user', 'assistant', 'tool')),
+      content TEXT NOT NULL,
+      created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+    )
+  `);
+  try {
+    await appkit.lakebase.query(`
+      CREATE INDEX IF NOT EXISTS idx_messages_chat_id_created_at
+        ON chat.messages(chat_id, created_at)
+    `);
+  } catch (err: unknown) {
+    const code = (err as { code?: string }).code;
+    if (code === "42501") {
+      console.log(
+        "[chat] Skipping index creation — table owned by another identity",
+      );
+    } else {
+      throw err;
+    }
+  }
+}
+
+export async function listChats(appkit: AppKitWithLakebase, userId: string) {
+  const result = await appkit.lakebase.query(
+    `SELECT id, user_id, title, created_at, updated_at
+     FROM chat.chats
+     WHERE user_id = $1
+     ORDER BY updated_at DESC`,
+    [userId],
+  );
+  return result.rows;
+}
+
 export async function createChat(
   appkit: AppKitWithLakebase,
   input: { userId: string; title: string },
@@ -60,6 +124,20 @@ export async function createChat(
   return result.rows[0];
 }
 
+export async function getChatMessages(
+  appkit: AppKitWithLakebase,
+  chatId: string,
+) {
+  const result = await appkit.lakebase.query(
+    `SELECT id, chat_id, role, content, created_at
+     FROM chat.messages
+     WHERE chat_id = $1
+     ORDER BY created_at ASC`,
+    [chatId],
+  );
+  return result.rows;
+}
+
 export async function appendMessage(
   appkit: AppKitWithLakebase,
   input: { chatId: string; role: string; content: string },
@@ -70,10 +148,16 @@ export async function appendMessage(
      RETURNING id, chat_id, role, content, created_at`,
     [input.chatId, input.role, input.content],
   );
+  await appkit.lakebase.query(
+    `UPDATE chat.chats SET updated_at = NOW() WHERE id = $1`,
+    [input.chatId],
+  );
   return result.rows[0];
 }
 ```
 
+Call `setupChatTables(appkit)` from `server.ts` before starting the server. The `listChats`, `getChatMessages`, and `appendMessage` functions are imported by the chat persistence routes and chat routes in later recipes.
+
 ### 4. Persist in the `/api/chat` flow
 
 In your chat route: