From 6e5c6c869e57a2979d7727dd267e272754d215a6 Mon Sep 17 00:00:00 2001 From: sid597 Date: Tue, 6 May 2025 21:27:09 +0530 Subject: [PATCH 01/19] initial migration from bigger base --- .../components/DiscourseContextOverlay.tsx | 233 ++++++++++++++++-- 1 file changed, 208 insertions(+), 25 deletions(-) diff --git a/apps/roam/src/components/DiscourseContextOverlay.tsx b/apps/roam/src/components/DiscourseContextOverlay.tsx index a9b9284e5..18fe2bc80 100644 --- a/apps/roam/src/components/DiscourseContextOverlay.tsx +++ b/apps/roam/src/components/DiscourseContextOverlay.tsx @@ -5,6 +5,7 @@ import { Position, Tooltip, ControlGroup, + Spinner, } from "@blueprintjs/core"; import React, { useCallback, useEffect, useMemo, useState } from "react"; import ReactDOM from "react-dom"; @@ -27,6 +28,42 @@ import getAllPageNames from "roamjs-components/queries/getAllPageNames"; import { Result } from "roamjs-components/types/query-builder"; import createBlock from "roamjs-components/writes/createBlock"; import { getBlockUidFromTarget } from "roamjs-components/dom"; +import { + findSimilarNodesUsingHyde, + SuggestedNode, + generateHypotheticalNode, + CandidateNodeWithEmbedding, + SearchResultItem, + EmbeddingVector, +} from "~/utils/hyde"; + +// Placeholder for future implementation +const mockCreateEmbedding = async (text: string): Promise => { + console.warn( + `mockCreateEmbedding called for text: "${text}". Using dummy random embedding.`, + ); + // Return a dummy embedding, matching the dimension (5) used elsewhere for candidate embeddings. + return Array.from({ length: 5 }, () => Math.random()); +}; + +// Placeholder for future implementation +const mockVectorSearch = async ( + _queryEmbedding: EmbeddingVector, + candidates: CandidateNodeWithEmbedding[], + options: { topK: number } = { topK: 3 }, +): Promise => { + const { topK } = options; + const numToReturn = Math.min(topK, candidates.length); + console.warn( + `mockVectorSearch called with ${candidates.length} candidates. Returning top ${numToReturn} as dummy results.`, + ); + // For a simple mock, return the first few candidates, wrapping them as SearchResultItem. + const resultCandidates = candidates.slice(0, numToReturn); + return resultCandidates.map((node) => ({ + object: { uid: node.uid, text: node.text, type: node.type }, + score: Math.random(), // Assign a dummy score + })); +}; type DiscourseData = { results: Awaited>; @@ -87,6 +124,71 @@ const getAllReferencesOnPage = (pageTitle: string) => { })) as Result[]; }; +interface Relation { + label: string; + source: string; + destination: string; +} + +interface DiscourseNodeInfo { + format: string; + text: string; + type: string; + [key: string]: any; +} + +const getUniqueLabelTypeTriplets = ( + relations: Relation[], + selfType: string, +): [string, string, string][] => { + const uniquePairStrings = new Set(); + const separator = "::"; + + const allNodes = getDiscourseNodes(); + const nodeMapByType = new Map(); + allNodes.forEach((node) => { + const discourseNode = node as DiscourseNodeInfo; + if (discourseNode.type) { + nodeMapByType.set(discourseNode.type, discourseNode); + } + }); + + for (const relation of relations) { + if (relation.label && relation.source && relation.source !== selfType) { + uniquePairStrings.add(`${relation.label}${separator}${relation.source}`); + } + if ( + relation.label && + relation.destination && + relation.destination !== selfType + ) { + uniquePairStrings.add( + `${relation.label}${separator}${relation.destination}`, + ); + } + } + + const uniqueTriplets: [string, string, string][] = []; + Array.from(uniquePairStrings).forEach((pairString) => { + const parts = pairString.split(separator); + const label = parts[0]; + const typeIdentifier = parts[1]; + + const node = nodeMapByType.get(typeIdentifier); + + if (node) { + uniqueTriplets.push([label, node.text, node.format]); + } else { + console.warn( + `Discourse node type "${typeIdentifier}" not found for relation label "${label}".`, + ); + } + }); + + console.log("uniqueTriplets", uniqueTriplets); + return uniqueTriplets; +}; + const DiscourseContextOverlay = ({ tag, id, @@ -102,6 +204,11 @@ const DiscourseContextOverlay = ({ const [results, setResults] = useState([]); const [refs, setRefs] = useState(0); const [score, setScore] = useState(0); + const [isSearchingHyde, setIsSearchingHyde] = useState(false); + const [suggestedNodes, setSuggestedNodes] = useState([]); + const [hydeFilteredNodes, setHydeFilteredNodes] = useState( + [], + ); const discourseNode = useMemo(() => findDiscourseNode(tagUid), [tagUid]); const relations = useMemo(() => getDiscourseRelations(), []); @@ -139,11 +246,17 @@ const DiscourseContextOverlay = ({ }, [refresh, getInfo]); // Suggestive Mode - const validTypes = useMemo(() => { - if (!discourseNode) return []; + const memoizedData = useMemo(() => { + if (!discourseNode) + return { + validTypes: [] as string[], + uniqueRelationTypeTriplets: [] as [string, string, string][], + }; const selfType = discourseNode.type; const validRelations = relations.filter((relation) => - [relation.source, relation.destination].includes(selfType), + [relation.source, relation.destination, relation.label].includes( + selfType, + ), ); const hasSelfRelation = validRelations.some( (relation) => @@ -157,10 +270,19 @@ const DiscourseContextOverlay = ({ ]), ), ); - return hasSelfRelation ? types : types.filter((type) => type !== selfType); + const filteredTypes = hasSelfRelation + ? types + : types.filter((type) => type !== selfType); + + const uniqueTriplets = getUniqueLabelTypeTriplets(validRelations, selfType); + return { + validTypes: filteredTypes, + uniqueRelationTypeTriplets: uniqueTriplets, + }; }, [discourseNode, relations]); - const [suggestedNodes, setSuggestedNodes] = useState([]); + const { validTypes, uniqueRelationTypeTriplets } = memoizedData; + const [currentPageInput, setCurrentPageInput] = useState(""); const [selectedPage, setSelectedPage] = useState(null); const allPages = useMemo(() => getAllPageNames(), []); @@ -171,6 +293,7 @@ const DiscourseContextOverlay = ({ useEffect(() => { if (!selectedPage) { setSuggestedNodes([]); + setHydeFilteredNodes([]); return; } const nodesOnPage = getAllReferencesOnPage(selectedPage); @@ -178,6 +301,7 @@ const DiscourseContextOverlay = ({ .map((n) => { const node = findDiscourseNode(n.uid); if (!node || node.backedBy === "default") return null; + if (!validTypes.includes(node.type)) return null; return { uid: n.uid, text: n.text, @@ -186,17 +310,72 @@ const DiscourseContextOverlay = ({ }) .filter((node) => node !== null) .filter((node) => validTypes.includes(node.type)) - .filter((node) => !results.some((r) => Object.values(r.results).some((result) => result.uid === node.uid))); + .filter((node): node is SuggestedNode => node !== null) + .filter( + (node) => + !results.some((r) => + Object.values(r.results).some((result) => result.uid === node.uid), + ), + ); setSuggestedNodes(nodes); - }, [selectedPage, discourseNode, relations]); - const handleCreateBlock = async (node: { uid: string; text: string }) => { + if (nodes.length > 0 && uniqueRelationTypeTriplets.length > 0) { + runHydeSearch(nodes, tag, uniqueRelationTypeTriplets); + } + }, [selectedPage, results, validTypes, tag, uniqueRelationTypeTriplets]); + + const handleCreateBlock = async (nodeText: string) => { await createBlock({ parentUid: blockUid, - node: { text: `[[${node.text}]]` }, + node: { text: `[[${nodeText}]]` }, }); - setSuggestedNodes(suggestedNodes.filter((n) => n.uid !== node.uid)); + }; + + const runHydeSearch = async ( + currentSuggestions: SuggestedNode[], + currentNodeText: string, + relationTriplets: [string, string, string][], + ) => { + if ( + !currentSuggestions.length || + !currentNodeText || + !relationTriplets.length + ) { + setHydeFilteredNodes([]); + return; + } + + setIsSearchingHyde(true); + setHydeFilteredNodes([]); + + try { + const candidateNodesWithEmbeddings: CandidateNodeWithEmbedding[] = + currentSuggestions.map((node) => ({ + ...node, + embedding: Array.from({ length: 5 }, () => Math.random()), + })); + + const options = { + hypotheticalNodeGenerator: generateHypotheticalNode, + embeddingFunction: mockCreateEmbedding, + searchFunction: mockVectorSearch, + }; + + const foundNodes: SuggestedNode[] = await findSimilarNodesUsingHyde( + candidateNodesWithEmbeddings, + currentNodeText, + relationTriplets, + options, + ); + + setHydeFilteredNodes(foundNodes); + } catch (error) { + console.error("Error during HyDE search:", error); + setHydeFilteredNodes([]); + } finally { + setIsSearchingHyde(false); + } }; return ( @@ -244,23 +423,27 @@ const DiscourseContextOverlay = ({ {selectedPage && (

- Suggested Relationships + Suggested Relationships (Ranked by HyDE)

+ {isSearchingHyde && ( + + )}
    - {suggestedNodes.length > 0 ? ( - suggestedNodes.map((node) => ( -
  • - {node.text} -
    • - )) - ) : ( -
  • No relations found
    • + {!isSearchingHyde && hydeFilteredNodes.length > 0 + ? hydeFilteredNodes.map((node) => ( +
  • + {node.text} +
    • + )) + : null} + {!isSearchingHyde && hydeFilteredNodes.length === 0 && ( +
  • No relevant relations found using HyDE.
    • )}
From fbcc36c029b2de43d76918161cf58618f4809d24 Mon Sep 17 00:00:00 2001 From: sid597 Date: Tue, 6 May 2025 21:36:32 +0530 Subject: [PATCH 02/19] gemini self review --- apps/roam/src/components/DiscourseContextOverlay.tsx | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/apps/roam/src/components/DiscourseContextOverlay.tsx b/apps/roam/src/components/DiscourseContextOverlay.tsx index 18fe2bc80..19523b1d3 100644 --- a/apps/roam/src/components/DiscourseContextOverlay.tsx +++ b/apps/roam/src/components/DiscourseContextOverlay.tsx @@ -124,18 +124,18 @@ const getAllReferencesOnPage = (pageTitle: string) => { })) as Result[]; }; -interface Relation { +type Relation = { label: string; source: string; destination: string; -} +}; -interface DiscourseNodeInfo { +type DiscourseNodeInfo = { format: string; text: string; type: string; [key: string]: any; -} +}; const getUniqueLabelTypeTriplets = ( relations: Relation[], @@ -185,7 +185,6 @@ const getUniqueLabelTypeTriplets = ( } }); - console.log("uniqueTriplets", uniqueTriplets); return uniqueTriplets; }; @@ -308,8 +307,6 @@ const DiscourseContextOverlay = ({ type: node.type, }; }) - .filter((node) => node !== null) - .filter((node) => validTypes.includes(node.type)) .filter((node): node is SuggestedNode => node !== null) .filter( (node) => From 4687390d20c0d51d424fa363acf5d1808af8d638 Mon Sep 17 00:00:00 2001 From: sid597 Date: Fri, 9 May 2025 11:07:24 +0530 Subject: [PATCH 03/19] insert working getting error since it works --- .../insert/content-embedding/route.ts | 58 +++++++ apps/website/app/utils/supabase/server.ts | 43 +++++ apps/website/package.json | 1 + package-lock.json | 150 +++++++++++++++--- 4 files changed, 232 insertions(+), 20 deletions(-) create mode 100644 apps/website/app/api/supabase/insert/content-embedding/route.ts create mode 100644 apps/website/app/utils/supabase/server.ts diff --git a/apps/website/app/api/supabase/insert/content-embedding/route.ts b/apps/website/app/api/supabase/insert/content-embedding/route.ts new file mode 100644 index 000000000..12867152a --- /dev/null +++ b/apps/website/app/api/supabase/insert/content-embedding/route.ts @@ -0,0 +1,58 @@ +import { createClient } from "~/utils/supabase/server"; +import { NextResponse } from "next/server"; + +export async function POST(request: Request) { + const supabase = await createClient(); + + try { + const body = await request.json(); + const { target_id, vector, obsolete } = body; + + if ( + target_id === undefined || + vector === undefined || + obsolete === undefined + ) { + return NextResponse.json( + { error: "Missing required fields" }, + { status: 400 }, + ); + } + + if (!Array.isArray(vector) || !vector.every((v) => typeof v === "number")) { + return NextResponse.json( + { error: "Invalid vector format. Expected an array of numbers." }, + { status: 400 }, + ); + } + const vectorString = JSON.stringify(vector); + + const { data, error } = await supabase + .from("ContentEmbedding_openai_text_embedding_3_small_1536") + .insert([ + { + target_id: target_id as number, + model: "openai_text_embedding_3_small_1536", + vector: vectorString, + obsolete: obsolete as boolean, + }, + ]) + .select(); + + if (error) { + console.error("Supabase insert error:", error); + return NextResponse.json({ error: error.message }, { status: 500 }); + } + + return NextResponse.json( + { message: "Data inserted successfully", data }, + { status: 201 }, + ); + } catch (e: any) { + console.error("API route error:", e); + return NextResponse.json( + { error: e.message || "An unexpected error occurred" }, + { status: 500 }, + ); + } +} diff --git a/apps/website/app/utils/supabase/server.ts b/apps/website/app/utils/supabase/server.ts new file mode 100644 index 000000000..ef8713bf7 --- /dev/null +++ b/apps/website/app/utils/supabase/server.ts @@ -0,0 +1,43 @@ +import { createServerClient, type CookieOptions } from "@supabase/ssr"; +import { cookies } from "next/headers"; + +export async function createClient() { + const cookieStore = await cookies(); + + return createServerClient( + process.env.NEXT_PUBLIC_SUPABASE_URL!, + process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY!, + { + cookies: { + getAll() { + return cookieStore.getAll(); + }, + setAll( + cookiesToSet: { + name: string; + value: string; + options: CookieOptions; + }[], + ) { + try { + cookiesToSet.forEach( + ({ + name, + value, + options, + }: { + name: string; + value: string; + options: CookieOptions; + }) => cookieStore.set(name, value, options), + ); + } catch { + // The `setAll` method was called from a Server Component. + // This can be ignored if you have middleware refreshing + // user sessions. + } + }, + }, + }, + ); +} diff --git a/apps/website/package.json b/apps/website/package.json index cb202fc63..7c4b40d38 100644 --- a/apps/website/package.json +++ b/apps/website/package.json @@ -13,6 +13,7 @@ "@repo/types": "*", "@repo/ui": "*", "@sindresorhus/slugify": "^2.2.1", + "@supabase/ssr": "^0.6.1", "gray-matter": "^4.0.3", "next": "^15.0.3", "openai": "^4.97.0", diff --git a/package-lock.json b/package-lock.json index 8d676b83c..3e46c8126 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6744,26 +6744,6 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "apps/roam/node_modules/ws": { - "version": "8.14.2", - "license": "MIT", - "peer": true, - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, "apps/roam/node_modules/xdg-basedir": { "version": "4.0.0", "license": "MIT", @@ -6907,6 +6887,7 @@ "@repo/types": "*", "@repo/ui": "*", "@sindresorhus/slugify": "^2.2.1", + "@supabase/ssr": "^0.6.1", "gray-matter": "^4.0.3", "next": "^15.0.3", "openai": "^4.97.0", @@ -9029,6 +9010,91 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@supabase/auth-js": { + "version": "2.69.1", + "resolved": "https://registry.npmjs.org/@supabase/auth-js/-/auth-js-2.69.1.tgz", + "integrity": "sha512-FILtt5WjCNzmReeRLq5wRs3iShwmnWgBvxHfqapC/VoljJl+W8hDAyFmf1NVw3zH+ZjZ05AKxiKxVeb0HNWRMQ==", + "peer": true, + "dependencies": { + "@supabase/node-fetch": "^2.6.14" + } + }, + "node_modules/@supabase/functions-js": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/@supabase/functions-js/-/functions-js-2.4.4.tgz", + "integrity": "sha512-WL2p6r4AXNGwop7iwvul2BvOtuJ1YQy8EbOd0dhG1oN1q8el/BIRSFCFnWAMM/vJJlHWLi4ad22sKbKr9mvjoA==", + "peer": true, + "dependencies": { + "@supabase/node-fetch": "^2.6.14" + } + }, + "node_modules/@supabase/node-fetch": { + "version": "2.6.15", + "resolved": "https://registry.npmjs.org/@supabase/node-fetch/-/node-fetch-2.6.15.tgz", + "integrity": "sha512-1ibVeYUacxWYi9i0cf5efil6adJ9WRyZBLivgjs+AUpewx1F3xPi7gLgaASI2SmIQxPoCEjAsLAzKPgMJVgOUQ==", + "peer": true, + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + } + }, + "node_modules/@supabase/postgrest-js": { + "version": "1.19.4", + "resolved": "https://registry.npmjs.org/@supabase/postgrest-js/-/postgrest-js-1.19.4.tgz", + "integrity": "sha512-O4soKqKtZIW3olqmbXXbKugUtByD2jPa8kL2m2c1oozAO11uCcGrRhkZL0kVxjBLrXHE0mdSkFsMj7jDSfyNpw==", + "peer": true, + "dependencies": { + "@supabase/node-fetch": "^2.6.14" + } + }, + "node_modules/@supabase/realtime-js": { + "version": "2.11.2", + "resolved": "https://registry.npmjs.org/@supabase/realtime-js/-/realtime-js-2.11.2.tgz", + "integrity": "sha512-u/XeuL2Y0QEhXSoIPZZwR6wMXgB+RQbJzG9VErA3VghVt7uRfSVsjeqd7m5GhX3JR6dM/WRmLbVR8URpDWG4+w==", + "peer": true, + "dependencies": { + "@supabase/node-fetch": "^2.6.14", + "@types/phoenix": "^1.5.4", + "@types/ws": "^8.5.10", + "ws": "^8.18.0" + } + }, + "node_modules/@supabase/ssr": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/@supabase/ssr/-/ssr-0.6.1.tgz", + "integrity": "sha512-QtQgEMvaDzr77Mk3vZ3jWg2/y+D8tExYF7vcJT+wQ8ysuvOeGGjYbZlvj5bHYsj/SpC0bihcisnwPrM4Gp5G4g==", + "dependencies": { + "cookie": "^1.0.1" + }, + "peerDependencies": { + "@supabase/supabase-js": "^2.43.4" + } + }, + "node_modules/@supabase/storage-js": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/@supabase/storage-js/-/storage-js-2.7.1.tgz", + "integrity": "sha512-asYHcyDR1fKqrMpytAS1zjyEfvxuOIp1CIXX7ji4lHHcJKqyk+sLl/Vxgm4sN6u8zvuUtae9e4kDxQP2qrwWBA==", + "peer": true, + "dependencies": { + "@supabase/node-fetch": "^2.6.14" + } + }, + "node_modules/@supabase/supabase-js": { + "version": "2.49.4", + "resolved": "https://registry.npmjs.org/@supabase/supabase-js/-/supabase-js-2.49.4.tgz", + "integrity": "sha512-jUF0uRUmS8BKt37t01qaZ88H9yV1mbGYnqLeuFWLcdV+x1P4fl0yP9DGtaEhFPZcwSom7u16GkLEH9QJZOqOkw==", + "peer": true, + "dependencies": { + "@supabase/auth-js": "2.69.1", + "@supabase/functions-js": "2.4.4", + "@supabase/node-fetch": "2.6.15", + "@supabase/postgrest-js": "1.19.4", + "@supabase/realtime-js": "2.11.2", + "@supabase/storage-js": "2.7.1" + } + }, "node_modules/@swc/counter": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz", @@ -9301,6 +9367,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/phoenix": { + "version": "1.6.6", + "resolved": "https://registry.npmjs.org/@types/phoenix/-/phoenix-1.6.6.tgz", + "integrity": "sha512-PIzZZlEppgrpoT2QgbnDU+MMzuR6BbCjllj0bM70lWoejMeNJAxCchxnv7J3XFkI8MpygtRpzXrIlmWUBclP5A==", + "peer": true + }, "node_modules/@types/prop-types": { "version": "15.7.14", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.14.tgz", @@ -9367,6 +9439,15 @@ "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", "license": "MIT" }, + "node_modules/@types/ws": { + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==", + "peer": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "7.18.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.18.0.tgz", @@ -10950,6 +11031,14 @@ "dev": true, "license": "MIT" }, + "node_modules/cookie": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.0.2.tgz", + "integrity": "sha512-9Kr/j4O16ISv8zBBhJoi4bXOYNTkFLOqSL3UDB0njXxCXNezjeyVrJyGOWtgfs/q2km1gwBcfH8q1yEGoMYunA==", + "engines": { + "node": ">=18" + } + }, "node_modules/core-js": { "version": "3.39.0", "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.39.0.tgz", @@ -20021,6 +20110,27 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "license": "ISC" }, + "node_modules/ws": { + "version": "8.18.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.2.tgz", + "integrity": "sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==", + "peer": true, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", From 7af13860ba6553486a6ea84c06ac07970ec0877a Mon Sep 17 00:00:00 2001 From: sid597 Date: Fri, 9 May 2025 12:09:56 +0530 Subject: [PATCH 04/19] cors --- .../insert/content-embedding/route.ts | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/apps/website/app/api/supabase/insert/content-embedding/route.ts b/apps/website/app/api/supabase/insert/content-embedding/route.ts index 12867152a..697317ba2 100644 --- a/apps/website/app/api/supabase/insert/content-embedding/route.ts +++ b/apps/website/app/api/supabase/insert/content-embedding/route.ts @@ -1,8 +1,10 @@ import { createClient } from "~/utils/supabase/server"; -import { NextResponse } from "next/server"; +import { NextResponse, NextRequest } from "next/server"; +import cors from "~/utils/llm/cors"; -export async function POST(request: Request) { +export async function POST(request: NextRequest) { const supabase = await createClient(); + let response: NextResponse; try { const body = await request.json(); @@ -13,17 +15,19 @@ export async function POST(request: Request) { vector === undefined || obsolete === undefined ) { - return NextResponse.json( + response = NextResponse.json( { error: "Missing required fields" }, { status: 400 }, ); + return cors(request, response) as NextResponse; } if (!Array.isArray(vector) || !vector.every((v) => typeof v === "number")) { - return NextResponse.json( + response = NextResponse.json( { error: "Invalid vector format. Expected an array of numbers." }, { status: 400 }, ); + return cors(request, response) as NextResponse; } const vectorString = JSON.stringify(vector); @@ -41,18 +45,24 @@ export async function POST(request: Request) { if (error) { console.error("Supabase insert error:", error); - return NextResponse.json({ error: error.message }, { status: 500 }); + response = NextResponse.json({ error: error.message }, { status: 500 }); + return cors(request, response) as NextResponse; } - return NextResponse.json( + response = NextResponse.json( { message: "Data inserted successfully", data }, { status: 201 }, ); } catch (e: any) { console.error("API route error:", e); - return NextResponse.json( + response = NextResponse.json( { error: e.message || "An unexpected error occurred" }, { status: 500 }, ); } + return cors(request, response) as NextResponse; +} + +export async function OPTIONS(req: NextRequest): Promise { + return cors(req, new NextResponse(null, { status: 204 })) as NextResponse; } From 079b7c2ee1b38e70a603a2b921f8a4e8e7537488 Mon Sep 17 00:00:00 2001 From: sid597 Date: Fri, 9 May 2025 15:12:51 +0530 Subject: [PATCH 05/19] insert discoursePlatform --- .../insert/DiscoursePlatform/route.ts | 191 ++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 apps/website/app/api/supabase/insert/DiscoursePlatform/route.ts diff --git a/apps/website/app/api/supabase/insert/DiscoursePlatform/route.ts b/apps/website/app/api/supabase/insert/DiscoursePlatform/route.ts new file mode 100644 index 000000000..b44d4da05 --- /dev/null +++ b/apps/website/app/api/supabase/insert/DiscoursePlatform/route.ts @@ -0,0 +1,191 @@ +import { createClient } from "~/utils/supabase/server"; +import { NextResponse } from "next/server"; +import type { SupabaseClient } from "@supabase/supabase-js"; + +interface PlatformResult { + platform: any | null; + error: string | null; + details?: string; + created?: boolean; +} + +async function getOrCreateDiscoursePlatform( + supabase: SupabaseClient, // Using a more specific type for SupabaseClient + currentContentURL: string, +): Promise { + let platformName: string | null = null; + let platformUrl: string | null = null; + const lowerCaseURL = currentContentURL.toLowerCase(); + + if (lowerCaseURL.includes("roamresearch.com")) { + platformName = "roamresearch"; + platformUrl = "https://roamresearch.com"; // Canonical URL + } else { + console.warn("Could not determine platform from URL:", currentContentURL); + return { + error: "Could not determine platform from URL", + platform: null, + created: false, + }; + } + + if (!platformName || !platformUrl) { + return { + error: "Platform name or URL could not be derived", + platform: null, + created: false, + }; + } + + // Try to find an existing platform by its canonical URL (which should be unique) + const { data: existingPlatform, error: fetchError } = await supabase + .from("DiscoursePlatform") + .select("id, name, url") + .eq("url", platformUrl) + .maybeSingle(); + + if (fetchError) { + console.error("Error fetching DiscoursePlatform:", fetchError); + return { + error: "Database error while fetching platform", + platform: null, + details: fetchError.message, + created: false, + }; + } + + if (existingPlatform) { + console.log("Found existing DiscoursePlatform:", existingPlatform); + return { error: null, platform: existingPlatform, created: false }; + } else { + console.log( + `DiscoursePlatform "${platformName}" (URL: ${platformUrl}) not found, creating new one...`, + ); + + const platformToInsert = { + name: platformName, + url: platformUrl, + }; + + const { data: newPlatform, error: insertError } = await supabase + .from("DiscoursePlatform") + .insert(platformToInsert) + .select() + .single(); // Expecting one row to be inserted and returned + + if (insertError) { + console.error("Error inserting new DiscoursePlatform:", insertError); + // Handle potential race condition where platform was created between fetch and insert + if (insertError.code === "23505") { + // Unique constraint violation + console.warn( + "Unique constraint violation on insert. Attempting to re-fetch platform by URL.", + ); + const { data: reFetchedPlatform, error: reFetchError } = await supabase + .from("DiscoursePlatform") + .select("id, name, url") + .eq("url", platformUrl) + .maybeSingle(); + + if (reFetchError) { + console.error( + "Error re-fetching DiscoursePlatform after unique constraint violation:", + reFetchError, + ); + return { + error: "Database error after unique constraint violation", + platform: null, + details: reFetchError.message, + created: false, + }; + } + if (reFetchedPlatform) { + console.log("Found platform on re-fetch:", reFetchedPlatform); + return { error: null, platform: reFetchedPlatform, created: false }; // It existed, wasn't "created" by this call + } + // If re-fetch also fails to find it, the original insert error stands + return { + error: + "Unique constraint violation on insert, and re-fetch failed to find the platform.", + platform: null, + details: insertError.message, + created: false, + }; + } + return { + error: "Database error while inserting platform", + platform: null, + details: insertError.message, + created: false, + }; + } + + console.log("Created new DiscoursePlatform:", newPlatform); + return { error: null, platform: newPlatform, created: true }; + } +} + +export async function POST(request: Request) { + const supabase = await createClient(); // Creates a server-side Supabase client + + try { + const body = await request.json(); + const { currentContentURL } = body; + + if (!currentContentURL || typeof currentContentURL !== "string") { + return NextResponse.json( + { error: "Missing or invalid currentContentURL in request body" }, + { status: 400 }, + ); + } + + const { platform, error, details, created } = + await getOrCreateDiscoursePlatform(supabase, currentContentURL); + + if (error) { + console.error( + `API Error for DiscoursePlatform (URL: ${currentContentURL}): ${error}`, + details || "", + ); + return NextResponse.json( + { error: error, details: details }, + { status: 500 }, + ); + } + + if (platform) { + return NextResponse.json(platform, { status: created ? 201 : 200 }); + } else { + // This case should ideally be caught by the 'error' field in the result + console.error( + `API Error for DiscoursePlatform (URL: ${currentContentURL}): Platform was null without an error flag.`, + ); + return NextResponse.json( + { + error: + "Failed to get or create DiscoursePlatform for an unknown reason", + }, + { status: 500 }, + ); + } + } catch (e: any) { + console.error( + "API route error in /api/supabase/insert/DiscoursePlatform:", + e, + ); + // Differentiate between JSON parsing errors and other errors + if (e instanceof SyntaxError && e.message.includes("JSON")) { + return NextResponse.json( + { error: "Invalid JSON in request body" }, + { status: 400 }, + ); + } + return NextResponse.json( + { + error: + e.message || "An unexpected error occurred processing your request", + }, + { status: 500 }, + ); + } +} From d171298f467113f96679b2404f7ba84048cab224 Mon Sep 17 00:00:00 2001 From: sid597 Date: Fri, 9 May 2025 15:26:25 +0530 Subject: [PATCH 06/19] api for inserting discourseSpace --- .../supabase/insert/DiscourseSpace/route.ts | 212 ++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 apps/website/app/api/supabase/insert/DiscourseSpace/route.ts diff --git a/apps/website/app/api/supabase/insert/DiscourseSpace/route.ts b/apps/website/app/api/supabase/insert/DiscourseSpace/route.ts new file mode 100644 index 000000000..8f54d35fd --- /dev/null +++ b/apps/website/app/api/supabase/insert/DiscourseSpace/route.ts @@ -0,0 +1,212 @@ +import { createClient } from "~/utils/supabase/server"; +import { NextResponse } from "next/server"; +import type { SupabaseClient } from "@supabase/supabase-js"; + +interface DiscourseSpaceData { + name: string; + url: string; + discourse_platform_id: number; +} + +interface DiscourseSpaceResult { + space: any | null; + error: string | null; + details?: string; + created?: boolean; +} + +async function getOrCreateDiscourseSpace( + supabase: SupabaseClient, + name: string, + url: string, + discoursePlatformId: number, +): Promise { + if ( + !name || + !url || + discoursePlatformId === undefined || + discoursePlatformId === null + ) { + return { + error: "DiscourseSpace name, URL, and discourse_platform_id are required", + space: null, + created: false, + }; + } + + const normalizedUrl = url.replace(/\/$/, ""); + + const { data: existingSpace, error: fetchError } = await supabase + .from("DiscourseSpace") + .select("id, name, url, discourse_platform_id") + .eq("url", normalizedUrl) + .eq("discourse_platform_id", discoursePlatformId) + .maybeSingle(); + + if (fetchError) { + console.error( + `Error fetching DiscourseSpace (URL: ${normalizedUrl}, PlatformID: ${discoursePlatformId}):`, + fetchError, + ); + return { + error: "Database error while fetching DiscourseSpace", + space: null, + details: fetchError.message, + created: false, + }; + } + + if (existingSpace) { + console.log("Found existing DiscourseSpace:", existingSpace); + return { error: null, space: existingSpace, created: false }; + } else { + console.log( + `DiscourseSpace "${name}" (URL: ${normalizedUrl}, PlatformID: ${discoursePlatformId}) not found, creating new one...`, + ); + + const spaceToInsert = { + name: name, + url: normalizedUrl, + discourse_platform_id: discoursePlatformId, + }; + + const { data: newSpace, error: insertError } = await supabase + .from("DiscourseSpace") + .insert(spaceToInsert) + .select("id, name, url, discourse_platform_id") + .single(); + + if (insertError) { + console.error( + `Error inserting new DiscourseSpace (Name: ${name}, URL: ${normalizedUrl}, PlatformID: ${discoursePlatformId}):`, + insertError, + ); + if (insertError.code === "23505") { + console.warn( + "Unique constraint violation on insert. Attempting to re-fetch DiscourseSpace.", + ); + const { data: reFetchedSpace, error: reFetchError } = await supabase + .from("DiscourseSpace") + .select("id, name, url, discourse_platform_id") + .eq("url", normalizedUrl) + .eq("discourse_platform_id", discoursePlatformId) + .maybeSingle(); + + if (reFetchError) { + console.error( + "Error re-fetching DiscourseSpace after unique constraint violation:", + reFetchError, + ); + return { + error: "Database error after unique constraint violation", + space: null, + details: reFetchError.message, + created: false, + }; + } + if (reFetchedSpace) { + console.log("Found DiscourseSpace on re-fetch:", reFetchedSpace); + return { error: null, space: reFetchedSpace, created: false }; + } + return { + error: + "Unique constraint violation on insert, and re-fetch failed to find the DiscourseSpace.", + space: null, + details: insertError.message, + created: false, + }; + } + return { + error: "Database error while inserting DiscourseSpace", + space: null, + details: insertError.message, + created: false, + }; + } + + console.log("Created new DiscourseSpace:", newSpace); + return { error: null, space: newSpace, created: true }; + } +} + +export async function POST(request: Request) { + const supabase = await createClient(); + + try { + const body: DiscourseSpaceData = await request.json(); + const { name, url, discourse_platform_id } = body; + + if (!name || typeof name !== "string" || name.trim() === "") { + return NextResponse.json( + { error: "Missing or invalid name in request body" }, + { status: 400 }, + ); + } + if (!url || typeof url !== "string" || url.trim() === "") { + return NextResponse.json( + { error: "Missing or invalid url in request body" }, + { status: 400 }, + ); + } + if ( + discourse_platform_id === undefined || + discourse_platform_id === null || + typeof discourse_platform_id !== "number" + ) { + return NextResponse.json( + { error: "Missing or invalid discourse_platform_id in request body" }, + { status: 400 }, + ); + } + + const { space, error, details, created } = await getOrCreateDiscourseSpace( + supabase, + name.trim(), + url.trim(), + discourse_platform_id, + ); + + if (error) { + console.error( + `API Error for DiscourseSpace (Name: ${name}, URL: ${url}, PlatformID: ${discourse_platform_id}): ${error}`, + details || "", + ); + const clientError = error.startsWith("Database error") + ? "An internal error occurred while processing the DiscourseSpace information." + : error; + return NextResponse.json( + { + error: clientError, + details: error.startsWith("Database error") ? undefined : details, + }, + { status: 500 }, + ); + } + + if (space) { + return NextResponse.json(space, { status: created ? 201 : 200 }); + } else { + console.error( + `API Error for DiscourseSpace (Name: ${name}, URL: ${url}, PlatformID: ${discourse_platform_id}): Space was null without an error flag.`, + ); + return NextResponse.json( + { + error: "Failed to get or create DiscourseSpace for an unknown reason", + }, + { status: 500 }, + ); + } + } catch (e: any) { + console.error("API route error in /api/supabase/insert/DiscourseSpace:", e); + if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { + return NextResponse.json( + { error: "Invalid JSON in request body" }, + { status: 400 }, + ); + } + return NextResponse.json( + { error: "An unexpected error occurred processing your request" }, + { status: 500 }, + ); + } +} From d8226474e9f632e255f535497a7bd48d69ad78d7 Mon Sep 17 00:00:00 2001 From: sid597 Date: Fri, 9 May 2025 16:07:10 +0530 Subject: [PATCH 07/19] person api --- .../app/api/supabase/insert/Person/route.ts | 283 ++++++++++++++++++ 1 file changed, 283 insertions(+) create mode 100644 apps/website/app/api/supabase/insert/Person/route.ts diff --git a/apps/website/app/api/supabase/insert/Person/route.ts b/apps/website/app/api/supabase/insert/Person/route.ts new file mode 100644 index 000000000..37b6d0253 --- /dev/null +++ b/apps/website/app/api/supabase/insert/Person/route.ts @@ -0,0 +1,283 @@ +import { createClient } from "@/utils/supabase/server"; // Using the previously established path alias +import { NextResponse } from "next/server"; +import type { SupabaseClient } from "@supabase/supabase-js"; + +// From LinkML and visual schema +interface PersonDataInput { + name: string; + email: string; + orcid?: string | null; + person_type?: string; // Corresponds to Agent.type, defaults to "Person" + account_platform_id: number; // DiscoursePlatform.id for the account + account_active?: boolean; // Defaults to true + account_write_permission?: boolean; // From visual schema, optional +} + +interface PersonResult { + person: any | null; + account: any | null; + error: string | null; + details?: string; + person_created?: boolean; + account_created?: boolean; +} + +// Helper function to get or create a Person +async function getOrCreatePerson( + supabase: SupabaseClient, + email: string, + name: string, + orcid: string | null | undefined, + personType: string, +): Promise<{ + person: any | null; + error: string | null; + details?: string; + created: boolean; +}> { + // Try to find an existing person by email + let { data: existingPerson, error: fetchError } = await supabase + .from("Person") + .select("id, name, email, orcid, type") // Assuming 'type' column exists on Person table for Agent.type + .eq("email", email) + .maybeSingle(); + + if (fetchError) { + console.error(`Error fetching Person by email (${email}):`, fetchError); + return { + person: null, + error: "Database error while fetching Person", + details: fetchError.message, + created: false, + }; + } + + if (existingPerson) { + console.log("Found existing Person:", existingPerson); + // Optionally, update name or orcid if they differ and are provided? For now, just return existing. + return { person: existingPerson, error: null, created: false }; + } else { + console.log(`Person with email "${email}" not found, creating new one...`); + const personToInsert = { + email: email, + name: name, + orcid: orcid, + type: personType, // Set the Agent type + }; + const { data: newPerson, error: insertError } = await supabase + .from("Person") + .insert(personToInsert) + .select("id, name, email, orcid, type") + .single(); + + if (insertError) { + console.error( + `Error inserting new Person (email: ${email}):`, + insertError, + ); + return { + person: null, + error: "Database error while inserting Person", + details: insertError.message, + created: false, + }; + } + console.log("Created new Person:", newPerson); + return { person: newPerson, error: null, created: true }; + } +} + +// Helper function to get or create an Account for a Person +async function getOrCreateAccount( + supabase: SupabaseClient, + personId: number, + platformId: number, + isActive: boolean, + writePermission?: boolean, // Optional based on visual schema +): Promise<{ + account: any | null; + error: string | null; + details?: string; + created: boolean; +}> { + let { data: existingAccount, error: fetchError } = await supabase + .from("Account") + .select("id, person_id, platform_id, active, write_permission") // 'platform_id' from visual schema + .eq("person_id", personId) + .eq("platform_id", platformId) + .maybeSingle(); + + if (fetchError) { + console.error( + `Error fetching Account (PersonID: ${personId}, PlatformID: ${platformId}):`, + fetchError, + ); + return { + account: null, + error: "Database error while fetching Account", + details: fetchError.message, + created: false, + }; + } + + if (existingAccount) { + console.log("Found existing Account:", existingAccount); + // Optionally, update active or write_permission status if needed? For now, just return existing. + return { account: existingAccount, error: null, created: false }; + } else { + console.log( + `Account for PersonID ${personId} on PlatformID ${platformId} not found, creating new one...`, + ); + const accountToInsert: any = { + person_id: personId, + platform_id: platformId, + active: isActive, + }; + if (writePermission !== undefined) { + accountToInsert.write_permission = writePermission; + } + + const { data: newAccount, error: insertError } = await supabase + .from("Account") + .insert(accountToInsert) + .select("id, person_id, platform_id, active, write_permission") + .single(); + + if (insertError) { + console.error( + `Error inserting new Account (PersonID: ${personId}, PlatformID: ${platformId}):`, + insertError, + ); + return { + account: null, + error: "Database error while inserting Account", + details: insertError.message, + created: false, + }; + } + console.log("Created new Account:", newAccount); + return { account: newAccount, error: null, created: true }; + } +} + +export async function POST(request: Request) { + const supabase = await createClient(); + + try { + const body: PersonDataInput = await request.json(); + const { + name, + email, + orcid = null, // Default to null if not provided + person_type = "Person", // Default Agent.type + account_platform_id, + account_active = true, // Default as per LinkML `ifabsent: true` + account_write_permission, // Optional + } = body; + + // Validate required fields for Person + if (!name || typeof name !== "string" || name.trim() === "") { + return NextResponse.json( + { error: "Missing or invalid name for Person" }, + { status: 400 }, + ); + } + if (!email || typeof email !== "string" || email.trim() === "") { + // Basic email validation could be added + return NextResponse.json( + { error: "Missing or invalid email for Person" }, + { status: 400 }, + ); + } + // Validate required fields for Account + if ( + account_platform_id === undefined || + account_platform_id === null || + typeof account_platform_id !== "number" + ) { + return NextResponse.json( + { error: "Missing or invalid account_platform_id for Account" }, + { status: 400 }, + ); + } + + // Step 1: Get or Create Person + const personResult = await getOrCreatePerson( + supabase, + email.trim(), + name.trim(), + orcid, + person_type, + ); + + if (personResult.error || !personResult.person) { + console.error( + `API Error during Person processing (Email: ${email}): ${personResult.error}`, + personResult.details || "", + ); + const clientError = personResult.error?.startsWith("Database error") + ? "An internal error occurred while processing Person." + : personResult.error; + return NextResponse.json( + { error: clientError, details: personResult.details }, + { status: 500 }, + ); + } + + // Step 2: Get or Create Account for this Person on the specified platform + const accountResult = await getOrCreateAccount( + supabase, + personResult.person.id, + account_platform_id, + account_active, + account_write_permission, + ); + + if (accountResult.error || !accountResult.account) { + console.error( + `API Error during Account processing (PersonID: ${personResult.person.id}, PlatformID: ${account_platform_id}): ${accountResult.error}`, + accountResult.details || "", + ); + // If person was just created, should we roll back or leave orphaned? For now, report error. + const clientError = accountResult.error?.startsWith("Database error") + ? "An internal error occurred while processing Account." + : accountResult.error; + return NextResponse.json( + { + error: clientError, + details: accountResult.details, + person: personResult.person, // Return person info even if account fails, for context + }, + { status: 500 }, + ); + } + + // Determine overall status code + // If both were created, 201. If one was created and other existed, still 201 for the "overall new entity" feel. + // If both existed, 200. + const statusCode = + personResult.created || accountResult.created ? 201 : 200; + + return NextResponse.json( + { + person: personResult.person, + account: accountResult.account, + person_created: personResult.created, + account_created: accountResult.created, + }, + { status: statusCode }, + ); + } catch (e: any) { + console.error("API route error in /api/supabase/insert/Person:", e); + if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { + return NextResponse.json( + { error: "Invalid JSON in request body" }, + { status: 400 }, + ); + } + return NextResponse.json( + { error: "An unexpected error occurred processing your request" }, + { status: 500 }, + ); + } +} From 5c0ed3ff2d88dda41c35140071754852581bbe2f Mon Sep 17 00:00:00 2001 From: sid597 Date: Fri, 9 May 2025 16:19:11 +0530 Subject: [PATCH 08/19] contentEmbedding route --- .../supabase/insert/ContentEmbedding/route.ts | 185 ++++++++++++++++++ .../insert/content-embedding/route.ts | 68 ------- 2 files changed, 185 insertions(+), 68 deletions(-) create mode 100644 apps/website/app/api/supabase/insert/ContentEmbedding/route.ts delete mode 100644 apps/website/app/api/supabase/insert/content-embedding/route.ts diff --git a/apps/website/app/api/supabase/insert/ContentEmbedding/route.ts b/apps/website/app/api/supabase/insert/ContentEmbedding/route.ts new file mode 100644 index 000000000..82982767d --- /dev/null +++ b/apps/website/app/api/supabase/insert/ContentEmbedding/route.ts @@ -0,0 +1,185 @@ +import { createClient } from "@/utils/supabase/server"; +import { NextResponse } from "next/server"; +import type { SupabaseClient } from "@supabase/supabase-js"; + +// Based on LinkML for Embedding +interface ContentEmbeddingDataInput { + target_id: number; // Foreign Key to Content.id + model: string; // e.g., "openai_txt_3_small_1536" (from EmbeddingName enum) + vector: number[]; // Array of numbers representing the embedding + obsolete?: boolean; // Defaults to false +} + +interface ContentEmbeddingResult { + embedding: any | null; + error: string | null; + details?: string; +} + +// Define the target table name based on your context. +// If you have a generic "ContentEmbedding" table where the "model" column distinguishes types, +// you'd use that. If it's specific like the one mentioned, use that. +const TARGET_EMBEDDING_TABLE = + "ContentEmbedding_openai_text_embedding_3_small_1536"; + +async function createContentEmbeddingEntry( + supabase: SupabaseClient, + data: ContentEmbeddingDataInput, +): Promise { + const { + target_id, + model, + vector, + obsolete = false, // Default from LinkML (ifabsent: false) + } = data; + + if (target_id === undefined || target_id === null || !model || !vector) { + return { + embedding: null, + error: "Missing required fields: target_id, model, or vector", + }; + } + + if (!Array.isArray(vector) || !vector.every((v) => typeof v === "number")) { + return { + embedding: null, + error: "Invalid vector format. Expected an array of numbers.", + }; + } + + // Supabase vector type usually expects a string representation like '[0.1,0.2,0.3]' + const vectorString = JSON.stringify(vector); + + const embeddingToInsert = { + target_id, + model, + vector: vectorString, + obsolete, + }; + + const { data: newEmbedding, error: insertError } = await supabase + .from(TARGET_EMBEDDING_TABLE) + .insert(embeddingToInsert) + .select() // Select all columns of the newly inserted row + .single(); + + if (insertError) { + console.error( + `Error inserting new ContentEmbedding into ${TARGET_EMBEDDING_TABLE}:`, + insertError, + ); + // Check for foreign key violation (target_id not in Content table) + if ( + insertError.code === "23503" && + insertError.message.includes("target_id_fkey") + ) { + // Or the specific FK name + return { + embedding: null, + error: `Invalid target_id: No Content record found for ID ${target_id}.`, + details: insertError.message, + }; + } + return { + embedding: null, + error: "Database error while inserting ContentEmbedding", + details: insertError.message, + }; + } + + console.log( + `Created new ContentEmbedding in ${TARGET_EMBEDDING_TABLE}:`, + newEmbedding, + ); + return { embedding: newEmbedding, error: null }; +} + +export async function POST(request: Request) { + const supabase = await createClient(); + + try { + const body: ContentEmbeddingDataInput = await request.json(); + + // Basic validation + if ( + body.target_id === undefined || + body.target_id === null || + typeof body.target_id !== "number" + ) { + return NextResponse.json( + { error: "Missing or invalid target_id" }, + { status: 400 }, + ); + } + if (!body.model || typeof body.model !== "string") { + // TODO: Validate against EmbeddingName enum + return NextResponse.json( + { error: "Missing or invalid model name" }, + { status: 400 }, + ); + } + if ( + !body.vector || + !Array.isArray(body.vector) || + !body.vector.every((v) => typeof v === "number") + ) { + return NextResponse.json( + { error: "Missing or invalid vector. Must be an array of numbers." }, + { status: 400 }, + ); + } + if (body.obsolete !== undefined && typeof body.obsolete !== "boolean") { + return NextResponse.json( + { error: "Invalid type for obsolete. Must be a boolean." }, + { status: 400 }, + ); + } + + const { embedding, error, details } = await createContentEmbeddingEntry( + supabase, + body, + ); + + if (error) { + console.error( + `API Error for ContentEmbedding creation: ${error}`, + details || "", + ); + // If it's a known client-side error (like invalid target_id), return 400 + if ( + error.startsWith("Invalid target_id") || + error.startsWith("Invalid vector format") + ) { + return NextResponse.json( + { error: error, details: details }, + { status: 400 }, + ); + } + // Otherwise, more likely a server/DB issue + const clientError = error.startsWith("Database error") + ? "An internal error occurred while processing ContentEmbedding." + : error; + return NextResponse.json( + { error: clientError, details: details }, + { status: 500 }, + ); + } + + return NextResponse.json(embedding, { status: 201 }); // 201 Created + } catch (e: any) { + console.error( + "API route error in /api/supabase/insert/ContentEmbedding:", + e, + ); + if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { + return NextResponse.json( + { error: "Invalid JSON in request body" }, + { status: 400 }, + ); + } + return NextResponse.json( + { error: "An unexpected error occurred processing your request" }, + { status: 500 }, + ); + } +} diff --git a/apps/website/app/api/supabase/insert/content-embedding/route.ts b/apps/website/app/api/supabase/insert/content-embedding/route.ts deleted file mode 100644 index 697317ba2..000000000 --- a/apps/website/app/api/supabase/insert/content-embedding/route.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { createClient } from "~/utils/supabase/server"; -import { NextResponse, NextRequest } from "next/server"; -import cors from "~/utils/llm/cors"; - -export async function POST(request: NextRequest) { - const supabase = await createClient(); - let response: NextResponse; - - try { - const body = await request.json(); - const { target_id, vector, obsolete } = body; - - if ( - target_id === undefined || - vector === undefined || - obsolete === undefined - ) { - response = NextResponse.json( - { error: "Missing required fields" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; - } - - if (!Array.isArray(vector) || !vector.every((v) => typeof v === "number")) { - response = NextResponse.json( - { error: "Invalid vector format. Expected an array of numbers." }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; - } - const vectorString = JSON.stringify(vector); - - const { data, error } = await supabase - .from("ContentEmbedding_openai_text_embedding_3_small_1536") - .insert([ - { - target_id: target_id as number, - model: "openai_text_embedding_3_small_1536", - vector: vectorString, - obsolete: obsolete as boolean, - }, - ]) - .select(); - - if (error) { - console.error("Supabase insert error:", error); - response = NextResponse.json({ error: error.message }, { status: 500 }); - return cors(request, response) as NextResponse; - } - - response = NextResponse.json( - { message: "Data inserted successfully", data }, - { status: 201 }, - ); - } catch (e: any) { - console.error("API route error:", e); - response = NextResponse.json( - { error: e.message || "An unexpected error occurred" }, - { status: 500 }, - ); - } - return cors(request, response) as NextResponse; -} - -export async function OPTIONS(req: NextRequest): Promise { - return cors(req, new NextResponse(null, { status: 204 })) as NextResponse; -} From 205074cf940c3afb19f087a296f7a5a945bb42a5 Mon Sep 17 00:00:00 2001 From: sid597 Date: Fri, 9 May 2025 16:23:04 +0530 Subject: [PATCH 09/19] contentEmbedding route --- .../supabase/insert/ContentEmbedding/route.ts | 31 +++++-------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/apps/website/app/api/supabase/insert/ContentEmbedding/route.ts b/apps/website/app/api/supabase/insert/ContentEmbedding/route.ts index 82982767d..334bfb954 100644 --- a/apps/website/app/api/supabase/insert/ContentEmbedding/route.ts +++ b/apps/website/app/api/supabase/insert/ContentEmbedding/route.ts @@ -1,13 +1,13 @@ -import { createClient } from "@/utils/supabase/server"; +import { createClient } from "~/utils/supabase/server"; import { NextResponse } from "next/server"; import type { SupabaseClient } from "@supabase/supabase-js"; // Based on LinkML for Embedding interface ContentEmbeddingDataInput { - target_id: number; // Foreign Key to Content.id - model: string; // e.g., "openai_txt_3_small_1536" (from EmbeddingName enum) - vector: number[]; // Array of numbers representing the embedding - obsolete?: boolean; // Defaults to false + target_id: number; + model: string; + vector: number[]; + obsolete?: boolean; } interface ContentEmbeddingResult { @@ -16,9 +16,6 @@ interface ContentEmbeddingResult { details?: string; } -// Define the target table name based on your context. -// If you have a generic "ContentEmbedding" table where the "model" column distinguishes types, -// you'd use that. If it's specific like the one mentioned, use that. const TARGET_EMBEDDING_TABLE = "ContentEmbedding_openai_text_embedding_3_small_1536"; @@ -26,12 +23,7 @@ async function createContentEmbeddingEntry( supabase: SupabaseClient, data: ContentEmbeddingDataInput, ): Promise { - const { - target_id, - model, - vector, - obsolete = false, // Default from LinkML (ifabsent: false) - } = data; + const { target_id, model, vector, obsolete = false } = data; if (target_id === undefined || target_id === null || !model || !vector) { return { @@ -47,7 +39,6 @@ async function createContentEmbeddingEntry( }; } - // Supabase vector type usually expects a string representation like '[0.1,0.2,0.3]' const vectorString = JSON.stringify(vector); const embeddingToInsert = { @@ -60,7 +51,7 @@ async function createContentEmbeddingEntry( const { data: newEmbedding, error: insertError } = await supabase .from(TARGET_EMBEDDING_TABLE) .insert(embeddingToInsert) - .select() // Select all columns of the newly inserted row + .select() .single(); if (insertError) { @@ -68,12 +59,10 @@ async function createContentEmbeddingEntry( `Error inserting new ContentEmbedding into ${TARGET_EMBEDDING_TABLE}:`, insertError, ); - // Check for foreign key violation (target_id not in Content table) if ( insertError.code === "23503" && insertError.message.includes("target_id_fkey") ) { - // Or the specific FK name return { embedding: null, error: `Invalid target_id: No Content record found for ID ${target_id}.`, @@ -100,7 +89,6 @@ export async function POST(request: Request) { try { const body: ContentEmbeddingDataInput = await request.json(); - // Basic validation if ( body.target_id === undefined || body.target_id === null || @@ -112,7 +100,6 @@ export async function POST(request: Request) { ); } if (!body.model || typeof body.model !== "string") { - // TODO: Validate against EmbeddingName enum return NextResponse.json( { error: "Missing or invalid model name" }, { status: 400 }, @@ -145,7 +132,6 @@ export async function POST(request: Request) { `API Error for ContentEmbedding creation: ${error}`, details || "", ); - // If it's a known client-side error (like invalid target_id), return 400 if ( error.startsWith("Invalid target_id") || error.startsWith("Invalid vector format") @@ -155,7 +141,6 @@ export async function POST(request: Request) { { status: 400 }, ); } - // Otherwise, more likely a server/DB issue const clientError = error.startsWith("Database error") ? "An internal error occurred while processing ContentEmbedding." : error; @@ -165,7 +150,7 @@ export async function POST(request: Request) { ); } - return NextResponse.json(embedding, { status: 201 }); // 201 Created + return NextResponse.json(embedding, { status: 201 }); } catch (e: any) { console.error( "API route error in /api/supabase/insert/ContentEmbedding:", From 709aa575b2998326f76af4a567a90a8e1cc56145 Mon Sep 17 00:00:00 2001 From: sid597 Date: Fri, 9 May 2025 22:22:42 +0530 Subject: [PATCH 10/19] cmts --- .../app/api/supabase/insert/Person/route.ts | 43 ++++++------------- 1 file changed, 14 insertions(+), 29 deletions(-) diff --git a/apps/website/app/api/supabase/insert/Person/route.ts b/apps/website/app/api/supabase/insert/Person/route.ts index 37b6d0253..2fe8cab98 100644 --- a/apps/website/app/api/supabase/insert/Person/route.ts +++ b/apps/website/app/api/supabase/insert/Person/route.ts @@ -1,16 +1,15 @@ -import { createClient } from "@/utils/supabase/server"; // Using the previously established path alias +import { createClient } from "~/utils/supabase/server"; import { NextResponse } from "next/server"; import type { SupabaseClient } from "@supabase/supabase-js"; -// From LinkML and visual schema interface PersonDataInput { name: string; email: string; orcid?: string | null; - person_type?: string; // Corresponds to Agent.type, defaults to "Person" - account_platform_id: number; // DiscoursePlatform.id for the account - account_active?: boolean; // Defaults to true - account_write_permission?: boolean; // From visual schema, optional + person_type?: string; + account_platform_id: number; + account_active?: boolean; + account_write_permission?: boolean; } interface PersonResult { @@ -22,7 +21,6 @@ interface PersonResult { account_created?: boolean; } -// Helper function to get or create a Person async function getOrCreatePerson( supabase: SupabaseClient, email: string, @@ -35,10 +33,9 @@ async function getOrCreatePerson( details?: string; created: boolean; }> { - // Try to find an existing person by email let { data: existingPerson, error: fetchError } = await supabase .from("Person") - .select("id, name, email, orcid, type") // Assuming 'type' column exists on Person table for Agent.type + .select("id, name, email, orcid, type") .eq("email", email) .maybeSingle(); @@ -54,7 +51,6 @@ async function getOrCreatePerson( if (existingPerson) { console.log("Found existing Person:", existingPerson); - // Optionally, update name or orcid if they differ and are provided? For now, just return existing. return { person: existingPerson, error: null, created: false }; } else { console.log(`Person with email "${email}" not found, creating new one...`); @@ -62,7 +58,7 @@ async function getOrCreatePerson( email: email, name: name, orcid: orcid, - type: personType, // Set the Agent type + type: personType, }; const { data: newPerson, error: insertError } = await supabase .from("Person") @@ -87,13 +83,12 @@ async function getOrCreatePerson( } } -// Helper function to get or create an Account for a Person async function getOrCreateAccount( supabase: SupabaseClient, personId: number, platformId: number, isActive: boolean, - writePermission?: boolean, // Optional based on visual schema + writePermission?: boolean, ): Promise<{ account: any | null; error: string | null; @@ -102,7 +97,7 @@ async function getOrCreateAccount( }> { let { data: existingAccount, error: fetchError } = await supabase .from("Account") - .select("id, person_id, platform_id, active, write_permission") // 'platform_id' from visual schema + .select("id, person_id, platform_id, active, write_permission") .eq("person_id", personId) .eq("platform_id", platformId) .maybeSingle(); @@ -122,7 +117,6 @@ async function getOrCreateAccount( if (existingAccount) { console.log("Found existing Account:", existingAccount); - // Optionally, update active or write_permission status if needed? For now, just return existing. return { account: existingAccount, error: null, created: false }; } else { console.log( @@ -168,14 +162,13 @@ export async function POST(request: Request) { const { name, email, - orcid = null, // Default to null if not provided - person_type = "Person", // Default Agent.type + orcid = null, + person_type = "Person", account_platform_id, - account_active = true, // Default as per LinkML `ifabsent: true` - account_write_permission, // Optional + account_active = true, + account_write_permission, } = body; - // Validate required fields for Person if (!name || typeof name !== "string" || name.trim() === "") { return NextResponse.json( { error: "Missing or invalid name for Person" }, @@ -183,13 +176,11 @@ export async function POST(request: Request) { ); } if (!email || typeof email !== "string" || email.trim() === "") { - // Basic email validation could be added return NextResponse.json( { error: "Missing or invalid email for Person" }, { status: 400 }, ); } - // Validate required fields for Account if ( account_platform_id === undefined || account_platform_id === null || @@ -201,7 +192,6 @@ export async function POST(request: Request) { ); } - // Step 1: Get or Create Person const personResult = await getOrCreatePerson( supabase, email.trim(), @@ -224,7 +214,6 @@ export async function POST(request: Request) { ); } - // Step 2: Get or Create Account for this Person on the specified platform const accountResult = await getOrCreateAccount( supabase, personResult.person.id, @@ -238,7 +227,6 @@ export async function POST(request: Request) { `API Error during Account processing (PersonID: ${personResult.person.id}, PlatformID: ${account_platform_id}): ${accountResult.error}`, accountResult.details || "", ); - // If person was just created, should we roll back or leave orphaned? For now, report error. const clientError = accountResult.error?.startsWith("Database error") ? "An internal error occurred while processing Account." : accountResult.error; @@ -246,15 +234,12 @@ export async function POST(request: Request) { { error: clientError, details: accountResult.details, - person: personResult.person, // Return person info even if account fails, for context + person: personResult.person, }, { status: 500 }, ); } - // Determine overall status code - // If both were created, 201. If one was created and other existed, still 201 for the "overall new entity" feel. - // If both existed, 200. const statusCode = personResult.created || accountResult.created ? 201 : 200; From cab4daf54b478a4de5afc45e9bf37c5a7b55beac Mon Sep 17 00:00:00 2001 From: sid597 Date: Sun, 18 May 2025 22:55:07 +0530 Subject: [PATCH 11/19] more with fixes --- .../app/api/supabase/insert/account/route.ts | 203 ++++++++++++ .../app/api/supabase/insert/agents/route.ts | 162 ++++++++++ .../batch/route.ts | 159 +++++++++ .../routes.ts} | 39 ++- .../supabase/insert/content/batch/route.ts | 177 ++++++++++ .../app/api/supabase/insert/content/route.ts | 301 ++++++++++++++++++ .../route.ts | 101 +++--- .../route.ts | 99 +++--- .../app/api/supabase/insert/document/route.ts | 186 +++++++++++ .../insert/{Person => person}/route.ts | 72 +++-- 10 files changed, 1389 insertions(+), 110 deletions(-) create mode 100644 apps/website/app/api/supabase/insert/account/route.ts create mode 100644 apps/website/app/api/supabase/insert/agents/route.ts create mode 100644 apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/batch/route.ts rename apps/website/app/api/supabase/insert/{ContentEmbedding/route.ts => content-embedding-openai-text-embedding-3-small-1536/routes.ts} (77%) create mode 100644 apps/website/app/api/supabase/insert/content/batch/route.ts create mode 100644 apps/website/app/api/supabase/insert/content/route.ts rename apps/website/app/api/supabase/insert/{DiscoursePlatform => discourse-platform}/route.ts (67%) rename apps/website/app/api/supabase/insert/{DiscourseSpace => discourse-space}/route.ts (69%) create mode 100644 apps/website/app/api/supabase/insert/document/route.ts rename apps/website/app/api/supabase/insert/{Person => person}/route.ts (87%) diff --git a/apps/website/app/api/supabase/insert/account/route.ts b/apps/website/app/api/supabase/insert/account/route.ts new file mode 100644 index 000000000..3b1b69cb1 --- /dev/null +++ b/apps/website/app/api/supabase/insert/account/route.ts @@ -0,0 +1,203 @@ +import { createClient } from "~/utils/supabase/server"; +import { NextResponse, NextRequest } from "next/server"; +import type { SupabaseClient } from "@supabase/supabase-js"; +import cors from "~/utils/llm/cors"; + +type AccountDataInput = { + person_id: number; + platform_id: number; + active?: boolean; + write_permission?: boolean; +}; + +// Represents the structure of an account record fetched from or inserted into the DB +type AccountRecord = { + id: number; + person_id: number; + platform_id: number; + active: boolean; + write_permission: boolean; +}; + +// Represents the payload for inserting a new account +type AccountInsertPayload = { + person_id: number; + platform_id: number; + active: boolean; + write_permission: boolean; +}; + +// Represents the return type of the getOrCreateAccountEntry function +type GetOrCreateAccountEntryReturn = { + account: AccountRecord | null; + error: string | null; + details?: string; + created: boolean; +}; + +async function getOrCreateAccountEntry( + supabase: SupabaseClient, + accountData: AccountDataInput, +): Promise { + const { + person_id, + platform_id, + active = true, + write_permission = true, + } = accountData; + + if ( + person_id === undefined || + person_id === null || + platform_id === undefined || + platform_id === null + ) { + return { + account: null, + error: "Missing required fields: person_id or platform_id", + details: "Both person_id and platform_id are required.", + created: false, + }; + } + + let { data: existingAccount, error: fetchError } = await supabase + .from("Account") + .select("id, person_id, platform_id, active, write_permission") + .eq("person_id", person_id) + .eq("platform_id", platform_id) + .maybeSingle(); + + if (fetchError) { + console.error( + `Error fetching Account (PersonID: ${person_id}, PlatformID: ${platform_id}):`, + fetchError, + ); + return { + account: null, + error: "Database error while fetching Account", + details: fetchError.message, + created: false, + }; + } + + if (existingAccount) { + return { account: existingAccount, error: null, created: false }; + } + + const accountToInsertData: AccountInsertPayload = { + person_id, + platform_id, + active, + write_permission, + }; + + const { data: newAccount, error: insertError } = await supabase + .from("Account") + .insert(accountToInsertData) + .select("id, person_id, platform_id, active, write_permission") + .single(); + + if (insertError) { + console.error( + `Error inserting new Account (PersonID: ${person_id}, PlatformID: ${platform_id}):`, + insertError, + ); + if (insertError.code === "23503") { + if (insertError.message.includes("Account_person_id_fkey")) { + return { + account: null, + error: `Invalid person_id: No Person record found for ID ${person_id}.`, + details: insertError.message, + created: false, + }; + } else if (insertError.message.includes("Account_platform_id_fkey")) { + return { + account: null, + error: `Invalid platform_id: No DiscoursePlatform record found for ID ${platform_id}.`, + details: insertError.message, + created: false, + }; + } + } + return { + account: null, + error: "Database error while inserting Account", + details: insertError.message, + created: false, + }; + } + + return { account: newAccount, error: null, created: true }; +} + +export async function POST(request: NextRequest): Promise { + const supabase = await createClient(); + let response: NextResponse; + + try { + const body: AccountDataInput = await request.json(); + + if ( + body.person_id === undefined || + body.person_id === null || + typeof body.person_id !== "number" + ) { + response = NextResponse.json( + { error: "Missing or invalid person_id" }, + { status: 400 }, + ); + return cors(request, response) as NextResponse; + } + if ( + body.platform_id === undefined || + body.platform_id === null || + typeof body.platform_id !== "number" + ) { + response = NextResponse.json( + { error: "Missing or invalid platform_id" }, + { status: 400 }, + ); + return cors(request, response) as NextResponse; + } + + const result = await getOrCreateAccountEntry(supabase, body); + + if (result.error || !result.account) { + console.error( + `API Error for Account creation (PersonID: ${body.person_id}, PlatformID: ${body.platform_id}): ${result.error}`, + result.details || "", + ); + const clientError = result.error?.startsWith("Database error") + ? "An internal error occurred." + : result.error; + const statusCode = result.error?.includes("Invalid") ? 400 : 500; + response = NextResponse.json( + { error: clientError, details: result.details }, + { status: statusCode }, + ); + } else { + response = NextResponse.json(result.account, { + status: result.created ? 201 : 200, + }); + } + } catch (e: unknown) { + console.error("API route error in /api/supabase/insert/Account:", e); + if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { + response = NextResponse.json( + { error: "Invalid JSON in request body" }, + { status: 400 }, + ); + } else { + response = NextResponse.json( + { error: "An unexpected error occurred" }, + { status: 500 }, + ); + } + } + return cors(request, response) as NextResponse; +} + +export async function OPTIONS(request: NextRequest): Promise { + const response = new NextResponse(null, { status: 204 }); + return cors(request, response) as NextResponse; +} diff --git a/apps/website/app/api/supabase/insert/agents/route.ts b/apps/website/app/api/supabase/insert/agents/route.ts new file mode 100644 index 000000000..0080017d7 --- /dev/null +++ b/apps/website/app/api/supabase/insert/agents/route.ts @@ -0,0 +1,162 @@ +import { createClient } from "~/utils/supabase/server"; +import { NextResponse, NextRequest } from "next/server"; +import type { SupabaseClient } from "@supabase/supabase-js"; +import cors from "~/utils/llm/cors"; // Adjust path if needed + +type AgentDataInput = { + type: string; // e.g., "Person", "Organization", "Software" + // Add any other fields directly belonging to the Agent table that are mandatory + // or that you want to set at creation time. + // For now, 'type' is the primary one from LinkML. +}; + +type AgentRecord = { + id: number; + type: string; + // Include other fields from your Agent table if they are selected + [key: string]: any; // Keeping this flexible for now if other fields are dynamically selected +}; + +type CreateAgentEntryReturn = { + agent: AgentRecord | null; + error: string | null; + details?: string; +}; + +// interface AgentResult { // Or just return the agent object or a simple wrapper +// agent: { id: number; type: string; [key: string]: any } | null; +// error: string | null; +// details?: string; +// } + +const createAgentEntry = async ( + supabase: SupabaseClient, + agentData: AgentDataInput, +): Promise => { + const { type } = agentData; + + if (!type || typeof type !== "string" || type.trim() === "") { + return { + agent: null, + error: "Missing or invalid 'type' for Agent", + details: "Agent 'type' is required.", + }; + } + + // Check if an agent with this type already exists? + // This depends on your business logic. If type should be unique, add a check. + // For now, we assume we always create a new agent. + // The ID is auto-generated by the database. + + const agentToInsert = { + type, + // any other default fields for Agent table + }; + + const { data: newAgent, error: insertError } = await supabase + .from("Agent") + .insert(agentToInsert) + .select("id, type") + .single(); + + if (insertError) { + console.error(`Error inserting new Agent (type: ${type}):`, insertError); + // Check for specific errors, e.g., if 'type' has a unique constraint + if (insertError.code === "23505") { + return { + agent: null, + error: `An Agent with type '${type}' might already exist or violates a unique constraint.`, + details: insertError.message, + }; + } + return { + agent: null, + error: "Database error while inserting Agent", + details: insertError.message, + }; + } + + if (!newAgent || newAgent.id === null || newAgent.id === undefined) { + // This case should ideally be caught by insertError, but as a safeguard: + console.error( + `New agent not returned or ID is null after insert (type: ${type})`, + newAgent, + ); + return { + agent: null, + error: "Failed to retrieve new Agent ID after insert.", + details: + "The insert operation might have appeared successful but returned no data or ID.", + }; + } + + console.log("Created new Agent:", newAgent); + return { agent: newAgent, error: null }; +}; + +export const POST = async (request: NextRequest): Promise => { + const supabase = await createClient(); + let response: NextResponse; + + try { + const body: AgentDataInput = await request.json(); + + if ( + !body.type || + typeof body.type !== "string" || + body.type.trim() === "" + ) { + response = NextResponse.json( + { error: "Validation Error: Missing or invalid type for Agent" }, + { status: 400 }, + ); + return cors(request, response) as NextResponse; + } + + const result = await createAgentEntry(supabase, body); + + if (result.error || !result.agent) { + console.error( + `API Error during Agent creation (type: ${body.type}): ${result.error}. Supabase Details: ${result.details === undefined || result.details === null ? "N/A" : result.details}`, + ); + const clientError = result.error?.startsWith("Database error") + ? "An internal error occurred." + : result.error; + const statusCode = result.error?.includes("already exist") ? 409 : 500; + response = NextResponse.json( + { error: clientError, details: result.details }, + { status: statusCode }, + ); + return cors(request, response) as NextResponse; + } else { + response = NextResponse.json(result.agent, { status: 201 }); // Agent created + return cors(request, response) as NextResponse; + } + } catch (e: unknown) { + console.error("API route error in /api/supabase/insert/Agents:", e); + let errorPayload: { error: string; details?: string } = { + error: "An unexpected server error occurred", + }; + let status = 500; + + if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { + response = NextResponse.json( + { error: "Invalid JSON in request body" }, + { status: 400 }, + ); + return cors(request, response) as NextResponse; + } else { + response = NextResponse.json( + { error: "An unexpected error occurred" }, + { status: 500 }, + ); + return cors(request, response) as NextResponse; + } + } + return cors(request, response) as NextResponse; +}; + +export const OPTIONS = async (request: NextRequest): Promise => { + const response = new NextResponse(null, { status: 204 }); + return cors(request, response) as NextResponse; +}; diff --git a/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/batch/route.ts b/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/batch/route.ts new file mode 100644 index 000000000..b87cbe98b --- /dev/null +++ b/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/batch/route.ts @@ -0,0 +1,159 @@ +import { createClient } from "~/utils/supabase/server"; +import { NextResponse, NextRequest } from "next/server"; +import type { SupabaseClient } from "@supabase/supabase-js"; +import cors from "~/utils/llm/cors"; + +// Based on LinkML for Embedding and the target table name +interface ContentEmbeddingBatchItemInput { + target_id: number; // Foreign key to Content.id + model: string; + vector: number[] | string; // Accept string for pre-formatted, or number[] + obsolete?: boolean; +} + +type ContentEmbeddingBatchRequestBody = ContentEmbeddingBatchItemInput[]; + +interface ContentEmbeddingBatchResult { + data?: any[]; // Array of successfully created embedding records + error?: string; + details?: string; +} + +const TARGET_EMBEDDING_TABLE = + "ContentEmbedding_openai_text_embedding_3_small_1536"; + +async function batchInsertEmbeddings( + supabase: SupabaseClient, + embeddingItems: ContentEmbeddingBatchRequestBody, +): Promise { + if (!Array.isArray(embeddingItems) || embeddingItems.length === 0) { + return { + error: "Request body must be a non-empty array of embedding items.", + }; + } + + const processedEmbeddingItems = embeddingItems.map((item, i) => { + if (!item) { + throw new Error( + `Validation Error: Item at index ${i} is undefined or null.`, + ); + } + if ( + item.target_id === undefined || + item.target_id === null || + !item.model || + !item.vector + ) { + throw new Error( + `Validation Error: Item at index ${i} is missing required fields (target_id, model, vector).`, + ); + } + if (!Array.isArray(item.vector) && typeof item.vector !== "string") { + throw new Error( + `Validation Error: Item.vector at index ${i} must be an array of numbers or a pre-formatted string.`, + ); + } + // Ensure vector is stringified if it's an array + const vectorString = Array.isArray(item.vector) + ? JSON.stringify(item.vector) + : item.vector; + + return { + target_id: item.target_id, + model: item.model, + vector: vectorString, + obsolete: item.obsolete === undefined ? false : item.obsolete, // Default to false + }; + }); + + const { data: newEmbeddings, error: insertError } = await supabase + .from(TARGET_EMBEDDING_TABLE) + .insert(processedEmbeddingItems) + .select(); + + if (insertError) { + console.error( + `Error batch inserting embeddings into ${TARGET_EMBEDDING_TABLE}:`, + insertError, + ); + return { + error: `Database error during batch insert of embeddings into ${TARGET_EMBEDDING_TABLE}.`, + details: insertError.message, + }; + } + + if ( + !newEmbeddings || + newEmbeddings.length !== processedEmbeddingItems.length + ) { + console.warn( + "Batch insert Embeddings: Mismatch between input and output count or no data returned.", + { + inputCount: processedEmbeddingItems.length, + outputCount: newEmbeddings?.length, + }, + ); + return { + error: + "Batch insert of Embeddings might have partially failed or returned unexpected data.", + }; + } + + console.log( + `Successfully batch inserted ${newEmbeddings.length} embedding records into ${TARGET_EMBEDDING_TABLE}.`, + ); + return { data: newEmbeddings }; +} + +export async function POST(request: NextRequest) { + const supabase = await createClient(); + let response: NextResponse; + + try { + const body: ContentEmbeddingBatchRequestBody = await request.json(); + const result = await batchInsertEmbeddings(supabase, body); + + if (result.error) { + console.error( + `API Error for batch Embedding creation: ${result.error}`, + result.details || "", + ); + const statusCode = result.error.startsWith("Validation Error:") + ? 400 + : 500; + response = NextResponse.json( + { error: result.error, details: result.details }, + { status: statusCode }, + ); + } else { + response = NextResponse.json(result.data, { status: 201 }); + } + } catch (e: any) { + console.error( + `API route error in /api/supabase/insert/${TARGET_EMBEDDING_TABLE}/batch:`, + e, + ); + if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { + response = NextResponse.json( + { + error: + "Invalid JSON in request body. Expected an array of embedding items.", + }, + { status: 400 }, + ); + } else if (e.message?.startsWith("Validation Error:")) { + response = NextResponse.json({ error: e.message }, { status: 400 }); + } else { + response = NextResponse.json( + { error: "An unexpected error occurred processing your request" }, + { status: 500 }, + ); + } + } + return cors(request, response) as NextResponse; +} + +export async function OPTIONS(request: NextRequest) { + const response = new NextResponse(null, { status: 204 }); + return cors(request, response) as NextResponse; +} diff --git a/apps/website/app/api/supabase/insert/ContentEmbedding/route.ts b/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts similarity index 77% rename from apps/website/app/api/supabase/insert/ContentEmbedding/route.ts rename to apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts index 334bfb954..13f8989a4 100644 --- a/apps/website/app/api/supabase/insert/ContentEmbedding/route.ts +++ b/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts @@ -1,6 +1,7 @@ import { createClient } from "~/utils/supabase/server"; -import { NextResponse } from "next/server"; +import { NextResponse, NextRequest } from "next/server"; import type { SupabaseClient } from "@supabase/supabase-js"; +import cors from "~/utils/llm/cors"; // Based on LinkML for Embedding interface ContentEmbeddingDataInput { @@ -83,9 +84,9 @@ async function createContentEmbeddingEntry( return { embedding: newEmbedding, error: null }; } -export async function POST(request: Request) { +export async function POST(request: NextRequest) { const supabase = await createClient(); - + let response: NextResponse; try { const body: ContentEmbeddingDataInput = await request.json(); @@ -94,32 +95,36 @@ export async function POST(request: Request) { body.target_id === null || typeof body.target_id !== "number" ) { - return NextResponse.json( + response = NextResponse.json( { error: "Missing or invalid target_id" }, { status: 400 }, ); + return cors(request, response) as NextResponse; } if (!body.model || typeof body.model !== "string") { - return NextResponse.json( + response = NextResponse.json( { error: "Missing or invalid model name" }, { status: 400 }, ); + return cors(request, response) as NextResponse; } if ( !body.vector || !Array.isArray(body.vector) || !body.vector.every((v) => typeof v === "number") ) { - return NextResponse.json( + response = NextResponse.json( { error: "Missing or invalid vector. Must be an array of numbers." }, { status: 400 }, ); + return cors(request, response) as NextResponse; } if (body.obsolete !== undefined && typeof body.obsolete !== "boolean") { - return NextResponse.json( + response = NextResponse.json( { error: "Invalid type for obsolete. Must be a boolean." }, { status: 400 }, ); + return cors(request, response) as NextResponse; } const { embedding, error, details } = await createContentEmbeddingEntry( @@ -136,35 +141,45 @@ export async function POST(request: Request) { error.startsWith("Invalid target_id") || error.startsWith("Invalid vector format") ) { - return NextResponse.json( + response = NextResponse.json( { error: error, details: details }, { status: 400 }, ); + return cors(request, response) as NextResponse; } const clientError = error.startsWith("Database error") ? "An internal error occurred while processing ContentEmbedding." : error; - return NextResponse.json( + response = NextResponse.json( { error: clientError, details: details }, { status: 500 }, ); + return cors(request, response) as NextResponse; } - return NextResponse.json(embedding, { status: 201 }); + response = NextResponse.json(embedding, { status: 201 }); + return cors(request, response) as NextResponse; } catch (e: any) { console.error( "API route error in /api/supabase/insert/ContentEmbedding:", e, ); if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { - return NextResponse.json( + response = NextResponse.json( { error: "Invalid JSON in request body" }, { status: 400 }, ); + return cors(request, response) as NextResponse; } - return NextResponse.json( + response = NextResponse.json( { error: "An unexpected error occurred processing your request" }, { status: 500 }, ); + return cors(request, response) as NextResponse; } } + +export async function OPTIONS(request: NextRequest) { + const response = new NextResponse(null, { status: 204 }); + return cors(request, response) as NextResponse; +} diff --git a/apps/website/app/api/supabase/insert/content/batch/route.ts b/apps/website/app/api/supabase/insert/content/batch/route.ts new file mode 100644 index 000000000..a979fcc89 --- /dev/null +++ b/apps/website/app/api/supabase/insert/content/batch/route.ts @@ -0,0 +1,177 @@ +import { createClient } from "~/utils/supabase/server"; +import { NextResponse, NextRequest } from "next/server"; +import type { SupabaseClient } from "@supabase/supabase-js"; +import cors from "~/utils/llm/cors"; + +// Based on the Content table schema and usage in embeddingWorkflow.ts +// This is for a single content item in the batch +type ContentBatchItemInput = { + text: string; + scale: string; + space_id: number; + author_id: number; // This is Person.id (Agent.id) + document_id: number; + source_local_id?: string; + metadata?: Record | string | null; // Allow string for pre-stringified, or null + created: string; // ISO 8601 date string + last_modified: string; // ISO 8601 date string + part_of_id?: number; +}; + +// The request body will be an array of these items +type ContentBatchRequestBody = ContentBatchItemInput[]; + +// Define a type for the actual record stored in/retrieved from DB for Content +type ContentRecord = { + id: number; + text: string; + scale: string; + space_id: number; + author_id: number; + document_id: number; + source_local_id: string | null; + metadata: Record | null; // Assuming metadata is stored as JSONB + created: string; // ISO 8601 date string + last_modified: string; // ISO 8601 date string + part_of_id: number | null; + // Add other fields from your Content table if they are selected +}; + +// The response will be an array of created content items (or an error object) +type ContentBatchResult = { + data?: ContentRecord[]; // Array of successfully created content records + error?: string; + details?: string; + partial_errors?: { index: number; error: string; details?: string }[]; +}; + +async function batchInsertContent( + supabase: SupabaseClient, + contentItems: ContentBatchRequestBody, +): Promise { + if (!Array.isArray(contentItems) || contentItems.length === 0) { + return { + error: "Request body must be a non-empty array of content items.", + }; + } + + const processedContentItems = contentItems.map((item, i) => { + if (!item) { + // This case should ideally not happen if contentItems is a valid array of objects + // but it satisfies the linter if it's worried about sparse arrays or undefined elements. + throw new Error( + `Validation Error: Item at index ${i} is undefined or null.`, + ); + } + if ( + !item.text || + !item.scale || + !item.space_id || + !item.author_id || + !item.document_id || + !item.created || + !item.last_modified + ) { + throw new Error( + `Validation Error: Item at index ${i} is missing required fields (text, scale, space_id, author_id, document_id, created, last_modified).`, + ); + } + + let metadataString: string | null = null; + if (item.metadata && typeof item.metadata === "object") { + metadataString = JSON.stringify(item.metadata); + } else if (typeof item.metadata === "string") { + metadataString = item.metadata; + } else { + metadataString = null; + } + + return { + ...item, + metadata: metadataString, + }; + }); + + const { data: newContents, error: insertError } = await supabase + .from("Content") + .insert(processedContentItems) // Use the validated and processed items + .select<"*", ContentRecord>(); // Select all columns of the inserted rows, including the 'id' + + if (insertError) { + console.error("Error batch inserting Content:", insertError); + return { + error: "Database error during batch insert of Content.", + details: insertError.message, + }; + } + + if (!newContents || newContents.length !== processedContentItems.length) { + console.warn( + "Batch insert Content: Mismatch between input and output count or no data returned.", + { + inputCount: processedContentItems.length, + outputCount: newContents?.length, + }, + ); + return { + error: + "Batch insert of Content might have partially failed or returned unexpected data.", + }; + } + + console.log( + `Successfully batch inserted ${newContents.length} Content records.`, + ); + return { data: newContents }; +} + +export async function POST(request: NextRequest): Promise { + const supabase = await createClient(); + let response: NextResponse; + + try { + const body: ContentBatchRequestBody = await request.json(); + const result = await batchInsertContent(supabase, body); + + if (result.error) { + console.error( + `API Error for batch Content creation: ${result.error}`, + result.details || "", + ); + const statusCode = result.error.startsWith("Validation Error:") + ? 400 + : 500; + response = NextResponse.json( + { error: result.error, details: result.details }, + { status: statusCode }, + ); + } else { + response = NextResponse.json(result.data, { status: 201 }); + } + } catch (e: unknown) { + console.error("API route error in /api/supabase/insert/Content/batch:", e); + if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { + response = NextResponse.json( + { + error: + "Invalid JSON in request body. Expected an array of content items.", + }, + { status: 400 }, + ); + } else if (e.message?.startsWith("Validation Error:")) { + // Catch errors thrown from batchInsertContent validation + response = NextResponse.json({ error: e.message }, { status: 400 }); + } else { + response = NextResponse.json( + { error: "An unexpected error occurred processing your request" }, + { status: 500 }, + ); + } + } + return cors(request, response) as NextResponse; +} + +export async function OPTIONS(request: NextRequest): Promise { + const response = new NextResponse(null, { status: 204 }); + return cors(request, response) as NextResponse; +} diff --git a/apps/website/app/api/supabase/insert/content/route.ts b/apps/website/app/api/supabase/insert/content/route.ts new file mode 100644 index 000000000..e8de1890b --- /dev/null +++ b/apps/website/app/api/supabase/insert/content/route.ts @@ -0,0 +1,301 @@ +import { createClient } from "~/utils/supabase/server"; +import { NextResponse, NextRequest } from "next/server"; +import type { SupabaseClient } from "@supabase/supabase-js"; +import cors from "~/utils/llm/cors"; + +// Based on the Content table schema and usage in embeddingWorkflow.ts +type ContentDataInput = { + text: string; + scale: string; + space_id: number; + author_id: number; + source_local_id?: string; + metadata?: Record; + created: string; // ISO 8601 date string + last_modified: string; // ISO 8601 date string + document_id?: number; + part_of_id?: number; +}; + +type ContentRecord = { + id: number; + text: string; + scale: string; + space_id: number; + author_id: number; + source_local_id: string | null; + metadata: Record | null; // Assuming metadata is stored as JSONB + created: string; // ISO 8601 date string + last_modified: string; // ISO 8601 date string + document_id: number | null; + part_of_id: number | null; + // Add other fields from your Content table if they are selected +}; + +type CreateContentEntryReturn = { + content: ContentRecord | null; + error: string | null; + details?: string; +}; + +async function createContentEntry( + supabase: SupabaseClient, + data: ContentDataInput, +): Promise { + const { + text, + scale, + space_id, + author_id, + source_local_id, + metadata, + created, + last_modified, + document_id, + part_of_id, + } = data; + + // Validate required fields + if ( + !text || + !scale || + !space_id || + !author_id || + !created || + !last_modified + ) { + return { + content: null, + error: + "Missing required fields: text, scale, space_id, author_id, created, or last_modified", + }; + } + + // Check for existing Content with same space_id and source_local_id + if (source_local_id) { + const { data: existingContent, error: existingError } = await supabase + .from("Content") + .select("*") // Consider selecting specific columns for ContentRecord + .eq("space_id", space_id) + .eq("source_local_id", source_local_id) + .maybeSingle(); + if (existingContent) { + return { content: existingContent, error: null }; + } + if (existingError && existingError.code !== "PGRST116") { + // PGRST116: No rows found + return { + content: null, + error: "Database error while checking for existing Content", + details: existingError.message, + }; + } + } + + // Validate field types + if (typeof text !== "string") { + return { + content: null, + error: "Invalid text format. Expected a string.", + }; + } + + if (typeof scale !== "string") { + return { + content: null, + error: "Invalid scale format. Expected a string.", + }; + } + + if (typeof space_id !== "number") { + return { + content: null, + error: "Invalid space_id format. Expected a number.", + }; + } + + if (typeof author_id !== "number") { + return { + content: null, + error: "Invalid author_id format. Expected a number.", + }; + } + + // Validate dates + try { + new Date(created); + new Date(last_modified); + } catch (e) { + return { + content: null, + error: "Invalid date format for created or last_modified", + }; + } + + const contentToInsert = { + text, + scale, + space_id, + author_id, + source_local_id, + metadata: metadata ? JSON.stringify(metadata) : null, + created, + last_modified, + document_id, + part_of_id, + }; + + const { data: newContent, error: insertError } = await supabase + .from("Content") + .insert(contentToInsert) + .select() // Consider selecting specific columns for ContentRecord + .single(); + + if (insertError) { + console.error("Error inserting new Content:", insertError); + + // Handle foreign key constraint violations + if (insertError.code === "23503") { + if (insertError.message.includes("space_id_fkey")) { + return { + content: null, + error: `Invalid space_id: No DiscourseSpace record found for ID ${space_id}.`, + details: insertError.message, + }; + } + if (insertError.message.includes("author_id_fkey")) { + return { + content: null, + error: `Invalid author_id: No Account record found for ID ${author_id}.`, + details: insertError.message, + }; + } + if (insertError.message.includes("document_id_fkey")) { + return { + content: null, + error: `Invalid document_id: No Document record found for ID ${document_id}.`, + details: insertError.message, + }; + } + if (insertError.message.includes("part_of_id_fkey")) { + return { + content: null, + error: `Invalid part_of_id: No Content record found for ID ${part_of_id}.`, + details: insertError.message, + }; + } + } + + return { + content: null, + error: "Database error while inserting Content", + details: insertError.message, + }; + } + + console.log("Created new Content:", newContent); + return { content: newContent, error: null }; +} + +export async function POST(request: NextRequest): Promise { + const supabase = await createClient(); + let response: NextResponse; + + try { + const body: ContentDataInput = await request.json(); + + // Validate required fields + if (!body.text) { + response = NextResponse.json( + { error: "Missing required field: text" }, + { status: 400 }, + ); + return cors(request, response) as NextResponse; + } + if (!body.scale) { + response = NextResponse.json( + { error: "Missing required field: scale" }, + { status: 400 }, + ); + return cors(request, response) as NextResponse; + } + if (body.space_id === undefined || body.space_id === null) { + response = NextResponse.json( + { error: "Missing required field: space_id" }, + { status: 400 }, + ); + return cors(request, response) as NextResponse; + } + if (body.author_id === undefined || body.author_id === null) { + response = NextResponse.json( + { error: "Missing required field: author_id" }, + { status: 400 }, + ); + return cors(request, response) as NextResponse; + } + if (!body.created) { + response = NextResponse.json( + { error: "Missing required field: created" }, + { status: 400 }, + ); + return cors(request, response) as NextResponse; + } + if (!body.last_modified) { + response = NextResponse.json( + { error: "Missing required field: last_modified" }, + { status: 400 }, + ); + return cors(request, response) as NextResponse; + } + + const { content, error, details } = await createContentEntry( + supabase, + body, + ); + + if (error) { + console.error(`API Error for Content creation: ${error}`, details || ""); + + // Handle validation errors + if ( + error.startsWith("Invalid") || + error.startsWith("Missing required fields") + ) { + response = NextResponse.json( + { error: error, details: details }, + { status: 400 }, + ); + } else { + // Handle database errors + const clientError = error.startsWith("Database error") + ? "An internal error occurred while processing Content." + : error; + response = NextResponse.json( + { error: clientError, details: details }, + { status: 500 }, + ); + } + } else { + response = NextResponse.json(content, { status: 201 }); + } + } catch (e: unknown) { + console.error("API route error in /api/supabase/insert/Content:", e); + if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { + response = NextResponse.json( + { error: "Invalid JSON in request body" }, + { status: 400 }, + ); + } else { + response = NextResponse.json( + { error: "An unexpected error occurred processing your request" }, + { status: 500 }, + ); + } + } + return cors(request, response) as NextResponse; +} + +export async function OPTIONS(request: NextRequest): Promise { + const response = new NextResponse(null, { status: 204 }); + return cors(request, response) as NextResponse; +} diff --git a/apps/website/app/api/supabase/insert/DiscoursePlatform/route.ts b/apps/website/app/api/supabase/insert/discourse-platform/route.ts similarity index 67% rename from apps/website/app/api/supabase/insert/DiscoursePlatform/route.ts rename to apps/website/app/api/supabase/insert/discourse-platform/route.ts index b44d4da05..40720e382 100644 --- a/apps/website/app/api/supabase/insert/DiscoursePlatform/route.ts +++ b/apps/website/app/api/supabase/insert/discourse-platform/route.ts @@ -1,18 +1,30 @@ import { createClient } from "~/utils/supabase/server"; -import { NextResponse } from "next/server"; +import { NextResponse, NextRequest } from "next/server"; import type { SupabaseClient } from "@supabase/supabase-js"; +import cors from "~/utils/llm/cors"; -interface PlatformResult { - platform: any | null; +type DiscoursePlatformRecord = { + id: number; + name: string; + url: string; + // Add other fields from your DiscoursePlatform table if they are selected +}; + +type GetOrCreateDiscoursePlatformReturn = { + platform: DiscoursePlatformRecord | null; error: string | null; details?: string; - created?: boolean; -} + created: boolean; // 'created' should always be boolean +}; + +type DiscoursePlatformDataInput = { + currentContentURL: string; +}; async function getOrCreateDiscoursePlatform( - supabase: SupabaseClient, // Using a more specific type for SupabaseClient + supabase: SupabaseClient, currentContentURL: string, -): Promise { +): Promise { let platformName: string | null = null; let platformUrl: string | null = null; const lowerCaseURL = currentContentURL.toLowerCase(); @@ -42,7 +54,7 @@ async function getOrCreateDiscoursePlatform( .from("DiscoursePlatform") .select("id, name, url") .eq("url", platformUrl) - .maybeSingle(); + .maybeSingle(); if (fetchError) { console.error("Error fetching DiscoursePlatform:", fetchError); @@ -70,8 +82,8 @@ async function getOrCreateDiscoursePlatform( const { data: newPlatform, error: insertError } = await supabase .from("DiscoursePlatform") .insert(platformToInsert) - .select() - .single(); // Expecting one row to be inserted and returned + .select("id, name, url") // Ensure selected fields match DiscoursePlatformRecord + .single(); // Expecting one row to be inserted and returned if (insertError) { console.error("Error inserting new DiscoursePlatform:", insertError); @@ -85,7 +97,7 @@ async function getOrCreateDiscoursePlatform( .from("DiscoursePlatform") .select("id, name, url") .eq("url", platformUrl) - .maybeSingle(); + .maybeSingle(); if (reFetchError) { console.error( @@ -125,42 +137,46 @@ async function getOrCreateDiscoursePlatform( } } -export async function POST(request: Request) { - const supabase = await createClient(); // Creates a server-side Supabase client +export async function POST(request: NextRequest): Promise { + const supabase = await createClient(); + let response: NextResponse; try { - const body = await request.json(); + const body: DiscoursePlatformDataInput = await request.json(); const { currentContentURL } = body; if (!currentContentURL || typeof currentContentURL !== "string") { - return NextResponse.json( + response = NextResponse.json( { error: "Missing or invalid currentContentURL in request body" }, { status: 400 }, ); + return cors(request, response) as NextResponse; } - const { platform, error, details, created } = - await getOrCreateDiscoursePlatform(supabase, currentContentURL); + const result = await getOrCreateDiscoursePlatform( + supabase, + currentContentURL, + ); - if (error) { + if (result.error) { console.error( - `API Error for DiscoursePlatform (URL: ${currentContentURL}): ${error}`, - details || "", + `API Error for DiscoursePlatform (URL: ${currentContentURL}): ${result.error}`, + result.details || "", ); - return NextResponse.json( - { error: error, details: details }, + response = NextResponse.json( + { error: result.error, details: result.details }, { status: 500 }, ); - } - - if (platform) { - return NextResponse.json(platform, { status: created ? 201 : 200 }); + } else if (result.platform) { + response = NextResponse.json(result.platform, { + status: result.created ? 201 : 200, + }); } else { // This case should ideally be caught by the 'error' field in the result console.error( `API Error for DiscoursePlatform (URL: ${currentContentURL}): Platform was null without an error flag.`, ); - return NextResponse.json( + response = NextResponse.json( { error: "Failed to get or create DiscoursePlatform for an unknown reason", @@ -168,24 +184,33 @@ export async function POST(request: Request) { { status: 500 }, ); } - } catch (e: any) { + } catch (e: unknown) { console.error( - "API route error in /api/supabase/insert/DiscoursePlatform:", + "API route error in /api/supabase/insert/discourse-platform:", e, ); // Differentiate between JSON parsing errors and other errors - if (e instanceof SyntaxError && e.message.includes("JSON")) { - return NextResponse.json( + if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { + response = NextResponse.json( { error: "Invalid JSON in request body" }, { status: 400 }, ); + } else { + response = NextResponse.json( + { + error: + e instanceof Error + ? e.message + : "An unexpected error occurred processing your request", + }, + { status: 500 }, + ); } - return NextResponse.json( - { - error: - e.message || "An unexpected error occurred processing your request", - }, - { status: 500 }, - ); } + return cors(request, response) as NextResponse; +} + +export async function OPTIONS(request: NextRequest): Promise { + const response = new NextResponse(null, { status: 204 }); + return cors(request, response) as NextResponse; } diff --git a/apps/website/app/api/supabase/insert/DiscourseSpace/route.ts b/apps/website/app/api/supabase/insert/discourse-space/route.ts similarity index 69% rename from apps/website/app/api/supabase/insert/DiscourseSpace/route.ts rename to apps/website/app/api/supabase/insert/discourse-space/route.ts index 8f54d35fd..03a3802c8 100644 --- a/apps/website/app/api/supabase/insert/DiscourseSpace/route.ts +++ b/apps/website/app/api/supabase/insert/discourse-space/route.ts @@ -1,26 +1,35 @@ import { createClient } from "~/utils/supabase/server"; -import { NextResponse } from "next/server"; +import { NextResponse, NextRequest } from "next/server"; import type { SupabaseClient } from "@supabase/supabase-js"; +import cors from "~/utils/llm/cors"; -interface DiscourseSpaceData { +type DiscourseSpaceDataInput = { name: string; url: string; discourse_platform_id: number; -} +}; + +type DiscourseSpaceRecord = { + id: number; + name: string; + url: string; + discourse_platform_id: number; + // Add other fields from your DiscourseSpace table if they are selected +}; -interface DiscourseSpaceResult { - space: any | null; +type GetOrCreateDiscourseSpaceReturn = { + space: DiscourseSpaceRecord | null; error: string | null; details?: string; - created?: boolean; -} + created: boolean; // 'created' should always be boolean +}; async function getOrCreateDiscourseSpace( supabase: SupabaseClient, name: string, url: string, discoursePlatformId: number, -): Promise { +): Promise { if ( !name || !url || @@ -41,7 +50,7 @@ async function getOrCreateDiscourseSpace( .select("id, name, url, discourse_platform_id") .eq("url", normalizedUrl) .eq("discourse_platform_id", discoursePlatformId) - .maybeSingle(); + .maybeSingle(); if (fetchError) { console.error( @@ -74,7 +83,7 @@ async function getOrCreateDiscourseSpace( .from("DiscourseSpace") .insert(spaceToInsert) .select("id, name, url, discourse_platform_id") - .single(); + .single(); if (insertError) { console.error( @@ -90,7 +99,7 @@ async function getOrCreateDiscourseSpace( .select("id, name, url, discourse_platform_id") .eq("url", normalizedUrl) .eq("discourse_platform_id", discoursePlatformId) - .maybeSingle(); + .maybeSingle(); if (reFetchError) { console.error( @@ -129,84 +138,102 @@ async function getOrCreateDiscourseSpace( } } -export async function POST(request: Request) { +export async function POST(request: NextRequest): Promise { const supabase = await createClient(); + let response: NextResponse; try { - const body: DiscourseSpaceData = await request.json(); + const body: DiscourseSpaceDataInput = await request.json(); const { name, url, discourse_platform_id } = body; if (!name || typeof name !== "string" || name.trim() === "") { - return NextResponse.json( + response = NextResponse.json( { error: "Missing or invalid name in request body" }, { status: 400 }, ); + return cors(request, response) as NextResponse; } if (!url || typeof url !== "string" || url.trim() === "") { - return NextResponse.json( + response = NextResponse.json( { error: "Missing or invalid url in request body" }, { status: 400 }, ); + return cors(request, response) as NextResponse; } if ( discourse_platform_id === undefined || discourse_platform_id === null || typeof discourse_platform_id !== "number" ) { - return NextResponse.json( + response = NextResponse.json( { error: "Missing or invalid discourse_platform_id in request body" }, { status: 400 }, ); + return cors(request, response) as NextResponse; } - const { space, error, details, created } = await getOrCreateDiscourseSpace( + const result = await getOrCreateDiscourseSpace( supabase, name.trim(), url.trim(), discourse_platform_id, ); - if (error) { + if (result.error) { console.error( - `API Error for DiscourseSpace (Name: ${name}, URL: ${url}, PlatformID: ${discourse_platform_id}): ${error}`, - details || "", + `API Error for DiscourseSpace (Name: ${name}, URL: ${url}, PlatformID: ${discourse_platform_id}): ${result.error}`, + result.details || "", ); - const clientError = error.startsWith("Database error") + const clientError = result.error.startsWith("Database error") ? "An internal error occurred while processing the DiscourseSpace information." - : error; - return NextResponse.json( + : result.error; + response = NextResponse.json( { error: clientError, - details: error.startsWith("Database error") ? undefined : details, + details: result.error.startsWith("Database error") + ? undefined + : result.details, }, { status: 500 }, ); - } - - if (space) { - return NextResponse.json(space, { status: created ? 201 : 200 }); + } else if (result.space) { + response = NextResponse.json(result.space, { + status: result.created ? 201 : 200, + }); } else { + // This case should ideally not be reached if error is null and space is null, + // but it's a safeguard. console.error( `API Error for DiscourseSpace (Name: ${name}, URL: ${url}, PlatformID: ${discourse_platform_id}): Space was null without an error flag.`, ); - return NextResponse.json( + response = NextResponse.json( { error: "Failed to get or create DiscourseSpace for an unknown reason", }, { status: 500 }, ); } - } catch (e: any) { - console.error("API route error in /api/supabase/insert/DiscourseSpace:", e); + } catch (e: unknown) { + console.error( + "API route error in /api/supabase/insert/discourse-space:", + e, + ); if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { - return NextResponse.json( + response = NextResponse.json( { error: "Invalid JSON in request body" }, { status: 400 }, ); + } else { + response = NextResponse.json( + { error: "An unexpected error occurred processing your request" }, + { status: 500 }, + ); } - return NextResponse.json( - { error: "An unexpected error occurred processing your request" }, - { status: 500 }, - ); } + return cors(request, response) as NextResponse; +} + +export async function OPTIONS(request: NextRequest): Promise { + const response = new NextResponse(null, { status: 204 }); + return cors(request, response) as NextResponse; } diff --git a/apps/website/app/api/supabase/insert/document/route.ts b/apps/website/app/api/supabase/insert/document/route.ts new file mode 100644 index 000000000..c516f5e7c --- /dev/null +++ b/apps/website/app/api/supabase/insert/document/route.ts @@ -0,0 +1,186 @@ +import { createClient } from "~/utils/supabase/server"; +import { NextResponse, NextRequest } from "next/server"; +import type { SupabaseClient } from "@supabase/supabase-js"; +import cors from "~/utils/llm/cors"; + +type DocumentDataInput = { + space_id: number; + source_local_id?: string; + url?: string; + metadata?: Record; + created: string; // ISO 8601 date string + last_modified: string; // ISO 8601 date string + author_id: number; +}; + +type DocumentRecord = { + id: number; + space_id: number; + source_local_id: string | null; + url: string | null; + metadata: Record | null; // Assuming metadata is stored as JSONB + created: string; // ISO 8601 date string + last_modified: string; // ISO 8601 date string + author_id: number; + // Add other fields from your Document table if they are selected +}; + +type CreateDocumentEntryReturn = { + document: DocumentRecord | null; + error: string | null; + details?: string; +}; + +async function createDocumentEntry( + supabase: SupabaseClient, + data: DocumentDataInput, +): Promise { + const { + space_id, + source_local_id, + url, + metadata, + created, + last_modified, + author_id, + } = data; + + // Validate required fields + if (!space_id || !created || !last_modified || !author_id) { + return { + document: null, + error: + "Missing required fields: space_id, created, last_modified, or author_id", + }; + } + + const documentToInsert = { + space_id, + source_local_id, + url, + metadata: metadata ? JSON.stringify(metadata) : "{}", + created, + last_modified, + author_id, + }; + + const { data: newDocument, error: insertError } = await supabase + .from("Document") + .insert(documentToInsert) + .select() // Consider selecting specific columns for DocumentRecord + .single(); + + if (insertError) { + console.error("Error inserting new Document:", insertError); + if (insertError.code === "23503") { + if (insertError.message.includes("space_id_fkey")) { + return { + document: null, + error: `Invalid space_id: No Space record found for ID ${space_id}.`, + details: insertError.message, + }; + } + if (insertError.message.includes("author_id_fkey")) { + return { + document: null, + error: `Invalid author_id: No Account record found for ID ${author_id}.`, + details: insertError.message, + }; + } + } + return { + document: null, + error: "Database error while inserting Document", + details: insertError.message, + }; + } + + console.log("Created new Document:", newDocument); + return { document: newDocument, error: null }; +} + +export async function POST(request: NextRequest): Promise { + const supabase = await createClient(); + let response: NextResponse; + + try { + const body: DocumentDataInput = await request.json(); + + // Validate required fields + if (body.space_id === undefined || body.space_id === null) { + response = NextResponse.json( + { error: "Missing required field: space_id" }, + { status: 400 }, + ); + return cors(request, response) as NextResponse; + } + if (!body.created) { + response = NextResponse.json( + { error: "Missing required field: created" }, + { status: 400 }, + ); + return cors(request, response) as NextResponse; + } + if (!body.last_modified) { + response = NextResponse.json( + { error: "Missing required field: last_modified" }, + { status: 400 }, + ); + return cors(request, response) as NextResponse; + } + if (body.author_id === undefined || body.author_id === null) { + response = NextResponse.json( + { error: "Missing required field: author_id" }, + { status: 400 }, + ); + return cors(request, response) as NextResponse; + } + + const { document, error, details } = await createDocumentEntry( + supabase, + body, + ); + + if (error) { + console.error(`API Error for Document creation: ${error}`, details || ""); + if ( + error.startsWith("Invalid") || + error.startsWith("Missing required fields") + ) { + response = NextResponse.json( + { error: error, details: details }, + { status: 400 }, + ); + } else { + const clientError = error.startsWith("Database error") + ? "An internal error occurred while processing Document." + : error; + response = NextResponse.json( + { error: clientError, details: details }, + { status: 500 }, + ); + } + } else { + response = NextResponse.json(document, { status: 201 }); + } + } catch (e: unknown) { + console.error("API route error in /api/supabase/insert/Document:", e); + if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { + response = NextResponse.json( + { error: "Invalid JSON in request body" }, + { status: 400 }, + ); + } else { + response = NextResponse.json( + { error: "An unexpected error occurred processing your request" }, + { status: 500 }, + ); + } + } + return cors(request, response) as NextResponse; +} + +export async function OPTIONS(request: NextRequest): Promise { + const response = new NextResponse(null, { status: 204 }); + return cors(request, response) as NextResponse; +} diff --git a/apps/website/app/api/supabase/insert/Person/route.ts b/apps/website/app/api/supabase/insert/person/route.ts similarity index 87% rename from apps/website/app/api/supabase/insert/Person/route.ts rename to apps/website/app/api/supabase/insert/person/route.ts index 2fe8cab98..0e6118417 100644 --- a/apps/website/app/api/supabase/insert/Person/route.ts +++ b/apps/website/app/api/supabase/insert/person/route.ts @@ -1,8 +1,8 @@ import { createClient } from "~/utils/supabase/server"; -import { NextResponse } from "next/server"; +import { NextResponse, NextRequest } from "next/server"; import type { SupabaseClient } from "@supabase/supabase-js"; -interface PersonDataInput { +type PersonDataInput = { name: string; email: string; orcid?: string | null; @@ -10,16 +10,46 @@ interface PersonDataInput { account_platform_id: number; account_active?: boolean; account_write_permission?: boolean; -} +}; + +type PersonRecord = { + id: number; + name: string; + email: string; + orcid: string | null; + type: string; // Assuming 'type' is the column name for person_type +}; + +type AccountRecord = { + id: number; + person_id: number; + platform_id: number; + active: boolean; + write_permission: boolean; +}; -interface PersonResult { +type PersonResult = { person: any | null; account: any | null; error: string | null; details?: string; person_created?: boolean; account_created?: boolean; -} +}; + +type GetOrCreatePersonReturn = { + person: PersonRecord | null; + error: string | null; + details?: string; + created: boolean; +}; + +type GetOrCreateAccountReturn = { + account: AccountRecord | null; + error: string | null; + details?: string; + created: boolean; +}; async function getOrCreatePerson( supabase: SupabaseClient, @@ -27,17 +57,12 @@ async function getOrCreatePerson( name: string, orcid: string | null | undefined, personType: string, -): Promise<{ - person: any | null; - error: string | null; - details?: string; - created: boolean; -}> { +): Promise { let { data: existingPerson, error: fetchError } = await supabase .from("Person") .select("id, name, email, orcid, type") .eq("email", email) - .maybeSingle(); + .maybeSingle(); if (fetchError) { console.error(`Error fetching Person by email (${email}):`, fetchError); @@ -64,7 +89,7 @@ async function getOrCreatePerson( .from("Person") .insert(personToInsert) .select("id, name, email, orcid, type") - .single(); + .single(); if (insertError) { console.error( @@ -89,18 +114,13 @@ async function getOrCreateAccount( platformId: number, isActive: boolean, writePermission?: boolean, -): Promise<{ - account: any | null; - error: string | null; - details?: string; - created: boolean; -}> { +): Promise { let { data: existingAccount, error: fetchError } = await supabase .from("Account") .select("id, person_id, platform_id, active, write_permission") .eq("person_id", personId) .eq("platform_id", platformId) - .maybeSingle(); + .maybeSingle(); if (fetchError) { console.error( @@ -122,7 +142,11 @@ async function getOrCreateAccount( console.log( `Account for PersonID ${personId} on PlatformID ${platformId} not found, creating new one...`, ); - const accountToInsert: any = { + const accountToInsert: Partial & { + person_id: number; + platform_id: number; + active: boolean; + } = { person_id: personId, platform_id: platformId, active: isActive, @@ -135,7 +159,7 @@ async function getOrCreateAccount( .from("Account") .insert(accountToInsert) .select("id, person_id, platform_id, active, write_permission") - .single(); + .single(); if (insertError) { console.error( @@ -154,7 +178,7 @@ async function getOrCreateAccount( } } -export async function POST(request: Request) { +export async function POST(request: NextRequest): Promise { const supabase = await createClient(); try { @@ -252,7 +276,7 @@ export async function POST(request: Request) { }, { status: statusCode }, ); - } catch (e: any) { + } catch (e: unknown) { console.error("API route error in /api/supabase/insert/Person:", e); if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { return NextResponse.json( From 30ea51a83aeff547aa4502786d845bd3d62572d6 Mon Sep 17 00:00:00 2001 From: sid597 Date: Mon, 19 May 2025 16:19:08 +0530 Subject: [PATCH 12/19] dry principles --- .../app/api/supabase/insert/account/route.ts | 214 +++------- .../app/api/supabase/insert/agents/route.ts | 178 +++----- .../batch/route.ts | 218 +++++----- .../routes.ts | 244 +++++------ .../supabase/insert/content/batch/route.ts | 207 ++++------ .../app/api/supabase/insert/content/route.ts | 385 ++++++++---------- .../insert/discourse-platform/route.ts | 213 +++------- .../supabase/insert/discourse-space/route.ts | 287 +++++-------- .../app/api/supabase/insert/document/route.ts | 218 +++++----- .../app/api/supabase/insert/person/route.ts | 327 ++++++--------- apps/website/app/utils/supabase/apiUtils.ts | 88 ++++ apps/website/app/utils/supabase/dbUtils.ts | 256 ++++++++++++ 12 files changed, 1272 insertions(+), 1563 deletions(-) create mode 100644 apps/website/app/utils/supabase/apiUtils.ts create mode 100644 apps/website/app/utils/supabase/dbUtils.ts diff --git a/apps/website/app/api/supabase/insert/account/route.ts b/apps/website/app/api/supabase/insert/account/route.ts index 3b1b69cb1..7ec574701 100644 --- a/apps/website/app/api/supabase/insert/account/route.ts +++ b/apps/website/app/api/supabase/insert/account/route.ts @@ -1,7 +1,14 @@ import { createClient } from "~/utils/supabase/server"; import { NextResponse, NextRequest } from "next/server"; -import type { SupabaseClient } from "@supabase/supabase-js"; -import cors from "~/utils/llm/cors"; +import { + getOrCreateEntity, + GetOrCreateEntityResult, +} from "~/utils/supabase/dbUtils"; +import { + createApiResponse, + handleRouteError, + defaultOptionsHandler, +} from "~/utils/supabase/apiUtils"; type AccountDataInput = { person_id: number; @@ -10,7 +17,6 @@ type AccountDataInput = { write_permission?: boolean; }; -// Represents the structure of an account record fetched from or inserted into the DB type AccountRecord = { id: number; person_id: number; @@ -19,26 +25,10 @@ type AccountRecord = { write_permission: boolean; }; -// Represents the payload for inserting a new account -type AccountInsertPayload = { - person_id: number; - platform_id: number; - active: boolean; - write_permission: boolean; -}; - -// Represents the return type of the getOrCreateAccountEntry function -type GetOrCreateAccountEntryReturn = { - account: AccountRecord | null; - error: string | null; - details?: string; - created: boolean; -}; - -async function getOrCreateAccountEntry( - supabase: SupabaseClient, +const getOrCreateAccount = async ( + supabasePromise: ReturnType, accountData: AccountDataInput, -): Promise { +): Promise> => { const { person_id, platform_id, @@ -53,151 +43,77 @@ async function getOrCreateAccountEntry( platform_id === null ) { return { - account: null, - error: "Missing required fields: person_id or platform_id", + entity: null, + error: "Missing required fields: person_id or platform_id.", details: "Both person_id and platform_id are required.", created: false, + status: 400, }; } - let { data: existingAccount, error: fetchError } = await supabase - .from("Account") - .select("id, person_id, platform_id, active, write_permission") - .eq("person_id", person_id) - .eq("platform_id", platform_id) - .maybeSingle(); + const supabase = await supabasePromise; - if (fetchError) { - console.error( - `Error fetching Account (PersonID: ${person_id}, PlatformID: ${platform_id}):`, - fetchError, - ); - return { - account: null, - error: "Database error while fetching Account", - details: fetchError.message, - created: false, - }; - } + const result = await getOrCreateEntity( + supabase, + "Account", + "id, person_id, platform_id, active, write_permission", + { person_id: person_id, platform_id: platform_id }, + { person_id, platform_id, active, write_permission }, + "Account", + ); - if (existingAccount) { - return { account: existingAccount, error: null, created: false }; - } - - const accountToInsertData: AccountInsertPayload = { - person_id, - platform_id, - active, - write_permission, - }; - - const { data: newAccount, error: insertError } = await supabase - .from("Account") - .insert(accountToInsertData) - .select("id, person_id, platform_id, active, write_permission") - .single(); - - if (insertError) { - console.error( - `Error inserting new Account (PersonID: ${person_id}, PlatformID: ${platform_id}):`, - insertError, - ); - if (insertError.code === "23503") { - if (insertError.message.includes("Account_person_id_fkey")) { - return { - account: null, - error: `Invalid person_id: No Person record found for ID ${person_id}.`, - details: insertError.message, - created: false, - }; - } else if (insertError.message.includes("Account_platform_id_fkey")) { - return { - account: null, - error: `Invalid platform_id: No DiscoursePlatform record found for ID ${platform_id}.`, - details: insertError.message, - created: false, - }; - } + if ( + result.error && + result.details && + result.status === 400 && + result.details.includes("violates foreign key constraint") + ) { + if (result.details.includes("Account_person_id_fkey")) { + return { + ...result, + error: `Invalid person_id: No Person record found for ID ${person_id}.`, + }; + } else if (result.details.includes("Account_platform_id_fkey")) { + return { + ...result, + error: `Invalid platform_id: No DiscoursePlatform record found for ID ${platform_id}.`, + }; } - return { - account: null, - error: "Database error while inserting Account", - details: insertError.message, - created: false, - }; } + return result; +}; - return { account: newAccount, error: null, created: true }; -} - -export async function POST(request: NextRequest): Promise { - const supabase = await createClient(); - let response: NextResponse; +export const POST = async (request: NextRequest): Promise => { + const supabasePromise = createClient(); try { const body: AccountDataInput = await request.json(); - if ( - body.person_id === undefined || - body.person_id === null || - typeof body.person_id !== "number" - ) { - response = NextResponse.json( - { error: "Missing or invalid person_id" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; + if (body.person_id === undefined || body.person_id === null) { + return createApiResponse(request, { + error: "Missing or invalid person_id.", + status: 400, + }); } - if ( - body.platform_id === undefined || - body.platform_id === null || - typeof body.platform_id !== "number" - ) { - response = NextResponse.json( - { error: "Missing or invalid platform_id" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; + if (body.platform_id === undefined || body.platform_id === null) { + return createApiResponse(request, { + error: "Missing or invalid platform_id.", + status: 400, + }); } - const result = await getOrCreateAccountEntry(supabase, body); + const result = await getOrCreateAccount(supabasePromise, body); - if (result.error || !result.account) { - console.error( - `API Error for Account creation (PersonID: ${body.person_id}, PlatformID: ${body.platform_id}): ${result.error}`, - result.details || "", - ); - const clientError = result.error?.startsWith("Database error") - ? "An internal error occurred." - : result.error; - const statusCode = result.error?.includes("Invalid") ? 400 : 500; - response = NextResponse.json( - { error: clientError, details: result.details }, - { status: statusCode }, - ); - } else { - response = NextResponse.json(result.account, { - status: result.created ? 201 : 200, - }); - } + return createApiResponse(request, { + data: result.entity, + error: result.error, + details: result.details, + status: result.status, + created: result.created, + }); } catch (e: unknown) { - console.error("API route error in /api/supabase/insert/Account:", e); - if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { - response = NextResponse.json( - { error: "Invalid JSON in request body" }, - { status: 400 }, - ); - } else { - response = NextResponse.json( - { error: "An unexpected error occurred" }, - { status: 500 }, - ); - } + return handleRouteError(request, e, "/api/supabase/insert/account"); } - return cors(request, response) as NextResponse; -} +}; -export async function OPTIONS(request: NextRequest): Promise { - const response = new NextResponse(null, { status: 204 }); - return cors(request, response) as NextResponse; -} +export const OPTIONS = defaultOptionsHandler; diff --git a/apps/website/app/api/supabase/insert/agents/route.ts b/apps/website/app/api/supabase/insert/agents/route.ts index 0080017d7..dd0792f76 100644 --- a/apps/website/app/api/supabase/insert/agents/route.ts +++ b/apps/website/app/api/supabase/insert/agents/route.ts @@ -1,162 +1,78 @@ import { createClient } from "~/utils/supabase/server"; import { NextResponse, NextRequest } from "next/server"; -import type { SupabaseClient } from "@supabase/supabase-js"; -import cors from "~/utils/llm/cors"; // Adjust path if needed +import { + getOrCreateEntity, + GetOrCreateEntityResult, +} from "~/utils/supabase/dbUtils"; +import { + createApiResponse, + handleRouteError, + defaultOptionsHandler, +} from "~/utils/supabase/apiUtils"; type AgentDataInput = { - type: string; // e.g., "Person", "Organization", "Software" - // Add any other fields directly belonging to the Agent table that are mandatory - // or that you want to set at creation time. - // For now, 'type' is the primary one from LinkML. + type: string; }; type AgentRecord = { id: number; type: string; - // Include other fields from your Agent table if they are selected - [key: string]: any; // Keeping this flexible for now if other fields are dynamically selected -}; - -type CreateAgentEntryReturn = { - agent: AgentRecord | null; - error: string | null; - details?: string; }; -// interface AgentResult { // Or just return the agent object or a simple wrapper -// agent: { id: number; type: string; [key: string]: any } | null; -// error: string | null; -// details?: string; -// } - -const createAgentEntry = async ( - supabase: SupabaseClient, - agentData: AgentDataInput, -): Promise => { - const { type } = agentData; - - if (!type || typeof type !== "string" || type.trim() === "") { - return { - agent: null, - error: "Missing or invalid 'type' for Agent", - details: "Agent 'type' is required.", - }; - } - - // Check if an agent with this type already exists? - // This depends on your business logic. If type should be unique, add a check. - // For now, we assume we always create a new agent. - // The ID is auto-generated by the database. - - const agentToInsert = { - type, - // any other default fields for Agent table - }; +const getOrCreateAgentByType = async ( + supabasePromise: ReturnType, + agentType: string, +): Promise> => { + const type = agentType.trim(); - const { data: newAgent, error: insertError } = await supabase - .from("Agent") - .insert(agentToInsert) - .select("id, type") - .single(); - - if (insertError) { - console.error(`Error inserting new Agent (type: ${type}):`, insertError); - // Check for specific errors, e.g., if 'type' has a unique constraint - if (insertError.code === "23505") { - return { - agent: null, - error: `An Agent with type '${type}' might already exist or violates a unique constraint.`, - details: insertError.message, - }; - } + if (!type) { return { - agent: null, - error: "Database error while inserting Agent", - details: insertError.message, + entity: null, + error: "Missing or invalid 'type' for Agent.", + details: "Agent 'type' is required and cannot be empty.", + created: false, + status: 400, }; } - if (!newAgent || newAgent.id === null || newAgent.id === undefined) { - // This case should ideally be caught by insertError, but as a safeguard: - console.error( - `New agent not returned or ID is null after insert (type: ${type})`, - newAgent, - ); - return { - agent: null, - error: "Failed to retrieve new Agent ID after insert.", - details: - "The insert operation might have appeared successful but returned no data or ID.", - }; - } + const supabase = await supabasePromise; - console.log("Created new Agent:", newAgent); - return { agent: newAgent, error: null }; + return getOrCreateEntity( + supabase, + "Agent", + "id, type", + { type: type }, + { type: type }, + "Agent", + ); }; export const POST = async (request: NextRequest): Promise => { - const supabase = await createClient(); - let response: NextResponse; + const supabasePromise = createClient(); try { const body: AgentDataInput = await request.json(); + const { type } = body; - if ( - !body.type || - typeof body.type !== "string" || - body.type.trim() === "" - ) { - response = NextResponse.json( - { error: "Validation Error: Missing or invalid type for Agent" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; + if (!type || typeof type !== "string" || type.trim() === "") { + return createApiResponse(request, { + error: "Validation Error: Missing or invalid type for Agent.", + status: 400, + }); } - const result = await createAgentEntry(supabase, body); + const result = await getOrCreateAgentByType(supabasePromise, type); - if (result.error || !result.agent) { - console.error( - `API Error during Agent creation (type: ${body.type}): ${result.error}. Supabase Details: ${result.details === undefined || result.details === null ? "N/A" : result.details}`, - ); - const clientError = result.error?.startsWith("Database error") - ? "An internal error occurred." - : result.error; - const statusCode = result.error?.includes("already exist") ? 409 : 500; - response = NextResponse.json( - { error: clientError, details: result.details }, - { status: statusCode }, - ); - return cors(request, response) as NextResponse; - } else { - response = NextResponse.json(result.agent, { status: 201 }); // Agent created - return cors(request, response) as NextResponse; - } + return createApiResponse(request, { + data: result.entity, + error: result.error, + details: result.details, + status: result.status, + created: result.created, + }); } catch (e: unknown) { - console.error("API route error in /api/supabase/insert/Agents:", e); - let errorPayload: { error: string; details?: string } = { - error: "An unexpected server error occurred", - }; - let status = 500; - - if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { - response = NextResponse.json( - { error: "Invalid JSON in request body" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; - } else { - response = NextResponse.json( - { error: "An unexpected error occurred" }, - { status: 500 }, - ); - return cors(request, response) as NextResponse; - } + return handleRouteError(request, e, "/api/supabase/insert/agents"); } - return cors(request, response) as NextResponse; }; -export const OPTIONS = async (request: NextRequest): Promise => { - const response = new NextResponse(null, { status: 204 }); - return cors(request, response) as NextResponse; -}; +export const OPTIONS = defaultOptionsHandler; diff --git a/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/batch/route.ts b/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/batch/route.ts index b87cbe98b..9197db679 100644 --- a/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/batch/route.ts +++ b/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/batch/route.ts @@ -1,9 +1,18 @@ import { createClient } from "~/utils/supabase/server"; import { NextResponse, NextRequest } from "next/server"; -import type { SupabaseClient } from "@supabase/supabase-js"; +import { + createApiResponse, + handleRouteError, + defaultOptionsHandler, +} from "~/utils/supabase/apiUtils"; +import { + processAndInsertBatch, + BatchItemValidator, + BatchProcessResult, +} from "~/utils/supabase/dbUtils"; import cors from "~/utils/llm/cors"; -// Based on LinkML for Embedding and the target table name +// Input type for a single item in the batch interface ContentEmbeddingBatchItemInput { target_id: number; // Foreign key to Content.id model: string; @@ -11,149 +20,122 @@ interface ContentEmbeddingBatchItemInput { obsolete?: boolean; } +// Request body is an array of these items type ContentEmbeddingBatchRequestBody = ContentEmbeddingBatchItemInput[]; -interface ContentEmbeddingBatchResult { - data?: any[]; // Array of successfully created embedding records - error?: string; - details?: string; +// Type for the record as stored/retrieved from DB (ensure fields match your table) +interface ContentEmbeddingRecord { + id: number; // Assuming auto-generated ID + target_id: number; + model: string; + vector: string; // Stored as string (JSON.stringify of number[]) + obsolete: boolean; + // created_at?: string; // If you have timestamps and want to select them } +// Type for the item after processing, ready for DB insert +type ProcessedEmbeddingItem = Omit & { + vector: string; // Vector is always stringified + obsolete: boolean; // Obsolete has a default +}; + const TARGET_EMBEDDING_TABLE = "ContentEmbedding_openai_text_embedding_3_small_1536"; -async function batchInsertEmbeddings( - supabase: SupabaseClient, - embeddingItems: ContentEmbeddingBatchRequestBody, -): Promise { - if (!Array.isArray(embeddingItems) || embeddingItems.length === 0) { +// Validator and processor for embedding items +const validateAndProcessEmbeddingItem: BatchItemValidator< + ContentEmbeddingBatchItemInput, + ProcessedEmbeddingItem +> = (item, index) => { + if (item.target_id === undefined || item.target_id === null) { return { - error: "Request body must be a non-empty array of embedding items.", + valid: false, + error: `Item at index ${index}: Missing required field target_id.`, }; } - - const processedEmbeddingItems = embeddingItems.map((item, i) => { - if (!item) { - throw new Error( - `Validation Error: Item at index ${i} is undefined or null.`, - ); - } - if ( - item.target_id === undefined || - item.target_id === null || - !item.model || - !item.vector - ) { - throw new Error( - `Validation Error: Item at index ${i} is missing required fields (target_id, model, vector).`, - ); - } - if (!Array.isArray(item.vector) && typeof item.vector !== "string") { - throw new Error( - `Validation Error: Item.vector at index ${i} must be an array of numbers or a pre-formatted string.`, - ); - } - // Ensure vector is stringified if it's an array - const vectorString = Array.isArray(item.vector) - ? JSON.stringify(item.vector) - : item.vector; - + if (!item.model) { return { - target_id: item.target_id, - model: item.model, - vector: vectorString, - obsolete: item.obsolete === undefined ? false : item.obsolete, // Default to false + valid: false, + error: `Item at index ${index}: Missing required field model.`, }; - }); - - const { data: newEmbeddings, error: insertError } = await supabase - .from(TARGET_EMBEDDING_TABLE) - .insert(processedEmbeddingItems) - .select(); - - if (insertError) { - console.error( - `Error batch inserting embeddings into ${TARGET_EMBEDDING_TABLE}:`, - insertError, - ); + } + if (!item.vector) { return { - error: `Database error during batch insert of embeddings into ${TARGET_EMBEDDING_TABLE}.`, - details: insertError.message, + valid: false, + error: `Item at index ${index}: Missing required field vector.`, }; } - - if ( - !newEmbeddings || - newEmbeddings.length !== processedEmbeddingItems.length - ) { - console.warn( - "Batch insert Embeddings: Mismatch between input and output count or no data returned.", - { - inputCount: processedEmbeddingItems.length, - outputCount: newEmbeddings?.length, - }, - ); + if (!Array.isArray(item.vector) && typeof item.vector !== "string") { return { - error: - "Batch insert of Embeddings might have partially failed or returned unexpected data.", + valid: false, + error: `Item at index ${index}: vector must be an array of numbers or a pre-formatted string.`, }; } - console.log( - `Successfully batch inserted ${newEmbeddings.length} embedding records into ${TARGET_EMBEDDING_TABLE}.`, + const vectorString = Array.isArray(item.vector) + ? JSON.stringify(item.vector) + : item.vector; + + return { + valid: true, + processedItem: { + target_id: item.target_id, + model: item.model, + vector: vectorString, + obsolete: item.obsolete === undefined ? false : item.obsolete, + }, + }; +}; + +const batchInsertEmbeddingsProcess = async ( + supabase: Awaited>, + embeddingItems: ContentEmbeddingBatchRequestBody, +): Promise> => { + return processAndInsertBatch< + ContentEmbeddingBatchItemInput, + ProcessedEmbeddingItem, + ContentEmbeddingRecord + >( + supabase, + embeddingItems, + TARGET_EMBEDDING_TABLE, + "*", // Select all fields, adjust if needed for ContentEmbeddingRecord + validateAndProcessEmbeddingItem, + "ContentEmbedding", ); - return { data: newEmbeddings }; -} +}; -export async function POST(request: NextRequest) { +export const POST = async (request: NextRequest): Promise => { const supabase = await createClient(); - let response: NextResponse; try { const body: ContentEmbeddingBatchRequestBody = await request.json(); - const result = await batchInsertEmbeddings(supabase, body); - - if (result.error) { - console.error( - `API Error for batch Embedding creation: ${result.error}`, - result.details || "", - ); - const statusCode = result.error.startsWith("Validation Error:") - ? 400 - : 500; - response = NextResponse.json( - { error: result.error, details: result.details }, - { status: statusCode }, - ); - } else { - response = NextResponse.json(result.data, { status: 201 }); + if (!Array.isArray(body)) { + return createApiResponse(request, { + error: "Request body must be an array of embedding items.", + status: 400, + }); } - } catch (e: any) { - console.error( - `API route error in /api/supabase/insert/${TARGET_EMBEDDING_TABLE}/batch:`, + + const result = await batchInsertEmbeddingsProcess(supabase, body); + + return createApiResponse(request, { + data: result.data, + error: result.error, + details: result.details, + ...(result.partial_errors && { + meta: { partial_errors: result.partial_errors }, + }), + status: result.status, + created: result.status === 201, + }); + } catch (e: unknown) { + return handleRouteError( + request, e, + `/api/supabase/insert/${TARGET_EMBEDDING_TABLE}/batch`, ); - if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { - response = NextResponse.json( - { - error: - "Invalid JSON in request body. Expected an array of embedding items.", - }, - { status: 400 }, - ); - } else if (e.message?.startsWith("Validation Error:")) { - response = NextResponse.json({ error: e.message }, { status: 400 }); - } else { - response = NextResponse.json( - { error: "An unexpected error occurred processing your request" }, - { status: 500 }, - ); - } } - return cors(request, response) as NextResponse; -} +}; -export async function OPTIONS(request: NextRequest) { - const response = new NextResponse(null, { status: 204 }); - return cors(request, response) as NextResponse; -} +export const OPTIONS = defaultOptionsHandler; diff --git a/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts b/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts index 13f8989a4..23c391b34 100644 --- a/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts +++ b/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts @@ -1,46 +1,88 @@ import { createClient } from "~/utils/supabase/server"; import { NextResponse, NextRequest } from "next/server"; -import type { SupabaseClient } from "@supabase/supabase-js"; -import cors from "~/utils/llm/cors"; - -// Based on LinkML for Embedding +import { + createApiResponse, + handleRouteError, + defaultOptionsHandler, +} from "~/utils/supabase/apiUtils"; +import { + getOrCreateEntity, + GetOrCreateEntityResult, +} from "~/utils/supabase/dbUtils"; + +// Input type interface ContentEmbeddingDataInput { target_id: number; model: string; - vector: number[]; + vector: number[]; // Singular route expects number[] directly obsolete?: boolean; } -interface ContentEmbeddingResult { - embedding: any | null; - error: string | null; - details?: string; +// DB record type +interface ContentEmbeddingRecord { + id: number; + target_id: number; + model: string; + vector: string; // Stored as stringified JSON + obsolete: boolean; } const TARGET_EMBEDDING_TABLE = "ContentEmbedding_openai_text_embedding_3_small_1536"; -async function createContentEmbeddingEntry( - supabase: SupabaseClient, +// Renamed and refactored +const processAndCreateEmbedding = async ( + supabasePromise: ReturnType, data: ContentEmbeddingDataInput, -): Promise { +): Promise> => { const { target_id, model, vector, obsolete = false } = data; - if (target_id === undefined || target_id === null || !model || !vector) { + // --- Start of validation --- + if ( + target_id === undefined || + target_id === null || + typeof target_id !== "number" + ) { return { - embedding: null, - error: "Missing required fields: target_id, model, or vector", + entity: null, + error: "Missing or invalid target_id.", + created: false, + status: 400, }; } - - if (!Array.isArray(vector) || !vector.every((v) => typeof v === "number")) { + if (!model || typeof model !== "string") { + return { + entity: null, + error: "Missing or invalid model name.", + created: false, + status: 400, + }; + } + if ( + !vector || + !Array.isArray(vector) || + !vector.every((v) => typeof v === "number") + ) { + return { + entity: null, + error: "Missing or invalid vector. Must be an array of numbers.", + created: false, + status: 400, + }; + } + if (data.obsolete !== undefined && typeof data.obsolete !== "boolean") { + // Check original data for obsolete presence return { - embedding: null, - error: "Invalid vector format. Expected an array of numbers.", + entity: null, + error: "Invalid type for obsolete. Must be a boolean.", + created: false, + status: 400, }; } + // --- End of validation --- const vectorString = JSON.stringify(vector); + const supabase = await supabasePromise; const embeddingToInsert = { target_id, @@ -49,137 +91,71 @@ async function createContentEmbeddingEntry( obsolete, }; - const { data: newEmbedding, error: insertError } = await supabase - .from(TARGET_EMBEDDING_TABLE) - .insert(embeddingToInsert) - .select() - .single(); + // Using getOrCreateEntity, forcing create path by providing non-matching criteria + // This standardizes return type and error handling (e.g., FK violations from dbUtils) + const result = await getOrCreateEntity( + supabase, + TARGET_EMBEDDING_TABLE, + "*", // Select all fields for the record + { id: -1 }, // Non-matching criteria to force "create" path + embeddingToInsert, + "ContentEmbedding", + ); - if (insertError) { - console.error( - `Error inserting new ContentEmbedding into ${TARGET_EMBEDDING_TABLE}:`, - insertError, - ); + // getOrCreateEntity handles general 23503, but we can make the message more specific if needed + if ( + result.error && + result.details && + result.status === 400 && + result.details.includes("violates foreign key constraint") + ) { if ( - insertError.code === "23503" && - insertError.message.includes("target_id_fkey") + result.details.toLowerCase().includes( + // Check for target_id FK, adapt if FK name is different + `${TARGET_EMBEDDING_TABLE.toLowerCase()}_target_id_fkey`.toLowerCase(), + ) || + result.details.toLowerCase().includes("target_id") ) { return { - embedding: null, + ...result, error: `Invalid target_id: No Content record found for ID ${target_id}.`, - details: insertError.message, }; } - return { - embedding: null, - error: "Database error while inserting ContentEmbedding", - details: insertError.message, - }; } - console.log( - `Created new ContentEmbedding in ${TARGET_EMBEDDING_TABLE}:`, - newEmbedding, - ); - return { embedding: newEmbedding, error: null }; -} + return result; +}; + +export const POST = async (request: NextRequest): Promise => { + const supabasePromise = createClient(); -export async function POST(request: NextRequest) { - const supabase = await createClient(); - let response: NextResponse; try { const body: ContentEmbeddingDataInput = await request.json(); - if ( - body.target_id === undefined || - body.target_id === null || - typeof body.target_id !== "number" - ) { - response = NextResponse.json( - { error: "Missing or invalid target_id" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; - } - if (!body.model || typeof body.model !== "string") { - response = NextResponse.json( - { error: "Missing or invalid model name" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; - } - if ( - !body.vector || - !Array.isArray(body.vector) || - !body.vector.every((v) => typeof v === "number") - ) { - response = NextResponse.json( - { error: "Missing or invalid vector. Must be an array of numbers." }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; - } - if (body.obsolete !== undefined && typeof body.obsolete !== "boolean") { - response = NextResponse.json( - { error: "Invalid type for obsolete. Must be a boolean." }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; - } - - const { embedding, error, details } = await createContentEmbeddingEntry( - supabase, - body, - ); - - if (error) { - console.error( - `API Error for ContentEmbedding creation: ${error}`, - details || "", - ); - if ( - error.startsWith("Invalid target_id") || - error.startsWith("Invalid vector format") - ) { - response = NextResponse.json( - { error: error, details: details }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; - } - const clientError = error.startsWith("Database error") - ? "An internal error occurred while processing ContentEmbedding." - : error; - response = NextResponse.json( - { error: clientError, details: details }, - { status: 500 }, - ); - return cors(request, response) as NextResponse; + // Minimal validation here, more detailed in processAndCreateEmbedding + if (!body || typeof body !== "object") { + return createApiResponse(request, { + error: "Invalid request body: expected a JSON object.", + status: 400, + }); } - response = NextResponse.json(embedding, { status: 201 }); - return cors(request, response) as NextResponse; - } catch (e: any) { - console.error( - "API route error in /api/supabase/insert/ContentEmbedding:", + const result = await processAndCreateEmbedding(supabasePromise, body); + + return createApiResponse(request, { + data: result.entity, + error: result.error, + details: result.details, + status: result.status, + created: result.created, // Will be true if successful create + }); + } catch (e: unknown) { + return handleRouteError( + request, e, + `/api/supabase/insert/${TARGET_EMBEDDING_TABLE}`, ); - if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { - response = NextResponse.json( - { error: "Invalid JSON in request body" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; - } - response = NextResponse.json( - { error: "An unexpected error occurred processing your request" }, - { status: 500 }, - ); - return cors(request, response) as NextResponse; } -} +}; -export async function OPTIONS(request: NextRequest) { - const response = new NextResponse(null, { status: 204 }); - return cors(request, response) as NextResponse; -} +export const OPTIONS = defaultOptionsHandler; diff --git a/apps/website/app/api/supabase/insert/content/batch/route.ts b/apps/website/app/api/supabase/insert/content/batch/route.ts index a979fcc89..129635a92 100644 --- a/apps/website/app/api/supabase/insert/content/batch/route.ts +++ b/apps/website/app/api/supabase/insert/content/batch/route.ts @@ -1,7 +1,15 @@ import { createClient } from "~/utils/supabase/server"; import { NextResponse, NextRequest } from "next/server"; -import type { SupabaseClient } from "@supabase/supabase-js"; -import cors from "~/utils/llm/cors"; +import { + createApiResponse, + handleRouteError, + defaultOptionsHandler, +} from "~/utils/supabase/apiUtils"; +import { + processAndInsertBatch, + BatchItemValidator, + BatchProcessResult, // Import BatchProcessResult for the return type +} from "~/utils/supabase/dbUtils"; // Ensure this path is correct // Based on the Content table schema and usage in embeddingWorkflow.ts // This is for a single content item in the batch @@ -37,141 +45,90 @@ type ContentRecord = { // Add other fields from your Content table if they are selected }; -// The response will be an array of created content items (or an error object) -type ContentBatchResult = { - data?: ContentRecord[]; // Array of successfully created content records - error?: string; - details?: string; - partial_errors?: { index: number; error: string; details?: string }[]; -}; - -async function batchInsertContent( - supabase: SupabaseClient, - contentItems: ContentBatchRequestBody, -): Promise { - if (!Array.isArray(contentItems) || contentItems.length === 0) { - return { - error: "Request body must be a non-empty array of content items.", - }; - } - - const processedContentItems = contentItems.map((item, i) => { - if (!item) { - // This case should ideally not happen if contentItems is a valid array of objects - // but it satisfies the linter if it's worried about sparse arrays or undefined elements. - throw new Error( - `Validation Error: Item at index ${i} is undefined or null.`, - ); - } - if ( - !item.text || - !item.scale || - !item.space_id || - !item.author_id || - !item.document_id || - !item.created || - !item.last_modified - ) { - throw new Error( - `Validation Error: Item at index ${i} is missing required fields (text, scale, space_id, author_id, document_id, created, last_modified).`, - ); - } - - let metadataString: string | null = null; - if (item.metadata && typeof item.metadata === "object") { - metadataString = JSON.stringify(item.metadata); - } else if (typeof item.metadata === "string") { - metadataString = item.metadata; - } else { - metadataString = null; - } - - return { - ...item, - metadata: metadataString, - }; - }); - - const { data: newContents, error: insertError } = await supabase - .from("Content") - .insert(processedContentItems) // Use the validated and processed items - .select<"*", ContentRecord>(); // Select all columns of the inserted rows, including the 'id' - - if (insertError) { - console.error("Error batch inserting Content:", insertError); +// Specific validator and processor for Content items +const validateAndProcessContentItem: BatchItemValidator< + ContentBatchItemInput, + Omit & { metadata: string | null } // TProcessed type +> = (item, index) => { + // No need to check for !item here, processAndInsertBatch handles null/undefined items in the array itself + const requiredFields = [ + "text", + "scale", + "space_id", + "author_id", + "document_id", + "created", + "last_modified", + ]; + const missingFields = requiredFields.filter( + (field) => !(item as any)[field] && (item as any)[field] !== 0, + ); // check for undefined, null, empty string but allow 0 + if (missingFields.length > 0) { return { - error: "Database error during batch insert of Content.", - details: insertError.message, + valid: false, + error: `Item at index ${index}: Missing required fields: ${missingFields.join(", ")}.`, }; } - if (!newContents || newContents.length !== processedContentItems.length) { - console.warn( - "Batch insert Content: Mismatch between input and output count or no data returned.", - { - inputCount: processedContentItems.length, - outputCount: newContents?.length, - }, - ); - return { - error: - "Batch insert of Content might have partially failed or returned unexpected data.", - }; - } + let metadataString: string | null = null; + if (item.metadata && typeof item.metadata === "object") { + metadataString = JSON.stringify(item.metadata); + } else if (typeof item.metadata === "string") { + metadataString = item.metadata; + } // item.metadata can also be null if provided as such, which is fine + + return { + valid: true, + processedItem: { ...item, metadata: metadataString }, + }; +}; - console.log( - `Successfully batch inserted ${newContents.length} Content records.`, +// Updated batchInsertContentProcess to use the generic utility +const batchInsertContentProcess = async ( + supabase: Awaited>, + contentItems: ContentBatchRequestBody, +): Promise> => { + return processAndInsertBatch< + ContentBatchItemInput, + Omit & { metadata: string | null }, + ContentRecord + >( + supabase, + contentItems, + "Content", // Table name + "*", // Select query (can be more specific, e.g., "id, text, scale, ...") + validateAndProcessContentItem, + "Content", // Entity name for logging ); - return { data: newContents }; -} +}; -export async function POST(request: NextRequest): Promise { +export const POST = async (request: NextRequest): Promise => { const supabase = await createClient(); - let response: NextResponse; try { const body: ContentBatchRequestBody = await request.json(); - const result = await batchInsertContent(supabase, body); - - if (result.error) { - console.error( - `API Error for batch Content creation: ${result.error}`, - result.details || "", - ); - const statusCode = result.error.startsWith("Validation Error:") - ? 400 - : 500; - response = NextResponse.json( - { error: result.error, details: result.details }, - { status: statusCode }, - ); - } else { - response = NextResponse.json(result.data, { status: 201 }); + if (!Array.isArray(body)) { + return createApiResponse(request, { + error: "Request body must be an array of content items.", + status: 400, + }); } + + const result = await batchInsertContentProcess(supabase, body); + + return createApiResponse(request, { + data: result.data, + error: result.error, + details: result.details, + ...(result.partial_errors && { + meta: { partial_errors: result.partial_errors }, + }), + status: result.status, + created: result.status === 201, + }); } catch (e: unknown) { - console.error("API route error in /api/supabase/insert/Content/batch:", e); - if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { - response = NextResponse.json( - { - error: - "Invalid JSON in request body. Expected an array of content items.", - }, - { status: 400 }, - ); - } else if (e.message?.startsWith("Validation Error:")) { - // Catch errors thrown from batchInsertContent validation - response = NextResponse.json({ error: e.message }, { status: 400 }); - } else { - response = NextResponse.json( - { error: "An unexpected error occurred processing your request" }, - { status: 500 }, - ); - } + return handleRouteError(request, e, "/api/supabase/insert/content/batch"); } - return cors(request, response) as NextResponse; -} +}; -export async function OPTIONS(request: NextRequest): Promise { - const response = new NextResponse(null, { status: 204 }); - return cors(request, response) as NextResponse; -} +export const OPTIONS = defaultOptionsHandler; diff --git a/apps/website/app/api/supabase/insert/content/route.ts b/apps/website/app/api/supabase/insert/content/route.ts index e8de1890b..e81251cd2 100644 --- a/apps/website/app/api/supabase/insert/content/route.ts +++ b/apps/website/app/api/supabase/insert/content/route.ts @@ -1,6 +1,14 @@ import { createClient } from "~/utils/supabase/server"; import { NextResponse, NextRequest } from "next/server"; -import type { SupabaseClient } from "@supabase/supabase-js"; +import { + getOrCreateEntity, + GetOrCreateEntityResult, +} from "~/utils/supabase/dbUtils"; // Ensure path is correct +import { + createApiResponse, + handleRouteError, + defaultOptionsHandler, +} from "~/utils/supabase/apiUtils"; // Ensure path is correct import cors from "~/utils/llm/cors"; // Based on the Content table schema and usage in embeddingWorkflow.ts @@ -10,7 +18,7 @@ type ContentDataInput = { space_id: number; author_id: number; source_local_id?: string; - metadata?: Record; + metadata?: Record | string; // Allow string for pre-stringified metadata created: string; // ISO 8601 date string last_modified: string; // ISO 8601 date string document_id?: number; @@ -24,278 +32,213 @@ type ContentRecord = { space_id: number; author_id: number; source_local_id: string | null; - metadata: Record | null; // Assuming metadata is stored as JSONB - created: string; // ISO 8601 date string - last_modified: string; // ISO 8601 date string + metadata: Record | null; + created: string; + last_modified: string; document_id: number | null; part_of_id: number | null; // Add other fields from your Content table if they are selected }; -type CreateContentEntryReturn = { - content: ContentRecord | null; - error: string | null; - details?: string; -}; - -async function createContentEntry( - supabase: SupabaseClient, +// Renamed and refactored +const processAndUpsertContentEntry = async ( + supabasePromise: ReturnType, data: ContentDataInput, -): Promise { +): Promise> => { const { text, scale, space_id, author_id, source_local_id, - metadata, + metadata: rawMetadata, created, last_modified, document_id, part_of_id, } = data; - // Validate required fields - if ( - !text || - !scale || - !space_id || - !author_id || - !created || - !last_modified - ) { + // --- Start of extensive validation --- + if (!text || typeof text !== "string") return { - content: null, - error: - "Missing required fields: text, scale, space_id, author_id, created, or last_modified", + entity: null, + error: "Invalid or missing text.", + created: false, + status: 400, }; - } - - // Check for existing Content with same space_id and source_local_id - if (source_local_id) { - const { data: existingContent, error: existingError } = await supabase - .from("Content") - .select("*") // Consider selecting specific columns for ContentRecord - .eq("space_id", space_id) - .eq("source_local_id", source_local_id) - .maybeSingle(); - if (existingContent) { - return { content: existingContent, error: null }; - } - if (existingError && existingError.code !== "PGRST116") { - // PGRST116: No rows found - return { - content: null, - error: "Database error while checking for existing Content", - details: existingError.message, - }; - } - } - - // Validate field types - if (typeof text !== "string") { + if (!scale || typeof scale !== "string") return { - content: null, - error: "Invalid text format. Expected a string.", + entity: null, + error: "Invalid or missing scale.", + created: false, + status: 400, }; - } - - if (typeof scale !== "string") { + if ( + space_id === undefined || + space_id === null || + typeof space_id !== "number" + ) return { - content: null, - error: "Invalid scale format. Expected a string.", + entity: null, + error: "Invalid or missing space_id.", + created: false, + status: 400, }; - } - - if (typeof space_id !== "number") { + if ( + author_id === undefined || + author_id === null || + typeof author_id !== "number" + ) return { - content: null, - error: "Invalid space_id format. Expected a number.", + entity: null, + error: "Invalid or missing author_id.", + created: false, + status: 400, }; - } - - if (typeof author_id !== "number") { + if (!created) return { - content: null, - error: "Invalid author_id format. Expected a number.", + entity: null, + error: "Missing created date.", + created: false, + status: 400, + }; + if (!last_modified) + return { + entity: null, + error: "Missing last_modified date.", + created: false, + status: 400, }; - } - // Validate dates try { - new Date(created); - new Date(last_modified); + new Date(created); // Validate date format + new Date(last_modified); // Validate date format } catch (e) { return { - content: null, - error: "Invalid date format for created or last_modified", + entity: null, + error: "Invalid date format for created or last_modified.", + created: false, + status: 400, }; } + // --- End of extensive validation --- + + const processedMetadata = + rawMetadata && typeof rawMetadata === "object" + ? JSON.stringify(rawMetadata) + : typeof rawMetadata === "string" + ? rawMetadata + : null; + + const supabase = await supabasePromise; - const contentToInsert = { + const contentToInsertOrUpdate = { text, scale, space_id, author_id, - source_local_id, - metadata: metadata ? JSON.stringify(metadata) : null, + source_local_id: source_local_id || null, + metadata: processedMetadata as any, created, last_modified, - document_id, - part_of_id, + document_id: document_id || null, + part_of_id: part_of_id || null, }; - const { data: newContent, error: insertError } = await supabase - .from("Content") - .insert(contentToInsert) - .select() // Consider selecting specific columns for ContentRecord - .single(); - - if (insertError) { - console.error("Error inserting new Content:", insertError); - - // Handle foreign key constraint violations - if (insertError.code === "23503") { - if (insertError.message.includes("space_id_fkey")) { - return { - content: null, - error: `Invalid space_id: No DiscourseSpace record found for ID ${space_id}.`, - details: insertError.message, - }; - } - if (insertError.message.includes("author_id_fkey")) { - return { - content: null, - error: `Invalid author_id: No Account record found for ID ${author_id}.`, - details: insertError.message, - }; - } - if (insertError.message.includes("document_id_fkey")) { - return { - content: null, - error: `Invalid document_id: No Document record found for ID ${document_id}.`, - details: insertError.message, - }; - } - if (insertError.message.includes("part_of_id_fkey")) { - return { - content: null, - error: `Invalid part_of_id: No Content record found for ID ${part_of_id}.`, - details: insertError.message, - }; - } + let matchCriteria: Record | null = null; + if (source_local_id && space_id !== undefined && space_id !== null) { + matchCriteria = { space_id: space_id, source_local_id: source_local_id }; + } + // If no solid matchCriteria for a "get", getOrCreateEntity will likely proceed to "create". + // If there are unique constraints other than (space_id, source_local_id), it will handle race conditions. + + const result = await getOrCreateEntity( + supabase, + "Content", + "*", // Select all fields for ContentRecord + matchCriteria || { id: -1 }, // Use a non-matching criteria if no specific lookup needed, to force create path if not found + contentToInsertOrUpdate, // This will be used for insert if not found or for update in some extended utilities. + "Content", + ); + + // Custom handling for specific foreign key errors + if ( + result.error && + result.details && + result.status === 400 && + result.details.includes("violates foreign key constraint") + ) { + const details = result.details.toLowerCase(); + if ( + details.includes("content_space_id_fkey") || + details.includes("space_id") + ) { + // Be more general with FK name if it changes + return { + ...result, + error: `Invalid space_id: No DiscourseSpace record found for ID ${space_id}.`, + }; + } + if ( + details.includes("content_author_id_fkey") || + details.includes("author_id") + ) { + return { + ...result, + error: `Invalid author_id: No Account record found for ID ${author_id}.`, + }; + } + if ( + document_id && + (details.includes("content_document_id_fkey") || + details.includes("document_id")) + ) { + return { + ...result, + error: `Invalid document_id: No Document record found for ID ${document_id}.`, + }; + } + if ( + part_of_id && + (details.includes("content_part_of_id_fkey") || + details.includes("part_of_id")) + ) { + return { + ...result, + error: `Invalid part_of_id: No Content record found for ID ${part_of_id}.`, + }; } - - return { - content: null, - error: "Database error while inserting Content", - details: insertError.message, - }; } + return result; +}; - console.log("Created new Content:", newContent); - return { content: newContent, error: null }; -} - -export async function POST(request: NextRequest): Promise { - const supabase = await createClient(); - let response: NextResponse; +export const POST = async (request: NextRequest): Promise => { + const supabasePromise = createClient(); try { const body: ContentDataInput = await request.json(); - // Validate required fields - if (!body.text) { - response = NextResponse.json( - { error: "Missing required field: text" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; - } - if (!body.scale) { - response = NextResponse.json( - { error: "Missing required field: scale" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; - } - if (body.space_id === undefined || body.space_id === null) { - response = NextResponse.json( - { error: "Missing required field: space_id" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; + // Most validation is now inside processAndUpsertContentEntry + // Minimal check here, or rely on processAndUpsertContentEntry for all field validation + if (!body || typeof body !== "object") { + return createApiResponse(request, { + error: "Invalid request body: expected a JSON object.", + status: 400, + }); } - if (body.author_id === undefined || body.author_id === null) { - response = NextResponse.json( - { error: "Missing required field: author_id" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; - } - if (!body.created) { - response = NextResponse.json( - { error: "Missing required field: created" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; - } - if (!body.last_modified) { - response = NextResponse.json( - { error: "Missing required field: last_modified" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; - } - - const { content, error, details } = await createContentEntry( - supabase, - body, - ); - if (error) { - console.error(`API Error for Content creation: ${error}`, details || ""); + const result = await processAndUpsertContentEntry(supabasePromise, body); - // Handle validation errors - if ( - error.startsWith("Invalid") || - error.startsWith("Missing required fields") - ) { - response = NextResponse.json( - { error: error, details: details }, - { status: 400 }, - ); - } else { - // Handle database errors - const clientError = error.startsWith("Database error") - ? "An internal error occurred while processing Content." - : error; - response = NextResponse.json( - { error: clientError, details: details }, - { status: 500 }, - ); - } - } else { - response = NextResponse.json(content, { status: 201 }); - } + return createApiResponse(request, { + data: result.entity, + error: result.error, + details: result.details, + status: result.status, + created: result.created, + }); } catch (e: unknown) { - console.error("API route error in /api/supabase/insert/Content:", e); - if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { - response = NextResponse.json( - { error: "Invalid JSON in request body" }, - { status: 400 }, - ); - } else { - response = NextResponse.json( - { error: "An unexpected error occurred processing your request" }, - { status: 500 }, - ); - } + return handleRouteError(request, e, "/api/supabase/insert/content"); } - return cors(request, response) as NextResponse; -} +}; -export async function OPTIONS(request: NextRequest): Promise { - const response = new NextResponse(null, { status: 204 }); - return cors(request, response) as NextResponse; -} +export const OPTIONS = defaultOptionsHandler; diff --git a/apps/website/app/api/supabase/insert/discourse-platform/route.ts b/apps/website/app/api/supabase/insert/discourse-platform/route.ts index 40720e382..ae5cb8407 100644 --- a/apps/website/app/api/supabase/insert/discourse-platform/route.ts +++ b/apps/website/app/api/supabase/insert/discourse-platform/route.ts @@ -1,216 +1,99 @@ import { createClient } from "~/utils/supabase/server"; import { NextResponse, NextRequest } from "next/server"; -import type { SupabaseClient } from "@supabase/supabase-js"; -import cors from "~/utils/llm/cors"; +import { + getOrCreateEntity, + GetOrCreateEntityResult, +} from "~/utils/supabase/dbUtils"; +import { + createApiResponse, + handleRouteError, + defaultOptionsHandler, +} from "~/utils/supabase/apiUtils"; type DiscoursePlatformRecord = { id: number; name: string; url: string; - // Add other fields from your DiscoursePlatform table if they are selected -}; - -type GetOrCreateDiscoursePlatformReturn = { - platform: DiscoursePlatformRecord | null; - error: string | null; - details?: string; - created: boolean; // 'created' should always be boolean }; type DiscoursePlatformDataInput = { currentContentURL: string; }; -async function getOrCreateDiscoursePlatform( - supabase: SupabaseClient, +const getOrCreateDiscoursePlatformFromURL = async ( + supabase: ReturnType, currentContentURL: string, -): Promise { +): Promise> => { let platformName: string | null = null; let platformUrl: string | null = null; const lowerCaseURL = currentContentURL.toLowerCase(); if (lowerCaseURL.includes("roamresearch.com")) { platformName = "roamresearch"; - platformUrl = "https://roamresearch.com"; // Canonical URL + platformUrl = "https://roamresearch.com"; } else { console.warn("Could not determine platform from URL:", currentContentURL); return { - error: "Could not determine platform from URL", - platform: null, + error: "Could not determine platform from URL.", + entity: null, created: false, + status: 400, }; } if (!platformName || !platformUrl) { return { - error: "Platform name or URL could not be derived", - platform: null, - created: false, - }; - } - - // Try to find an existing platform by its canonical URL (which should be unique) - const { data: existingPlatform, error: fetchError } = await supabase - .from("DiscoursePlatform") - .select("id, name, url") - .eq("url", platformUrl) - .maybeSingle(); - - if (fetchError) { - console.error("Error fetching DiscoursePlatform:", fetchError); - return { - error: "Database error while fetching platform", - platform: null, - details: fetchError.message, + error: "Platform name or URL could not be derived.", + entity: null, created: false, + status: 400, }; } - if (existingPlatform) { - console.log("Found existing DiscoursePlatform:", existingPlatform); - return { error: null, platform: existingPlatform, created: false }; - } else { - console.log( - `DiscoursePlatform "${platformName}" (URL: ${platformUrl}) not found, creating new one...`, - ); - - const platformToInsert = { - name: platformName, - url: platformUrl, - }; - - const { data: newPlatform, error: insertError } = await supabase - .from("DiscoursePlatform") - .insert(platformToInsert) - .select("id, name, url") // Ensure selected fields match DiscoursePlatformRecord - .single(); // Expecting one row to be inserted and returned - - if (insertError) { - console.error("Error inserting new DiscoursePlatform:", insertError); - // Handle potential race condition where platform was created between fetch and insert - if (insertError.code === "23505") { - // Unique constraint violation - console.warn( - "Unique constraint violation on insert. Attempting to re-fetch platform by URL.", - ); - const { data: reFetchedPlatform, error: reFetchError } = await supabase - .from("DiscoursePlatform") - .select("id, name, url") - .eq("url", platformUrl) - .maybeSingle(); - - if (reFetchError) { - console.error( - "Error re-fetching DiscoursePlatform after unique constraint violation:", - reFetchError, - ); - return { - error: "Database error after unique constraint violation", - platform: null, - details: reFetchError.message, - created: false, - }; - } - if (reFetchedPlatform) { - console.log("Found platform on re-fetch:", reFetchedPlatform); - return { error: null, platform: reFetchedPlatform, created: false }; // It existed, wasn't "created" by this call - } - // If re-fetch also fails to find it, the original insert error stands - return { - error: - "Unique constraint violation on insert, and re-fetch failed to find the platform.", - platform: null, - details: insertError.message, - created: false, - }; - } - return { - error: "Database error while inserting platform", - platform: null, - details: insertError.message, - created: false, - }; - } - - console.log("Created new DiscoursePlatform:", newPlatform); - return { error: null, platform: newPlatform, created: true }; - } -} + const resolvedSupabaseClient = await supabase; + return getOrCreateEntity( + resolvedSupabaseClient, + "DiscoursePlatform", + "id, name, url", + { url: platformUrl }, + { name: platformName, url: platformUrl }, + "DiscoursePlatform", + ); +}; -export async function POST(request: NextRequest): Promise { - const supabase = await createClient(); - let response: NextResponse; +export const POST = async (request: NextRequest): Promise => { + const supabase = createClient(); try { const body: DiscoursePlatformDataInput = await request.json(); const { currentContentURL } = body; if (!currentContentURL || typeof currentContentURL !== "string") { - response = NextResponse.json( - { error: "Missing or invalid currentContentURL in request body" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; + return createApiResponse(request, { + error: "Missing or invalid currentContentURL in request body.", + status: 400, + }); } - const result = await getOrCreateDiscoursePlatform( + const result = await getOrCreateDiscoursePlatformFromURL( supabase, currentContentURL, ); - if (result.error) { - console.error( - `API Error for DiscoursePlatform (URL: ${currentContentURL}): ${result.error}`, - result.details || "", - ); - response = NextResponse.json( - { error: result.error, details: result.details }, - { status: 500 }, - ); - } else if (result.platform) { - response = NextResponse.json(result.platform, { - status: result.created ? 201 : 200, - }); - } else { - // This case should ideally be caught by the 'error' field in the result - console.error( - `API Error for DiscoursePlatform (URL: ${currentContentURL}): Platform was null without an error flag.`, - ); - response = NextResponse.json( - { - error: - "Failed to get or create DiscoursePlatform for an unknown reason", - }, - { status: 500 }, - ); - } + return createApiResponse(request, { + data: result.entity, + error: result.error, + details: result.details, + status: result.status, + created: result.created, + }); } catch (e: unknown) { - console.error( - "API route error in /api/supabase/insert/discourse-platform:", + return handleRouteError( + request, e, + "/api/supabase/insert/discourse-platform", ); - // Differentiate between JSON parsing errors and other errors - if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { - response = NextResponse.json( - { error: "Invalid JSON in request body" }, - { status: 400 }, - ); - } else { - response = NextResponse.json( - { - error: - e instanceof Error - ? e.message - : "An unexpected error occurred processing your request", - }, - { status: 500 }, - ); - } } - return cors(request, response) as NextResponse; -} +}; -export async function OPTIONS(request: NextRequest): Promise { - const response = new NextResponse(null, { status: 204 }); - return cors(request, response) as NextResponse; -} +export const OPTIONS = defaultOptionsHandler; diff --git a/apps/website/app/api/supabase/insert/discourse-space/route.ts b/apps/website/app/api/supabase/insert/discourse-space/route.ts index 03a3802c8..19bfed8fe 100644 --- a/apps/website/app/api/supabase/insert/discourse-space/route.ts +++ b/apps/website/app/api/supabase/insert/discourse-space/route.ts @@ -1,7 +1,14 @@ import { createClient } from "~/utils/supabase/server"; import { NextResponse, NextRequest } from "next/server"; -import type { SupabaseClient } from "@supabase/supabase-js"; -import cors from "~/utils/llm/cors"; +import { + getOrCreateEntity, + GetOrCreateEntityResult, +} from "~/utils/supabase/dbUtils"; +import { + createApiResponse, + handleRouteError, + defaultOptionsHandler, +} from "~/utils/supabase/apiUtils"; type DiscourseSpaceDataInput = { name: string; @@ -17,223 +24,113 @@ type DiscourseSpaceRecord = { // Add other fields from your DiscourseSpace table if they are selected }; -type GetOrCreateDiscourseSpaceReturn = { - space: DiscourseSpaceRecord | null; - error: string | null; - details?: string; - created: boolean; // 'created' should always be boolean -}; +// Renamed and refactored helper function +const processAndGetOrCreateDiscourseSpace = async ( + supabasePromise: ReturnType, + data: DiscourseSpaceDataInput, +): Promise> => { + const { name, url, discourse_platform_id } = data; -async function getOrCreateDiscourseSpace( - supabase: SupabaseClient, - name: string, - url: string, - discoursePlatformId: number, -): Promise { - if ( - !name || - !url || - discoursePlatformId === undefined || - discoursePlatformId === null - ) { + // --- Start of validation --- + if (!name || typeof name !== "string" || name.trim() === "") { return { - error: "DiscourseSpace name, URL, and discourse_platform_id are required", - space: null, + entity: null, + error: "Missing or invalid name.", created: false, + status: 400, }; } - - const normalizedUrl = url.replace(/\/$/, ""); - - const { data: existingSpace, error: fetchError } = await supabase - .from("DiscourseSpace") - .select("id, name, url, discourse_platform_id") - .eq("url", normalizedUrl) - .eq("discourse_platform_id", discoursePlatformId) - .maybeSingle(); - - if (fetchError) { - console.error( - `Error fetching DiscourseSpace (URL: ${normalizedUrl}, PlatformID: ${discoursePlatformId}):`, - fetchError, - ); + if (!url || typeof url !== "string" || url.trim() === "") { return { - error: "Database error while fetching DiscourseSpace", - space: null, - details: fetchError.message, + entity: null, + error: "Missing or invalid URL.", created: false, + status: 400, }; } - - if (existingSpace) { - console.log("Found existing DiscourseSpace:", existingSpace); - return { error: null, space: existingSpace, created: false }; - } else { - console.log( - `DiscourseSpace "${name}" (URL: ${normalizedUrl}, PlatformID: ${discoursePlatformId}) not found, creating new one...`, - ); - - const spaceToInsert = { - name: name, - url: normalizedUrl, - discourse_platform_id: discoursePlatformId, + if ( + discourse_platform_id === undefined || + discourse_platform_id === null || + typeof discourse_platform_id !== "number" + ) { + return { + entity: null, + error: "Missing or invalid discourse_platform_id.", + created: false, + status: 400, }; + } + // --- End of validation --- + + const normalizedUrl = url.trim().replace(/\/$/, ""); + const trimmedName = name.trim(); + const supabase = await supabasePromise; + + const result = await getOrCreateEntity( + supabase, + "DiscourseSpace", + "id, name, url, discourse_platform_id", + { url: normalizedUrl, discourse_platform_id: discourse_platform_id }, + { + name: trimmedName, + url: normalizedUrl, + discourse_platform_id: discourse_platform_id, + }, + "DiscourseSpace", + ); - const { data: newSpace, error: insertError } = await supabase - .from("DiscourseSpace") - .insert(spaceToInsert) - .select("id, name, url, discourse_platform_id") - .single(); - - if (insertError) { - console.error( - `Error inserting new DiscourseSpace (Name: ${name}, URL: ${normalizedUrl}, PlatformID: ${discoursePlatformId}):`, - insertError, - ); - if (insertError.code === "23505") { - console.warn( - "Unique constraint violation on insert. Attempting to re-fetch DiscourseSpace.", - ); - const { data: reFetchedSpace, error: reFetchError } = await supabase - .from("DiscourseSpace") - .select("id, name, url, discourse_platform_id") - .eq("url", normalizedUrl) - .eq("discourse_platform_id", discoursePlatformId) - .maybeSingle(); - - if (reFetchError) { - console.error( - "Error re-fetching DiscourseSpace after unique constraint violation:", - reFetchError, - ); - return { - error: "Database error after unique constraint violation", - space: null, - details: reFetchError.message, - created: false, - }; - } - if (reFetchedSpace) { - console.log("Found DiscourseSpace on re-fetch:", reFetchedSpace); - return { error: null, space: reFetchedSpace, created: false }; - } - return { - error: - "Unique constraint violation on insert, and re-fetch failed to find the DiscourseSpace.", - space: null, - details: insertError.message, - created: false, - }; - } + // Custom handling for specific foreign key error related to discourse_platform_id + if ( + result.error && + result.details && + result.status === 400 && + result.details.includes("violates foreign key constraint") + ) { + if ( + result.details + .toLowerCase() + .includes("discoursespace_discourse_platform_id_fkey") || + result.details.toLowerCase().includes("discourse_platform_id") + ) { return { - error: "Database error while inserting DiscourseSpace", - space: null, - details: insertError.message, - created: false, + ...result, + error: `Invalid discourse_platform_id: No DiscoursePlatform record found for ID ${discourse_platform_id}.`, }; } - - console.log("Created new DiscourseSpace:", newSpace); - return { error: null, space: newSpace, created: true }; } -} -export async function POST(request: NextRequest): Promise { - const supabase = await createClient(); - let response: NextResponse; + return result; +}; + +export const POST = async (request: NextRequest): Promise => { + const supabasePromise = createClient(); try { const body: DiscourseSpaceDataInput = await request.json(); - const { name, url, discourse_platform_id } = body; - if (!name || typeof name !== "string" || name.trim() === "") { - response = NextResponse.json( - { error: "Missing or invalid name in request body" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; - } - if (!url || typeof url !== "string" || url.trim() === "") { - response = NextResponse.json( - { error: "Missing or invalid url in request body" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; - } - if ( - discourse_platform_id === undefined || - discourse_platform_id === null || - typeof discourse_platform_id !== "number" - ) { - response = NextResponse.json( - { error: "Missing or invalid discourse_platform_id in request body" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; + // Minimal validation here, more detailed in the helper + if (!body || typeof body !== "object") { + return createApiResponse(request, { + error: "Invalid request body: expected a JSON object.", + status: 400, + }); } - const result = await getOrCreateDiscourseSpace( - supabase, - name.trim(), - url.trim(), - discourse_platform_id, + const result = await processAndGetOrCreateDiscourseSpace( + supabasePromise, + body, ); - if (result.error) { - console.error( - `API Error for DiscourseSpace (Name: ${name}, URL: ${url}, PlatformID: ${discourse_platform_id}): ${result.error}`, - result.details || "", - ); - const clientError = result.error.startsWith("Database error") - ? "An internal error occurred while processing the DiscourseSpace information." - : result.error; - response = NextResponse.json( - { - error: clientError, - details: result.error.startsWith("Database error") - ? undefined - : result.details, - }, - { status: 500 }, - ); - } else if (result.space) { - response = NextResponse.json(result.space, { - status: result.created ? 201 : 200, - }); - } else { - // This case should ideally not be reached if error is null and space is null, - // but it's a safeguard. - console.error( - `API Error for DiscourseSpace (Name: ${name}, URL: ${url}, PlatformID: ${discourse_platform_id}): Space was null without an error flag.`, - ); - response = NextResponse.json( - { - error: "Failed to get or create DiscourseSpace for an unknown reason", - }, - { status: 500 }, - ); - } + return createApiResponse(request, { + data: result.entity, + error: result.error, + details: result.details, + status: result.status, + created: result.created, + }); } catch (e: unknown) { - console.error( - "API route error in /api/supabase/insert/discourse-space:", - e, - ); - if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { - response = NextResponse.json( - { error: "Invalid JSON in request body" }, - { status: 400 }, - ); - } else { - response = NextResponse.json( - { error: "An unexpected error occurred processing your request" }, - { status: 500 }, - ); - } + return handleRouteError(request, e, "/api/supabase/insert/discourse-space"); } - return cors(request, response) as NextResponse; -} +}; -export async function OPTIONS(request: NextRequest): Promise { - const response = new NextResponse(null, { status: 204 }); - return cors(request, response) as NextResponse; -} +export const OPTIONS = defaultOptionsHandler; diff --git a/apps/website/app/api/supabase/insert/document/route.ts b/apps/website/app/api/supabase/insert/document/route.ts index c516f5e7c..a60ac2cc4 100644 --- a/apps/website/app/api/supabase/insert/document/route.ts +++ b/apps/website/app/api/supabase/insert/document/route.ts @@ -1,13 +1,21 @@ import { createClient } from "~/utils/supabase/server"; import { NextResponse, NextRequest } from "next/server"; -import type { SupabaseClient } from "@supabase/supabase-js"; +import { + getOrCreateEntity, + GetOrCreateEntityResult, +} from "~/utils/supabase/dbUtils"; +import { + createApiResponse, + handleRouteError, + defaultOptionsHandler, +} from "~/utils/supabase/apiUtils"; import cors from "~/utils/llm/cors"; type DocumentDataInput = { space_id: number; source_local_id?: string; url?: string; - metadata?: Record; + metadata?: Record | string; created: string; // ISO 8601 date string last_modified: string; // ISO 8601 date string author_id: number; @@ -18,169 +26,137 @@ type DocumentRecord = { space_id: number; source_local_id: string | null; url: string | null; - metadata: Record | null; // Assuming metadata is stored as JSONB + metadata: Record | null; created: string; // ISO 8601 date string last_modified: string; // ISO 8601 date string author_id: number; - // Add other fields from your Document table if they are selected }; -type CreateDocumentEntryReturn = { - document: DocumentRecord | null; - error: string | null; - details?: string; -}; - -async function createDocumentEntry( - supabase: SupabaseClient, +const createDocument = async ( + supabasePromise: ReturnType, data: DocumentDataInput, -): Promise { +): Promise> => { const { space_id, source_local_id, url, - metadata, + metadata: rawMetadata, created, last_modified, author_id, } = data; - // Validate required fields - if (!space_id || !created || !last_modified || !author_id) { + if ( + space_id === undefined || + space_id === null || + !created || + !last_modified || + author_id === undefined || + author_id === null + ) { return { - document: null, + entity: null, error: - "Missing required fields: space_id, created, last_modified, or author_id", + "Missing required fields: space_id, created, last_modified, or author_id.", + created: false, + status: 400, }; } + const processedMetadata = + rawMetadata && typeof rawMetadata === "object" + ? JSON.stringify(rawMetadata) + : typeof rawMetadata === "string" + ? rawMetadata + : null; + const documentToInsert = { space_id, - source_local_id, - url, - metadata: metadata ? JSON.stringify(metadata) : "{}", + source_local_id: source_local_id || null, + url: url || null, + metadata: processedMetadata as any, created, last_modified, author_id, }; - const { data: newDocument, error: insertError } = await supabase - .from("Document") - .insert(documentToInsert) - .select() // Consider selecting specific columns for DocumentRecord - .single(); - - if (insertError) { - console.error("Error inserting new Document:", insertError); - if (insertError.code === "23503") { - if (insertError.message.includes("space_id_fkey")) { - return { - document: null, - error: `Invalid space_id: No Space record found for ID ${space_id}.`, - details: insertError.message, - }; - } - if (insertError.message.includes("author_id_fkey")) { - return { - document: null, - error: `Invalid author_id: No Account record found for ID ${author_id}.`, - details: insertError.message, - }; - } + const supabase = await supabasePromise; + + const result = await getOrCreateEntity( + supabase, + "Document", + "id, space_id, source_local_id, url, metadata, created, last_modified, author_id", + { id: -1 }, + documentToInsert, + "Document", + ); + + if ( + result.error && + result.details && + result.status === 400 && + result.details.includes("violates foreign key constraint") + ) { + if (result.details.includes("space_id_fkey")) { + return { + ...result, + error: `Invalid space_id: No Space record found for ID ${space_id}.`, + }; + } + if (result.details.includes("author_id_fkey")) { + return { + ...result, + error: `Invalid author_id: No Account record found for ID ${author_id}.`, + }; } - return { - document: null, - error: "Database error while inserting Document", - details: insertError.message, - }; } - console.log("Created new Document:", newDocument); - return { document: newDocument, error: null }; -} + return result; +}; -export async function POST(request: NextRequest): Promise { - const supabase = await createClient(); - let response: NextResponse; +export const POST = async (request: NextRequest): Promise => { + const supabasePromise = createClient(); try { const body: DocumentDataInput = await request.json(); - // Validate required fields if (body.space_id === undefined || body.space_id === null) { - response = NextResponse.json( - { error: "Missing required field: space_id" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; + return createApiResponse(request, { + error: "Missing required field: space_id.", + status: 400, + }); } if (!body.created) { - response = NextResponse.json( - { error: "Missing required field: created" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; + return createApiResponse(request, { + error: "Missing required field: created.", + status: 400, + }); } if (!body.last_modified) { - response = NextResponse.json( - { error: "Missing required field: last_modified" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; + return createApiResponse(request, { + error: "Missing required field: last_modified.", + status: 400, + }); } if (body.author_id === undefined || body.author_id === null) { - response = NextResponse.json( - { error: "Missing required field: author_id" }, - { status: 400 }, - ); - return cors(request, response) as NextResponse; + return createApiResponse(request, { + error: "Missing required field: author_id.", + status: 400, + }); } - const { document, error, details } = await createDocumentEntry( - supabase, - body, - ); - - if (error) { - console.error(`API Error for Document creation: ${error}`, details || ""); - if ( - error.startsWith("Invalid") || - error.startsWith("Missing required fields") - ) { - response = NextResponse.json( - { error: error, details: details }, - { status: 400 }, - ); - } else { - const clientError = error.startsWith("Database error") - ? "An internal error occurred while processing Document." - : error; - response = NextResponse.json( - { error: clientError, details: details }, - { status: 500 }, - ); - } - } else { - response = NextResponse.json(document, { status: 201 }); - } + const result = await createDocument(supabasePromise, body); + + return createApiResponse(request, { + data: result.entity, + error: result.error, + details: result.details, + status: result.status, + created: result.created, + }); } catch (e: unknown) { - console.error("API route error in /api/supabase/insert/Document:", e); - if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { - response = NextResponse.json( - { error: "Invalid JSON in request body" }, - { status: 400 }, - ); - } else { - response = NextResponse.json( - { error: "An unexpected error occurred processing your request" }, - { status: 500 }, - ); - } + return handleRouteError(request, e, "/api/supabase/insert/document"); } - return cors(request, response) as NextResponse; -} +}; -export async function OPTIONS(request: NextRequest): Promise { - const response = new NextResponse(null, { status: 204 }); - return cors(request, response) as NextResponse; -} +export const OPTIONS = defaultOptionsHandler; diff --git a/apps/website/app/api/supabase/insert/person/route.ts b/apps/website/app/api/supabase/insert/person/route.ts index 0e6118417..a29fdc882 100644 --- a/apps/website/app/api/supabase/insert/person/route.ts +++ b/apps/website/app/api/supabase/insert/person/route.ts @@ -1,6 +1,14 @@ import { createClient } from "~/utils/supabase/server"; import { NextResponse, NextRequest } from "next/server"; -import type { SupabaseClient } from "@supabase/supabase-js"; +import { + getOrCreateEntity, + GetOrCreateEntityResult, +} from "~/utils/supabase/dbUtils"; +import { + createApiResponse, + handleRouteError, + defaultOptionsHandler, // Assuming OPTIONS might be added later +} from "~/utils/supabase/apiUtils"; type PersonDataInput = { name: string; @@ -17,7 +25,7 @@ type PersonRecord = { name: string; email: string; orcid: string | null; - type: string; // Assuming 'type' is the column name for person_type + type: string; }; type AccountRecord = { @@ -28,265 +36,176 @@ type AccountRecord = { write_permission: boolean; }; -type PersonResult = { - person: any | null; - account: any | null; - error: string | null; - details?: string; - person_created?: boolean; - account_created?: boolean; -}; - -type GetOrCreatePersonReturn = { +// Kept for the final API response structure +type PersonWithAccountResult = { person: PersonRecord | null; - error: string | null; - details?: string; - created: boolean; -}; - -type GetOrCreateAccountReturn = { account: AccountRecord | null; - error: string | null; - details?: string; - created: boolean; + person_created?: boolean; + account_created?: boolean; }; -async function getOrCreatePerson( - supabase: SupabaseClient, +const getOrCreatePersonInternal = async ( + supabasePromise: ReturnType, email: string, name: string, orcid: string | null | undefined, personType: string, -): Promise { - let { data: existingPerson, error: fetchError } = await supabase - .from("Person") - .select("id, name, email, orcid, type") - .eq("email", email) - .maybeSingle(); - - if (fetchError) { - console.error(`Error fetching Person by email (${email}):`, fetchError); - return { - person: null, - error: "Database error while fetching Person", - details: fetchError.message, - created: false, - }; - } - - if (existingPerson) { - console.log("Found existing Person:", existingPerson); - return { person: existingPerson, error: null, created: false }; - } else { - console.log(`Person with email "${email}" not found, creating new one...`); - const personToInsert = { - email: email, - name: name, - orcid: orcid, +): Promise> => { + const supabase = await supabasePromise; + return getOrCreateEntity( + supabase, + "Person", + "id, name, email, orcid, type", + { email: email.trim() }, + { + email: email.trim(), + name: name.trim(), + orcid: orcid || null, type: personType, - }; - const { data: newPerson, error: insertError } = await supabase - .from("Person") - .insert(personToInsert) - .select("id, name, email, orcid, type") - .single(); - - if (insertError) { - console.error( - `Error inserting new Person (email: ${email}):`, - insertError, - ); - return { - person: null, - error: "Database error while inserting Person", - details: insertError.message, - created: false, - }; - } - console.log("Created new Person:", newPerson); - return { person: newPerson, error: null, created: true }; - } -} + }, + "Person", + ); +}; -async function getOrCreateAccount( - supabase: SupabaseClient, +const getOrCreateAccountInternal = async ( + supabasePromise: ReturnType, personId: number, platformId: number, isActive: boolean, writePermission?: boolean, -): Promise { - let { data: existingAccount, error: fetchError } = await supabase - .from("Account") - .select("id, person_id, platform_id, active, write_permission") - .eq("person_id", personId) - .eq("platform_id", platformId) - .maybeSingle(); - - if (fetchError) { - console.error( - `Error fetching Account (PersonID: ${personId}, PlatformID: ${platformId}):`, - fetchError, - ); - return { - account: null, - error: "Database error while fetching Account", - details: fetchError.message, - created: false, - }; - } - - if (existingAccount) { - console.log("Found existing Account:", existingAccount); - return { account: existingAccount, error: null, created: false }; - } else { - console.log( - `Account for PersonID ${personId} on PlatformID ${platformId} not found, creating new one...`, - ); - const accountToInsert: Partial & { - person_id: number; - platform_id: number; - active: boolean; - } = { +): Promise> => { + const supabase = await supabasePromise; + const result = await getOrCreateEntity( + supabase, + "Account", + "id, person_id, platform_id, active, write_permission", + { person_id: personId, platform_id: platformId }, + { person_id: personId, platform_id: platformId, active: isActive, - }; - if (writePermission !== undefined) { - accountToInsert.write_permission = writePermission; + write_permission: writePermission === undefined ? true : writePermission, // Default to true if undefined + }, + "Account", + ); + + // Custom handling for specific foreign key errors + if ( + result.error && + result.details && + result.status === 400 && + result.details.includes("violates foreign key constraint") + ) { + if (result.details.includes("Account_person_id_fkey")) { + return { + ...result, + error: `Invalid person_id for Account: No Person record found for ID ${personId}.`, + }; } - - const { data: newAccount, error: insertError } = await supabase - .from("Account") - .insert(accountToInsert) - .select("id, person_id, platform_id, active, write_permission") - .single(); - - if (insertError) { - console.error( - `Error inserting new Account (PersonID: ${personId}, PlatformID: ${platformId}):`, - insertError, - ); + if (result.details.includes("Account_platform_id_fkey")) { return { - account: null, - error: "Database error while inserting Account", - details: insertError.message, - created: false, + ...result, + error: `Invalid platform_id for Account: No DiscoursePlatform record found for ID ${platformId}.`, }; } - console.log("Created new Account:", newAccount); - return { account: newAccount, error: null, created: true }; } -} + return result; +}; -export async function POST(request: NextRequest): Promise { - const supabase = await createClient(); +export const POST = async (request: NextRequest): Promise => { + const supabasePromise = createClient(); try { const body: PersonDataInput = await request.json(); const { name, email, - orcid = null, - person_type = "Person", + orcid = null, // Default from input + person_type = "Person", // Default from input account_platform_id, - account_active = true, - account_write_permission, + account_active = true, // Default from input + account_write_permission, // No default here, handled in getOrCreateAccountInternal } = body; + // Initial input validation if (!name || typeof name !== "string" || name.trim() === "") { - return NextResponse.json( - { error: "Missing or invalid name for Person" }, - { status: 400 }, - ); + return createApiResponse(request, { + error: "Missing or invalid name for Person.", + status: 400, + }); } if (!email || typeof email !== "string" || email.trim() === "") { - return NextResponse.json( - { error: "Missing or invalid email for Person" }, - { status: 400 }, - ); + return createApiResponse(request, { + error: "Missing or invalid email for Person.", + status: 400, + }); } if ( account_platform_id === undefined || account_platform_id === null || typeof account_platform_id !== "number" ) { - return NextResponse.json( - { error: "Missing or invalid account_platform_id for Account" }, - { status: 400 }, - ); + return createApiResponse(request, { + error: "Missing or invalid account_platform_id for Account.", + status: 400, + }); } - const personResult = await getOrCreatePerson( - supabase, - email.trim(), - name.trim(), + // Get or Create Person + const personResult = await getOrCreatePersonInternal( + supabasePromise, // Pass the promise + email, + name, orcid, person_type, ); - if (personResult.error || !personResult.person) { - console.error( - `API Error during Person processing (Email: ${email}): ${personResult.error}`, - personResult.details || "", - ); - const clientError = personResult.error?.startsWith("Database error") - ? "An internal error occurred while processing Person." - : personResult.error; - return NextResponse.json( - { error: clientError, details: personResult.details }, - { status: 500 }, - ); + if (personResult.error || !personResult.entity) { + return createApiResponse(request, { + error: personResult.error || "Failed to process Person.", + details: personResult.details, + status: personResult.status || 500, + }); } - const accountResult = await getOrCreateAccount( - supabase, - personResult.person.id, + // Get or Create Account + const accountResult = await getOrCreateAccountInternal( + supabasePromise, // Pass the promise again, it will resolve the same client or a new one if needed by createClient impl. + personResult.entity.id, account_platform_id, account_active, account_write_permission, ); - if (accountResult.error || !accountResult.account) { - console.error( - `API Error during Account processing (PersonID: ${personResult.person.id}, PlatformID: ${account_platform_id}): ${accountResult.error}`, - accountResult.details || "", - ); - const clientError = accountResult.error?.startsWith("Database error") - ? "An internal error occurred while processing Account." - : accountResult.error; - return NextResponse.json( - { - error: clientError, - details: accountResult.details, - person: personResult.person, - }, - { status: 500 }, - ); + if (accountResult.error || !accountResult.entity) { + // If account creation fails, return error but include successfully processed person + return createApiResponse(request, { + error: accountResult.error || "Failed to process Account.", + details: accountResult.details, + status: accountResult.status || 500, + // Optionally include person data if account failed + // data: { person: personResult.entity, person_created: personResult.created } + }); } - const statusCode = + const responsePayload: PersonWithAccountResult = { + person: personResult.entity, + account: accountResult.entity, + person_created: personResult.created, + account_created: accountResult.created, + }; + + const overallStatus = personResult.created || accountResult.created ? 201 : 200; - return NextResponse.json( - { - person: personResult.person, - account: accountResult.account, - person_created: personResult.created, - account_created: accountResult.created, - }, - { status: statusCode }, - ); + return createApiResponse(request, { + data: responsePayload, + status: overallStatus, + }); } catch (e: unknown) { - console.error("API route error in /api/supabase/insert/Person:", e); - if (e instanceof SyntaxError && e.message.toLowerCase().includes("json")) { - return NextResponse.json( - { error: "Invalid JSON in request body" }, - { status: 400 }, - ); - } - return NextResponse.json( - { error: "An unexpected error occurred processing your request" }, - { status: 500 }, - ); + return handleRouteError(request, e, "/api/supabase/insert/person"); } -} +}; + +// If you need an OPTIONS handler for this route: +// export const OPTIONS = defaultOptionsHandler; diff --git a/apps/website/app/utils/supabase/apiUtils.ts b/apps/website/app/utils/supabase/apiUtils.ts new file mode 100644 index 000000000..1703d27ec --- /dev/null +++ b/apps/website/app/utils/supabase/apiUtils.ts @@ -0,0 +1,88 @@ +import { NextResponse, NextRequest } from "next/server"; +import cors from "~/utils/llm/cors"; // Assuming this path is correct and accessible + +/** + * Sends a standardized JSON response. + * @param request The original NextRequest. + * @param data The data payload for successful responses. + * @param error An error message string if the operation failed. + * @param details Optional detailed error information. + * @param status The HTTP status code for the response. + * @param created A boolean indicating if a resource was created (influences status code: 201 vs 200). + */ +export function createApiResponse( + request: NextRequest, + payload: { + data?: T | null; + error?: string | null; + details?: string | null; + status: number; + created?: boolean; + }, +): NextResponse { + let response: NextResponse; + const { data, error, details, status, created } = payload; + + if (error) { + // console.error(`API Error (status ${status}): ${error}`, details || ""); // Logging done by callers or specific error handlers + response = NextResponse.json( + { error, details: details || undefined }, + { status }, + ); + } else if (data !== undefined && data !== null) { + const effectiveStatus = created ? 201 : status === 201 ? 200 : status; + response = NextResponse.json(data, { status: effectiveStatus }); + } else { + // Fallback for unexpected state (e.g. no error, but no data for a success status) + console.error( + `API Response Error: Attempted to send success response (status ${status}) with no data and no error.`, + ); + response = NextResponse.json( + { + error: + "An unexpected server error occurred during response generation.", + }, + { status: 500 }, + ); + } + return cors(request, response) as NextResponse; +} + +/** + * Handles errors caught in the main try-catch block of an API route. + * Differentiates JSON parsing errors from other errors. + */ +export function handleRouteError( + request: NextRequest, + error: unknown, + routeName: string, +): NextResponse { + console.error(`API route error in ${routeName}:`, error); + if ( + error instanceof SyntaxError && + error.message.toLowerCase().includes("json") + ) { + return createApiResponse(request, { + error: "Invalid JSON in request body.", + status: 400, + }); + } + const message = + error instanceof Error + ? error.message + : "An unexpected error occurred processing your request."; + return createApiResponse(request, { + error: message, + status: 500, + }); +} + +/** + * Default OPTIONS handler for CORS preflight requests. + */ +export async function defaultOptionsHandler( + request: NextRequest, +): Promise { + const response = new NextResponse(null, { status: 204 }); + return cors(request, response) as NextResponse; +} diff --git a/apps/website/app/utils/supabase/dbUtils.ts b/apps/website/app/utils/supabase/dbUtils.ts new file mode 100644 index 000000000..8663e1a71 --- /dev/null +++ b/apps/website/app/utils/supabase/dbUtils.ts @@ -0,0 +1,256 @@ +import type { SupabaseClient, PostgrestError } from "@supabase/supabase-js"; + +export type GetOrCreateEntityResult = { + entity: T | null; + error: string | null; + details?: string; // For detailed error messages, e.g., from Supabase + created: boolean; + status: number; // HTTP status code to suggest +}; + +/** + * Generic function to get an entity or create it if it doesn't exist. + * Handles common race conditions for unique constraint violations. + * + * @param supabase Supabase client instance. + * @param tableName The name of the table. + * @param selectQuery The select string, e.g., "id, name, url". + * @param matchCriteria An object representing the WHERE clause for lookup, e.g., { url: "some-url" }. + * @param insertData Data to insert if the entity is not found. + * @param entityName A friendly name for the entity, used in logging and error messages. + * @returns Promise> + */ +export async function getOrCreateEntity( + supabase: SupabaseClient, + tableName: string, + selectQuery: string, + matchCriteria: Record, + insertData: Omit & Record, // Flexible insert data + entityName: string = tableName, +): Promise> { + // 1. Try to fetch existing entity + let queryBuilder = supabase.from(tableName).select(selectQuery); + for (const key in matchCriteria) { + queryBuilder = queryBuilder.eq(key, matchCriteria[key]); + } + const { data: existingEntity, error: fetchError } = + await queryBuilder.maybeSingle(); + + if (fetchError) { + console.error(`Error fetching ${entityName} by`, matchCriteria, fetchError); + return { + entity: null, + error: `Database error while fetching ${entityName}.`, + details: fetchError.message, + created: false, + status: 500, + }; + } + + if (existingEntity) { + console.log(`Found existing ${entityName}:`, existingEntity); + return { + entity: existingEntity, + error: null, + created: false, + status: 200, + }; + } + + // 2. Create new entity if not found + console.log( + `${entityName} not found with criteria`, + matchCriteria, + `creating new one...`, + ); + const { data: newEntity, error: insertError } = await supabase + .from(tableName) + .insert(insertData) + .select(selectQuery) + .single(); + + if (insertError) { + console.error( + `Error inserting new ${entityName}:`, + insertData, + insertError, + ); + // Handle race condition: unique constraint violation (PostgreSQL error code '23505') + if (insertError.code === "23505") { + console.warn( + `Unique constraint violation on ${entityName} insert for`, + matchCriteria, + `Attempting to re-fetch.`, + ); + let reFetchQueryBuilder = supabase.from(tableName).select(selectQuery); + for (const key in matchCriteria) { + reFetchQueryBuilder = reFetchQueryBuilder.eq(key, matchCriteria[key]); + } + const { data: reFetchedEntity, error: reFetchError } = + await reFetchQueryBuilder.maybeSingle(); + + if (reFetchError) { + console.error( + `Error re-fetching ${entityName} after unique constraint violation:`, + reFetchError, + ); + return { + entity: null, + error: `Database error after unique constraint violation for ${entityName}.`, + details: reFetchError.message, + created: false, + status: 500, + }; + } + if (reFetchedEntity) { + console.log(`Found ${entityName} on re-fetch:`, reFetchedEntity); + return { + entity: reFetchedEntity, + error: null, + created: false, + status: 200, // Successfully fetched, though not created by this call + }; + } + return { + entity: null, + error: `Unique constraint violation on ${entityName} insert, and re-fetch failed to find the entity.`, + details: insertError.message, // Original insert error + created: false, + status: 409, // Conflict, and couldn't resolve by re-fetching + }; + } + // Handle foreign key constraint violations (PostgreSQL error code '23503') + if (insertError.code === "23503") { + return { + entity: null, + error: `Invalid reference: A foreign key constraint was violated while creating ${entityName}.`, + details: insertError.message, // Specific FK details are in the message + created: false, + status: 400, // Usually due to bad input ID + }; + } + + return { + entity: null, + error: `Database error while inserting ${entityName}.`, + details: insertError.message, + created: false, + status: 500, + }; + } + + if (!newEntity) { + // This case should ideally not be reached if insertError is null and .single() is used. + console.error( + `New ${entityName} was not returned after insert, despite no reported Supabase error.`, + ); + return { + entity: null, + error: `Failed to retrieve new ${entityName} after insert operation.`, + details: + "The insert operation might have appeared successful but returned no data.", + created: false, // Unsure if created + status: 500, + }; + } + + console.log(`Created new ${entityName}:`, newEntity); + return { entity: newEntity, error: null, created: true, status: 201 }; +} + +export type BatchItemValidator = ( + item: TInput, + index: number, +) => { valid: boolean; error?: string; processedItem?: TProcessed }; + +export type BatchProcessResult = { + data?: TRecord[]; + error?: string; + details?: string; // For DB error details + partial_errors?: { index: number; error: string }[]; + status: number; // HTTP status to suggest +}; + +export async function processAndInsertBatch( + supabase: SupabaseClient, + items: TInput[], + tableName: string, + selectQuery: string, // e.g., "id, field1, field2" or "*" + itemValidatorAndProcessor: BatchItemValidator, + entityName: string = tableName, // For logging +): Promise> { + if (!Array.isArray(items) || items.length === 0) { + return { + error: `Request body must be a non-empty array of ${entityName} items.`, + status: 400, + }; + } + + const validationErrors: { index: number; error: string }[] = []; + const processedForDb: TProcessed[] = []; + + for (let i = 0; i < items.length; i++) { + const item = items[i]; + if (!item) { + // Handles undefined/null items in the array itself + validationErrors.push({ index: i, error: "Item is undefined or null." }); + continue; + } + const { valid, error, processedItem } = itemValidatorAndProcessor(item, i); + if (!valid || !processedItem) { + validationErrors.push({ index: i, error: error || "Validation failed." }); + } else { + processedForDb.push(processedItem); + } + } + + if (validationErrors.length > 0) { + return { + error: `Validation failed for one or more ${entityName} items.`, + partial_errors: validationErrors, + status: 400, + }; + } + + if (processedForDb.length === 0 && items.length > 0) { + return { + error: `All ${entityName} items in the batch failed validation or processing.`, + partial_errors: + validationErrors.length > 0 + ? validationErrors + : [{ index: 0, error: `No valid ${entityName} items to process.` }], + status: 400, + }; + } + + const { data: newRecords, error: insertError } = await supabase + .from(tableName) + .insert(processedForDb as any) // Cast as any if TProcessed is not directly insertable; ensure TProcessed matches table + .select(selectQuery); // Use the provided select query + + if (insertError) { + console.error(`Error batch inserting ${entityName}:`, insertError); + return { + error: `Database error during batch insert of ${entityName}.`, + details: insertError.message, + status: 500, + }; + } + + const newRecordsTyped = newRecords as TRecord[]; // Assert type + + if (!newRecordsTyped || newRecordsTyped.length !== processedForDb.length) { + console.warn( + `Batch insert ${entityName}: Mismatch between processed count (${processedForDb.length}) and DB returned count (${newRecordsTyped?.length || 0}).`, + ); + return { + error: `Batch insert of ${entityName} might have partially failed or returned unexpected data.`, + status: 500, // Or a more specific error + }; + } + + console.log( + `Successfully batch inserted ${newRecordsTyped.length} ${entityName} records.`, + ); + return { data: newRecordsTyped, status: 201 }; +} From 06d53048258135072a7109d5096137f555451932 Mon Sep 17 00:00:00 2001 From: sid597 Date: Tue, 20 May 2025 13:55:23 +0530 Subject: [PATCH 13/19] fx --- apps/roam/src/components/DiscourseContextOverlay.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/roam/src/components/DiscourseContextOverlay.tsx b/apps/roam/src/components/DiscourseContextOverlay.tsx index ac326325b..346fb3d8e 100644 --- a/apps/roam/src/components/DiscourseContextOverlay.tsx +++ b/apps/roam/src/components/DiscourseContextOverlay.tsx @@ -294,7 +294,7 @@ const DiscourseContextOverlay = ({ const handleCreateBlock = async (node: SuggestedNode) => { await createBlock({ parentUid: blockUid, - node: { text: `[[${nodeText}]]` }, + node: { text: `[[${node.text}]]` }, }); setHydeFilteredNodes(hydeFilteredNodes.filter((n) => n.uid !== node.uid)); }; @@ -386,7 +386,7 @@ const DiscourseContextOverlay = ({ {selectedPage && (

- Suggested Relationships (Ranked by HyDE) + Suggested Relationships

{isSearchingHyde && ( From aef9fd07c1517859598b568dfb346055c4f697fa Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Mon, 19 May 2025 11:21:04 -0400 Subject: [PATCH 14/19] New version with generated types --- .../app/api/supabase/insert/account/route.ts | 19 +- .../app/api/supabase/insert/agents/route.ts | 23 +- .../batch/route.ts | 36 +- .../routes.ts | 38 +- .../supabase/insert/content/batch/route.ts | 44 +- .../app/api/supabase/insert/content/route.ts | 36 +- .../insert/discourse-platform/route.ts | 31 +- .../supabase/insert/discourse-space/route.ts | 18 +- .../app/api/supabase/insert/document/route.ts | 26 +- .../app/api/supabase/insert/person/route.ts | 148 ++-- apps/website/app/utils/supabase/dbUtils.ts | 35 +- apps/website/app/utils/supabase/server.ts | 3 +- apps/website/app/utils/supabase/types.gen.ts | 796 ++++++++++++++++++ 13 files changed, 957 insertions(+), 296 deletions(-) create mode 100644 apps/website/app/utils/supabase/types.gen.ts diff --git a/apps/website/app/api/supabase/insert/account/route.ts b/apps/website/app/api/supabase/insert/account/route.ts index 7ec574701..25a727b46 100644 --- a/apps/website/app/api/supabase/insert/account/route.ts +++ b/apps/website/app/api/supabase/insert/account/route.ts @@ -9,21 +9,10 @@ import { handleRouteError, defaultOptionsHandler, } from "~/utils/supabase/apiUtils"; +import { Tables, TablesInsert } from "~/utils/supabase/types.gen"; -type AccountDataInput = { - person_id: number; - platform_id: number; - active?: boolean; - write_permission?: boolean; -}; - -type AccountRecord = { - id: number; - person_id: number; - platform_id: number; - active: boolean; - write_permission: boolean; -}; +type AccountDataInput = TablesInsert<"Account">; +type AccountRecord = Tables<"Account">; const getOrCreateAccount = async ( supabasePromise: ReturnType, @@ -53,7 +42,7 @@ const getOrCreateAccount = async ( const supabase = await supabasePromise; - const result = await getOrCreateEntity( + const result = await getOrCreateEntity<"Account">( supabase, "Account", "id, person_id, platform_id, active, write_permission", diff --git a/apps/website/app/api/supabase/insert/agents/route.ts b/apps/website/app/api/supabase/insert/agents/route.ts index dd0792f76..ea82d3829 100644 --- a/apps/website/app/api/supabase/insert/agents/route.ts +++ b/apps/website/app/api/supabase/insert/agents/route.ts @@ -9,23 +9,16 @@ import { handleRouteError, defaultOptionsHandler, } from "~/utils/supabase/apiUtils"; +import { Database, Tables, TablesInsert } from "~/utils/supabase/types.gen"; -type AgentDataInput = { - type: string; -}; - -type AgentRecord = { - id: number; - type: string; -}; +type AgentDataInput = TablesInsert<"Agent">; +type AgentRecord = Tables<"Agent">; const getOrCreateAgentByType = async ( supabasePromise: ReturnType, - agentType: string, + agentType: Database["public"]["Enums"]["EntityType"], ): Promise> => { - const type = agentType.trim(); - - if (!type) { + if (!agentType) { return { entity: null, error: "Missing or invalid 'type' for Agent.", @@ -37,12 +30,12 @@ const getOrCreateAgentByType = async ( const supabase = await supabasePromise; - return getOrCreateEntity( + return getOrCreateEntity<"Agent">( supabase, "Agent", "id, type", - { type: type }, - { type: type }, + { type: agentType }, + { type: agentType }, "Agent", ); }; diff --git a/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/batch/route.ts b/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/batch/route.ts index 9197db679..986bbae7f 100644 --- a/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/batch/route.ts +++ b/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/batch/route.ts @@ -10,33 +10,20 @@ import { BatchItemValidator, BatchProcessResult, } from "~/utils/supabase/dbUtils"; -import cors from "~/utils/llm/cors"; +import { Tables, TablesInsert } from "~/utils/supabase/types.gen"; -// Input type for a single item in the batch -interface ContentEmbeddingBatchItemInput { - target_id: number; // Foreign key to Content.id - model: string; - vector: number[] | string; // Accept string for pre-formatted, or number[] - obsolete?: boolean; -} +type ContentEmbeddingDataInput = + TablesInsert<"ContentEmbedding_openai_text_embedding_3_small_1536">; +type ContentEmbeddingRecord = + Tables<"ContentEmbedding_openai_text_embedding_3_small_1536">; // Request body is an array of these items -type ContentEmbeddingBatchRequestBody = ContentEmbeddingBatchItemInput[]; - -// Type for the record as stored/retrieved from DB (ensure fields match your table) -interface ContentEmbeddingRecord { - id: number; // Assuming auto-generated ID - target_id: number; - model: string; - vector: string; // Stored as string (JSON.stringify of number[]) - obsolete: boolean; - // created_at?: string; // If you have timestamps and want to select them -} +type ContentEmbeddingBatchRequestBody = ContentEmbeddingDataInput[]; // Type for the item after processing, ready for DB insert -type ProcessedEmbeddingItem = Omit & { +type ProcessedEmbeddingItem = Omit & { vector: string; // Vector is always stringified - obsolete: boolean; // Obsolete has a default + obsolete: boolean | null; // Obsolete has a default }; const TARGET_EMBEDDING_TABLE = @@ -44,7 +31,7 @@ const TARGET_EMBEDDING_TABLE = // Validator and processor for embedding items const validateAndProcessEmbeddingItem: BatchItemValidator< - ContentEmbeddingBatchItemInput, + ContentEmbeddingDataInput, ProcessedEmbeddingItem > = (item, index) => { if (item.target_id === undefined || item.target_id === null) { @@ -92,9 +79,8 @@ const batchInsertEmbeddingsProcess = async ( embeddingItems: ContentEmbeddingBatchRequestBody, ): Promise> => { return processAndInsertBatch< - ContentEmbeddingBatchItemInput, - ProcessedEmbeddingItem, - ContentEmbeddingRecord + "ContentEmbedding_openai_text_embedding_3_small_1536", + ProcessedEmbeddingItem >( supabase, embeddingItems, diff --git a/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts b/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts index 23c391b34..1a2d3379a 100644 --- a/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts +++ b/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts @@ -9,23 +9,12 @@ import { getOrCreateEntity, GetOrCreateEntityResult, } from "~/utils/supabase/dbUtils"; +import { Tables, TablesInsert } from "~/utils/supabase/types.gen"; -// Input type -interface ContentEmbeddingDataInput { - target_id: number; - model: string; - vector: number[]; // Singular route expects number[] directly - obsolete?: boolean; -} - -// DB record type -interface ContentEmbeddingRecord { - id: number; - target_id: number; - model: string; - vector: string; // Stored as stringified JSON - obsolete: boolean; -} +type ContentEmbeddingDataInput = + TablesInsert<"ContentEmbedding_openai_text_embedding_3_small_1536">; +type ContentEmbeddingRecord = + Tables<"ContentEmbedding_openai_text_embedding_3_small_1536">; const TARGET_EMBEDDING_TABLE = "ContentEmbedding_openai_text_embedding_3_small_1536"; @@ -93,14 +82,15 @@ const processAndCreateEmbedding = async ( // Using getOrCreateEntity, forcing create path by providing non-matching criteria // This standardizes return type and error handling (e.g., FK violations from dbUtils) - const result = await getOrCreateEntity( - supabase, - TARGET_EMBEDDING_TABLE, - "*", // Select all fields for the record - { id: -1 }, // Non-matching criteria to force "create" path - embeddingToInsert, - "ContentEmbedding", - ); + const result = + await getOrCreateEntity<"ContentEmbedding_openai_text_embedding_3_small_1536">( + supabase, + TARGET_EMBEDDING_TABLE, + "*", // Select all fields for the record + { id: -1 }, // Non-matching criteria to force "create" path + embeddingToInsert, + "ContentEmbedding", + ); // getOrCreateEntity handles general 23503, but we can make the message more specific if needed if ( diff --git a/apps/website/app/api/supabase/insert/content/batch/route.ts b/apps/website/app/api/supabase/insert/content/batch/route.ts index 129635a92..cf4d8e997 100644 --- a/apps/website/app/api/supabase/insert/content/batch/route.ts +++ b/apps/website/app/api/supabase/insert/content/batch/route.ts @@ -10,45 +10,18 @@ import { BatchItemValidator, BatchProcessResult, // Import BatchProcessResult for the return type } from "~/utils/supabase/dbUtils"; // Ensure this path is correct +import { Tables, TablesInsert } from "~/utils/supabase/types.gen"; -// Based on the Content table schema and usage in embeddingWorkflow.ts -// This is for a single content item in the batch -type ContentBatchItemInput = { - text: string; - scale: string; - space_id: number; - author_id: number; // This is Person.id (Agent.id) - document_id: number; - source_local_id?: string; - metadata?: Record | string | null; // Allow string for pre-stringified, or null - created: string; // ISO 8601 date string - last_modified: string; // ISO 8601 date string - part_of_id?: number; -}; +type ContentDataInput = TablesInsert<"Content">; +type ContentRecord = Tables<"Content">; // The request body will be an array of these items -type ContentBatchRequestBody = ContentBatchItemInput[]; - -// Define a type for the actual record stored in/retrieved from DB for Content -type ContentRecord = { - id: number; - text: string; - scale: string; - space_id: number; - author_id: number; - document_id: number; - source_local_id: string | null; - metadata: Record | null; // Assuming metadata is stored as JSONB - created: string; // ISO 8601 date string - last_modified: string; // ISO 8601 date string - part_of_id: number | null; - // Add other fields from your Content table if they are selected -}; +type ContentBatchRequestBody = ContentDataInput[]; // Specific validator and processor for Content items const validateAndProcessContentItem: BatchItemValidator< - ContentBatchItemInput, - Omit & { metadata: string | null } // TProcessed type + ContentDataInput, + Omit & { metadata: string | null } // TProcessed type > = (item, index) => { // No need to check for !item here, processAndInsertBatch handles null/undefined items in the array itself const requiredFields = [ @@ -89,9 +62,8 @@ const batchInsertContentProcess = async ( contentItems: ContentBatchRequestBody, ): Promise> => { return processAndInsertBatch< - ContentBatchItemInput, - Omit & { metadata: string | null }, - ContentRecord + "Content", + Omit & { metadata: string | null } >( supabase, contentItems, diff --git a/apps/website/app/api/supabase/insert/content/route.ts b/apps/website/app/api/supabase/insert/content/route.ts index e81251cd2..e7d3ebfd1 100644 --- a/apps/website/app/api/supabase/insert/content/route.ts +++ b/apps/website/app/api/supabase/insert/content/route.ts @@ -9,36 +9,10 @@ import { handleRouteError, defaultOptionsHandler, } from "~/utils/supabase/apiUtils"; // Ensure path is correct -import cors from "~/utils/llm/cors"; - -// Based on the Content table schema and usage in embeddingWorkflow.ts -type ContentDataInput = { - text: string; - scale: string; - space_id: number; - author_id: number; - source_local_id?: string; - metadata?: Record | string; // Allow string for pre-stringified metadata - created: string; // ISO 8601 date string - last_modified: string; // ISO 8601 date string - document_id?: number; - part_of_id?: number; -}; +import { Tables, TablesInsert } from "~/utils/supabase/types.gen"; -type ContentRecord = { - id: number; - text: string; - scale: string; - space_id: number; - author_id: number; - source_local_id: string | null; - metadata: Record | null; - created: string; - last_modified: string; - document_id: number | null; - part_of_id: number | null; - // Add other fields from your Content table if they are selected -}; +type ContentDataInput = TablesInsert<"Content">; +type ContentRecord = Tables<"Content">; // Renamed and refactored const processAndUpsertContentEntry = async ( @@ -141,7 +115,7 @@ const processAndUpsertContentEntry = async ( metadata: processedMetadata as any, created, last_modified, - document_id: document_id || null, + document_id: document_id, part_of_id: part_of_id || null, }; @@ -152,7 +126,7 @@ const processAndUpsertContentEntry = async ( // If no solid matchCriteria for a "get", getOrCreateEntity will likely proceed to "create". // If there are unique constraints other than (space_id, source_local_id), it will handle race conditions. - const result = await getOrCreateEntity( + const result = await getOrCreateEntity<"Content">( supabase, "Content", "*", // Select all fields for ContentRecord diff --git a/apps/website/app/api/supabase/insert/discourse-platform/route.ts b/apps/website/app/api/supabase/insert/discourse-platform/route.ts index ae5cb8407..1daf7ad79 100644 --- a/apps/website/app/api/supabase/insert/discourse-platform/route.ts +++ b/apps/website/app/api/supabase/insert/discourse-platform/route.ts @@ -9,30 +9,24 @@ import { handleRouteError, defaultOptionsHandler, } from "~/utils/supabase/apiUtils"; +import { Tables, TablesInsert } from "~/utils/supabase/types.gen"; -type DiscoursePlatformRecord = { - id: number; - name: string; - url: string; -}; - -type DiscoursePlatformDataInput = { - currentContentURL: string; -}; +type DiscoursePlatformDataInput = TablesInsert<"DiscoursePlatform">; +type DiscoursePlatformRecord = Tables<"DiscoursePlatform">; const getOrCreateDiscoursePlatformFromURL = async ( supabase: ReturnType, - currentContentURL: string, + url: string, ): Promise> => { let platformName: string | null = null; let platformUrl: string | null = null; - const lowerCaseURL = currentContentURL.toLowerCase(); + const lowerCaseURL = url.toLowerCase(); if (lowerCaseURL.includes("roamresearch.com")) { platformName = "roamresearch"; platformUrl = "https://roamresearch.com"; } else { - console.warn("Could not determine platform from URL:", currentContentURL); + console.warn("Could not determine platform from URL:", url); return { error: "Could not determine platform from URL.", entity: null, @@ -51,7 +45,7 @@ const getOrCreateDiscoursePlatformFromURL = async ( } const resolvedSupabaseClient = await supabase; - return getOrCreateEntity( + return getOrCreateEntity<"DiscoursePlatform">( resolvedSupabaseClient, "DiscoursePlatform", "id, name, url", @@ -66,19 +60,16 @@ export const POST = async (request: NextRequest): Promise => { try { const body: DiscoursePlatformDataInput = await request.json(); - const { currentContentURL } = body; + const { url } = body; - if (!currentContentURL || typeof currentContentURL !== "string") { + if (!url || typeof url !== "string") { return createApiResponse(request, { - error: "Missing or invalid currentContentURL in request body.", + error: "Missing or invalid url in request body.", status: 400, }); } - const result = await getOrCreateDiscoursePlatformFromURL( - supabase, - currentContentURL, - ); + const result = await getOrCreateDiscoursePlatformFromURL(supabase, url); return createApiResponse(request, { data: result.entity, diff --git a/apps/website/app/api/supabase/insert/discourse-space/route.ts b/apps/website/app/api/supabase/insert/discourse-space/route.ts index 19bfed8fe..49aa80f90 100644 --- a/apps/website/app/api/supabase/insert/discourse-space/route.ts +++ b/apps/website/app/api/supabase/insert/discourse-space/route.ts @@ -9,20 +9,10 @@ import { handleRouteError, defaultOptionsHandler, } from "~/utils/supabase/apiUtils"; +import { Tables, TablesInsert } from "~/utils/supabase/types.gen"; -type DiscourseSpaceDataInput = { - name: string; - url: string; - discourse_platform_id: number; -}; - -type DiscourseSpaceRecord = { - id: number; - name: string; - url: string; - discourse_platform_id: number; - // Add other fields from your DiscourseSpace table if they are selected -}; +type DiscourseSpaceDataInput = TablesInsert<"DiscourseSpace">; +type DiscourseSpaceRecord = Tables<"DiscourseSpace">; // Renamed and refactored helper function const processAndGetOrCreateDiscourseSpace = async ( @@ -66,7 +56,7 @@ const processAndGetOrCreateDiscourseSpace = async ( const trimmedName = name.trim(); const supabase = await supabasePromise; - const result = await getOrCreateEntity( + const result = await getOrCreateEntity<"DiscourseSpace">( supabase, "DiscourseSpace", "id, name, url, discourse_platform_id", diff --git a/apps/website/app/api/supabase/insert/document/route.ts b/apps/website/app/api/supabase/insert/document/route.ts index a60ac2cc4..faa1ec9c9 100644 --- a/apps/website/app/api/supabase/insert/document/route.ts +++ b/apps/website/app/api/supabase/insert/document/route.ts @@ -9,28 +9,10 @@ import { handleRouteError, defaultOptionsHandler, } from "~/utils/supabase/apiUtils"; -import cors from "~/utils/llm/cors"; +import { Tables, TablesInsert } from "~/utils/supabase/types.gen"; -type DocumentDataInput = { - space_id: number; - source_local_id?: string; - url?: string; - metadata?: Record | string; - created: string; // ISO 8601 date string - last_modified: string; // ISO 8601 date string - author_id: number; -}; - -type DocumentRecord = { - id: number; - space_id: number; - source_local_id: string | null; - url: string | null; - metadata: Record | null; - created: string; // ISO 8601 date string - last_modified: string; // ISO 8601 date string - author_id: number; -}; +type DocumentDataInput = TablesInsert<"Document">; +type DocumentRecord = Tables<"Document">; const createDocument = async ( supabasePromise: ReturnType, @@ -82,7 +64,7 @@ const createDocument = async ( const supabase = await supabasePromise; - const result = await getOrCreateEntity( + const result = await getOrCreateEntity<"Document">( supabase, "Document", "id, space_id, source_local_id, url, metadata, created, last_modified, author_id", diff --git a/apps/website/app/api/supabase/insert/person/route.ts b/apps/website/app/api/supabase/insert/person/route.ts index a29fdc882..a4fb765a2 100644 --- a/apps/website/app/api/supabase/insert/person/route.ts +++ b/apps/website/app/api/supabase/insert/person/route.ts @@ -9,59 +9,47 @@ import { handleRouteError, defaultOptionsHandler, // Assuming OPTIONS might be added later } from "~/utils/supabase/apiUtils"; +import { Tables, TablesInsert } from "~/utils/supabase/types.gen"; -type PersonDataInput = { - name: string; - email: string; - orcid?: string | null; - person_type?: string; - account_platform_id: number; - account_active?: boolean; - account_write_permission?: boolean; -}; - -type PersonRecord = { - id: number; - name: string; - email: string; - orcid: string | null; - type: string; -}; - -type AccountRecord = { - id: number; - person_id: number; - platform_id: number; - active: boolean; - write_permission: boolean; -}; +type PersonDataInput = TablesInsert<"Person">; +type PersonRecord = Tables<"Person">; +type AccountRecord = Tables<"Account">; // Kept for the final API response structure -type PersonWithAccountResult = { - person: PersonRecord | null; - account: AccountRecord | null; - person_created?: boolean; - account_created?: boolean; -}; +// type PersonWithAccountResult = { +// person: PersonRecord | null; +// account: AccountRecord | null; +// person_created?: boolean; +// account_created?: boolean; +// }; const getOrCreatePersonInternal = async ( supabasePromise: ReturnType, email: string, name: string, orcid: string | null | undefined, - personType: string, ): Promise> => { const supabase = await supabasePromise; - return getOrCreateEntity( + const agent_response = await getOrCreateEntity<"Agent">( + supabase, + "Agent", + "id, type", + { type: "Person" }, + { type: "Person" }, + "Agent", + ); + if (agent_response.error || agent_response.entity === null) + return agent_response as any as GetOrCreateEntityResult; + return getOrCreateEntity<"Person">( supabase, "Person", "id, name, email, orcid, type", { email: email.trim() }, { + id: agent_response.entity.id, email: email.trim(), name: name.trim(), orcid: orcid || null, - type: personType, }, "Person", ); @@ -75,7 +63,7 @@ const getOrCreateAccountInternal = async ( writePermission?: boolean, ): Promise> => { const supabase = await supabasePromise; - const result = await getOrCreateEntity( + const result = await getOrCreateEntity<"Account">( supabase, "Account", "id, person_id, platform_id, active, write_permission", @@ -115,16 +103,16 @@ const getOrCreateAccountInternal = async ( export const POST = async (request: NextRequest): Promise => { const supabasePromise = createClient(); + // MAP: Punting the joint creation of person and account. Create the account after the person. try { const body: PersonDataInput = await request.json(); const { name, email, orcid = null, // Default from input - person_type = "Person", // Default from input - account_platform_id, - account_active = true, // Default from input - account_write_permission, // No default here, handled in getOrCreateAccountInternal + // account_platform_id, + // account_active = true, // Default from input + // account_write_permission, // No default here, handled in getOrCreateAccountInternal } = body; // Initial input validation @@ -140,16 +128,16 @@ export const POST = async (request: NextRequest): Promise => { status: 400, }); } - if ( - account_platform_id === undefined || - account_platform_id === null || - typeof account_platform_id !== "number" - ) { - return createApiResponse(request, { - error: "Missing or invalid account_platform_id for Account.", - status: 400, - }); - } + // if ( + // account_platform_id === undefined || + // account_platform_id === null || + // typeof account_platform_id !== "number" + // ) { + // return createApiResponse(request, { + // error: "Missing or invalid account_platform_id for Account.", + // status: 400, + // }); + // } // Get or Create Person const personResult = await getOrCreatePersonInternal( @@ -157,7 +145,6 @@ export const POST = async (request: NextRequest): Promise => { email, name, orcid, - person_type, ); if (personResult.error || !personResult.entity) { @@ -168,38 +155,39 @@ export const POST = async (request: NextRequest): Promise => { }); } - // Get or Create Account - const accountResult = await getOrCreateAccountInternal( - supabasePromise, // Pass the promise again, it will resolve the same client or a new one if needed by createClient impl. - personResult.entity.id, - account_platform_id, - account_active, - account_write_permission, - ); - - if (accountResult.error || !accountResult.entity) { - // If account creation fails, return error but include successfully processed person - return createApiResponse(request, { - error: accountResult.error || "Failed to process Account.", - details: accountResult.details, - status: accountResult.status || 500, - // Optionally include person data if account failed - // data: { person: personResult.entity, person_created: personResult.created } - }); - } - - const responsePayload: PersonWithAccountResult = { - person: personResult.entity, - account: accountResult.entity, - person_created: personResult.created, - account_created: accountResult.created, - }; - - const overallStatus = - personResult.created || accountResult.created ? 201 : 200; + // // Get or Create Account + // const accountResult = await getOrCreateAccountInternal( + // supabasePromise, // Pass the promise again, it will resolve the same client or a new one if needed by createClient impl. + // personResult.entity.id, + // account_platform_id, + // account_active, + // account_write_permission, + // ); + + // if (accountResult.error || !accountResult.entity) { + // // If account creation fails, return error but include successfully processed person + // return createApiResponse(request, { + // error: accountResult.error || "Failed to process Account.", + // details: accountResult.details, + // status: accountResult.status || 500, + // // Optionally include person data if account failed + // // data: { person: personResult.entity, person_created: personResult.created } + // }); + // } + + // const responsePayload: PersonWithAccountResult = { + // person: personResult.entity, + // account: accountResult.entity, + // person_created: personResult.created, + // account_created: accountResult.created, + // }; + + // const overallStatus = + // personResult.created || accountResult.created ? 201 : 200; + const overallStatus = personResult.created ? 201 : 200; return createApiResponse(request, { - data: responsePayload, + data: personResult, status: overallStatus, }); } catch (e: unknown) { diff --git a/apps/website/app/utils/supabase/dbUtils.ts b/apps/website/app/utils/supabase/dbUtils.ts index 8663e1a71..eb36c408b 100644 --- a/apps/website/app/utils/supabase/dbUtils.ts +++ b/apps/website/app/utils/supabase/dbUtils.ts @@ -1,4 +1,5 @@ import type { SupabaseClient, PostgrestError } from "@supabase/supabase-js"; +import { Database, Tables, TablesInsert } from "~/utils/supabase/types.gen"; export type GetOrCreateEntityResult = { entity: T | null; @@ -20,21 +21,23 @@ export type GetOrCreateEntityResult = { * @param entityName A friendly name for the entity, used in logging and error messages. * @returns Promise> */ -export async function getOrCreateEntity( - supabase: SupabaseClient, - tableName: string, +export async function getOrCreateEntity< + TableName extends keyof Database["public"]["Tables"], +>( + supabase: SupabaseClient, + tableName: keyof Database["public"]["Tables"], selectQuery: string, matchCriteria: Record, - insertData: Omit & Record, // Flexible insert data + insertData: TablesInsert, // Flexible insert data entityName: string = tableName, -): Promise> { +): Promise>> { // 1. Try to fetch existing entity let queryBuilder = supabase.from(tableName).select(selectQuery); for (const key in matchCriteria) { queryBuilder = queryBuilder.eq(key, matchCriteria[key]); } const { data: existingEntity, error: fetchError } = - await queryBuilder.maybeSingle(); + await queryBuilder.maybeSingle>(); if (fetchError) { console.error(`Error fetching ${entityName} by`, matchCriteria, fetchError); @@ -67,7 +70,7 @@ export async function getOrCreateEntity( .from(tableName) .insert(insertData) .select(selectQuery) - .single(); + .single>(); if (insertError) { console.error( @@ -87,7 +90,7 @@ export async function getOrCreateEntity( reFetchQueryBuilder = reFetchQueryBuilder.eq(key, matchCriteria[key]); } const { data: reFetchedEntity, error: reFetchError } = - await reFetchQueryBuilder.maybeSingle(); + await reFetchQueryBuilder.maybeSingle>(); if (reFetchError) { console.error( @@ -171,14 +174,20 @@ export type BatchProcessResult = { status: number; // HTTP status to suggest }; -export async function processAndInsertBatch( +export async function processAndInsertBatch< + TableName extends keyof Database["public"]["Tables"], + TProcessed, +>( supabase: SupabaseClient, - items: TInput[], + items: TablesInsert[], tableName: string, selectQuery: string, // e.g., "id, field1, field2" or "*" - itemValidatorAndProcessor: BatchItemValidator, + itemValidatorAndProcessor: BatchItemValidator< + TablesInsert, + TProcessed + >, entityName: string = tableName, // For logging -): Promise> { +): Promise>> { if (!Array.isArray(items) || items.length === 0) { return { error: `Request body must be a non-empty array of ${entityName} items.`, @@ -237,7 +246,7 @@ export async function processAndInsertBatch( }; } - const newRecordsTyped = newRecords as TRecord[]; // Assert type + const newRecordsTyped = newRecords as Tables[]; // Assert type if (!newRecordsTyped || newRecordsTyped.length !== processedForDb.length) { console.warn( diff --git a/apps/website/app/utils/supabase/server.ts b/apps/website/app/utils/supabase/server.ts index ef8713bf7..1434dbebf 100644 --- a/apps/website/app/utils/supabase/server.ts +++ b/apps/website/app/utils/supabase/server.ts @@ -1,10 +1,11 @@ import { createServerClient, type CookieOptions } from "@supabase/ssr"; import { cookies } from "next/headers"; +import { Database } from "~/utils/supabase/types.gen"; export async function createClient() { const cookieStore = await cookies(); - return createServerClient( + return createServerClient( process.env.NEXT_PUBLIC_SUPABASE_URL!, process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY!, { diff --git a/apps/website/app/utils/supabase/types.gen.ts b/apps/website/app/utils/supabase/types.gen.ts new file mode 100644 index 000000000..593bf7458 --- /dev/null +++ b/apps/website/app/utils/supabase/types.gen.ts @@ -0,0 +1,796 @@ +export type Json = + | string + | number + | boolean + | null + | { [key: string]: Json | undefined } + | Json[]; + +export type Database = { + public: { + Tables: { + Account: { + Row: { + active: boolean; + id: number; + person_id: number; + platform_id: number; + write_permission: boolean; + }; + Insert: { + active?: boolean; + id?: number; + person_id: number; + platform_id: number; + write_permission: boolean; + }; + Update: { + active?: boolean; + id?: number; + person_id?: number; + platform_id?: number; + write_permission?: boolean; + }; + Relationships: [ + { + foreignKeyName: "Account_person_id_fkey"; + columns: ["person_id"]; + isOneToOne: false; + referencedRelation: "Agent"; + referencedColumns: ["id"]; + }, + { + foreignKeyName: "Account_platform_id_fkey"; + columns: ["platform_id"]; + isOneToOne: false; + referencedRelation: "DiscoursePlatform"; + referencedColumns: ["id"]; + }, + ]; + }; + Agent: { + Row: { + id: number; + type: Database["public"]["Enums"]["EntityType"]; + }; + Insert: { + id?: number; + type: Database["public"]["Enums"]["EntityType"]; + }; + Update: { + id?: number; + type?: Database["public"]["Enums"]["EntityType"]; + }; + Relationships: []; + }; + AutomatedAgent: { + Row: { + deterministic: boolean | null; + id: number; + metadata: Json; + name: string; + version: string | null; + }; + Insert: { + deterministic?: boolean | null; + id: number; + metadata?: Json; + name: string; + version?: string | null; + }; + Update: { + deterministic?: boolean | null; + id?: number; + metadata?: Json; + name?: string; + version?: string | null; + }; + Relationships: [ + { + foreignKeyName: "automated_agent_id_fkey"; + columns: ["id"]; + isOneToOne: true; + referencedRelation: "Agent"; + referencedColumns: ["id"]; + }, + ]; + }; + Concept: { + Row: { + arity: number; + author_id: number | null; + content: Json; + created: string; + description: string | null; + epistemic_status: Database["public"]["Enums"]["EpistemicStatus"]; + id: number; + is_schema: boolean; + last_modified: string; + name: string; + represented_by_id: number | null; + schema_id: number | null; + space_id: number | null; + }; + Insert: { + arity?: number; + author_id?: number | null; + content?: Json; + created: string; + description?: string | null; + epistemic_status?: Database["public"]["Enums"]["EpistemicStatus"]; + id?: number; + is_schema?: boolean; + last_modified: string; + name: string; + represented_by_id?: number | null; + schema_id?: number | null; + space_id?: number | null; + }; + Update: { + arity?: number; + author_id?: number | null; + content?: Json; + created?: string; + description?: string | null; + epistemic_status?: Database["public"]["Enums"]["EpistemicStatus"]; + id?: number; + is_schema?: boolean; + last_modified?: string; + name?: string; + represented_by_id?: number | null; + schema_id?: number | null; + space_id?: number | null; + }; + Relationships: [ + { + foreignKeyName: "Concept_author_id_fkey"; + columns: ["author_id"]; + isOneToOne: false; + referencedRelation: "Agent"; + referencedColumns: ["id"]; + }, + { + foreignKeyName: "Concept_represented_by_id_fkey"; + columns: ["represented_by_id"]; + isOneToOne: false; + referencedRelation: "Content"; + referencedColumns: ["id"]; + }, + { + foreignKeyName: "Concept_schema_id_fkey"; + columns: ["schema_id"]; + isOneToOne: false; + referencedRelation: "Concept"; + referencedColumns: ["id"]; + }, + { + foreignKeyName: "Concept_space_id_fkey"; + columns: ["space_id"]; + isOneToOne: false; + referencedRelation: "DiscourseSpace"; + referencedColumns: ["id"]; + }, + ]; + }; + concept_contributors: { + Row: { + concept_id: number; + contributor_id: number; + }; + Insert: { + concept_id: number; + contributor_id: number; + }; + Update: { + concept_id?: number; + contributor_id?: number; + }; + Relationships: [ + { + foreignKeyName: "concept_contributors_concept_id_fkey"; + columns: ["concept_id"]; + isOneToOne: false; + referencedRelation: "Concept"; + referencedColumns: ["id"]; + }, + { + foreignKeyName: "concept_contributors_contributor_id_fkey"; + columns: ["contributor_id"]; + isOneToOne: false; + referencedRelation: "Agent"; + referencedColumns: ["id"]; + }, + ]; + }; + Content: { + Row: { + author_id: number | null; + created: string; + creator_id: number | null; + document_id: number; + id: number; + last_modified: string; + metadata: Json; + part_of_id: number | null; + scale: Database["public"]["Enums"]["Scale"]; + source_local_id: string | null; + space_id: number | null; + text: string; + }; + Insert: { + author_id?: number | null; + created: string; + creator_id?: number | null; + document_id: number; + id?: number; + last_modified: string; + metadata?: Json; + part_of_id?: number | null; + scale: Database["public"]["Enums"]["Scale"]; + source_local_id?: string | null; + space_id?: number | null; + text: string; + }; + Update: { + author_id?: number | null; + created?: string; + creator_id?: number | null; + document_id?: number; + id?: number; + last_modified?: string; + metadata?: Json; + part_of_id?: number | null; + scale?: Database["public"]["Enums"]["Scale"]; + source_local_id?: string | null; + space_id?: number | null; + text?: string; + }; + Relationships: [ + { + foreignKeyName: "Content_author_id_fkey"; + columns: ["author_id"]; + isOneToOne: false; + referencedRelation: "Agent"; + referencedColumns: ["id"]; + }, + { + foreignKeyName: "Content_creator_id_fkey"; + columns: ["creator_id"]; + isOneToOne: false; + referencedRelation: "Agent"; + referencedColumns: ["id"]; + }, + { + foreignKeyName: "Content_document_id_fkey"; + columns: ["document_id"]; + isOneToOne: false; + referencedRelation: "Document"; + referencedColumns: ["id"]; + }, + { + foreignKeyName: "Content_part_of_id_fkey"; + columns: ["part_of_id"]; + isOneToOne: false; + referencedRelation: "Content"; + referencedColumns: ["id"]; + }, + { + foreignKeyName: "Content_space_id_fkey"; + columns: ["space_id"]; + isOneToOne: false; + referencedRelation: "DiscourseSpace"; + referencedColumns: ["id"]; + }, + ]; + }; + content_contributors: { + Row: { + content_id: number; + contributor_id: number; + }; + Insert: { + content_id: number; + contributor_id: number; + }; + Update: { + content_id?: number; + contributor_id?: number; + }; + Relationships: [ + { + foreignKeyName: "content_contributors_content_id_fkey"; + columns: ["content_id"]; + isOneToOne: false; + referencedRelation: "Content"; + referencedColumns: ["id"]; + }, + { + foreignKeyName: "content_contributors_contributor_id_fkey"; + columns: ["contributor_id"]; + isOneToOne: false; + referencedRelation: "Agent"; + referencedColumns: ["id"]; + }, + ]; + }; + ContentEmbedding_openai_text_embedding_3_small_1536: { + Row: { + model: Database["public"]["Enums"]["EmbeddingName"]; + obsolete: boolean | null; + target_id: number; + vector: string; + }; + Insert: { + model?: Database["public"]["Enums"]["EmbeddingName"]; + obsolete?: boolean | null; + target_id: number; + vector: string; + }; + Update: { + model?: Database["public"]["Enums"]["EmbeddingName"]; + obsolete?: boolean | null; + target_id?: number; + vector?: string; + }; + Relationships: [ + { + foreignKeyName: "ContentEmbedding_openai_text_embedding_3_small_1_target_id_fkey"; + columns: ["target_id"]; + isOneToOne: true; + referencedRelation: "Content"; + referencedColumns: ["id"]; + }, + ]; + }; + DiscoursePlatform: { + Row: { + id: number; + name: string; + url: string; + }; + Insert: { + id?: number; + name: string; + url: string; + }; + Update: { + id?: number; + name?: string; + url?: string; + }; + Relationships: []; + }; + DiscourseSpace: { + Row: { + discourse_platform_id: number; + id: number; + name: string; + url: string | null; + }; + Insert: { + discourse_platform_id: number; + id?: number; + name: string; + url?: string | null; + }; + Update: { + discourse_platform_id?: number; + id?: number; + name?: string; + url?: string | null; + }; + Relationships: [ + { + foreignKeyName: "DiscourseSpace_discourse_platform_id_fkey"; + columns: ["discourse_platform_id"]; + isOneToOne: false; + referencedRelation: "DiscoursePlatform"; + referencedColumns: ["id"]; + }, + ]; + }; + Document: { + Row: { + author_id: number; + contents: unknown | null; + created: string; + id: number; + last_modified: string; + metadata: Json; + source_local_id: string | null; + space_id: number | null; + url: string | null; + }; + Insert: { + author_id: number; + contents?: unknown | null; + created: string; + id?: number; + last_modified: string; + metadata?: Json; + source_local_id?: string | null; + space_id?: number | null; + url?: string | null; + }; + Update: { + author_id?: number; + contents?: unknown | null; + created?: string; + id?: number; + last_modified?: string; + metadata?: Json; + source_local_id?: string | null; + space_id?: number | null; + url?: string | null; + }; + Relationships: [ + { + foreignKeyName: "Document_author_id_fkey"; + columns: ["author_id"]; + isOneToOne: false; + referencedRelation: "Agent"; + referencedColumns: ["id"]; + }, + { + foreignKeyName: "Document_space_id_fkey"; + columns: ["space_id"]; + isOneToOne: false; + referencedRelation: "DiscourseSpace"; + referencedColumns: ["id"]; + }, + ]; + }; + Person: { + Row: { + email: string; + id: number; + name: string; + orcid: string | null; + }; + Insert: { + email: string; + id: number; + name: string; + orcid?: string | null; + }; + Update: { + email?: string; + id?: number; + name?: string; + orcid?: string | null; + }; + Relationships: [ + { + foreignKeyName: "person_id_fkey"; + columns: ["id"]; + isOneToOne: true; + referencedRelation: "Agent"; + referencedColumns: ["id"]; + }, + ]; + }; + SpaceAccess: { + Row: { + account_id: number; + editor: boolean; + id: number; + space_id: number | null; + }; + Insert: { + account_id: number; + editor: boolean; + id?: number; + space_id?: number | null; + }; + Update: { + account_id?: number; + editor?: boolean; + id?: number; + space_id?: number | null; + }; + Relationships: [ + { + foreignKeyName: "SpaceAccess_account_id_fkey"; + columns: ["account_id"]; + isOneToOne: false; + referencedRelation: "Account"; + referencedColumns: ["id"]; + }, + { + foreignKeyName: "SpaceAccess_space_id_fkey"; + columns: ["space_id"]; + isOneToOne: false; + referencedRelation: "DiscourseSpace"; + referencedColumns: ["id"]; + }, + ]; + }; + sync_info: { + Row: { + failure_count: number | null; + id: number; + last_task_end: string | null; + last_task_start: string | null; + status: Database["public"]["Enums"]["task_status"] | null; + sync_function: string | null; + sync_target: number | null; + task_times_out_at: string | null; + worker: string; + }; + Insert: { + failure_count?: number | null; + id?: number; + last_task_end?: string | null; + last_task_start?: string | null; + status?: Database["public"]["Enums"]["task_status"] | null; + sync_function?: string | null; + sync_target?: number | null; + task_times_out_at?: string | null; + worker: string; + }; + Update: { + failure_count?: number | null; + id?: number; + last_task_end?: string | null; + last_task_start?: string | null; + status?: Database["public"]["Enums"]["task_status"] | null; + sync_function?: string | null; + sync_target?: number | null; + task_times_out_at?: string | null; + worker?: string; + }; + Relationships: []; + }; + }; + Views: { + [_ in never]: never; + }; + Functions: { + end_sync_task: { + Args: { + s_target: number; + s_function: string; + s_worker: string; + s_status: Database["public"]["Enums"]["task_status"]; + }; + Returns: undefined; + }; + match_content_embeddings: { + Args: { + query_embedding: string; + match_threshold: number; + match_count: number; + current_document_id?: number; + }; + Returns: { + content_id: number; + roam_uid: string; + text_content: string; + similarity: number; + }[]; + }; + match_embeddings_for_subset_nodes: { + Args: { p_query_embedding: string; p_subset_roam_uids: string[] }; + Returns: { + content_id: number; + roam_uid: string; + text_content: string; + similarity: number; + }[]; + }; + propose_sync_task: { + Args: { + s_target: number; + s_function: string; + s_worker: string; + timeout: unknown; + task_interval: unknown; + }; + Returns: unknown; + }; + }; + Enums: { + EmbeddingName: + | "openai_text_embedding_ada2_1536" + | "openai_text_embedding_3_small_512" + | "openai_text_embedding_3_small_1536" + | "openai_text_embedding_3_large_256" + | "openai_text_embedding_3_large_1024" + | "openai_text_embedding_3_large_3072"; + EntityType: + | "Platform" + | "Space" + | "Account" + | "Person" + | "AutomatedAgent" + | "Document" + | "Content" + | "Concept" + | "ConceptSchema" + | "ContentLink" + | "Occurrence"; + EpistemicStatus: + | "certainly_not" + | "strong_evidence_against" + | "could_be_false" + | "unknown" + | "uncertain" + | "contentious" + | "could_be_true" + | "strong_evidence_for" + | "certain"; + Scale: + | "document" + | "post" + | "chunk_unit" + | "section" + | "block" + | "field" + | "paragraph" + | "quote" + | "sentence" + | "phrase"; + task_status: "active" | "timeout" | "complete" | "failed"; + }; + CompositeTypes: { + [_ in never]: never; + }; + }; +}; + +type DefaultSchema = Database[Extract]; + +export type Tables< + DefaultSchemaTableNameOrOptions extends + | keyof (DefaultSchema["Tables"] & DefaultSchema["Views"]) + | { schema: keyof Database }, + TableName extends DefaultSchemaTableNameOrOptions extends { + schema: keyof Database; + } + ? keyof (Database[DefaultSchemaTableNameOrOptions["schema"]]["Tables"] & + Database[DefaultSchemaTableNameOrOptions["schema"]]["Views"]) + : never = never, +> = DefaultSchemaTableNameOrOptions extends { schema: keyof Database } + ? (Database[DefaultSchemaTableNameOrOptions["schema"]]["Tables"] & + Database[DefaultSchemaTableNameOrOptions["schema"]]["Views"])[TableName] extends { + Row: infer R; + } + ? R + : never + : DefaultSchemaTableNameOrOptions extends keyof (DefaultSchema["Tables"] & + DefaultSchema["Views"]) + ? (DefaultSchema["Tables"] & + DefaultSchema["Views"])[DefaultSchemaTableNameOrOptions] extends { + Row: infer R; + } + ? R + : never + : never; + +export type TablesInsert< + DefaultSchemaTableNameOrOptions extends + | keyof DefaultSchema["Tables"] + | { schema: keyof Database }, + TableName extends DefaultSchemaTableNameOrOptions extends { + schema: keyof Database; + } + ? keyof Database[DefaultSchemaTableNameOrOptions["schema"]]["Tables"] + : never = never, +> = DefaultSchemaTableNameOrOptions extends { schema: keyof Database } + ? Database[DefaultSchemaTableNameOrOptions["schema"]]["Tables"][TableName] extends { + Insert: infer I; + } + ? I + : never + : DefaultSchemaTableNameOrOptions extends keyof DefaultSchema["Tables"] + ? DefaultSchema["Tables"][DefaultSchemaTableNameOrOptions] extends { + Insert: infer I; + } + ? I + : never + : never; + +export type TablesUpdate< + DefaultSchemaTableNameOrOptions extends + | keyof DefaultSchema["Tables"] + | { schema: keyof Database }, + TableName extends DefaultSchemaTableNameOrOptions extends { + schema: keyof Database; + } + ? keyof Database[DefaultSchemaTableNameOrOptions["schema"]]["Tables"] + : never = never, +> = DefaultSchemaTableNameOrOptions extends { schema: keyof Database } + ? Database[DefaultSchemaTableNameOrOptions["schema"]]["Tables"][TableName] extends { + Update: infer U; + } + ? U + : never + : DefaultSchemaTableNameOrOptions extends keyof DefaultSchema["Tables"] + ? DefaultSchema["Tables"][DefaultSchemaTableNameOrOptions] extends { + Update: infer U; + } + ? U + : never + : never; + +export type Enums< + DefaultSchemaEnumNameOrOptions extends + | keyof DefaultSchema["Enums"] + | { schema: keyof Database }, + EnumName extends DefaultSchemaEnumNameOrOptions extends { + schema: keyof Database; + } + ? keyof Database[DefaultSchemaEnumNameOrOptions["schema"]]["Enums"] + : never = never, +> = DefaultSchemaEnumNameOrOptions extends { schema: keyof Database } + ? Database[DefaultSchemaEnumNameOrOptions["schema"]]["Enums"][EnumName] + : DefaultSchemaEnumNameOrOptions extends keyof DefaultSchema["Enums"] + ? DefaultSchema["Enums"][DefaultSchemaEnumNameOrOptions] + : never; + +export type CompositeTypes< + PublicCompositeTypeNameOrOptions extends + | keyof DefaultSchema["CompositeTypes"] + | { schema: keyof Database }, + CompositeTypeName extends PublicCompositeTypeNameOrOptions extends { + schema: keyof Database; + } + ? keyof Database[PublicCompositeTypeNameOrOptions["schema"]]["CompositeTypes"] + : never = never, +> = PublicCompositeTypeNameOrOptions extends { schema: keyof Database } + ? Database[PublicCompositeTypeNameOrOptions["schema"]]["CompositeTypes"][CompositeTypeName] + : PublicCompositeTypeNameOrOptions extends keyof DefaultSchema["CompositeTypes"] + ? DefaultSchema["CompositeTypes"][PublicCompositeTypeNameOrOptions] + : never; + +export const Constants = { + public: { + Enums: { + EmbeddingName: [ + "openai_text_embedding_ada2_1536", + "openai_text_embedding_3_small_512", + "openai_text_embedding_3_small_1536", + "openai_text_embedding_3_large_256", + "openai_text_embedding_3_large_1024", + "openai_text_embedding_3_large_3072", + ], + EntityType: [ + "Platform", + "Space", + "Account", + "Person", + "AutomatedAgent", + "Document", + "Content", + "Concept", + "ConceptSchema", + "ContentLink", + "Occurrence", + ], + EpistemicStatus: [ + "certainly_not", + "strong_evidence_against", + "could_be_false", + "unknown", + "uncertain", + "contentious", + "could_be_true", + "strong_evidence_for", + "certain", + ], + Scale: [ + "document", + "post", + "chunk_unit", + "section", + "block", + "field", + "paragraph", + "quote", + "sentence", + "phrase", + ], + task_status: ["active", "timeout", "complete", "failed"], + }, + }, +} as const; From ea8eb6ce50ea2fe670dd4241a43cbcce86dda5b0 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Tue, 20 May 2025 13:56:57 -0400 Subject: [PATCH 15/19] correct base ContentEmbedding insert. Also make it model-independent. --- .../routes.ts | 79 ++++++++++++++++--- 1 file changed, 67 insertions(+), 12 deletions(-) diff --git a/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts b/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts index 1a2d3379a..816bd4dae 100644 --- a/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts +++ b/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts @@ -9,21 +9,39 @@ import { getOrCreateEntity, GetOrCreateEntityResult, } from "~/utils/supabase/dbUtils"; -import { Tables, TablesInsert } from "~/utils/supabase/types.gen"; +import { Database, Tables, TablesInsert } from "~/utils/supabase/types.gen"; +// Use the first known ContentEmbedding table, as they have the same structure type ContentEmbeddingDataInput = TablesInsert<"ContentEmbedding_openai_text_embedding_3_small_1536">; type ContentEmbeddingRecord = Tables<"ContentEmbedding_openai_text_embedding_3_small_1536">; -const TARGET_EMBEDDING_TABLE = - "ContentEmbedding_openai_text_embedding_3_small_1536"; +const known_embedding_tables: { + [key: string]: { + table_name: keyof Database["public"]["Tables"]; + table_size: number; + }; +} = { + openai_text_embedding_3_small_1536: { + table_name: "ContentEmbedding_openai_text_embedding_3_small_1536", + table_size: 1536, + }, +}; + +type ApiInputEmbeddingItem = Omit & { + vector: number[]; // Vector is passed in as a number[] +}; + +type ApiOutputEmbeddingRecord = Omit & { + vector: number[]; // Vector is passed in as a number[] +}; // Renamed and refactored const processAndCreateEmbedding = async ( supabasePromise: ReturnType, - data: ContentEmbeddingDataInput, -): Promise> => { + data: ApiInputEmbeddingItem, +): Promise> => { const { target_id, model, vector, obsolete = false } = data; // --- Start of validation --- @@ -39,7 +57,11 @@ const processAndCreateEmbedding = async ( status: 400, }; } - if (!model || typeof model !== "string") { + if ( + !model || + typeof model !== "string" || + known_embedding_tables[model] == undefined + ) { return { entity: null, error: "Missing or invalid model name.", @@ -47,9 +69,13 @@ const processAndCreateEmbedding = async ( status: 400, }; } + + const { table_name, table_size } = known_embedding_tables[model]; + if ( !vector || !Array.isArray(vector) || + vector.length != table_size || !vector.every((v) => typeof v === "number") ) { return { @@ -73,7 +99,7 @@ const processAndCreateEmbedding = async ( const vectorString = JSON.stringify(vector); const supabase = await supabasePromise; - const embeddingToInsert = { + const embeddingToInsert: ContentEmbeddingDataInput = { target_id, model, vector: vectorString, @@ -85,7 +111,7 @@ const processAndCreateEmbedding = async ( const result = await getOrCreateEntity<"ContentEmbedding_openai_text_embedding_3_small_1536">( supabase, - TARGET_EMBEDDING_TABLE, + table_name, "*", // Select all fields for the record { id: -1 }, // Non-matching criteria to force "create" path embeddingToInsert, @@ -102,25 +128,53 @@ const processAndCreateEmbedding = async ( if ( result.details.toLowerCase().includes( // Check for target_id FK, adapt if FK name is different - `${TARGET_EMBEDDING_TABLE.toLowerCase()}_target_id_fkey`.toLowerCase(), + `${table_name.toLowerCase()}_target_id_fkey`, ) || result.details.toLowerCase().includes("target_id") ) { return { ...result, + entity: null, error: `Invalid target_id: No Content record found for ID ${target_id}.`, }; } } - return result; + try { + const decoded_json = JSON.parse(result.entity?.vector || ""); + if ( + result.entity && + Array.isArray(decoded_json) && + decoded_json.length == table_size && + decoded_json.every((v) => typeof v === "number") + ) { + return { + ...result, + entity: { + ...result.entity, + vector: decoded_json, + }, + }; + } + } catch (Exception) { + return { + ...result, + entity: null, + error: `Resulting entity does not have the right vector shape`, + }; + } + return { + ...result, + entity: null, + error: `Error creating the database object`, + }; }; export const POST = async (request: NextRequest): Promise => { const supabasePromise = createClient(); try { - const body: ContentEmbeddingDataInput = await request.json(); + const body: ApiInputEmbeddingItem = await request.json(); // Minimal validation here, more detailed in processAndCreateEmbedding if (!body || typeof body !== "object") { @@ -143,7 +197,8 @@ export const POST = async (request: NextRequest): Promise => { return handleRouteError( request, e, - `/api/supabase/insert/${TARGET_EMBEDDING_TABLE}`, + `/api/supabase/insert/ContentEmbedding_openai_text_embedding_3_small_1536`, + // TODO replace with a generic name ); } }; From b263de693e2ca9b781377d1c8adea0d1aca46e5e Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Wed, 21 May 2025 20:53:54 -0400 Subject: [PATCH 16/19] commit the new functions --- .../routes.ts | 15 +- apps/website/app/utils/supabase/dbUtils.ts | 250 ++++++++++++++++++ 2 files changed, 252 insertions(+), 13 deletions(-) diff --git a/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts b/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts index 816bd4dae..5c18b148d 100644 --- a/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts +++ b/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts @@ -8,8 +8,9 @@ import { import { getOrCreateEntity, GetOrCreateEntityResult, + known_embedding_tables, } from "~/utils/supabase/dbUtils"; -import { Database, Tables, TablesInsert } from "~/utils/supabase/types.gen"; +import { Tables, TablesInsert } from "~/utils/supabase/types.gen"; // Use the first known ContentEmbedding table, as they have the same structure type ContentEmbeddingDataInput = @@ -17,18 +18,6 @@ type ContentEmbeddingDataInput = type ContentEmbeddingRecord = Tables<"ContentEmbedding_openai_text_embedding_3_small_1536">; -const known_embedding_tables: { - [key: string]: { - table_name: keyof Database["public"]["Tables"]; - table_size: number; - }; -} = { - openai_text_embedding_3_small_1536: { - table_name: "ContentEmbedding_openai_text_embedding_3_small_1536", - table_size: 1536, - }, -}; - type ApiInputEmbeddingItem = Omit & { vector: number[]; // Vector is passed in as a number[] }; diff --git a/apps/website/app/utils/supabase/dbUtils.ts b/apps/website/app/utils/supabase/dbUtils.ts index eb36c408b..d8a87d7cd 100644 --- a/apps/website/app/utils/supabase/dbUtils.ts +++ b/apps/website/app/utils/supabase/dbUtils.ts @@ -1,6 +1,19 @@ import type { SupabaseClient, PostgrestError } from "@supabase/supabase-js"; +import { OK } from "zod"; import { Database, Tables, TablesInsert } from "~/utils/supabase/types.gen"; +export const known_embedding_tables: { + [key: string]: { + table_name: keyof Database["public"]["Tables"]; + table_size: number; + }; +} = { + openai_text_embedding_3_small_1536: { + table_name: "ContentEmbedding_openai_text_embedding_3_small_1536", + table_size: 1536, + }, +}; + export type GetOrCreateEntityResult = { entity: T | null; error: string | null; @@ -166,6 +179,14 @@ export type BatchItemValidator = ( index: number, ) => { valid: boolean; error?: string; processedItem?: TProcessed }; +export type ItemProcessor = (item: TInput) => { + valid: boolean; + error?: string; + processedItem?: TProcessed; +}; + +export type ItemValidator = (item: T) => string | null; + export type BatchProcessResult = { data?: TRecord[]; error?: string; @@ -263,3 +284,232 @@ export async function processAndInsertBatch< ); return { data: newRecordsTyped, status: 201 }; } + +export async function InsertValidatedBatch< + TableName extends keyof Database["public"]["Tables"], +>( + supabase: SupabaseClient, + items: TablesInsert[], + tableName: string, + selectQuery: string, // e.g., "id, field1, field2" or "*" + entityName: string = tableName, // For logging +): Promise>> { + const { data: newRecords, error: insertError } = await supabase + .from(tableName) + .insert(items) + .select(selectQuery); // Use the provided select query + + if (insertError) { + console.error(`Error batch inserting ${entityName}:`, insertError); + return { + error: `Database error during batch insert of ${entityName}.`, + details: insertError.message, + status: 500, + }; + } + + const newRecordsTyped = newRecords as Tables[]; // Assert type + + if (!newRecordsTyped || newRecordsTyped.length !== items.length) { + console.warn( + `Batch insert ${entityName}: Mismatch between processed count (${items.length}) and DB returned count (${newRecordsTyped?.length || 0}).`, + ); + return { + error: `Batch insert of ${entityName} might have partially failed or returned unexpected data.`, + status: 500, // Or a more specific error + }; + } + + console.log( + `Successfully batch inserted ${newRecordsTyped.length} ${entityName} records.`, + ); + return { data: newRecordsTyped, status: 201 }; +} + +export async function validateAndInsertBatch< + TableName extends keyof Database["public"]["Tables"], +>( + supabase: SupabaseClient, + items: TablesInsert[], + tableName: string, + selectQuery: string, // e.g., "id, field1, field2" or "*" + entityName: string = tableName, // For logging + inputValidator: ItemValidator> | null, + outputValidator: ItemValidator> | null, +): Promise>> { + let validatedItems: TablesInsert[] = []; + const validationErrors: { index: number; error: string }[] = []; + if (!Array.isArray(items) || items.length === 0) { + return { + error: `Request body must be a non-empty array of ${entityName} items.`, + status: 400, + }; + } + + if (inputValidator !== null) { + for (let i = 0; i < items.length; i++) { + const item = items[i]; + if (!item) { + // Handles undefined/null items in the array itself + validationErrors.push({ + index: i, + error: "Item is undefined or null.", + }); + continue; + } + const error = inputValidator(item); + if (error !== null) { + validationErrors.push({ + index: i, + error: error || "Validation failed.", + }); + } else { + validatedItems.push(item); + } + } + + if (validationErrors.length > 0) { + return { + error: `Validation failed for one or more ${entityName} items.`, + partial_errors: validationErrors, + status: 400, + }; + } + } else { + validatedItems = items; + } + const result = await InsertValidatedBatch( + supabase, + validatedItems, + tableName, + selectQuery, + entityName, + ); + if (result.error || !result.data) { + return result; + } + if (outputValidator !== null) { + const validatedResults: Tables[] = []; + for (let i = 0; i < result.data.length; i++) { + const item = result.data[i]; + if (!item) { + // Handles undefined/null items in the array itself + validationErrors.push({ + index: i, + error: "Returned item is undefined or null.", + }); + continue; + } + const error = outputValidator(item); + if (error !== null) { + validationErrors.push({ + index: i, + error: error || "Validation failed.", + }); + } else { + validatedResults.push(item); + } + } + + if (validationErrors.length > 0) { + return { + error: `Validation failed for one or more ${entityName} results.`, + partial_errors: validationErrors, + status: 500, + }; + } + } + return result; +} + +export async function processAndInsertBatch_new< + TableName extends keyof Database["public"]["Tables"], + InputType, + OutputType, +>( + supabase: SupabaseClient, + items: InputType[], + tableName: string, + selectQuery: string, // e.g., "id, field1, field2" or "*" + entityName: string = tableName, // For logging + inputProcessor: ItemProcessor>, + outputProcessor: ItemProcessor, OutputType>, +): Promise> { + let processedItems: TablesInsert[] = []; + const validationErrors: { index: number; error: string }[] = []; + if (!Array.isArray(items) || items.length === 0) { + return { + error: `Request body must be a non-empty array of ${entityName} items.`, + status: 400, + }; + } + + for (let i = 0; i < items.length; i++) { + const item = items[i]; + if (!item) { + // Handles undefined/null items in the array itself + validationErrors.push({ + index: i, + error: "Item is undefined or null.", + }); + continue; + } + const { valid, error, processedItem } = inputProcessor(item); + if (!valid || !processedItem) { + validationErrors.push({ + index: i, + error: error || "Validation failed.", + }); + } else { + processedItems.push(processedItem); + } + } + + if (validationErrors.length > 0) { + return { + error: `Validation failed for one or more ${entityName} items.`, + partial_errors: validationErrors, + status: 400, + }; + } + const result = await InsertValidatedBatch( + supabase, + processedItems, + tableName, + selectQuery, + entityName, + ); + if (result.error || !result.data) { + return { ...result, data: [] }; + } + const processedResults: OutputType[] = []; + for (let i = 0; i < result.data.length; i++) { + const item = result.data[i]; + if (!item) { + // Handles undefined/null items in the array itself + validationErrors.push({ + index: i, + error: "Returned item is undefined or null.", + }); + continue; + } + const { valid, error, processedItem } = outputProcessor(item); + if (!valid || !processedItem) { + validationErrors.push({ + index: i, + error: error || "Result validation failed.", + }); + } else { + processedResults.push(processedItem); + } + } + + if (validationErrors.length > 0) { + return { + error: `Validation failed for one or more ${entityName} results.`, + partial_errors: validationErrors, + status: 500, + }; + } + return { ...result, data: processedResults }; +} From 6cd6ca3abe5bb740e951f4416ca42bea97488026 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Thu, 22 May 2025 01:21:25 -0400 Subject: [PATCH 17/19] use validation functions --- .../batch/route.ts | 132 ++++++------- .../routes.ts | 180 ++++++++++++------ .../supabase/insert/content/batch/route.ts | 65 ++----- .../app/api/supabase/insert/content/route.ts | 124 ++++-------- apps/website/app/utils/supabase/dbUtils.ts | 92 +-------- 5 files changed, 223 insertions(+), 370 deletions(-) diff --git a/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/batch/route.ts b/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/batch/route.ts index 986bbae7f..0c6f5012d 100644 --- a/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/batch/route.ts +++ b/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/batch/route.ts @@ -7,95 +7,73 @@ import { } from "~/utils/supabase/apiUtils"; import { processAndInsertBatch, - BatchItemValidator, BatchProcessResult, } from "~/utils/supabase/dbUtils"; -import { Tables, TablesInsert } from "~/utils/supabase/types.gen"; - -type ContentEmbeddingDataInput = - TablesInsert<"ContentEmbedding_openai_text_embedding_3_small_1536">; -type ContentEmbeddingRecord = - Tables<"ContentEmbedding_openai_text_embedding_3_small_1536">; - -// Request body is an array of these items -type ContentEmbeddingBatchRequestBody = ContentEmbeddingDataInput[]; - -// Type for the item after processing, ready for DB insert -type ProcessedEmbeddingItem = Omit & { - vector: string; // Vector is always stringified - obsolete: boolean | null; // Obsolete has a default -}; - -const TARGET_EMBEDDING_TABLE = - "ContentEmbedding_openai_text_embedding_3_small_1536"; +import { + inputProcessing, + outputProcessing, + type ApiInputEmbeddingItem, + type ApiOutputEmbeddingRecord, +} from "../routes"; -// Validator and processor for embedding items -const validateAndProcessEmbeddingItem: BatchItemValidator< - ContentEmbeddingDataInput, - ProcessedEmbeddingItem -> = (item, index) => { - if (item.target_id === undefined || item.target_id === null) { - return { - valid: false, - error: `Item at index ${index}: Missing required field target_id.`, - }; - } - if (!item.model) { - return { - valid: false, - error: `Item at index ${index}: Missing required field model.`, - }; - } - if (!item.vector) { - return { - valid: false, - error: `Item at index ${index}: Missing required field vector.`, - }; +const batchInsertEmbeddingsProcess = async ( + supabase: Awaited>, + embeddingItems: ApiInputEmbeddingItem[], +): Promise> => { + // groupBy is node21 only. Group by model. + // Note: This means that later index values may be totally wrong. + const by_model: { [key: string]: ApiInputEmbeddingItem[] } = {}; + for (let i = 0; i < embeddingItems.length; i++) { + const inputItem = embeddingItems[i]; + if (inputItem !== undefined && inputItem.model !== undefined) { + if (by_model[inputItem.model] === undefined) { + by_model[inputItem.model] = [inputItem]; + } else { + by_model[inputItem.model]!.push(inputItem); + } + } else { + return { + status: 400, + error: `Element ${i} undefined or does not have a model`, + }; + } } - if (!Array.isArray(item.vector) && typeof item.vector !== "string") { - return { - valid: false, - error: `Item at index ${index}: vector must be an array of numbers or a pre-formatted string.`, - }; + const globalResults: ApiOutputEmbeddingRecord[] = []; + const partial_errors = []; + let created = true; // TODO: Maybe transmit from below + for (const table_name of Object.keys(by_model)) { + const embeddingItemsSet = by_model[table_name]; + const results = await processAndInsertBatch< + "ContentEmbedding_openai_text_embedding_3_small_1536", + ApiInputEmbeddingItem, + ApiOutputEmbeddingRecord + >( + supabase, + embeddingItemsSet!, + table_name, + "*", // Select all fields, adjust if needed for ContentEmbeddingRecord + "ContentEmbedding", + inputProcessing!, + outputProcessing, + ); + if (results.error || results.data === undefined) + return { ...results, data: undefined }; + globalResults.push(...results.data); + if (results.partial_errors !== undefined) + partial_errors.push(...results.partial_errors); } - - const vectorString = Array.isArray(item.vector) - ? JSON.stringify(item.vector) - : item.vector; - return { - valid: true, - processedItem: { - target_id: item.target_id, - model: item.model, - vector: vectorString, - obsolete: item.obsolete === undefined ? false : item.obsolete, - }, + data: globalResults, + partial_errors, + status: created ? 201 : 200, }; }; -const batchInsertEmbeddingsProcess = async ( - supabase: Awaited>, - embeddingItems: ContentEmbeddingBatchRequestBody, -): Promise> => { - return processAndInsertBatch< - "ContentEmbedding_openai_text_embedding_3_small_1536", - ProcessedEmbeddingItem - >( - supabase, - embeddingItems, - TARGET_EMBEDDING_TABLE, - "*", // Select all fields, adjust if needed for ContentEmbeddingRecord - validateAndProcessEmbeddingItem, - "ContentEmbedding", - ); -}; - export const POST = async (request: NextRequest): Promise => { const supabase = await createClient(); try { - const body: ContentEmbeddingBatchRequestBody = await request.json(); + const body: ApiInputEmbeddingItem[] = await request.json(); if (!Array.isArray(body)) { return createApiResponse(request, { error: "Request body must be an array of embedding items.", @@ -119,7 +97,7 @@ export const POST = async (request: NextRequest): Promise => { return handleRouteError( request, e, - `/api/supabase/insert/${TARGET_EMBEDDING_TABLE}/batch`, + `/api/supabase/insert/content-embedding/batch`, ); } }; diff --git a/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts b/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts index 5c18b148d..7e551738a 100644 --- a/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts +++ b/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts @@ -9,29 +9,33 @@ import { getOrCreateEntity, GetOrCreateEntityResult, known_embedding_tables, + ItemProcessor, + ItemValidator, } from "~/utils/supabase/dbUtils"; import { Tables, TablesInsert } from "~/utils/supabase/types.gen"; // Use the first known ContentEmbedding table, as they have the same structure -type ContentEmbeddingDataInput = +export type ContentEmbeddingDataInput = TablesInsert<"ContentEmbedding_openai_text_embedding_3_small_1536">; -type ContentEmbeddingRecord = +export type ContentEmbeddingRecord = Tables<"ContentEmbedding_openai_text_embedding_3_small_1536">; -type ApiInputEmbeddingItem = Omit & { +export type ApiInputEmbeddingItem = Omit< + ContentEmbeddingDataInput, + "vector" +> & { vector: number[]; // Vector is passed in as a number[] }; -type ApiOutputEmbeddingRecord = Omit & { +export type ApiOutputEmbeddingRecord = Omit< + ContentEmbeddingRecord, + "vector" +> & { vector: number[]; // Vector is passed in as a number[] }; -// Renamed and refactored -const processAndCreateEmbedding = async ( - supabasePromise: ReturnType, - data: ApiInputEmbeddingItem, -): Promise> => { - const { target_id, model, vector, obsolete = false } = data; +export const inputValidation: ItemValidator = (data) => { + const { target_id, model, vector } = data; // --- Start of validation --- if ( @@ -39,62 +43,121 @@ const processAndCreateEmbedding = async ( target_id === null || typeof target_id !== "number" ) { - return { - entity: null, - error: "Missing or invalid target_id.", - created: false, - status: 400, - }; + return "Missing or invalid target_id."; } if ( !model || typeof model !== "string" || known_embedding_tables[model] == undefined ) { - return { - entity: null, - error: "Missing or invalid model name.", - created: false, - status: 400, - }; + return "Missing or invalid model name."; } - - const { table_name, table_size } = known_embedding_tables[model]; + const { table_size } = known_embedding_tables[model]; if ( !vector || !Array.isArray(vector) || - vector.length != table_size || !vector.every((v) => typeof v === "number") + ) { + return "Missing or invalid vector. Must be an array of numbers."; + } + if (vector.length != table_size) { + return `Invalid vector length. Expected ${table_size}, got ${vector.length}.`; + } + if (data.obsolete !== undefined && typeof data.obsolete !== "boolean") { + // Check original data for obsolete presence + return "Invalid type for obsolete. Must be a boolean."; + } + return null; +}; + +export const inputProcessing: ItemProcessor< + ApiInputEmbeddingItem, + ContentEmbeddingDataInput +> = (data) => { + const error = inputValidation(data); + if (error) { + return { valid: false, error }; + } + return { + valid: true, + data: { ...data, vector: JSON.stringify(data.vector) }, + }; +}; + +export const outputValidation: ItemValidator = ( + data, +) => { + const { model, vector } = data; + if ( + !model || + typeof model !== "string" || + known_embedding_tables[model] == undefined + ) { + return "Missing or invalid model name."; + } + + const { table_size } = known_embedding_tables[model]; + + if (vector.length != table_size) { + return `Invalid vector length. Expected ${table_size}, got ${vector.length}.`; + } + return null; +}; + +export const outputProcessing: ItemProcessor< + ContentEmbeddingRecord, + ApiOutputEmbeddingRecord +> = (data) => { + try { + const processedData = { ...data, vector: JSON.parse(data.vector) }; + const error = outputValidation(processedData); + if (error) { + return { valid: false, error }; + } + return { + valid: true, + data: processedData, + }; + } catch (error) { + if (error instanceof SyntaxError) { + return { valid: false, error: error.message }; + } + throw error; + } +}; + +// Renamed and refactored +const processAndCreateEmbedding = async ( + supabasePromise: ReturnType, + data: ApiInputEmbeddingItem, +): Promise> => { + const { valid, error, processedItem } = inputProcessing(data); + if ( + !valid || + processedItem === undefined || + processedItem.model === undefined ) { return { entity: null, - error: "Missing or invalid vector. Must be an array of numbers.", + error: error || "unknown error", created: false, status: 400, }; } - if (data.obsolete !== undefined && typeof data.obsolete !== "boolean") { - // Check original data for obsolete presence + const supabase = await supabasePromise; + const table_data = known_embedding_tables[processedItem.model]; + + if (!table_data) { return { entity: null, - error: "Invalid type for obsolete. Must be a boolean.", + error: "unknown model", created: false, status: 400, }; } - // --- End of validation --- - - const vectorString = JSON.stringify(vector); - const supabase = await supabasePromise; - - const embeddingToInsert: ContentEmbeddingDataInput = { - target_id, - model, - vector: vectorString, - obsolete, - }; + const { table_name } = table_data; // Using getOrCreateEntity, forcing create path by providing non-matching criteria // This standardizes return type and error handling (e.g., FK violations from dbUtils) const result = @@ -103,7 +166,7 @@ const processAndCreateEmbedding = async ( table_name, "*", // Select all fields for the record { id: -1 }, // Non-matching criteria to force "create" path - embeddingToInsert, + processedItem, "ContentEmbedding", ); @@ -124,38 +187,29 @@ const processAndCreateEmbedding = async ( return { ...result, entity: null, - error: `Invalid target_id: No Content record found for ID ${target_id}.`, + error: `Invalid target_id: No Content record found for ID ${processedItem.target_id}.`, }; } } - try { - const decoded_json = JSON.parse(result.entity?.vector || ""); - if ( - result.entity && - Array.isArray(decoded_json) && - decoded_json.length == table_size && - decoded_json.every((v) => typeof v === "number") - ) { - return { - ...result, - entity: { - ...result.entity, - vector: decoded_json, - }, - }; - } - } catch (Exception) { + if (result.error || !result.entity) { return { ...result, entity: null, - error: `Resulting entity does not have the right vector shape`, }; } + + const processedResult = outputProcessing(result.entity); + if (!processedResult.valid || !processedResult.processedItem) + return { + ...result, + error: processedResult.error || "unknown error", + entity: null, + status: 500, + }; return { ...result, - entity: null, - error: `Error creating the database object`, + entity: processedResult.processedItem, }; }; @@ -186,7 +240,7 @@ export const POST = async (request: NextRequest): Promise => { return handleRouteError( request, e, - `/api/supabase/insert/ContentEmbedding_openai_text_embedding_3_small_1536`, + `/api/supabase/insert/content-embedding`, // TODO replace with a generic name ); } diff --git a/apps/website/app/api/supabase/insert/content/batch/route.ts b/apps/website/app/api/supabase/insert/content/batch/route.ts index cf4d8e997..200fa720e 100644 --- a/apps/website/app/api/supabase/insert/content/batch/route.ts +++ b/apps/website/app/api/supabase/insert/content/batch/route.ts @@ -6,71 +6,28 @@ import { defaultOptionsHandler, } from "~/utils/supabase/apiUtils"; import { - processAndInsertBatch, - BatchItemValidator, + validateAndInsertBatch, BatchProcessResult, // Import BatchProcessResult for the return type } from "~/utils/supabase/dbUtils"; // Ensure this path is correct -import { Tables, TablesInsert } from "~/utils/supabase/types.gen"; - -type ContentDataInput = TablesInsert<"Content">; -type ContentRecord = Tables<"Content">; - -// The request body will be an array of these items -type ContentBatchRequestBody = ContentDataInput[]; - -// Specific validator and processor for Content items -const validateAndProcessContentItem: BatchItemValidator< - ContentDataInput, - Omit & { metadata: string | null } // TProcessed type -> = (item, index) => { - // No need to check for !item here, processAndInsertBatch handles null/undefined items in the array itself - const requiredFields = [ - "text", - "scale", - "space_id", - "author_id", - "document_id", - "created", - "last_modified", - ]; - const missingFields = requiredFields.filter( - (field) => !(item as any)[field] && (item as any)[field] !== 0, - ); // check for undefined, null, empty string but allow 0 - if (missingFields.length > 0) { - return { - valid: false, - error: `Item at index ${index}: Missing required fields: ${missingFields.join(", ")}.`, - }; - } - - let metadataString: string | null = null; - if (item.metadata && typeof item.metadata === "object") { - metadataString = JSON.stringify(item.metadata); - } else if (typeof item.metadata === "string") { - metadataString = item.metadata; - } // item.metadata can also be null if provided as such, which is fine - - return { - valid: true, - processedItem: { ...item, metadata: metadataString }, - }; -}; +import { + inputValidation, + type ContentDataInput, + type ContentRecord, +} from "../route"; // Updated batchInsertContentProcess to use the generic utility const batchInsertContentProcess = async ( supabase: Awaited>, - contentItems: ContentBatchRequestBody, + contentItems: ContentDataInput[], ): Promise> => { - return processAndInsertBatch< - "Content", - Omit & { metadata: string | null } - >( + return validateAndInsertBatch<"Content">( supabase, contentItems, "Content", // Table name "*", // Select query (can be more specific, e.g., "id, text, scale, ...") - validateAndProcessContentItem, "Content", // Entity name for logging + inputValidation, + null, ); }; @@ -78,7 +35,7 @@ export const POST = async (request: NextRequest): Promise => { const supabase = await createClient(); try { - const body: ContentBatchRequestBody = await request.json(); + const body: ContentDataInput[] = await request.json(); if (!Array.isArray(body)) { return createApiResponse(request, { error: "Request body must be an array of content items.", diff --git a/apps/website/app/api/supabase/insert/content/route.ts b/apps/website/app/api/supabase/insert/content/route.ts index e7d3ebfd1..7174690c5 100644 --- a/apps/website/app/api/supabase/insert/content/route.ts +++ b/apps/website/app/api/supabase/insert/content/route.ts @@ -3,6 +3,7 @@ import { NextResponse, NextRequest } from "next/server"; import { getOrCreateEntity, GetOrCreateEntityResult, + ItemValidator, } from "~/utils/supabase/dbUtils"; // Ensure path is correct import { createApiResponse, @@ -11,114 +12,67 @@ import { } from "~/utils/supabase/apiUtils"; // Ensure path is correct import { Tables, TablesInsert } from "~/utils/supabase/types.gen"; -type ContentDataInput = TablesInsert<"Content">; -type ContentRecord = Tables<"Content">; +export type ContentDataInput = TablesInsert<"Content">; +export type ContentRecord = Tables<"Content">; -// Renamed and refactored -const processAndUpsertContentEntry = async ( - supabasePromise: ReturnType, +export const inputValidation: ItemValidator = ( data: ContentDataInput, -): Promise> => { - const { - text, - scale, - space_id, - author_id, - source_local_id, - metadata: rawMetadata, - created, - last_modified, - document_id, - part_of_id, - } = data; +) => { + const { author_id, created, last_modified, scale, space_id, text } = data; // --- Start of extensive validation --- - if (!text || typeof text !== "string") - return { - entity: null, - error: "Invalid or missing text.", - created: false, - status: 400, - }; - if (!scale || typeof scale !== "string") - return { - entity: null, - error: "Invalid or missing scale.", - created: false, - status: 400, - }; + if (!text || typeof text !== "string") return "Invalid or missing text."; + if (!scale || typeof scale !== "string") return "Invalid or missing scale."; if ( space_id === undefined || space_id === null || typeof space_id !== "number" ) - return { - entity: null, - error: "Invalid or missing space_id.", - created: false, - status: 400, - }; + return "Invalid or missing space_id."; if ( author_id === undefined || author_id === null || typeof author_id !== "number" ) - return { - entity: null, - error: "Invalid or missing author_id.", - created: false, - status: 400, - }; - if (!created) - return { - entity: null, - error: "Missing created date.", - created: false, - status: 400, - }; - if (!last_modified) - return { - entity: null, - error: "Missing last_modified date.", - created: false, - status: 400, - }; + return "Invalid or missing author_id."; + if (created) + try { + new Date(created); // Validate date format + new Date(last_modified); // Validate date format + } catch (e) { + return "Invalid date format for created or last_modified."; + } + if (last_modified) + try { + new Date(last_modified); // Validate date format + } catch (e) { + return "Invalid date format for created or last_modified."; + } + // --- End of extensive validation --- - try { - new Date(created); // Validate date format - new Date(last_modified); // Validate date format - } catch (e) { + return null; +}; + +// Renamed and refactored +const processAndUpsertContentEntry = async ( + supabasePromise: ReturnType, + data: ContentDataInput, +): Promise> => { + const { space_id, author_id, source_local_id, document_id, part_of_id } = + data; + + const error = inputValidation(data); + if (error !== null) { return { entity: null, - error: "Invalid date format for created or last_modified.", + error, created: false, status: 400, }; } - // --- End of extensive validation --- - - const processedMetadata = - rawMetadata && typeof rawMetadata === "object" - ? JSON.stringify(rawMetadata) - : typeof rawMetadata === "string" - ? rawMetadata - : null; const supabase = await supabasePromise; - const contentToInsertOrUpdate = { - text, - scale, - space_id, - author_id, - source_local_id: source_local_id || null, - metadata: processedMetadata as any, - created, - last_modified, - document_id: document_id, - part_of_id: part_of_id || null, - }; - let matchCriteria: Record | null = null; if (source_local_id && space_id !== undefined && space_id !== null) { matchCriteria = { space_id: space_id, source_local_id: source_local_id }; @@ -131,7 +85,7 @@ const processAndUpsertContentEntry = async ( "Content", "*", // Select all fields for ContentRecord matchCriteria || { id: -1 }, // Use a non-matching criteria if no specific lookup needed, to force create path if not found - contentToInsertOrUpdate, // This will be used for insert if not found or for update in some extended utilities. + data, // This will be used for insert if not found or for update in some extended utilities. "Content", ); diff --git a/apps/website/app/utils/supabase/dbUtils.ts b/apps/website/app/utils/supabase/dbUtils.ts index d8a87d7cd..daf8533cb 100644 --- a/apps/website/app/utils/supabase/dbUtils.ts +++ b/apps/website/app/utils/supabase/dbUtils.ts @@ -195,96 +195,6 @@ export type BatchProcessResult = { status: number; // HTTP status to suggest }; -export async function processAndInsertBatch< - TableName extends keyof Database["public"]["Tables"], - TProcessed, ->( - supabase: SupabaseClient, - items: TablesInsert[], - tableName: string, - selectQuery: string, // e.g., "id, field1, field2" or "*" - itemValidatorAndProcessor: BatchItemValidator< - TablesInsert, - TProcessed - >, - entityName: string = tableName, // For logging -): Promise>> { - if (!Array.isArray(items) || items.length === 0) { - return { - error: `Request body must be a non-empty array of ${entityName} items.`, - status: 400, - }; - } - - const validationErrors: { index: number; error: string }[] = []; - const processedForDb: TProcessed[] = []; - - for (let i = 0; i < items.length; i++) { - const item = items[i]; - if (!item) { - // Handles undefined/null items in the array itself - validationErrors.push({ index: i, error: "Item is undefined or null." }); - continue; - } - const { valid, error, processedItem } = itemValidatorAndProcessor(item, i); - if (!valid || !processedItem) { - validationErrors.push({ index: i, error: error || "Validation failed." }); - } else { - processedForDb.push(processedItem); - } - } - - if (validationErrors.length > 0) { - return { - error: `Validation failed for one or more ${entityName} items.`, - partial_errors: validationErrors, - status: 400, - }; - } - - if (processedForDb.length === 0 && items.length > 0) { - return { - error: `All ${entityName} items in the batch failed validation or processing.`, - partial_errors: - validationErrors.length > 0 - ? validationErrors - : [{ index: 0, error: `No valid ${entityName} items to process.` }], - status: 400, - }; - } - - const { data: newRecords, error: insertError } = await supabase - .from(tableName) - .insert(processedForDb as any) // Cast as any if TProcessed is not directly insertable; ensure TProcessed matches table - .select(selectQuery); // Use the provided select query - - if (insertError) { - console.error(`Error batch inserting ${entityName}:`, insertError); - return { - error: `Database error during batch insert of ${entityName}.`, - details: insertError.message, - status: 500, - }; - } - - const newRecordsTyped = newRecords as Tables[]; // Assert type - - if (!newRecordsTyped || newRecordsTyped.length !== processedForDb.length) { - console.warn( - `Batch insert ${entityName}: Mismatch between processed count (${processedForDb.length}) and DB returned count (${newRecordsTyped?.length || 0}).`, - ); - return { - error: `Batch insert of ${entityName} might have partially failed or returned unexpected data.`, - status: 500, // Or a more specific error - }; - } - - console.log( - `Successfully batch inserted ${newRecordsTyped.length} ${entityName} records.`, - ); - return { data: newRecordsTyped, status: 201 }; -} - export async function InsertValidatedBatch< TableName extends keyof Database["public"]["Tables"], >( @@ -422,7 +332,7 @@ export async function validateAndInsertBatch< return result; } -export async function processAndInsertBatch_new< +export async function processAndInsertBatch< TableName extends keyof Database["public"]["Tables"], InputType, OutputType, From c018a0e39bf7d3629edb82685e5be1330eb07fd0 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Thu, 22 May 2025 09:52:30 -0400 Subject: [PATCH 18/19] rename content-embedding route, which is general enough. --- .../batch/route.ts | 0 .../routes.ts | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename apps/website/app/api/supabase/insert/{content-embedding-openai-text-embedding-3-small-1536 => content-embedding}/batch/route.ts (100%) rename apps/website/app/api/supabase/insert/{content-embedding-openai-text-embedding-3-small-1536 => content-embedding}/routes.ts (100%) diff --git a/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/batch/route.ts b/apps/website/app/api/supabase/insert/content-embedding/batch/route.ts similarity index 100% rename from apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/batch/route.ts rename to apps/website/app/api/supabase/insert/content-embedding/batch/route.ts diff --git a/apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts b/apps/website/app/api/supabase/insert/content-embedding/routes.ts similarity index 100% rename from apps/website/app/api/supabase/insert/content-embedding-openai-text-embedding-3-small-1536/routes.ts rename to apps/website/app/api/supabase/insert/content-embedding/routes.ts From 6c3eff91b673721e3c8163d56a452f3d0d059503 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Thu, 22 May 2025 22:39:07 -0400 Subject: [PATCH 19/19] Update with schema changes --- .../app/api/supabase/insert/account/route.ts | 2 +- .../insert/content-embedding/batch/route.ts | 2 +- .../content-embedding/{routes.ts => route.ts} | 0 .../app/api/supabase/insert/content/route.ts | 2 +- .../app/api/supabase/insert/person/route.ts | 2 +- .../{discourse-platform => platform}/route.ts | 24 +- .../{discourse-space => space}/route.ts | 51 +- apps/website/app/utils/supabase/types.gen.ts | 995 +++++++++--------- 8 files changed, 538 insertions(+), 540 deletions(-) rename apps/website/app/api/supabase/insert/content-embedding/{routes.ts => route.ts} (100%) rename apps/website/app/api/supabase/insert/{discourse-platform => platform}/route.ts (75%) rename apps/website/app/api/supabase/insert/{discourse-space => space}/route.ts (60%) diff --git a/apps/website/app/api/supabase/insert/account/route.ts b/apps/website/app/api/supabase/insert/account/route.ts index 25a727b46..a5d642398 100644 --- a/apps/website/app/api/supabase/insert/account/route.ts +++ b/apps/website/app/api/supabase/insert/account/route.ts @@ -65,7 +65,7 @@ const getOrCreateAccount = async ( } else if (result.details.includes("Account_platform_id_fkey")) { return { ...result, - error: `Invalid platform_id: No DiscoursePlatform record found for ID ${platform_id}.`, + error: `Invalid platform_id: No Space record found for ID ${platform_id}.`, }; } } diff --git a/apps/website/app/api/supabase/insert/content-embedding/batch/route.ts b/apps/website/app/api/supabase/insert/content-embedding/batch/route.ts index 0c6f5012d..6e28cadf2 100644 --- a/apps/website/app/api/supabase/insert/content-embedding/batch/route.ts +++ b/apps/website/app/api/supabase/insert/content-embedding/batch/route.ts @@ -14,7 +14,7 @@ import { outputProcessing, type ApiInputEmbeddingItem, type ApiOutputEmbeddingRecord, -} from "../routes"; +} from "../route"; const batchInsertEmbeddingsProcess = async ( supabase: Awaited>, diff --git a/apps/website/app/api/supabase/insert/content-embedding/routes.ts b/apps/website/app/api/supabase/insert/content-embedding/route.ts similarity index 100% rename from apps/website/app/api/supabase/insert/content-embedding/routes.ts rename to apps/website/app/api/supabase/insert/content-embedding/route.ts diff --git a/apps/website/app/api/supabase/insert/content/route.ts b/apps/website/app/api/supabase/insert/content/route.ts index 7174690c5..dfba27b52 100644 --- a/apps/website/app/api/supabase/insert/content/route.ts +++ b/apps/website/app/api/supabase/insert/content/route.ts @@ -104,7 +104,7 @@ const processAndUpsertContentEntry = async ( // Be more general with FK name if it changes return { ...result, - error: `Invalid space_id: No DiscourseSpace record found for ID ${space_id}.`, + error: `Invalid space_id: No Space record found for ID ${space_id}.`, }; } if ( diff --git a/apps/website/app/api/supabase/insert/person/route.ts b/apps/website/app/api/supabase/insert/person/route.ts index a4fb765a2..c4f53f7e6 100644 --- a/apps/website/app/api/supabase/insert/person/route.ts +++ b/apps/website/app/api/supabase/insert/person/route.ts @@ -93,7 +93,7 @@ const getOrCreateAccountInternal = async ( if (result.details.includes("Account_platform_id_fkey")) { return { ...result, - error: `Invalid platform_id for Account: No DiscoursePlatform record found for ID ${platformId}.`, + error: `Invalid platform_id for Account: No Space record found for ID ${platformId}.`, }; } } diff --git a/apps/website/app/api/supabase/insert/discourse-platform/route.ts b/apps/website/app/api/supabase/insert/platform/route.ts similarity index 75% rename from apps/website/app/api/supabase/insert/discourse-platform/route.ts rename to apps/website/app/api/supabase/insert/platform/route.ts index 1daf7ad79..45a543af8 100644 --- a/apps/website/app/api/supabase/insert/discourse-platform/route.ts +++ b/apps/website/app/api/supabase/insert/platform/route.ts @@ -11,13 +11,13 @@ import { } from "~/utils/supabase/apiUtils"; import { Tables, TablesInsert } from "~/utils/supabase/types.gen"; -type DiscoursePlatformDataInput = TablesInsert<"DiscoursePlatform">; -type DiscoursePlatformRecord = Tables<"DiscoursePlatform">; +type PlatformDataInput = TablesInsert<"Platform">; +type PlatformRecord = Tables<"Platform">; -const getOrCreateDiscoursePlatformFromURL = async ( +const getOrCreatePlatformFromURL = async ( supabase: ReturnType, url: string, -): Promise> => { +): Promise> => { let platformName: string | null = null; let platformUrl: string | null = null; const lowerCaseURL = url.toLowerCase(); @@ -45,13 +45,13 @@ const getOrCreateDiscoursePlatformFromURL = async ( } const resolvedSupabaseClient = await supabase; - return getOrCreateEntity<"DiscoursePlatform">( + return getOrCreateEntity<"Platform">( resolvedSupabaseClient, - "DiscoursePlatform", + "Platform", "id, name, url", { url: platformUrl }, { name: platformName, url: platformUrl }, - "DiscoursePlatform", + "Platform", ); }; @@ -59,7 +59,7 @@ export const POST = async (request: NextRequest): Promise => { const supabase = createClient(); try { - const body: DiscoursePlatformDataInput = await request.json(); + const body: PlatformDataInput = await request.json(); const { url } = body; if (!url || typeof url !== "string") { @@ -69,7 +69,7 @@ export const POST = async (request: NextRequest): Promise => { }); } - const result = await getOrCreateDiscoursePlatformFromURL(supabase, url); + const result = await getOrCreatePlatformFromURL(supabase, url); return createApiResponse(request, { data: result.entity, @@ -79,11 +79,7 @@ export const POST = async (request: NextRequest): Promise => { created: result.created, }); } catch (e: unknown) { - return handleRouteError( - request, - e, - "/api/supabase/insert/discourse-platform", - ); + return handleRouteError(request, e, "/api/supabase/insert/platform"); } }; diff --git a/apps/website/app/api/supabase/insert/discourse-space/route.ts b/apps/website/app/api/supabase/insert/space/route.ts similarity index 60% rename from apps/website/app/api/supabase/insert/discourse-space/route.ts rename to apps/website/app/api/supabase/insert/space/route.ts index 49aa80f90..23b928259 100644 --- a/apps/website/app/api/supabase/insert/discourse-space/route.ts +++ b/apps/website/app/api/supabase/insert/space/route.ts @@ -11,15 +11,15 @@ import { } from "~/utils/supabase/apiUtils"; import { Tables, TablesInsert } from "~/utils/supabase/types.gen"; -type DiscourseSpaceDataInput = TablesInsert<"DiscourseSpace">; -type DiscourseSpaceRecord = Tables<"DiscourseSpace">; +type SpaceDataInput = TablesInsert<"Space">; +type SpaceRecord = Tables<"Space">; // Renamed and refactored helper function -const processAndGetOrCreateDiscourseSpace = async ( +export const processAndGetOrCreateSpace = async ( supabasePromise: ReturnType, - data: DiscourseSpaceDataInput, -): Promise> => { - const { name, url, discourse_platform_id } = data; + data: SpaceDataInput, +): Promise> => { + const { name, url, platform_id } = data; // --- Start of validation --- if (!name || typeof name !== "string" || name.trim() === "") { @@ -39,13 +39,13 @@ const processAndGetOrCreateDiscourseSpace = async ( }; } if ( - discourse_platform_id === undefined || - discourse_platform_id === null || - typeof discourse_platform_id !== "number" + platform_id === undefined || + platform_id === null || + typeof platform_id !== "number" ) { return { entity: null, - error: "Missing or invalid discourse_platform_id.", + error: "Missing or invalid platform_id.", created: false, status: 400, }; @@ -56,20 +56,20 @@ const processAndGetOrCreateDiscourseSpace = async ( const trimmedName = name.trim(); const supabase = await supabasePromise; - const result = await getOrCreateEntity<"DiscourseSpace">( + const result = await getOrCreateEntity<"Space">( supabase, - "DiscourseSpace", - "id, name, url, discourse_platform_id", - { url: normalizedUrl, discourse_platform_id: discourse_platform_id }, + "Space", + "id, name, url, platform_id", + { url: normalizedUrl, platform_id: platform_id }, { name: trimmedName, url: normalizedUrl, - discourse_platform_id: discourse_platform_id, + platform_id: platform_id, }, - "DiscourseSpace", + "Space", ); - // Custom handling for specific foreign key error related to discourse_platform_id + // Custom handling for specific foreign key error related to platform_id if ( result.error && result.details && @@ -77,14 +77,12 @@ const processAndGetOrCreateDiscourseSpace = async ( result.details.includes("violates foreign key constraint") ) { if ( - result.details - .toLowerCase() - .includes("discoursespace_discourse_platform_id_fkey") || - result.details.toLowerCase().includes("discourse_platform_id") + result.details.toLowerCase().includes("platform_id_fkey") || + result.details.toLowerCase().includes("platform_id") ) { return { ...result, - error: `Invalid discourse_platform_id: No DiscoursePlatform record found for ID ${discourse_platform_id}.`, + error: `Invalid platform_id: No Space record found for ID ${platform_id}.`, }; } } @@ -96,7 +94,7 @@ export const POST = async (request: NextRequest): Promise => { const supabasePromise = createClient(); try { - const body: DiscourseSpaceDataInput = await request.json(); + const body: SpaceDataInput = await request.json(); // Minimal validation here, more detailed in the helper if (!body || typeof body !== "object") { @@ -106,10 +104,7 @@ export const POST = async (request: NextRequest): Promise => { }); } - const result = await processAndGetOrCreateDiscourseSpace( - supabasePromise, - body, - ); + const result = await processAndGetOrCreateSpace(supabasePromise, body); return createApiResponse(request, { data: result.entity, @@ -119,7 +114,7 @@ export const POST = async (request: NextRequest): Promise => { created: result.created, }); } catch (e: unknown) { - return handleRouteError(request, e, "/api/supabase/insert/discourse-space"); + return handleRouteError(request, e, "/api/supabase/insert/space"); } }; diff --git a/apps/website/app/utils/supabase/types.gen.ts b/apps/website/app/utils/supabase/types.gen.ts index 593bf7458..a3981b363 100644 --- a/apps/website/app/utils/supabase/types.gen.ts +++ b/apps/website/app/utils/supabase/types.gen.ts @@ -4,591 +4,597 @@ export type Json = | boolean | null | { [key: string]: Json | undefined } - | Json[]; + | Json[] export type Database = { public: { Tables: { Account: { Row: { - active: boolean; - id: number; - person_id: number; - platform_id: number; - write_permission: boolean; - }; + active: boolean + id: number + person_id: number + platform_id: number + write_permission: boolean + } Insert: { - active?: boolean; - id?: number; - person_id: number; - platform_id: number; - write_permission: boolean; - }; + active?: boolean + id?: number + person_id: number + platform_id: number + write_permission: boolean + } Update: { - active?: boolean; - id?: number; - person_id?: number; - platform_id?: number; - write_permission?: boolean; - }; + active?: boolean + id?: number + person_id?: number + platform_id?: number + write_permission?: boolean + } Relationships: [ { - foreignKeyName: "Account_person_id_fkey"; - columns: ["person_id"]; - isOneToOne: false; - referencedRelation: "Agent"; - referencedColumns: ["id"]; + foreignKeyName: "Account_person_id_fkey" + columns: ["person_id"] + isOneToOne: false + referencedRelation: "Agent" + referencedColumns: ["id"] }, { - foreignKeyName: "Account_platform_id_fkey"; - columns: ["platform_id"]; - isOneToOne: false; - referencedRelation: "DiscoursePlatform"; - referencedColumns: ["id"]; + foreignKeyName: "Account_platform_id_fkey" + columns: ["platform_id"] + isOneToOne: false + referencedRelation: "Platform" + referencedColumns: ["id"] }, - ]; - }; + ] + } Agent: { Row: { - id: number; - type: Database["public"]["Enums"]["EntityType"]; - }; + id: number + type: Database["public"]["Enums"]["EntityType"] + } Insert: { - id?: number; - type: Database["public"]["Enums"]["EntityType"]; - }; + id?: number + type: Database["public"]["Enums"]["EntityType"] + } Update: { - id?: number; - type?: Database["public"]["Enums"]["EntityType"]; - }; - Relationships: []; - }; + id?: number + type?: Database["public"]["Enums"]["EntityType"] + } + Relationships: [] + } AutomatedAgent: { Row: { - deterministic: boolean | null; - id: number; - metadata: Json; - name: string; - version: string | null; - }; + deterministic: boolean | null + id: number + metadata: Json + name: string + version: string | null + } Insert: { - deterministic?: boolean | null; - id: number; - metadata?: Json; - name: string; - version?: string | null; - }; + deterministic?: boolean | null + id: number + metadata?: Json + name: string + version?: string | null + } Update: { - deterministic?: boolean | null; - id?: number; - metadata?: Json; - name?: string; - version?: string | null; - }; + deterministic?: boolean | null + id?: number + metadata?: Json + name?: string + version?: string | null + } Relationships: [ { - foreignKeyName: "automated_agent_id_fkey"; - columns: ["id"]; - isOneToOne: true; - referencedRelation: "Agent"; - referencedColumns: ["id"]; + foreignKeyName: "automated_agent_id_fkey" + columns: ["id"] + isOneToOne: true + referencedRelation: "Agent" + referencedColumns: ["id"] }, - ]; - }; + ] + } Concept: { Row: { - arity: number; - author_id: number | null; - content: Json; - created: string; - description: string | null; - epistemic_status: Database["public"]["Enums"]["EpistemicStatus"]; - id: number; - is_schema: boolean; - last_modified: string; - name: string; - represented_by_id: number | null; - schema_id: number | null; - space_id: number | null; - }; + arity: number + author_id: number | null + content: Json + created: string + description: string | null + epistemic_status: Database["public"]["Enums"]["EpistemicStatus"] + id: number + is_schema: boolean + last_modified: string + name: string + represented_by_id: number | null + schema_id: number | null + space_id: number | null + } Insert: { - arity?: number; - author_id?: number | null; - content?: Json; - created: string; - description?: string | null; - epistemic_status?: Database["public"]["Enums"]["EpistemicStatus"]; - id?: number; - is_schema?: boolean; - last_modified: string; - name: string; - represented_by_id?: number | null; - schema_id?: number | null; - space_id?: number | null; - }; + arity?: number + author_id?: number | null + content?: Json + created: string + description?: string | null + epistemic_status?: Database["public"]["Enums"]["EpistemicStatus"] + id?: number + is_schema?: boolean + last_modified: string + name: string + represented_by_id?: number | null + schema_id?: number | null + space_id?: number | null + } Update: { - arity?: number; - author_id?: number | null; - content?: Json; - created?: string; - description?: string | null; - epistemic_status?: Database["public"]["Enums"]["EpistemicStatus"]; - id?: number; - is_schema?: boolean; - last_modified?: string; - name?: string; - represented_by_id?: number | null; - schema_id?: number | null; - space_id?: number | null; - }; + arity?: number + author_id?: number | null + content?: Json + created?: string + description?: string | null + epistemic_status?: Database["public"]["Enums"]["EpistemicStatus"] + id?: number + is_schema?: boolean + last_modified?: string + name?: string + represented_by_id?: number | null + schema_id?: number | null + space_id?: number | null + } Relationships: [ { - foreignKeyName: "Concept_author_id_fkey"; - columns: ["author_id"]; - isOneToOne: false; - referencedRelation: "Agent"; - referencedColumns: ["id"]; + foreignKeyName: "Concept_author_id_fkey" + columns: ["author_id"] + isOneToOne: false + referencedRelation: "Agent" + referencedColumns: ["id"] }, { - foreignKeyName: "Concept_represented_by_id_fkey"; - columns: ["represented_by_id"]; - isOneToOne: false; - referencedRelation: "Content"; - referencedColumns: ["id"]; + foreignKeyName: "Concept_represented_by_id_fkey" + columns: ["represented_by_id"] + isOneToOne: false + referencedRelation: "Content" + referencedColumns: ["id"] }, { - foreignKeyName: "Concept_schema_id_fkey"; - columns: ["schema_id"]; - isOneToOne: false; - referencedRelation: "Concept"; - referencedColumns: ["id"]; + foreignKeyName: "Concept_schema_id_fkey" + columns: ["schema_id"] + isOneToOne: false + referencedRelation: "Concept" + referencedColumns: ["id"] }, { - foreignKeyName: "Concept_space_id_fkey"; - columns: ["space_id"]; - isOneToOne: false; - referencedRelation: "DiscourseSpace"; - referencedColumns: ["id"]; + foreignKeyName: "Concept_space_id_fkey" + columns: ["space_id"] + isOneToOne: false + referencedRelation: "Space" + referencedColumns: ["id"] }, - ]; - }; + ] + } concept_contributors: { Row: { - concept_id: number; - contributor_id: number; - }; + concept_id: number + contributor_id: number + } Insert: { - concept_id: number; - contributor_id: number; - }; + concept_id: number + contributor_id: number + } Update: { - concept_id?: number; - contributor_id?: number; - }; + concept_id?: number + contributor_id?: number + } Relationships: [ { - foreignKeyName: "concept_contributors_concept_id_fkey"; - columns: ["concept_id"]; - isOneToOne: false; - referencedRelation: "Concept"; - referencedColumns: ["id"]; + foreignKeyName: "concept_contributors_concept_id_fkey" + columns: ["concept_id"] + isOneToOne: false + referencedRelation: "Concept" + referencedColumns: ["id"] }, { - foreignKeyName: "concept_contributors_contributor_id_fkey"; - columns: ["contributor_id"]; - isOneToOne: false; - referencedRelation: "Agent"; - referencedColumns: ["id"]; + foreignKeyName: "concept_contributors_contributor_id_fkey" + columns: ["contributor_id"] + isOneToOne: false + referencedRelation: "Agent" + referencedColumns: ["id"] }, - ]; - }; + ] + } Content: { Row: { - author_id: number | null; - created: string; - creator_id: number | null; - document_id: number; - id: number; - last_modified: string; - metadata: Json; - part_of_id: number | null; - scale: Database["public"]["Enums"]["Scale"]; - source_local_id: string | null; - space_id: number | null; - text: string; - }; + author_id: number | null + created: string + creator_id: number | null + document_id: number + id: number + last_modified: string + metadata: Json + part_of_id: number | null + scale: Database["public"]["Enums"]["Scale"] + source_local_id: string | null + space_id: number | null + text: string + } Insert: { - author_id?: number | null; - created: string; - creator_id?: number | null; - document_id: number; - id?: number; - last_modified: string; - metadata?: Json; - part_of_id?: number | null; - scale: Database["public"]["Enums"]["Scale"]; - source_local_id?: string | null; - space_id?: number | null; - text: string; - }; + author_id?: number | null + created: string + creator_id?: number | null + document_id: number + id?: number + last_modified: string + metadata?: Json + part_of_id?: number | null + scale: Database["public"]["Enums"]["Scale"] + source_local_id?: string | null + space_id?: number | null + text: string + } Update: { - author_id?: number | null; - created?: string; - creator_id?: number | null; - document_id?: number; - id?: number; - last_modified?: string; - metadata?: Json; - part_of_id?: number | null; - scale?: Database["public"]["Enums"]["Scale"]; - source_local_id?: string | null; - space_id?: number | null; - text?: string; - }; + author_id?: number | null + created?: string + creator_id?: number | null + document_id?: number + id?: number + last_modified?: string + metadata?: Json + part_of_id?: number | null + scale?: Database["public"]["Enums"]["Scale"] + source_local_id?: string | null + space_id?: number | null + text?: string + } Relationships: [ { - foreignKeyName: "Content_author_id_fkey"; - columns: ["author_id"]; - isOneToOne: false; - referencedRelation: "Agent"; - referencedColumns: ["id"]; + foreignKeyName: "Content_author_id_fkey" + columns: ["author_id"] + isOneToOne: false + referencedRelation: "Agent" + referencedColumns: ["id"] }, { - foreignKeyName: "Content_creator_id_fkey"; - columns: ["creator_id"]; - isOneToOne: false; - referencedRelation: "Agent"; - referencedColumns: ["id"]; + foreignKeyName: "Content_creator_id_fkey" + columns: ["creator_id"] + isOneToOne: false + referencedRelation: "Agent" + referencedColumns: ["id"] }, { - foreignKeyName: "Content_document_id_fkey"; - columns: ["document_id"]; - isOneToOne: false; - referencedRelation: "Document"; - referencedColumns: ["id"]; + foreignKeyName: "Content_document_id_fkey" + columns: ["document_id"] + isOneToOne: false + referencedRelation: "Document" + referencedColumns: ["id"] }, { - foreignKeyName: "Content_part_of_id_fkey"; - columns: ["part_of_id"]; - isOneToOne: false; - referencedRelation: "Content"; - referencedColumns: ["id"]; + foreignKeyName: "Content_part_of_id_fkey" + columns: ["part_of_id"] + isOneToOne: false + referencedRelation: "Content" + referencedColumns: ["id"] }, { - foreignKeyName: "Content_space_id_fkey"; - columns: ["space_id"]; - isOneToOne: false; - referencedRelation: "DiscourseSpace"; - referencedColumns: ["id"]; + foreignKeyName: "Content_space_id_fkey" + columns: ["space_id"] + isOneToOne: false + referencedRelation: "Space" + referencedColumns: ["id"] }, - ]; - }; + ] + } content_contributors: { Row: { - content_id: number; - contributor_id: number; - }; + content_id: number + contributor_id: number + } Insert: { - content_id: number; - contributor_id: number; - }; + content_id: number + contributor_id: number + } Update: { - content_id?: number; - contributor_id?: number; - }; + content_id?: number + contributor_id?: number + } Relationships: [ { - foreignKeyName: "content_contributors_content_id_fkey"; - columns: ["content_id"]; - isOneToOne: false; - referencedRelation: "Content"; - referencedColumns: ["id"]; + foreignKeyName: "content_contributors_content_id_fkey" + columns: ["content_id"] + isOneToOne: false + referencedRelation: "Content" + referencedColumns: ["id"] }, { - foreignKeyName: "content_contributors_contributor_id_fkey"; - columns: ["contributor_id"]; - isOneToOne: false; - referencedRelation: "Agent"; - referencedColumns: ["id"]; + foreignKeyName: "content_contributors_contributor_id_fkey" + columns: ["contributor_id"] + isOneToOne: false + referencedRelation: "Agent" + referencedColumns: ["id"] }, - ]; - }; + ] + } ContentEmbedding_openai_text_embedding_3_small_1536: { Row: { - model: Database["public"]["Enums"]["EmbeddingName"]; - obsolete: boolean | null; - target_id: number; - vector: string; - }; + model: Database["public"]["Enums"]["EmbeddingName"] + obsolete: boolean | null + target_id: number + vector: string + } Insert: { - model?: Database["public"]["Enums"]["EmbeddingName"]; - obsolete?: boolean | null; - target_id: number; - vector: string; - }; + model?: Database["public"]["Enums"]["EmbeddingName"] + obsolete?: boolean | null + target_id: number + vector: string + } Update: { - model?: Database["public"]["Enums"]["EmbeddingName"]; - obsolete?: boolean | null; - target_id?: number; - vector?: string; - }; + model?: Database["public"]["Enums"]["EmbeddingName"] + obsolete?: boolean | null + target_id?: number + vector?: string + } Relationships: [ { - foreignKeyName: "ContentEmbedding_openai_text_embedding_3_small_1_target_id_fkey"; - columns: ["target_id"]; - isOneToOne: true; - referencedRelation: "Content"; - referencedColumns: ["id"]; + foreignKeyName: "ContentEmbedding_openai_text_embedding_3_small_1_target_id_fkey" + columns: ["target_id"] + isOneToOne: true + referencedRelation: "Content" + referencedColumns: ["id"] }, - ]; - }; - DiscoursePlatform: { - Row: { - id: number; - name: string; - url: string; - }; - Insert: { - id?: number; - name: string; - url: string; - }; - Update: { - id?: number; - name?: string; - url?: string; - }; - Relationships: []; - }; - DiscourseSpace: { + ] + } + Document: { Row: { - discourse_platform_id: number; - id: number; - name: string; - url: string | null; - }; + author_id: number + contents: unknown | null + created: string + id: number + last_modified: string + metadata: Json + source_local_id: string | null + space_id: number | null + url: string | null + } Insert: { - discourse_platform_id: number; - id?: number; - name: string; - url?: string | null; - }; + author_id: number + contents?: unknown | null + created: string + id?: number + last_modified: string + metadata?: Json + source_local_id?: string | null + space_id?: number | null + url?: string | null + } Update: { - discourse_platform_id?: number; - id?: number; - name?: string; - url?: string | null; - }; + author_id?: number + contents?: unknown | null + created?: string + id?: number + last_modified?: string + metadata?: Json + source_local_id?: string | null + space_id?: number | null + url?: string | null + } Relationships: [ { - foreignKeyName: "DiscourseSpace_discourse_platform_id_fkey"; - columns: ["discourse_platform_id"]; - isOneToOne: false; - referencedRelation: "DiscoursePlatform"; - referencedColumns: ["id"]; + foreignKeyName: "Document_author_id_fkey" + columns: ["author_id"] + isOneToOne: false + referencedRelation: "Agent" + referencedColumns: ["id"] }, - ]; - }; - Document: { + { + foreignKeyName: "Document_space_id_fkey" + columns: ["space_id"] + isOneToOne: false + referencedRelation: "Space" + referencedColumns: ["id"] + }, + ] + } + Person: { Row: { - author_id: number; - contents: unknown | null; - created: string; - id: number; - last_modified: string; - metadata: Json; - source_local_id: string | null; - space_id: number | null; - url: string | null; - }; + email: string + id: number + name: string + orcid: string | null + } Insert: { - author_id: number; - contents?: unknown | null; - created: string; - id?: number; - last_modified: string; - metadata?: Json; - source_local_id?: string | null; - space_id?: number | null; - url?: string | null; - }; + email: string + id: number + name: string + orcid?: string | null + } Update: { - author_id?: number; - contents?: unknown | null; - created?: string; - id?: number; - last_modified?: string; - metadata?: Json; - source_local_id?: string | null; - space_id?: number | null; - url?: string | null; - }; + email?: string + id?: number + name?: string + orcid?: string | null + } Relationships: [ { - foreignKeyName: "Document_author_id_fkey"; - columns: ["author_id"]; - isOneToOne: false; - referencedRelation: "Agent"; - referencedColumns: ["id"]; + foreignKeyName: "person_id_fkey" + columns: ["id"] + isOneToOne: true + referencedRelation: "Agent" + referencedColumns: ["id"] }, - { - foreignKeyName: "Document_space_id_fkey"; - columns: ["space_id"]; - isOneToOne: false; - referencedRelation: "DiscourseSpace"; - referencedColumns: ["id"]; - }, - ]; - }; - Person: { + ] + } + Platform: { + Row: { + id: number + name: string + url: string + } + Insert: { + id?: number + name: string + url: string + } + Update: { + id?: number + name?: string + url?: string + } + Relationships: [] + } + Space: { Row: { - email: string; - id: number; - name: string; - orcid: string | null; - }; + id: number + name: string + platform_id: number + url: string | null + } Insert: { - email: string; - id: number; - name: string; - orcid?: string | null; - }; + id?: number + name: string + platform_id: number + url?: string | null + } Update: { - email?: string; - id?: number; - name?: string; - orcid?: string | null; - }; + id?: number + name?: string + platform_id?: number + url?: string | null + } Relationships: [ { - foreignKeyName: "person_id_fkey"; - columns: ["id"]; - isOneToOne: true; - referencedRelation: "Agent"; - referencedColumns: ["id"]; + foreignKeyName: "Space_platform_id_fkey" + columns: ["platform_id"] + isOneToOne: false + referencedRelation: "Platform" + referencedColumns: ["id"] }, - ]; - }; + ] + } SpaceAccess: { Row: { - account_id: number; - editor: boolean; - id: number; - space_id: number | null; - }; + account_id: number + editor: boolean + id: number + space_id: number | null + } Insert: { - account_id: number; - editor: boolean; - id?: number; - space_id?: number | null; - }; + account_id: number + editor: boolean + id?: number + space_id?: number | null + } Update: { - account_id?: number; - editor?: boolean; - id?: number; - space_id?: number | null; - }; + account_id?: number + editor?: boolean + id?: number + space_id?: number | null + } Relationships: [ { - foreignKeyName: "SpaceAccess_account_id_fkey"; - columns: ["account_id"]; - isOneToOne: false; - referencedRelation: "Account"; - referencedColumns: ["id"]; + foreignKeyName: "SpaceAccess_account_id_fkey" + columns: ["account_id"] + isOneToOne: false + referencedRelation: "Account" + referencedColumns: ["id"] }, { - foreignKeyName: "SpaceAccess_space_id_fkey"; - columns: ["space_id"]; - isOneToOne: false; - referencedRelation: "DiscourseSpace"; - referencedColumns: ["id"]; + foreignKeyName: "SpaceAccess_space_id_fkey" + columns: ["space_id"] + isOneToOne: false + referencedRelation: "Space" + referencedColumns: ["id"] }, - ]; - }; + ] + } sync_info: { Row: { - failure_count: number | null; - id: number; - last_task_end: string | null; - last_task_start: string | null; - status: Database["public"]["Enums"]["task_status"] | null; - sync_function: string | null; - sync_target: number | null; - task_times_out_at: string | null; - worker: string; - }; + failure_count: number | null + id: number + last_task_end: string | null + last_task_start: string | null + status: Database["public"]["Enums"]["task_status"] | null + sync_function: string | null + sync_target: number | null + task_times_out_at: string | null + worker: string + } Insert: { - failure_count?: number | null; - id?: number; - last_task_end?: string | null; - last_task_start?: string | null; - status?: Database["public"]["Enums"]["task_status"] | null; - sync_function?: string | null; - sync_target?: number | null; - task_times_out_at?: string | null; - worker: string; - }; + failure_count?: number | null + id?: number + last_task_end?: string | null + last_task_start?: string | null + status?: Database["public"]["Enums"]["task_status"] | null + sync_function?: string | null + sync_target?: number | null + task_times_out_at?: string | null + worker: string + } Update: { - failure_count?: number | null; - id?: number; - last_task_end?: string | null; - last_task_start?: string | null; - status?: Database["public"]["Enums"]["task_status"] | null; - sync_function?: string | null; - sync_target?: number | null; - task_times_out_at?: string | null; - worker?: string; - }; - Relationships: []; - }; - }; + failure_count?: number | null + id?: number + last_task_end?: string | null + last_task_start?: string | null + status?: Database["public"]["Enums"]["task_status"] | null + sync_function?: string | null + sync_target?: number | null + task_times_out_at?: string | null + worker?: string + } + Relationships: [] + } + } Views: { - [_ in never]: never; - }; + [_ in never]: never + } Functions: { end_sync_task: { Args: { - s_target: number; - s_function: string; - s_worker: string; - s_status: Database["public"]["Enums"]["task_status"]; - }; - Returns: undefined; - }; + s_target: number + s_function: string + s_worker: string + s_status: Database["public"]["Enums"]["task_status"] + } + Returns: undefined + } + get_nodes_needing_sync: { + Args: { nodes_from_roam: Json } + Returns: { + uid_to_sync: string + }[] + } match_content_embeddings: { Args: { - query_embedding: string; - match_threshold: number; - match_count: number; - current_document_id?: number; - }; + query_embedding: string + match_threshold: number + match_count: number + current_document_id?: number + } Returns: { - content_id: number; - roam_uid: string; - text_content: string; - similarity: number; - }[]; - }; + content_id: number + roam_uid: string + text_content: string + similarity: number + }[] + } match_embeddings_for_subset_nodes: { - Args: { p_query_embedding: string; p_subset_roam_uids: string[] }; + Args: { p_query_embedding: string; p_subset_roam_uids: string[] } Returns: { - content_id: number; - roam_uid: string; - text_content: string; - similarity: number; - }[]; - }; + content_id: number + roam_uid: string + text_content: string + similarity: number + }[] + } propose_sync_task: { Args: { - s_target: number; - s_function: string; - s_worker: string; - timeout: unknown; - task_interval: unknown; - }; - Returns: unknown; - }; - }; + s_target: number + s_function: string + s_worker: string + timeout: unknown + task_interval: unknown + } + Returns: unknown + } + } Enums: { EmbeddingName: | "openai_text_embedding_ada2_1536" @@ -596,7 +602,7 @@ export type Database = { | "openai_text_embedding_3_small_1536" | "openai_text_embedding_3_large_256" | "openai_text_embedding_3_large_1024" - | "openai_text_embedding_3_large_3072"; + | "openai_text_embedding_3_large_3072" EntityType: | "Platform" | "Space" @@ -608,7 +614,7 @@ export type Database = { | "Concept" | "ConceptSchema" | "ContentLink" - | "Occurrence"; + | "Occurrence" EpistemicStatus: | "certainly_not" | "strong_evidence_against" @@ -618,7 +624,7 @@ export type Database = { | "contentious" | "could_be_true" | "strong_evidence_for" - | "certain"; + | "certain" Scale: | "document" | "post" @@ -629,23 +635,23 @@ export type Database = { | "paragraph" | "quote" | "sentence" - | "phrase"; - task_status: "active" | "timeout" | "complete" | "failed"; - }; + | "phrase" + task_status: "active" | "timeout" | "complete" | "failed" + } CompositeTypes: { - [_ in never]: never; - }; - }; -}; + [_ in never]: never + } + } +} -type DefaultSchema = Database[Extract]; +type DefaultSchema = Database[Extract] export type Tables< DefaultSchemaTableNameOrOptions extends | keyof (DefaultSchema["Tables"] & DefaultSchema["Views"]) | { schema: keyof Database }, TableName extends DefaultSchemaTableNameOrOptions extends { - schema: keyof Database; + schema: keyof Database } ? keyof (Database[DefaultSchemaTableNameOrOptions["schema"]]["Tables"] & Database[DefaultSchemaTableNameOrOptions["schema"]]["Views"]) @@ -653,7 +659,7 @@ export type Tables< > = DefaultSchemaTableNameOrOptions extends { schema: keyof Database } ? (Database[DefaultSchemaTableNameOrOptions["schema"]]["Tables"] & Database[DefaultSchemaTableNameOrOptions["schema"]]["Views"])[TableName] extends { - Row: infer R; + Row: infer R } ? R : never @@ -661,64 +667,64 @@ export type Tables< DefaultSchema["Views"]) ? (DefaultSchema["Tables"] & DefaultSchema["Views"])[DefaultSchemaTableNameOrOptions] extends { - Row: infer R; + Row: infer R } ? R : never - : never; + : never export type TablesInsert< DefaultSchemaTableNameOrOptions extends | keyof DefaultSchema["Tables"] | { schema: keyof Database }, TableName extends DefaultSchemaTableNameOrOptions extends { - schema: keyof Database; + schema: keyof Database } ? keyof Database[DefaultSchemaTableNameOrOptions["schema"]]["Tables"] : never = never, > = DefaultSchemaTableNameOrOptions extends { schema: keyof Database } ? Database[DefaultSchemaTableNameOrOptions["schema"]]["Tables"][TableName] extends { - Insert: infer I; + Insert: infer I } ? I : never : DefaultSchemaTableNameOrOptions extends keyof DefaultSchema["Tables"] ? DefaultSchema["Tables"][DefaultSchemaTableNameOrOptions] extends { - Insert: infer I; + Insert: infer I } ? I : never - : never; + : never export type TablesUpdate< DefaultSchemaTableNameOrOptions extends | keyof DefaultSchema["Tables"] | { schema: keyof Database }, TableName extends DefaultSchemaTableNameOrOptions extends { - schema: keyof Database; + schema: keyof Database } ? keyof Database[DefaultSchemaTableNameOrOptions["schema"]]["Tables"] : never = never, > = DefaultSchemaTableNameOrOptions extends { schema: keyof Database } ? Database[DefaultSchemaTableNameOrOptions["schema"]]["Tables"][TableName] extends { - Update: infer U; + Update: infer U } ? U : never : DefaultSchemaTableNameOrOptions extends keyof DefaultSchema["Tables"] ? DefaultSchema["Tables"][DefaultSchemaTableNameOrOptions] extends { - Update: infer U; + Update: infer U } ? U : never - : never; + : never export type Enums< DefaultSchemaEnumNameOrOptions extends | keyof DefaultSchema["Enums"] | { schema: keyof Database }, EnumName extends DefaultSchemaEnumNameOrOptions extends { - schema: keyof Database; + schema: keyof Database } ? keyof Database[DefaultSchemaEnumNameOrOptions["schema"]]["Enums"] : never = never, @@ -726,14 +732,14 @@ export type Enums< ? Database[DefaultSchemaEnumNameOrOptions["schema"]]["Enums"][EnumName] : DefaultSchemaEnumNameOrOptions extends keyof DefaultSchema["Enums"] ? DefaultSchema["Enums"][DefaultSchemaEnumNameOrOptions] - : never; + : never export type CompositeTypes< PublicCompositeTypeNameOrOptions extends | keyof DefaultSchema["CompositeTypes"] | { schema: keyof Database }, CompositeTypeName extends PublicCompositeTypeNameOrOptions extends { - schema: keyof Database; + schema: keyof Database } ? keyof Database[PublicCompositeTypeNameOrOptions["schema"]]["CompositeTypes"] : never = never, @@ -741,7 +747,7 @@ export type CompositeTypes< ? Database[PublicCompositeTypeNameOrOptions["schema"]]["CompositeTypes"][CompositeTypeName] : PublicCompositeTypeNameOrOptions extends keyof DefaultSchema["CompositeTypes"] ? DefaultSchema["CompositeTypes"][PublicCompositeTypeNameOrOptions] - : never; + : never export const Constants = { public: { @@ -793,4 +799,5 @@ export const Constants = { task_status: ["active", "timeout", "complete", "failed"], }, }, -} as const; +} as const +