From d9fb9dbeb98630c37eb5da867994bf42afc81429 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Fri, 23 May 2025 07:49:35 -0400 Subject: [PATCH 01/27] Squash of eng-294-create-supabase-insertupdate-route Work by Sid: Hyde utility, embedding route, supabase insert routes Marc-Antoine: introduce generated types, further work on insert routes --- .../components/DiscourseContextOverlay.tsx | 356 +- apps/roam/src/utils/hyde.ts | 373 + .../app/api/embeddings/openai/small/route.ts | 97 + .../app/api/supabase/insert/account/route.ts | 108 + .../app/api/supabase/insert/agents/route.ts | 71 + .../insert/content-embedding/batch/route.ts | 105 + .../insert/content-embedding/route.ts | 249 + .../supabase/insert/content/batch/route.ts | 63 + .../app/api/supabase/insert/content/route.ts | 172 + .../app/api/supabase/insert/document/route.ts | 144 + .../app/api/supabase/insert/person/route.ts | 199 + .../app/api/supabase/insert/platform/route.ts | 86 + .../app/api/supabase/insert/space/route.ts | 121 + apps/website/app/utils/supabase/apiUtils.ts | 88 + apps/website/app/utils/supabase/dbUtils.ts | 425 + apps/website/app/utils/supabase/server.ts | 44 + apps/website/app/utils/supabase/types.gen.ts | 803 + apps/website/package.json | 2 + package-lock.json | 13262 ++++++---------- 19 files changed, 8538 insertions(+), 8230 deletions(-) create mode 100644 apps/roam/src/utils/hyde.ts create mode 100644 apps/website/app/api/embeddings/openai/small/route.ts create mode 100644 apps/website/app/api/supabase/insert/account/route.ts create mode 100644 apps/website/app/api/supabase/insert/agents/route.ts create mode 100644 apps/website/app/api/supabase/insert/content-embedding/batch/route.ts create mode 100644 apps/website/app/api/supabase/insert/content-embedding/route.ts create mode 100644 apps/website/app/api/supabase/insert/content/batch/route.ts create mode 100644 apps/website/app/api/supabase/insert/content/route.ts create mode 100644 apps/website/app/api/supabase/insert/document/route.ts create mode 100644 apps/website/app/api/supabase/insert/person/route.ts create mode 100644 apps/website/app/api/supabase/insert/platform/route.ts create mode 100644 apps/website/app/api/supabase/insert/space/route.ts create mode 100644 apps/website/app/utils/supabase/apiUtils.ts create mode 100644 apps/website/app/utils/supabase/dbUtils.ts create mode 100644 apps/website/app/utils/supabase/server.ts create mode 100644 apps/website/app/utils/supabase/types.gen.ts diff --git a/apps/roam/src/components/DiscourseContextOverlay.tsx b/apps/roam/src/components/DiscourseContextOverlay.tsx index 25220de50..9f986aecd 100644 --- a/apps/roam/src/components/DiscourseContextOverlay.tsx +++ b/apps/roam/src/components/DiscourseContextOverlay.tsx @@ -1,4 +1,12 @@ -import { Button, Icon, Popover, Position, Tooltip } from "@blueprintjs/core"; +import { + Button, + Icon, + Popover, + Position, + Tooltip, + ControlGroup, + Spinner, +} from "@blueprintjs/core"; import React, { useCallback, useEffect, useMemo, useState } from "react"; import ReactDOM from "react-dom"; import { ContextContent } from "./DiscourseContext"; @@ -11,10 +19,23 @@ import nanoid from "nanoid"; import getPageUidByPageTitle from "roamjs-components/queries/getPageUidByPageTitle"; import getDiscourseContextResults from "~/utils/getDiscourseContextResults"; import findDiscourseNode from "~/utils/findDiscourseNode"; -import getDiscourseNodes from "~/utils/getDiscourseNodes"; -import getDiscourseRelations from "~/utils/getDiscourseRelations"; +import getDiscourseNodes, { DiscourseNode } from "~/utils/getDiscourseNodes"; +import getDiscourseRelations, { + DiscourseRelation, +} from "~/utils/getDiscourseRelations"; import ExtensionApiContextProvider from "roamjs-components/components/ExtensionApiContext"; import { OnloadArgs } from "roamjs-components/types/native"; +import AutocompleteInput from "roamjs-components/components/AutocompleteInput"; +import getAllPageNames from "roamjs-components/queries/getAllPageNames"; +import { Result } from "roamjs-components/types/query-builder"; +import createBlock from "roamjs-components/writes/createBlock"; +import { getBlockUidFromTarget } from "roamjs-components/dom"; +import { + findSimilarNodesUsingHyde, + SuggestedNode, + RelationDetails, + CandidateNodeWithEmbedding, +} from "~/utils/hyde"; type DiscourseData = { results: Awaited>; @@ -25,11 +46,13 @@ const cache: { [tag: string]: DiscourseData; } = {}; -const getOverlayInfo = async (tag: string): Promise => { +const getOverlayInfo = async ( + tag: string, + relations: ReturnType, +): Promise => { try { if (cache[tag]) return cache[tag]; - const relations = getDiscourseRelations(); const nodes = getDiscourseNodes(relations); const [results, refs] = await Promise.all([ @@ -57,43 +80,267 @@ const getOverlayInfo = async (tag: string): Promise => { } }; -const DiscourseContextOverlay = ({ tag, id }: { tag: string; id: string }) => { +const getAllReferencesOnPage = (pageTitle: string) => { + const referencedPages = window.roamAlphaAPI.data.q( + `[:find ?uid ?text + :where + [?page :node/title "${normalizePageTitle(pageTitle)}"] + [?b :block/page ?page] + [?b :block/refs ?refPage] + [?refPage :block/uid ?uid] + [?refPage :node/title ?text]]`, + ); + return referencedPages.map(([uid, text]) => ({ + uid, + text, + })) as Result[]; +}; + +const DiscourseContextOverlay = ({ + tag, + id, + parentEl, +}: { + tag: string; + id: string; + parentEl: HTMLElement; +}) => { const tagUid = useMemo(() => getPageUidByPageTitle(tag), [tag]); + const blockUid = useMemo(() => getBlockUidFromTarget(parentEl), [parentEl]); const [loading, setLoading] = useState(true); const [results, setResults] = useState([]); const [refs, setRefs] = useState(0); const [score, setScore] = useState(0); + const [isSearchingHyde, setIsSearchingHyde] = useState(false); + const [hydeFilteredNodes, setHydeFilteredNodes] = useState( + [], + ); + + const discourseNode = useMemo(() => findDiscourseNode(tagUid), [tagUid]); + const relations = useMemo(() => getDiscourseRelations(), []); + const allNodes = useMemo(() => getDiscourseNodes(), []); + const getInfo = useCallback( () => - getOverlayInfo(tag) + getOverlayInfo(tag, relations) .then(({ refs, results }) => { - const discourseNode = findDiscourseNode(tagUid); - if (discourseNode) { - const attribute = getSettingValueFromTree({ - tree: getBasicTreeByParentUid(discourseNode.type), - key: "Overlay", - defaultValue: "Overlay", - }); - return deriveDiscourseNodeAttribute({ - uid: tagUid, - attribute, - }).then((score) => { - setResults(results); - setRefs(refs); - setScore(score); - }); - } + if (!discourseNode) return; + const attribute = getSettingValueFromTree({ + tree: getBasicTreeByParentUid(discourseNode.type), + key: "Overlay", + defaultValue: "Overlay", + }); + return deriveDiscourseNodeAttribute({ + uid: tagUid, + attribute, + }).then((score) => { + setResults(results); + setRefs(refs); + setScore(score); + }); }) .finally(() => setLoading(false)), [tag, setResults, setLoading, setRefs, setScore], ); + const refresh = useCallback(() => { setLoading(true); getInfo(); }, [getInfo, setLoading]); + useEffect(() => { getInfo(); }, [refresh, getInfo]); + + const suggestionContextData = useMemo(() => { + const selfNode = discourseNode; + + if (!selfNode || !selfNode.text || !selfNode.format) { + return { + validTypes: [], + uniqueRelationTypeTriplets: [], + }; + } + const selfType = selfNode.type; + + const relationsConnectingToSelf = relations.filter( + (relation) => + relation.source === selfType || relation.destination === selfType, + ); + + const uniqueTriplets = useMemo(() => { + const relatedNodeType = selfNode.type; + + return relationsConnectingToSelf.flatMap((relation) => { + const isSelfSource = relation.source === relatedNodeType; + const isSelfDestination = relation.destination === relatedNodeType; + + let targetNodeType: string; + let currentRelationLabel: string; + + if (isSelfSource) { + targetNodeType = relation.destination; + currentRelationLabel = relation.label; + } else if (isSelfDestination) { + targetNodeType = relation.source; + currentRelationLabel = relation.complement; + } else { + return []; + } + + const identifiedTargetNode = allNodes.find( + (node) => node.type === targetNodeType, + ); + + if (!identifiedTargetNode) { + return []; + } + + const mappedItem: RelationDetails = { + relationLabel: currentRelationLabel, + relatedNodeText: identifiedTargetNode.text, + relatedNodeFormat: identifiedTargetNode.format, + }; + return [mappedItem]; + }); + }, [relationsConnectingToSelf, selfNode.type, allNodes]); + + const relationsInvolvingSelfBroadly = relations.filter((relation) => + [relation.source, relation.destination, relation.label].includes( + selfType, + ), + ); + const hasSelfRelation = relationsInvolvingSelfBroadly.some( + (relation) => + relation.source === selfType && relation.destination === selfType, + ); + const types = Array.from( + new Set( + relationsInvolvingSelfBroadly.flatMap((relation) => [ + relation.source, + relation.destination, + ]), + ), + ); + const filteredTypes = hasSelfRelation + ? types + : types.filter((type) => type !== selfType); + + return { + validTypes: filteredTypes, + uniqueRelationTypeTriplets: uniqueTriplets, + }; + }, [discourseNode, relations, allNodes]); + + const { validTypes, uniqueRelationTypeTriplets } = suggestionContextData; + + const [currentPageInput, setCurrentPageInput] = useState(""); + const [selectedPage, setSelectedPage] = useState(null); + const allPages = useMemo(() => getAllPageNames(), []); + useEffect(() => { + setSelectedPage(null); + }, [currentPageInput]); + + useEffect(() => { + if (!selectedPage) { + setHydeFilteredNodes([]); + return; + } + const nodesOnPage = getAllReferencesOnPage(selectedPage); + const nodes = nodesOnPage + .map((n) => { + const node = findDiscourseNode(n.uid); + if (!node || node.backedBy === "default") return null; + if (!validTypes.includes(node.type)) return null; + return { + uid: n.uid, + text: n.text, + type: node.type, + }; + }) + .filter((node): node is SuggestedNode => node !== null) + .filter( + (node) => + !results.some((r) => + Object.values(r.results).some((result) => result.uid === node.uid), + ), + ); + + if (nodes.length > 0 && uniqueRelationTypeTriplets.length > 0) { + const performSearch = async () => { + setIsSearchingHyde(true); + setHydeFilteredNodes([]); + try { + const foundNodes = await runHydeSearch({ + currentSuggestions: nodes, + currentNodeText: tag, + relationDetails: uniqueRelationTypeTriplets, + }); + setHydeFilteredNodes(foundNodes); + } catch (error) { + console.error( + "Error during HyDE search operation in useEffect:", + error, + ); + setHydeFilteredNodes([]); + } finally { + setIsSearchingHyde(false); + } + }; + performSearch(); + } + }, [selectedPage, results, validTypes, tag, uniqueRelationTypeTriplets]); + + const handleCreateBlock = async (node: SuggestedNode) => { + await createBlock({ + parentUid: blockUid, + node: { text: `[[${node.text}]]` }, + }); + setHydeFilteredNodes(hydeFilteredNodes.filter((n) => n.uid !== node.uid)); + }; + + type RunHydeSearchArgs = { + currentSuggestions: SuggestedNode[]; + currentNodeText: string; + relationDetails: RelationDetails[]; + }; + + const runHydeSearch = async ({ + currentSuggestions, + currentNodeText, + relationDetails, + }: RunHydeSearchArgs): Promise => { + if ( + !currentSuggestions.length || + !currentNodeText || + !relationDetails.length + ) { + return []; + } + + try { + const candidateNodesForHyde = currentSuggestions.map((node) => ({ + uid: node.uid, + text: node.text, + type: node.type, + })); + + const foundNodes: SuggestedNode[] = await findSimilarNodesUsingHyde({ + candidateNodes: candidateNodesForHyde, + currentNodeText: currentNodeText, + relationDetails: relationDetails, + }); + + return foundNodes; + } catch (error) { + console.error( + "Error during HyDE search with default RPC subset search:", + error, + ); + return []; + } + }; + return ( { }`} > + {/* Suggestive Mode */} +
+
+ + { + if (e.key === "Enter") { + setSelectedPage(currentPageInput); + } + }} + > + +
+ {selectedPage && ( +
+

+ Suggested Relationships +

+ {isSearchingHyde && ( + + )} +
    + {!isSearchingHyde && hydeFilteredNodes.length > 0 + ? hydeFilteredNodes.map((node) => ( +
  • + {node.text} +
  • + )) + : null} + {!isSearchingHyde && hydeFilteredNodes.length === 0 && ( +
  • No relevant relations found.
  • + )} +
+
+ )} +
} target={ @@ -143,7 +451,7 @@ const Wrapper = ({ parent, tag }: { parent: HTMLElement; tag: string }) => { {}, ); return inViewport ? ( - + ) : (