From e2b9b377c6a5bc25f8a1433f0856309ee260c085 Mon Sep 17 00:00:00 2001 From: sid597 Date: Sat, 9 Aug 2025 01:40:50 +0530 Subject: [PATCH 01/30] settings for context and page groups --- .../components/settings/PageGroupPanel.tsx | 196 ++++++++++++++++++ .../roam/src/components/settings/Settings.tsx | 7 + .../settings/SuggestiveModeSettings.tsx | 53 +++++ apps/roam/src/utils/configPageTabs.ts | 28 +++ apps/roam/src/utils/discourseConfigRef.ts | 45 ++++ 5 files changed, 329 insertions(+) create mode 100644 apps/roam/src/components/settings/PageGroupPanel.tsx create mode 100644 apps/roam/src/components/settings/SuggestiveModeSettings.tsx diff --git a/apps/roam/src/components/settings/PageGroupPanel.tsx b/apps/roam/src/components/settings/PageGroupPanel.tsx new file mode 100644 index 000000000..99db30c42 --- /dev/null +++ b/apps/roam/src/components/settings/PageGroupPanel.tsx @@ -0,0 +1,196 @@ +import React, { useState, useCallback, useMemo } from "react"; +import { Label, Button, Intent, Tag } from "@blueprintjs/core"; +import Description from "roamjs-components/components/Description"; +import AutocompleteInput from "roamjs-components/components/AutocompleteInput"; +import getBasicTreeByParentUid from "roamjs-components/queries/getBasicTreeByParentUid"; +import createBlock from "roamjs-components/writes/createBlock"; +import deleteBlock from "roamjs-components/writes/deleteBlock"; +import getDiscourseNodes from "~/utils/getDiscourseNodes"; +import getAllPageNames from "roamjs-components/queries/getAllPageNames"; + +type PageGroupData = { + uid: string; + name: string; + pages: { uid: string; name: string }[]; +}; + +const PageGroupsPanel = ({ uid }: { uid: string }) => { + const [pageGroups, setPageGroups] = useState(() => + getBasicTreeByParentUid(uid).map((node) => ({ + uid: node.uid, + name: node.text, + pages: node.children.map((c) => ({ uid: c.uid, name: c.text })), + })), + ); + + const refreshGroups = useCallback(() => { + setPageGroups( + getBasicTreeByParentUid(uid).map((node) => ({ + uid: node.uid, + name: node.text, + pages: node.children.map((c) => ({ uid: c.uid, name: c.text })), + })), + ); + }, [uid, setPageGroups]); + + const [newGroupName, setNewGroupName] = useState(""); + const [newPageInputs, setNewPageInputs] = useState>( + {}, + ); + const [autocompleteKeys, setAutocompleteKeys] = useState< + Record + >({}); + + const addGroup = async (name: string) => { + if (!name || pageGroups.some((g) => g.name === name)) return; + await createBlock({ parentUid: uid, node: { text: name } }).then(() => { + refreshGroups(); + setNewGroupName(""); + }); + }; + + const removeGroup = async (groupUid: string) => { + await deleteBlock(groupUid).then(() => { + refreshGroups(); + }); + }; + + const addPageToGroup = async (groupUid: string, page: string) => { + const group = pageGroups.find((g) => g.uid === groupUid); + if (!page || group?.pages.some((p) => p.name === page)) { + return; + } + await createBlock({ parentUid: groupUid, node: { text: page } }).then( + () => { + refreshGroups(); + setNewPageInputs((prev) => ({ + ...prev, + [groupUid]: "", + })); + setAutocompleteKeys((prev) => ({ + ...prev, + [groupUid]: (prev[groupUid] || 0) + 1, + })); + }, + ); + }; + + const removePageFromGroup = async (pageUid: string) => { + await deleteBlock(pageUid).then(() => { + refreshGroups(); + }); + }; + + const getPageInput = (groupUid: string) => newPageInputs[groupUid] || ""; + const setPageInput = useCallback((groupUid: string, value: string) => { + setTimeout(() => { + setNewPageInputs((prev) => ({ + ...prev, + [groupUid]: value, + })); + }, 0); + }, []); + const getAutocompleteKey = (groupUid: string) => + autocompleteKeys[groupUid] || 0; + + return ( +
+ +
+ ); +}; + +export default PageGroupsPanel; diff --git a/apps/roam/src/components/settings/Settings.tsx b/apps/roam/src/components/settings/Settings.tsx index 6046fa158..ed5e89533 100644 --- a/apps/roam/src/components/settings/Settings.tsx +++ b/apps/roam/src/components/settings/Settings.tsx @@ -25,6 +25,7 @@ import sendErrorEmail from "~/utils/sendErrorEmail"; import HomePersonalSettings from "./HomePersonalSettings"; import refreshConfigTree from "~/utils/refreshConfigTree"; import { FeedbackWidget } from "~/components/BirdEatsBugs"; +import SuggestiveModeSettings from "./SuggestiveModeSettings"; type SectionHeaderProps = { children: React.ReactNode; @@ -160,6 +161,12 @@ export const SettingsDialog = ({ className="overflow-y-auto" panel={} /> + } + /> Grammar { + const settings = useMemo(() => { + refreshConfigTree(); + return getFormattedConfigTree(); + }, []); + + return ( +
+
+ Discourse Suggestions +
+
+
+
+ + +
+
+ +
+
+ ); +}; + +export default SuggestiveModeSettings; diff --git a/apps/roam/src/utils/configPageTabs.ts b/apps/roam/src/utils/configPageTabs.ts index 0fdf8a4d6..3112a6a63 100644 --- a/apps/roam/src/utils/configPageTabs.ts +++ b/apps/roam/src/utils/configPageTabs.ts @@ -14,6 +14,7 @@ import { CustomField, SelectField, } from "roamjs-components/components/ConfigPanels/types"; +import PageGroupsPanel from "~/components/settings/PageGroupPanel"; export const configPageTabs = (args: OnloadArgs): ConfigTab[] => [ { @@ -116,4 +117,31 @@ export const configPageTabs = (args: OnloadArgs): ConfigTab[] => [ }, ], }, + { + id: "suggestive-mode", + fields: [ + { + title: "Include Current Page Relations", + // @ts-ignore + Panel: FlagPanel, + description: + "Include relations from pages referenced on the current page", + }, + { + title: "Include Parent and Child Blocks", + // @ts-ignore + Panel: FlagPanel, + description: "Include relations from parent and child blocks", + }, + { + title: "Page Groups", + // @ts-ignore + Panel: CustomPanel, + description: "Set page groups to use for discourse suggestions", + options: { + component: PageGroupsPanel, + }, + }, + ], + }, ]; diff --git a/apps/roam/src/utils/discourseConfigRef.ts b/apps/roam/src/utils/discourseConfigRef.ts index 3cfc91cdc..7c4a38920 100644 --- a/apps/roam/src/utils/discourseConfigRef.ts +++ b/apps/roam/src/utils/discourseConfigRef.ts @@ -4,15 +4,56 @@ import { StringSetting, ExportConfigWithUids, getUidAndStringSetting, + BooleanSetting, + getUidAndBooleanSetting, } from "./getExportSettings"; import { DISCOURSE_CONFIG_PAGE_TITLE } from "~/utils/renderNodeConfigPage"; import getPageUidByPageTitle from "roamjs-components/queries/getPageUidByPageTitle"; +import { getSubTree } from "roamjs-components/util"; const configTreeRef: { tree: RoamBasicNode[]; nodes: { [uid: string]: { text: string; children: RoamBasicNode[] } }; } = { tree: [], nodes: {} }; +type SuggestiveModeConfigWithUids = { + parentUid: string; + grabFromReferencedPages: BooleanSetting; + grabParentAndChildren: BooleanSetting; + pageGroups: { + uid: string; + name: string; + pages: { uid: string; name: string }[]; + }; +}; + +const getSuggestiveModeConfigAndUids = (): SuggestiveModeConfigWithUids => { + const suggestiveModeNode = getSubTree({ + tree: configTreeRef.tree, + key: "suggestive-mode", + }); + const pageGroupsNode = suggestiveModeNode.children.find( + (node) => node.text === "Page Groups", + ); + + return { + parentUid: suggestiveModeNode.uid, + grabFromReferencedPages: getUidAndBooleanSetting({ + tree: suggestiveModeNode.children, + text: "Include Current Page Relations", + }), + grabParentAndChildren: getUidAndBooleanSetting({ + tree: suggestiveModeNode.children, + text: "Include Parent and Child Blocks", + }), + pageGroups: { + uid: pageGroupsNode?.uid || "", + name: "page groups", + pages: [], + }, + }; +}; + type FormattedConfigTree = { settingsUid: string; grammarUid: string; @@ -21,13 +62,16 @@ type FormattedConfigTree = { trigger: StringSetting; export: ExportConfigWithUids; canvasPageFormat: StringSetting; + suggestiveMode: SuggestiveModeConfigWithUids; }; export const getFormattedConfigTree = (): FormattedConfigTree => { const settingsUid = getPageUidByPageTitle(DISCOURSE_CONFIG_PAGE_TITLE); + console.log("settingsUid", settingsUid); const grammarNode = configTreeRef.tree.find( (node) => node.text === "grammar", ); + console.log("grammarNode", grammarNode); const relationsNode = grammarNode?.children.find( (node) => node.text === "relations", ); @@ -47,6 +91,7 @@ export const getFormattedConfigTree = (): FormattedConfigTree => { tree: configTreeRef.tree, text: "Canvas Page Format", }), + suggestiveMode: getSuggestiveModeConfigAndUids(), }; }; export default configTreeRef; From da54c72b65b412e296d4185131c5b929443a17ea Mon Sep 17 00:00:00 2001 From: sid597 Date: Sun, 10 Aug 2025 03:10:03 +0530 Subject: [PATCH 02/30] node settings --- .../settings/DiscourseNodeSuggestiveRules.tsx | 159 ++++++++++++++++++ .../src/components/settings/NodeConfig.tsx | 17 ++ apps/roam/src/utils/discourseConfigRef.ts | 2 - apps/roam/src/utils/renderNodeConfigPage.ts | 13 ++ 4 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx diff --git a/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx new file mode 100644 index 000000000..a4eb4adbd --- /dev/null +++ b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx @@ -0,0 +1,159 @@ +import React, { + useState, + useMemo, + useEffect, + useRef, + useCallback, +} from "react"; +import { Button, Intent, Tooltip, Position } from "@blueprintjs/core"; +import BlocksPanel from "roamjs-components/components/ConfigPanels/BlocksPanel"; +import getBasicTreeByParentUid from "roamjs-components/queries/getBasicTreeByParentUid"; +import refreshConfigTree from "~/utils/refreshConfigTree"; +import FlagPanel from "roamjs-components/components/ConfigPanels/FlagPanel"; +import { getUidAndBooleanSetting } from "~/utils/getExportSettings"; +import TextPanel from "roamjs-components/components/ConfigPanels/TextPanel"; +import getSubTree from "roamjs-components/util/getSubTree"; + +type DiscourseNode = { + type: string; + text: string; + template?: string; + uid: string; +}; + +const BlockRenderer = ({ uid }: { uid: string }) => { + const containerRef = useRef(null); + + useEffect(() => { + const container = containerRef.current; + if (container) { + while (container.firstChild) { + container.removeChild(container.firstChild); + } + + if (uid) { + window.roamAlphaAPI.ui.components.renderBlock({ + uid: uid, + el: container, + }); + } + } + }, [uid]); + + return
; +}; + +const DiscourseNodeSuggestiveRules = ({ + node, + parentUid, +}: { + node: DiscourseNode; + parentUid: string; +}) => { + const nodeUid = node.type; + const nodeConfigTree = useMemo(() => { + refreshConfigTree(); + const tree = getBasicTreeByParentUid(parentUid); + const embeddingRefNode = tree.find((n) => + n.text.startsWith("Embedding Block Ref"), + ); + const match = embeddingRefNode?.children?.[0]?.text?.match(/\(\((.*)\)\)/); + const blockRef = match ? `((${match[1]}))` : ""; + return { + embeddingRef: blockRef, + embeddingRefUid: embeddingRefNode?.uid || "", + isFirstChild: getUidAndBooleanSetting({ + tree: tree, + text: "First Child", + }), + }; + }, [parentUid]); + + const [embeddingRef, setEmbeddingRef] = useState(nodeConfigTree.embeddingRef); + + useEffect(() => { + if (nodeConfigTree.embeddingRef !== embeddingRef) { + setEmbeddingRef(nodeConfigTree.embeddingRef); + } + }, [nodeConfigTree.embeddingRef]); + + const blockUidToRender = useMemo(() => { + const match = embeddingRef?.match(/\(\((.*)\)\)/); + return match ? match[1] : ""; + }, [embeddingRef]); + + const templateUid = useMemo( + () => + getSubTree({ + parentUid: nodeUid, + key: "Template", + }).uid, + [nodeUid], + ); + + const handleEmbeddingRefChange = useCallback( + (e: React.ChangeEvent) => { + const newValue = e.target.value; + setEmbeddingRef(newValue); + }, + [], + ); + + return ( +
+ + + + + {blockUidToRender && ( +
+
Preview:
+ +
+ )} + + + +
+ +
+
+ ); +}; + +export default DiscourseNodeSuggestiveRules; diff --git a/apps/roam/src/components/settings/NodeConfig.tsx b/apps/roam/src/components/settings/NodeConfig.tsx index 0b1bd5eca..8f49869e0 100644 --- a/apps/roam/src/components/settings/NodeConfig.tsx +++ b/apps/roam/src/components/settings/NodeConfig.tsx @@ -15,6 +15,7 @@ import { OnloadArgs } from "roamjs-components/types"; import getBasicTreeByParentUid from "roamjs-components/queries/getBasicTreeByParentUid"; import createBlock from "roamjs-components/writes/createBlock"; import updateBlock from "roamjs-components/writes/updateBlock"; +import DiscourseNodeSuggestiveRules from "./DiscourseNodeSuggestiveRules"; const ValidatedInputPanel = ({ label, @@ -327,6 +328,22 @@ const NodeConfig = ({
} /> + + n.text === "Suggestive Rules", + )?.uid || "" + } + /> + + } + /> ); diff --git a/apps/roam/src/utils/discourseConfigRef.ts b/apps/roam/src/utils/discourseConfigRef.ts index 7c4a38920..4bdb4e784 100644 --- a/apps/roam/src/utils/discourseConfigRef.ts +++ b/apps/roam/src/utils/discourseConfigRef.ts @@ -67,11 +67,9 @@ type FormattedConfigTree = { export const getFormattedConfigTree = (): FormattedConfigTree => { const settingsUid = getPageUidByPageTitle(DISCOURSE_CONFIG_PAGE_TITLE); - console.log("settingsUid", settingsUid); const grammarNode = configTreeRef.tree.find( (node) => node.text === "grammar", ); - console.log("grammarNode", grammarNode); const relationsNode = grammarNode?.children.find( (node) => node.text === "relations", ); diff --git a/apps/roam/src/utils/renderNodeConfigPage.ts b/apps/roam/src/utils/renderNodeConfigPage.ts index de2ee07b6..0e5d223dc 100644 --- a/apps/roam/src/utils/renderNodeConfigPage.ts +++ b/apps/roam/src/utils/renderNodeConfigPage.ts @@ -24,6 +24,7 @@ import { } from "~/components"; import getDiscourseNodes from "~/utils/getDiscourseNodes"; import { render as configPageRender } from "roamjs-components/components/ConfigPage"; +import DiscourseNodeSuggestiveRules from "~/components/settings/DiscourseNodeSuggestiveRules"; export const DISCOURSE_CONFIG_PAGE_TITLE = "roam/js/discourse-graph"; export const NODE_CONFIG_PAGE_TITLE = "discourse-graph/nodes/"; @@ -147,6 +148,18 @@ export const renderNodeConfigPage = ({ description: `Whether to color the node in the graph overview based on canvas color. This is based on the node's plain title as described by a \`has title\` condition in its specification.`, defaultValue: true, } as FieldPanel, + // @ts-ignore + { + title: "Suggestive Rules", + Panel: CustomPanel, + options: { + component: ({ uid }) => + React.createElement(DiscourseNodeSuggestiveRules, { + node, + parentUid: uid, + }), + }, + } as Field, ], }); From f83a0079e53c38f06b2d43f0cf2612a1eb1ee59e Mon Sep 17 00:00:00 2001 From: sid597 Date: Sun, 10 Aug 2025 11:38:02 +0530 Subject: [PATCH 03/30] address review --- .../settings/DiscourseNodeSuggestiveRules.tsx | 2 +- .../src/components/settings/NodeConfig.tsx | 7 ++- .../components/settings/PageGroupPanel.tsx | 58 +++++++++++-------- .../settings/SuggestiveModeSettings.tsx | 7 ++- apps/roam/src/utils/configPageTabs.ts | 4 +- apps/roam/src/utils/discourseConfigRef.ts | 7 ++- 6 files changed, 48 insertions(+), 37 deletions(-) diff --git a/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx index a4eb4adbd..d75f4604d 100644 --- a/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx +++ b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx @@ -105,7 +105,7 @@ const DiscourseNodeSuggestiveRules = ({ title="Template" description={`The template that auto fills ${node.text} page when generated.`} order={0} - parentUid={node.uid} + parentUid={nodeUid} uid={templateUid} defaultValue={node.template ? [{ text: node.template }] : undefined} /> diff --git a/apps/roam/src/components/settings/NodeConfig.tsx b/apps/roam/src/components/settings/NodeConfig.tsx index 8f49869e0..0261bb3b5 100644 --- a/apps/roam/src/components/settings/NodeConfig.tsx +++ b/apps/roam/src/components/settings/NodeConfig.tsx @@ -336,9 +336,10 @@ const NodeConfig = ({ n.text === "Suggestive Rules", - )?.uid || "" + getSubTree({ + parentUid: node.type, + key: "Suggestive Rules", + }).uid } /> diff --git a/apps/roam/src/components/settings/PageGroupPanel.tsx b/apps/roam/src/components/settings/PageGroupPanel.tsx index 99db30c42..38b20311e 100644 --- a/apps/roam/src/components/settings/PageGroupPanel.tsx +++ b/apps/roam/src/components/settings/PageGroupPanel.tsx @@ -43,16 +43,22 @@ const PageGroupsPanel = ({ uid }: { uid: string }) => { const addGroup = async (name: string) => { if (!name || pageGroups.some((g) => g.name === name)) return; - await createBlock({ parentUid: uid, node: { text: name } }).then(() => { + try { + await createBlock({ parentUid: uid, node: { text: name } }); refreshGroups(); setNewGroupName(""); - }); + } catch (e) { + console.error("Error adding group", e); + } }; const removeGroup = async (groupUid: string) => { - await deleteBlock(groupUid).then(() => { + try { + await deleteBlock(groupUid); refreshGroups(); - }); + } catch (e) { + console.error("Error removing group", e); + } }; const addPageToGroup = async (groupUid: string, page: string) => { @@ -60,35 +66,37 @@ const PageGroupsPanel = ({ uid }: { uid: string }) => { if (!page || group?.pages.some((p) => p.name === page)) { return; } - await createBlock({ parentUid: groupUid, node: { text: page } }).then( - () => { - refreshGroups(); - setNewPageInputs((prev) => ({ - ...prev, - [groupUid]: "", - })); - setAutocompleteKeys((prev) => ({ - ...prev, - [groupUid]: (prev[groupUid] || 0) + 1, - })); - }, - ); + try { + await createBlock({ parentUid: groupUid, node: { text: page } }); + refreshGroups(); + setNewPageInputs((prev) => ({ + ...prev, + [groupUid]: "", + })); + setAutocompleteKeys((prev) => ({ + ...prev, + [groupUid]: (prev[groupUid] || 0) + 1, + })); + } catch (e) { + console.error("Error adding page to group", e); + } }; const removePageFromGroup = async (pageUid: string) => { - await deleteBlock(pageUid).then(() => { + try { + await deleteBlock(pageUid); refreshGroups(); - }); + } catch (e) { + console.error("Error removing page from group", e); + } }; const getPageInput = (groupUid: string) => newPageInputs[groupUid] || ""; const setPageInput = useCallback((groupUid: string, value: string) => { - setTimeout(() => { - setNewPageInputs((prev) => ({ - ...prev, - [groupUid]: value, - })); - }, 0); + setNewPageInputs((prev) => ({ + ...prev, + [groupUid]: value, + })); }, []); const getAutocompleteKey = (groupUid: string) => autocompleteKeys[groupUid] || 0; diff --git a/apps/roam/src/components/settings/SuggestiveModeSettings.tsx b/apps/roam/src/components/settings/SuggestiveModeSettings.tsx index 0371b48f1..3d2bc3316 100644 --- a/apps/roam/src/components/settings/SuggestiveModeSettings.tsx +++ b/apps/roam/src/components/settings/SuggestiveModeSettings.tsx @@ -1,4 +1,4 @@ -import React, { useMemo } from "react"; +import React, { useEffect, useState } from "react"; import { Button, Intent } from "@blueprintjs/core"; import { getFormattedConfigTree } from "~/utils/discourseConfigRef"; import refreshConfigTree from "~/utils/refreshConfigTree"; @@ -6,9 +6,10 @@ import FlagPanel from "roamjs-components/components/ConfigPanels/FlagPanel"; import PageGroupsPanel from "./PageGroupPanel"; const SuggestiveModeSettings = () => { - const settings = useMemo(() => { + const [settings, setSettings] = useState(() => getFormattedConfigTree()); + useEffect(() => { refreshConfigTree(); - return getFormattedConfigTree(); + setSettings(getFormattedConfigTree()); }, []); return ( diff --git a/apps/roam/src/utils/configPageTabs.ts b/apps/roam/src/utils/configPageTabs.ts index 3112a6a63..c2fe5a75a 100644 --- a/apps/roam/src/utils/configPageTabs.ts +++ b/apps/roam/src/utils/configPageTabs.ts @@ -133,15 +133,15 @@ export const configPageTabs = (args: OnloadArgs): ConfigTab[] => [ Panel: FlagPanel, description: "Include relations from parent and child blocks", }, + // @ts-ignore { title: "Page Groups", - // @ts-ignore Panel: CustomPanel, description: "Set page groups to use for discourse suggestions", options: { component: PageGroupsPanel, }, - }, + } as Field, ], }, ]; diff --git a/apps/roam/src/utils/discourseConfigRef.ts b/apps/roam/src/utils/discourseConfigRef.ts index 4bdb4e784..25239b9f1 100644 --- a/apps/roam/src/utils/discourseConfigRef.ts +++ b/apps/roam/src/utils/discourseConfigRef.ts @@ -32,9 +32,10 @@ const getSuggestiveModeConfigAndUids = (): SuggestiveModeConfigWithUids => { tree: configTreeRef.tree, key: "suggestive-mode", }); - const pageGroupsNode = suggestiveModeNode.children.find( - (node) => node.text === "Page Groups", - ); + const pageGroupsNode = getSubTree({ + parentUid: suggestiveModeNode.uid, + key: "page groups", + }); return { parentUid: suggestiveModeNode.uid, From ee58296419ef1c03c50498dfc5a3f319e858f1a0 Mon Sep 17 00:00:00 2001 From: sid597 Date: Sun, 10 Aug 2025 13:12:35 +0530 Subject: [PATCH 04/30] fix node creation if it does not exist --- .../components/settings/PageGroupPanel.tsx | 10 ++++---- .../settings/SuggestiveModeSettings.tsx | 24 +++++++++++++++---- apps/roam/src/utils/configPageTabs.ts | 2 +- apps/roam/src/utils/discourseConfigRef.ts | 2 +- 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/apps/roam/src/components/settings/PageGroupPanel.tsx b/apps/roam/src/components/settings/PageGroupPanel.tsx index 38b20311e..a9b928f03 100644 --- a/apps/roam/src/components/settings/PageGroupPanel.tsx +++ b/apps/roam/src/components/settings/PageGroupPanel.tsx @@ -93,10 +93,12 @@ const PageGroupsPanel = ({ uid }: { uid: string }) => { const getPageInput = (groupUid: string) => newPageInputs[groupUid] || ""; const setPageInput = useCallback((groupUid: string, value: string) => { - setNewPageInputs((prev) => ({ - ...prev, - [groupUid]: value, - })); + setTimeout(() => { + setNewPageInputs((prev) => ({ + ...prev, + [groupUid]: value, + })); + }, 0); }, []); const getAutocompleteKey = (groupUid: string) => autocompleteKeys[groupUid] || 0; diff --git a/apps/roam/src/components/settings/SuggestiveModeSettings.tsx b/apps/roam/src/components/settings/SuggestiveModeSettings.tsx index 3d2bc3316..bb7097035 100644 --- a/apps/roam/src/components/settings/SuggestiveModeSettings.tsx +++ b/apps/roam/src/components/settings/SuggestiveModeSettings.tsx @@ -4,13 +4,29 @@ import { getFormattedConfigTree } from "~/utils/discourseConfigRef"; import refreshConfigTree from "~/utils/refreshConfigTree"; import FlagPanel from "roamjs-components/components/ConfigPanels/FlagPanel"; import PageGroupsPanel from "./PageGroupPanel"; +import createBlock from "roamjs-components/writes/createBlock"; +import getPageUidByPageTitle from "roamjs-components/queries/getPageUidByPageTitle"; +import { DISCOURSE_CONFIG_PAGE_TITLE } from "~/utils/renderNodeConfigPage"; const SuggestiveModeSettings = () => { - const [settings, setSettings] = useState(() => getFormattedConfigTree()); - useEffect(() => { + const [settings, setSettings] = useState(() => { refreshConfigTree(); - setSettings(getFormattedConfigTree()); - }, []); + return getFormattedConfigTree(); + }); + + useEffect(() => { + const ensureSettings = async () => { + if (!settings.suggestiveMode.parentUid) { + await createBlock({ + parentUid: getPageUidByPageTitle(DISCOURSE_CONFIG_PAGE_TITLE), + node: { text: "Suggestive mode" }, + }); + refreshConfigTree(); + setSettings(getFormattedConfigTree()); + } + }; + ensureSettings(); + }, [settings.suggestiveMode.parentUid]); return (
diff --git a/apps/roam/src/utils/configPageTabs.ts b/apps/roam/src/utils/configPageTabs.ts index c2fe5a75a..28aebf28a 100644 --- a/apps/roam/src/utils/configPageTabs.ts +++ b/apps/roam/src/utils/configPageTabs.ts @@ -118,7 +118,7 @@ export const configPageTabs = (args: OnloadArgs): ConfigTab[] => [ ], }, { - id: "suggestive-mode", + id: "Suggestive mode", fields: [ { title: "Include Current Page Relations", diff --git a/apps/roam/src/utils/discourseConfigRef.ts b/apps/roam/src/utils/discourseConfigRef.ts index 25239b9f1..9f11e2a38 100644 --- a/apps/roam/src/utils/discourseConfigRef.ts +++ b/apps/roam/src/utils/discourseConfigRef.ts @@ -30,7 +30,7 @@ type SuggestiveModeConfigWithUids = { const getSuggestiveModeConfigAndUids = (): SuggestiveModeConfigWithUids => { const suggestiveModeNode = getSubTree({ tree: configTreeRef.tree, - key: "suggestive-mode", + key: "Suggestive mode", }); const pageGroupsNode = getSubTree({ parentUid: suggestiveModeNode.uid, From 87f38d30a2abc24da93d9f3c1d4b768509cac945 Mon Sep 17 00:00:00 2001 From: sid597 Date: Sun, 10 Aug 2025 13:28:13 +0530 Subject: [PATCH 05/30] remove unused, refactor --- .../settings/DiscourseNodeSuggestiveRules.tsx | 8 +--- .../components/settings/PageGroupPanel.tsx | 3 +- apps/roam/src/utils/discourseConfigRef.ts | 46 ++----------------- .../utils/getSuggestiveModeConfigSettings.ts | 43 +++++++++++++++++ 4 files changed, 49 insertions(+), 51 deletions(-) create mode 100644 apps/roam/src/utils/getSuggestiveModeConfigSettings.ts diff --git a/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx index d75f4604d..b3b03cf47 100644 --- a/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx +++ b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx @@ -13,13 +13,7 @@ import FlagPanel from "roamjs-components/components/ConfigPanels/FlagPanel"; import { getUidAndBooleanSetting } from "~/utils/getExportSettings"; import TextPanel from "roamjs-components/components/ConfigPanels/TextPanel"; import getSubTree from "roamjs-components/util/getSubTree"; - -type DiscourseNode = { - type: string; - text: string; - template?: string; - uid: string; -}; +import { DiscourseNode } from "~/utils/getDiscourseNodes"; const BlockRenderer = ({ uid }: { uid: string }) => { const containerRef = useRef(null); diff --git a/apps/roam/src/components/settings/PageGroupPanel.tsx b/apps/roam/src/components/settings/PageGroupPanel.tsx index a9b928f03..11fa99769 100644 --- a/apps/roam/src/components/settings/PageGroupPanel.tsx +++ b/apps/roam/src/components/settings/PageGroupPanel.tsx @@ -1,11 +1,10 @@ -import React, { useState, useCallback, useMemo } from "react"; +import React, { useState, useCallback } from "react"; import { Label, Button, Intent, Tag } from "@blueprintjs/core"; import Description from "roamjs-components/components/Description"; import AutocompleteInput from "roamjs-components/components/AutocompleteInput"; import getBasicTreeByParentUid from "roamjs-components/queries/getBasicTreeByParentUid"; import createBlock from "roamjs-components/writes/createBlock"; import deleteBlock from "roamjs-components/writes/deleteBlock"; -import getDiscourseNodes from "~/utils/getDiscourseNodes"; import getAllPageNames from "roamjs-components/queries/getAllPageNames"; type PageGroupData = { diff --git a/apps/roam/src/utils/discourseConfigRef.ts b/apps/roam/src/utils/discourseConfigRef.ts index 9f11e2a38..bc4041bcd 100644 --- a/apps/roam/src/utils/discourseConfigRef.ts +++ b/apps/roam/src/utils/discourseConfigRef.ts @@ -4,57 +4,19 @@ import { StringSetting, ExportConfigWithUids, getUidAndStringSetting, - BooleanSetting, - getUidAndBooleanSetting, } from "./getExportSettings"; import { DISCOURSE_CONFIG_PAGE_TITLE } from "~/utils/renderNodeConfigPage"; import getPageUidByPageTitle from "roamjs-components/queries/getPageUidByPageTitle"; -import { getSubTree } from "roamjs-components/util"; +import { + getSuggestiveModeConfigAndUids, + SuggestiveModeConfigWithUids, +} from "./getSuggestiveModeConfigSettings"; const configTreeRef: { tree: RoamBasicNode[]; nodes: { [uid: string]: { text: string; children: RoamBasicNode[] } }; } = { tree: [], nodes: {} }; -type SuggestiveModeConfigWithUids = { - parentUid: string; - grabFromReferencedPages: BooleanSetting; - grabParentAndChildren: BooleanSetting; - pageGroups: { - uid: string; - name: string; - pages: { uid: string; name: string }[]; - }; -}; - -const getSuggestiveModeConfigAndUids = (): SuggestiveModeConfigWithUids => { - const suggestiveModeNode = getSubTree({ - tree: configTreeRef.tree, - key: "Suggestive mode", - }); - const pageGroupsNode = getSubTree({ - parentUid: suggestiveModeNode.uid, - key: "page groups", - }); - - return { - parentUid: suggestiveModeNode.uid, - grabFromReferencedPages: getUidAndBooleanSetting({ - tree: suggestiveModeNode.children, - text: "Include Current Page Relations", - }), - grabParentAndChildren: getUidAndBooleanSetting({ - tree: suggestiveModeNode.children, - text: "Include Parent and Child Blocks", - }), - pageGroups: { - uid: pageGroupsNode?.uid || "", - name: "page groups", - pages: [], - }, - }; -}; - type FormattedConfigTree = { settingsUid: string; grammarUid: string; diff --git a/apps/roam/src/utils/getSuggestiveModeConfigSettings.ts b/apps/roam/src/utils/getSuggestiveModeConfigSettings.ts new file mode 100644 index 000000000..16a58b7a0 --- /dev/null +++ b/apps/roam/src/utils/getSuggestiveModeConfigSettings.ts @@ -0,0 +1,43 @@ +import { getSubTree } from "roamjs-components/util"; +import configTreeRef from "./discourseConfigRef"; +import { BooleanSetting, getUidAndBooleanSetting } from "./getExportSettings"; + +export type SuggestiveModeConfigWithUids = { + parentUid: string; + grabFromReferencedPages: BooleanSetting; + grabParentAndChildren: BooleanSetting; + pageGroups: { + uid: string; + name: string; + pages: { uid: string; name: string }[]; + }; +}; + +export const getSuggestiveModeConfigAndUids = + (): SuggestiveModeConfigWithUids => { + const suggestiveModeNode = getSubTree({ + tree: configTreeRef.tree, + key: "Suggestive mode", + }); + const pageGroupsNode = getSubTree({ + parentUid: suggestiveModeNode.uid, + key: "page groups", + }); + + return { + parentUid: suggestiveModeNode.uid, + grabFromReferencedPages: getUidAndBooleanSetting({ + tree: suggestiveModeNode.children, + text: "Include Current Page Relations", + }), + grabParentAndChildren: getUidAndBooleanSetting({ + tree: suggestiveModeNode.children, + text: "Include Parent and Child Blocks", + }), + pageGroups: { + uid: pageGroupsNode?.uid || "", + name: "page groups", + pages: [], + }, + }; + }; From 5a6bfddfbf4dda00af7871c888aca8d6c08a5f88 Mon Sep 17 00:00:00 2001 From: sid597 Date: Sun, 10 Aug 2025 19:07:55 +0530 Subject: [PATCH 06/30] address coderabbit review --- .../settings/DiscourseNodeSuggestiveRules.tsx | 42 +++++++++++-------- apps/roam/src/utils/configPageTabs.ts | 2 +- apps/roam/src/utils/discourseConfigRef.ts | 2 +- .../utils/getSuggestiveModeConfigSettings.ts | 12 +++--- 4 files changed, 33 insertions(+), 25 deletions(-) diff --git a/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx index b3b03cf47..23e948bcb 100644 --- a/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx +++ b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx @@ -15,6 +15,23 @@ import TextPanel from "roamjs-components/components/ConfigPanels/TextPanel"; import getSubTree from "roamjs-components/util/getSubTree"; import { DiscourseNode } from "~/utils/getDiscourseNodes"; +const getNodeConfig = (parentUid: string) => { + const tree = getBasicTreeByParentUid(parentUid); + const embeddingRefNode = tree.find((n) => + n.text.startsWith("Embedding Block Ref"), + ); + const match = embeddingRefNode?.children?.[0]?.text?.match(/\(\((.*)\)\)/); + const blockRef = match ? `((${match[1]}))` : ""; + return { + embeddingRef: blockRef, + embeddingRefUid: embeddingRefNode?.uid || "", + isFirstChild: getUidAndBooleanSetting({ + tree: tree, + text: "First Child", + }), + }; +}; + const BlockRenderer = ({ uid }: { uid: string }) => { const containerRef = useRef(null); @@ -45,22 +62,13 @@ const DiscourseNodeSuggestiveRules = ({ parentUid: string; }) => { const nodeUid = node.type; - const nodeConfigTree = useMemo(() => { + const [nodeConfigTree, setNodeConfigTree] = useState(() => + getNodeConfig(parentUid), + ); + + useEffect(() => { refreshConfigTree(); - const tree = getBasicTreeByParentUid(parentUid); - const embeddingRefNode = tree.find((n) => - n.text.startsWith("Embedding Block Ref"), - ); - const match = embeddingRefNode?.children?.[0]?.text?.match(/\(\((.*)\)\)/); - const blockRef = match ? `((${match[1]}))` : ""; - return { - embeddingRef: blockRef, - embeddingRefUid: embeddingRefNode?.uid || "", - isFirstChild: getUidAndBooleanSetting({ - tree: tree, - text: "First Child", - }), - }; + setNodeConfigTree(getNodeConfig(parentUid)); }, [parentUid]); const [embeddingRef, setEmbeddingRef] = useState(nodeConfigTree.embeddingRef); @@ -101,7 +109,7 @@ const DiscourseNodeSuggestiveRules = ({ order={0} parentUid={nodeUid} uid={templateUid} - defaultValue={node.template ? [{ text: node.template }] : undefined} + defaultValue={node.template} />
diff --git a/apps/roam/src/utils/configPageTabs.ts b/apps/roam/src/utils/configPageTabs.ts index 28aebf28a..cb7d8aff1 100644 --- a/apps/roam/src/utils/configPageTabs.ts +++ b/apps/roam/src/utils/configPageTabs.ts @@ -128,7 +128,7 @@ export const configPageTabs = (args: OnloadArgs): ConfigTab[] => [ "Include relations from pages referenced on the current page", }, { - title: "Include Parent and Child Blocks", + title: "Include Parent And Child Blocks", // @ts-ignore Panel: FlagPanel, description: "Include relations from parent and child blocks", diff --git a/apps/roam/src/utils/discourseConfigRef.ts b/apps/roam/src/utils/discourseConfigRef.ts index bc4041bcd..48b2c1a1c 100644 --- a/apps/roam/src/utils/discourseConfigRef.ts +++ b/apps/roam/src/utils/discourseConfigRef.ts @@ -52,7 +52,7 @@ export const getFormattedConfigTree = (): FormattedConfigTree => { tree: configTreeRef.tree, text: "Canvas Page Format", }), - suggestiveMode: getSuggestiveModeConfigAndUids(), + suggestiveMode: getSuggestiveModeConfigAndUids(configTreeRef.tree), }; }; export default configTreeRef; diff --git a/apps/roam/src/utils/getSuggestiveModeConfigSettings.ts b/apps/roam/src/utils/getSuggestiveModeConfigSettings.ts index 16a58b7a0..a405c9a52 100644 --- a/apps/roam/src/utils/getSuggestiveModeConfigSettings.ts +++ b/apps/roam/src/utils/getSuggestiveModeConfigSettings.ts @@ -1,6 +1,6 @@ import { getSubTree } from "roamjs-components/util"; -import configTreeRef from "./discourseConfigRef"; import { BooleanSetting, getUidAndBooleanSetting } from "./getExportSettings"; +import { RoamBasicNode } from "roamjs-components/types"; export type SuggestiveModeConfigWithUids = { parentUid: string; @@ -14,14 +14,14 @@ export type SuggestiveModeConfigWithUids = { }; export const getSuggestiveModeConfigAndUids = - (): SuggestiveModeConfigWithUids => { + (tree: RoamBasicNode[]): SuggestiveModeConfigWithUids => { const suggestiveModeNode = getSubTree({ - tree: configTreeRef.tree, + tree, key: "Suggestive mode", }); const pageGroupsNode = getSubTree({ parentUid: suggestiveModeNode.uid, - key: "page groups", + key: "Page Groups", }); return { @@ -32,11 +32,11 @@ export const getSuggestiveModeConfigAndUids = }), grabParentAndChildren: getUidAndBooleanSetting({ tree: suggestiveModeNode.children, - text: "Include Parent and Child Blocks", + text: "Include Parent And Child Blocks", }), pageGroups: { uid: pageGroupsNode?.uid || "", - name: "page groups", + name: "Page Groups", pages: [], }, }; From 9815656da9d28a57434ae6b7b70537741fafb252 Mon Sep 17 00:00:00 2001 From: sid597 Date: Mon, 11 Aug 2025 12:59:22 +0530 Subject: [PATCH 07/30] unify spelling, handle fallback --- .../settings/DiscourseNodeSuggestiveRules.tsx | 2 +- .../settings/SuggestiveModeSettings.tsx | 2 +- apps/roam/src/utils/configPageTabs.ts | 2 +- .../utils/getSuggestiveModeConfigSettings.ts | 53 ++++++++++--------- 4 files changed, 30 insertions(+), 29 deletions(-) diff --git a/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx index 23e948bcb..ec1eabc0c 100644 --- a/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx +++ b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx @@ -89,7 +89,7 @@ const DiscourseNodeSuggestiveRules = ({ getSubTree({ parentUid: nodeUid, key: "Template", - }).uid, + }).uid || "", [nodeUid], ); diff --git a/apps/roam/src/components/settings/SuggestiveModeSettings.tsx b/apps/roam/src/components/settings/SuggestiveModeSettings.tsx index bb7097035..7d5baa69a 100644 --- a/apps/roam/src/components/settings/SuggestiveModeSettings.tsx +++ b/apps/roam/src/components/settings/SuggestiveModeSettings.tsx @@ -19,7 +19,7 @@ const SuggestiveModeSettings = () => { if (!settings.suggestiveMode.parentUid) { await createBlock({ parentUid: getPageUidByPageTitle(DISCOURSE_CONFIG_PAGE_TITLE), - node: { text: "Suggestive mode" }, + node: { text: "Suggestive Mode" }, }); refreshConfigTree(); setSettings(getFormattedConfigTree()); diff --git a/apps/roam/src/utils/configPageTabs.ts b/apps/roam/src/utils/configPageTabs.ts index cb7d8aff1..1d43df445 100644 --- a/apps/roam/src/utils/configPageTabs.ts +++ b/apps/roam/src/utils/configPageTabs.ts @@ -118,7 +118,7 @@ export const configPageTabs = (args: OnloadArgs): ConfigTab[] => [ ], }, { - id: "Suggestive mode", + id: "Suggestive Mode", fields: [ { title: "Include Current Page Relations", diff --git a/apps/roam/src/utils/getSuggestiveModeConfigSettings.ts b/apps/roam/src/utils/getSuggestiveModeConfigSettings.ts index a405c9a52..982038443 100644 --- a/apps/roam/src/utils/getSuggestiveModeConfigSettings.ts +++ b/apps/roam/src/utils/getSuggestiveModeConfigSettings.ts @@ -13,31 +13,32 @@ export type SuggestiveModeConfigWithUids = { }; }; -export const getSuggestiveModeConfigAndUids = - (tree: RoamBasicNode[]): SuggestiveModeConfigWithUids => { - const suggestiveModeNode = getSubTree({ - tree, - key: "Suggestive mode", - }); - const pageGroupsNode = getSubTree({ - parentUid: suggestiveModeNode.uid, - key: "Page Groups", - }); +export const getSuggestiveModeConfigAndUids = ( + tree: RoamBasicNode[], +): SuggestiveModeConfigWithUids => { + const suggestiveModeNode = getSubTree({ + tree, + key: "Suggestive Mode", + }); + const pageGroupsNode = getSubTree({ + parentUid: suggestiveModeNode.uid, + key: "Page Groups", + }); - return { - parentUid: suggestiveModeNode.uid, - grabFromReferencedPages: getUidAndBooleanSetting({ - tree: suggestiveModeNode.children, - text: "Include Current Page Relations", - }), - grabParentAndChildren: getUidAndBooleanSetting({ - tree: suggestiveModeNode.children, - text: "Include Parent And Child Blocks", - }), - pageGroups: { - uid: pageGroupsNode?.uid || "", - name: "Page Groups", - pages: [], - }, - }; + return { + parentUid: suggestiveModeNode.uid, + grabFromReferencedPages: getUidAndBooleanSetting({ + tree: suggestiveModeNode.children, + text: "Include Current Page Relations", + }), + grabParentAndChildren: getUidAndBooleanSetting({ + tree: suggestiveModeNode.children, + text: "Include Parent And Child Blocks", + }), + pageGroups: { + uid: pageGroupsNode?.uid || "", + name: "Page Groups", + pages: [], + }, }; +}; From 37aa7e01d304ca54803556f2c68d101dcc050c01 Mon Sep 17 00:00:00 2001 From: sid597 Date: Sun, 17 Aug 2025 16:04:26 +0530 Subject: [PATCH 08/30] sync --- apps/roam/src/utils/cleanupOrphanedNodes.ts | 332 +++++++++++++-- .../roam/src/utils/fetchEmbeddingsForNodes.ts | 27 +- .../src/utils/getAllDiscourseNodesSince.ts | 177 ++++++-- apps/roam/src/utils/syncDgNodesToSupabase.ts | 397 ++++++++++++++++++ .../upsertNodesAsContentWithEmbeddings.ts | 168 ++++++++ 5 files changed, 1034 insertions(+), 67 deletions(-) create mode 100644 apps/roam/src/utils/syncDgNodesToSupabase.ts create mode 100644 apps/roam/src/utils/upsertNodesAsContentWithEmbeddings.ts diff --git a/apps/roam/src/utils/cleanupOrphanedNodes.ts b/apps/roam/src/utils/cleanupOrphanedNodes.ts index dfdb78e39..31d6f3418 100644 --- a/apps/roam/src/utils/cleanupOrphanedNodes.ts +++ b/apps/roam/src/utils/cleanupOrphanedNodes.ts @@ -1,28 +1,92 @@ +import { + getSupabaseContext, + getLoggedInClient, + type SupabaseContext, +} from "./supabaseContext"; import { type SupabaseClient } from "@supabase/supabase-js"; -import { getSupabaseContext, getLoggedInClient } from "./supabaseContext"; +import { type Database } from "@repo/database/types.gen"; + +type DGSupabaseClient = SupabaseClient; const getAllNodesFromSupabase = async ( - spaceId: number, - supabaseClient: SupabaseClient, + supabaseClient: DGSupabaseClient, + context: SupabaseContext, ): Promise => { try { - const allNodeInstanceIds = await supabaseClient + if (!context) { + console.error("Failed to get Supabase context"); + return []; + } + const { spaceId } = context; + + const { data: schemas, error: schemasError } = await supabaseClient .from("Concept") + .select("id") + .eq("space_id", spaceId) + .eq("is_schema", true) + .eq("arity", 0); + + if (schemasError) { + console.error( + "Failed to get all discourse node schemas from Supabase:", + schemasError, + ); + return []; + } + + const schemaIds = schemas.map((s) => s.id); + let nodeResult: string[] = []; + + if (schemaIds.length > 0) { + const conceptResponse = await supabaseClient + .from("Concept") + .select( + ` + Content!inner ( + source_local_id + ) + `, + ) + .eq("space_id", spaceId) + .eq("is_schema", false) + .in("schema_id", schemaIds) + .not("Content.source_local_id", "is", null); + + if (conceptResponse.error) { + console.error( + "Failed to get concepts from Supabase:", + conceptResponse.error, + ); + return []; + } + nodeResult = + conceptResponse.data + ?.map((c) => c.Content?.source_local_id) + .filter((id): id is string => !!id) || []; + } + + const blockContentResponse = await supabaseClient + .from("Content") .select("source_local_id") - .not("schema_id", "is", null); + .eq("space_id", spaceId) + .eq("scale", "block") + .not("source_local_id", "is", null); - if (allNodeInstanceIds.error) { + if (blockContentResponse.error) { console.error( - "Failed to get all discourse node instances from Supabase:", - allNodeInstanceIds.error, + "Failed to get block content from Supabase:", + blockContentResponse.error, ); return []; } - const result = - allNodeInstanceIds.data + + const blockResult = + blockContentResponse.data ?.map((c) => c.source_local_id) .filter((id): id is string => !!id) || []; + const result = [...new Set([...nodeResult, ...blockResult])]; + return result; } catch (error) { console.error("Error in getAllNodesFromSupabase:", error); @@ -30,6 +94,49 @@ const getAllNodesFromSupabase = async ( } }; +const getAllNodeSchemasFromSupabase = async ( + supabaseClient: DGSupabaseClient, + context: SupabaseContext, +): Promise => { + try { + if (!context) { + console.error("Failed to get Supabase context"); + return []; + } + + const { data, error } = await supabaseClient + .from("Concept") + .select( + ` + Content!inner ( + source_local_id + ) + `, + ) + .eq("space_id", context.spaceId) + .eq("is_schema", true) + .eq("arity", 0) + .not("Content.source_local_id", "is", null); + + if (error) { + console.error( + "Failed to get all discourse node schemas from Supabase:", + error, + ); + return []; + } + + return ( + data + ?.map((c) => c.Content?.source_local_id) + .filter((id): id is string => !!id) || [] + ); + } catch (error) { + console.error("Error in getAllNodeSchemasFromSupabase:", error); + return []; + } +}; + const getNonExistentRoamUids = (nodeUids: string[]): string[] => { try { if (nodeUids.length === 0) { @@ -53,7 +160,7 @@ const getNonExistentRoamUids = (nodeUids: string[]): string[] => { const deleteNodesFromSupabase = async ( uids: string[], spaceId: number, - supabaseClient: SupabaseClient, + supabaseClient: DGSupabaseClient, ): Promise => { try { const { data: contentData, error: contentError } = await supabaseClient @@ -96,25 +203,200 @@ const deleteNodesFromSupabase = async ( } }; -export const cleanupOrphanedNodes = async (): Promise => { - const context = await getSupabaseContext(); - if (!context) { - console.error("Failed to get Supabase context"); - return; +const deleteNodeSchemasFromSupabase = async ( + uids: string[], +): Promise => { + try { + const context = await getSupabaseContext(); + if (!context) { + console.error("Failed to get Supabase context"); + return 0; + } + if (uids.length === 0) return 0; + + const supabaseClient = await getLoggedInClient(); + const { spaceId } = context; + + const { data: schemaContentData, error: contentLookupError } = + await supabaseClient + .from("Content") + .select("id, source_local_id") + .eq("space_id", spaceId) + .in("source_local_id", uids); + + if (contentLookupError) { + console.error( + "deleteNodeSchemasFromSupabase: content lookup failed:", + contentLookupError, + ); + return 0; + } + + if (!schemaContentData || schemaContentData.length === 0) { + return 0; + } + + const schemaContentIds = schemaContentData.map((c) => c.id); + + const { data: schemaConceptData, error: schemaConceptError } = + await supabaseClient + .from("Concept") + .select("id") + .eq("space_id", spaceId) + .eq("is_schema", true) + .in("represented_by_id", schemaContentIds); + + if (schemaConceptError) { + console.error( + "deleteNodeSchemasFromSupabase: schema concept lookup failed:", + schemaConceptError, + ); + return 0; + } + + const schemaConceptIds = (schemaConceptData || []).map((c) => c.id); + + let instanceConceptIds: number[] = []; + let instanceContentIds: number[] = []; + let instanceSourceLocalIds: string[] = []; + + if (schemaConceptIds.length > 0) { + const { data: instanceConceptData, error: instanceConceptError } = + await supabaseClient + .from("Concept") + .select("id, represented_by_id") + .eq("space_id", spaceId) + .eq("is_schema", false) + .in("schema_id", schemaConceptIds); + + if (instanceConceptError) { + console.error( + "deleteNodeSchemasFromSupabase: instance concept lookup failed:", + instanceConceptError, + ); + return 0; + } + + instanceConceptIds = (instanceConceptData || []).map((ic) => ic.id); + instanceContentIds = (instanceConceptData || []) + .map((ic) => ic.represented_by_id) + .filter((x): x is number => typeof x === "number"); + + if (instanceContentIds.length > 0) { + const { data: instanceContentData, error: instanceContentLookupError } = + await supabaseClient + .from("Content") + .select("source_local_id") + .in("id", instanceContentIds); + + if (instanceContentLookupError) { + console.error( + "deleteNodeSchemasFromSupabase: instance content lookup failed:", + instanceContentLookupError, + ); + return 0; + } + instanceSourceLocalIds = (instanceContentData || []) + .map((c) => c.source_local_id) + .filter((id): id is string => !!id); + } + } + + if (instanceConceptIds.length > 0) { + const { error: deleteInstanceConceptError } = await supabaseClient + .from("Concept") + .delete() + .in("id", instanceConceptIds); + if (deleteInstanceConceptError) { + console.error( + "deleteNodeSchemasFromSupabase: delete instance concepts failed:", + deleteInstanceConceptError, + ); + return 0; + } + } + + if (schemaConceptIds.length > 0) { + const { error: deleteSchemaConceptError } = await supabaseClient + .from("Concept") + .delete() + .in("id", schemaConceptIds); + if (deleteSchemaConceptError) { + console.error( + "deleteNodeSchemasFromSupabase: delete schema concepts failed:", + deleteSchemaConceptError, + ); + return 0; + } + } + + const allContentIds = [...schemaContentIds, ...instanceContentIds]; + if (allContentIds.length > 0) { + const { error: deleteContentError } = await supabaseClient + .from("Content") + .delete() + .in("id", allContentIds); + if (deleteContentError) { + console.error( + "deleteNodeSchemasFromSupabase: delete content failed:", + deleteContentError, + ); + return 0; + } + } + + const docLocalIds = [...uids, ...instanceSourceLocalIds]; + let deletedDocsCount = 0; + if (docLocalIds.length > 0) { + const { error: docError, count } = await supabaseClient + .from("Document") + .delete({ count: "exact" }) + .eq("space_id", spaceId) + .in("source_local_id", docLocalIds); + if (docError) { + console.error( + "deleteNodeSchemasFromSupabase: delete documents failed:", + docError, + ); + return 0; + } + deletedDocsCount = count ?? 0; + } + + return deletedDocsCount; + } catch (error) { + console.error("Error in deleteNodeSchemasFromSupabase:", error); + return 0; } - const spaceId = context.spaceId; +}; - const supabaseClient = await getLoggedInClient(); +export const cleanupOrphanedNodes = async ( + supabaseClient: DGSupabaseClient, + context: SupabaseContext, +): Promise => { try { - const supabaseUids = await getAllNodesFromSupabase(spaceId, supabaseClient); - if (supabaseUids.length === 0) { - return; + const supabaseUids = await getAllNodesFromSupabase(supabaseClient, context); + if (supabaseUids.length > 0) { + const orphanedUids = getNonExistentRoamUids(supabaseUids); + if (orphanedUids.length > 0) { + await deleteNodesFromSupabase( + orphanedUids, + context.spaceId, + supabaseClient, + ); + } } - const orphanedUids = getNonExistentRoamUids(supabaseUids); - if (orphanedUids.length === 0) { - return; + + const supabaseSchemaUids = await getAllNodeSchemasFromSupabase( + supabaseClient, + context, + ); + if (supabaseSchemaUids.length > 0) { + const orphanedSchemaUids = getNonExistentRoamUids(supabaseSchemaUids); + if (orphanedSchemaUids.length > 0) { + await deleteNodeSchemasFromSupabase(orphanedSchemaUids); + } } - await deleteNodesFromSupabase(orphanedUids, spaceId, supabaseClient); } catch (error) { console.error("Error in cleanupOrphanedNodes:", error); } diff --git a/apps/roam/src/utils/fetchEmbeddingsForNodes.ts b/apps/roam/src/utils/fetchEmbeddingsForNodes.ts index 3d01e78a7..bff740f8f 100644 --- a/apps/roam/src/utils/fetchEmbeddingsForNodes.ts +++ b/apps/roam/src/utils/fetchEmbeddingsForNodes.ts @@ -1,16 +1,8 @@ -type DiscourseGraphContent = { - author_local_id: string; - source_local_id: string; - scale: string; - created: string; - last_modified: string; - text: string; - model: string; - vector: number[]; -}; +import { RoamDiscourseNodeData } from "./getAllDiscourseNodesSince"; -const EMBEDDING_BATCH_SIZE = 100; +const EMBEDDING_BATCH_SIZE = 200; const API_URL = `https://discoursegraphs.com/api/embeddings/openai/small`; +const EMBEDDING_MODEL = "openai_text_embedding_3_small_1536"; type EmbeddingApiResponse = { data: { @@ -19,13 +11,18 @@ type EmbeddingApiResponse = { }; export const fetchEmbeddingsForNodes = async ( - nodes: DiscourseGraphContent[], -): Promise => { + nodes: RoamDiscourseNodeData[], +): Promise => { const allEmbeddings: number[][] = []; - const allNodesTexts = nodes.map((node) => node.text); + const allNodesTexts = nodes.map((node) => + node.node_title ? `${node.node_title} ${node.text}` : node.text, + ); for (let i = 0; i < allNodesTexts.length; i += EMBEDDING_BATCH_SIZE) { const batch = allNodesTexts.slice(i, i + EMBEDDING_BATCH_SIZE); + console.log( + `fetchEmbeddingsForNodes: Fetching batch ${i / EMBEDDING_BATCH_SIZE + 1} of ${allNodesTexts.length / EMBEDDING_BATCH_SIZE}`, + ); const response = await fetch(API_URL, { method: "POST", @@ -67,7 +64,7 @@ export const fetchEmbeddingsForNodes = async ( } return nodes.map((node, i) => ({ ...node, - model: "openai_text_embedding_3_small_1536", + model: EMBEDDING_MODEL, vector: allEmbeddings[i], })); }; diff --git a/apps/roam/src/utils/getAllDiscourseNodesSince.ts b/apps/roam/src/utils/getAllDiscourseNodesSince.ts index b474f224e..115efc850 100644 --- a/apps/roam/src/utils/getAllDiscourseNodesSince.ts +++ b/apps/roam/src/utils/getAllDiscourseNodesSince.ts @@ -1,48 +1,171 @@ -import getDiscourseNodes from "./getDiscourseNodes"; +import getDiscourseNodes, { DiscourseNode } from "./getDiscourseNodes"; import findDiscourseNode from "./findDiscourseNode"; +import { OnloadArgs } from "roamjs-components/types"; +import getDiscourseNodeFormatExpression from "./getDiscourseNodeFormatExpression"; -type RoamDiscourseNodeData = { +type ISODateString = string; + +export type RoamDiscourseNodeData = { author_local_id: string; + author_name: string; source_local_id: string; created: string; + vector: number[]; last_modified: string; - author_name: string; text: string; + type: string; + node_title?: string; }; -type ISODateString = string; +export type DiscourseNodesSinceResult = { + pageNodes: RoamDiscourseNodeData[]; + blockNodes: RoamDiscourseNodeData[]; +}; + +export const getDiscourseNodeTypeWithSettingsBlockNodes = ( + node: DiscourseNode, + sinceMs: number, + extensionAPI: OnloadArgs["extensionAPI"], +): RoamDiscourseNodeData[] => { + const settingsKey = `discourse-graph-node-rule-${node.type}`; + const settings = extensionAPI.settings.get(settingsKey) as { + embeddingRef: string; + isFirstChild: boolean; + }; + const regex = getDiscourseNodeFormatExpression(node.format); + const regexPattern = regex.source.replace(/\\/g, "\\\\").replace(/"/g, '\\"'); + const firstChildUid = + settings.embeddingRef?.match(/\(\((.*?)\)\)/)?.[1] ?? ""; + const queryBlock = `[ + :find ?childString ?nodeUid ?nodeCreateTime ?nodeEditTime ?author_local_id ?type ?author_name ?node-title + :keys text source_local_id created last_modified author_local_id type author_name node_title + :in $ ?firstChildUid ?type ?since + :where + [(re-pattern "${regexPattern}") ?title-regex] + [?node :node/title ?node-title] + [(re-find ?title-regex ?node-title)] + [?node :block/uid ?nodeUid] + [?node :create/time ?nodeCreateTime] + [?node :edit/time ?nodeEditTime] + [?s :block/uid ?firstChildUid] + [?s :block/string ?firstChildString] + [?bg :block/page ?node] + [?bg :block/string ?firstChildString] + [?bg :block/children ?child] + [?child :block/order 0] + [?child :block/string ?childString] + [?child :edit/time ?childEditTime] + [?child :create/user ?user-eid] + [?user-eid :user/uid ?author_local_id] + [?child :edit/user ?eu] + [?eu :user/display-name ?author_name] + [or + [(> ?childEditTime ?since)] + [(> ?nodeEditTime ?since)]] + ]`; + + const blockNode = window.roamAlphaAPI.data.q( + queryBlock, + String(firstChildUid), + String(node.type), + sinceMs, + ) as unknown as Omit[]; + return blockNode.map((node) => ({ ...node, vector: [] })); +}; export const getAllDiscourseNodesSince = async ( since: ISODateString, + nodeTypes: DiscourseNode[], + extensionAPI: OnloadArgs["extensionAPI"], ): Promise => { const sinceMs = new Date(since).getTime(); + const result: RoamDiscourseNodeData[] = []; + + if (nodeTypes.length > 0) { + for (const node of nodeTypes) { + const blockNode = getDiscourseNodeTypeWithSettingsBlockNodes( + node, + sinceMs, + extensionAPI, + ); + if (blockNode) { + result.push(...blockNode); + } + } + } - const query = `[:find ?uid ?create-time ?edit-time ?user-uuid ?username ?title - :keys source_local_id created last_modified author_local_id author_name text - :in $ ?since - :where - [?e :node/title ?title] - [?e :block/uid ?uid] - [?e :create/user ?user-id] - [?user-id :user/uid ?user-uuid] - [?user-id :user/display-name ?username] - [?e :create/time ?create-time] - [?e :edit/time ?edit-time] - [(> ?edit-time ?since)]]`; - - // @ts-ignore - backend to be added to roamjs-components - const result = (await window.roamAlphaAPI.data.backend.q( + const query = `[ + :find ?node-title ?uid ?nodeCreateTime ?nodeEditTime ?author_local_id ?author_name + :keys text source_local_id created last_modified author_local_id author_name + :in $ ?since + :where + [?node :node/title ?node-title] + [?node :block/uid ?uid] + [?node :create/time ?nodeCreateTime] + [?node :edit/time ?nodeEditTime] + [?node :create/user ?user-eid] + [?user-eid :user/uid ?author_local_id] + [?node :edit/user ?eu] + [(get-else $ ?eu :user/display-name "Unknown-person") ?author_name] + [(> ?nodeEditTime ?since)] +]`; + + //@ts-ignore - backend to be added to roamjs-components + const allNodes = (await window.roamAlphaAPI.data.backend.q( query, sinceMs, - )) as unknown[][] as RoamDiscourseNodeData[]; + )) as unknown as RoamDiscourseNodeData[]; const discourseNodes = getDiscourseNodes(); + const nodeTypesSet = new Set(nodeTypes.map((nodeType) => nodeType.type)); + + result.push( + ...allNodes + .map((entity) => { + if (!entity.source_local_id) { + return null; + } + const node = findDiscourseNode(entity.source_local_id, discourseNodes); + if ( + !node || + node.backedBy === "default" || + !entity.text || + entity.text.trim() === "" || + nodeTypesSet.has(node.type) + ) { + return null; + } + return { + ...entity, + type: node.type, + }; + }) + .filter((n): n is RoamDiscourseNodeData => n !== null), + ); + return result; +}; + +export const nodeTypeSince = async ( + since: ISODateString, + nodeTypes: DiscourseNode[], +) => { + const sinceMs = new Date(since).getTime(); + const filterMap = await Promise.all( + nodeTypes.map((node) => { + const query = ` + [:find ?node-title + :in $ ?since ?type + :where + [?node :block/uid ?type] + [?node :node/title ?node-title] + [?node :edit/time ?nodeEditTime] + [(> ?nodeEditTime ?since)]] + `; + const result = window.roamAlphaAPI.data.q(query, sinceMs, node.type); + return result.length > 0; + }), + ); - return result.filter((entity) => { - if (!entity.source_local_id) return false; - const node = findDiscourseNode(entity.source_local_id, discourseNodes); - if (!node) return false; - if (node.backedBy === "default") return false; - return Boolean(entity.text && entity.text.trim() !== ""); - }); + const nodesSince = nodeTypes.filter((_, index) => filterMap[index]); + return nodesSince; }; diff --git a/apps/roam/src/utils/syncDgNodesToSupabase.ts b/apps/roam/src/utils/syncDgNodesToSupabase.ts new file mode 100644 index 000000000..b45c43cef --- /dev/null +++ b/apps/roam/src/utils/syncDgNodesToSupabase.ts @@ -0,0 +1,397 @@ +import { + getAllDiscourseNodesSince, + nodeTypeSince, +} from "./getAllDiscourseNodesSince"; +import { cleanupOrphanedNodes } from "./cleanupOrphanedNodes"; +import { + getLoggedInClient, + getSupabaseContext, + SupabaseContext, +} from "./supabaseContext"; +import { fetchEmbeddingsForNodes } from "./fetchEmbeddingsForNodes"; +import { LocalContentDataInput } from "@repo/database/inputTypes"; +import { RoamDiscourseNodeData } from "./getAllDiscourseNodesSince"; +import getDiscourseRelations from "./getDiscourseRelations"; +import getDiscourseNodes, { DiscourseNode } from "./getDiscourseNodes"; +import { + discourseNodeBlockToLocalConcept, + discourseNodeSchemaToLocalConcept, + orderConceptsByDependency, + discourseRelationSchemaToLocalConcept, + discourseRelationDataToLocalConcept, +} from "./conceptConversion"; +import getDiscourseRelationTriples from "./getDiscourseRelationTriples"; +import { OnloadArgs } from "roamjs-components/types"; +import { DGSupabaseClient } from "@repo/ui/lib/supabase/client"; + +const SYNC_FUNCTION = "embedding"; +const SYNC_INTERVAL = "45s"; +const SYNC_TIMEOUT = "20s"; +const BATCH_SIZE = 200; +const DEFAULT_TIME = "1970-01-01"; +const EMBEDDING_MODEL = "openai_text_embedding_3_small_1536"; + +type SyncTaskInfo = { + lastUpdateTime: string | null; + spaceId: number; + worker: string; + shouldProceed: boolean; +}; + +export const endSyncTask = async ( + worker: string, + status: "complete" | "failed", +): Promise => { + try { + const supabaseClient = await getLoggedInClient(); + const context = await getSupabaseContext(); + if (!context) { + console.error("endSyncTask: Unable to obtain Supabase context."); + return; + } + const { error } = await supabaseClient.rpc("end_sync_task", { + s_target: context.spaceId, + s_function: "embedding", + s_worker: worker, + s_status: status, + }); + if (error) { + console.error("endSyncTask: Error calling end_sync_task:", error); + } + } catch (error) { + console.error("endSyncTask: Error calling end_sync_task:", error); + } +}; + +export const proposeSyncTask = async (): Promise => { + try { + const supabaseClient = await getLoggedInClient(); + const context = await getSupabaseContext(); + if (!context) { + console.error("proposeSyncTask: Unable to obtain Supabase context."); + return { + lastUpdateTime: null, + spaceId: 0, + worker: "", + shouldProceed: false, + }; + } + const worker = window.roamAlphaAPI.user.uid(); + + const { data, error } = await supabaseClient.rpc("propose_sync_task", { + s_target: context.spaceId, + s_function: SYNC_FUNCTION, + s_worker: worker, + task_interval: SYNC_INTERVAL, + timeout: SYNC_TIMEOUT, + }); + + const { spaceId } = context; + + if (error) { + console.error( + `proposeSyncTask: propose_sync_task failed – ${error.message}`, + ); + return { lastUpdateTime: null, spaceId, worker, shouldProceed: false }; + } + + if (typeof data === "string") { + const timestamp = new Date(data); + const now = new Date(); + + if (timestamp > now) { + console.log( + "proposeSyncTask: Another worker is already running this task", + ); + return { lastUpdateTime: null, spaceId, worker, shouldProceed: false }; + } else { + return { lastUpdateTime: data, spaceId, worker, shouldProceed: true }; + } + } + + return { lastUpdateTime: null, spaceId, worker, shouldProceed: true }; + } catch (error) { + console.error( + `proposeSyncTask: Unexpected error while contacting sync-task API:`, + error, + ); + return { + lastUpdateTime: null, + spaceId: 0, + worker: "", + shouldProceed: false, + }; + } +}; + +const upsertNodeSchemaToContent = async ( + nodeTypesUids: string[], + spaceId: number, + userId: number, + supabaseClient: DGSupabaseClient, +) => { + const query = `[ + :find ?uid ?create-time ?edit-time ?user-uuid ?title ?author-name + :keys source_local_id created last_modified author_local_id text author_name + :in $ [?uid ...] + :where + [?e :block/uid ?uid] + [?e :node/title ?title] + [?e :create/user ?user-eid] + [?user-eid :user/uid ?user-uuid] + [?e :create/time ?create-time] + [?e :edit/time ?edit-time] + [?e :edit/user ?eu] + [(get-else $ ?eu :user/display-name "Unknown-person") ?author-name] + + ] + `; + //@ts-ignore - backend to be added to roamjs-components + const result = await window.roamAlphaAPI.data.backend.q( + query, + nodeTypesUids, + ) as unknown as RoamDiscourseNodeData[]; + + const contentData: LocalContentDataInput[] = result.map((node) => ({ + author_id: userId, + account_local_id: node.author_local_id, + source_local_id: node.source_local_id, + created: new Date(node.created || Date.now()).toISOString(), + last_modified: new Date(node.last_modified || Date.now()).toISOString(), + text: node.text, + embedding_inline: { + model: EMBEDDING_MODEL, + vector: node.vector, + }, + scale: "document", + })); + const { error } = await supabaseClient.rpc("upsert_content", { + data: contentData as any, + v_space_id: spaceId, + v_creator_id: userId, + content_as_document: true, + }); + if (error) { + console.error("upsert_content failed:", error); + } +}; + +export const convertDgToSupabaseConcepts = async ( + nodesSince: RoamDiscourseNodeData[], + since: string, + allNodeTypes: DiscourseNode[], + supabaseClient: DGSupabaseClient, + context: SupabaseContext, +) => { + const nodeTypes = await nodeTypeSince(since, allNodeTypes); + await upsertNodeSchemaToContent( + nodeTypes.map((node) => node.type), + context.spaceId, + context.userId, + supabaseClient, + ); + + const nodesTypesToLocalConcepts = nodeTypes.map((node) => { + return discourseNodeSchemaToLocalConcept(context, node); + }); + + const relationSchemas = getDiscourseRelations(); + + const relationsToEmbed = relationSchemas.map((relation) => { + const localConcept = discourseRelationSchemaToLocalConcept( + context, + relation, + ); + return localConcept; + }); + + const nodeBlockToLocalConcepts = nodesSince.map((node) => { + const localConcept = discourseNodeBlockToLocalConcept(context, { + nodeUid: node.source_local_id, + schemaUid: node.type, + text: node.node_title ? `${node.node_title} ${node.text}` : node.text, + }); + return localConcept; + }); + + const relationTriples = getDiscourseRelationTriples(); + const relationLabelToId = Object.fromEntries( + relationSchemas.map((r) => [r.label, r.id]), + ); + const relationBlockToLocalConcepts = relationTriples + .map(({ relation, source, target }) => { + const relationSchemaUid = relationLabelToId[relation]; + if (!relationSchemaUid) { + return null; + } + return discourseRelationDataToLocalConcept(context, relationSchemaUid, { + source, + target, + }); + }) + .filter((x): x is NonNullable => x !== null); + + const conceptsToUpsert = [ + ...nodesTypesToLocalConcepts, + ...relationsToEmbed, + ...nodeBlockToLocalConcepts, + ...relationBlockToLocalConcepts, + ]; + const { ordered } = orderConceptsByDependency(conceptsToUpsert); + const { error } = await supabaseClient.rpc("upsert_concepts", { + data: ordered, + v_space_id: context.spaceId, + }); + if (error) { + throw new Error( + `upsert_concepts failed: ${JSON.stringify(error, null, 2)}`, + ); + } +}; + +export const upsertNodesToSupabaseAsContentWithEmbeddings = async ( + roamNodes: RoamDiscourseNodeData[], + supabaseClient: DGSupabaseClient, + context: SupabaseContext, +): Promise => { + const { spaceId, userId } = context; + + if (roamNodes.length === 0) { + return; + } + + let nodesWithEmbeddings: RoamDiscourseNodeData[]; + try { + nodesWithEmbeddings = await fetchEmbeddingsForNodes(roamNodes); + } catch (error: any) { + console.error( + `upsertNodesToSupabaseAsContentWithEmbeddings: Embedding service failed – ${error.message}`, + ); + return; + } + + if (nodesWithEmbeddings.length !== roamNodes.length) { + console.error( + "upsertNodesToSupabaseAsContentWithEmbeddings: Mismatch between node and embedding counts.", + ); + return; + } + + const chunk = (array: T[], size: number): T[][] => { + const chunks: T[][] = []; + for (let i = 0; i < array.length; i += size) { + chunks.push(array.slice(i, i + size)); + } + return chunks; + }; + + const uploadBatches = async (batches: RoamDiscourseNodeData[][]) => { + for (let idx = 0; idx < batches.length; idx++) { + const batch = batches[idx]; + + const contents: LocalContentDataInput[] = batch.map((node) => { + const variant = node.node_title ? "direct_and_description" : "direct"; + const text = node.node_title + ? `${node.node_title} ${node.text}` + : node.text; + + return { + author_id: userId, + account_local_id: node.author_local_id, + source_local_id: node.source_local_id, + created: new Date(node.created || Date.now()).toISOString(), + last_modified: new Date( + node.last_modified || Date.now(), + ).toISOString(), + text: text, + variant: variant, + embedding_inline: { + model: EMBEDDING_MODEL, + vector: node.vector, + }, + scale: "document", + }; + }); + + const { error } = await supabaseClient.rpc("upsert_content", { + data: contents as any, + v_space_id: spaceId, + v_creator_id: userId, + content_as_document: true, + }); + + if (error) { + console.error(`upsert_content failed for batch ${idx + 1}:`, error); + throw error; + } + } + }; + + await uploadBatches(chunk(nodesWithEmbeddings, BATCH_SIZE)); +}; + +const getDgNodeTypes = (extensionAPI: OnloadArgs["extensionAPI"]) => { + const allDgNodeTypes = getDiscourseNodes().filter( + (n) => n.backedBy === "user", + ); + const dgNodeTypesWithSettings = allDgNodeTypes.filter((n) => { + const settingsKey = `discourse-graph-node-rule-${n.type}`; + const settings = extensionAPI.settings.get(settingsKey) as + | { + isFirstChild?: boolean; + embeddingRef?: string; + } + | undefined; + return settings?.isFirstChild || settings?.embeddingRef; + }); + return { allDgNodeTypes, dgNodeTypesWithSettings }; +}; + +export const createOrUpdateDiscourseEmbedding = async ( + extensionAPI: OnloadArgs["extensionAPI"], +) => { + const { shouldProceed, lastUpdateTime, worker } = await proposeSyncTask(); + + if (!shouldProceed) { + console.log( + "createOrUpdateDiscourseEmbedding: Task already running or failed to acquire lock. Exiting.", + ); + return; + } + + try { + const time = lastUpdateTime === null ? DEFAULT_TIME : lastUpdateTime; + const { allDgNodeTypes, dgNodeTypesWithSettings } = + getDgNodeTypes(extensionAPI); + + const allNodeInstances = await getAllDiscourseNodesSince( + time, + dgNodeTypesWithSettings, + extensionAPI, + ); + const supabaseClient = await getLoggedInClient(); + const context = await getSupabaseContext(); + if (!context) { + console.error("No Supabase context found."); + return; + } + await upsertNodesToSupabaseAsContentWithEmbeddings( + allNodeInstances, + supabaseClient, + context, + ); + await convertDgToSupabaseConcepts( + allNodeInstances, + time, + allDgNodeTypes, + supabaseClient, + context, + ); + await cleanupOrphanedNodes(supabaseClient, context); + await endSyncTask(worker, "complete"); + } catch (error) { + console.error("createOrUpdateDiscourseEmbedding: Process failed:", error); + await endSyncTask(worker, "failed"); + throw error; + } +}; diff --git a/apps/roam/src/utils/upsertNodesAsContentWithEmbeddings.ts b/apps/roam/src/utils/upsertNodesAsContentWithEmbeddings.ts new file mode 100644 index 000000000..fb929c086 --- /dev/null +++ b/apps/roam/src/utils/upsertNodesAsContentWithEmbeddings.ts @@ -0,0 +1,168 @@ +import { RoamDiscourseNodeData } from "./getAllDiscourseNodesSince"; +import { SupabaseContext } from "./supabaseContext"; +import { LocalContentDataInput } from "@repo/database/inputTypes"; +import { DGSupabaseClient } from "@repo/ui/lib/supabase/client"; + + +const EMBEDDING_BATCH_SIZE = 200; +const API_URL = `https://discoursegraphs.com/api/embeddings/openai/small`; + +type EmbeddingApiResponse = { + data: { + embedding: number[]; + }[]; +}; + +export const fetchEmbeddingsForNodes = async ( + nodes: RoamDiscourseNodeData[], +): Promise => { + const allEmbeddings: number[][] = []; + console.log("nodes", nodes); + const allNodesTexts = nodes.map((node) => + node.node_title ? `${node.node_title} ${node.text}` : node.text, + ); + + for (let i = 0; i < allNodesTexts.length; i += EMBEDDING_BATCH_SIZE) { + const batch = allNodesTexts.slice(i, i + EMBEDDING_BATCH_SIZE); + console.log( + `fetchEmbeddingsForNodes: Fetching batch ${i / EMBEDDING_BATCH_SIZE + 1} of ${allNodesTexts.length / EMBEDDING_BATCH_SIZE}`, + ); + + const response = await fetch(API_URL, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ input: batch }), + }); + + if (!response.ok) { + let errorData; + try { + errorData = await response.json(); + } catch (e) { + errorData = { + error: `Server responded with ${response.status}: ${await response.text()}`, + }; + } + throw new Error( + `API Error (${response.status}) processing batch ${ + i / EMBEDDING_BATCH_SIZE + 1 + }: ${errorData.error || "Failed to fetch embeddings"}`, + ); + } + + const data: EmbeddingApiResponse = await response.json(); + if (!data || !Array.isArray(data.data)) { + throw new Error( + `Invalid API response format for batch ${ + i / EMBEDDING_BATCH_SIZE + 1 + }. Expected 'data' array.`, + ); + } + const batchEmbeddings = data.data.map((item) => item.embedding); + allEmbeddings.push(...batchEmbeddings); + } + if (nodes.length !== allEmbeddings.length) { + throw new Error( + `Mismatch between nodes (${nodes.length}) and embeddings (${allEmbeddings.length})`, + ); + } + return nodes.map((node, i) => ({ + ...node, + model: "openai_text_embedding_3_small_1536", + vector: allEmbeddings[i], + })); +}; + +const uploadBatches = async ( + batches: RoamDiscourseNodeData[][], + supabaseClient: DGSupabaseClient, + context: SupabaseContext, +) => { + const { spaceId, userId } = context; + for (let idx = 0; idx < batches.length; idx++) { + const batch = batches[idx]; + + const contents: LocalContentDataInput[] = batch.map((node) => { + const variant = node.node_title ? "direct_and_description" : "direct"; + const text = node.node_title + ? `${node.node_title} ${node.text}` + : node.text; + + return { + author_id: userId, + account_local_id: node.author_local_id, + source_local_id: node.source_local_id, + created: new Date(node.created || Date.now()).toISOString(), + last_modified: new Date(node.last_modified || Date.now()).toISOString(), + text: text, + variant: variant, + embedding_inline: { + model: "openai_text_embedding_3_small_1536", + vector: node.vector, + }, + scale: "document", + }; + }); + + const { error } = await supabaseClient.rpc("upsert_content", { + data: contents as any, + v_space_id: spaceId, + v_creator_id: userId, + content_as_document: true, + }); + + if (error) { + console.error(`upsert_content failed for batch ${idx + 1}:`, error); + throw error; + } + } +}; + +export const upsertNodesToSupabaseAsContentWithEmbeddings = async ( + roamNodes: RoamDiscourseNodeData[], + supabaseClient: DGSupabaseClient, + context: SupabaseContext, +): Promise => { + if (!context) { + console.error("No Supabase context found."); + return; + } + const { spaceId, userId } = context; + + if (roamNodes.length === 0) { + return; + } + + let nodesWithEmbeddings: RoamDiscourseNodeData[]; + try { + nodesWithEmbeddings = await fetchEmbeddingsForNodes(roamNodes); + } catch (error: any) { + console.error( + `upsertNodesToSupabaseAsContentWithEmbeddings: Embedding service failed – ${error.message}`, + ); + return; + } + + if (nodesWithEmbeddings.length !== roamNodes.length) { + console.error( + "upsertNodesToSupabaseAsContentWithEmbeddings: Mismatch between node and embedding counts.", + ); + return; + } + + const batchSize = 200; + + const chunk = (array: T[], size: number): T[][] => { + const chunks: T[][] = []; + for (let i = 0; i < array.length; i += size) { + chunks.push(array.slice(i, i + size)); + } + return chunks; + }; + + await uploadBatches( + chunk(nodesWithEmbeddings, batchSize), + supabaseClient, + context, + ); +}; From 4f2807b8a7ef4b9e969b12c3b2c419dfe8bea8b5 Mon Sep 17 00:00:00 2001 From: sid597 Date: Sun, 17 Aug 2025 22:46:04 +0530 Subject: [PATCH 09/30] address coderabbit code --- apps/roam/src/utils/cleanupOrphanedNodes.ts | 45 +++++++------------ .../roam/src/utils/fetchEmbeddingsForNodes.ts | 2 +- apps/roam/src/utils/syncDgNodesToSupabase.ts | 7 +-- 3 files changed, 21 insertions(+), 33 deletions(-) diff --git a/apps/roam/src/utils/cleanupOrphanedNodes.ts b/apps/roam/src/utils/cleanupOrphanedNodes.ts index 31d6f3418..9bb850f26 100644 --- a/apps/roam/src/utils/cleanupOrphanedNodes.ts +++ b/apps/roam/src/utils/cleanupOrphanedNodes.ts @@ -1,8 +1,4 @@ -import { - getSupabaseContext, - getLoggedInClient, - type SupabaseContext, -} from "./supabaseContext"; +import { type SupabaseContext } from "./supabaseContext"; import { type SupabaseClient } from "@supabase/supabase-js"; import { type Database } from "@repo/database/types.gen"; @@ -10,14 +6,9 @@ type DGSupabaseClient = SupabaseClient; const getAllNodesFromSupabase = async ( supabaseClient: DGSupabaseClient, - context: SupabaseContext, + spaceId: number, ): Promise => { try { - if (!context) { - console.error("Failed to get Supabase context"); - return []; - } - const { spaceId } = context; const { data: schemas, error: schemasError } = await supabaseClient .from("Concept") @@ -96,14 +87,9 @@ const getAllNodesFromSupabase = async ( const getAllNodeSchemasFromSupabase = async ( supabaseClient: DGSupabaseClient, - context: SupabaseContext, + spaceId: number, ): Promise => { try { - if (!context) { - console.error("Failed to get Supabase context"); - return []; - } - const { data, error } = await supabaseClient .from("Concept") .select( @@ -113,7 +99,7 @@ const getAllNodeSchemasFromSupabase = async ( ) `, ) - .eq("space_id", context.spaceId) + .eq("space_id", spaceId) .eq("is_schema", true) .eq("arity", 0) .not("Content.source_local_id", "is", null); @@ -205,18 +191,12 @@ const deleteNodesFromSupabase = async ( const deleteNodeSchemasFromSupabase = async ( uids: string[], + supabaseClient: DGSupabaseClient, + spaceId: number, ): Promise => { try { - const context = await getSupabaseContext(); - if (!context) { - console.error("Failed to get Supabase context"); - return 0; - } if (uids.length === 0) return 0; - const supabaseClient = await getLoggedInClient(); - const { spaceId } = context; - const { data: schemaContentData, error: contentLookupError } = await supabaseClient .from("Content") @@ -375,7 +355,10 @@ export const cleanupOrphanedNodes = async ( context: SupabaseContext, ): Promise => { try { - const supabaseUids = await getAllNodesFromSupabase(supabaseClient, context); + const supabaseUids = await getAllNodesFromSupabase( + supabaseClient, + context.spaceId, + ); if (supabaseUids.length > 0) { const orphanedUids = getNonExistentRoamUids(supabaseUids); if (orphanedUids.length > 0) { @@ -389,12 +372,16 @@ export const cleanupOrphanedNodes = async ( const supabaseSchemaUids = await getAllNodeSchemasFromSupabase( supabaseClient, - context, + context.spaceId, ); if (supabaseSchemaUids.length > 0) { const orphanedSchemaUids = getNonExistentRoamUids(supabaseSchemaUids); if (orphanedSchemaUids.length > 0) { - await deleteNodeSchemasFromSupabase(orphanedSchemaUids); + await deleteNodeSchemasFromSupabase( + orphanedSchemaUids, + supabaseClient, + context.spaceId, + ); } } } catch (error) { diff --git a/apps/roam/src/utils/fetchEmbeddingsForNodes.ts b/apps/roam/src/utils/fetchEmbeddingsForNodes.ts index bff740f8f..61727dc34 100644 --- a/apps/roam/src/utils/fetchEmbeddingsForNodes.ts +++ b/apps/roam/src/utils/fetchEmbeddingsForNodes.ts @@ -21,7 +21,7 @@ export const fetchEmbeddingsForNodes = async ( for (let i = 0; i < allNodesTexts.length; i += EMBEDDING_BATCH_SIZE) { const batch = allNodesTexts.slice(i, i + EMBEDDING_BATCH_SIZE); console.log( - `fetchEmbeddingsForNodes: Fetching batch ${i / EMBEDDING_BATCH_SIZE + 1} of ${allNodesTexts.length / EMBEDDING_BATCH_SIZE}`, + `fetchEmbeddingsForNodes: Fetching batch ${i / EMBEDDING_BATCH_SIZE + 1} of ${Math.ceil(allNodesTexts.length / EMBEDDING_BATCH_SIZE)}`, ); const response = await fetch(API_URL, { diff --git a/apps/roam/src/utils/syncDgNodesToSupabase.ts b/apps/roam/src/utils/syncDgNodesToSupabase.ts index b45c43cef..95880e317 100644 --- a/apps/roam/src/utils/syncDgNodesToSupabase.ts +++ b/apps/roam/src/utils/syncDgNodesToSupabase.ts @@ -147,10 +147,10 @@ const upsertNodeSchemaToContent = async ( ] `; //@ts-ignore - backend to be added to roamjs-components - const result = await window.roamAlphaAPI.data.backend.q( + const result = (await window.roamAlphaAPI.data.backend.q( query, nodeTypesUids, - ) as unknown as RoamDiscourseNodeData[]; + )) as unknown as RoamDiscourseNodeData[]; const contentData: LocalContentDataInput[] = result.map((node) => ({ author_id: userId, @@ -238,7 +238,7 @@ export const convertDgToSupabaseConcepts = async ( ...relationBlockToLocalConcepts, ]; const { ordered } = orderConceptsByDependency(conceptsToUpsert); - const { error } = await supabaseClient.rpc("upsert_concepts", { + const { error } = await supabaseClient.rpc("upsert_concepts", { data: ordered, v_space_id: context.spaceId, }); @@ -373,6 +373,7 @@ export const createOrUpdateDiscourseEmbedding = async ( const context = await getSupabaseContext(); if (!context) { console.error("No Supabase context found."); + await endSyncTask(worker, "failed"); return; } await upsertNodesToSupabaseAsContentWithEmbeddings( From 3e430c9dff093b02bfff27598d17b64dd501f2a2 Mon Sep 17 00:00:00 2001 From: sid597 Date: Tue, 19 Aug 2025 21:56:16 +0530 Subject: [PATCH 10/30] address lint errors --- .../roam/src/utils/fetchEmbeddingsForNodes.ts | 11 ++- .../src/utils/getAllDiscourseNodesSince.ts | 1 + apps/roam/src/utils/syncDgNodesToSupabase.ts | 67 +++++++++++-------- .../upsertNodesAsContentWithEmbeddings.ts | 15 +++-- 4 files changed, 53 insertions(+), 41 deletions(-) diff --git a/apps/roam/src/utils/fetchEmbeddingsForNodes.ts b/apps/roam/src/utils/fetchEmbeddingsForNodes.ts index 61727dc34..8e3ace77e 100644 --- a/apps/roam/src/utils/fetchEmbeddingsForNodes.ts +++ b/apps/roam/src/utils/fetchEmbeddingsForNodes.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/naming-convention */ import { RoamDiscourseNodeData } from "./getAllDiscourseNodesSince"; const EMBEDDING_BATCH_SIZE = 200; @@ -20,10 +21,6 @@ export const fetchEmbeddingsForNodes = async ( for (let i = 0; i < allNodesTexts.length; i += EMBEDDING_BATCH_SIZE) { const batch = allNodesTexts.slice(i, i + EMBEDDING_BATCH_SIZE); - console.log( - `fetchEmbeddingsForNodes: Fetching batch ${i / EMBEDDING_BATCH_SIZE + 1} of ${Math.ceil(allNodesTexts.length / EMBEDDING_BATCH_SIZE)}`, - ); - const response = await fetch(API_URL, { method: "POST", headers: { "Content-Type": "application/json" }, @@ -31,9 +28,9 @@ export const fetchEmbeddingsForNodes = async ( }); if (!response.ok) { - let errorData; + let errorData: { error: string }; try { - errorData = await response.json(); + errorData = (await response.json()) as { error: string }; } catch (e) { errorData = { error: `Server responded with ${response.status}: ${await response.text()}`, @@ -46,7 +43,7 @@ export const fetchEmbeddingsForNodes = async ( ); } - const data: EmbeddingApiResponse = await response.json(); + const data = (await response.json()) as EmbeddingApiResponse; if (!data || !Array.isArray(data.data)) { throw new Error( `Invalid API response format for batch ${ diff --git a/apps/roam/src/utils/getAllDiscourseNodesSince.ts b/apps/roam/src/utils/getAllDiscourseNodesSince.ts index 115efc850..242ac456e 100644 --- a/apps/roam/src/utils/getAllDiscourseNodesSince.ts +++ b/apps/roam/src/utils/getAllDiscourseNodesSince.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/naming-convention */ import getDiscourseNodes, { DiscourseNode } from "./getDiscourseNodes"; import findDiscourseNode from "./findDiscourseNode"; import { OnloadArgs } from "roamjs-components/types"; diff --git a/apps/roam/src/utils/syncDgNodesToSupabase.ts b/apps/roam/src/utils/syncDgNodesToSupabase.ts index 95880e317..fd9c489ed 100644 --- a/apps/roam/src/utils/syncDgNodesToSupabase.ts +++ b/apps/roam/src/utils/syncDgNodesToSupabase.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/naming-convention */ import { getAllDiscourseNodesSince, nodeTypeSince, @@ -124,12 +125,17 @@ export const proposeSyncTask = async (): Promise => { } }; -const upsertNodeSchemaToContent = async ( - nodeTypesUids: string[], - spaceId: number, - userId: number, - supabaseClient: DGSupabaseClient, -) => { +const upsertNodeSchemaToContent = async ({ + nodeTypesUids, + spaceId, + userId, + supabaseClient, +}: { + nodeTypesUids: string[]; + spaceId: number; + userId: number; + supabaseClient: DGSupabaseClient; +}) => { const query = `[ :find ?uid ?create-time ?edit-time ?user-uuid ?title ?author-name :keys source_local_id created last_modified author_local_id text author_name @@ -166,7 +172,7 @@ const upsertNodeSchemaToContent = async ( scale: "document", })); const { error } = await supabaseClient.rpc("upsert_content", { - data: contentData as any, + data: contentData, v_space_id: spaceId, v_creator_id: userId, content_as_document: true, @@ -176,20 +182,26 @@ const upsertNodeSchemaToContent = async ( } }; -export const convertDgToSupabaseConcepts = async ( - nodesSince: RoamDiscourseNodeData[], - since: string, - allNodeTypes: DiscourseNode[], - supabaseClient: DGSupabaseClient, - context: SupabaseContext, -) => { +export const convertDgToSupabaseConcepts = async ({ + nodesSince, + since, + allNodeTypes, + supabaseClient, + context, +}: { + nodesSince: RoamDiscourseNodeData[]; + since: string; + allNodeTypes: DiscourseNode[]; + supabaseClient: DGSupabaseClient; + context: SupabaseContext; +}) => { const nodeTypes = await nodeTypeSince(since, allNodeTypes); - await upsertNodeSchemaToContent( - nodeTypes.map((node) => node.type), - context.spaceId, - context.userId, + await upsertNodeSchemaToContent({ + nodeTypesUids: nodeTypes.map((node) => node.type), + spaceId: context.spaceId, + userId: context.userId, supabaseClient, - ); + }); const nodesTypesToLocalConcepts = nodeTypes.map((node) => { return discourseNodeSchemaToLocalConcept(context, node); @@ -263,9 +275,10 @@ export const upsertNodesToSupabaseAsContentWithEmbeddings = async ( let nodesWithEmbeddings: RoamDiscourseNodeData[]; try { nodesWithEmbeddings = await fetchEmbeddingsForNodes(roamNodes); - } catch (error: any) { + } catch (error) { + const message = error instanceof Error ? error.message : String(error); console.error( - `upsertNodesToSupabaseAsContentWithEmbeddings: Embedding service failed – ${error.message}`, + `upsertNodesToSupabaseAsContentWithEmbeddings: Embedding service failed – ${message}`, ); return; } @@ -314,7 +327,7 @@ export const upsertNodesToSupabaseAsContentWithEmbeddings = async ( }); const { error } = await supabaseClient.rpc("upsert_content", { - data: contents as any, + data: contents, v_space_id: spaceId, v_creator_id: userId, content_as_document: true, @@ -381,13 +394,13 @@ export const createOrUpdateDiscourseEmbedding = async ( supabaseClient, context, ); - await convertDgToSupabaseConcepts( - allNodeInstances, - time, - allDgNodeTypes, + await convertDgToSupabaseConcepts({ + nodesSince: allNodeInstances, + since: time, + allNodeTypes: allDgNodeTypes, supabaseClient, context, - ); + }); await cleanupOrphanedNodes(supabaseClient, context); await endSyncTask(worker, "complete"); } catch (error) { diff --git a/apps/roam/src/utils/upsertNodesAsContentWithEmbeddings.ts b/apps/roam/src/utils/upsertNodesAsContentWithEmbeddings.ts index fb929c086..d31f1367e 100644 --- a/apps/roam/src/utils/upsertNodesAsContentWithEmbeddings.ts +++ b/apps/roam/src/utils/upsertNodesAsContentWithEmbeddings.ts @@ -1,8 +1,9 @@ +/* eslint-disable @typescript-eslint/naming-convention */ import { RoamDiscourseNodeData } from "./getAllDiscourseNodesSince"; import { SupabaseContext } from "./supabaseContext"; import { LocalContentDataInput } from "@repo/database/inputTypes"; import { DGSupabaseClient } from "@repo/ui/lib/supabase/client"; - +import { Json } from "@repo/database/types.gen"; const EMBEDDING_BATCH_SIZE = 200; const API_URL = `https://discoursegraphs.com/api/embeddings/openai/small`; @@ -37,7 +38,7 @@ export const fetchEmbeddingsForNodes = async ( if (!response.ok) { let errorData; try { - errorData = await response.json(); + errorData = (await response.json()) as { error: string }; } catch (e) { errorData = { error: `Server responded with ${response.status}: ${await response.text()}`, @@ -50,7 +51,7 @@ export const fetchEmbeddingsForNodes = async ( ); } - const data: EmbeddingApiResponse = await response.json(); + const data = (await response.json()) as EmbeddingApiResponse; if (!data || !Array.isArray(data.data)) { throw new Error( `Invalid API response format for batch ${ @@ -105,7 +106,7 @@ const uploadBatches = async ( }); const { error } = await supabaseClient.rpc("upsert_content", { - data: contents as any, + data: contents as unknown as Json, v_space_id: spaceId, v_creator_id: userId, content_as_document: true, @@ -127,7 +128,6 @@ export const upsertNodesToSupabaseAsContentWithEmbeddings = async ( console.error("No Supabase context found."); return; } - const { spaceId, userId } = context; if (roamNodes.length === 0) { return; @@ -136,9 +136,10 @@ export const upsertNodesToSupabaseAsContentWithEmbeddings = async ( let nodesWithEmbeddings: RoamDiscourseNodeData[]; try { nodesWithEmbeddings = await fetchEmbeddingsForNodes(roamNodes); - } catch (error: any) { + } catch (error: unknown) { + const errorMessage = error instanceof Error ? error.message : String(error); console.error( - `upsertNodesToSupabaseAsContentWithEmbeddings: Embedding service failed – ${error.message}`, + `upsertNodesToSupabaseAsContentWithEmbeddings: Embedding service failed – ${errorMessage}`, ); return; } From 0425eeefa88799b5b1b554200b274b492ca6b92a Mon Sep 17 00:00:00 2001 From: sid597 Date: Tue, 19 Aug 2025 21:58:48 +0530 Subject: [PATCH 11/30] use async instead of backend --- apps/roam/src/components/DiscourseContextOverlay.tsx | 2 +- apps/roam/src/utils/getAllDiscourseNodesSince.ts | 2 +- apps/roam/src/utils/syncDgNodesToSupabase.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/roam/src/components/DiscourseContextOverlay.tsx b/apps/roam/src/components/DiscourseContextOverlay.tsx index 25220de50..64dd8c6c8 100644 --- a/apps/roam/src/components/DiscourseContextOverlay.tsx +++ b/apps/roam/src/components/DiscourseContextOverlay.tsx @@ -39,7 +39,7 @@ const getOverlayInfo = async (tag: string): Promise => { relations, }), // @ts-ignore - backend to be added to roamjs-components - window.roamAlphaAPI.data.backend.q( + window.roamAlphaAPI.data.async.q( `[:find ?a :where [?b :node/title "${normalizePageTitle(tag)}"] [?a :block/refs ?b]]`, ), ]); diff --git a/apps/roam/src/utils/getAllDiscourseNodesSince.ts b/apps/roam/src/utils/getAllDiscourseNodesSince.ts index 242ac456e..0c0d6b75a 100644 --- a/apps/roam/src/utils/getAllDiscourseNodesSince.ts +++ b/apps/roam/src/utils/getAllDiscourseNodesSince.ts @@ -112,7 +112,7 @@ export const getAllDiscourseNodesSince = async ( ]`; //@ts-ignore - backend to be added to roamjs-components - const allNodes = (await window.roamAlphaAPI.data.backend.q( + const allNodes = (await window.roamAlphaAPI.data.async.q( query, sinceMs, )) as unknown as RoamDiscourseNodeData[]; diff --git a/apps/roam/src/utils/syncDgNodesToSupabase.ts b/apps/roam/src/utils/syncDgNodesToSupabase.ts index fd9c489ed..f2095781e 100644 --- a/apps/roam/src/utils/syncDgNodesToSupabase.ts +++ b/apps/roam/src/utils/syncDgNodesToSupabase.ts @@ -153,7 +153,7 @@ const upsertNodeSchemaToContent = async ({ ] `; //@ts-ignore - backend to be added to roamjs-components - const result = (await window.roamAlphaAPI.data.backend.q( + const result = (await window.roamAlphaAPI.data.async.q( query, nodeTypesUids, )) as unknown as RoamDiscourseNodeData[]; From 5622c64e65a8204a913c82a5880e5fe476010afa Mon Sep 17 00:00:00 2001 From: sid597 Date: Thu, 21 Aug 2025 23:52:05 +0530 Subject: [PATCH 12/30] address review --- .../roam/src/utils/fetchEmbeddingsForNodes.ts | 67 -------------- .../src/utils/getAllDiscourseNodesSince.ts | 43 +++++---- apps/roam/src/utils/syncDgNodesToSupabase.ts | 75 ++++++---------- .../upsertNodesAsContentWithEmbeddings.ts | 87 ++++++++++--------- packages/database/package.json | 1 + 5 files changed, 95 insertions(+), 178 deletions(-) delete mode 100644 apps/roam/src/utils/fetchEmbeddingsForNodes.ts diff --git a/apps/roam/src/utils/fetchEmbeddingsForNodes.ts b/apps/roam/src/utils/fetchEmbeddingsForNodes.ts deleted file mode 100644 index 8e3ace77e..000000000 --- a/apps/roam/src/utils/fetchEmbeddingsForNodes.ts +++ /dev/null @@ -1,67 +0,0 @@ -/* eslint-disable @typescript-eslint/naming-convention */ -import { RoamDiscourseNodeData } from "./getAllDiscourseNodesSince"; - -const EMBEDDING_BATCH_SIZE = 200; -const API_URL = `https://discoursegraphs.com/api/embeddings/openai/small`; -const EMBEDDING_MODEL = "openai_text_embedding_3_small_1536"; - -type EmbeddingApiResponse = { - data: { - embedding: number[]; - }[]; -}; - -export const fetchEmbeddingsForNodes = async ( - nodes: RoamDiscourseNodeData[], -): Promise => { - const allEmbeddings: number[][] = []; - const allNodesTexts = nodes.map((node) => - node.node_title ? `${node.node_title} ${node.text}` : node.text, - ); - - for (let i = 0; i < allNodesTexts.length; i += EMBEDDING_BATCH_SIZE) { - const batch = allNodesTexts.slice(i, i + EMBEDDING_BATCH_SIZE); - const response = await fetch(API_URL, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ input: batch }), - }); - - if (!response.ok) { - let errorData: { error: string }; - try { - errorData = (await response.json()) as { error: string }; - } catch (e) { - errorData = { - error: `Server responded with ${response.status}: ${await response.text()}`, - }; - } - throw new Error( - `API Error (${response.status}) processing batch ${ - i / EMBEDDING_BATCH_SIZE + 1 - }: ${errorData.error || "Failed to fetch embeddings"}`, - ); - } - - const data = (await response.json()) as EmbeddingApiResponse; - if (!data || !Array.isArray(data.data)) { - throw new Error( - `Invalid API response format for batch ${ - i / EMBEDDING_BATCH_SIZE + 1 - }. Expected 'data' array.`, - ); - } - const batchEmbeddings = data.data.map((item) => item.embedding); - allEmbeddings.push(...batchEmbeddings); - } - if (nodes.length !== allEmbeddings.length) { - throw new Error( - `Mismatch between nodes (${nodes.length}) and embeddings (${allEmbeddings.length})`, - ); - } - return nodes.map((node, i) => ({ - ...node, - model: EMBEDDING_MODEL, - vector: allEmbeddings[i], - })); -}; diff --git a/apps/roam/src/utils/getAllDiscourseNodesSince.ts b/apps/roam/src/utils/getAllDiscourseNodesSince.ts index 0c0d6b75a..bb72063a8 100644 --- a/apps/roam/src/utils/getAllDiscourseNodesSince.ts +++ b/apps/roam/src/utils/getAllDiscourseNodesSince.ts @@ -11,7 +11,6 @@ export type RoamDiscourseNodeData = { author_name: string; source_local_id: string; created: string; - vector: number[]; last_modified: string; text: string; type: string; @@ -70,8 +69,8 @@ export const getDiscourseNodeTypeWithSettingsBlockNodes = ( String(firstChildUid), String(node.type), sinceMs, - ) as unknown as Omit[]; - return blockNode.map((node) => ({ ...node, vector: [] })); + ) as unknown as RoamDiscourseNodeData[]; + return blockNode; }; export const getAllDiscourseNodesSince = async ( @@ -121,27 +120,27 @@ export const getAllDiscourseNodesSince = async ( const nodeTypesSet = new Set(nodeTypes.map((nodeType) => nodeType.type)); result.push( - ...allNodes - .map((entity) => { - if (!entity.source_local_id) { - return null; - } - const node = findDiscourseNode(entity.source_local_id, discourseNodes); - if ( - !node || - node.backedBy === "default" || - !entity.text || - entity.text.trim() === "" || - nodeTypesSet.has(node.type) - ) { - return null; - } - return { + ...allNodes.flatMap((entity) => { + if (!entity.source_local_id) { + return []; + } + const node = findDiscourseNode(entity.source_local_id, discourseNodes); + if ( + !node || + node.backedBy === "default" || + !entity.text || + entity.text.trim() === "" || + nodeTypesSet.has(node.type) + ) { + return []; + } + return [ + { ...entity, type: node.type, - }; - }) - .filter((n): n is RoamDiscourseNodeData => n !== null), + }, + ]; + }), ); return result; }; diff --git a/apps/roam/src/utils/syncDgNodesToSupabase.ts b/apps/roam/src/utils/syncDgNodesToSupabase.ts index f2095781e..2feeee774 100644 --- a/apps/roam/src/utils/syncDgNodesToSupabase.ts +++ b/apps/roam/src/utils/syncDgNodesToSupabase.ts @@ -9,7 +9,6 @@ import { getSupabaseContext, SupabaseContext, } from "./supabaseContext"; -import { fetchEmbeddingsForNodes } from "./fetchEmbeddingsForNodes"; import { LocalContentDataInput } from "@repo/database/inputTypes"; import { RoamDiscourseNodeData } from "./getAllDiscourseNodesSince"; import getDiscourseRelations from "./getDiscourseRelations"; @@ -24,13 +23,15 @@ import { import getDiscourseRelationTriples from "./getDiscourseRelationTriples"; import { OnloadArgs } from "roamjs-components/types"; import { DGSupabaseClient } from "@repo/ui/lib/supabase/client"; +import { fetchEmbeddingsForNodes } from "./upsertNodesAsContentWithEmbeddings"; +import { Json } from "@repo/database/types.gen"; +import { convertRoamNodeToLocalContent } from "./upsertNodesAsContentWithEmbeddings"; const SYNC_FUNCTION = "embedding"; const SYNC_INTERVAL = "45s"; const SYNC_TIMEOUT = "20s"; const BATCH_SIZE = 200; const DEFAULT_TIME = "1970-01-01"; -const EMBEDDING_MODEL = "openai_text_embedding_3_small_1536"; type SyncTaskInfo = { lastUpdateTime: string | null; @@ -78,6 +79,15 @@ export const proposeSyncTask = async (): Promise => { }; } const worker = window.roamAlphaAPI.user.uid(); + if (!worker) { + console.error("proposeSyncTask: Unable to obtain user UID."); + return { + lastUpdateTime: null, + spaceId: 0, + worker: "", + shouldProceed: false, + }; + } const { data, error } = await supabaseClient.rpc("propose_sync_task", { s_target: context.spaceId, @@ -158,21 +168,12 @@ const upsertNodeSchemaToContent = async ({ nodeTypesUids, )) as unknown as RoamDiscourseNodeData[]; - const contentData: LocalContentDataInput[] = result.map((node) => ({ - author_id: userId, - account_local_id: node.author_local_id, - source_local_id: node.source_local_id, - created: new Date(node.created || Date.now()).toISOString(), - last_modified: new Date(node.last_modified || Date.now()).toISOString(), - text: node.text, - embedding_inline: { - model: EMBEDDING_MODEL, - vector: node.vector, - }, - scale: "document", - })); + const contentData: LocalContentDataInput[] = convertRoamNodeToLocalContent({ + nodes: result, + userId, + }); const { error } = await supabaseClient.rpc("upsert_content", { - data: contentData, + data: contentData as Json, v_space_id: spaceId, v_creator_id: userId, content_as_document: true, @@ -266,15 +267,19 @@ export const upsertNodesToSupabaseAsContentWithEmbeddings = async ( supabaseClient: DGSupabaseClient, context: SupabaseContext, ): Promise => { - const { spaceId, userId } = context; + const { userId } = context; if (roamNodes.length === 0) { return; } + const allNodeInstancesAsLocalContent = convertRoamNodeToLocalContent({ + nodes: roamNodes, + userId: context.userId, + }); - let nodesWithEmbeddings: RoamDiscourseNodeData[]; + let nodesWithEmbeddings: LocalContentDataInput[]; try { - nodesWithEmbeddings = await fetchEmbeddingsForNodes(roamNodes); + nodesWithEmbeddings = await fetchEmbeddingsForNodes(allNodeInstancesAsLocalContent); } catch (error) { const message = error instanceof Error ? error.message : String(error); console.error( @@ -283,7 +288,7 @@ export const upsertNodesToSupabaseAsContentWithEmbeddings = async ( return; } - if (nodesWithEmbeddings.length !== roamNodes.length) { + if (nodesWithEmbeddings.length !== allNodeInstancesAsLocalContent.length) { console.error( "upsertNodesToSupabaseAsContentWithEmbeddings: Mismatch between node and embedding counts.", ); @@ -298,37 +303,13 @@ export const upsertNodesToSupabaseAsContentWithEmbeddings = async ( return chunks; }; - const uploadBatches = async (batches: RoamDiscourseNodeData[][]) => { + const uploadBatches = async (batches: LocalContentDataInput[][]) => { for (let idx = 0; idx < batches.length; idx++) { const batch = batches[idx]; - const contents: LocalContentDataInput[] = batch.map((node) => { - const variant = node.node_title ? "direct_and_description" : "direct"; - const text = node.node_title - ? `${node.node_title} ${node.text}` - : node.text; - - return { - author_id: userId, - account_local_id: node.author_local_id, - source_local_id: node.source_local_id, - created: new Date(node.created || Date.now()).toISOString(), - last_modified: new Date( - node.last_modified || Date.now(), - ).toISOString(), - text: text, - variant: variant, - embedding_inline: { - model: EMBEDDING_MODEL, - vector: node.vector, - }, - scale: "document", - }; - }); - const { error } = await supabaseClient.rpc("upsert_content", { - data: contents, - v_space_id: spaceId, + data: batch as Json, + v_space_id: context.spaceId, v_creator_id: userId, content_as_document: true, }); diff --git a/apps/roam/src/utils/upsertNodesAsContentWithEmbeddings.ts b/apps/roam/src/utils/upsertNodesAsContentWithEmbeddings.ts index d31f1367e..f00704201 100644 --- a/apps/roam/src/utils/upsertNodesAsContentWithEmbeddings.ts +++ b/apps/roam/src/utils/upsertNodesAsContentWithEmbeddings.ts @@ -4,9 +4,10 @@ import { SupabaseContext } from "./supabaseContext"; import { LocalContentDataInput } from "@repo/database/inputTypes"; import { DGSupabaseClient } from "@repo/ui/lib/supabase/client"; import { Json } from "@repo/database/types.gen"; +import { nextApiRoot } from "@repo/ui/lib/execContext"; const EMBEDDING_BATCH_SIZE = 200; -const API_URL = `https://discoursegraphs.com/api/embeddings/openai/small`; +const EMBEDDING_MODEL = "openai_text_embedding_3_small_1536"; type EmbeddingApiResponse = { data: { @@ -14,22 +15,40 @@ type EmbeddingApiResponse = { }[]; }; +export const convertRoamNodeToLocalContent = ({ + nodes, + userId, +}: { + nodes: RoamDiscourseNodeData[]; + userId: number; +}): LocalContentDataInput[] => { + return nodes.map((node) => { + const variant = node.node_title ? "direct_and_description" : "direct"; + const text = node.node_title + ? `${node.node_title} ${node.text}` + : node.text; + return { + author_id: userId, + author_local_id: node.author_local_id, + source_local_id: node.source_local_id, + created: new Date(node.created || Date.now()).toISOString(), + last_modified: new Date(node.last_modified || Date.now()).toISOString(), + text: text, + variant: variant, + scale: "document", + }; + }); +}; + export const fetchEmbeddingsForNodes = async ( - nodes: RoamDiscourseNodeData[], -): Promise => { + nodes: LocalContentDataInput[], +): Promise => { const allEmbeddings: number[][] = []; - console.log("nodes", nodes); - const allNodesTexts = nodes.map((node) => - node.node_title ? `${node.node_title} ${node.text}` : node.text, - ); + const allNodesTexts = nodes.map((node) => node.text || ""); for (let i = 0; i < allNodesTexts.length; i += EMBEDDING_BATCH_SIZE) { const batch = allNodesTexts.slice(i, i + EMBEDDING_BATCH_SIZE); - console.log( - `fetchEmbeddingsForNodes: Fetching batch ${i / EMBEDDING_BATCH_SIZE + 1} of ${allNodesTexts.length / EMBEDDING_BATCH_SIZE}`, - ); - - const response = await fetch(API_URL, { + const response = await fetch(nextApiRoot() + "/embeddings/openai/small", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ input: batch }), @@ -67,46 +86,26 @@ export const fetchEmbeddingsForNodes = async ( `Mismatch between nodes (${nodes.length}) and embeddings (${allEmbeddings.length})`, ); } + return nodes.map((node, i) => ({ ...node, - model: "openai_text_embedding_3_small_1536", - vector: allEmbeddings[i], + embedding_inline: { + model: EMBEDDING_MODEL, + vector: allEmbeddings[i], + }, })); }; const uploadBatches = async ( - batches: RoamDiscourseNodeData[][], + batches: LocalContentDataInput[][], supabaseClient: DGSupabaseClient, context: SupabaseContext, ) => { const { spaceId, userId } = context; for (let idx = 0; idx < batches.length; idx++) { const batch = batches[idx]; - - const contents: LocalContentDataInput[] = batch.map((node) => { - const variant = node.node_title ? "direct_and_description" : "direct"; - const text = node.node_title - ? `${node.node_title} ${node.text}` - : node.text; - - return { - author_id: userId, - account_local_id: node.author_local_id, - source_local_id: node.source_local_id, - created: new Date(node.created || Date.now()).toISOString(), - last_modified: new Date(node.last_modified || Date.now()).toISOString(), - text: text, - variant: variant, - embedding_inline: { - model: "openai_text_embedding_3_small_1536", - vector: node.vector, - }, - scale: "document", - }; - }); - const { error } = await supabaseClient.rpc("upsert_content", { - data: contents as unknown as Json, + data: batch as unknown as Json, v_space_id: spaceId, v_creator_id: userId, content_as_document: true, @@ -124,7 +123,7 @@ export const upsertNodesToSupabaseAsContentWithEmbeddings = async ( supabaseClient: DGSupabaseClient, context: SupabaseContext, ): Promise => { - if (!context) { + if (!context?.userId) { console.error("No Supabase context found."); return; } @@ -132,10 +131,14 @@ export const upsertNodesToSupabaseAsContentWithEmbeddings = async ( if (roamNodes.length === 0) { return; } + const localContentNodes = convertRoamNodeToLocalContent({ + nodes: roamNodes, + userId: context.userId, + }); - let nodesWithEmbeddings: RoamDiscourseNodeData[]; + let nodesWithEmbeddings: LocalContentDataInput[]; try { - nodesWithEmbeddings = await fetchEmbeddingsForNodes(roamNodes); + nodesWithEmbeddings = await fetchEmbeddingsForNodes(localContentNodes); } catch (error: unknown) { const errorMessage = error instanceof Error ? error.message : String(error); console.error( diff --git a/packages/database/package.json b/packages/database/package.json index 97fa9d2cf..c4e4bb2d7 100644 --- a/packages/database/package.json +++ b/packages/database/package.json @@ -7,6 +7,7 @@ "exports": { "./types.gen.ts": "./types.gen.ts", "./types.gen": "./types.gen.ts", + "./inputTypes": "./inputTypes.ts", "./dbDotEnv": "./dbDotEnv.ts" }, "scripts": { From 590848c33b287f9f1f1f2ccad53cc442107d6e48 Mon Sep 17 00:00:00 2001 From: sid597 Date: Fri, 22 Aug 2025 02:12:44 +0530 Subject: [PATCH 13/30] address review --- .../settings/DiscourseNodeSuggestiveRules.tsx | 61 ++++--------- .../src/components/settings/NodeConfig.tsx | 13 +-- .../components/settings/PageGroupPanel.tsx | 91 ++++++++++--------- .../roam/src/components/settings/Settings.tsx | 2 +- .../settings/SuggestiveModeSettings.tsx | 56 ++++++------ apps/roam/src/utils/getDiscourseNodes.ts | 34 +++++++ .../utils/getSuggestiveModeConfigSettings.ts | 29 ++++-- 7 files changed, 154 insertions(+), 132 deletions(-) diff --git a/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx index ec1eabc0c..3aff00f6e 100644 --- a/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx +++ b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx @@ -1,3 +1,5 @@ +/* eslint-disable @typescript-eslint/naming-convention */ + import React, { useState, useMemo, @@ -7,30 +9,11 @@ import React, { } from "react"; import { Button, Intent, Tooltip, Position } from "@blueprintjs/core"; import BlocksPanel from "roamjs-components/components/ConfigPanels/BlocksPanel"; -import getBasicTreeByParentUid from "roamjs-components/queries/getBasicTreeByParentUid"; -import refreshConfigTree from "~/utils/refreshConfigTree"; import FlagPanel from "roamjs-components/components/ConfigPanels/FlagPanel"; -import { getUidAndBooleanSetting } from "~/utils/getExportSettings"; import TextPanel from "roamjs-components/components/ConfigPanels/TextPanel"; import getSubTree from "roamjs-components/util/getSubTree"; import { DiscourseNode } from "~/utils/getDiscourseNodes"; - -const getNodeConfig = (parentUid: string) => { - const tree = getBasicTreeByParentUid(parentUid); - const embeddingRefNode = tree.find((n) => - n.text.startsWith("Embedding Block Ref"), - ); - const match = embeddingRefNode?.children?.[0]?.text?.match(/\(\((.*)\)\)/); - const blockRef = match ? `((${match[1]}))` : ""; - return { - embeddingRef: blockRef, - embeddingRefUid: embeddingRefNode?.uid || "", - isFirstChild: getUidAndBooleanSetting({ - tree: tree, - text: "First Child", - }), - }; -}; +import extractRef from "roamjs-components/util/extractRef"; const BlockRenderer = ({ uid }: { uid: string }) => { const containerRef = useRef(null); @@ -38,9 +21,7 @@ const BlockRenderer = ({ uid }: { uid: string }) => { useEffect(() => { const container = containerRef.current; if (container) { - while (container.firstChild) { - container.removeChild(container.firstChild); - } + container.innerHTML = ""; if (uid) { window.roamAlphaAPI.ui.components.renderBlock({ @@ -62,27 +43,19 @@ const DiscourseNodeSuggestiveRules = ({ parentUid: string; }) => { const nodeUid = node.type; - const [nodeConfigTree, setNodeConfigTree] = useState(() => - getNodeConfig(parentUid), - ); - - useEffect(() => { - refreshConfigTree(); - setNodeConfigTree(getNodeConfig(parentUid)); - }, [parentUid]); - const [embeddingRef, setEmbeddingRef] = useState(nodeConfigTree.embeddingRef); + const [embeddingRef, setEmbeddingRef] = useState(node.embeddingRef || ""); useEffect(() => { - if (nodeConfigTree.embeddingRef !== embeddingRef) { - setEmbeddingRef(nodeConfigTree.embeddingRef); + if (node.embeddingRef !== embeddingRef) { + setEmbeddingRef(node.embeddingRef || ""); } - }, [nodeConfigTree.embeddingRef]); + }, [node.embeddingRef, embeddingRef]); - const blockUidToRender = useMemo(() => { - const match = embeddingRef?.match(/\(\((.*)\)\)/); - return match ? match[1] : ""; - }, [embeddingRef]); + const blockUidToRender = useMemo( + () => extractRef(embeddingRef), + [embeddingRef], + ); const templateUid = useMemo( () => @@ -116,9 +89,9 @@ const DiscourseNodeSuggestiveRules = ({ title="Embedding Block Ref" description="Copy block ref from template which you want to be embedded and ranked." order={0} - uid={nodeConfigTree.embeddingRefUid} + uid={node.embeddingRefUid || ""} parentUid={parentUid} - defaultValue={nodeConfigTree.embeddingRef} + defaultValue={node.embeddingRef || ""} options={{ placeholder: "((block-uid))", onChange: handleEmbeddingRefChange, @@ -134,11 +107,11 @@ const DiscourseNodeSuggestiveRules = ({
diff --git a/apps/roam/src/components/settings/NodeConfig.tsx b/apps/roam/src/components/settings/NodeConfig.tsx index 0261bb3b5..0bcda01c9 100644 --- a/apps/roam/src/components/settings/NodeConfig.tsx +++ b/apps/roam/src/components/settings/NodeConfig.tsx @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/naming-convention */ import React, { useState, useCallback, useRef, useEffect } from "react"; import { DiscourseNode } from "~/utils/getDiscourseNodes"; import FlagPanel from "roamjs-components/components/ConfigPanels/FlagPanel"; @@ -122,6 +123,7 @@ const NodeConfig = ({ const graphOverviewUid = getUid("Graph Overview"); const specificationUid = getUid("Specification"); const indexUid = getUid("Index"); + const suggestiveRulesUid = getUid("Suggestive Rules"); const attributeNode = getSubTree({ parentUid: node.type, key: "Attributes", @@ -329,18 +331,13 @@ const NodeConfig = ({ } />
} diff --git a/apps/roam/src/components/settings/PageGroupPanel.tsx b/apps/roam/src/components/settings/PageGroupPanel.tsx index 11fa99769..f9f5b1027 100644 --- a/apps/roam/src/components/settings/PageGroupPanel.tsx +++ b/apps/roam/src/components/settings/PageGroupPanel.tsx @@ -1,36 +1,23 @@ +/* eslint-disable @typescript-eslint/naming-convention */ import React, { useState, useCallback } from "react"; import { Label, Button, Intent, Tag } from "@blueprintjs/core"; import Description from "roamjs-components/components/Description"; import AutocompleteInput from "roamjs-components/components/AutocompleteInput"; -import getBasicTreeByParentUid from "roamjs-components/queries/getBasicTreeByParentUid"; import createBlock from "roamjs-components/writes/createBlock"; import deleteBlock from "roamjs-components/writes/deleteBlock"; import getAllPageNames from "roamjs-components/queries/getAllPageNames"; +import { PageGroup } from "~/utils/getSuggestiveModeConfigSettings"; -type PageGroupData = { - uid: string; - name: string; - pages: { uid: string; name: string }[]; -}; +type PageGroupData = PageGroup; -const PageGroupsPanel = ({ uid }: { uid: string }) => { - const [pageGroups, setPageGroups] = useState(() => - getBasicTreeByParentUid(uid).map((node) => ({ - uid: node.uid, - name: node.text, - pages: node.children.map((c) => ({ uid: c.uid, name: c.text })), - })), - ); - - const refreshGroups = useCallback(() => { - setPageGroups( - getBasicTreeByParentUid(uid).map((node) => ({ - uid: node.uid, - name: node.text, - pages: node.children.map((c) => ({ uid: c.uid, name: c.text })), - })), - ); - }, [uid, setPageGroups]); +const PageGroupsPanel = ({ + uid, + initialGroups, +}: { + uid: string; + initialGroups: PageGroupData[]; +}) => { + const [pageGroups, setPageGroups] = useState(initialGroups); const [newGroupName, setNewGroupName] = useState(""); const [newPageInputs, setNewPageInputs] = useState>( @@ -43,8 +30,11 @@ const PageGroupsPanel = ({ uid }: { uid: string }) => { const addGroup = async (name: string) => { if (!name || pageGroups.some((g) => g.name === name)) return; try { - await createBlock({ parentUid: uid, node: { text: name } }); - refreshGroups(); + const newGroupUid = await createBlock({ + parentUid: uid, + node: { text: name }, + }); + setPageGroups([...pageGroups, { uid: newGroupUid, name, pages: [] }]); setNewGroupName(""); } catch (e) { console.error("Error adding group", e); @@ -54,7 +44,7 @@ const PageGroupsPanel = ({ uid }: { uid: string }) => { const removeGroup = async (groupUid: string) => { try { await deleteBlock(groupUid); - refreshGroups(); + setPageGroups(pageGroups.filter((g) => g.uid !== groupUid)); } catch (e) { console.error("Error removing group", e); } @@ -66,8 +56,17 @@ const PageGroupsPanel = ({ uid }: { uid: string }) => { return; } try { - await createBlock({ parentUid: groupUid, node: { text: page } }); - refreshGroups(); + const newPageUid = await createBlock({ + parentUid: groupUid, + node: { text: page }, + }); + setPageGroups( + pageGroups.map((g) => + g.uid === groupUid + ? { ...g, pages: [...g.pages, { uid: newPageUid, name: page }] } + : g, + ), + ); setNewPageInputs((prev) => ({ ...prev, [groupUid]: "", @@ -81,10 +80,16 @@ const PageGroupsPanel = ({ uid }: { uid: string }) => { } }; - const removePageFromGroup = async (pageUid: string) => { + const removePageFromGroup = async (groupUid: string, pageUid: string) => { try { await deleteBlock(pageUid); - refreshGroups(); + setPageGroups( + pageGroups.map((g) => + g.uid === groupUid + ? { ...g, pages: g.pages.filter((p) => p.uid !== pageUid) } + : g, + ), + ); } catch (e) { console.error("Error removing page from group", e); } @@ -103,21 +108,21 @@ const PageGroupsPanel = ({ uid }: { uid: string }) => { autocompleteKeys[groupUid] || 0; return ( -
+
); }; diff --git a/apps/roam/src/index.ts b/apps/roam/src/index.ts index fb44ababa..88ad00365 100644 --- a/apps/roam/src/index.ts +++ b/apps/roam/src/index.ts @@ -120,7 +120,7 @@ export default runExtension(async (onloadArgs) => { document.addEventListener("input", discourseNodeSearchTriggerListener); document.addEventListener("selectionchange", nodeCreationPopoverListener); - initializeSupabaseSync(); + await initializeSupabaseSync(); const { extensionAPI } = onloadArgs; window.roamjs.extension.queryBuilder = { diff --git a/apps/roam/src/utils/getDiscourseNodes.ts b/apps/roam/src/utils/getDiscourseNodes.ts index 5b1a86db1..affb06a45 100644 --- a/apps/roam/src/utils/getDiscourseNodes.ts +++ b/apps/roam/src/utils/getDiscourseNodes.ts @@ -5,7 +5,6 @@ import getDiscourseRelations from "./getDiscourseRelations"; import { roamNodeToCondition } from "./parseQuery"; import { Condition } from "./types"; import { InputTextNode, RoamBasicNode } from "roamjs-components/types"; -import extractRef from "roamjs-components/util/extractRef"; export const excludeDefaultNodes = (node: DiscourseNode) => { return node.backedBy !== "default"; diff --git a/apps/roam/src/utils/syncDgNodesToSupabase.ts b/apps/roam/src/utils/syncDgNodesToSupabase.ts index 8b2002c7a..a5ff7a37b 100644 --- a/apps/roam/src/utils/syncDgNodesToSupabase.ts +++ b/apps/roam/src/utils/syncDgNodesToSupabase.ts @@ -139,11 +139,6 @@ export const proposeSyncTask = async (): Promise => { const now = new Date(); if (timestamp > now) { - console.log( - "proposeSyncTask: Another worker is already running this task", - timestamp, - now, - ); return { lastUpdateTime: null, spaceId, worker, shouldProceed: false }; } else { return { lastUpdateTime: data, spaceId, worker, shouldProceed: true }; @@ -374,9 +369,6 @@ export const createOrUpdateDiscourseEmbedding = async () => { const { shouldProceed, lastUpdateTime, worker } = await proposeSyncTask(); if (!shouldProceed) { - console.log( - "createOrUpdateDiscourseEmbedding: Task already running or failed to acquire lock. Exiting.", - ); return; } @@ -426,7 +418,6 @@ export const initializeSupabaseSync = async () => { .eq("url", getRoamUrl()) .maybeSingle(); if (!result.data) { - console.log("initializeSupabaseSync: No space found."); return; } else { createOrUpdateDiscourseEmbedding(); From 3e142dedf64abf37ef57b261bc52e3cd442f05a1 Mon Sep 17 00:00:00 2001 From: sid597 Date: Fri, 29 Aug 2025 22:03:54 +0530 Subject: [PATCH 28/30] update embeddings --- .../settings/DiscourseNodeSuggestiveRules.tsx | 48 +++++++++++++++++-- .../src/utils/getAllDiscourseNodesSince.ts | 28 +++++------ 2 files changed, 57 insertions(+), 19 deletions(-) diff --git a/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx index 9fd592eac..700116610 100644 --- a/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx +++ b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx @@ -13,6 +13,10 @@ import TextPanel from "roamjs-components/components/ConfigPanels/TextPanel"; import getSubTree from "roamjs-components/util/getSubTree"; import { DiscourseNode } from "~/utils/getDiscourseNodes"; import extractRef from "roamjs-components/util/extractRef"; +import { getDiscourseNodeTypeWithSettingsBlockNodes } from "~/utils/getAllDiscourseNodesSince"; +import { upsertNodesToSupabaseAsContentWithEmbeddings } from "~/utils/syncDgNodesToSupabase"; +import { discourseNodeBlockToLocalConcept } from "~/utils/conceptConversion"; +import { getLoggedInClient, getSupabaseContext } from "~/utils/supabaseContext"; const BlockRenderer = ({ uid }: { uid: string }) => { const containerRef = useRef(null); @@ -69,6 +73,41 @@ const DiscourseNodeSuggestiveRules = ({ [], ); + const [isUpdating, setIsUpdating] = useState(false); + + const handleUpdateEmbeddings = async () => { + setIsUpdating(true); + const blockNodesSince = getDiscourseNodeTypeWithSettingsBlockNodes(node, 0); + const supabaseClient = await getLoggedInClient(); + const context = await getSupabaseContext(); + if (context && blockNodesSince) { + await upsertNodesToSupabaseAsContentWithEmbeddings( + blockNodesSince, + supabaseClient, + context, + ); + const nodeBlockToLocalConcepts = blockNodesSince.map((node) => { + const localConcept = discourseNodeBlockToLocalConcept(context, { + nodeUid: node.source_local_id, + schemaUid: node.type, + text: node.node_title ? `${node.node_title} ${node.text}` : node.text, + }); + return localConcept; + }); + + const { error } = await supabaseClient.rpc("upsert_concepts", { + data: nodeBlockToLocalConcepts, + v_space_id: context.spaceId, + }); + if (error) { + throw new Error( + `upsert_concepts failed: ${JSON.stringify(error, null, 2)}`, + ); + } + } + setIsUpdating(false); + }; + return (
console.log("Not implemented")} + onClick={() => void handleUpdateEmbeddings()} + loading={isUpdating} className="w-52" />
diff --git a/apps/roam/src/utils/getAllDiscourseNodesSince.ts b/apps/roam/src/utils/getAllDiscourseNodesSince.ts index 8533c2305..b74a0cc1c 100644 --- a/apps/roam/src/utils/getAllDiscourseNodesSince.ts +++ b/apps/roam/src/utils/getAllDiscourseNodesSince.ts @@ -73,18 +73,6 @@ export const getAllDiscourseNodesSince = async ( const sinceMs = new Date(since).getTime(); const result: RoamDiscourseNodeData[] = []; - if (nodeTypes.length > 0) { - for (const node of nodeTypes) { - const blockNode = getDiscourseNodeTypeWithSettingsBlockNodes( - node, - sinceMs, - ); - if (blockNode) { - result.push(...blockNode); - } - } - } - const query = `[ :find ?node-title ?uid ?nodeCreateTime ?nodeEditTime ?author_local_id ?author_name :keys text source_local_id created last_modified author_local_id author_name @@ -108,7 +96,6 @@ export const getAllDiscourseNodesSince = async ( )) as unknown as RoamDiscourseNodeData[]; const discourseNodes = getDiscourseNodes(); - const nodeTypesSet = new Set(nodeTypes.map((nodeType) => nodeType.type)); result.push( ...allNodes.flatMap((entity) => { @@ -120,8 +107,7 @@ export const getAllDiscourseNodesSince = async ( !node || node.backedBy === "default" || !entity.text || - entity.text.trim() === "" || - nodeTypesSet.has(node.type) + entity.text.trim() === "" ) { return []; } @@ -133,6 +119,18 @@ export const getAllDiscourseNodesSince = async ( ]; }), ); + + if (nodeTypes.length > 0) { + for (const node of nodeTypes) { + const blockNode = getDiscourseNodeTypeWithSettingsBlockNodes( + node, + sinceMs, + ); + if (blockNode) { + result.push(...blockNode); + } + } + } return result; }; From 569df7a62b646e10b83b1703760b0cf4b06dbcc6 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Mon, 1 Sep 2025 11:21:03 -0400 Subject: [PATCH 29/30] ENG-818: Declare dbDotEnv as mjs explicitly --- packages/database/package.json | 6 +++--- packages/database/src/{dbDotEnv.js => dbDotEnv.mjs} | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) rename packages/database/src/{dbDotEnv.js => dbDotEnv.mjs} (92%) diff --git a/packages/database/package.json b/packages/database/package.json index 8a83e6e72..9a0ae4706 100644 --- a/packages/database/package.json +++ b/packages/database/package.json @@ -11,9 +11,9 @@ }, "./dbDotEnv": { "types": "./types/dbDotEnv.d.ts", - "import": "./src/dbDotEnv.js", - "require": "./src/dbDotEnv.js", - "default": "./src/dbDotEnv.js" + "import": "./src/dbDotEnv.mjs", + "require": "./src/dbDotEnv.mjs", + "default": "./src/dbDotEnv.mjs" }, "./dbTypes": { "types": "./dist/src/dbTypes.d.ts", diff --git a/packages/database/src/dbDotEnv.js b/packages/database/src/dbDotEnv.mjs similarity index 92% rename from packages/database/src/dbDotEnv.js rename to packages/database/src/dbDotEnv.mjs index 2e446e524..d2720a089 100644 --- a/packages/database/src/dbDotEnv.js +++ b/packages/database/src/dbDotEnv.mjs @@ -3,6 +3,10 @@ import { join, dirname, basename } from "node:path"; import { fileURLToPath } from "node:url"; import dotenv from "dotenv"; +// Note: This file is written as mjs so it can be used before typescript compilation. +// This means the corresponding .d.ts file is currently maintained by hand. +// Remember to update it as needed. + const findRoot = () => { let dir = fileURLToPath(import.meta.url); while (basename(dir) !== "database") { From b91efed754a47c172ebed6912763b7c580cadb84 Mon Sep 17 00:00:00 2001 From: sid597 Date: Tue, 9 Sep 2025 00:17:05 +0530 Subject: [PATCH 30/30] reverting create a new pr for update embeddings --- .../settings/DiscourseNodeSuggestiveRules.tsx | 50 +++---------------- 1 file changed, 6 insertions(+), 44 deletions(-) diff --git a/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx index 700116610..8271a056a 100644 --- a/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx +++ b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx @@ -13,10 +13,6 @@ import TextPanel from "roamjs-components/components/ConfigPanels/TextPanel"; import getSubTree from "roamjs-components/util/getSubTree"; import { DiscourseNode } from "~/utils/getDiscourseNodes"; import extractRef from "roamjs-components/util/extractRef"; -import { getDiscourseNodeTypeWithSettingsBlockNodes } from "~/utils/getAllDiscourseNodesSince"; -import { upsertNodesToSupabaseAsContentWithEmbeddings } from "~/utils/syncDgNodesToSupabase"; -import { discourseNodeBlockToLocalConcept } from "~/utils/conceptConversion"; -import { getLoggedInClient, getSupabaseContext } from "~/utils/supabaseContext"; const BlockRenderer = ({ uid }: { uid: string }) => { const containerRef = useRef(null); @@ -73,41 +69,6 @@ const DiscourseNodeSuggestiveRules = ({ [], ); - const [isUpdating, setIsUpdating] = useState(false); - - const handleUpdateEmbeddings = async () => { - setIsUpdating(true); - const blockNodesSince = getDiscourseNodeTypeWithSettingsBlockNodes(node, 0); - const supabaseClient = await getLoggedInClient(); - const context = await getSupabaseContext(); - if (context && blockNodesSince) { - await upsertNodesToSupabaseAsContentWithEmbeddings( - blockNodesSince, - supabaseClient, - context, - ); - const nodeBlockToLocalConcepts = blockNodesSince.map((node) => { - const localConcept = discourseNodeBlockToLocalConcept(context, { - nodeUid: node.source_local_id, - schemaUid: node.type, - text: node.node_title ? `${node.node_title} ${node.text}` : node.text, - }); - return localConcept; - }); - - const { error } = await supabaseClient.rpc("upsert_concepts", { - data: nodeBlockToLocalConcepts, - v_space_id: context.spaceId, - }); - if (error) { - throw new Error( - `upsert_concepts failed: ${JSON.stringify(error, null, 2)}`, - ); - } - } - setIsUpdating(false); - }; - return (
+ {/* TODO: Add a button to update embeddings in seperate PR */}
);