From 7537500721d796f581af7a855a5137dac7409b92 Mon Sep 17 00:00:00 2001
From: sid597 <sid597athens@gmail.com>
Date: Thu, 6 Nov 2025 00:15:17 +0530
Subject: [PATCH 1/2] Add duplicate node alert during nod creation via modal

---
 apps/roam/src/components/CreateNodeDialog.tsx |  94 ++++++++++-
 apps/roam/src/utils/hyde.ts                   | 151 ++++++++++++++++++
 2 files changed, 239 insertions(+), 6 deletions(-)

diff --git a/apps/roam/src/components/CreateNodeDialog.tsx b/apps/roam/src/components/CreateNodeDialog.tsx
index 4211f7c56..9e64d2e46 100644
--- a/apps/roam/src/components/CreateNodeDialog.tsx
+++ b/apps/roam/src/components/CreateNodeDialog.tsx
@@ -12,6 +12,8 @@ import getDiscourseNodes, {
 import { getNewDiscourseNodeText } from "~/utils/formatUtils";
 import MenuItemSelect from "roamjs-components/components/MenuItemSelect";
 import createBlock from "roamjs-components/writes/createBlock";
+import { findSimilarNodes, SuggestedNode } from "~/utils/hyde";
+import { Spinner } from "@blueprintjs/core";
 
 export type CreateNodeDialogProps = {
   onClose: () => void;
@@ -37,6 +39,9 @@ const CreateNodeDialog = ({
   const [selectedType, setSelectedType] =
     useState<DiscourseNode>(defaultNodeType);
   const [loading, setLoading] = useState(false);
+  const [suggestions, setSuggestions] = useState<SuggestedNode[]>([]);
+  const [suggestionsLoading, setSuggestionsLoading] = useState(false);
+  const [formattedTitle, setFormattedTitle] = useState("");
   const inputRef = useRef<HTMLInputElement>(null);
 
   useEffect(() => {
@@ -45,16 +50,53 @@ const CreateNodeDialog = ({
     }
   }, []);
 
+  useEffect(() => {
+    let isCancelled = false;
+    const compute = async () => {
+      const base = title.trim();
+      if (!base) {
+        setFormattedTitle("");
+        return;
+      }
+      const ft = await getNewDiscourseNodeText({
+        text: base,
+        nodeType: selectedType.type,
+        blockUid: sourceBlockUid,
+      });
+      if (!isCancelled) setFormattedTitle(ft || "");
+    };
+    void compute();
+    return () => {
+      isCancelled = true;
+    };
+  }, [title, selectedType.type, sourceBlockUid]);
+
+  useEffect(() => {
+    const fetchSuggestions = async () => {
+      if (formattedTitle.trim()) {
+        setSuggestionsLoading(true);
+        console.log(
+          "fetching suggestions for",
+          formattedTitle,
+          selectedType.type,
+        );
+        const similarNodes = await findSimilarNodes({
+          text: formattedTitle,
+          nodeType: selectedType.type,
+        });
+        setSuggestions(similarNodes);
+        setSuggestionsLoading(false);
+      } else {
+        setSuggestions([]);
+      }
+    };
+    void fetchSuggestions();
+  }, [formattedTitle, selectedType.type]);
+
   const onCreate = async () => {
     if (!title.trim()) return;
     setLoading(true);
 
-    const formattedTitle = await getNewDiscourseNodeText({
-      text: title.trim(),
-      nodeType: selectedType.type,
-      blockUid: sourceBlockUid,
-    });
-
     if (!formattedTitle) {
       setLoading(false);
       return;
@@ -138,6 +180,46 @@ const CreateNodeDialog = ({
             />
           </div>
 
+          {suggestionsLoading && (
+            <div className="flex items-center gap-2">
+              <Spinner size={16} />
+              <span>Fetching possible duplicates...</span>
+            </div>
+          )}
+          {suggestions.length > 0 && (
+            <div className="flex flex-col gap-1">
+              <h4 className="font-bold">Possible duplicates</h4>
+              <ul className="flex flex-col gap-1">
+                {suggestions.slice(0, 5).map((node) => (
+                  <li key={node.uid}>
+                    <a
+                      onClick={async () => {
+                        if (sourceBlockUid) {
+                          const pageRef = `[[${node.text}]]`;
+                          await updateBlock({
+                            uid: sourceBlockUid,
+                            text: pageRef,
+                          });
+                          await createBlock({
+                            parentUid: sourceBlockUid,
+                            order: 0,
+                            node: {
+                              text: initialTitle,
+                            },
+                          });
+                        }
+                        onClose();
+                      }}
+                      className="cursor-pointer text-blue-500 hover:underline"
+                    >
+                      {node.text}
+                    </a>
+                  </li>
+                ))}
+              </ul>
+            </div>
+          )}
+
           <Label>
             Type
             <MenuItemSelect
diff --git a/apps/roam/src/utils/hyde.ts b/apps/roam/src/utils/hyde.ts
index 361d38725..eaf682fc0 100644
--- a/apps/roam/src/utils/hyde.ts
+++ b/apps/roam/src/utils/hyde.ts
@@ -292,6 +292,79 @@ const rankNodes = ({
   return combinedResults.map((item) => item.node);
 };
 
+const filterAndRerankByLlm = async ({
+  originalText,
+  candidates,
+}: {
+  originalText: string;
+  candidates: SuggestedNode[];
+}): Promise<SuggestedNode[]> => {
+  if (candidates.length === 0) {
+    return [];
+  }
+
+  const candidateList = candidates
+    .map((c, i) => `${i + 1}. "${c.text}"`)
+    .join("\n");
+
+  const userPromptContent = `Given the original text for a new node: "${originalText}".
+
+Here is a list of existing nodes that might be duplicates:
+${candidateList}
+
+Your task is to identify which of these existing nodes are strong potential duplicates for the new node. A strong potential duplicate is one that covers the same core concepts or ideas as the new node, not just a partial or superficial match.
+
+Please return a JSON array of strings, containing the exact text of only the nodes you've identified as strong potential duplicates. The list should be ordered from the most likely duplicate to the least likely. If you think none of the candidates are strong duplicates, return an empty JSON array.
+
+For example, if you decide only candidates 3 and 1 are strong duplicates, and 3 is more likely, your response should be:
+["text of candidate 3", "text of candidate 1"]
+
+Only return the JSON array.`;
+
+  const requestBody = {
+    documents: [{ role: "user", content: userPromptContent }],
+    passphrase: "",
+    settings: {
+      model: API_CONFIG.LLM.MODEL,
+      maxTokens: 500,
+      temperature: 0.2,
+    },
+  };
+
+  let response: Response | null = null;
+  try {
+    const signal = AbortSignal.timeout(API_CONFIG.LLM.TIMEOUT_MS);
+    response = await fetch(API_CONFIG.LLM.URL, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify(requestBody),
+      signal,
+    });
+
+    if (!response.ok) {
+      await handleApiError(response, "LLM reranking");
+      return candidates;
+    }
+
+    const responseText = await response.text();
+    const rerankedTextList = JSON.parse(responseText) as string[];
+
+    const originalCandidatesMap = new Map(candidates.map((c) => [c.text, c]));
+    const rerankedNodes = rerankedTextList
+      .map((text) => originalCandidatesMap.get(text))
+      .filter((node): node is SuggestedNode => !!node);
+
+    console.log("rerankedNodes", rerankedNodes);
+
+    return rerankedNodes;
+  } catch (error: unknown) {
+    console.error("LLM reranking failed:", error);
+    return candidates;
+  }
+};
+
 export const findSimilarNodesUsingHyde = async ({
   candidateNodes,
   currentNodeText,
@@ -530,3 +603,81 @@ export const performHydeSearch = async ({
   }
   return [];
 };
+
+export const findSimilarNodes = async ({
+  text,
+  nodeType,
+}: {
+  text: string;
+  nodeType: string;
+}): Promise<SuggestedNode[]> => {
+  if (!text.trim() || !nodeType) {
+    return [];
+  }
+
+  try {
+    const context = await getSupabaseContext();
+    if (!context) return [];
+    const supabase = await getLoggedInClient();
+    const { spaceId } = context;
+    if (!supabase) return [];
+
+    const candidateNodesForHyde = (
+      await getNodesByType({
+        supabase,
+        spaceId,
+        fields: { content: ["source_local_id", "text"] },
+        ofTypes: [nodeType],
+        pagination: { limit: 10000 },
+      })
+    )
+      .map((c) => {
+        const node = findDiscourseNode(c.Content?.source_local_id || "");
+        return {
+          uid: c.Content?.source_local_id || "",
+          text: c.Content?.text || "",
+          type: node ? node.type : "",
+        };
+      })
+      .filter((n) => n.uid && n.text && n.type);
+
+    if (candidateNodesForHyde.length === 0) {
+      return [];
+    }
+
+    const queryEmbedding = await createEmbedding(text);
+    const searchResults = await searchEmbeddings({
+      queryEmbedding,
+      indexData: candidateNodesForHyde,
+    });
+    console.log("searchResults", searchResults);
+
+    const nodeMap = new Map<string, CandidateNodeWithEmbedding>(
+      candidateNodesForHyde.map((node) => [node.uid, node]),
+    );
+
+    const combinedResults: { node: SuggestedNode; score: number }[] = [];
+    searchResults.forEach((result) => {
+      const fullNode = nodeMap.get(result.object.uid);
+      if (fullNode) {
+        combinedResults.push({ node: fullNode, score: result.score });
+      }
+    });
+
+    combinedResults.sort((a, b) => b.score - a.score);
+    const top5Candidates = combinedResults.slice(0, 7).map((item) => item.node);
+
+    if (top5Candidates.length === 0) {
+      return [];
+    }
+    console.log("top5Candidates", top5Candidates);
+
+    return await filterAndRerankByLlm({
+      originalText: text,
+      candidates: top5Candidates,
+    });
+  } catch (error) {
+    console.error("Error finding similar nodes:", error);
+    return [];
+  }
+};

From b5752693ea090afb03f26416792cad2c5ed68afd Mon Sep 17 00:00:00 2001
From: sid597 <sid597athens@gmail.com>
Date: Thu, 6 Nov 2025 21:13:29 +0530
Subject: [PATCH 2/2] show both llm and normal semantic searched queries

---
 apps/roam/src/components/CreateNodeDialog.tsx | 74 +++++++++++++------
 apps/roam/src/utils/hyde.ts                   | 27 ++++---
 2 files changed, 66 insertions(+), 35 deletions(-)

diff --git a/apps/roam/src/components/CreateNodeDialog.tsx b/apps/roam/src/components/CreateNodeDialog.tsx
index 9e64d2e46..32e35da67 100644
--- a/apps/roam/src/components/CreateNodeDialog.tsx
+++ b/apps/roam/src/components/CreateNodeDialog.tsx
@@ -39,7 +39,10 @@ const CreateNodeDialog = ({
   const [selectedType, setSelectedType] =
     useState<DiscourseNode>(defaultNodeType);
   const [loading, setLoading] = useState(false);
-  const [suggestions, setSuggestions] = useState<SuggestedNode[]>([]);
+  const [filteredSuggestions, setFilteredSuggestions] = useState<
+    SuggestedNode[]
+  >([]);
+  const [rawSuggestions, setRawSuggestions] = useState<SuggestedNode[]>([]);
   const [suggestionsLoading, setSuggestionsLoading] = useState(false);
   const [formattedTitle, setFormattedTitle] = useState("");
   const inputRef = useRef<HTMLInputElement>(null);
@@ -80,14 +83,16 @@ const CreateNodeDialog = ({
           formattedTitle,
           selectedType.type,
         );
-        const similarNodes = await findSimilarNodes({
+        const { raw, filtered } = await findSimilarNodes({
           text: formattedTitle,
           nodeType: selectedType.type,
         });
-        setSuggestions(similarNodes);
+        setRawSuggestions(raw);
+        setFilteredSuggestions(filtered);
         setSuggestionsLoading(false);
       } else {
-        setSuggestions([]);
+        setRawSuggestions([]);
+        setFilteredSuggestions([]);
       }
     };
     void fetchSuggestions();
@@ -161,6 +166,24 @@ const CreateNodeDialog = ({
     onClose();
   };
 
+  const handleSuggestionClick = async (node: SuggestedNode) => {
+    if (sourceBlockUid) {
+      const pageRef = `[[${node.text}]]`;
+      await updateBlock({
+        uid: sourceBlockUid,
+        text: pageRef,
+      });
+      await createBlock({
+        parentUid: sourceBlockUid,
+        order: 0,
+        node: {
+          text: initialTitle,
+        },
+      });
+    }
+    onClose();
+  };
+
   return (
     <Dialog
       isOpen={true}
@@ -186,29 +209,34 @@ const CreateNodeDialog = ({
               <span>Fetching possible duplicates...</span>
             </div>
           )}
-          {suggestions.length > 0 && (
+          {rawSuggestions.length > 0 && (
+            <div className="flex flex-col gap-1">
+              <h4 className="font-bold">Possible duplicates (Semantic)</h4>
+              <ul className="flex flex-col gap-1">
+                {rawSuggestions.map((node) => (
+                  <li key={node.uid}>
+                    <a
+                      onClick={() => {
+                        void handleSuggestionClick(node);
+                      }}
+                      className="cursor-pointer text-blue-500 hover:underline"
+                    >
+                      {node.text}
+                    </a>
+                  </li>
+                ))}
+              </ul>
+            </div>
+          )}
+          {filteredSuggestions.length > 0 && (
             <div className="flex flex-col gap-1">
-              <h4 className="font-bold">Possible duplicates</h4>
+              <h4 className="font-bold">Possible duplicates (LLM Filtered)</h4>
               <ul className="flex flex-col gap-1">
-                {suggestions.slice(0, 5).map((node) => (
+                {filteredSuggestions.map((node) => (
                   <li key={node.uid}>
                     <a
-                      onClick={async () => {
-                        if (sourceBlockUid) {
-                          const pageRef = `[[${node.text}]]`;
-                          await updateBlock({
-                            uid: sourceBlockUid,
-                            text: pageRef,
-                          });
-                          await createBlock({
-                            parentUid: sourceBlockUid,
-                            order: 0,
-                            node: {
-                              text: initialTitle,
-                            },
-                          });
-                        }
-                        onClose();
+                      onClick={() => {
+                        void handleSuggestionClick(node);
                       }}
                       className="cursor-pointer text-blue-500 hover:underline"
                     >
diff --git a/apps/roam/src/utils/hyde.ts b/apps/roam/src/utils/hyde.ts
index eaf682fc0..ebac4f29d 100644
--- a/apps/roam/src/utils/hyde.ts
+++ b/apps/roam/src/utils/hyde.ts
@@ -610,17 +610,18 @@ export const findSimilarNodes = async ({
 }: {
   text: string;
   nodeType: string;
-}): Promise<SuggestedNode[]> => {
+}): Promise<{ raw: SuggestedNode[]; filtered: SuggestedNode[] }> => {
+  const emptyResult = { raw: [], filtered: [] };
   if (!text.trim() || !nodeType) {
-    return [];
+    return emptyResult;
   }
 
   try {
     const context = await getSupabaseContext();
-    if (!context) return [];
+    if (!context) return emptyResult;
     const supabase = await getLoggedInClient();
     const { spaceId } = context;
-    if (!supabase) return [];
+    if (!supabase) return emptyResult;
 
     const candidateNodesForHyde = (
       await getNodesByType({
@@ -642,7 +643,7 @@ export const findSimilarNodes = async ({
       .filter((n) => n.uid && n.text && n.type);
 
     if (candidateNodesForHyde.length === 0) {
-      return [];
+      return emptyResult;
     }
 
     const queryEmbedding = await createEmbedding(text);
@@ -665,19 +666,21 @@ export const findSimilarNodes = async ({
     });
 
     combinedResults.sort((a, b) => b.score - a.score);
-    const top5Candidates = combinedResults.slice(0, 7).map((item) => item.node);
+    const topCandidates = combinedResults.slice(0, 7).map((item) => item.node);
 
-    if (top5Candidates.length === 0) {
-      return [];
+    if (topCandidates.length === 0) {
+      return emptyResult;
     }
-    console.log("top5Candidates", top5Candidates);
+    console.log("topCandidates", topCandidates);
 
-    return await filterAndRerankByLlm({
+    const filteredResults = await filterAndRerankByLlm({
       originalText: text,
-      candidates: top5Candidates,
+      candidates: topCandidates,
     });
+
+    return { raw: topCandidates, filtered: filteredResults };
   } catch (error) {
     console.error("Error finding similar nodes:", error);
-    return [];
+    return { raw: [], filtered: [] };
   }
 };