Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
138 commits
Select commit Hold shift + click to select a range
079af15
init
mdroidian Apr 15, 2025
e4c0cc5
exclude nodes already suggested in results
mdroidian Apr 16, 2025
0f9ad1c
add api endpoint
sid597 May 6, 2025
8d0cf3e
address coderabbit review
sid597 May 6, 2025
349a0cb
hyde utility
sid597 May 6, 2025
7c4685c
address review
sid597 May 6, 2025
7c0e97d
Merge branch 'add-embedding-endpoint' into hyde-utility
sid597 May 6, 2025
eaa9c8f
redundant code
sid597 May 6, 2025
8e7a928
Merge branch 'add-embedding-endpoint' into hyde-utility
sid597 May 6, 2025
0033fb1
address coderabbit review
sid597 May 6, 2025
6e5c6c8
initial migration from bigger base
sid597 May 6, 2025
fbcc36c
gemini self review
sid597 May 6, 2025
2cc7e23
fx
sid597 May 7, 2025
7054186
use arrow functions
sid597 May 8, 2025
1be7a55
build off the existing type
sid597 May 8, 2025
7060255
named parameters
sid597 May 8, 2025
4687390
insert working getting error since it works
sid597 May 9, 2025
7af1386
cors
sid597 May 9, 2025
079b7c2
insert discoursePlatform
sid597 May 9, 2025
d171298
api for inserting discourseSpace
sid597 May 9, 2025
d822647
person api
sid597 May 9, 2025
5c0ed3f
contentEmbedding route
sid597 May 9, 2025
205074c
contentEmbedding route
sid597 May 9, 2025
709aa57
cmts
sid597 May 9, 2025
848a11c
Merge branch 'hyde-utility' into store-in-supabase
sid597 May 9, 2025
7147ffd
get all dg nodes, create embeddings
sid597 May 9, 2025
842501d
hack to get all discourse nodes
sid597 May 11, 2025
292c87d
fix cors
sid597 May 11, 2025
bd038f6
add cors
sid597 May 11, 2025
6a20994
more routes based on the dependency
sid597 May 11, 2025
fce6d3c
agents working
sid597 May 12, 2025
4bd158a
account working
sid597 May 12, 2025
1c53e01
every api before embedding is working
sid597 May 12, 2025
8cc474d
content
sid597 May 12, 2025
713f9c9
full but bugs
sid597 May 12, 2025
995c895
document
sid597 May 12, 2025
be0d4bc
full emdedding process
sid597 May 12, 2025
7816506
all routes working but embedding one
sid597 May 12, 2025
3c97861
cors embedding
sid597 May 12, 2025
ff3aa6b
ALL DONE, now can upsert all the embeddings for a new graph
sid597 May 12, 2025
7544e95
created supabase pgvector functions for subset nodes sorted return, r…
sid597 May 13, 2025
4b531fc
batch
sid597 May 13, 2025
f8c98c3
batch insert
sid597 May 13, 2025
3e42a29
rpc route for matching the subset nodes, use openi since anthropic is…
sid597 May 13, 2025
c79ea69
increased batch size, batching for insert
sid597 May 13, 2025
59724d8
show all results
sid597 May 13, 2025
788f137
updated url to local deployment
sid597 May 15, 2025
11ca52f
base url
sid597 May 15, 2025
d2c46a5
fixt chat url
sid597 May 15, 2025
24babfe
return next respones
sid597 May 15, 2025
814446d
last_synced got removed
sid597 May 15, 2025
04d880e
last_synced got removed
sid597 May 15, 2025
aa31c23
remove represents_id
sid597 May 15, 2025
3ea57c4
make batch size small got content too large error
sid597 May 15, 2025
1a32003
even smaller batch size
sid597 May 15, 2025
6c167a3
400 batch size
sid597 May 15, 2025
1258c2e
300
sid597 May 15, 2025
90a8cb6
stick to 200
sid597 May 15, 2025
87abb21
moving out
sid597 May 28, 2025
e0cde7b
Merge branch 'main' into merge-main-to-store-in-supabase
sid597 May 28, 2025
9564409
so many changes its confusing now, should merge and work off marc-ant…
sid597 May 28, 2025
657b3bb
rpc for upserting content and content-embedding
sid597 May 29, 2025
14b2014
all working now new function, ulsin sync table
sid597 May 30, 2025
6463ef6
undo changes not compaitable with base
sid597 May 30, 2025
852a1c6
fix build, lets see
sid597 May 30, 2025
3fb9375
fix package lock
sid597 May 30, 2025
03c7fc3
fix for supabase vercel
sid597 May 30, 2025
2eb8c3a
fix the base url
sid597 May 30, 2025
41692b4
merge:
sid597 May 30, 2025
dfb463f
clickable
sid597 May 30, 2025
66366b5
add all pages
sid597 May 30, 2025
22fa5a0
seperate sugestion blocks, reset input, only search on button click, …
sid597 May 30, 2025
2679e4b
filter, fix visuals, bugs
sid597 May 30, 2025
46aa973
fir route
sid597 May 31, 2025
d3350ff
done, more modification
sid597 May 31, 2025
cb90778
button
sid597 Jun 14, 2025
576ffbd
sidebar
sid597 Jun 14, 2025
6011dc0
working same as now
sid597 Jun 15, 2025
e10fd06
highlight
sid597 Jun 17, 2025
fe4f8cc
3 views for the suggestive mode
sid597 Jun 17, 2025
be68945
inline works
sid597 Jun 19, 2025
abc3f77
Merge branch 'main' into merge-main-to-alpha
sid597 Jun 22, 2025
8676d7e
fix first time upload
sid597 Jun 22, 2025
e357aaf
cache, fix highlight, hover flivker
sid597 Jun 22, 2025
96efe8c
fix bug
sid597 Jun 22, 2025
4e62510
fix type
sid597 Jun 23, 2025
9ab1f49
fix type
sid597 Jun 23, 2025
ca128d5
fix inline add bug issue, fix ui, remove logs
sid597 Jun 23, 2025
1582b2b
fix createclient
sid597 Jun 23, 2025
7bc21d0
include linked refs and and target node data as context by default
sid597 Jun 26, 2025
722248c
add groups for default pages, fix text, seperate group of setting
sid597 Jun 26, 2025
611a3dd
select groups and extract pages needed from each group
sid597 Jun 26, 2025
bcd7045
seperate section with headers, use template to extract context we wan…
sid597 Jun 30, 2025
9f3af37
Merge branch 'main' into eng-356-want-to-define-node-specific-rules-f…
sid597 Jun 30, 2025
c457fea
extract node-specific embedding
sid597 Jun 30, 2025
dd8e06a
upload new content with marking
sid597 Jul 1, 2025
1a2ad28
Merge branch 'main' into eng-356-want-to-define-node-specific-rules-f…
sid597 Jul 1, 2025
fb5e732
rpc route for node-specific matching
sid597 Jul 2, 2025
5809d3b
Merge branch 'main' into eng-356-want-to-define-node-specific-rules-f…
sid597 Jul 3, 2025
40d70b5
pull migrations locally, test lcoally
sid597 Jul 3, 2025
5da00c5
working full flow
sid597 Jul 5, 2025
fe0d937
prelim
sid597 Jul 7, 2025
82bb758
dev route
sid597 Jul 10, 2025
d71c432
Merge branch 'main' into eng-472-update-callers-of-temp-alpha-functions
sid597 Jul 10, 2025
70e1be4
using upsert_content and upsert_document
sid597 Jul 10, 2025
f5f0089
consolidate updating new content, use propose sync info for creating …
sid597 Jul 12, 2025
a2d021e
logs and migration
sid597 Jul 12, 2025
b66e800
concept upsert working
sid597 Jul 13, 2025
361e418
finally concepts working, first have to create content for the corres…
sid597 Jul 13, 2025
aa869a0
delete and fetch only based on discourse context instead of content
sid597 Jul 13, 2025
5e262c1
increase batch size, add comments, seperate update and irst time logi…
sid597 Jul 14, 2025
4ab7905
Merge branch 'main' into store-in-supabase
sid597 Jul 14, 2025
4aaab69
fix deploy errors
sid597 Jul 14, 2025
87fbcab
Merge branch 'store-in-supabase' into use-concept-table
sid597 Jul 14, 2025
c6b687c
moving out fixed logic but the process.env not working
sid597 Jul 16, 2025
25ff2e1
content uploading working now for both documetn and block tye, tooooo…
sid597 Jul 17, 2025
3668857
finally all is working from initial load and button click upload pov,…
sid597 Jul 18, 2025
6f42029
modify match embeddings to now use block to get the data otherwise fa…
sid597 Jul 18, 2025
c31e7d4
delete content from content as well, preciously only document and the…
sid597 Jul 18, 2025
eb2292b
use match embeddings intsead of calculating on client
sid597 Jul 18, 2025
5ccd270
content, concept,content,concept, bug, fix, content, concept, bug, ba…
sid597 Jul 18, 2025
9f7dd30
simplified content extraaction logic
sid597 Jul 19, 2025
36ef373
Merge branch 'main' into store-in-supabase
sid597 Jul 20, 2025
aa380cd
using variants
sid597 Jul 20, 2025
1c5d687
reverse embedding schema
sid597 Jul 20, 2025
9a95d9c
fix nodetypesince
sid597 Jul 20, 2025
7f45abb
Merge branch 'main' into store-in-supabase
sid597 Jul 25, 2025
13352ab
Merge branch 'main' into store-in-supabase
sid597 Jul 28, 2025
63980e5
fix bug
sid597 Jul 28, 2025
86e9ace
add first to stack
sid597 Jul 28, 2025
6abf000
moving out
sid597 Jul 28, 2025
4ee6898
Working now: async all pages, observer for width of main window when …
sid597 Jul 29, 2025
4b9817a
fix some build bugs
sid597 Jul 29, 2025
b6c122d
bug fixes
sid597 Aug 2, 2025
dec9a78
fix bug was not uploading content not sure why, on load check if exists
sid597 Aug 3, 2025
4e4df5b
delete orghaned
sid597 Aug 3, 2025
bf018fe
Merge branch 'main' into store-in-supabase
sid597 Aug 4, 2025
49deba6
Merge branch 'main' into store-in-supabase
sid597 Aug 4, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/roam/scripts/dev.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import esbuild from "esbuild";
import dotenv from "dotenv";
import { compile, args } from "./compile";

dotenv.config();
dotenv.config({ path: ".env.local" });

const dev = () => {
process.env.NODE_ENV = process.env.NODE_ENV || "development";
Expand Down
197 changes: 113 additions & 84 deletions apps/roam/src/components/DiscourseContextOverlay.tsx
Original file line number Diff line number Diff line change
@@ -1,20 +1,18 @@
import { Button, Icon, Popover, Position, Tooltip } from "@blueprintjs/core";
import React, { useCallback, useEffect, useMemo, useState } from "react";
import React, { useMemo } from "react";
import ReactDOM from "react-dom";
import { ContextContent } from "./DiscourseContext";
import useInViewport from "react-in-viewport/dist/es/lib/useInViewport";
import normalizePageTitle from "roamjs-components/queries/normalizePageTitle";
import deriveDiscourseNodeAttribute from "~/utils/deriveDiscourseNodeAttribute";
import getSettingValueFromTree from "roamjs-components/util/getSettingValueFromTree";
import getBasicTreeByParentUid from "roamjs-components/queries/getBasicTreeByParentUid";
import nanoid from "nanoid";
import getPageUidByPageTitle from "roamjs-components/queries/getPageUidByPageTitle";
import getDiscourseContextResults from "~/utils/getDiscourseContextResults";
import findDiscourseNode from "~/utils/findDiscourseNode";
import getDiscourseNodes from "~/utils/getDiscourseNodes";
import getDiscourseRelations from "~/utils/getDiscourseRelations";
import ExtensionApiContextProvider from "roamjs-components/components/ExtensionApiContext";
import { OnloadArgs } from "roamjs-components/types/native";
import createBlock from "roamjs-components/writes/createBlock";
import { getBlockUidFromTarget } from "roamjs-components/dom";
import { DiscourseSuggestionsPanel } from "./DiscourseSuggestionsPanel";
import useSuggestionsDisplaySettings from "~/utils/useSuggestionsDisplayMode";
import SuggestionsBody from "./SuggestionsBody";
import { useDiscourseData } from "~/utils/useDiscourseData";

type DiscourseData = {
results: Awaited<ReturnType<typeof getDiscourseContextResults>>;
Expand All @@ -25,75 +23,34 @@ const cache: {
[tag: string]: DiscourseData;
} = {};

const getOverlayInfo = async (tag: string): Promise<DiscourseData> => {
try {
if (cache[tag]) return cache[tag];
const DiscourseContextOverlay = ({
tag,
id,
parentEl,
onloadArgs,
}: {
tag: string;
id: string;
parentEl: HTMLElement;
onloadArgs: OnloadArgs;
}) => {
const blockUid = useMemo(() => getBlockUidFromTarget(parentEl), [parentEl]);

const relations = getDiscourseRelations();
const nodes = getDiscourseNodes(relations);
const { loading, results, refs, score, tagUid } = useDiscourseData(tag);

const [results, refs] = await Promise.all([
getDiscourseContextResults({
uid: getPageUidByPageTitle(tag),
nodes,
relations,
}),
// @ts-ignore - backend to be added to roamjs-components
window.roamAlphaAPI.data.backend.q(
`[:find ?a :where [?b :node/title "${normalizePageTitle(tag)}"] [?a :block/refs ?b]]`,
),
]);
// Determine how suggestions should be displayed
const {
split: splitEnabled,
overlay: overlayEnabled,
inline: inlineEnabled,
} = useSuggestionsDisplaySettings();

return (cache[tag] = {
results,
refs: refs.length,
});
} catch (error) {
console.error(`Error getting overlay info for ${tag}:`, error);
return {
results: [],
refs: 0,
};
}
};
const toggleHighlight = (uid: string, on: boolean) => {
document
.querySelectorAll(`[data-dg-block-uid="${uid}"]`)
.forEach((el) => el.classList.toggle("dg-highlight", on));
};

const DiscourseContextOverlay = ({ tag, id }: { tag: string; id: string }) => {
const tagUid = useMemo(() => getPageUidByPageTitle(tag), [tag]);
const [loading, setLoading] = useState(true);
const [results, setResults] = useState<DiscourseData["results"]>([]);
const [refs, setRefs] = useState(0);
const [score, setScore] = useState<number | string>(0);
const getInfo = useCallback(
() =>
getOverlayInfo(tag)
.then(({ refs, results }) => {
const discourseNode = findDiscourseNode(tagUid);
if (discourseNode) {
const attribute = getSettingValueFromTree({
tree: getBasicTreeByParentUid(discourseNode.type),
key: "Overlay",
defaultValue: "Overlay",
});
return deriveDiscourseNodeAttribute({
uid: tagUid,
attribute,
}).then((score) => {
setResults(results);
setRefs(refs);
setScore(score);
});
}
})
.finally(() => setLoading(false)),
[tag, setResults, setLoading, setRefs, setScore],
);
const refresh = useCallback(() => {
setLoading(true);
getInfo();
}, [getInfo, setLoading]);
useEffect(() => {
getInfo();
}, [refresh, getInfo]);
return (
<Popover
autoFocus={false}
Expand All @@ -104,15 +61,25 @@ const DiscourseContextOverlay = ({ tag, id }: { tag: string; id: string }) => {
}`}
>
<ContextContent uid={tagUid} results={results} />
{overlayEnabled && (
<div className="mt-4 border-t pt-4">
<SuggestionsBody
tag={tag}
blockUid={blockUid}
existingResults={results}
/>
</div>
)}
</div>
}
target={
<Button
small
id={id}
className={`roamjs-discourse-context-overlay ${
loading ? "animate-pulse" : ""
}`}
{...{ "data-dg-block-uid": blockUid }}
onMouseEnter={() => toggleHighlight(blockUid, true)}
onMouseLeave={() => toggleHighlight(blockUid, false)}
className={"roamjs-discourse-context-overlay"}
style={{
minHeight: "initial",
paddingTop: ".25rem",
Expand All @@ -123,9 +90,58 @@ const DiscourseContextOverlay = ({ tag, id }: { tag: string; id: string }) => {
>
<div className="flex items-center gap-1.5">
<Icon icon={"diagram-tree"} />
<span className="mr-1 leading-none">{loading ? "-" : score}</span>
<span className="mr-1 leading-none">{score}</span>
<Icon icon={"link"} />
<span className="leading-none">{loading ? "-" : refs}</span>
<span className="leading-none">{refs}</span>
{splitEnabled && (
<Tooltip
content="Open suggestions panel"
hoverOpenDelay={200}
hoverCloseDelay={0}
position={Position.RIGHT}
>
<Button
icon="panel-stats"
minimal
small
onClick={(e) => {
e.stopPropagation();
e.preventDefault();
DiscourseSuggestionsPanel.toggle(
tag,
id,
parentEl,
onloadArgs,
);
}}
/>
</Tooltip>
)}
{inlineEnabled && (
<Tooltip
content="Insert inline suggestions"
hoverOpenDelay={200}
position={Position.RIGHT}
>
<Button
icon="insert"
minimal
small
onClick={async (e) => {
e.stopPropagation();
e.preventDefault();
await createBlock({
parentUid: blockUid,
order: Infinity,
node: {
text: `{{inline-suggestive-mode}}`,
children: [{ text: tag }],
},
});
}}
/>
</Tooltip>
)}
</div>
</Button>
}
Expand All @@ -134,7 +150,15 @@ const DiscourseContextOverlay = ({ tag, id }: { tag: string; id: string }) => {
);
};

const Wrapper = ({ parent, tag }: { parent: HTMLElement; tag: string }) => {
const Wrapper = ({
parent,
tag,
onloadArgs,
}: {
parent: HTMLElement;
tag: string;
onloadArgs: OnloadArgs;
}) => {
const id = useMemo(() => nanoid(), []);
const { inViewport } = useInViewport(
{ current: parent },
Expand All @@ -143,7 +167,12 @@ const Wrapper = ({ parent, tag }: { parent: HTMLElement; tag: string }) => {
{},
);
return inViewport ? (
<DiscourseContextOverlay tag={tag} id={id} />
<DiscourseContextOverlay
tag={tag}
id={id}
parentEl={parent}
onloadArgs={onloadArgs}
/>
) : (
<Button
small
Expand All @@ -154,9 +183,9 @@ const Wrapper = ({ parent, tag }: { parent: HTMLElement; tag: string }) => {
>
<div className="flex items-center gap-1.5">
<Icon icon={"diagram-tree"} />
<span className="mr-1">-</span>
<span className="mr-1">0</span>
<Icon icon={"link"} />
<span>-</span>
<span>0</span>
</div>
</Button>
);
Expand All @@ -175,7 +204,7 @@ export const render = ({
parent.onmousedown = (e) => e.stopPropagation();
ReactDOM.render(
<ExtensionApiContextProvider {...onloadArgs}>
<Wrapper tag={tag} parent={parent} />
<Wrapper tag={tag} parent={parent} onloadArgs={onloadArgs} />
</ExtensionApiContextProvider>,
parent,
);
Expand Down
Loading