Skip to content

Eng 415 bulk content upsert function for sync #202

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 23, 2025
Merged

Eng 415 bulk content upsert function for sync #202

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions packages/database/doc/upsert_content.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
# A few examples of using upsert_content / upsert_document etc.

In general, for external references, you can either embed it inline, reference it by `X_local_id` (where local means the platform) or use the db_ids as in the tables.

```typescript
import type { LocalDocumentDataInput, LocalContentDataInput } from '@repo/database/input_types.ts';

const docs: LocalDocumentDataInput[] = [{ "source_local_id": "page1_uid", "created": "2000/01/01", "last_modified": "2001/01/02", author_local_id:"sR22zZ470dNPkIf9PpjQXXdTBjG2"}];
const contents: LocalContentDataInput[] = [
{
"author_local_id": "sR22zZ470dNPkIf9PpjQXXdTBjG2",
"author_inline": {
"account_local_id": "sR22zZ470dNPkIf9PpjQXXdTBjG2",
"name": "maparent"
},
"document_inline": {
"source_local_id": "page1_uid",
"created": "2000/01/01",
"last_modified": "2001/01/02",
"author_local_id": "sR22zZ470dNPkIf9PpjQXXdTBjG2"
},
"source_local_id": "a_roam_uid",
"scale": "document",
"created": "2000/01/01",
"last_modified": "2001/01/02",
"text": "Some text"
},
{ "author_local_id": "sR22zZ470dNPkIf9PpjQXXdTBjG2", "document_local_id":"page1_uid", "source_local_id": "a_roam_uid2", "scale":"document", "created": "2000/01/02", "last_modified": "2001/01/03", "part_of_local_id":"a_roam_uid", "text": "Some subtext" },
{
"author_local_id": "sR22zZ470dNPkIf9PpjQXXdTBjG2", "document_inline": docs[0], "source_local_id": "a_roam_uid3", "scale": "document", "created": "2000/01/02", "last_modified": "2001/01/03", "part_of_local_id": "a_roam_uid2", "text": "Some subsubtext", "embedding_inline": {
"model":"openai_text_embedding_3_small_1536", "vector":[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
} }
];

// optional preliminary step: upsert documents.
{
const { data, error } = await client.rpc("upsert_documents", { v_space_id: 12, data: docs as Json });
if (error) console.error(error);
console.log(data);
}

// upsert content
{
const { data, error } = await client.rpc("upsert_content", { v_space_id: 12, data: contents as Json, v_creator_id: 63 });
if (error) console.error(error);
console.log(data);
}

// more compact version if all content is known to also be a page
const page_contents: LocalContentDataInput[] = [
{ "author_local_id": "sR22zZ470dNPkIf9PpjQXXdTBjG2", "source_local_id": "a_page_uid2", "created": "2000/01/02", "last_modified": "2001/01/03", "text": "Some other page" },
];

{
const { data, error } = await client.rpc("upsert_content", { v_space_id: 12, data: page_contents as Json, v_creator_id: 63, content_as_document: true });
if (error) console.error(error);
console.log(data);
}

```
13 changes: 13 additions & 0 deletions packages/database/input_types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import { Database, TablesInsert } from "./types.gen.ts";
Copy link
Contributor

@mdroidian mdroidian Jun 23, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: Typescript files are generally camelCase. If you change it, also update the .md file that references it.

Copy link
Collaborator Author

@maparent maparent Jun 23, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah, missed that before merging. I'll make a small PR for this.


export type LocalDocumentDataInput = Partial<
Omit<Database['public']["CompositeTypes"]["document_local_input"], "author_inline">
& { author_inline: Partial<TablesInsert<"PlatformAccount">> }>;
export type LocalContentDataInput = Partial<
Omit<Database['public']["CompositeTypes"]["content_local_input"], "document_inline" | "author_inline" | "creator_inline">
& {
document_inline: LocalDocumentDataInput,
author_inline: Partial<TablesInsert<"PlatformAccount">>,
creator_inline: Partial<TablesInsert<"PlatformAccount">>
}>;
export type LocalConceptDataInput = Partial<Database['public']["CompositeTypes"]["concept_local_input"]>;
8 changes: 7 additions & 1 deletion packages/database/schema.puml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,12 +74,18 @@ class "AgentIdentifier" [[{An identifying attribute associated with an agent, ca
"Concept" --> "0..*" "PlatformAccount" : "contributors"
"Content" --> "1" "PlatformAccount" : "creator"
"Content" --> "1" "Document" : "document"
class "ContentEmbedding" [[{An embedding for a piece of content.}]] {
abstract "ContentEmbedding" [[{An embedding for a piece of content.}]] {
{field} model : EmbeddingName
{field} vector : vector
{field} obsolete : boolean
}
class "ContentEmbedding_openai_text_embedding_3_small_1536" [[{The table for the openai text_embedding_3_small model (1536 dimensions)}]] {
{field} model(i) : EmbeddingName
{field} vector(i) : vector
{field} obsolete(i) : boolean
}
"ContentEmbedding" --> "1" "Content" : "target"
"ContentEmbedding" ^-- "ContentEmbedding_openai_text_embedding_3_small_1536"
"Content" --> "0..1" "Content" : "part_of"
"Concept" --> "0..1" "Content" : "represented_by"
class "ConceptSchema" [[{A Concept that describes a schema (type) for other concepts}]] {
Expand Down
2 changes: 1 addition & 1 deletion packages/database/schema.svg
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
5 changes: 4 additions & 1 deletion packages/database/schema.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,7 @@ classes:
# - abstract
ContentEmbedding:
description: An embedding for a piece of content.
# abstract: true
abstract: true
attributes:
target:
range: Content
Expand All @@ -299,6 +299,9 @@ classes:
description: Whether this embedding is obsolete (becauses the Content was modified)
range: boolean
ifabsent: 'false'
ContentEmbedding_openai_text_embedding_3_small_1536:
is_a: ContentEmbedding
description: The table for the openai text_embedding_3_small model (1536 dimensions)
Concept:
description: An abstract concept, claim or relation
slots:
Expand Down
Loading