ENG-431 Separate literal and references in concept data

packages/database/schema.puml

@@ -2,7 +2,7 @@
 skinparam nodesep 10
 hide circle
 hide empty members
-abstract "PlatformAccount" [[{An account for an agent on a platform}]] {
+class "PlatformAccount" [[{An account for an agent on a platform}]] {
     {field} id : integer  
     {field} name : string  
     {field} platform : Platform  
@@ -53,7 +53,8 @@ class "Concept" [[{An abstract concept, claim or relation}]] {
     {field} created : datetime  
     {field} last_modified : datetime  
     {field} arity : integer  
-    {field} content : JSON  
+    {field} reference_content : JSON  
+    {field} literal_content : JSON  
     {field} is_schema : boolean  
 }
 "Content" --> "0..1" "Space" : "space"
@@ -89,7 +90,8 @@ class "ConceptSchema" [[{A Concept that describes a schema (type) for other conc
     {field} created(i) : datetime  
     {field} last_modified(i) : datetime  
     {field} arity(i) : integer  
-    {field} content(i) : JSON  
+    {field} reference_content(i) : JSON  
+    {field} literal_content(i) : JSON  
     {field} is_schema(i) : boolean  
 }
 "Concept" --> "1" "ConceptSchema" : "schema"

packages/database/schema.yaml

@@ -320,9 +320,16 @@ classes:
       schema:
         range: ConceptSchema
         required: true
-      content:
+      reference_content:
+        description: "Aspects of the concept that reference other concepts. `{[key: string]: number|number[]}`"
         range: JSON
         required: true
+        ifabsent: '{}'
+      literal_content:
+        range: JSON
+        required: true
+        description: "Aspects of the concept that have literal values. `{[key: string]: any}`"
+        ifabsent: '{}'
       is_schema:
         range: boolean
         required: true

packages/database/supabase/migrations/20250609140958_concept_indexing.sql

@@ -0,0 +1,16 @@
+CREATE OR REPLACE FUNCTION extract_references(refs JSONB) RETURNS BIGINT [] LANGUAGE SQL IMMUTABLE AS $$
+  SELECT COALESCE(array_agg(i::bigint), '{}') FROM (SELECT jsonb_array_elements(jsonb_path_query_array(refs, '$.*[*]')) i) exrefs;
+$$;
+
+ALTER TABLE PUBLIC."Concept" RENAME COLUMN CONTENT TO LITERAL_CONTENT;
+DROP INDEX "Concept_content";
+CREATE INDEX CONCEPT_LITERAL_CONTENT_IDX ON PUBLIC."Concept" USING GIN (
+    LITERAL_CONTENT JSONB_OPS
+);
+
+ALTER TABLE PUBLIC."Concept" ADD COLUMN REFERENCE_CONTENT JSONB NOT NULL DEFAULT '{}'::JSONB;
+
+ALTER TABLE PUBLIC."Concept" ADD COLUMN REFS BIGINT [] NOT NULL GENERATED ALWAYS AS (extract_references(REFERENCE_CONTENT)) STORED;
+
+CREATE INDEX CONCEPT_REFS_IDX ON PUBLIC."Concept" USING GIN (REFS);
+

packages/database/supabase/schemas/concept.sql

@@ -12,6 +12,9 @@ CREATE TYPE public."EpistemicStatus" AS ENUM (
 
 ALTER TYPE public."EpistemicStatus" OWNER TO postgres;
 
+CREATE OR REPLACE FUNCTION extract_references(refs JSONB) RETURNS BIGINT [] LANGUAGE sql IMMUTABLE AS $$
+  SELECT COALESCE(array_agg(i::bigint), '{}') FROM (SELECT jsonb_array_elements(jsonb_path_query_array(refs, '$.*[*]')) i) exrefs;
+$$;
 
 CREATE TABLE IF NOT EXISTS public."Concept" (
     id bigint DEFAULT nextval(
@@ -26,7 +29,9 @@ CREATE TABLE IF NOT EXISTS public."Concept" (
     space_id bigint NOT NULL,
     arity smallint DEFAULT 0 NOT NULL,
     schema_id bigint,
-    content jsonb DEFAULT '{}'::jsonb NOT NULL,
+    literal_content jsonb NOT NULL DEFAULT '{}'::jsonb,
+    reference_content jsonb NOT NULL DEFAULT '{}'::jsonb,
+    refs BIGINT [] NOT NULL GENERATED ALWAYS AS (extract_references(reference_content)) STORED,
     is_schema boolean DEFAULT false NOT NULL,
     represented_by_id bigint
 );
@@ -52,10 +57,12 @@ ADD FOREIGN KEY (represented_by_id) REFERENCES public."Content" (
     id
 ) ON DELETE SET NULL ON UPDATE CASCADE;
 
-CREATE INDEX "Concept_content" ON public."Concept" USING gin (
-    content jsonb_path_ops
+CREATE INDEX concept_literal_content_idx ON public."Concept" USING gin (
+    literal_content jsonb_ops
 );
 
+CREATE INDEX concept_refs_idx ON public."Concept" USING gin (refs);
+
 CREATE INDEX "Concept_schema" ON public."Concept" USING btree (schema_id);
 
 CREATE INDEX "Concept_space" ON public."Concept" USING btree (space_id);
@@ -64,6 +71,7 @@ CREATE UNIQUE INDEX "Concept_represented_by" ON public."Concept" (
     represented_by_id
 );
 
+-- maybe make that for schemas only?
 CREATE UNIQUE INDEX concept_space_and_name_idx ON public."Concept" (space_id, name);
 
 

packages/database/types.gen.ts

@@ -74,44 +74,50 @@ export type Database = {
         Row: {
           arity: number
           author_id: number | null
-          content: Json
           created: string
           description: string | null
           epistemic_status: Database["public"]["Enums"]["EpistemicStatus"]
           id: number
           is_schema: boolean
           last_modified: string
+          literal_content: Json
           name: string
+          reference_content: Json
+          refs: number[]
           represented_by_id: number | null
           schema_id: number | null
           space_id: number
         }
         Insert: {
           arity?: number
           author_id?: number | null
-          content?: Json
           created: string
           description?: string | null
           epistemic_status?: Database["public"]["Enums"]["EpistemicStatus"]
           id?: number
           is_schema?: boolean
           last_modified: string
+          literal_content?: Json
           name: string
+          reference_content?: Json
+          refs?: number[]
           represented_by_id?: number | null
           schema_id?: number | null
           space_id: number
         }
         Update: {
           arity?: number
           author_id?: number | null
-          content?: Json
           created?: string
           description?: string | null
           epistemic_status?: Database["public"]["Enums"]["EpistemicStatus"]
           id?: number
           is_schema?: boolean
           last_modified?: string
+          literal_content?: Json
           name?: string
+          reference_content?: Json
+          refs?: number[]
           represented_by_id?: number | null
           schema_id?: number | null
           space_id?: number
@@ -527,6 +533,10 @@ export type Database = {
         }
         Returns: undefined
       }
+      extract_references: {
+        Args: { refs: Json }
+        Returns: number[]
+      }
       get_nodes_needing_sync: {
         Args: { nodes_from_roam: Json }
         Returns: {