From 499055dad3c911eff07123fbef76185e1f393a7e Mon Sep 17 00:00:00 2001 From: Michael Gartner Date: Mon, 13 Oct 2025 09:31:18 -0600 Subject: [PATCH 1/7] WIP --- .../src/components/settings/AdminPanel.tsx | 6 +- .../features/step-definitions/stepdefs.ts | 7 +- packages/database/src/lib/queries.ts | 768 +++++++++++++++++- 3 files changed, 765 insertions(+), 16 deletions(-) diff --git a/apps/roam/src/components/settings/AdminPanel.tsx b/apps/roam/src/components/settings/AdminPanel.tsx index 96ef9506a..4b637e7f6 100644 --- a/apps/roam/src/components/settings/AdminPanel.tsx +++ b/apps/roam/src/components/settings/AdminPanel.tsx @@ -7,7 +7,7 @@ import { SupabaseContext, } from "~/utils/supabaseContext"; import { - getConcepts, + getNodesByType, getSchemaConcepts, nodeSchemaSignature, type NodeSignature, @@ -157,10 +157,10 @@ const AdminPanel = () => { try { setLoadingNodes(true); setNodes( - await getConcepts({ + await getNodesByType({ supabase, spaceId, - schemaLocalIds: showingSchema.sourceLocalId, + nodeTypes: [showingSchema.sourceLocalId], }), ); } catch (e) { diff --git a/packages/database/features/step-definitions/stepdefs.ts b/packages/database/features/step-definitions/stepdefs.ts index c7c8bb903..555fb9761 100644 --- a/packages/database/features/step-definitions/stepdefs.ts +++ b/packages/database/features/step-definitions/stepdefs.ts @@ -9,7 +9,10 @@ import { type Json, } from "@repo/database/dbTypes"; import { getVariant, config } from "@repo/database/dbDotEnv"; -import { getConcepts, initNodeSchemaCache } from "@repo/database/lib/queries"; +import { + getConceptsLegacy, + initNodeSchemaCache, +} from "@repo/database/lib/queries"; import { spaceAnonUserEmail, @@ -358,7 +361,7 @@ Given( if (spaceId === undefined) assert.fail("spaceId"); const supabase = await getLoggedinDatabase(spaceId); // note that we supply spaceId and supabase, they do not need to be part of the incoming Json - const nodes = await getConcepts({ ...params, supabase, spaceId }); + const nodes = await getConceptsLegacy({ ...params, supabase, spaceId }); nodes.sort((a, b) => a.id! - b.id!); world.queryResults = nodes; }, diff --git a/packages/database/src/lib/queries.ts b/packages/database/src/lib/queries.ts index aa1f3921e..e02dc33c4 100644 --- a/packages/database/src/lib/queries.ts +++ b/packages/database/src/lib/queries.ts @@ -2,7 +2,46 @@ import { PostgrestResponse } from "@supabase/supabase-js"; import type { Tables } from "../dbTypes"; import { DGSupabaseClient } from "./client"; -// the functions you are most likely to use are getSchemaConcepts and getConcepts. +// Query API Overview: +// +// NEW API (Recommended): +// - getConcepts(): Main function with grouped parameters for better DX +// - Primitive functions: getAllNodes, getNodesByType, getAllRelations, etc. +// - Always includes metadata (created/edited user/times) by default +// +// LEGACY API (Backward compatibility): +// - getConceptsLegacy(): Original function with 17 parameters +// +// Examples: +// +// // Query all nodes of a specific type +// const nodes = await getNodesByType({ +// supabase, spaceId, nodeTypes: ["my-node-type"] +// }); +// +// // Query all relations containing a specific node +// const relations = await getRelationsContainingNode({ +// supabase, spaceId, nodeIds: ["node-123"] +// }); +// +// // Query nodes in relations of specific types +// const nodes = await getNodesInRelations({ +// supabase, spaceId, relationTypes: ["cites", "references"] +// }); +// +// // Get discourse context (all nodes connected to a node) +// const context = await getDiscourseContext({ +// supabase, spaceId, nodeId: "node-123" +// }); +// +// // Use the main getConcepts function for complex queries +// const results = await getConcepts({ +// supabase, spaceId, +// scope: { type: 'nodes', nodeTypes: ['type1', 'type2'] }, +// relations: { ofType: ['cites'], authoredBy: 'user123' }, +// author: 'creator456', +// pagination: { limit: 50, offset: 0 } +// }); type Concept = Tables<"Concept">; type Content = Tables<"Content">; @@ -310,6 +349,22 @@ const getLocalToDbIdMapping = async ( return dbIds; }; +/** + * Default concept fields that include metadata (created/edited user and times). + * These fields are returned by default for all queries unless overridden. + * + * @example + * ```typescript + * // Use default fields + * const nodes = await getAllNodes({ supabase }); + * + * // Override with custom fields + * const nodes = await getAllNodes({ + * supabase, + * fields: { concepts: CONCEPT_FIELDS_MINIMAL } + * }); + * ``` + */ export const CONCEPT_FIELDS: (keyof Concept)[] = [ "id", "name", @@ -327,6 +382,28 @@ export const CONCEPT_FIELDS: (keyof Concept)[] = [ "represented_by_id", ]; +/** + * Minimal concept fields for lightweight queries. + * Includes only essential fields: id, name, space_id, author_id, created, last_modified. + * + * @example + * ```typescript + * // Use minimal fields for performance + * const nodes = await getAllNodes({ + * supabase, + * fields: { concepts: CONCEPT_FIELDS_MINIMAL } + * }); + * ``` + */ +export const CONCEPT_FIELDS_MINIMAL: (keyof Concept)[] = [ + "id", + "name", + "space_id", + "author_id", + "created", + "last_modified", +]; + export const CONTENT_FIELDS: (keyof Content)[] = [ "id", "source_local_id", @@ -355,16 +432,685 @@ export const DOCUMENT_FIELDS: (keyof Document)[] = [ // instrumentation for benchmarking export const LAST_QUERY_DATA = { duration: 0 }; -// Main entry point to query Concepts and related data: -// related sub-objects can be provided as: -// Content, Content.Document, author (PlatformAccount), relations (Concept), -// relations.subnodes (Concept), relations.subnodes.author, relations.subnodes.Content -// Which fields of these subobjects are fetched is controlled by the respective Fields parameters -// (except the last two, which would have just enough data for query filters.) -// If the fields are empty, the sub-object will not be fetched (unless needed for matching query parameters) -// Any parameter called "local" expects platform Ids (source_local_id) of the corresponding Content. -// In the case of node/relation definitions, schema refers to the page Id of the definition. -export const getConcepts = async ({ +/** + * Defines what type of concepts to query and any specific constraints. + * + * @example + * ```typescript + * // Query all nodes of specific types + * { type: "nodes", nodeTypes: ["page", "note"] } + * + * // Query all relations + * { type: "relations" } + * + * // Query specific nodes by their local IDs + * { type: "specific", nodeIds: ["node-123", "node-456"] } + * ``` + */ +export type QueryScope = { + /** The type of concepts to retrieve */ + type: "all" | "nodes" | "relations" | "schemas" | "specific"; + /** + * Schema local IDs to filter by (e.g., ["page", "note", "relation"]). + * Only used when type is "nodes" or "relations". + */ + nodeTypes?: string[]; + /** + * Specific node local IDs to retrieve. + * Only used when type is "specific". + */ + nodeIds?: string[]; +}; + +/** + * Filters for querying concepts based on their relationships. + * All filters are optional and can be combined. + * + * @example + * ```typescript + * // Find relations of type "cites" containing specific nodes + * { + * ofType: ["cites", "references"], + * containingNodes: ["node-123", "node-456"] + * } + * + * // Find concepts connected to nodes of specific types + * { + * toNodeTypes: ["page", "note"] + * } + * ``` + */ +export type RelationFilters = { + /** Find relations containing this specific node (single node) */ + containingNode?: string; + /** Find relations containing any of these nodes (multiple nodes) */ + containingNodes?: string[]; + /** Find concepts participating in relations of these types */ + ofType?: string[]; + /** Find concepts connected to nodes of these types */ + toNodeTypes?: string[]; + /** Find concepts in relations authored by this user */ + authoredBy?: string; +}; + +/** + * Controls which fields are returned in the response. + * Each field array specifies which columns to fetch from the respective table. + * If a field array is empty or undefined, that data won't be fetched. + * + * @example + * ```typescript + * // Get minimal concept data with full content + * { + * concepts: ["id", "name", "created"], + * content: ["source_local_id", "text", "metadata"] + * } + * ``` + */ +export type FieldSelection = { + /** Fields to return from the Concept table */ + concepts?: (keyof Concept)[]; + /** Fields to return from the Content table */ + content?: (keyof Content)[]; + /** Fields to return from the Document table */ + documents?: (keyof Document)[]; + /** Fields to return for relation concepts */ + relations?: (keyof Concept)[]; + /** Fields to return for nodes in relations */ + relationNodes?: (keyof Concept)[]; +}; + +/** + * Pagination options for controlling result set size and offset. + * + * @example + * ```typescript + * // Get first 50 results + * { limit: 50 } + * + * // Get next 50 results (pagination) + * { limit: 50, offset: 50 } + * ``` + */ +export type PaginationOptions = { + /** Maximum number of results to return (default: 100) */ + limit?: number; + /** Number of results to skip (default: 0) */ + offset?: number; +}; + +/** + * Main parameters for querying concepts with the new grouped API. + * Provides better developer experience with logical parameter grouping. + * + * @example + * ```typescript + * const results = await getConcepts({ + * supabase, + * spaceId: 123, + * scope: { type: "nodes", nodeTypes: ["page"] }, + * relations: { ofType: ["cites"] }, + * author: "user123", + * pagination: { limit: 50 } + * }); + * ``` + */ +export type GetConceptsParams = { + /** Supabase client instance (must be authenticated) */ + supabase: DGSupabaseClient; + /** Space ID to query within (optional, uses client's default space) */ + spaceId?: number; + + /** What type of concepts to query and any constraints */ + scope: QueryScope; + + /** Optional filters based on relationships */ + relations?: RelationFilters; + + /** Filter results by the author who created the concepts */ + author?: string; + + /** Control which fields are returned in the response */ + fields?: FieldSelection; + + /** Pagination options for result set control */ + pagination?: PaginationOptions; +}; + +// Primitive query functions for common use cases +// These provide a simpler API for the most common query patterns + +/** + * Retrieves all discourse nodes (non-relation concepts) in a space. + * + * @param params - Query parameters + * @param params.supabase - Authenticated Supabase client + * @param params.spaceId - Space ID to query (optional) + * @param params.author - Filter by author (optional) + * @param params.fields - Fields to return (defaults to full concept + content) + * @param params.pagination - Pagination options (defaults to limit 100) + * @returns Promise resolving to array of concept objects with full metadata + * + * @example + * ```typescript + * // Get all nodes in the space + * const allNodes = await getAllNodes({ supabase, spaceId: 123 }); + * + * // Get nodes by a specific author + * const myNodes = await getAllNodes({ + * supabase, + * spaceId: 123, + * author: "user123" + * }); + * ``` + */ +export const getAllNodes = async ({ + supabase, + spaceId, + author, + fields = { concepts: CONCEPT_FIELDS, content: CONTENT_FIELDS }, + pagination = { limit: 100 }, +}: { + supabase: DGSupabaseClient; + spaceId?: number; + author?: string; + fields?: FieldSelection; + pagination?: PaginationOptions; +}): Promise => { + return getConceptsNew({ + supabase, + spaceId, + scope: { type: "nodes" }, + author, + fields, + pagination, + }); +}; + +/** + * Retrieves all discourse nodes of specific types. + * + * @param params - Query parameters + * @param params.supabase - Authenticated Supabase client + * @param params.spaceId - Space ID to query (optional) + * @param params.nodeTypes - Array of node type local IDs to filter by + * @param params.author - Filter by author (optional) + * @param params.fields - Fields to return (defaults to full concept + content) + * @param params.pagination - Pagination options (defaults to limit 100) + * @returns Promise resolving to array of concept objects with full metadata + * + * @example + * ```typescript + * // Get all pages and notes + * const nodes = await getNodesByType({ + * supabase, + * spaceId: 123, + * nodeTypes: ["page", "note"] + * }); + * + * // Get pages by a specific author + * const myPages = await getNodesByType({ + * supabase, + * spaceId: 123, + * nodeTypes: ["page"], + * author: "user123" + * }); + * ``` + */ +export const getNodesByType = async ({ + supabase, + spaceId, + nodeTypes, + author, + fields = { concepts: CONCEPT_FIELDS, content: CONTENT_FIELDS }, + pagination = { limit: 100 }, +}: { + supabase: DGSupabaseClient; + spaceId?: number; + nodeTypes: string[]; + author?: string; + fields?: FieldSelection; + pagination?: PaginationOptions; +}): Promise => { + return getConceptsNew({ + supabase, + spaceId, + scope: { type: "nodes", nodeTypes }, + author, + fields, + pagination, + }); +}; + +/** + * Retrieves all discourse relations (concepts with arity > 0) in a space. + * + * @param params - Query parameters + * @param params.supabase - Authenticated Supabase client + * @param params.spaceId - Space ID to query (optional) + * @param params.author - Filter by author (optional) + * @param params.fields - Fields to return (defaults to full concept + content) + * @param params.pagination - Pagination options (defaults to limit 100) + * @returns Promise resolving to array of relation concept objects with full metadata + * + * @example + * ```typescript + * // Get all relations in the space + * const relations = await getAllRelations({ supabase, spaceId: 123 }); + * + * // Get relations by a specific author + * const myRelations = await getAllRelations({ + * supabase, + * spaceId: 123, + * author: "user123" + * }); + * ``` + */ +export const getAllRelations = async ({ + supabase, + spaceId, + author, + fields = { concepts: CONCEPT_FIELDS, content: CONTENT_FIELDS }, + pagination = { limit: 100 }, +}: { + supabase: DGSupabaseClient; + spaceId?: number; + author?: string; + fields?: FieldSelection; + pagination?: PaginationOptions; +}): Promise => { + return getConceptsNew({ + supabase, + spaceId, + scope: { type: "relations" }, + author, + fields, + pagination, + }); +}; + +/** + * Retrieves all relations that contain nodes of specific types. + * + * @param params - Query parameters + * @param params.supabase - Authenticated Supabase client + * @param params.spaceId - Space ID to query (optional) + * @param params.nodeTypes - Array of node type local IDs that must be in the relations + * @param params.relationTypes - Optional array of relation types to filter by + * @param params.authoredBy - Optional filter by relation author + * @param params.fields - Fields to return (defaults to full concept + content) + * @param params.pagination - Pagination options (defaults to limit 100) + * @returns Promise resolving to array of relation concept objects + * + * @example + * ```typescript + * // Find all relations containing page nodes + * const relations = await getRelationsContainingNodeType({ + * supabase, + * spaceId: 123, + * nodeTypes: ["page"] + * }); + * + * // Find citation relations containing note nodes + * const citations = await getRelationsContainingNodeType({ + * supabase, + * spaceId: 123, + * nodeTypes: ["note"], + * relationTypes: ["cites"] + * }); + * ``` + */ +export const getRelationsContainingNodeType = async ({ + supabase, + spaceId, + nodeTypes, + relationTypes, + authoredBy, + fields = { concepts: CONCEPT_FIELDS, content: CONTENT_FIELDS }, + pagination = { limit: 100 }, +}: { + supabase: DGSupabaseClient; + spaceId?: number; + nodeTypes: string[]; + relationTypes?: string[]; + authoredBy?: string; + fields?: FieldSelection; + pagination?: PaginationOptions; +}): Promise => { + return getConceptsNew({ + supabase, + spaceId, + scope: { type: "relations" }, + relations: { + toNodeTypes: nodeTypes, + ofType: relationTypes, + authoredBy, + }, + fields, + pagination, + }); +}; + +/** + * Retrieves all relations that contain specific nodes. + * + * @param params - Query parameters + * @param params.supabase - Authenticated Supabase client + * @param params.spaceId - Space ID to query (optional) + * @param params.nodeIds - Array of specific node local IDs that must be in the relations + * @param params.relationTypes - Optional array of relation types to filter by + * @param params.authoredBy - Optional filter by relation author + * @param params.fields - Fields to return (defaults to full concept + content) + * @param params.pagination - Pagination options (defaults to limit 100) + * @returns Promise resolving to array of relation concept objects + * + * @example + * ```typescript + * // Find all relations containing specific nodes + * const relations = await getRelationsContainingNode({ + * supabase, + * spaceId: 123, + * nodeIds: ["node-123", "node-456"] + * }); + * + * // Find citation relations containing a specific node + * const citations = await getRelationsContainingNode({ + * supabase, + * spaceId: 123, + * nodeIds: ["node-123"], + * relationTypes: ["cites"] + * }); + * ``` + */ +export const getRelationsContainingNode = async ({ + supabase, + spaceId, + nodeIds, + relationTypes, + authoredBy, + fields = { concepts: CONCEPT_FIELDS, content: CONTENT_FIELDS }, + pagination = { limit: 100 }, +}: { + supabase: DGSupabaseClient; + spaceId?: number; + nodeIds: string[]; + relationTypes?: string[]; + authoredBy?: string; + fields?: FieldSelection; + pagination?: PaginationOptions; +}): Promise => { + return getConceptsNew({ + supabase, + spaceId, + scope: { type: "relations" }, + relations: { + containingNodes: nodeIds, + ofType: relationTypes, + authoredBy, + }, + fields, + pagination, + }); +}; + +/** + * Retrieves nodes that participate in relations of specific types. + * + * @param params - Query parameters + * @param params.supabase - Authenticated Supabase client + * @param params.spaceId - Space ID to query (optional) + * @param params.relationTypes - Array of relation types to filter by + * @param params.toNodes - Optional array of node local IDs that must be connected to + * @param params.authoredBy - Optional filter by relation author + * @param params.author - Optional filter by node author + * @param params.fields - Fields to return (defaults to full concept + content) + * @param params.pagination - Pagination options (defaults to limit 100) + * @returns Promise resolving to array of node concept objects + * + * @example + * ```typescript + * // Find all nodes that are cited + * const citedNodes = await getNodesInRelations({ + * supabase, + * spaceId: 123, + * relationTypes: ["cites"] + * }); + * + * // Find nodes that cite specific other nodes + * const citingNodes = await getNodesInRelations({ + * supabase, + * spaceId: 123, + * relationTypes: ["cites"], + * toNodes: ["node-123", "node-456"] + * }); + * ``` + */ +export const getNodesInRelations = async ({ + supabase, + spaceId, + relationTypes, + toNodes, + authoredBy, + author, + fields = { concepts: CONCEPT_FIELDS, content: CONTENT_FIELDS }, + pagination = { limit: 100 }, +}: { + supabase: DGSupabaseClient; + spaceId?: number; + relationTypes: string[]; + toNodes?: string[]; + authoredBy?: string; + author?: string; + fields?: FieldSelection; + pagination?: PaginationOptions; +}): Promise => { + return getConceptsNew({ + supabase, + spaceId, + scope: { type: "nodes" }, + relations: { + ofType: relationTypes, + containingNodes: toNodes, + authoredBy, + }, + author, + fields, + pagination, + }); +}; + +/** + * Retrieves the discourse context of a node - all nodes and relations connected to it. + * This is useful for understanding the full context around a specific concept. + * + * @param params - Query parameters + * @param params.supabase - Authenticated Supabase client + * @param params.spaceId - Space ID to query (optional) + * @param params.nodeId - Local ID of the node to get context for + * @param params.fields - Fields to return (defaults to full concept + content + minimal relations) + * @param params.pagination - Pagination options (defaults to limit 100) + * @returns Promise resolving to array of concept objects including relations + * + * @example + * ```typescript + * // Get full context around a node + * const context = await getDiscourseContext({ + * supabase, + * spaceId: 123, + * nodeId: "node-123" + * }); + * + * // The result will include: + * // - The target node itself + * // - All nodes connected to it via relations + * // - The relation information connecting them + * ``` + */ +export const getDiscourseContext = async ({ + supabase, + spaceId, + nodeId, + fields = { + concepts: CONCEPT_FIELDS, + content: CONTENT_FIELDS, + relations: CONCEPT_FIELDS_MINIMAL, + }, + pagination = { limit: 100 }, +}: { + supabase: DGSupabaseClient; + spaceId?: number; + nodeId: string; + fields?: FieldSelection; + pagination?: PaginationOptions; +}): Promise => { + return getConceptsNew({ + supabase, + spaceId, + scope: { type: "all" }, + relations: { + containingNode: nodeId, + }, + fields, + pagination, + }); +}; + +/** + * Internal implementation of getConcepts with grouped parameters. + * Converts the new API to legacy parameters and delegates to getConceptsLegacy. + * + * @internal + */ +export const getConceptsNew = async ({ + supabase, + spaceId, + scope, + relations, + author, + fields = { concepts: CONCEPT_FIELDS, content: CONTENT_FIELDS }, + pagination = { limit: 100, offset: 0 }, +}: GetConceptsParams): Promise => { + // Convert new API to old API parameters + const oldParams: Parameters[0] = { + supabase, + spaceId, + nodeAuthor: author, + conceptFields: fields.concepts, + contentFields: fields.content, + documentFields: fields.documents, + relationFields: fields.relations, + relationSubNodesFields: fields.relationNodes, + limit: pagination.limit, + offset: pagination.offset, + }; + + // Map scope to old parameters + switch (scope.type) { + case "all": + oldParams.schemaLocalIds = []; + oldParams.fetchNodes = null; + break; + case "nodes": + oldParams.schemaLocalIds = scope.nodeTypes || []; + oldParams.fetchNodes = true; + break; + case "relations": + oldParams.schemaLocalIds = []; + oldParams.fetchNodes = false; + break; + case "schemas": + oldParams.schemaLocalIds = NODE_SCHEMAS; + oldParams.fetchNodes = null; + break; + case "specific": + oldParams.baseNodeLocalIds = scope.nodeIds || []; + oldParams.schemaLocalIds = []; + oldParams.fetchNodes = null; + break; + } + + // Map relation filters to old parameters + if (relations) { + if (relations.ofType) { + oldParams.inRelsOfTypeLocal = relations.ofType; + } + if (relations.toNodeTypes) { + oldParams.inRelsToNodesOfTypeLocal = relations.toNodeTypes; + } + if (relations.authoredBy) { + oldParams.inRelsToNodesOfAuthor = relations.authoredBy; + } + if (relations.containingNode) { + oldParams.inRelsToNodeLocalIds = [relations.containingNode]; + } + if (relations.containingNodes) { + oldParams.inRelsToNodeLocalIds = relations.containingNodes; + } + } + + return getConceptsLegacy(oldParams); +}; + +/** + * Main function for querying concepts with the new grouped parameter API. + * Provides better developer experience with logical parameter grouping and comprehensive IntelliSense. + * + * @param params - Query parameters with grouped structure + * @returns Promise resolving to array of concept objects with full metadata + * + * @example + * ```typescript + * // Query nodes of specific types with relation filters + * const results = await getConcepts({ + * supabase, + * spaceId: 123, + * scope: { type: "nodes", nodeTypes: ["page", "note"] }, + * relations: { ofType: ["cites", "references"] }, + * author: "user123", + * pagination: { limit: 50, offset: 0 } + * }); + * + * // Query relations containing specific nodes + * const relations = await getConcepts({ + * supabase, + * spaceId: 123, + * scope: { type: "relations" }, + * relations: { containingNodes: ["node-123", "node-456"] } + * }); + * ``` + */ +export const getConcepts = async ( + params: GetConceptsParams, +): Promise => { + return getConceptsNew(params); +}; + +/** + * Legacy getConcepts function with the original 17-parameter API. + * Kept for backward compatibility. Consider using the new getConcepts() or primitive functions instead. + * + * @deprecated Use the new getConcepts() function with grouped parameters for better DX + * @param params - Legacy parameters with individual fields + * @returns Promise resolving to array of concept objects with full metadata + * + * @example + * ```typescript + * // Legacy usage (not recommended) + * const results = await getConceptsLegacy({ + * supabase, + * spaceId: 123, + * schemaLocalIds: ["page", "note"], + * fetchNodes: true, + * nodeAuthor: "user123", + * inRelsOfTypeLocal: ["cites"], + * conceptFields: ["id", "name", "created"], + * contentFields: ["source_local_id", "text"], + * limit: 100, + * offset: 0 + * }); + * ``` + */ +export const getConceptsLegacy = async ({ supabase, // An instance of a logged-in client spaceId, // the numeric id of the space being queried baseNodeLocalIds = [], // If we are specifying the Concepts being queried directly. From 62b91174884f5ac5d7c3ec23f182a1dc09d6cb74 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Thu, 16 Oct 2025 13:19:33 -0400 Subject: [PATCH 2/7] Pushed author into scope. Removed unneeded slow functions. Pushed grouping into query builder funcition --- .../src/components/settings/AdminPanel.tsx | 12 +- .../database/features/queryConcepts.feature | 16 +- packages/database/src/lib/queries.ts | 980 ++++++------------ 3 files changed, 359 insertions(+), 649 deletions(-) diff --git a/apps/roam/src/components/settings/AdminPanel.tsx b/apps/roam/src/components/settings/AdminPanel.tsx index 4b637e7f6..03b90435d 100644 --- a/apps/roam/src/components/settings/AdminPanel.tsx +++ b/apps/roam/src/components/settings/AdminPanel.tsx @@ -7,7 +7,7 @@ import { SupabaseContext, } from "~/utils/supabaseContext"; import { - getNodesByType, + getConcepts, getSchemaConcepts, nodeSchemaSignature, type NodeSignature, @@ -157,10 +157,16 @@ const AdminPanel = () => { try { setLoadingNodes(true); setNodes( - await getNodesByType({ + await getConcepts({ supabase, spaceId, - nodeTypes: [showingSchema.sourceLocalId], + scope: { + schemas: + showingSchema.sourceLocalId === + nodeSchemaSignature.sourceLocalId, + type: "nodes", + ofType: [showingSchema.sourceLocalId], + }, }), ); } catch (e) { diff --git a/packages/database/features/queryConcepts.feature b/packages/database/features/queryConcepts.feature index 15e3b5ffa..9e1b04aff 100644 --- a/packages/database/features/queryConcepts.feature +++ b/packages/database/features/queryConcepts.feature @@ -48,7 +48,7 @@ Feature: Concept queries | c9 | opposes 2 | s1 | user2 | 2025/01/01 | 2025/01/01 | false | c5 | {} | {"target": "c8", "source": "c2"} | Scenario Outline: Query all nodes - And a user logged in space s1 and calling getConcepts with these parameters: '{"schemaLocalIds":[],"fetchNodes":null}' + And a user logged in space s1 and calling getConcepts with these parameters: '{"scope":{"type":"all"}}' Then query results should look like this | _id | name | _space_id | _author_id | @is_schema | _schema_id | @_reference_content | | c2 | claim 1 | s1 | user1 | false | c1 | {} | @@ -59,7 +59,7 @@ Feature: Concept queries | c9 | opposes 2 | s1 | user2 | false | c5 | {"target": "c8", "source": "c2"} | Scenario Outline: Query node schemas - And a user logged in space s1 and calling getConcepts with these parameters: '{"fetchNodes":null}' + And a user logged in space s1 and calling getConcepts with these parameters: '{"scope":{"schemas":true}}' Then query results should look like this | _id | name | _space_id | _author_id | @is_schema | _schema_id | @literal_content | @reference_content | _represented_by_id | | c1 | Claim | s1 | user1 | true | | {} | {} | ct1 | @@ -67,7 +67,7 @@ Feature: Concept queries | c7 | Hypothesis | s1 | user1 | true | | {} | {} | ct7 | Scenario Outline: Query by node types - And a user logged in space s1 and calling getConcepts with these parameters: '{"schemaLocalIds":["lct1"]}' + And a user logged in space s1 and calling getConcepts with these parameters: '{"scope":{"ofType":["lct1"]}}' Then query results should look like this | _id | name | _space_id | _author_id | @is_schema | _schema_id | @literal_content | @reference_content | | c2 | claim 1 | s1 | user1 | false | c1 | {} | {} | @@ -75,7 +75,7 @@ Feature: Concept queries | c4 | claim 3 | s1 | user3 | false | c1 | {} | {} | Scenario Outline: Query by author - And a user logged in space s1 and calling getConcepts with these parameters: '{"nodeAuthor":"user2","schemaLocalIds":[],"fetchNodes":null}' + And a user logged in space s1 and calling getConcepts with these parameters: '{"scope":{"author":"user2","type":"all"}}' Then query results should look like this | _id | name | _space_id | _author_id | @is_schema | _schema_id | @literal_content | @_reference_content | | c3 | claim 2 | s1 | user2 | false | c1 | {} | {} | @@ -83,7 +83,7 @@ Feature: Concept queries | c9 | opposes 2 | s1 | user2 | false | c5 | {} | {"target": "c8", "source": "c2"} | Scenario Outline: Query by relation type - And a user logged in space s1 and calling getConcepts with these parameters: '{"inRelsOfTypeLocal":["lct5"],"schemaLocalIds":[]}' + And a user logged in space s1 and calling getConcepts with these parameters: '{"relations":{"ofType":["lct5"]}}' Then query results should look like this | _id | name | _space_id | _author_id | @is_schema | _schema_id | @literal_content | @_reference_content | | c2 | claim 1 | s1 | user1 | false | c1 | {} | {} | @@ -91,7 +91,7 @@ Feature: Concept queries | c8 | hypothesis 1 | s1 | user3 | false | c7 | {} | {} | Scenario Outline: Query by related node type - And a user logged in space s1 and calling getConcepts with these parameters: '{"inRelsToNodesOfTypeLocal":["lct7"],"schemaLocalIds":[]}' + And a user logged in space s1 and calling getConcepts with these parameters: '{"relations":{"toNodeType":["lct7"]}}' Then query results should look like this | _id | name | _space_id | _author_id | @is_schema | _schema_id | @literal_content | @_reference_content | | c2 | claim 1 | s1 | user1 | false | c1 | {} | {} | @@ -99,14 +99,14 @@ Feature: Concept queries # Note that the node is related to itself, unfortunate but hard to solve. Scenario Outline: Query by author of related node - And a user logged in space s1 and calling getConcepts with these parameters: '{"schemaLocalIds":[],"inRelsToNodesOfAuthor":"user3","relationFields":["id"],"relationSubNodesFields":["id"]}' + And a user logged in space s1 and calling getConcepts with these parameters: '{"relations":{"author":"user3"},"fields":{"relations":["id"],"relationNodes":["id"]}}' Then query results should look like this | _id | name | _space_id | _author_id | @is_schema | _schema_id | @literal_content | @_reference_content | | c2 | claim 1 | s1 | user1 | false | c1 | {} | {} | | c8 | hypothesis 1 | s1 | user3 | false | c7 | {} | {} | Scenario Outline: Query by related node - And a user logged in space s1 and calling getConcepts with these parameters: '{"schemaLocalIds":[],"inRelsToNodeLocalIds":["lct2"]}' + And a user logged in space s1 and calling getConcepts with these parameters: '{"relations":{"toNodeId":["lct2"]}}' Then query results should look like this | _id | name | _space_id | _author_id | @is_schema | _schema_id | @literal_content | @_reference_content | | c2 | claim 1 | s1 | user1 | false | c1 | {} | {} | diff --git a/packages/database/src/lib/queries.ts b/packages/database/src/lib/queries.ts index e02dc33c4..db372dbc4 100644 --- a/packages/database/src/lib/queries.ts +++ b/packages/database/src/lib/queries.ts @@ -4,41 +4,38 @@ import { DGSupabaseClient } from "./client"; // Query API Overview: // -// NEW API (Recommended): -// - getConcepts(): Main function with grouped parameters for better DX -// - Primitive functions: getAllNodes, getNodesByType, getAllRelations, etc. +// - getConcepts(): Generic API which handles many combinations of filters +// - Single-filter functions: getAllNodes, getNodesByType, getAllRelations, etc. +// Helps understand each filter in isolation. // - Always includes metadata (created/edited user/times) by default // -// LEGACY API (Backward compatibility): -// - getConceptsLegacy(): Original function with 17 parameters -// // Examples: // // // Query all nodes of a specific type // const nodes = await getNodesByType({ -// supabase, spaceId, nodeTypes: ["my-node-type"] +// supabase, spaceId, ofType: ["my-node-type"] // }); // -// // Query all relations containing a specific node -// const relations = await getRelationsContainingNode({ +// // Query a specific node and its relations +// const relations = await getRelationsFromNode({ // supabase, spaceId, nodeIds: ["node-123"] // }); // -// // Query nodes in relations of specific types -// const nodes = await getNodesInRelations({ -// supabase, spaceId, relationTypes: ["cites", "references"] +// // Query nodes of specific types and their (filtered) relations +// const nodes = await getRelationsFromNodeType({ +// supabase, spaceId, ofType: ["claim"], relationTypes: ["cites", "references"] // }); // -// // Get discourse context (all nodes connected to a node) +// // Get discourse context (all nodes connected to a node or set of nodes) // const context = await getDiscourseContext({ -// supabase, spaceId, nodeId: "node-123" +// supabase, spaceId, nodeIds: ["node-123"] // }); // // // Use the main getConcepts function for complex queries // const results = await getConcepts({ // supabase, spaceId, -// scope: { type: 'nodes', nodeTypes: ['type1', 'type2'] }, -// relations: { ofType: ['cites'], authoredBy: 'user123' }, +// scope: { type: 'nodes', ofType: ['type1', 'type2'] }, +// relations: { ofType: ['cites'], author: 'user123' }, // author: 'creator456', // pagination: { limit: 50, offset: 0 } // }); @@ -104,62 +101,202 @@ type DefaultQueryShape = { }; /* eslint-enable @typescript-eslint/naming-convention */ +/** + * Defines what type of concepts to query and any specific constraints. + * + * @example + * ```typescript + * // Query all nodes of specific types + * { type: "nodes", ofSchemaLocal: ["page", "note"] } + * + * // Query all relations + * { type: "relations" } + * + * // Query specific nodes by their local IDs + * { type: "nodes", nodeIds: ["node-123", "node-456"] } + * + * // Query relation schemas + * { type: "relations", schemas: true } + * ``` + */ +export type NodeFilters = { + /** The type of concept frames to retrieve */ + type: "all" | "nodes" | "relations"; + /* Whether we are retrieving schemas or instances */ + schemas?: boolean; + /** + * Retrieve instances of those types (aka schemas) only. + * Given as a list of local Ids (eg Roam page Ids of the node or relation types) + * Only used when schemas=false. + */ + ofType?: string[]; + /** + * Specific node local IDs to retrieve. + */ + nodeIds?: string[]; + + /** Filter results by the author who created the concepts. Use the local Id for the author. */ + author?: string; +}; + +type NodeFiltersDb = Omit & { ofType?: number[]}; + + +/** + * Filters for querying concepts based on their relationships. + * All filters are optional and can be combined. + * + * @example + * ```typescript + * // Find relations of type "cites" containing specific nodes + * { + * ofType: ["cites", "references"], + * toNodeId: ["node-123", "node-456"] + * } + * + * // Find concepts connected to nodes of specific types + * { + * toNodeType: ["page", "note"] + * } + * ``` + */ +export type RelationFilters = { + /** Find relations containing any of these nodes (multiple nodes) */ + /* SLOW. Avoid using unless you have strong constraints on base node of search */ + toNodeId?: string[]; + /** Find concepts participating in relations of these types */ + ofType?: string[]; + /** Find concepts connected to nodes of these types. */ + toNodeType?: string[]; + /** Find concepts in relations authored by this user */ + /* SLOW. Avoid using unless you have strong constraints on base node of search */ + author?: string; +}; + +export type RelationFiltersDb = Omit&{ofType?: number[], toNodeType?: number[]}; + +/** + * Controls which fields are returned in the response. + * Each field array specifies which columns to fetch from the respective table. + * If a field array is empty or undefined, that data won't be fetched. + * + * @example + * ```typescript + * // Get minimal concept data with full content + * { + * concepts: ["id", "name", "created"], + * content: ["source_local_id", "text", "metadata"] + * } + * ``` + */ +export type FieldSelection = { + /** Fields to return from the Concept table */ + concepts?: (keyof Concept)[]; + /** Fields to return from the Content table */ + content?: (keyof Content)[]; + /** Fields to return from the Document table */ + documents?: (keyof Document)[]; + /** Fields to return for relation concepts */ + relations?: (keyof Concept)[]; + /** Fields to return for nodes in relations */ + relationNodes?: (keyof Concept)[]; +}; + +/** + * Pagination options for controlling result set size and offset. + * + * @example + * ```typescript + * // Get first 50 results + * { limit: 50 } + * + * // Get next 50 results (pagination) + * { limit: 50, offset: 50 } + * ``` + */ +export type PaginationOptions = { + /** Maximum number of results to return (default: 100) */ + limit?: number; + /** Number of results to skip (default: 0) */ + offset?: number; +}; + +/** + * Main parameters for querying concepts with the new grouped API. + * Provides better developer experience with logical parameter grouping. + * + * @example + * ```typescript + * const results = await getConcepts({ + * supabase, + * spaceId: 123, + * scope: { type: "nodes", ofTypes: ["page"] }, + * relations: { ofType: ["cites"] }, + * author: "user123", + * pagination: { limit: 50 } + * }); + * ``` + */ +export type GetConceptsParams = { + /** Supabase client instance (must be authenticated) */ + supabase: DGSupabaseClient; + /** Space ID to query within (optional, uses client's default space) */ + spaceId?: number; + + /** What type of concepts to query and any constraints */ + scope?: NodeFilters; + + /** Optional filters based on relationships */ + relations?: RelationFilters; + + /** Control which fields are returned in the response */ + fields?: FieldSelection; + + /** Pagination options for result set control */ + pagination?: PaginationOptions; +}; + +type GetConceptsParamsDb = Omit&{scope?: NodeFiltersDb, relations?: RelationFiltersDb}; + // Utility function to compose a generic query to fetch concepts, content and document. // Arguments are as in getConcepts, except we use numeric db ids of concepts for schemas instead // their respective content's source_local_id. const composeConceptQuery = ({ supabase, spaceId, - baseNodeLocalIds = [], - schemaDbIds = 0, - fetchNodes = true, - nodeAuthor = undefined, - inRelsOfType = undefined, - inRelsToNodesOfType = undefined, - inRelsToNodesOfAuthor = undefined, - inRelsToNodeLocalIds = undefined, - conceptFields = ["id", "name", "space_id"], - contentFields = ["source_local_id"], - documentFields = [], - relationFields = undefined, - relationSubNodesFields = undefined, - limit = 100, - offset = 0, -}: { - supabase: DGSupabaseClient; - spaceId?: number; - schemaDbIds?: number | number[]; - baseNodeLocalIds?: string[]; - fetchNodes?: boolean | null; - nodeAuthor?: string; - inRelsOfType?: number[]; - relationSubNodesFields?: (keyof Concept)[]; - inRelsToNodesOfType?: number[]; - inRelsToNodesOfAuthor?: string; - conceptFields?: (keyof Concept)[]; - contentFields?: (keyof Content)[]; - documentFields?: (keyof Document)[]; - relationFields?: (keyof Concept)[]; - inRelsToNodeLocalIds?: string[]; - limit?: number; - offset?: number; -}) => { - let q = conceptFields.join(",\n"); - const innerContent = schemaDbIds === 0 || baseNodeLocalIds.length > 0; - if (innerContent && !contentFields.includes("source_local_id")) { - contentFields = contentFields.slice(); - contentFields.push("source_local_id"); + scope= { + type: "nodes", + }, + relations= {}, + fields= { + concepts: ["id", "name", "space_id"], + content: ["source_local_id"], + }, + pagination= { + offset: 0, + limit: 100, + }, +}: GetConceptsParamsDb) => { + const baseNodeLocalIds = scope.nodeIds || []; + const inRelsOfType = relations.ofType; + const inRelsToNodesOfType = relations.toNodeType; + const inRelsToNodeLocalIds = relations.toNodeId; + const inRelsToNodesOfAuthor = relations.author; + + let q = (fields.concepts || CONCEPT_FIELDS).join(",\n"); + const innerContent = scope.schemas || baseNodeLocalIds.length > 0; + const ctArgs: string[] = (fields.content || []).slice(); + if (innerContent && !ctArgs.includes("source_local_id")) { + ctArgs.push("source_local_id"); } - if (contentFields.length > 0) { - const args: string[] = contentFields.slice(); + if (ctArgs.length > 0) { + const documentFields = fields.documents || []; if (documentFields.length > 0) { - args.push( - `Document:my_documents!document_id${innerContent ? "!inner" : ""} (\n${documentFields.join(",\n")})`, - ); + ctArgs.push("Document:my_documents!document_id${innerContent ? "!inner" : ""} (\n" + documentFields.join(",\n") + ")"); } - q += `,\nContent:my_contents!represented_by_id${innerContent ? "!inner" : ""} (\n${args.join(",\n")})`; + q += `,\nContent:my_contents!represented_by_id${innerContent ? "!inner" : ""} (\n${ctArgs.join(",\n")})`; } - if (nodeAuthor !== undefined) { + if (scope.author !== undefined) { q += ", author:my_accounts!author_id!inner(account_local_id)"; } if ( @@ -168,7 +305,7 @@ const composeConceptQuery = ({ inRelsToNodesOfAuthor !== undefined || inRelsToNodeLocalIds !== undefined ) { - const args: string[] = (relationFields || []).slice(); + const args: string[] = (fields.relations || []).slice(); if (inRelsOfType !== undefined && !args.includes("schema_id")) args.push("schema_id"); if ( @@ -176,13 +313,11 @@ const composeConceptQuery = ({ inRelsToNodesOfAuthor !== undefined || inRelsToNodeLocalIds !== undefined ) { - const args2: string[] = (relationSubNodesFields || []).slice(); + const args2: string[] = (fields.relationNodes || []).slice(); if (inRelsToNodesOfType !== undefined && !args2.includes("schema_id")) args2.push("schema_id"); if (inRelsToNodeLocalIds !== undefined) - args2.push( - "Content:my_contents!represented_by_id!inner(source_local_id)", - ); + args2.push("Content:my_contents!represented_by_id!inner(source_local_id)"); if (inRelsToNodesOfAuthor !== undefined) { if (!args2.includes("author_id")) args2.push("author_id"); args2.push("author:my_accounts!author_id!inner(account_local_id)"); @@ -192,50 +327,71 @@ const composeConceptQuery = ({ q += `, relations:concept_in_relations!inner(${args.join(",\n")})`; } let query = supabase.from("my_concepts").select(q); - if (fetchNodes === true) { + if (scope.type === 'nodes') { query = query.eq("arity", 0); - } else if (fetchNodes === false) { + } else if (scope.type === 'relations') { query = query.gt("arity", 0); } // else fetch both if (spaceId !== undefined) query = query.eq("space_id", spaceId); - if (nodeAuthor !== undefined) { - query = query.eq("author.account_local_id", nodeAuthor); + if (scope.author !== undefined) { + query = query.eq("author.account_local_id", scope.author); } - if (schemaDbIds === 0) { + if (scope.schemas) { query = query.eq("is_schema", true); } else { query = query.eq("is_schema", false); - if (Array.isArray(schemaDbIds)) { - if (schemaDbIds.length > 0) query = query.in("schema_id", schemaDbIds); - // else we'll get all nodes - } else if (typeof schemaDbIds === "number") - query = query.eq("schema_id", schemaDbIds); - else throw new Error("schemaDbIds should be a number or number[]"); + const schemaDbIds = scope.ofType || []; + if (schemaDbIds.length > 0) { + if (schemaDbIds.length === 1) + query = query.eq("schema_id", schemaDbIds[0]!); + else + query = query.in("schema_id", schemaDbIds); + } + // else we'll get all nodes + } + if (baseNodeLocalIds.length > 0) { + if (baseNodeLocalIds.length === 1) + query = query.eq("Content.source_local_id", baseNodeLocalIds[0]!); + else + query = query.in("Content.source_local_id", baseNodeLocalIds); + } + if (inRelsOfType !== undefined && inRelsOfType.length > 0) { + if (inRelsOfType.length === 1) + query = query.eq("relations.schema_id", inRelsOfType[0]!); + else + query = query.in("relations.schema_id", inRelsOfType); + } + if (inRelsToNodesOfType !== undefined && inRelsToNodesOfType.length > 0) { + if (inRelsToNodesOfType.length === 1) + query = query.eq("relations.subnodes.schema_id", inRelsToNodesOfType[0]!); + else + query = query.in("relations.subnodes.schema_id", inRelsToNodesOfType); } - if (baseNodeLocalIds.length > 0) - query = query.in("Content.source_local_id", baseNodeLocalIds); - if (inRelsOfType !== undefined && inRelsOfType.length > 0) - query = query.in("relations.schema_id", inRelsOfType); - if (inRelsToNodesOfType !== undefined && inRelsToNodesOfType.length > 0) - query = query.in("relations.subnodes.schema_id", inRelsToNodesOfType); if (inRelsToNodesOfAuthor !== undefined) { query = query.eq( "relations.subnodes.author.account_local_id", inRelsToNodesOfAuthor, ); } - if (inRelsToNodeLocalIds !== undefined) { - query = query.in( - "relations.subnodes.Content.source_local_id", - inRelsToNodeLocalIds, - ); + if (inRelsToNodeLocalIds !== undefined && inRelsToNodeLocalIds.length > 0) { + if (inRelsToNodeLocalIds.length === 1) + query = query.eq( + "relations.subnodes.Content.source_local_id", + inRelsToNodeLocalIds[0]!, + ); + else + query = query.in( + "relations.subnodes.Content.source_local_id", + inRelsToNodeLocalIds, + ); } + const limit = Math.min(pagination.limit || 100, 1000); + const offset = pagination.offset || 0; if (limit > 0 || offset > 0) { query = query.order("id"); if (offset > 0) { - limit = Math.min(limit, 1000); query = query.range(offset, offset + limit); } else if (limit > 0) { query = query.limit(limit); @@ -255,7 +411,7 @@ export const getSchemaConcepts = async ( .filter((x) => typeof x === "object") .filter((x) => x.spaceId === spaceId || x.spaceId === 0); if (forceCacheReload || result.length === 1) { - const q = composeConceptQuery({ supabase, spaceId, fetchNodes: null }); + const q = composeConceptQuery({ supabase, spaceId, scope: {type:"all", schemas: true} }); const res = (await q) as PostgrestResponse; if (res.error) { console.error("getSchemaConcepts failed", res.error); @@ -312,7 +468,7 @@ const getLocalToDbIdMapping = async ( console.warn("Cannot populate cache without spaceId"); return dbIds; } - let q = composeConceptQuery({ supabase, spaceId, fetchNodes: null }); + let q = composeConceptQuery({ supabase, spaceId, scope: {type:"all", schemas: true} }); if (Object.keys(NODE_SCHEMA_CACHE).length > 1) { // Non-empty cache, query selectively q = q @@ -350,21 +506,27 @@ const getLocalToDbIdMapping = async ( }; /** - * Default concept fields that include metadata (created/edited user and times). - * These fields are returned by default for all queries unless overridden. + * Minimal concept fields for lightweight queries. + * Includes only essential fields: id, name, space_id, author_id, created, last_modified. * * @example * ```typescript - * // Use default fields - * const nodes = await getAllNodes({ supabase }); - * - * // Override with custom fields + * // Use minimal fields for performance * const nodes = await getAllNodes({ * supabase, * fields: { concepts: CONCEPT_FIELDS_MINIMAL } * }); * ``` */ +export const CONCEPT_FIELDS_MINIMAL: (keyof Concept)[] = [ + "id", + "name", + "space_id", + "author_id", + "created", + "last_modified", +]; + export const CONCEPT_FIELDS: (keyof Concept)[] = [ "id", "name", @@ -382,28 +544,6 @@ export const CONCEPT_FIELDS: (keyof Concept)[] = [ "represented_by_id", ]; -/** - * Minimal concept fields for lightweight queries. - * Includes only essential fields: id, name, space_id, author_id, created, last_modified. - * - * @example - * ```typescript - * // Use minimal fields for performance - * const nodes = await getAllNodes({ - * supabase, - * fields: { concepts: CONCEPT_FIELDS_MINIMAL } - * }); - * ``` - */ -export const CONCEPT_FIELDS_MINIMAL: (keyof Concept)[] = [ - "id", - "name", - "space_id", - "author_id", - "created", - "last_modified", -]; - export const CONTENT_FIELDS: (keyof Content)[] = [ "id", "source_local_id", @@ -429,154 +569,6 @@ export const DOCUMENT_FIELDS: (keyof Document)[] = [ "author_id", ]; -// instrumentation for benchmarking -export const LAST_QUERY_DATA = { duration: 0 }; - -/** - * Defines what type of concepts to query and any specific constraints. - * - * @example - * ```typescript - * // Query all nodes of specific types - * { type: "nodes", nodeTypes: ["page", "note"] } - * - * // Query all relations - * { type: "relations" } - * - * // Query specific nodes by their local IDs - * { type: "specific", nodeIds: ["node-123", "node-456"] } - * ``` - */ -export type QueryScope = { - /** The type of concepts to retrieve */ - type: "all" | "nodes" | "relations" | "schemas" | "specific"; - /** - * Schema local IDs to filter by (e.g., ["page", "note", "relation"]). - * Only used when type is "nodes" or "relations". - */ - nodeTypes?: string[]; - /** - * Specific node local IDs to retrieve. - * Only used when type is "specific". - */ - nodeIds?: string[]; -}; - -/** - * Filters for querying concepts based on their relationships. - * All filters are optional and can be combined. - * - * @example - * ```typescript - * // Find relations of type "cites" containing specific nodes - * { - * ofType: ["cites", "references"], - * containingNodes: ["node-123", "node-456"] - * } - * - * // Find concepts connected to nodes of specific types - * { - * toNodeTypes: ["page", "note"] - * } - * ``` - */ -export type RelationFilters = { - /** Find relations containing this specific node (single node) */ - containingNode?: string; - /** Find relations containing any of these nodes (multiple nodes) */ - containingNodes?: string[]; - /** Find concepts participating in relations of these types */ - ofType?: string[]; - /** Find concepts connected to nodes of these types */ - toNodeTypes?: string[]; - /** Find concepts in relations authored by this user */ - authoredBy?: string; -}; - -/** - * Controls which fields are returned in the response. - * Each field array specifies which columns to fetch from the respective table. - * If a field array is empty or undefined, that data won't be fetched. - * - * @example - * ```typescript - * // Get minimal concept data with full content - * { - * concepts: ["id", "name", "created"], - * content: ["source_local_id", "text", "metadata"] - * } - * ``` - */ -export type FieldSelection = { - /** Fields to return from the Concept table */ - concepts?: (keyof Concept)[]; - /** Fields to return from the Content table */ - content?: (keyof Content)[]; - /** Fields to return from the Document table */ - documents?: (keyof Document)[]; - /** Fields to return for relation concepts */ - relations?: (keyof Concept)[]; - /** Fields to return for nodes in relations */ - relationNodes?: (keyof Concept)[]; -}; - -/** - * Pagination options for controlling result set size and offset. - * - * @example - * ```typescript - * // Get first 50 results - * { limit: 50 } - * - * // Get next 50 results (pagination) - * { limit: 50, offset: 50 } - * ``` - */ -export type PaginationOptions = { - /** Maximum number of results to return (default: 100) */ - limit?: number; - /** Number of results to skip (default: 0) */ - offset?: number; -}; - -/** - * Main parameters for querying concepts with the new grouped API. - * Provides better developer experience with logical parameter grouping. - * - * @example - * ```typescript - * const results = await getConcepts({ - * supabase, - * spaceId: 123, - * scope: { type: "nodes", nodeTypes: ["page"] }, - * relations: { ofType: ["cites"] }, - * author: "user123", - * pagination: { limit: 50 } - * }); - * ``` - */ -export type GetConceptsParams = { - /** Supabase client instance (must be authenticated) */ - supabase: DGSupabaseClient; - /** Space ID to query within (optional, uses client's default space) */ - spaceId?: number; - - /** What type of concepts to query and any constraints */ - scope: QueryScope; - - /** Optional filters based on relationships */ - relations?: RelationFilters; - - /** Filter results by the author who created the concepts */ - author?: string; - - /** Control which fields are returned in the response */ - fields?: FieldSelection; - - /** Pagination options for result set control */ - pagination?: PaginationOptions; -}; - // Primitive query functions for common use cases // These provide a simpler API for the most common query patterns @@ -609,7 +601,7 @@ export const getAllNodes = async ({ spaceId, author, fields = { concepts: CONCEPT_FIELDS, content: CONTENT_FIELDS }, - pagination = { limit: 100 }, + pagination, }: { supabase: DGSupabaseClient; spaceId?: number; @@ -617,11 +609,10 @@ export const getAllNodes = async ({ fields?: FieldSelection; pagination?: PaginationOptions; }): Promise => { - return getConceptsNew({ + return getConcepts({ supabase, spaceId, - scope: { type: "nodes" }, - author, + scope: { type: "nodes", author }, fields, pagination, }); @@ -645,14 +636,14 @@ export const getAllNodes = async ({ * const nodes = await getNodesByType({ * supabase, * spaceId: 123, - * nodeTypes: ["page", "note"] + * ofType: ["page", "note"] * }); * * // Get pages by a specific author * const myPages = await getNodesByType({ * supabase, * spaceId: 123, - * nodeTypes: ["page"], + * ofType: ["page"], * author: "user123" * }); * ``` @@ -660,23 +651,22 @@ export const getAllNodes = async ({ export const getNodesByType = async ({ supabase, spaceId, - nodeTypes, + ofType, author, fields = { concepts: CONCEPT_FIELDS, content: CONTENT_FIELDS }, - pagination = { limit: 100 }, + pagination, }: { supabase: DGSupabaseClient; spaceId?: number; - nodeTypes: string[]; + ofType: string[]; author?: string; fields?: FieldSelection; pagination?: PaginationOptions; }): Promise => { - return getConceptsNew({ + return getConcepts({ supabase, spaceId, - scope: { type: "nodes", nodeTypes }, - author, + scope: { type: "nodes", ofType, author }, fields, pagination, }); @@ -711,7 +701,7 @@ export const getAllRelations = async ({ spaceId, author, fields = { concepts: CONCEPT_FIELDS, content: CONTENT_FIELDS }, - pagination = { limit: 100 }, + pagination, }: { supabase: DGSupabaseClient; spaceId?: number; @@ -719,25 +709,24 @@ export const getAllRelations = async ({ fields?: FieldSelection; pagination?: PaginationOptions; }): Promise => { - return getConceptsNew({ + return getConcepts({ supabase, spaceId, - scope: { type: "relations" }, - author, + scope: { type: "relations", author }, fields, pagination, }); }; /** - * Retrieves all relations that contain nodes of specific types. + * Retrieves all relations that start from nodes of specific types. * * @param params - Query parameters * @param params.supabase - Authenticated Supabase client * @param params.spaceId - Space ID to query (optional) - * @param params.nodeTypes - Array of node type local IDs that must be in the relations + * @param params.ofType - Array of node type local IDs in the relation * @param params.relationTypes - Optional array of relation types to filter by - * @param params.authoredBy - Optional filter by relation author + * @param params.nodeAuthoredBy - Optional filter by target node author (SLOW) * @param params.fields - Fields to return (defaults to full concept + content) * @param params.pagination - Pagination options (defaults to limit 100) * @returns Promise resolving to array of relation concept objects @@ -745,108 +734,45 @@ export const getAllRelations = async ({ * @example * ```typescript * // Find all relations containing page nodes - * const relations = await getRelationsContainingNodeType({ + * const relations = await getRelationsFromNodeType({ * supabase, * spaceId: 123, - * nodeTypes: ["page"] + * ofType: ["page"] * }); * * // Find citation relations containing note nodes - * const citations = await getRelationsContainingNodeType({ - * supabase, - * spaceId: 123, - * nodeTypes: ["note"], - * relationTypes: ["cites"] - * }); - * ``` - */ -export const getRelationsContainingNodeType = async ({ - supabase, - spaceId, - nodeTypes, - relationTypes, - authoredBy, - fields = { concepts: CONCEPT_FIELDS, content: CONTENT_FIELDS }, - pagination = { limit: 100 }, -}: { - supabase: DGSupabaseClient; - spaceId?: number; - nodeTypes: string[]; - relationTypes?: string[]; - authoredBy?: string; - fields?: FieldSelection; - pagination?: PaginationOptions; -}): Promise => { - return getConceptsNew({ - supabase, - spaceId, - scope: { type: "relations" }, - relations: { - toNodeTypes: nodeTypes, - ofType: relationTypes, - authoredBy, - }, - fields, - pagination, - }); -}; - -/** - * Retrieves all relations that contain specific nodes. - * - * @param params - Query parameters - * @param params.supabase - Authenticated Supabase client - * @param params.spaceId - Space ID to query (optional) - * @param params.nodeIds - Array of specific node local IDs that must be in the relations - * @param params.relationTypes - Optional array of relation types to filter by - * @param params.authoredBy - Optional filter by relation author - * @param params.fields - Fields to return (defaults to full concept + content) - * @param params.pagination - Pagination options (defaults to limit 100) - * @returns Promise resolving to array of relation concept objects - * - * @example - * ```typescript - * // Find all relations containing specific nodes - * const relations = await getRelationsContainingNode({ - * supabase, - * spaceId: 123, - * nodeIds: ["node-123", "node-456"] - * }); - * - * // Find citation relations containing a specific node - * const citations = await getRelationsContainingNode({ + * const citations = await getRelationsFromNodeType({ * supabase, * spaceId: 123, - * nodeIds: ["node-123"], + * ofType: ["note"], * relationTypes: ["cites"] * }); * ``` */ -export const getRelationsContainingNode = async ({ +export const getRelationsFromNodeType = async ({ supabase, spaceId, - nodeIds, + ofType, relationTypes, - authoredBy, + nodeAuthoredBy, fields = { concepts: CONCEPT_FIELDS, content: CONTENT_FIELDS }, - pagination = { limit: 100 }, + pagination, }: { supabase: DGSupabaseClient; spaceId?: number; - nodeIds: string[]; + ofType: string[]; relationTypes?: string[]; - authoredBy?: string; + nodeAuthoredBy?: string; fields?: FieldSelection; pagination?: PaginationOptions; }): Promise => { - return getConceptsNew({ + return getConcepts({ supabase, spaceId, - scope: { type: "relations" }, + scope: { type:"nodes", ofType, }, relations: { - containingNodes: nodeIds, ofType: relationTypes, - authoredBy, + author: nodeAuthoredBy, }, fields, pagination, @@ -854,79 +780,13 @@ export const getRelationsContainingNode = async ({ }; /** - * Retrieves nodes that participate in relations of specific types. - * - * @param params - Query parameters - * @param params.supabase - Authenticated Supabase client - * @param params.spaceId - Space ID to query (optional) - * @param params.relationTypes - Array of relation types to filter by - * @param params.toNodes - Optional array of node local IDs that must be connected to - * @param params.authoredBy - Optional filter by relation author - * @param params.author - Optional filter by node author - * @param params.fields - Fields to return (defaults to full concept + content) - * @param params.pagination - Pagination options (defaults to limit 100) - * @returns Promise resolving to array of node concept objects - * - * @example - * ```typescript - * // Find all nodes that are cited - * const citedNodes = await getNodesInRelations({ - * supabase, - * spaceId: 123, - * relationTypes: ["cites"] - * }); - * - * // Find nodes that cite specific other nodes - * const citingNodes = await getNodesInRelations({ - * supabase, - * spaceId: 123, - * relationTypes: ["cites"], - * toNodes: ["node-123", "node-456"] - * }); - * ``` - */ -export const getNodesInRelations = async ({ - supabase, - spaceId, - relationTypes, - toNodes, - authoredBy, - author, - fields = { concepts: CONCEPT_FIELDS, content: CONTENT_FIELDS }, - pagination = { limit: 100 }, -}: { - supabase: DGSupabaseClient; - spaceId?: number; - relationTypes: string[]; - toNodes?: string[]; - authoredBy?: string; - author?: string; - fields?: FieldSelection; - pagination?: PaginationOptions; -}): Promise => { - return getConceptsNew({ - supabase, - spaceId, - scope: { type: "nodes" }, - relations: { - ofType: relationTypes, - containingNodes: toNodes, - authoredBy, - }, - author, - fields, - pagination, - }); -}; - -/** - * Retrieves the discourse context of a node - all nodes and relations connected to it. + * Retrieves the discourse context of a set of nodes - all nodes and relations connected to it. * This is useful for understanding the full context around a specific concept. * * @param params - Query parameters * @param params.supabase - Authenticated Supabase client * @param params.spaceId - Space ID to query (optional) - * @param params.nodeId - Local ID of the node to get context for + * @param params.nodeIds - Local ID of the nodes to get context for * @param params.fields - Fields to return (defaults to full concept + content + minimal relations) * @param params.pagination - Pagination options (defaults to limit 100) * @returns Promise resolving to array of concept objects including relations @@ -937,7 +797,7 @@ export const getNodesInRelations = async ({ * const context = await getDiscourseContext({ * supabase, * spaceId: 123, - * nodeId: "node-123" + * nodeIds: ["node-123"] * }); * * // The result will include: @@ -949,219 +809,70 @@ export const getNodesInRelations = async ({ export const getDiscourseContext = async ({ supabase, spaceId, - nodeId, + nodeIds, fields = { concepts: CONCEPT_FIELDS, content: CONTENT_FIELDS, relations: CONCEPT_FIELDS_MINIMAL, + relationNodes: CONCEPT_FIELDS_MINIMAL, }, - pagination = { limit: 100 }, + pagination, }: { supabase: DGSupabaseClient; spaceId?: number; - nodeId: string; + nodeIds: string[]; fields?: FieldSelection; pagination?: PaginationOptions; }): Promise => { - return getConceptsNew({ + return getConcepts({ supabase, spaceId, - scope: { type: "all" }, - relations: { - containingNode: nodeId, - }, + scope: { type: "all", nodeIds }, fields, pagination, }); }; -/** - * Internal implementation of getConcepts with grouped parameters. - * Converts the new API to legacy parameters and delegates to getConceptsLegacy. - * - * @internal - */ -export const getConceptsNew = async ({ - supabase, - spaceId, - scope, - relations, - author, - fields = { concepts: CONCEPT_FIELDS, content: CONTENT_FIELDS }, - pagination = { limit: 100, offset: 0 }, -}: GetConceptsParams): Promise => { - // Convert new API to old API parameters - const oldParams: Parameters[0] = { - supabase, - spaceId, - nodeAuthor: author, - conceptFields: fields.concepts, - contentFields: fields.content, - documentFields: fields.documents, - relationFields: fields.relations, - relationSubNodesFields: fields.relationNodes, - limit: pagination.limit, - offset: pagination.offset, - }; - - // Map scope to old parameters - switch (scope.type) { - case "all": - oldParams.schemaLocalIds = []; - oldParams.fetchNodes = null; - break; - case "nodes": - oldParams.schemaLocalIds = scope.nodeTypes || []; - oldParams.fetchNodes = true; - break; - case "relations": - oldParams.schemaLocalIds = []; - oldParams.fetchNodes = false; - break; - case "schemas": - oldParams.schemaLocalIds = NODE_SCHEMAS; - oldParams.fetchNodes = null; - break; - case "specific": - oldParams.baseNodeLocalIds = scope.nodeIds || []; - oldParams.schemaLocalIds = []; - oldParams.fetchNodes = null; - break; - } - // Map relation filters to old parameters - if (relations) { - if (relations.ofType) { - oldParams.inRelsOfTypeLocal = relations.ofType; - } - if (relations.toNodeTypes) { - oldParams.inRelsToNodesOfTypeLocal = relations.toNodeTypes; - } - if (relations.authoredBy) { - oldParams.inRelsToNodesOfAuthor = relations.authoredBy; - } - if (relations.containingNode) { - oldParams.inRelsToNodeLocalIds = [relations.containingNode]; - } - if (relations.containingNodes) { - oldParams.inRelsToNodeLocalIds = relations.containingNodes; - } - } - - return getConceptsLegacy(oldParams); -}; +// instrumentation for benchmarking +export const LAST_QUERY_DATA = { duration: 0 }; -/** - * Main function for querying concepts with the new grouped parameter API. - * Provides better developer experience with logical parameter grouping and comprehensive IntelliSense. - * - * @param params - Query parameters with grouped structure - * @returns Promise resolving to array of concept objects with full metadata - * - * @example - * ```typescript - * // Query nodes of specific types with relation filters - * const results = await getConcepts({ - * supabase, - * spaceId: 123, - * scope: { type: "nodes", nodeTypes: ["page", "note"] }, - * relations: { ofType: ["cites", "references"] }, - * author: "user123", - * pagination: { limit: 50, offset: 0 } - * }); - * - * // Query relations containing specific nodes - * const relations = await getConcepts({ - * supabase, - * spaceId: 123, - * scope: { type: "relations" }, - * relations: { containingNodes: ["node-123", "node-456"] } - * }); - * ``` - */ +// Main entry point to query Concepts and related data: +// related sub-objects can be provided as: +// Content, Content.Document, author (PlatformAccount), relations (Concept), +// relations.subnodes (Concept), relations.subnodes.author, relations.subnodes.Content +// Which fields of these subobjects are fetched is controlled by the respective Fields parameters +// (except the last two, which would have just enough data for query filters.) +// If the fields are empty, the sub-object will not be fetched (unless needed for matching query parameters) +// Any parameter called "local" expects platform Ids (source_local_id) of the corresponding Content. +// In the case of node/relation definitions, schema refers to the page Id of the definition. export const getConcepts = async ( - params: GetConceptsParams, + { + supabase, + spaceId, + scope= { + type: "nodes", + }, + relations= {}, + fields= { + concepts: CONCEPT_FIELDS, + content: CONTENT_FIELDS, + documents: DOCUMENT_FIELDS + }, + pagination= { + offset: 0, + limit: 100, + }, + }: GetConceptsParams ): Promise => { - return getConceptsNew(params); -}; - -/** - * Legacy getConcepts function with the original 17-parameter API. - * Kept for backward compatibility. Consider using the new getConcepts() or primitive functions instead. - * - * @deprecated Use the new getConcepts() function with grouped parameters for better DX - * @param params - Legacy parameters with individual fields - * @returns Promise resolving to array of concept objects with full metadata - * - * @example - * ```typescript - * // Legacy usage (not recommended) - * const results = await getConceptsLegacy({ - * supabase, - * spaceId: 123, - * schemaLocalIds: ["page", "note"], - * fetchNodes: true, - * nodeAuthor: "user123", - * inRelsOfTypeLocal: ["cites"], - * conceptFields: ["id", "name", "created"], - * contentFields: ["source_local_id", "text"], - * limit: 100, - * offset: 0 - * }); - * ``` - */ -export const getConceptsLegacy = async ({ - supabase, // An instance of a logged-in client - spaceId, // the numeric id of the space being queried - baseNodeLocalIds = [], // If we are specifying the Concepts being queried directly. - schemaLocalIds = NODE_SCHEMAS, // the type of Concepts being queried - // • ALL schemas: schemaLocalIds = NODE_SCHEMAS (default, "__schemas") - // • ALL instances (nodes and/or relations): schemaLocalIds = [] - // • Nodes from X,Y schemas: schemaLocalIds = ["localIdX","localIdY",...] - fetchNodes = true, // are we fetching nodes or relations? - // true for nodes, false for relations, null for both - nodeAuthor = undefined, // filter on Content author - inRelsOfTypeLocal = undefined, // filter on Concepts that participate in a relation of a given type - inRelsToNodesOfTypeLocal = undefined, // filter on Concepts that are in a relation with another node of a given type - inRelsToNodesOfAuthor = undefined, // filter on Concepts that are in a relation with another Concept by a given author - inRelsToNodeLocalIds = undefined, // filter on Concepts that are in relation with a Concept from a given list - conceptFields = CONCEPT_FIELDS, // which fields are returned for the given Concept - contentFields = CONTENT_FIELDS, // which fields are returned for the corresponding Content - documentFields = DOCUMENT_FIELDS, // which fields are returned for the Content's corresponding Document - relationFields = undefined, // which fields are returned for the relation the node is part of - relationSubNodesFields = undefined, // which fields are returned for the other nodes in the relation the target node is part of - limit = 100, // query limit - offset = 0, // query offset -}: { - supabase: DGSupabaseClient; - spaceId?: number; - baseNodeLocalIds?: string[]; - schemaLocalIds?: string | string[]; - fetchNodes?: boolean | null; - nodeAuthor?: string; - inRelsOfTypeLocal?: string[]; - inRelsToNodesOfTypeLocal?: string[]; - inRelsToNodesOfAuthor?: string; - inRelsToNodeLocalIds?: string[]; - conceptFields?: (keyof Concept)[]; - contentFields?: (keyof Content)[]; - documentFields?: (keyof Document)[]; - relationFields?: (keyof Concept)[]; - relationSubNodesFields?: (keyof Concept)[]; - limit?: number; - offset?: number; -}): Promise => { - const schemaLocalIdsArray = - typeof schemaLocalIds === "string" ? [schemaLocalIds] : schemaLocalIds; // translate schema local content Ids to concept database Ids. - const localIds = new Set(schemaLocalIdsArray); - if (inRelsOfTypeLocal !== undefined) - inRelsOfTypeLocal.map((k) => localIds.add(k)); - if (inRelsToNodesOfTypeLocal !== undefined) - inRelsToNodesOfTypeLocal.map((k) => localIds.add(k)); + const localSchemaIds = new Set(); + (scope.ofType || []).map((k) => localSchemaIds.add(k)); + (relations.ofType || []).map((k) => localSchemaIds.add(k)); + (relations.toNodeType || []).map((k) => localSchemaIds.add(k)); const dbIdsMapping = await getLocalToDbIdMapping( supabase, - new Array(...localIds.keys()), + new Array(...localSchemaIds.keys()), spaceId, ); const localToDbArray = (a: string[] | undefined): number[] | undefined => { @@ -1177,27 +888,20 @@ export const getConceptsLegacy = async ({ } return r; }; - const schemaDbIds = - schemaLocalIds === NODE_SCHEMAS ? 0 : localToDbArray(schemaLocalIdsArray); - const q = composeConceptQuery({ supabase, spaceId, - baseNodeLocalIds, - schemaDbIds, - conceptFields, - contentFields, - documentFields, - nodeAuthor, - fetchNodes, - inRelsOfType: localToDbArray(inRelsOfTypeLocal), - relationFields, - relationSubNodesFields, - inRelsToNodesOfType: localToDbArray(inRelsToNodesOfTypeLocal), - inRelsToNodesOfAuthor, - inRelsToNodeLocalIds, - limit, - offset, + scope: { + ...scope, + ofType: localToDbArray(scope.ofType) + }, + relations: { + ...relations, + ofType: localToDbArray(relations.ofType), + toNodeType: localToDbArray(relations.toNodeType) + }, + fields, + pagination }); const before = Date.now(); const { error, data } = (await q) as PostgrestResponse; From 4666fe1752cc940ac0c60191d4f880aad479280c Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Thu, 16 Oct 2025 13:23:32 -0400 Subject: [PATCH 3/7] forgot to revert changes to stepdefs --- packages/database/features/step-definitions/stepdefs.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/database/features/step-definitions/stepdefs.ts b/packages/database/features/step-definitions/stepdefs.ts index 555fb9761..c7c8bb903 100644 --- a/packages/database/features/step-definitions/stepdefs.ts +++ b/packages/database/features/step-definitions/stepdefs.ts @@ -9,10 +9,7 @@ import { type Json, } from "@repo/database/dbTypes"; import { getVariant, config } from "@repo/database/dbDotEnv"; -import { - getConceptsLegacy, - initNodeSchemaCache, -} from "@repo/database/lib/queries"; +import { getConcepts, initNodeSchemaCache } from "@repo/database/lib/queries"; import { spaceAnonUserEmail, @@ -361,7 +358,7 @@ Given( if (spaceId === undefined) assert.fail("spaceId"); const supabase = await getLoggedinDatabase(spaceId); // note that we supply spaceId and supabase, they do not need to be part of the incoming Json - const nodes = await getConceptsLegacy({ ...params, supabase, spaceId }); + const nodes = await getConcepts({ ...params, supabase, spaceId }); nodes.sort((a, b) => a.id! - b.id!); world.queryResults = nodes; }, From dea98bf812248b9db8d51f7b05964e06ef410c00 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Thu, 16 Oct 2025 14:40:45 -0400 Subject: [PATCH 4/7] rebase error --- packages/database/src/lib/queries.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/database/src/lib/queries.ts b/packages/database/src/lib/queries.ts index db372dbc4..11db6a755 100644 --- a/packages/database/src/lib/queries.ts +++ b/packages/database/src/lib/queries.ts @@ -292,7 +292,7 @@ const composeConceptQuery = ({ if (ctArgs.length > 0) { const documentFields = fields.documents || []; if (documentFields.length > 0) { - ctArgs.push("Document:my_documents!document_id${innerContent ? "!inner" : ""} (\n" + documentFields.join(",\n") + ")"); + ctArgs.push(`Document:my_documents!document_id${innerContent ? "!inner" : ""} (\n" + documentFields.join(",\n") + ")`); } q += `,\nContent:my_contents!represented_by_id${innerContent ? "!inner" : ""} (\n${ctArgs.join(",\n")})`; } From bffce9741ce6ea9136649c0455703f53a1841a70 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Sat, 18 Oct 2025 08:33:16 -0400 Subject: [PATCH 5/7] reinstate and expand jdoc for getConcepts --- packages/database/src/lib/queries.ts | 52 +++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/packages/database/src/lib/queries.ts b/packages/database/src/lib/queries.ts index 11db6a755..6d5283ebf 100644 --- a/packages/database/src/lib/queries.ts +++ b/packages/database/src/lib/queries.ts @@ -837,15 +837,49 @@ export const getDiscourseContext = async ({ // instrumentation for benchmarking export const LAST_QUERY_DATA = { duration: 0 }; -// Main entry point to query Concepts and related data: -// related sub-objects can be provided as: -// Content, Content.Document, author (PlatformAccount), relations (Concept), -// relations.subnodes (Concept), relations.subnodes.author, relations.subnodes.Content -// Which fields of these subobjects are fetched is controlled by the respective Fields parameters -// (except the last two, which would have just enough data for query filters.) -// If the fields are empty, the sub-object will not be fetched (unless needed for matching query parameters) -// Any parameter called "local" expects platform Ids (source_local_id) of the corresponding Content. -// In the case of node/relation definitions, schema refers to the page Id of the definition. +/** + * Main entry point to query Concepts and related data: + * related sub-objects can be provided as: + * Content, Content.Document, author (PlatformAccount), relations (Concept), + * relations.subnodes (Concept), relations.subnodes.author, relations.subnodes.Content + * Which fields of these subobjects are fetched is controlled by the fields sub-parameters + * (except the last two, which would have just enough data for query filters.) + * If the fields are empty, the sub-object will not be fetched (unless needed for matching query parameters) + * Node identifiers are platform Ids (source_local_id) of the corresponding Content. + * In the case of node/relation definitions, schema refers to the page Id of the definition. + * Author identifiers are also source local ids. + * + * @param params - Query parameters with grouped structure + * @returns Promise resolving to array of concept objects with full metadata + * + * @example + * ```typescript + * // Query nodes of specific types with relation filters + * const results = await getConcepts({ + * supabase, + * spaceId: 123, + * scope: { type: "nodes", ofType: ["pageId", "noteId"], author: "user123Id" }, + * relations: { ofType: ["citesId", "referencesId"] }, + * pagination: { limit: 50, offset: 0 } + * }); + * + * // Query relations from specific nodes + * const relations = await getConcepts({ + * supabase, + * spaceId: 123, + * scope: { type: "nodes", nodeId: ["node-123", "node-456"] }, + * fields: { relations: CONCEPT_FIELDS_MINIMAL, relationNodes: CONCEPT_FIELDS_MINIMAL } + * }); + * + * // Query relations linking specific nodes + * const relations = await getConcepts({ + * supabase, + * spaceId: 123, + * scope: { type: "nodes", nodeId: ["node-123", "node-456"] }, + * relations: { toNodeId: ["node-789"] } + * }); + * ``` + */ export const getConcepts = async ( { supabase, From 85768193ab3e170bff43cf3ac258dd1cce9be364 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Mon, 20 Oct 2025 09:42:12 -0400 Subject: [PATCH 6/7] coderabbit comments, pluralization --- .../database/features/queryConcepts.feature | 44 +++---- packages/database/src/lib/queries.ts | 109 +++++++++--------- 2 files changed, 76 insertions(+), 77 deletions(-) diff --git a/packages/database/features/queryConcepts.feature b/packages/database/features/queryConcepts.feature index 9e1b04aff..b584262eb 100644 --- a/packages/database/features/queryConcepts.feature +++ b/packages/database/features/queryConcepts.feature @@ -17,35 +17,35 @@ Feature: Concept queries # Note: table syntax is explained in features/step-definitions/stepdefs.ts, look for `added to the database`. And Document are added to the database: | $id | source_local_id | created | last_modified | _author_id | _space_id | - | d1 | ld1 | 2025/01/01 | 2025/01/01 | user1 | s1 | - | d2 | ld2 | 2025/01/01 | 2025/01/01 | user1 | s1 | - | d5 | ld5 | 2025/01/01 | 2025/01/01 | user2 | s1 | - | d7 | ld7 | 2025/01/01 | 2025/01/01 | user1 | s1 | + | d1 | ld1 | 2025/01/01 | 2025/01/01 | user1 | s1 | + | d2 | ld2 | 2025/01/01 | 2025/01/01 | user1 | s1 | + | d5 | ld5 | 2025/01/01 | 2025/01/01 | user2 | s1 | + | d7 | ld7 | 2025/01/01 | 2025/01/01 | user1 | s1 | # Add Content as support for the Concept objects, esp. schemas And Content are added to the database: | $id | source_local_id | _document_id | text | created | last_modified | scale | _author_id | _space_id | - | ct1 | lct1 | d1 | Claim | 2025/01/01 | 2025/01/01 | document | user1 | s1 | - | ct2 | lct2 | d2 | claim 1 | 2025/01/01 | 2025/01/01 | document | user1 | s1 | - | ct5 | lct5 | d5 | Opposes | 2025/01/01 | 2025/01/01 | document | user2 | s1 | - | ct7 | lct7 | d7 | Hypothesis | 2025/01/01 | 2025/01/01 | document | user1 | s1 | + | ct1 | lct1 | d1 | Claim | 2025/01/01 | 2025/01/01 | document | user1 | s1 | + | ct2 | lct2 | d2 | claim 1 | 2025/01/01 | 2025/01/01 | document | user1 | s1 | + | ct5 | lct5 | d5 | Opposes | 2025/01/01 | 2025/01/01 | document | user2 | s1 | + | ct7 | lct7 | d7 | Hypothesis | 2025/01/01 | 2025/01/01 | document | user1 | s1 | # First add schemas And Concept are added to the database: | $id | name | _space_id | _author_id | _represented_by_id | created | last_modified | @is_schema | _schema_id | @literal_content | @reference_content | - | c1 | Claim | s1 | user1 | ct1 | 2025/01/01 | 2025/01/01 | true | | {} | {} | - | c5 | Opposes | s1 | user1 | ct5 | 2025/01/01 | 2025/01/01 | true | | {"roles": ["target", "source"]} | {} | - | c7 | Hypothesis | s1 | user1 | ct7 | 2025/01/01 | 2025/01/01 | true | | {} | {} | + | c1 | Claim | s1 | user1 | ct1 | 2025/01/01 | 2025/01/01 | true | | {} | {} | + | c5 | Opposes | s1 | user1 | ct5 | 2025/01/01 | 2025/01/01 | true | | {"roles": ["target", "source"]} | {} | + | c7 | Hypothesis | s1 | user1 | ct7 | 2025/01/01 | 2025/01/01 | true | | {} | {} | # Then nodes referring to the schemas And Concept are added to the database: | $id | name | _space_id | _author_id | created | last_modified | @is_schema | _schema_id | @literal_content | @reference_content | _represented_by_id | - | c2 | claim 1 | s1 | user1 | 2025/01/01 | 2025/01/01 | false | c1 | {} | {} | ct2 | - | c3 | claim 2 | s1 | user2 | 2025/01/01 | 2025/01/01 | false | c1 | {} | {} | | - | c4 | claim 3 | s1 | user3 | 2025/01/01 | 2025/01/01 | false | c1 | {} | {} | | - | c8 | hypothesis 1 | s1 | user3 | 2025/01/01 | 2025/01/01 | false | c7 | {} | {} | | + | c2 | claim 1 | s1 | user1 | 2025/01/01 | 2025/01/01 | false | c1 | {} | {} | ct2 | + | c3 | claim 2 | s1 | user2 | 2025/01/01 | 2025/01/01 | false | c1 | {} | {} | | + | c4 | claim 3 | s1 | user3 | 2025/01/01 | 2025/01/01 | false | c1 | {} | {} | | + | c8 | hypothesis 1 | s1 | user3 | 2025/01/01 | 2025/01/01 | false | c7 | {} | {} | | # Then relations (which refer to nodes) And Concept are added to the database: | $id | name | _space_id | _author_id | created | last_modified | @is_schema | _schema_id | @literal_content | @_reference_content | - | c6 | opposes 1 | s1 | user2 | 2025/01/01 | 2025/01/01 | false | c5 | {} | {"target": "c3", "source": "c2"} | - | c9 | opposes 2 | s1 | user2 | 2025/01/01 | 2025/01/01 | false | c5 | {} | {"target": "c8", "source": "c2"} | + | c6 | opposes 1 | s1 | user2 | 2025/01/01 | 2025/01/01 | false | c5 | {} | {"target": "c3", "source": "c2"} | + | c9 | opposes 2 | s1 | user2 | 2025/01/01 | 2025/01/01 | false | c5 | {} | {"target": "c8", "source": "c2"} | Scenario Outline: Query all nodes And a user logged in space s1 and calling getConcepts with these parameters: '{"scope":{"type":"all"}}' @@ -67,7 +67,7 @@ Feature: Concept queries | c7 | Hypothesis | s1 | user1 | true | | {} | {} | ct7 | Scenario Outline: Query by node types - And a user logged in space s1 and calling getConcepts with these parameters: '{"scope":{"ofType":["lct1"]}}' + And a user logged in space s1 and calling getConcepts with these parameters: '{"scope":{"ofTypes":["lct1"]}}' Then query results should look like this | _id | name | _space_id | _author_id | @is_schema | _schema_id | @literal_content | @reference_content | | c2 | claim 1 | s1 | user1 | false | c1 | {} | {} | @@ -83,7 +83,7 @@ Feature: Concept queries | c9 | opposes 2 | s1 | user2 | false | c5 | {} | {"target": "c8", "source": "c2"} | Scenario Outline: Query by relation type - And a user logged in space s1 and calling getConcepts with these parameters: '{"relations":{"ofType":["lct5"]}}' + And a user logged in space s1 and calling getConcepts with these parameters: '{"relations":{"ofTypes":["lct5"]}}' Then query results should look like this | _id | name | _space_id | _author_id | @is_schema | _schema_id | @literal_content | @_reference_content | | c2 | claim 1 | s1 | user1 | false | c1 | {} | {} | @@ -91,13 +91,13 @@ Feature: Concept queries | c8 | hypothesis 1 | s1 | user3 | false | c7 | {} | {} | Scenario Outline: Query by related node type - And a user logged in space s1 and calling getConcepts with these parameters: '{"relations":{"toNodeType":["lct7"]}}' + And a user logged in space s1 and calling getConcepts with these parameters: '{"relations":{"toNodeTypes":["lct7"]}}' Then query results should look like this | _id | name | _space_id | _author_id | @is_schema | _schema_id | @literal_content | @_reference_content | | c2 | claim 1 | s1 | user1 | false | c1 | {} | {} | | c8 | hypothesis 1 | s1 | user3 | false | c7 | {} | {} | - # Note that the node is related to itself, unfortunate but hard to solve. + Scenario Outline: Query by author of related node And a user logged in space s1 and calling getConcepts with these parameters: '{"relations":{"author":"user3"},"fields":{"relations":["id"],"relationNodes":["id"]}}' Then query results should look like this @@ -106,7 +106,7 @@ Feature: Concept queries | c8 | hypothesis 1 | s1 | user3 | false | c7 | {} | {} | Scenario Outline: Query by related node - And a user logged in space s1 and calling getConcepts with these parameters: '{"relations":{"toNodeId":["lct2"]}}' + And a user logged in space s1 and calling getConcepts with these parameters: '{"relations":{"toNodeIds":["lct2"]}}' Then query results should look like this | _id | name | _space_id | _author_id | @is_schema | _schema_id | @literal_content | @_reference_content | | c2 | claim 1 | s1 | user1 | false | c1 | {} | {} | diff --git a/packages/database/src/lib/queries.ts b/packages/database/src/lib/queries.ts index 6d5283ebf..945f43f1d 100644 --- a/packages/database/src/lib/queries.ts +++ b/packages/database/src/lib/queries.ts @@ -13,17 +13,12 @@ import { DGSupabaseClient } from "./client"; // // // Query all nodes of a specific type // const nodes = await getNodesByType({ -// supabase, spaceId, ofType: ["my-node-type"] -// }); -// -// // Query a specific node and its relations -// const relations = await getRelationsFromNode({ -// supabase, spaceId, nodeIds: ["node-123"] +// supabase, spaceId, ofTypes: ["my-node-type"] // }); // // // Query nodes of specific types and their (filtered) relations -// const nodes = await getRelationsFromNodeType({ -// supabase, spaceId, ofType: ["claim"], relationTypes: ["cites", "references"] +// const nodes = await getNodesOfTypeWithRelations({ +// supabase, spaceId, ofTypes: ["claim"], relationTypes: ["cites", "references"] // }); // // // Get discourse context (all nodes connected to a node or set of nodes) @@ -34,8 +29,8 @@ import { DGSupabaseClient } from "./client"; // // Use the main getConcepts function for complex queries // const results = await getConcepts({ // supabase, spaceId, -// scope: { type: 'nodes', ofType: ['type1', 'type2'] }, -// relations: { ofType: ['cites'], author: 'user123' }, +// scope: { type: 'nodes', ofTypes: ['type1', 'type2'] }, +// relations: { ofTypes: ['cites'], author: 'user123' }, // author: 'creator456', // pagination: { limit: 50, offset: 0 } // }); @@ -62,12 +57,15 @@ export const nodeSchemaSignature: NodeSignature = { type CacheMissTimestamp = number; type CacheEntry = NodeSignature | CacheMissTimestamp; +// Cache of nodes keyed by sourceLocalId +// TODO: Consider including the source_id in the key to avoid collisions +// I see a case for shared Ids, but this is not handled well here either. const NODE_SCHEMA_CACHE: Record = { [NODE_SCHEMAS]: nodeSchemaSignature, }; export const initNodeSchemaCache = () => { - Object.keys(NODE_SCHEMA_CACHE).map((k) => { + Object.keys(NODE_SCHEMA_CACHE).forEach((k) => { if (k !== NODE_SCHEMAS) delete NODE_SCHEMA_CACHE[k]; }); }; @@ -107,7 +105,7 @@ type DefaultQueryShape = { * @example * ```typescript * // Query all nodes of specific types - * { type: "nodes", ofSchemaLocal: ["page", "note"] } + * { type: "nodes", ofTypes: ["page", "note"] } * * // Query all relations * { type: "relations" } @@ -129,7 +127,7 @@ export type NodeFilters = { * Given as a list of local Ids (eg Roam page Ids of the node or relation types) * Only used when schemas=false. */ - ofType?: string[]; + ofTypes?: string[]; /** * Specific node local IDs to retrieve. */ @@ -139,7 +137,7 @@ export type NodeFilters = { author?: string; }; -type NodeFiltersDb = Omit & { ofType?: number[]}; +type NodeFiltersDb = Omit & { ofTypes?: number[]}; /** @@ -150,30 +148,30 @@ type NodeFiltersDb = Omit & { ofType?: number[]}; * ```typescript * // Find relations of type "cites" containing specific nodes * { - * ofType: ["cites", "references"], - * toNodeId: ["node-123", "node-456"] + * ofTypes: ["cites", "references"], + * toNodeIds: ["node-123", "node-456"] * } * * // Find concepts connected to nodes of specific types * { - * toNodeType: ["page", "note"] + * toNodeTypes: ["page", "note"] * } * ``` */ export type RelationFilters = { /** Find relations containing any of these nodes (multiple nodes) */ /* SLOW. Avoid using unless you have strong constraints on base node of search */ - toNodeId?: string[]; + toNodeIds?: string[]; /** Find concepts participating in relations of these types */ - ofType?: string[]; + ofTypes?: string[]; /** Find concepts connected to nodes of these types. */ - toNodeType?: string[]; + toNodeTypes?: string[]; /** Find concepts in relations authored by this user */ /* SLOW. Avoid using unless you have strong constraints on base node of search */ author?: string; }; -export type RelationFiltersDb = Omit&{ofType?: number[], toNodeType?: number[]}; +export type RelationFiltersDb = Omit&{ofTypes?: number[], toNodeTypes?: number[]}; /** * Controls which fields are returned in the response. @@ -231,7 +229,7 @@ export type PaginationOptions = { * supabase, * spaceId: 123, * scope: { type: "nodes", ofTypes: ["page"] }, - * relations: { ofType: ["cites"] }, + * relations: { ofTypes: ["cites"] }, * author: "user123", * pagination: { limit: 50 } * }); @@ -278,9 +276,9 @@ const composeConceptQuery = ({ }, }: GetConceptsParamsDb) => { const baseNodeLocalIds = scope.nodeIds || []; - const inRelsOfType = relations.ofType; - const inRelsToNodesOfType = relations.toNodeType; - const inRelsToNodeLocalIds = relations.toNodeId; + const inRelsOfType = relations.ofTypes; + const inRelsToNodesOfType = relations.toNodeTypes; + const inRelsToNodeLocalIds = relations.toNodeIds; const inRelsToNodesOfAuthor = relations.author; let q = (fields.concepts || CONCEPT_FIELDS).join(",\n"); @@ -292,7 +290,7 @@ const composeConceptQuery = ({ if (ctArgs.length > 0) { const documentFields = fields.documents || []; if (documentFields.length > 0) { - ctArgs.push(`Document:my_documents!document_id${innerContent ? "!inner" : ""} (\n" + documentFields.join(",\n") + ")`); + ctArgs.push(`Document:my_documents!document_id${innerContent ? "!inner" : ""} (\n ${documentFields.join(",\n")} )`); } q += `,\nContent:my_contents!represented_by_id${innerContent ? "!inner" : ""} (\n${ctArgs.join(",\n")})`; } @@ -342,7 +340,7 @@ const composeConceptQuery = ({ query = query.eq("is_schema", true); } else { query = query.eq("is_schema", false); - const schemaDbIds = scope.ofType || []; + const schemaDbIds = scope.ofTypes || []; if (schemaDbIds.length > 0) { if (schemaDbIds.length === 1) query = query.eq("schema_id", schemaDbIds[0]!); @@ -392,7 +390,8 @@ const composeConceptQuery = ({ if (limit > 0 || offset > 0) { query = query.order("id"); if (offset > 0) { - query = query.range(offset, offset + limit); + const to = Math.max(offset, offset + limit - 1); + query = query.range(offset, to); } else if (limit > 0) { query = query.limit(limit); } @@ -636,14 +635,14 @@ export const getAllNodes = async ({ * const nodes = await getNodesByType({ * supabase, * spaceId: 123, - * ofType: ["page", "note"] + * ofTypes: ["page", "note"] * }); * * // Get pages by a specific author * const myPages = await getNodesByType({ * supabase, * spaceId: 123, - * ofType: ["page"], + * ofTypes: ["page"], * author: "user123" * }); * ``` @@ -651,14 +650,14 @@ export const getAllNodes = async ({ export const getNodesByType = async ({ supabase, spaceId, - ofType, + ofTypes, author, fields = { concepts: CONCEPT_FIELDS, content: CONTENT_FIELDS }, pagination, }: { supabase: DGSupabaseClient; spaceId?: number; - ofType: string[]; + ofTypes: string[]; author?: string; fields?: FieldSelection; pagination?: PaginationOptions; @@ -666,7 +665,7 @@ export const getNodesByType = async ({ return getConcepts({ supabase, spaceId, - scope: { type: "nodes", ofType, author }, + scope: { type: "nodes", ofTypes, author }, fields, pagination, }); @@ -719,12 +718,12 @@ export const getAllRelations = async ({ }; /** - * Retrieves all relations that start from nodes of specific types. + * Retrieves all relations that start from nodes of specific types. Centered on the node. * * @param params - Query parameters * @param params.supabase - Authenticated Supabase client * @param params.spaceId - Space ID to query (optional) - * @param params.ofType - Array of node type local IDs in the relation + * @param params.ofTypes - Array of node type local IDs in the relation * @param params.relationTypes - Optional array of relation types to filter by * @param params.nodeAuthoredBy - Optional filter by target node author (SLOW) * @param params.fields - Fields to return (defaults to full concept + content) @@ -734,25 +733,25 @@ export const getAllRelations = async ({ * @example * ```typescript * // Find all relations containing page nodes - * const relations = await getRelationsFromNodeType({ + * const relations = await getNodesOfTypeWithRelations({ * supabase, * spaceId: 123, - * ofType: ["page"] + * ofTypes: ["page"] * }); * * // Find citation relations containing note nodes - * const citations = await getRelationsFromNodeType({ + * const citations = await getNodesOfTypeWithRelations({ * supabase, * spaceId: 123, - * ofType: ["note"], + * ofTypes: ["note"], * relationTypes: ["cites"] * }); * ``` */ -export const getRelationsFromNodeType = async ({ +export const getNodesOfTypeWithRelations = async ({ supabase, spaceId, - ofType, + ofTypes, relationTypes, nodeAuthoredBy, fields = { concepts: CONCEPT_FIELDS, content: CONTENT_FIELDS }, @@ -760,7 +759,7 @@ export const getRelationsFromNodeType = async ({ }: { supabase: DGSupabaseClient; spaceId?: number; - ofType: string[]; + ofTypes: string[]; relationTypes?: string[]; nodeAuthoredBy?: string; fields?: FieldSelection; @@ -769,9 +768,9 @@ export const getRelationsFromNodeType = async ({ return getConcepts({ supabase, spaceId, - scope: { type:"nodes", ofType, }, + scope: { type:"nodes", ofTypes, }, // we still start from the node relations: { - ofType: relationTypes, + ofTypes: relationTypes, author: nodeAuthoredBy, }, fields, @@ -858,8 +857,8 @@ export const LAST_QUERY_DATA = { duration: 0 }; * const results = await getConcepts({ * supabase, * spaceId: 123, - * scope: { type: "nodes", ofType: ["pageId", "noteId"], author: "user123Id" }, - * relations: { ofType: ["citesId", "referencesId"] }, + * scope: { type: "nodes", ofTypes: ["pageId", "noteId"], author: "user123Id" }, + * relations: { ofTypes: ["citesId", "referencesId"] }, * pagination: { limit: 50, offset: 0 } * }); * @@ -867,7 +866,7 @@ export const LAST_QUERY_DATA = { duration: 0 }; * const relations = await getConcepts({ * supabase, * spaceId: 123, - * scope: { type: "nodes", nodeId: ["node-123", "node-456"] }, + * scope: { type: "nodes", nodeIds: ["node-123", "node-456"] }, * fields: { relations: CONCEPT_FIELDS_MINIMAL, relationNodes: CONCEPT_FIELDS_MINIMAL } * }); * @@ -875,8 +874,8 @@ export const LAST_QUERY_DATA = { duration: 0 }; * const relations = await getConcepts({ * supabase, * spaceId: 123, - * scope: { type: "nodes", nodeId: ["node-123", "node-456"] }, - * relations: { toNodeId: ["node-789"] } + * scope: { type: "nodes", nodeIds: ["node-123", "node-456"] }, + * relations: { toNodeIds: ["node-789"] } * }); * ``` */ @@ -901,9 +900,9 @@ export const getConcepts = async ( ): Promise => { // translate schema local content Ids to concept database Ids. const localSchemaIds = new Set(); - (scope.ofType || []).map((k) => localSchemaIds.add(k)); - (relations.ofType || []).map((k) => localSchemaIds.add(k)); - (relations.toNodeType || []).map((k) => localSchemaIds.add(k)); + (scope.ofTypes || []).map((k) => localSchemaIds.add(k)); + (relations.ofTypes || []).map((k) => localSchemaIds.add(k)); + (relations.toNodeTypes || []).map((k) => localSchemaIds.add(k)); const dbIdsMapping = await getLocalToDbIdMapping( supabase, new Array(...localSchemaIds.keys()), @@ -927,12 +926,12 @@ export const getConcepts = async ( spaceId, scope: { ...scope, - ofType: localToDbArray(scope.ofType) + ofTypes: localToDbArray(scope.ofTypes) }, relations: { ...relations, - ofType: localToDbArray(relations.ofType), - toNodeType: localToDbArray(relations.toNodeType) + ofTypes: localToDbArray(relations.ofTypes), + toNodeTypes: localToDbArray(relations.toNodeTypes) }, fields, pagination From 6396b1931573eea0d52935f4a7176a5f1c0175ef Mon Sep 17 00:00:00 2001 From: Marc-Antoine Parent Date: Mon, 20 Oct 2025 09:45:27 -0400 Subject: [PATCH 7/7] forgot a rename --- apps/roam/src/components/settings/AdminPanel.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/roam/src/components/settings/AdminPanel.tsx b/apps/roam/src/components/settings/AdminPanel.tsx index 03b90435d..924b37806 100644 --- a/apps/roam/src/components/settings/AdminPanel.tsx +++ b/apps/roam/src/components/settings/AdminPanel.tsx @@ -165,7 +165,7 @@ const AdminPanel = () => { showingSchema.sourceLocalId === nodeSchemaSignature.sourceLocalId, type: "nodes", - ofType: [showingSchema.sourceLocalId], + ofTypes: [showingSchema.sourceLocalId], }, }), );