diff --git a/.changeset/well-traveled-refactors-0276.md b/.changeset/well-traveled-refactors-0276.md new file mode 100644 index 000000000..1620b5802 --- /dev/null +++ b/.changeset/well-traveled-refactors-0276.md @@ -0,0 +1,17 @@ +--- +'@xnetjs/core': minor +'@xnetjs/data': patch +'@xnetjs/plugins': patch +'@xnetjs/react': patch +--- + +Add the shared Last-Write-Wins ordering module to `@xnetjs/core` +(`compareChangeApplicationOrder`, `compareLwwStamps`, `lwwWins`, +`lwwUpdateGuardSql`, `LwwStamp`) — the single canonical LWW comparison used +across the stack (protocol §L1.7). + +`@xnetjs/data`, `@xnetjs/plugins`, and `@xnetjs/react` adopt it and receive +internal decompositions of their most-churned modules (NodeStore query +compiler/hydration/transaction execution, ai-surface tool registry and +resource URI router, XNetProvider provider units). No public API changes in +those packages. diff --git a/CLAUDE.md b/CLAUDE.md index 17c3674ce..7384d7ac8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,5 +1,24 @@ # xNet — agent conventions +## Barrel exports (index.ts) — sub-barrel policy (0276) + +The `react`/`data`/`plugins` root barrels are the highest-churn files in the +repo (90/87/47 commits in 8 months) — every feature appending re-exports there +creates standing merge conflicts and degrades tree-shaking. + +- **New surface lands in a scoped sub-barrel**, not the root barrel: add (or + extend) a feature-area file — e.g. `packages/react/src/hooks/index.ts`, + `packages/data/src/store/index.ts` — and re-export the _area_ from the root + with ONE grouped block, so the root barrel gains at most one line per area, + not five lines per feature. +- **Never `export *` from the root barrel** — named re-exports only (keeps + tree-shaking and makes API-surface diffs reviewable). +- **Internal modules don't get barrel exports at all.** If nothing outside the + package imports it (e.g. `packages/react/src/provider/*` units), leave it + out of every barrel. +- Removing/renaming anything already exported from a root barrel is a + **major** bump (see Changesets below) — bump from the diff. + ## Changesets (npm release intent) Every change to a **publishable** `packages/*` library MUST produce a diff --git a/apps/electron/src/renderer/App.tsx b/apps/electron/src/renderer/App.tsx index abaab0da0..48de98737 100644 --- a/apps/electron/src/renderer/App.tsx +++ b/apps/electron/src/renderer/App.tsx @@ -1,145 +1,66 @@ /** * Electron App - Main component + * + * The shell orchestration lives in `./shell/`: `shell-state.ts` is the pure + * ShellState reducer, `useDocumentShell` owns the shell state, home canvas, + * document queries and transition handlers, and `useShellPaletteCommands` + * the command-palette table. This component composes those hooks and renders + * per shell state. */ -import type { LinkedDocumentItem } from './lib/canvas-shell' -import type { PaletteCommand } from '@xnetjs/ui' -import { CANVAS_PLANNING_TEMPLATE_DEFINITIONS } from '@xnetjs/canvas' -import { PageSchema, DatabaseSchema, CanvasSchema } from '@xnetjs/data' -import { useDevTools } from '@xnetjs/devtools' -import { useQuery, useMutate } from '@xnetjs/react' -import { CommandPalette, useCommandPalette, usePrefersReducedMotion } from '@xnetjs/ui' -import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import type { ConnectHubRequest } from './components/ConnectHubDialog' +import { useCommandPalette, CommandPalette } from '@xnetjs/ui' +import React, { useCallback, useEffect, useState } from 'react' import { ActionDock } from './components/ActionDock' -import { AddSharedDialog, type AddSharedInput } from './components/AddSharedDialog' +import { AddSharedDialog } from './components/AddSharedDialog' import { BundledPluginInstaller } from './components/BundledPluginInstaller' -import { - CanvasView, - type CanvasViewCommandState, - type CanvasViewHandle -} from './components/CanvasView' -import { ConnectHubDialog, type ConnectHubRequest } from './components/ConnectHubDialog' +import { CanvasView } from './components/CanvasView' +import { ConnectHubDialog } from './components/ConnectHubDialog' import { DatabaseView } from './components/DatabaseView' -import { DataWorkspaceView, type SavedViewCanvasFrameInput } from './components/DataWorkspaceView' +import { DataWorkspaceView } from './components/DataWorkspaceView' import { PageView } from './components/PageView' import { SettingsView } from './components/SettingsView' import { SocialImportView } from './components/SocialImportView' import { StorybookView } from './components/StorybookView' import { SystemMenu } from './components/SystemMenu' import { setPersistedHubUrl } from './lib/hub-url' - -type DocType = 'page' | 'database' | 'canvas' - -type ViewportSnapshot = { - x: number - y: number - zoom: number -} - -type ShellState = - | { kind: 'canvas-home' } - | { kind: 'page-focus'; docId: string; returnViewport: ViewportSnapshot | null } - | { kind: 'database-focus'; docId: string; returnViewport: ViewportSnapshot | null } - | { kind: 'database-split'; docId: string } - | { kind: 'settings' } - | { kind: 'data-workspace' } - | { kind: 'social-import' } - | { kind: 'stories' } - -type DocumentItem = { - id: string - title: string - type: DocType - createdAt?: number - updatedAt?: number -} - -const OVERLAY_OPEN_DELAY_MS = 180 -const STORIES_ENABLED = import.meta.env.DEV -const MOD_ENTER_SHORTCUT = navigator.platform.includes('Mac') ? '⌘↩' : 'Ctrl+Enter' -const EMPTY_CANVAS_COMMAND_STATE: CanvasViewCommandState = { - selectionCount: 0, - selectedNodeId: null, - selectedSourceId: null, - selectedSourceType: null, - selectedDisplayType: null, - selectedTitle: null, - selectedIsQueryFrame: false, - selectionAllLocked: false, - selectionAnyLocked: false, - shortcutHelpOpen: false -} - -function toError(error: unknown): Error { - return error instanceof Error ? error : new Error(String(error)) -} +import { STORIES_ENABLED, useDocumentShell } from './shell/use-document-shell' +import { useShellPaletteCommands } from './shell/use-shell-palette-commands' export function App(): React.ReactElement { - const [homeCanvasId, setHomeCanvasId] = useState(null) - const [homeCanvasBootstrapError, setHomeCanvasBootstrapError] = useState(null) - const [shellState, setShellState] = useState({ kind: 'canvas-home' }) - const [pendingCanvasInsert, setPendingCanvasInsert] = useState<{ - requestId: string - document: LinkedDocumentItem - } | null>(null) - const [canvasCommandState, setCanvasCommandState] = useState( - EMPTY_CANVAS_COMMAND_STATE - ) + const { + shellState, + overlayTitle, + isCanvasInteractiveShell, + prefersReducedMotion, + homeCanvasId, + homeCanvasBootstrapError, + documents, + recentDocuments, + isLoading, + pendingCanvasInsert, + canvasCommandState, + canvasViewRef, + bootstrapHomeCanvas, + focusDocument, + handleOpenDocument, + handleCreateLinkedDocument, + handleCreateCanvasNote, + handleReturnHome, + handleAddShared, + openDatabaseSplit, + handleOpenSettings, + handleOpenSocialImport, + handleOpenDataWorkspace, + handleOpenStories, + handleInsertSavedLensAsCanvasFrame, + handleCommandStateChange, + handlePendingInsertConsumed + } = useDocumentShell() const [showAddSharedDialog, setShowAddSharedDialog] = useState(false) const [prefilledShareValue, setPrefilledShareValue] = useState('') const [connectRequest, setConnectRequest] = useState(null) - const { setActiveNodeId } = useDevTools() - const { create } = useMutate() const { open: paletteOpen, setOpen: setPaletteOpen, show: showPalette } = useCommandPalette() - const prefersReducedMotion = usePrefersReducedMotion() - const canvasViewRef = useRef(null) - const creatingHomeCanvasRef = useRef(false) - const transitionTimerRef = useRef(null) - - const { data: pages, loading: pagesLoading } = useQuery(PageSchema, { limit: 100 }) - const { data: databases, loading: databasesLoading } = useQuery(DatabaseSchema, { limit: 100 }) - const { data: canvases, loading: canvasesLoading } = useQuery(CanvasSchema, { limit: 100 }) - - const documents: DocumentItem[] = useMemo( - () => - [ - ...pages.map((page) => ({ - id: page.id, - title: page.title || 'Untitled Page', - type: 'page' as const, - createdAt: page.createdAt, - updatedAt: page.updatedAt - })), - ...databases.map((database) => ({ - id: database.id, - title: database.title || 'Untitled Database', - type: 'database' as const, - createdAt: database.createdAt, - updatedAt: database.updatedAt - })), - ...canvases.map((canvas) => ({ - id: canvas.id, - title: canvas.title || 'Workspace Canvas', - type: 'canvas' as const, - createdAt: canvas.createdAt, - updatedAt: canvas.updatedAt - })) - ].sort( - (left, right) => - (right.updatedAt || right.createdAt || 0) - (left.updatedAt || left.createdAt || 0) - ), - [canvases, databases, pages] - ) - - const isLoading = pagesLoading || databasesLoading || canvasesLoading - const recentDocuments = useMemo(() => documents.slice(0, 6), [documents]) - - const clearTransitionTimer = useCallback(() => { - if (transitionTimerRef.current !== null) { - window.clearTimeout(transitionTimerRef.current) - transitionTimerRef.current = null - } - }, []) useEffect(() => { const cleanup = window.xnet.onSharePayload((payload) => { @@ -167,777 +88,20 @@ export function App(): React.ReactElement { await window.__xnetIpcSyncManager?.configureShareSession({ signalingUrl: request.hub }) }, []) - useEffect(() => { - return () => { - clearTransitionTimer() - } - }, [clearTransitionTimer]) - - const bootstrapHomeCanvas = useCallback(async () => { - if (creatingHomeCanvasRef.current) return - - creatingHomeCanvasRef.current = true - setHomeCanvasBootstrapError(null) - - try { - const canvas = await create(CanvasSchema, { title: 'Workspace Canvas' }) - if (!canvas) { - throw new Error('Home canvas was not created') - } - - setHomeCanvasId(canvas.id) - setActiveNodeId(canvas.id) - } catch (error) { - const normalizedError = toError(error) - console.error('Failed to create home canvas', normalizedError) - setHomeCanvasBootstrapError(normalizedError) - } finally { - creatingHomeCanvasRef.current = false - } - }, [create, setActiveNodeId]) - - useEffect(() => { - if (isLoading) return - - if (canvases.length === 0) { - if (homeCanvasBootstrapError) return - if (homeCanvasId) { - setHomeCanvasId(null) - } - void bootstrapHomeCanvas() - return - } - - if (homeCanvasBootstrapError) { - setHomeCanvasBootstrapError(null) - } - - if (!homeCanvasId || !canvases.some((canvas) => canvas.id === homeCanvasId)) { - const defaultCanvas = [...canvases].sort( - (left, right) => - (right.updatedAt || right.createdAt || 0) - (left.updatedAt || left.createdAt || 0) - )[0] - - if (defaultCanvas) { - setHomeCanvasId(defaultCanvas.id) - setActiveNodeId(defaultCanvas.id) - } - } - }, [ - bootstrapHomeCanvas, - canvases, - homeCanvasBootstrapError, - homeCanvasId, - isLoading, - setActiveNodeId - ]) - - const focusDocument = useCallback( - (docId: string, docType: Exclude, animateFromCanvas: boolean) => { - clearTransitionTimer() - - const shouldAnimateFromCanvas = animateFromCanvas && !prefersReducedMotion - const returnViewport = - shouldAnimateFromCanvas && canvasViewRef.current - ? canvasViewRef.current.focusLinkedDocument(docId) - : null - - const openOverlay = () => { - setShellState( - docType === 'page' - ? { kind: 'page-focus', docId, returnViewport } - : { kind: 'database-focus', docId, returnViewport } - ) - setActiveNodeId(docId) - } - - if (returnViewport && !prefersReducedMotion) { - transitionTimerRef.current = window.setTimeout(openOverlay, OVERLAY_OPEN_DELAY_MS) - return - } - - openOverlay() - }, - [clearTransitionTimer, prefersReducedMotion, setActiveNodeId] - ) - - const handleOpenDocument = useCallback( - (docId: string) => { - const document = documents.find((entry) => entry.id === docId) - if (!document) return - - if (document.type === 'canvas') { - clearTransitionTimer() - setHomeCanvasId(document.id) - setShellState({ kind: 'canvas-home' }) - setActiveNodeId(document.id) - return - } - - focusDocument(document.id, document.type, true) - }, - [clearTransitionTimer, documents, focusDocument, setActiveNodeId] - ) - - const handleCreateLinkedDocument = useCallback( - async (type: Exclude) => { - clearTransitionTimer() - - try { - const schema = type === 'page' ? PageSchema : DatabaseSchema - const title = type === 'page' ? 'Untitled Page' : 'Untitled Database' - const newDocument = await create(schema, { title }) - if (!newDocument) return - - setPendingCanvasInsert({ - requestId: `${type}-${newDocument.id}-${Date.now()}`, - document: { - id: newDocument.id, - title, - type - } - }) - setShellState({ kind: 'canvas-home' }) - setActiveNodeId(homeCanvasId) - } catch (error) { - console.error('Failed to create linked document', toError(error)) - } - }, - [clearTransitionTimer, create, homeCanvasId, setActiveNodeId] - ) - - const handleCreateCanvasNote = useCallback(() => { - const createCanvasNote = async () => { - clearTransitionTimer() - - try { - const note = await create(PageSchema, { title: 'Untitled Note' }) - if (!note) return - - setPendingCanvasInsert({ - requestId: `note-${note.id}-${Date.now()}`, - document: { - id: note.id, - title: note.title || 'Untitled Note', - type: 'page', - canvasKind: 'note' - } - }) - setShellState({ kind: 'canvas-home' }) - setActiveNodeId(homeCanvasId) - } catch (error) { - console.error('Failed to create canvas note', toError(error)) - } - } - - void createCanvasNote() - }, [clearTransitionTimer, create, homeCanvasId, setActiveNodeId]) - - const handleReturnHome = useCallback(() => { - clearTransitionTimer() - if (shellState.kind === 'page-focus' || shellState.kind === 'database-focus') { - if (shellState.returnViewport) { - canvasViewRef.current?.restoreViewport(shellState.returnViewport) - } - } - - setShellState({ kind: 'canvas-home' }) - setActiveNodeId(homeCanvasId) - }, [clearTransitionTimer, homeCanvasId, setActiveNodeId, shellState]) - - const handleAddShared = useCallback( - async (input: AddSharedInput) => { - if (input.share) { - try { - await window.__xnetIpcSyncManager?.configureShareSession({ - signalingUrl: input.share.endpoint, - ucanToken: input.share.token, - transport: input.share.transport, - iceServers: input.share.iceServers - }) - } catch (error) { - console.error('Failed to configure shared session', toError(error)) - } - } - - if (input.docType === 'canvas') { - clearTransitionTimer() - setHomeCanvasId(input.docId) - setShellState({ kind: 'canvas-home' }) - setActiveNodeId(input.docId) - return - } - - focusDocument(input.docId, input.docType, false) - }, - [clearTransitionTimer, focusDocument, setActiveNodeId] - ) - - const overlayTitle = useMemo(() => { - if (shellState.kind === 'page-focus') return 'Document' - if (shellState.kind === 'database-focus') return 'Database' - if (shellState.kind === 'settings') return 'Settings' - if (shellState.kind === 'data-workspace') return 'Data Workspace' - if (shellState.kind === 'social-import') return 'Social Import' - if (shellState.kind === 'stories') return 'Stories' - return null - }, [shellState.kind]) - const isCanvasInteractiveShell = - shellState.kind === 'canvas-home' || shellState.kind === 'database-split' - - const openDatabaseSplit = useCallback( - (docId: string) => { - clearTransitionTimer() - setShellState({ kind: 'database-split', docId }) - setActiveNodeId(docId) - }, - [clearTransitionTimer, setActiveNodeId] - ) - - const handleOpenSettings = useCallback(() => { - clearTransitionTimer() - setShellState({ kind: 'settings' }) - }, [clearTransitionTimer]) - - const handleOpenSocialImport = useCallback(() => { - clearTransitionTimer() - setShellState({ kind: 'social-import' }) - }, [clearTransitionTimer]) - - const handleOpenDataWorkspace = useCallback(() => { - clearTransitionTimer() - setShellState({ kind: 'data-workspace' }) - }, [clearTransitionTimer]) - - const handleInsertSavedLensAsCanvasFrame = useCallback( - (view: SavedViewCanvasFrameInput) => { - const inserted = - canvasViewRef.current?.createQueryFrameFromSavedView({ - viewId: view.id, - title: view.title ?? 'Saved lens', - descriptorJson: view.descriptor ?? null - }) ?? false - - if (!inserted) { - console.error('Failed to insert saved lens as a canvas query frame', view.id) - return - } - - clearTransitionTimer() - setShellState({ kind: 'canvas-home' }) - setActiveNodeId(homeCanvasId) - }, - [clearTransitionTimer, homeCanvasId, setActiveNodeId] - ) - - const handleOpenStories = useCallback(() => { - if (!STORIES_ENABLED) return - - clearTransitionTimer() - setShellState({ kind: 'stories' }) - }, [clearTransitionTimer]) - - const paletteCommands = useMemo( - () => [ - { - id: 'create-page', - name: 'Create Page', - description: 'Create a new page and place it on the canvas', - icon: 'file-text', - shortcut: 'P', - group: 'Canvas', - keywords: ['page', 'canvas', 'create'], - execute: () => void handleCreateLinkedDocument('page') - }, - { - id: 'create-database', - name: 'Create Database', - description: 'Create a new database and place it on the canvas', - icon: 'database', - shortcut: 'D', - group: 'Canvas', - keywords: ['database', 'canvas', 'create'], - execute: () => void handleCreateLinkedDocument('database') - }, - { - id: 'create-note', - name: 'Create Canvas Note', - description: 'Create a page-backed note and place it on the canvas', - icon: 'sparkles', - shortcut: 'N', - group: 'Canvas', - keywords: ['note', 'canvas', 'create'], - execute: () => handleCreateCanvasNote() - }, - { - id: 'create-rectangle', - name: 'Create Rectangle', - description: 'Create a canvas-native rectangle on the current board', - icon: 'square', - shortcut: 'R', - group: 'Canvas', - keywords: ['shape', 'rectangle', 'canvas', 'create'], - when: () => isCanvasInteractiveShell, - execute: () => { - canvasViewRef.current?.createShape('rectangle') - } - }, - { - id: 'create-frame', - name: 'Create Frame', - description: 'Create an empty frame container on the current board', - icon: 'layout', - shortcut: 'F', - group: 'Canvas', - keywords: ['frame', 'group', 'canvas', 'create'], - when: () => isCanvasInteractiveShell, - execute: () => { - canvasViewRef.current?.createFrame() - } - }, - ...CANVAS_PLANNING_TEMPLATE_DEFINITIONS.map((template) => ({ - id: `create-canvas-template-${template.id}`, - name: `Create ${template.name}`, - description: template.description, - icon: 'layout', - group: 'Canvas', - keywords: ['template', template.category, template.name, 'canvas', 'planning'], - when: () => isCanvasInteractiveShell, - execute: () => { - canvasViewRef.current?.createPlanningTemplate(template.id) - } - })), - { - id: 'frame-selection', - name: 'Frame Selection', - description: 'Wrap the selected canvas objects in a frame container', - icon: 'layout', - shortcut: 'Mod+Shift+F', - group: 'Canvas', - keywords: ['frame', 'group', 'selection', 'canvas'], - when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 0, - execute: () => { - canvasViewRef.current?.wrapSelectionInFrame() - } - }, - { - id: 'canvas-refresh-query-frame', - name: 'Refresh Query Frame', - description: - canvasCommandState.selectedTitle && canvasCommandState.selectedIsQueryFrame - ? `Refresh ${canvasCommandState.selectedTitle}` - : 'Refresh the selected query frame', - icon: 'refresh-cw', - group: 'Canvas', - keywords: ['refresh', 'query', 'frame', 'lens', 'canvas'], - when: () => isCanvasInteractiveShell && canvasCommandState.selectedIsQueryFrame, - execute: () => { - canvasViewRef.current?.refreshSelectedQueryFrame() - } - }, - { - id: 'canvas-connect-selection', - name: 'Connect Selection', - description: 'Create a connector between the two selected canvas objects', - icon: 'link', - shortcut: 'Mod+Shift+K', - group: 'Canvas', - keywords: ['connect', 'connector', 'edge', 'selection', 'canvas'], - when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount === 2, - execute: () => { - canvasViewRef.current?.connectSelection() - } - }, - { - id: 'canvas-rename-alias', - name: 'Rename Canvas Alias', - description: - canvasCommandState.selectedTitle && canvasCommandState.selectionCount === 1 - ? `Rename the canvas copy of ${canvasCommandState.selectedTitle}` - : 'Rename the selected canvas object without changing the source title', - icon: 'pencil', - shortcut: 'Mod+Shift+A', - group: 'Canvas', - keywords: ['alias', 'rename', 'selection', 'canvas'], - when: () => - isCanvasInteractiveShell && - canvasCommandState.selectionCount === 1 && - Boolean(canvasCommandState.selectedSourceId), - execute: () => { - canvasViewRef.current?.openAliasEditor() - } - }, - { - id: 'canvas-clear-alias', - name: 'Clear Canvas Alias', - description: 'Remove the canvas-local alias from the selected object', - icon: 'x', - group: 'Canvas', - keywords: ['alias', 'clear', 'selection', 'canvas'], - when: () => - isCanvasInteractiveShell && - canvasCommandState.selectionCount === 1 && - Boolean(canvasCommandState.selectedSourceId), - execute: () => { - canvasViewRef.current?.clearSelectionAlias() - } - }, - { - id: 'canvas-comment-selection', - name: 'Comment on Selection', - description: - canvasCommandState.selectedTitle && canvasCommandState.selectionCount === 1 - ? `Add a canvas-anchored comment to ${canvasCommandState.selectedTitle}` - : 'Add a canvas-anchored comment to the selected object', - icon: 'message-square', - shortcut: 'Mod+Shift+C', - group: 'Canvas', - keywords: ['comment', 'selection', 'canvas', 'feedback'], - when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount === 1, - execute: () => { - canvasViewRef.current?.openCommentComposer() - } - }, - { - id: 'canvas-show-linked-copies', - name: 'Show Linked Copies', - description: 'Inspect other canvas objects that point at the same source node', - icon: 'copy', - group: 'Canvas', - keywords: ['references', 'copies', 'linked', 'canvas'], - when: () => - isCanvasInteractiveShell && - canvasCommandState.selectionCount === 1 && - Boolean(canvasCommandState.selectedSourceId), - execute: () => { - canvasViewRef.current?.toggleSourceReferences(true) - } - }, - { - id: 'canvas-peek-selection', - name: 'Peek Selected Object', - description: - canvasCommandState.selectedTitle && canvasCommandState.selectionCount === 1 - ? `Center and activate ${canvasCommandState.selectedTitle}` - : 'Center and activate the current canvas selection', - icon: 'eye', - shortcut: 'Enter', - group: 'Canvas', - keywords: ['peek', 'edit', 'selection', 'canvas'], - when: () => shellState.kind === 'canvas-home' && canvasCommandState.selectionCount === 1, - execute: () => { - canvasViewRef.current?.openSelection('peek') - } - }, - { - id: 'canvas-open-selection', - name: 'Open Selected Object', - description: - canvasCommandState.selectedTitle && canvasCommandState.selectionCount === 1 - ? `Open ${canvasCommandState.selectedTitle} in a focused surface` - : 'Open the current canvas selection in a focused surface', - icon: 'external-link', - shortcut: MOD_ENTER_SHORTCUT, - group: 'Canvas', - keywords: ['open', 'focus', 'selection', 'canvas'], - when: () => - isCanvasInteractiveShell && - canvasCommandState.selectionCount === 1 && - Boolean(canvasCommandState.selectedSourceId && canvasCommandState.selectedSourceType), - execute: () => { - canvasViewRef.current?.openSelection('focus') - } - }, - { - id: 'canvas-open-database-split', - name: 'Open Database in Split View', - description: - canvasCommandState.selectedTitle && canvasCommandState.selectionCount === 1 - ? `Keep ${canvasCommandState.selectedTitle} open beside the canvas` - : 'Open the selected database in a split view beside the canvas', - icon: 'columns', - shortcut: 'Alt+Enter', - group: 'Canvas', - keywords: ['split', 'database', 'canvas', 'preview'], - when: () => - isCanvasInteractiveShell && - canvasCommandState.selectionCount === 1 && - canvasCommandState.selectedDisplayType === 'database' && - Boolean(canvasCommandState.selectedSourceId), - execute: () => { - canvasViewRef.current?.openSelection('split') - } - }, - { - id: 'canvas-fit-selection', - name: 'Fit Selected Object', - description: 'Center the current canvas selection in view', - icon: 'layout', - group: 'Canvas', - keywords: ['fit', 'selection', 'zoom', 'canvas'], - when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 0, - execute: () => { - canvasViewRef.current?.fitSelection() - } - }, - { - id: 'canvas-toggle-lock', - name: canvasCommandState.selectionAllLocked ? 'Unlock Selection' : 'Lock Selection', - description: canvasCommandState.selectionAllLocked - ? 'Allow the current selection to move and resize again' - : 'Protect the current selection from accidental moves and nudges', - icon: 'lock', - shortcut: 'Mod+Shift+L', - group: 'Canvas', - keywords: ['lock', 'unlock', 'selection', 'canvas'], - when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 0, - execute: () => { - canvasViewRef.current?.toggleSelectionLock() - } - }, - { - id: 'canvas-align-left', - name: 'Align Selection Left', - description: 'Snap the selected objects to a shared left edge', - icon: 'align-start-horizontal', - shortcut: 'Mod+Shift+Left', - group: 'Canvas', - keywords: ['align', 'left', 'selection', 'canvas'], - when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 1, - execute: () => { - canvasViewRef.current?.alignSelection('left') - } - }, - { - id: 'canvas-align-right', - name: 'Align Selection Right', - description: 'Snap the selected objects to a shared right edge', - icon: 'align-end-horizontal', - shortcut: 'Mod+Shift+Right', - group: 'Canvas', - keywords: ['align', 'right', 'selection', 'canvas'], - when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 1, - execute: () => { - canvasViewRef.current?.alignSelection('right') - } - }, - { - id: 'canvas-align-top', - name: 'Align Selection Top', - description: 'Snap the selected objects to a shared top edge', - icon: 'align-start-vertical', - shortcut: 'Mod+Shift+Up', - group: 'Canvas', - keywords: ['align', 'top', 'selection', 'canvas'], - when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 1, - execute: () => { - canvasViewRef.current?.alignSelection('top') - } - }, - { - id: 'canvas-align-bottom', - name: 'Align Selection Bottom', - description: 'Snap the selected objects to a shared bottom edge', - icon: 'align-end-vertical', - shortcut: 'Mod+Shift+Down', - group: 'Canvas', - keywords: ['align', 'bottom', 'selection', 'canvas'], - when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 1, - execute: () => { - canvasViewRef.current?.alignSelection('bottom') - } - }, - { - id: 'canvas-distribute-horizontal', - name: 'Distribute Selection Horizontally', - description: 'Even out the horizontal spacing between selected objects', - icon: 'columns', - group: 'Canvas', - keywords: ['distribute', 'horizontal', 'selection', 'canvas'], - when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 2, - execute: () => { - canvasViewRef.current?.distributeSelection('horizontal') - } - }, - { - id: 'canvas-distribute-vertical', - name: 'Distribute Selection Vertically', - description: 'Even out the vertical spacing between selected objects', - icon: 'rows', - group: 'Canvas', - keywords: ['distribute', 'vertical', 'selection', 'canvas'], - when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 2, - execute: () => { - canvasViewRef.current?.distributeSelection('vertical') - } - }, - { - id: 'canvas-tidy-selection', - name: 'Tidy Selection', - description: 'Pack the selected objects into a clean reading grid', - icon: 'sparkles', - group: 'Canvas', - keywords: ['tidy', 'arrange', 'selection', 'canvas'], - when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 1, - execute: () => { - canvasViewRef.current?.tidySelection() - } - }, - { - id: 'canvas-cluster-selection', - name: 'Cluster Selection', - description: 'Pull selected objects into a compact planning cluster', - icon: 'sparkles', - group: 'Canvas', - keywords: ['cluster', 'arrange', 'selection', 'canvas'], - when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 1, - execute: () => { - canvasViewRef.current?.clusterSelection() - } - }, - { - id: 'canvas-stack-selection', - name: 'Stack Selection', - description: 'Stack selected objects into an offset pile', - icon: 'layers', - group: 'Canvas', - keywords: ['stack', 'pile', 'arrange', 'selection', 'canvas'], - when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 1, - execute: () => { - canvasViewRef.current?.stackSelection() - } - }, - { - id: 'canvas-convert-selection-mind-map', - name: 'Convert Selection To Mind Map', - description: 'Create a mind-map root and convert the selected objects into branches', - icon: 'git-branch', - group: 'Canvas', - keywords: ['convert', 'mind map', 'selection', 'canvas'], - when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 0, - execute: () => { - canvasViewRef.current?.convertSelectionToMindMap() - } - }, - { - id: 'canvas-send-backward', - name: 'Send Selection Backward', - description: 'Move the selected objects back one layer', - icon: 'minus', - shortcut: '[', - group: 'Canvas', - keywords: ['backward', 'z-index', 'layer', 'selection', 'canvas'], - when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 0, - execute: () => { - canvasViewRef.current?.shiftSelectionLayer('backward') - } - }, - { - id: 'canvas-bring-forward', - name: 'Bring Selection Forward', - description: 'Move the selected objects forward one layer', - icon: 'plus', - shortcut: ']', - group: 'Canvas', - keywords: ['forward', 'z-index', 'layer', 'selection', 'canvas'], - when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 0, - execute: () => { - canvasViewRef.current?.shiftSelectionLayer('forward') - } - }, - { - id: 'canvas-clear-selection', - name: 'Clear Selection', - description: 'Clear the current canvas selection', - icon: 'x', - shortcut: 'Esc', - group: 'Canvas', - keywords: ['clear', 'selection', 'canvas'], - when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 0, - execute: () => { - canvasViewRef.current?.clearSelection() - } - }, - { - id: 'canvas-shortcut-help', - name: canvasCommandState.shortcutHelpOpen - ? 'Hide Canvas Shortcuts' - : 'Show Canvas Shortcuts', - description: 'Toggle the canvas shortcut help overlay', - icon: 'help-circle', - shortcut: '?', - group: 'Canvas', - keywords: ['help', 'shortcuts', 'canvas', 'hotkeys'], - when: () => isCanvasInteractiveShell, - execute: () => { - canvasViewRef.current?.toggleShortcutHelp() - } - }, - { - id: 'open-settings', - name: 'Open Settings', - description: 'Open the system settings overlay', - icon: 'settings', - execute: handleOpenSettings - }, - { - id: 'open-social-import', - name: 'Import Social Archive', - description: 'Open the social graph archive importer', - icon: 'upload', - group: 'Data', - keywords: ['social', 'archive', 'instagram', 'grok', 'import'], - execute: handleOpenSocialImport - }, - { - id: 'open-data-workspace', - name: 'Open Data Workspace', - description: 'Explore saved views, graph lenses, and imported data counts', - icon: 'database', - group: 'Data', - keywords: ['data', 'workspace', 'social', 'saved views', 'lenses'], - execute: handleOpenDataWorkspace - }, - ...(STORIES_ENABLED - ? [ - { - id: 'open-stories', - name: 'Open Stories', - description: 'Open the dev-only embedded Storybook surface', - icon: 'layout', - group: 'Developer', - execute: handleOpenStories - } satisfies PaletteCommand - ] - : []), - ...recentDocuments.map((document) => ({ - id: `open-${document.id}`, - name: document.title, - description: `Open ${document.type}`, - icon: - document.type === 'page' - ? 'file-text' - : document.type === 'database' - ? 'database' - : 'layout', - group: 'Recent', - execute: () => handleOpenDocument(document.id) - })) - ], - [ - handleCreateCanvasNote, - handleCreateLinkedDocument, - handleOpenDocument, - handleOpenDataWorkspace, - handleOpenSettings, - handleOpenSocialImport, - handleOpenStories, - canvasCommandState, - isCanvasInteractiveShell, - recentDocuments, - shellState.kind - ] - ) + const paletteCommands = useShellPaletteCommands({ + canvasViewRef, + canvasCommandState, + isCanvasInteractiveShell, + shellKind: shellState.kind, + recentDocuments, + handleCreateLinkedDocument, + handleCreateCanvasNote, + handleOpenDocument, + handleOpenSettings, + handleOpenSocialImport, + handleOpenDataWorkspace, + handleOpenStories + }) const renderOverlay = () => { const overlaySurfaceClassName = [ @@ -1122,12 +286,8 @@ export function App(): React.ReactElement { onCreatePage={() => void handleCreateLinkedDocument('page')} onCreateDatabase={() => void handleCreateLinkedDocument('database')} onCreateNote={handleCreateCanvasNote} - onCommandStateChange={setCanvasCommandState} - onPendingInsertConsumed={(requestId) => { - setPendingCanvasInsert((current) => - current?.requestId === requestId ? null : current - ) - }} + onCommandStateChange={handleCommandStateChange} + onPendingInsertConsumed={handlePendingInsertConsumed} onOpenDocument={(docId, docType) => focusDocument(docId, docType, true)} onOpenDatabaseSplit={openDatabaseSplit} /> diff --git a/apps/electron/src/renderer/components/DataWorkspaceView.tsx b/apps/electron/src/renderer/components/DataWorkspaceView.tsx index 9f50aa345..59de23f66 100644 --- a/apps/electron/src/renderer/components/DataWorkspaceView.tsx +++ b/apps/electron/src/renderer/components/DataWorkspaceView.tsx @@ -1,561 +1,49 @@ -import type { SavedViewDescriptor } from '@xnetjs/data' -import { SavedViewSchema, validateSavedViewDescriptor } from '@xnetjs/data' -import { - SavedViewRunner, - useMutate, - useQuery, - type MutateOp, - type SavedViewLensDraft, - type SavedViewSchemaRegistry, - type SavedViewVisualCanvasProjectionRequest -} from '@xnetjs/react' -import { - listSocialImportJobs, - subscribeSocialImportJobs, - upsertSocialImportJobProgress, - type SocialImportJobProgress -} from '@xnetjs/social/import/core' -import { createDefaultSocialGraphAtlas, type SocialGraphAtlasEntry } from '@xnetjs/social/lenses' -import { - createSocialPatternSavedViewDraft, - detectSocialPatterns, - type SocialPatternKind, - type SocialPatternSuggestion -} from '@xnetjs/social/patterns' -import { - SocialActorSchema, - SocialCollectionSchema, - SocialContentSchema, - SocialConversationSchema, - SocialImportRunSchema, - SocialInteractionSchema, - SocialMessageSchema, - socialSchemas -} from '@xnetjs/social/schemas' -import { - recommendSocialAnalyticsCache, - type SocialAnalyticsCacheRecommendation -} from '@xnetjs/social/workspace' -import { - AlertTriangle, - BarChart3, - Database, - GitBranch, - Import, - Layout, - Loader2, - MessageSquare, - Network, - Save, - Search, - Shield, - Table, - UserRound, - X -} from 'lucide-react' -import React, { useEffect, useMemo, useState } from 'react' -import { - getDefaultSocialWorkspaceSeeds, - upsertDefaultSocialWorkspace, - type SocialWorkspaceSeedSummary -} from '../lib/social-workspace' +/** + * DataWorkspaceView (desktop) — overlay chrome around the shared Data + * Workspace core (@xnetjs/views, exploration 0276). Desktop-specific + * concerns: IPC-backed seeding + import-job progress, the close affordance, + * and inserting saved lenses onto the canvas as frames. + */ +import { upsertSocialImportJobProgress } from '@xnetjs/social/import/core' +import { useDataWorkspace, DataWorkspaceBody, type SavedViewCanvasFrameInput } from '@xnetjs/views' +import { Database, Import, Loader2, X } from 'lucide-react' +import React, { useEffect } from 'react' + +export type { SavedViewCanvasFrameInput } type DataWorkspaceViewProps = { onClose: () => void onInsertSavedLensAsCanvasFrame?: (input: SavedViewCanvasFrameInput) => void } -export type SavedViewCanvasFrameInput = { - id: string - title?: string - description?: string - descriptor?: string -} - -type SavedViewRow = { - id: string - title?: string - description?: string - descriptor?: string - scope?: string -} - -type ParsedDescriptor = { - valid: boolean - queryKind: string - queryMode: string | null - primarySchemaId: string | null -} - -type WorkspaceMetric = { - id: string - label: string - value: number | null - icon: typeof UserRound -} - -type GraphAtlasRow = { - entry: SocialGraphAtlasEntry - savedView: SavedViewRow | null -} - -const SOCIAL_SCHEMA_REGISTRY = socialSchemas as unknown as SavedViewSchemaRegistry -const PATTERN_QUERY_LIMIT = 300 -const DISMISSED_PATTERN_STORAGE_KEY = 'xnet:data-workspace:dismissed-patterns' - -function getCount(input: { totalCount: number | null; data: unknown[] }): number | null { - return input.totalCount ?? (input.data.length > 0 ? input.data.length : null) -} - -function parseSavedViewDescriptor(value: string | undefined): ParsedDescriptor { - if (!value) { - return { - valid: false, - queryKind: 'unknown', - queryMode: null, - primarySchemaId: null - } - } - - try { - const descriptor = JSON.parse(value) as SavedViewDescriptor - const validation = validateSavedViewDescriptor(descriptor) - const query = descriptor.query as Record - const queryKind = typeof query.kind === 'string' ? query.kind : 'unknown' - const queryMode = typeof query.mode === 'string' ? query.mode : null - const primarySchemaId = - queryKind === 'query-set' ? primarySchemaIdForQuerySet(query) : primarySchemaIdForQuery(query) - - return { - valid: validation.valid, - queryKind, - queryMode, - primarySchemaId - } - } catch { - return { - valid: false, - queryKind: 'invalid-json', - queryMode: null, - primarySchemaId: null - } - } -} - -function parseSavedViewDescriptorObject(value: string | undefined): SavedViewDescriptor | null { - if (!value) return null - - try { - const descriptor = JSON.parse(value) as SavedViewDescriptor - return validateSavedViewDescriptor(descriptor).valid ? descriptor : null - } catch { - return null - } -} - -function primarySchemaIdForQuery(query: Record): string | null { - const schema = query.schema as Record | undefined - return typeof schema?.id === 'string' - ? schema.id - : typeof schema?.['@id'] === 'string' - ? schema['@id'] - : typeof query.schemaId === 'string' - ? query.schemaId - : null -} - -function primarySchemaIdForQuerySet(query: Record): string | null { - const queries = query.queries as Record> | undefined - const firstQuery = queries ? Object.values(queries)[0] : null - return firstQuery ? primarySchemaIdForQuery(firstQuery) : null -} - -function metricValueLabel(value: number | null): string { - return value === null ? '-' : value.toLocaleString() -} - -function sumKnownCounts(values: readonly (number | null)[]): number { - return values.reduce((total, value) => total + (value ?? 0), 0) -} - -function descriptorKindLabel(descriptor: ParsedDescriptor): string { - if (!descriptor.valid) return 'Invalid' - if (descriptor.queryKind === 'query-set') return descriptor.queryMode ?? 'query set' - return descriptor.queryKind -} - -function isVisibleSocialImportJob(job: SocialImportJobProgress): boolean { - if (job.status !== 'completed') return true - return Date.now() - job.updatedAt < 5 * 60 * 1000 -} - -function socialImportJobPercent(job: SocialImportJobProgress): number { - if (!job.totalRecords || job.totalRecords <= 0) return job.status === 'completed' ? 100 : 0 - return Math.min(100, Math.max(0, (job.processedRecords / job.totalRecords) * 100)) -} - -function socialImportJobStatusLabel(job: SocialImportJobProgress): string { - if (job.status === 'queued') return 'Queued' - if (job.status === 'running') return 'Running' - if (job.status === 'paused') return 'Paused' - if (job.status === 'completed') return 'Complete' - if (job.status === 'failed') return 'Failed' - return 'Cancelled' -} - -function socialImportJobRecordLabel(job: SocialImportJobProgress): string { - if (!job.totalRecords) return job.processedRecords.toLocaleString() - return `${job.processedRecords.toLocaleString()} / ${job.totalRecords.toLocaleString()}` -} - -function socialImportJobRateLabel(job: SocialImportJobProgress): string { - const recordsPerSecond = job.metrics?.recordsPerSecond ?? 0 - if (!Number.isFinite(recordsPerSecond) || recordsPerSecond <= 0) return '0/s' - return `${Math.round(recordsPerSecond).toLocaleString()}/s` -} - -function readDismissedPatternIds(): string[] { - if (typeof localStorage === 'undefined') return [] - - try { - const value = JSON.parse(localStorage.getItem(DISMISSED_PATTERN_STORAGE_KEY) ?? '[]') - return Array.isArray(value) - ? value.flatMap((item) => (typeof item === 'string' ? [item] : [])) - : [] - } catch { - return [] - } -} - -function writeDismissedPatternIds(ids: readonly string[]): void { - if (typeof localStorage === 'undefined') return - - localStorage.setItem(DISMISSED_PATTERN_STORAGE_KEY, JSON.stringify([...new Set(ids)].sort())) -} - -function toPatternRows(rows: readonly unknown[]): Record[] { - return rows as unknown as Record[] -} - -function patternIconFor(kind: SocialPatternKind): typeof BarChart3 { - if (kind === 'privacy-hotspots') return Shield - if (kind === 'cross-source-overlap') return Search - if (kind === 'bridge-actors') return Network - if (kind === 'unrevisited-saves') return Import - if (kind === 'attention-bursts') return BarChart3 - return BarChart3 -} - export function DataWorkspaceView({ onClose, onInsertSavedLensAsCanvasFrame }: DataWorkspaceViewProps): React.ReactElement { - const { create, mutate } = useMutate() - const [socialImportJobs, setSocialImportJobs] = - useState(listSocialImportJobs) - const [seedSummary, setSeedSummary] = useState(null) - const [seeding, setSeeding] = useState(false) - const [seedError, setSeedError] = useState(null) - const [saveLensMessage, setSaveLensMessage] = useState(null) - const [saveLensError, setSaveLensError] = useState(null) - const [selectedViewId, setSelectedViewId] = useState(null) - const [dismissedPatternIds, setDismissedPatternIds] = useState(readDismissedPatternIds) - const { data: savedViews, loading: savedViewsLoading } = useQuery(SavedViewSchema, { - orderBy: { title: 'asc' }, - limit: 200 - }) - const actorQuery = useQuery(SocialActorSchema, { page: { first: 1, count: 'estimate' } }) - const contentQuery = useQuery(SocialContentSchema, { - page: { first: PATTERN_QUERY_LIMIT, count: 'estimate' }, - orderBy: { importedAt: 'desc' } - }) - const interactionQuery = useQuery(SocialInteractionSchema, { - page: { first: PATTERN_QUERY_LIMIT, count: 'estimate' }, - orderBy: { importedAt: 'desc' } - }) - const messageQuery = useQuery(SocialMessageSchema, { page: { first: 1, count: 'estimate' } }) - const conversationQuery = useQuery(SocialConversationSchema, { - page: { first: 1, count: 'estimate' } - }) - const collectionQuery = useQuery(SocialCollectionSchema, { - page: { first: 1, count: 'estimate' } - }) - const importRunQuery = useQuery(SocialImportRunSchema, { - page: { first: 50, count: 'estimate' }, - orderBy: { startedAt: 'desc' } - }) - - const defaultSeeds = useMemo(() => getDefaultSocialWorkspaceSeeds(), []) - const defaultSeedIds = useMemo( - () => new Set(defaultSeeds.map((seed) => seed.deterministicId)), - [defaultSeeds] - ) - const defaultSeedBySourceId = useMemo( - () => new Map(defaultSeeds.map((seed) => [seed.id, seed])), - [defaultSeeds] - ) - const graphAtlasEntries = useMemo(() => createDefaultSocialGraphAtlas({ pageSize: 100 }), []) - const socialWorkspaceViews = useMemo( - () => (savedViews as SavedViewRow[]).filter((view) => defaultSeedIds.has(view.id)), - [defaultSeedIds, savedViews] - ) - const otherSavedViews = useMemo( - () => (savedViews as SavedViewRow[]).filter((view) => !defaultSeedIds.has(view.id)), - [defaultSeedIds, savedViews] - ) - const allSavedViews = useMemo( - () => [...socialWorkspaceViews, ...otherSavedViews], - [otherSavedViews, socialWorkspaceViews] - ) - const selectedView = useMemo( - () => - allSavedViews.find((view) => view.id === selectedViewId) ?? - socialWorkspaceViews[0] ?? - allSavedViews[0] ?? - null, - [allSavedViews, selectedViewId, socialWorkspaceViews] - ) - const metrics: WorkspaceMetric[] = [ - { - id: 'actors', - label: 'People', - value: getCount(actorQuery), - icon: UserRound - }, - { - id: 'content', - label: 'Content', - value: getCount(contentQuery), - icon: Table - }, - { - id: 'interactions', - label: 'Interactions', - value: getCount(interactionQuery), - icon: Network - }, - { - id: 'messages', - label: 'Messages', - value: getCount(messageQuery), - icon: MessageSquare - }, - { - id: 'conversations', - label: 'Conversations', - value: getCount(conversationQuery), - icon: GitBranch - }, - { - id: 'collections', - label: 'Collections', - value: getCount(collectionQuery), - icon: Database - }, - { - id: 'import-runs', - label: 'Import Runs', - value: getCount(importRunQuery), - icon: Import - } - ] - const analyticsCacheRecommendation = recommendSocialAnalyticsCache({ - rowCount: sumKnownCounts(metrics.map((metric) => metric.value)), - columnCount: 12, - relationCount: getCount(interactionQuery) ?? 0 + const workspace = useDataWorkspace({ + getExistingNode: (id) => window.xnetNodes.getNode(id), + onInsertSavedLensAsCanvasFrame }) - const dismissedPatternIdSet = useMemo(() => new Set(dismissedPatternIds), [dismissedPatternIds]) - const patternSuggestions = useMemo( - () => - detectSocialPatterns({ - content: toPatternRows(contentQuery.data), - interactions: toPatternRows(interactionQuery.data), - importRuns: toPatternRows(importRunQuery.data) - }).filter((pattern) => !dismissedPatternIdSet.has(pattern.id)), - [contentQuery.data, dismissedPatternIdSet, importRunQuery.data, interactionQuery.data] - ) - const graphAtlasRows = useMemo( - () => - graphAtlasEntries.map((entry) => { - const seed = defaultSeedBySourceId.get(entry.id) - const savedView = seed - ? (socialWorkspaceViews.find((view) => view.id === seed.deterministicId) ?? null) - : null - - return { entry, savedView } - }), - [defaultSeedBySourceId, graphAtlasEntries, socialWorkspaceViews] - ) - const visibleSocialImportJobs = useMemo( - () => socialImportJobs.filter(isVisibleSocialImportJob).slice(0, 3), - [socialImportJobs] - ) + const { seeding, handleSeedWorkspace, refreshSocialImportJobs } = workspace + // Bridge main-process commit jobs into the renderer's import-job store so + // progress started from the main process shows up in the shared panel. useEffect(() => { - if (!selectedViewId && selectedView) { - setSelectedViewId(selectedView.id) - return - } - - if (selectedViewId && !allSavedViews.some((view) => view.id === selectedViewId)) { - setSelectedViewId(selectedView?.id ?? null) - } - }, [allSavedViews, selectedView, selectedViewId]) - - useEffect(() => { - const syncSocialImportJobs = () => setSocialImportJobs(listSocialImportJobs()) - const unsubscribeLocal = subscribeSocialImportJobs(syncSocialImportJobs) - void window.xnetSocialImport .listCommitJobs() .then((jobs) => { jobs.forEach(upsertSocialImportJobProgress) - syncSocialImportJobs() + refreshSocialImportJobs() }) .catch(() => undefined) - const unsubscribeElectron = window.xnetSocialImport.onCommitJob((job) => { + const unsubscribe = window.xnetSocialImport.onCommitJob((job) => { upsertSocialImportJobProgress(job) - syncSocialImportJobs() - }) - - return () => { - unsubscribeLocal() - unsubscribeElectron() - } - }, []) - - async function handleSeedWorkspace(): Promise { - setSeeding(true) - setSeedError(null) - - try { - const summary = await upsertDefaultSocialWorkspace({ - mutate, - getExisting: (id) => window.xnetNodes.getNode(id) - }) - setSeedSummary(summary) - } catch (error) { - setSeedError(error instanceof Error ? error.message : String(error)) - } finally { - setSeeding(false) - } - } - - async function handleSaveLens(draft: SavedViewLensDraft): Promise { - setSaveLensMessage(null) - setSaveLensError(null) - - try { - const savedView = await create(SavedViewSchema, { - title: draft.title, - description: draft.description, - descriptor: JSON.stringify(draft.descriptor), - scope: draft.descriptor.scope ?? 'workspace' - }) - - if (!savedView) { - throw new Error('Saved lens could not be created.') - } - - setSelectedViewId(savedView.id) - setSaveLensMessage(`Saved lens: ${draft.title}.`) - } catch (error) { - setSaveLensError(error instanceof Error ? error.message : String(error)) - throw error - } - } - - function handleOpenPattern(pattern: SocialPatternSuggestion): void { - const view = socialWorkspaceViews.find((candidate) => candidate.title === pattern.viewHint) - if (view) { - setSelectedViewId(view.id) - } - } - - async function upsertPatternSavedView( - pattern: SocialPatternSuggestion - ): Promise { - setSaveLensMessage(null) - setSaveLensError(null) - - const baseView = socialWorkspaceViews.find((candidate) => candidate.title === pattern.viewHint) - const baseDescriptor = parseSavedViewDescriptorObject(baseView?.descriptor) - - if (!baseView || !baseDescriptor) { - setSaveLensError(`Seed the ${pattern.viewHint} view before saving this pattern.`) - return null - } - - const draft = createSocialPatternSavedViewDraft({ pattern, baseDescriptor }) - if (!draft) { - setSaveLensError('Pattern lens could not be created from the base view.') - return null - } - - const existing = allSavedViews.some((view) => view.id === draft.deterministicId) - const operation: MutateOp = existing - ? { - type: 'update', - id: draft.deterministicId, - data: draft.savedViewProperties - } - : { - type: 'create', - id: draft.deterministicId, - schema: SavedViewSchema, - data: draft.savedViewProperties - } - - await mutate([operation]) - - const savedView = { - id: draft.deterministicId, - ...draft.savedViewProperties - } - setSelectedViewId(savedView.id) - setSaveLensMessage(`${existing ? 'Updated' : 'Saved'} pattern lens: ${draft.title}.`) - return savedView - } - - async function handleSavePattern(pattern: SocialPatternSuggestion): Promise { - await upsertPatternSavedView(pattern) - } - - async function handlePinPattern(pattern: SocialPatternSuggestion): Promise { - if (!onInsertSavedLensAsCanvasFrame) return - - const savedView = await upsertPatternSavedView(pattern) - if (!savedView) return - - onInsertSavedLensAsCanvasFrame(savedView) - } - - function handleOpenVisualCanvasProjection(request: SavedViewVisualCanvasProjectionRequest): void { - if (!onInsertSavedLensAsCanvasFrame) return - - const descriptorJson = - typeof request.descriptor === 'string' - ? request.descriptor - : request.descriptor - ? JSON.stringify(request.descriptor) - : selectedView?.descriptor - - onInsertSavedLensAsCanvasFrame({ - id: selectedView?.id ?? request.id, - title: request.title, - ...(request.description ? { description: request.description } : {}), - ...(descriptorJson ? { descriptor: descriptorJson } : {}) + refreshSocialImportJobs() }) - } - function handleDismissPattern(patternId: string): void { - setDismissedPatternIds((current) => { - const next = [...new Set([...current, patternId])] - writeDismissedPatternIds(next) - return next - }) - } + return unsubscribe + }, [refreshSocialImportJobs]) return (
@@ -596,594 +84,9 @@ export function DataWorkspaceView({

- {seedSummary ? ( - - ) : null} - {seedError ? : null} - {saveLensMessage ? : null} - {saveLensError ? : null} - - -
- {metrics.map((metric) => { - const Icon = metric.icon - - return ( -
-
- {metric.label} - -
-
- {metricValueLabel(metric.value)} -
-
- ) - })} -
- - setSelectedViewId(view.id)} - onInsertCanvasFrame={onInsertSavedLensAsCanvasFrame} - /> - -
- - -
-
-
-
-

Social Starter Lenses

-

- Schema views and graph-lens query sets persisted as saved views. -

-
- {savedViewsLoading ? ( -
- - Loading -
- ) : null} -
- -
- - - -
-
-

Other Saved Views

-

- General saved views will use the same workspace surface as more importers land. -

-
- -
-
-
- - - - ) -} - -function SocialImportJobsPanel({ - jobs -}: { - jobs: SocialImportJobProgress[] -}): React.ReactElement | null { - if (jobs.length === 0) return null - - return ( -
- -
- {jobs.map((job) => { - const percent = socialImportJobPercent(job) - const statusLabel = socialImportJobStatusLabel(job) - - return ( -
-
-
-
- {job.status === 'running' || job.status === 'queued' ? ( - - ) : ( - - )} -
{job.archiveName}
-
-
- {job.platform} / {statusLabel} / {job.phase} -
-
-
- {Math.floor(percent)}% -
-
-
-
-
-
- - - - -
- {job.error ? ( -
- - {job.error} -
- ) : null} -
- ) - })} -
-
- ) -} - -function JobMetric({ label, value }: { label: string; value: string }): React.ReactElement { - return ( -
-
{label}
-
{value}
-
- ) -} - -function SavedViewTable({ - views, - selectedViewId, - emptyLabel, - onSelect, - onInsertCanvasFrame -}: { - views: SavedViewRow[] - selectedViewId: string | null - emptyLabel: string - onSelect: (viewId: string) => void - onInsertCanvasFrame?: (input: SavedViewCanvasFrameInput) => void -}): React.ReactElement { - if (views.length === 0) { - return ( -
- {emptyLabel} -
- ) - } - - return ( -
- - - - - - - - {onInsertCanvasFrame ? : null} - - - - {views.map((view) => { - const descriptor = parseSavedViewDescriptor(view.descriptor) - const selected = view.id === selectedViewId - - return ( - - - - - - {onInsertCanvasFrame ? ( - - ) : null} - - ) - })} - -
ViewKindScopeSchemaCanvas
- - - {descriptorKindLabel(descriptor)} - {view.scope ?? '-'} - {descriptor.primarySchemaId ?? '-'} - - -
-
- ) -} - -function GraphAtlasPanel({ - rows, - selectedViewId, - onOpen, - onInsertCanvasFrame -}: { - rows: GraphAtlasRow[] - selectedViewId: string | null - onOpen: (view: SavedViewRow) => void - onInsertCanvasFrame?: (input: SavedViewCanvasFrameInput) => void -}): React.ReactElement { - return ( -
-
-
-

Graph Atlas

-

- Starter graph lenses organized by node roles, relationship rules, and saved-view state. -

-
- - {rows.filter((row) => row.savedView).length}/{rows.length} seeded - -
-
- {rows.map((row) => ( - - ))} -
-
- ) -} - -function GraphAtlasCard({ - row, - selected, - onOpen, - onInsertCanvasFrame -}: { - row: GraphAtlasRow - selected: boolean - onOpen: (view: SavedViewRow) => void - onInsertCanvasFrame?: (input: SavedViewCanvasFrameInput) => void -}): React.ReactElement { - const { entry, savedView } = row - - return ( -
-
-
-
- -

{entry.title}

-
-

{entry.description}

-
- - {savedView ? 'saved' : 'seed'} - -
-
- - - -
-
- {entry.nodeRoles.slice(0, 3).map((role) => ( - - {role.role} - - ))} - {entry.relationshipKinds.slice(0, 3).map((kind) => ( - - {kind} - - ))} -
-
- - {onInsertCanvasFrame ? ( - - ) : null} -
-
- ) -} - -function GraphAtlasMetric({ label, value }: { label: string; value: number }): React.ReactElement { - return ( -
-
{label}
-
{value.toLocaleString()}
-
- ) -} - -function SectionLabel({ label }: { label: string }): React.ReactElement { - return ( -
{label}
- ) -} - -function SourceRow({ label, value }: { label: string; value: string }): React.ReactElement { - return ( -
- {label} - {value} -
- ) -} - -function AnalyticsCacheRow({ - recommendation -}: { - recommendation: SocialAnalyticsCacheRecommendation -}): React.ReactElement { - return ( -
-
- Scale cache - {recommendation.label} -
-

{recommendation.reason}

-
- - {recommendation.estimatedRows.toLocaleString()} rows - - - {recommendation.estimatedCells.toLocaleString()} cells - -
-
- ) -} - -function PatternRow({ - icon: Icon, - pattern, - onOpen, - onSave, - onPin, - onDismiss -}: { - icon: typeof BarChart3 - pattern: SocialPatternSuggestion - onOpen: (pattern: SocialPatternSuggestion) => void - onSave: (pattern: SocialPatternSuggestion) => void - onPin?: (pattern: SocialPatternSuggestion) => void - onDismiss: (patternId: string) => void -}): React.ReactElement { - return ( -
-
- -
-
{pattern.title}
-
- {pattern.description} -
-
-
-
- - {pattern.evidenceCount.toLocaleString()} evidence - - {pattern.platforms.slice(0, 2).map((platform) => ( - - {platform} - - ))} - {pattern.privacyClasses.slice(0, 2).map((privacyClass) => ( - - {privacyClass} - - ))} - {pattern.sourceImportRunIds.length > 0 ? ( - - {pattern.sourceImportRunIds.length} runs - - ) : null} -
- {pattern.evidence.length > 0 ? ( -
- {pattern.evidence.slice(0, 2).map((item) => ( -
- {item.value} - {item.count.toLocaleString()} -
- ))} +
- ) : null} -
- - - {onPin ? ( - - ) : null} -
) } - -function StatusBanner({ - message, - tone -}: { - message: string - tone: 'error' | 'success' | 'warning' -}): React.ReactElement { - const toneClassName = { - error: 'border-destructive/40 bg-destructive/10 text-destructive', - success: 'border-emerald-500/30 bg-emerald-500/10 text-emerald-700 dark:text-emerald-300', - warning: 'border-amber-500/30 bg-amber-500/10 text-amber-700 dark:text-amber-300' - }[tone] - - const Icon = tone === 'success' ? Shield : AlertTriangle - - return ( -
- - {message} -
- ) -} diff --git a/apps/electron/src/renderer/components/PageView.tsx b/apps/electron/src/renderer/components/PageView.tsx index 588787cfd..97e00c703 100644 --- a/apps/electron/src/renderer/components/PageView.tsx +++ b/apps/electron/src/renderer/components/PageView.tsx @@ -4,36 +4,29 @@ * Features: * - Collaborative editing via Yjs * - Comment system with inline popover + * (state machine shared with web via usePageComments, 0276) * - Real-time presence indicators */ import type { SyncStatus } from '@xnetjs/react' import { PageSchema } from '@xnetjs/data' -import { CommentMark, CommentPlugin, restoreCommentMarks } from '@xnetjs/editor/extensions' import { EditorSurface, buildTaskMentionSuggestions, useImageUpload, useFileUpload, useFileDownload, - type Editor + usePageComments } from '@xnetjs/editor/react' import { TaskCollectionEmbed, useNode, useIdentity, useEditorExtensionsSafe, - useComments, usePluginRegistryOptional, usePageTaskSync } from '@xnetjs/react' -import { - CommentPopover, - CommentsSidebar, - OrphanedThreadList, - type CommentThreadData, - type OrphanedThread -} from '@xnetjs/ui' +import { CommentPopover, CommentsSidebar, OrphanedThreadList } from '@xnetjs/ui' import React, { useState, useCallback, useMemo, useRef, useEffect } from 'react' import { DocumentHeader } from './DocumentHeader' import { resolvePageEditorFocusPosition } from './page-editor-focus' @@ -47,31 +40,6 @@ interface PageViewProps { type EditorExtensions = NonNullable['extensions']> -// ─── Comment Popover State ────────────────────────────────────────────────────── - -interface PopoverState { - visible: boolean - mode: 'preview' | 'full' - threadId: string | null - anchor: HTMLElement | null -} - -const INITIAL_POPOVER_STATE: PopoverState = { - visible: false, - mode: 'preview', - threadId: null, - anchor: null -} - -/** State for creating a new comment (before submission) */ -interface NewCommentState { - visible: boolean - anchorData: string - /** Selection range to restore when applying the mark */ - selectionFrom: number - selectionTo: number -} - export function PageView({ docId, minimalChrome = false }: PageViewProps) { const { did } = useIdentity() const onImageUpload = useImageUpload() @@ -112,78 +80,83 @@ export function PageView({ docId, minimalChrome = false }: PageViewProps) { [did, presence] ) - // ─── Comments Integration ───────────────────────────────────────────────────── + // ─── Comments Integration (shared state machine, 0276) ─────────────────────── - // Load comments for this page, filtered to text anchors only const { threads, - addComment, - replyTo, - resolveThread, - reopenThread, - deleteComment, - editComment, - unresolvedCount - } = useComments({ nodeId: docId, anchorType: 'text' }) - - // Popover state for comment interactions - const [popoverState, setPopoverState] = useState(INITIAL_POPOVER_STATE) - const [newCommentState, setNewCommentState] = useState(null) - const [orphanedIds, setOrphanedIds] = useState([]) - const [orphanedCollapsed, setOrphanedCollapsed] = useState(false) + unresolvedCount, + threadDataMap, + sidebarThreads, + currentThread, + orphanedThreads, + orphanedCollapsed, + toggleOrphanedCollapsed, + popoverState, + newCommentState, + editorRef, + handleEditorReady, + commentExtensions, + showThreadPopover, + handlePopoverMouseEnter, + handlePopoverMouseLeave, + handleDismiss, + handleUpgradeToFull, + handleReply, + handleResolve, + handleReopen, + handleDelete, + handleEdit, + handleCreateComment, + handleSubmitNewComment, + handleCancelNewComment, + handleSidebarSelectThread, + handleSidebarReply, + handleSidebarResolve, + handleSidebarReopen, + handleSidebarDelete, + handleSidebarEdit, + handleDismissOrphaned, + handleReattachOrphaned + } = usePageComments({ docId, dismissPopoverOnCaretExit: true }) + const [sidebarOpen, setSidebarOpen] = useState(false) - const hoverTimeoutRef = useRef(null) - const dismissTimeoutRef = useRef(null) - const editorRef = useRef(null) const titleInputRef = useRef(null) - const marksRestoredRef = useRef(false) - const [editorReady, setEditorReady] = useState(false) - - // Reset mark restoration state when switching documents - useEffect(() => { - marksRestoredRef.current = false - editorRef.current = null - setEditorReady(false) - }, [docId]) - - // Handle editor ready - store ref and trigger mark restoration - const handleEditorReady = useCallback((editor: Editor) => { - editorRef.current = editor - setEditorReady(true) - }, []) - const handleEditorSurfaceMouseDown = useCallback((event: React.MouseEvent) => { - const { target } = event - if (!(target instanceof HTMLElement)) return - - const interactiveTarget = target.closest( - [ - '[contenteditable="true"]', - 'a', - 'button', - 'input', - 'select', - 'textarea', - '[role="button"]', - '[data-page-editor-ignore-focus="true"]' - ].join(',') - ) + const handleEditorSurfaceMouseDown = useCallback( + (event: React.MouseEvent) => { + const { target } = event + if (!(target instanceof HTMLElement)) return + + const interactiveTarget = target.closest( + [ + '[contenteditable="true"]', + 'a', + 'button', + 'input', + 'select', + 'textarea', + '[role="button"]', + '[data-page-editor-ignore-focus="true"]' + ].join(',') + ) - if (interactiveTarget || !editorRef.current) { - return - } + if (interactiveTarget || !editorRef.current) { + return + } - event.preventDefault() - const focusPosition = resolvePageEditorFocusPosition( - event.clientY, - editorRef.current.view.dom.getBoundingClientRect() - ) - editorRef.current.commands.focus(focusPosition) - }, []) + event.preventDefault() + const focusPosition = resolvePageEditorFocusPosition( + event.clientY, + editorRef.current.view.dom.getBoundingClientRect() + ) + editorRef.current.commands.focus(focusPosition) + }, + [editorRef] + ) const handleTitleSubmit = useCallback(() => { editorRef.current?.commands.focus('start') - }, []) + }, [editorRef]) const handleBodyBackspaceAtStart = useCallback(() => { const titleInput = titleInputRef.current @@ -195,443 +168,10 @@ export function PageView({ docId, minimalChrome = false }: PageViewProps) { return true }, []) - // Restore comment marks when editor is ready and threads are loaded. - // Both editorReady and threads are in the dependency array so the effect - // fires regardless of which one becomes available first. - useEffect(() => { - if (!editorRef.current || marksRestoredRef.current || threads.length === 0) return - - // Convert threads to format expected by restoreCommentMarks - const commentsToRestore = threads.map((t) => ({ - id: t.root.id, - properties: { - anchorType: t.root.properties.anchorType, - anchorData: t.root.properties.anchorData, - resolved: t.root.properties.resolved - } - })) - - // Restore marks - this will highlight the commented text - const { resolved, orphaned } = restoreCommentMarks(editorRef.current, commentsToRestore) - - if (resolved.length > 0 || orphaned.length > 0) { - marksRestoredRef.current = true - setOrphanedIds(orphaned) - console.log(`[Comments] Restored ${resolved.length} marks, ${orphaned.length} orphaned`) - } - }, [threads, editorReady]) - - // Dismiss the comment popover when the caret moves out of comment marks. - // TipTap's onSelectionUpdate fires after every cursor movement. - useEffect(() => { - const editor = editorRef.current - if (!editor) return - - const onSelectionUpdate = () => { - setPopoverState((prev) => { - if (!prev.visible || !prev.threadId) return prev - const { from } = editor.state.selection - const resolved = editor.state.doc.resolve(from) - const inComment = resolved.marks().some((mark) => { - const typedMark = mark as { - type?: { name?: string } - attrs?: { commentId?: string } - } - return typedMark.type?.name === 'comment' && typedMark.attrs?.commentId === prev.threadId - }) - if (!inComment && !markHoveredRef.current && !popoverHoveredRef.current) { - return INITIAL_POPOVER_STATE - } - return prev - }) - } - - editor.on('selectionUpdate', onSelectionUpdate) - return () => { - editor.off('selectionUpdate', onSelectionUpdate) - } - }, [editorReady]) - - // Build orphaned threads list for display - const orphanedThreads = useMemo((): OrphanedThread[] => { - const result: OrphanedThread[] = [] - - for (const id of orphanedIds) { - const thread = threads.find((t) => t.root.id === id) - if (!thread) continue - - // Parse anchor data to get context - let context: string | undefined - try { - const anchor = JSON.parse(thread.root.properties.anchorData) - context = anchor.quotedText - } catch { - // Ignore parse errors - } - - result.push({ - comment: { - id: thread.root.id, - author: thread.root.properties.createdBy, - authorDisplayName: undefined, - content: thread.root.properties.content, - createdAt: thread.root.createdAt, - replyCount: thread.replies.length - }, - reason: 'text-deleted', - context - }) - } - - return result - }, [orphanedIds, threads]) - - // Convert threads to format expected by CommentPopover - const threadDataMap = useMemo(() => { - const map = new Map() - for (const thread of threads) { - map.set(thread.root.id, { - root: { - id: thread.root.id, - author: thread.root.properties.createdBy, - authorDisplayName: undefined, // TODO: lookup display name - content: thread.root.properties.content, - createdAt: thread.root.createdAt, - edited: thread.root.properties.edited, - editedAt: thread.root.properties.editedAt, - replyToUser: thread.root.properties.replyToUser, - replyToCommentId: thread.root.properties.replyToCommentId - }, - replies: thread.replies.map((r) => ({ - id: r.id, - author: r.properties.createdBy, - authorDisplayName: undefined, - content: r.properties.content, - createdAt: r.createdAt, - edited: r.properties.edited, - editedAt: r.properties.editedAt, - replyToUser: r.properties.replyToUser, - replyToCommentId: r.properties.replyToCommentId - })), - resolved: thread.root.properties.resolved - }) - } - return map - }, [threads]) - - // ─── Popover Handlers ───────────────────────────────────────────────────────── - - // Track whether the cursor is over the mark or the popover. - // The popover stays open as long as either is true. - const markHoveredRef = useRef(false) - const popoverHoveredRef = useRef(false) - - /** Check if the editor caret is currently inside a comment mark matching the popover thread. */ - const isCaretInComment = useCallback((): boolean => { - const editor = editorRef.current - if (!editor) return false - const { from } = editor.state.selection - const resolved = editor.state.doc.resolve(from) - return resolved.marks().some((mark) => { - const typedMark = mark as { type?: { name?: string } } - return typedMark.type?.name === 'comment' - }) - }, []) - - /** Schedule a dismiss after a short delay, unless mark/popover is hovered or caret is in comment. */ - const scheduleDismiss = useCallback(() => { - if (dismissTimeoutRef.current) clearTimeout(dismissTimeoutRef.current) - dismissTimeoutRef.current = setTimeout(() => { - if (!markHoveredRef.current && !popoverHoveredRef.current && !isCaretInComment()) { - setPopoverState(INITIAL_POPOVER_STATE) - } - }, 200) - }, [isCaretInComment]) - - const handleClickComment = useCallback((commentId: string, anchorEl: HTMLElement) => { - if (hoverTimeoutRef.current) { - clearTimeout(hoverTimeoutRef.current) - hoverTimeoutRef.current = null - } - if (dismissTimeoutRef.current) { - clearTimeout(dismissTimeoutRef.current) - dismissTimeoutRef.current = null - } - setPopoverState((prev) => { - // Already showing for this comment — keep as-is to avoid flicker - if (prev.visible && prev.mode === 'full' && prev.threadId === commentId) { - return prev - } - return { visible: true, mode: 'full', threadId: commentId, anchor: anchorEl } - }) - }, []) - - const handleHoverComment = useCallback((commentId: string, anchorEl: HTMLElement) => { - markHoveredRef.current = true - // Cancel any pending dismiss - if (dismissTimeoutRef.current) { - clearTimeout(dismissTimeoutRef.current) - dismissTimeoutRef.current = null - } - // Delay showing to avoid flicker on quick mouse passes - if (hoverTimeoutRef.current) clearTimeout(hoverTimeoutRef.current) - hoverTimeoutRef.current = setTimeout(() => { - setPopoverState((prev) => { - if (prev.visible && prev.threadId === commentId) return prev - return { visible: true, mode: 'full', threadId: commentId, anchor: anchorEl } - }) - }, 300) - }, []) - - const handleLeaveComment = useCallback(() => { - markHoveredRef.current = false - if (hoverTimeoutRef.current) { - clearTimeout(hoverTimeoutRef.current) - hoverTimeoutRef.current = null - } - scheduleDismiss() - }, [scheduleDismiss]) - - const handlePopoverMouseEnter = useCallback(() => { - popoverHoveredRef.current = true - if (dismissTimeoutRef.current) { - clearTimeout(dismissTimeoutRef.current) - dismissTimeoutRef.current = null - } - }, []) - - const handlePopoverMouseLeave = useCallback(() => { - popoverHoveredRef.current = false - scheduleDismiss() - }, [scheduleDismiss]) - - const handleDismiss = useCallback(() => { - if (hoverTimeoutRef.current) { - clearTimeout(hoverTimeoutRef.current) - hoverTimeoutRef.current = null - } - if (dismissTimeoutRef.current) { - clearTimeout(dismissTimeoutRef.current) - dismissTimeoutRef.current = null - } - markHoveredRef.current = false - popoverHoveredRef.current = false - setPopoverState(INITIAL_POPOVER_STATE) - }, []) - - const handleUpgradeToFull = useCallback(() => { - setPopoverState((prev) => ({ ...prev, mode: 'full' })) - }, []) - - // ─── Comment Actions ────────────────────────────────────────────────────────── - - const handleReply = useCallback( - async (content: string) => { - if (!popoverState.threadId) return - await replyTo(popoverState.threadId, content) - }, - [popoverState.threadId, replyTo] - ) - - const handleResolve = useCallback(async () => { - if (!popoverState.threadId) return - await resolveThread(popoverState.threadId) - // Update the mark visual state to resolved (amber -> green) - editorRef.current?.commands.setCommentResolved(popoverState.threadId, true) - }, [popoverState.threadId, resolveThread]) - - const handleReopen = useCallback(async () => { - if (!popoverState.threadId) return - await reopenThread(popoverState.threadId) - // Update the mark visual state back to active (green -> amber) - editorRef.current?.commands.setCommentResolved(popoverState.threadId, false) - }, [popoverState.threadId, reopenThread]) - - const handleDelete = useCallback( - async (commentId: string) => { - await deleteComment(commentId) - // If deleting root with no replies, remove the mark from the document and close popover - const thread = threadDataMap.get(popoverState.threadId || '') - if (thread && commentId === thread.root.id && thread.replies.length === 0) { - // Remove the comment mark from the editor document - const editor = editorRef.current - if (editor) { - const { tr, doc } = editor.state - const markType = editor.schema.marks.comment - if (markType) { - doc.descendants((node, pos) => { - node.marks.forEach((mark) => { - if (mark.type === markType && mark.attrs.commentId === commentId) { - tr.removeMark(pos, pos + node.nodeSize, mark) - } - }) - }) - editor.view.dispatch(tr) - } - } - handleDismiss() - } - }, - [deleteComment, threadDataMap, popoverState.threadId, handleDismiss] - ) - - const handleEdit = useCallback( - async (commentId: string, newContent: string) => { - await editComment(commentId, newContent) - }, - [editComment] - ) - - // Handler for initiating comment creation from toolbar selection - // This shows the input UI; actual comment creation happens on submit - const handleCreateComment = useCallback(async (anchorData: string): Promise => { - if (!editorRef.current) return null - - // Capture the current selection range so we can apply the mark later - const { from, to } = editorRef.current.state.selection - - if (from === to) return null // No selection - - // Show the new comment input modal - setNewCommentState({ - visible: true, - anchorData, - selectionFrom: from, - selectionTo: to - }) - - // Return null - we're not creating the comment yet - // The actual creation happens when user submits the new comment form - return null - }, []) - - // Handler for submitting a new comment - const handleSubmitNewComment = useCallback( - async (content: string) => { - if (!newCommentState || !content.trim() || !editorRef.current) return - - const commentId = await addComment({ - content: content.trim(), - anchorType: 'text', - anchorData: newCommentState.anchorData, - targetSchema: PageSchema.schema['@id'] - }) - - if (commentId) { - console.log('[Comments] Applying mark:', { - commentId, - from: newCommentState.selectionFrom, - to: newCommentState.selectionTo - }) - - // Use the editor command to apply the mark - // First, set selection to the original range, then apply mark - editorRef.current - .chain() - .focus() - .setTextSelection({ - from: newCommentState.selectionFrom, - to: newCommentState.selectionTo - }) - .setComment(commentId) - .run() - - console.log('[Comments] Mark command executed, checking DOM...') - setTimeout(() => { - const el = document.querySelector(`[data-comment-id="${commentId}"]`) - console.log('[Comments] DOM element found:', el) - if (el) { - console.log('[Comments] Element HTML:', el.outerHTML) - } - }, 50) - - // After a short delay, find the mark element and show the popover - // This gives time for the DOM and threads state to update - const showPopover = () => { - const markEl = document.querySelector( - `[data-comment-id="${commentId}"]` - ) as HTMLElement | null - if (markEl) { - setPopoverState({ - visible: true, - mode: 'full', - threadId: commentId, - anchor: markEl - }) - } - } - // Try immediately, then retry after a delay if needed - setTimeout(showPopover, 50) - setTimeout(showPopover, 200) - } - - // Close the new comment UI - setNewCommentState(null) - }, - [newCommentState, addComment] - ) - - // Handler for canceling new comment creation - const handleCancelNewComment = useCallback(() => { - setNewCommentState(null) - }, []) - - // ─── Sidebar Handlers ───────────────────────────────────────────────────────── - - const handleSidebarSelectThread = useCallback((threadId: string) => { - // Find and scroll to the comment mark in the editor - const markEl = document.querySelector(`[data-comment-id="${threadId}"]`) as HTMLElement | null - if (markEl) { - markEl.scrollIntoView({ behavior: 'smooth', block: 'center' }) - // Flash the highlight by briefly selecting it - setPopoverState({ - visible: true, - mode: 'full', - threadId, - anchor: markEl - }) - } - }, []) - - const handleSidebarReply = useCallback( - async (threadId: string, content: string) => { - await replyTo(threadId, content) - }, - [replyTo] - ) - - const handleSidebarResolve = useCallback( - async (threadId: string) => { - await resolveThread(threadId) - editorRef.current?.commands.setCommentResolved(threadId, true) - }, - [resolveThread] - ) - - const handleSidebarReopen = useCallback( - async (threadId: string) => { - await reopenThread(threadId) - editorRef.current?.commands.setCommentResolved(threadId, false) - }, - [reopenThread] - ) - - const handleSidebarDelete = useCallback( - async (commentId: string) => { - await deleteComment(commentId) - }, - [deleteComment] - ) - - const handleSidebarEdit = useCallback( - async (commentId: string, newContent: string) => { - await editComment(commentId, newContent) - }, - [editComment] - ) + // ─── Sidebar hover highlights (desktop-only affordance) ────────────────────── const hoveredThreadRef = useRef(null) - const leaveTimerRef = useRef(null) + const leaveTimerRef = useRef | null>(null) const handleSidebarHoverThread = useCallback((threadId: string) => { // Cancel any pending leave — user moved to another thread or re-entered @@ -671,30 +211,12 @@ export function PageView({ docId, minimalChrome = false }: PageViewProps) { }, 150) }, []) - // ─── Comment Extensions ─────────────────────────────────────────────────────── - - const commentExtensions = useMemo( - () => [ - CommentMark, - CommentPlugin.configure({ - onClickComment: handleClickComment, - onHoverComment: handleHoverComment, - onLeaveComment: handleLeaveComment - }) - ], - [handleClickComment, handleHoverComment, handleLeaveComment] - ) - // Combine plugin extensions with comment extensions const allExtensions = useMemo( () => [...pluginExtensions, ...commentExtensions], [pluginExtensions, commentExtensions] ) - // Get the current thread for the popover - // If thread not found in map yet (newly created), it will show once threads update - const currentThread = popoverState.threadId ? threadDataMap.get(popoverState.threadId) : null - // Debug: log when popover should show but thread not found useEffect(() => { if (popoverState.visible && popoverState.threadId && !currentThread) { @@ -706,52 +228,18 @@ export function PageView({ docId, minimalChrome = false }: PageViewProps) { } }, [popoverState, currentThread, threads]) - // ─── Orphaned Comment Handlers ───────────────────────────────────────────────── - - const handleDismissOrphaned = useCallback( - async (commentId: string) => { - // Delete the orphaned thread entirely - const thread = threads.find((t) => t.root.id === commentId) - if (thread) { - // Delete replies first, then root - for (const reply of thread.replies) { - await deleteComment(reply.id) - } - await deleteComment(commentId) - } - // Remove from orphaned list - setOrphanedIds((prev) => prev.filter((id) => id !== commentId)) - }, - [threads, deleteComment] - ) - - const handleReattachOrphaned = useCallback((commentId: string) => { - // For now, just show a message - reattachment requires selecting new text - // In a full implementation, this would open a mode to select new anchor text - console.log(`[Comments] Reattach not yet implemented for ${commentId}`) - // TODO: Implement reattachment UI - enter "select text" mode - }, []) - const handleSelectOrphaned = useCallback( (commentId: string) => { // Open the popover for this orphaned comment const thread = threadDataMap.get(commentId) if (thread) { // Since orphaned comments don't have anchor elements, use coordinates - setPopoverState({ - visible: true, - mode: 'full', - threadId: commentId, - anchor: null // Will need to position differently - }) + showThreadPopover(commentId, null) // Will need to position differently } }, - [threadDataMap] + [threadDataMap, showThreadPopover] ) - // Sidebar thread list (derived from threadDataMap) - const sidebarThreads = useMemo(() => Array.from(threadDataMap.values()), [threadDataMap]) - if (loading || !doc || !pluginsReady) { return (
@@ -834,7 +322,7 @@ export function PageView({ docId, minimalChrome = false }: PageViewProps) { setOrphanedCollapsed((prev) => !prev)} + onToggleCollapse={toggleOrphanedCollapsed} onDismiss={handleDismissOrphaned} onReattach={handleReattachOrphaned} onSelect={handleSelectOrphaned} diff --git a/apps/electron/src/renderer/lib/social-workspace.ts b/apps/electron/src/renderer/lib/social-workspace.ts index d87d63a6e..79739c429 100644 --- a/apps/electron/src/renderer/lib/social-workspace.ts +++ b/apps/electron/src/renderer/lib/social-workspace.ts @@ -1,59 +1,9 @@ -import type { MutateOp } from '@xnetjs/react' -import { SavedViewSchema } from '@xnetjs/data' -import { createDefaultSocialWorkspaceSavedViewSeeds } from '@xnetjs/social/workspace' - -export type SocialWorkspaceSeedSummary = { - created: number - updated: number - total: number -} - -type SocialWorkspaceSeedOperationResult = { - action: 'created' | 'updated' - operation: MutateOp -} - -export function getDefaultSocialWorkspaceSeeds() { - return createDefaultSocialWorkspaceSavedViewSeeds({ pageSize: 100 }) -} - -export async function upsertDefaultSocialWorkspace(input: { - mutate: (ops: MutateOp[]) => Promise - getExisting: (id: string) => Promise -}): Promise { - const seeds = getDefaultSocialWorkspaceSeeds() - const operationResults = await Promise.all( - seeds.map(async (seed): Promise => { - const existing = await input.getExisting(seed.deterministicId) - if (existing) { - return { - action: 'updated', - operation: { - type: 'update', - id: seed.deterministicId, - data: seed.savedViewProperties - } - } - } - - return { - action: 'created', - operation: { - type: 'create', - id: seed.deterministicId, - schema: SavedViewSchema, - data: seed.savedViewProperties - } as MutateOp - } - }) - ) - - const operations = operationResults.map((result) => result.operation) - await input.mutate(operations) - - return { - created: operationResults.filter((result) => result.action === 'created').length, - updated: operationResults.filter((result) => result.action === 'updated').length, - total: seeds.length - } -} +/** + * Social workspace seeding — moved into the shared Data Workspace core + * (@xnetjs/views, exploration 0276). Re-exported here for existing app imports. + */ +export { + getDefaultSocialWorkspaceSeeds, + upsertDefaultSocialWorkspace, + type SocialWorkspaceSeedSummary +} from '@xnetjs/views' diff --git a/apps/electron/src/renderer/shell/shell-state.ts b/apps/electron/src/renderer/shell/shell-state.ts new file mode 100644 index 000000000..a97ec4a91 --- /dev/null +++ b/apps/electron/src/renderer/shell/shell-state.ts @@ -0,0 +1,92 @@ +/** + * Desktop shell state — the ShellState union plus the pure transition + * reducer, extracted from App.tsx. Adding a new shell view means adding a + * kind here (state + action + reducer arm + overlay title) instead of + * threading it through App.tsx callbacks. + * + * The overlay transition-timer semantics (OVERLAY_OPEN_DELAY_MS) are owned by + * `use-document-shell.ts`; the reducer itself is pure and synchronous. + */ + +export type DocType = 'page' | 'database' | 'canvas' + +export type ViewportSnapshot = { + x: number + y: number + zoom: number +} + +export type ShellState = + | { kind: 'canvas-home' } + | { kind: 'page-focus'; docId: string; returnViewport: ViewportSnapshot | null } + | { kind: 'database-focus'; docId: string; returnViewport: ViewportSnapshot | null } + | { kind: 'database-split'; docId: string } + | { kind: 'settings' } + | { kind: 'data-workspace' } + | { kind: 'social-import' } + | { kind: 'stories' } + +export type DocumentItem = { + id: string + title: string + type: DocType + createdAt?: number + updatedAt?: number +} + +/** + * When a focus transition animates from the canvas (the camera glide toward + * the linked document), the overlay opens after this delay so the glide is + * visible before the focused surface covers it. + */ +export const OVERLAY_OPEN_DELAY_MS = 180 + +export type ShellAction = + | { type: 'return-home' } + | { + type: 'focus-document' + docType: Exclude + docId: string + returnViewport: ViewportSnapshot | null + } + | { type: 'open-database-split'; docId: string } + | { type: 'open-settings' } + | { type: 'open-data-workspace' } + | { type: 'open-social-import' } + | { type: 'open-stories' } + +export function shellReducer(_state: ShellState, action: ShellAction): ShellState { + switch (action.type) { + case 'return-home': + return { kind: 'canvas-home' } + case 'focus-document': + return action.docType === 'page' + ? { kind: 'page-focus', docId: action.docId, returnViewport: action.returnViewport } + : { kind: 'database-focus', docId: action.docId, returnViewport: action.returnViewport } + case 'open-database-split': + return { kind: 'database-split', docId: action.docId } + case 'open-settings': + return { kind: 'settings' } + case 'open-data-workspace': + return { kind: 'data-workspace' } + case 'open-social-import': + return { kind: 'social-import' } + case 'open-stories': + return { kind: 'stories' } + } +} + +export function overlayTitleFor(kind: ShellState['kind']): string | null { + if (kind === 'page-focus') return 'Document' + if (kind === 'database-focus') return 'Database' + if (kind === 'settings') return 'Settings' + if (kind === 'data-workspace') return 'Data Workspace' + if (kind === 'social-import') return 'Social Import' + if (kind === 'stories') return 'Stories' + return null +} + +/** The canvas stays interactive underneath these shell states. */ +export function isCanvasInteractiveShellKind(kind: ShellState['kind']): boolean { + return kind === 'canvas-home' || kind === 'database-split' +} diff --git a/apps/electron/src/renderer/shell/use-document-shell.ts b/apps/electron/src/renderer/shell/use-document-shell.ts new file mode 100644 index 000000000..98d785a0c --- /dev/null +++ b/apps/electron/src/renderer/shell/use-document-shell.ts @@ -0,0 +1,442 @@ +/** + * Desktop document shell hook, extracted from App.tsx. Owns the shell state + * (via the pure reducer in `shell-state.ts`), the home-canvas bootstrap, the + * pending canvas insert, the canvas command state, the document queries, and + * every document/view transition handler — including the overlay + * transition-timer semantics (OVERLAY_OPEN_DELAY_MS). App.tsx composes this + * hook and renders per shell state. + */ +import type { + DocType, + DocumentItem, + ShellAction, + ShellState, + ViewportSnapshot +} from './shell-state' +import type { AddSharedInput } from '../components/AddSharedDialog' +import type { CanvasViewCommandState, CanvasViewHandle } from '../components/CanvasView' +import type { SavedViewCanvasFrameInput } from '../components/DataWorkspaceView' +import type { LinkedDocumentItem } from '../lib/canvas-shell' +import type { Dispatch, RefObject, SetStateAction } from 'react' +import { PageSchema, DatabaseSchema, CanvasSchema } from '@xnetjs/data' +import { useDevTools } from '@xnetjs/devtools' +import { useQuery, useMutate } from '@xnetjs/react' +import { usePrefersReducedMotion } from '@xnetjs/ui' +import { useCallback, useEffect, useMemo, useReducer, useRef, useState } from 'react' +import { + OVERLAY_OPEN_DELAY_MS, + isCanvasInteractiveShellKind, + overlayTitleFor, + shellReducer +} from './shell-state' + +export const STORIES_ENABLED = import.meta.env.DEV + +export const EMPTY_CANVAS_COMMAND_STATE: CanvasViewCommandState = { + selectionCount: 0, + selectedNodeId: null, + selectedSourceId: null, + selectedSourceType: null, + selectedDisplayType: null, + selectedTitle: null, + selectedIsQueryFrame: false, + selectionAllLocked: false, + selectionAnyLocked: false, + shortcutHelpOpen: false +} + +function toError(error: unknown): Error { + return error instanceof Error ? error : new Error(String(error)) +} + +export type PendingCanvasInsert = { + requestId: string + document: LinkedDocumentItem +} | null + +export interface DocumentShell { + shellState: ShellState + overlayTitle: string | null + isCanvasInteractiveShell: boolean + prefersReducedMotion: boolean + homeCanvasId: string | null + homeCanvasBootstrapError: Error | null + documents: DocumentItem[] + recentDocuments: DocumentItem[] + isLoading: boolean + pendingCanvasInsert: PendingCanvasInsert + canvasCommandState: CanvasViewCommandState + canvasViewRef: RefObject + bootstrapHomeCanvas: () => Promise + focusDocument: ( + docId: string, + docType: Exclude, + animateFromCanvas: boolean + ) => void + handleOpenDocument: (docId: string) => void + handleCreateLinkedDocument: (type: Exclude) => Promise + handleCreateCanvasNote: () => void + handleReturnHome: () => void + handleAddShared: (input: AddSharedInput) => Promise + openDatabaseSplit: (docId: string) => void + handleOpenSettings: () => void + handleOpenSocialImport: () => void + handleOpenDataWorkspace: () => void + handleOpenStories: () => void + handleInsertSavedLensAsCanvasFrame: (view: SavedViewCanvasFrameInput) => void + handleCommandStateChange: Dispatch> + handlePendingInsertConsumed: (requestId: string) => void +} + +export function useDocumentShell(): DocumentShell { + const [homeCanvasId, setHomeCanvasId] = useState(null) + const [homeCanvasBootstrapError, setHomeCanvasBootstrapError] = useState(null) + const [shellState, dispatchShell] = useReducer(shellReducer, { kind: 'canvas-home' }) + const [pendingCanvasInsert, setPendingCanvasInsert] = useState(null) + const [canvasCommandState, setCanvasCommandState] = useState( + EMPTY_CANVAS_COMMAND_STATE + ) + const { setActiveNodeId } = useDevTools() + const { create } = useMutate() + const prefersReducedMotion = usePrefersReducedMotion() + const canvasViewRef = useRef(null) + const creatingHomeCanvasRef = useRef(false) + const transitionTimerRef = useRef(null) + + const { data: pages, loading: pagesLoading } = useQuery(PageSchema, { limit: 100 }) + const { data: databases, loading: databasesLoading } = useQuery(DatabaseSchema, { limit: 100 }) + const { data: canvases, loading: canvasesLoading } = useQuery(CanvasSchema, { limit: 100 }) + + const documents: DocumentItem[] = useMemo( + () => + [ + ...pages.map((page) => ({ + id: page.id, + title: page.title || 'Untitled Page', + type: 'page' as const, + createdAt: page.createdAt, + updatedAt: page.updatedAt + })), + ...databases.map((database) => ({ + id: database.id, + title: database.title || 'Untitled Database', + type: 'database' as const, + createdAt: database.createdAt, + updatedAt: database.updatedAt + })), + ...canvases.map((canvas) => ({ + id: canvas.id, + title: canvas.title || 'Workspace Canvas', + type: 'canvas' as const, + createdAt: canvas.createdAt, + updatedAt: canvas.updatedAt + })) + ].sort( + (left, right) => + (right.updatedAt || right.createdAt || 0) - (left.updatedAt || left.createdAt || 0) + ), + [canvases, databases, pages] + ) + + const isLoading = pagesLoading || databasesLoading || canvasesLoading + const recentDocuments = useMemo(() => documents.slice(0, 6), [documents]) + + const clearTransitionTimer = useCallback(() => { + if (transitionTimerRef.current !== null) { + window.clearTimeout(transitionTimerRef.current) + transitionTimerRef.current = null + } + }, []) + + /** Clear any pending overlay timer, then apply a shell transition now. */ + const transitionShell = useCallback( + (action: ShellAction) => { + clearTransitionTimer() + dispatchShell(action) + }, + [clearTransitionTimer] + ) + + useEffect(() => { + return () => { + clearTransitionTimer() + } + }, [clearTransitionTimer]) + + const bootstrapHomeCanvas = useCallback(async () => { + if (creatingHomeCanvasRef.current) return + + creatingHomeCanvasRef.current = true + setHomeCanvasBootstrapError(null) + + try { + const canvas = await create(CanvasSchema, { title: 'Workspace Canvas' }) + if (!canvas) { + throw new Error('Home canvas was not created') + } + + setHomeCanvasId(canvas.id) + setActiveNodeId(canvas.id) + } catch (error) { + const normalizedError = toError(error) + console.error('Failed to create home canvas', normalizedError) + setHomeCanvasBootstrapError(normalizedError) + } finally { + creatingHomeCanvasRef.current = false + } + }, [create, setActiveNodeId]) + + useEffect(() => { + if (isLoading) return + + if (canvases.length === 0) { + if (homeCanvasBootstrapError) return + if (homeCanvasId) { + setHomeCanvasId(null) + } + void bootstrapHomeCanvas() + return + } + + if (homeCanvasBootstrapError) { + setHomeCanvasBootstrapError(null) + } + + if (!homeCanvasId || !canvases.some((canvas) => canvas.id === homeCanvasId)) { + const defaultCanvas = [...canvases].sort( + (left, right) => + (right.updatedAt || right.createdAt || 0) - (left.updatedAt || left.createdAt || 0) + )[0] + + if (defaultCanvas) { + setHomeCanvasId(defaultCanvas.id) + setActiveNodeId(defaultCanvas.id) + } + } + }, [ + bootstrapHomeCanvas, + canvases, + homeCanvasBootstrapError, + homeCanvasId, + isLoading, + setActiveNodeId + ]) + + const focusDocument = useCallback( + (docId: string, docType: Exclude, animateFromCanvas: boolean) => { + clearTransitionTimer() + + const shouldAnimateFromCanvas = animateFromCanvas && !prefersReducedMotion + const returnViewport: ViewportSnapshot | null = + shouldAnimateFromCanvas && canvasViewRef.current + ? canvasViewRef.current.focusLinkedDocument(docId) + : null + + const openOverlay = () => { + dispatchShell({ type: 'focus-document', docType, docId, returnViewport }) + setActiveNodeId(docId) + } + + if (returnViewport && !prefersReducedMotion) { + transitionTimerRef.current = window.setTimeout(openOverlay, OVERLAY_OPEN_DELAY_MS) + return + } + + openOverlay() + }, + [clearTransitionTimer, prefersReducedMotion, setActiveNodeId] + ) + + const handleOpenDocument = useCallback( + (docId: string) => { + const document = documents.find((entry) => entry.id === docId) + if (!document) return + + if (document.type === 'canvas') { + setHomeCanvasId(document.id) + transitionShell({ type: 'return-home' }) + setActiveNodeId(document.id) + return + } + + focusDocument(document.id, document.type, true) + }, + [documents, focusDocument, setActiveNodeId, transitionShell] + ) + + const handleCreateLinkedDocument = useCallback( + async (type: Exclude) => { + clearTransitionTimer() + + try { + const schema = type === 'page' ? PageSchema : DatabaseSchema + const title = type === 'page' ? 'Untitled Page' : 'Untitled Database' + const newDocument = await create(schema, { title }) + if (!newDocument) return + + setPendingCanvasInsert({ + requestId: `${type}-${newDocument.id}-${Date.now()}`, + document: { + id: newDocument.id, + title, + type + } + }) + dispatchShell({ type: 'return-home' }) + setActiveNodeId(homeCanvasId) + } catch (error) { + console.error('Failed to create linked document', toError(error)) + } + }, + [clearTransitionTimer, create, homeCanvasId, setActiveNodeId] + ) + + const handleCreateCanvasNote = useCallback(() => { + const createCanvasNote = async () => { + clearTransitionTimer() + + try { + const note = await create(PageSchema, { title: 'Untitled Note' }) + if (!note) return + + setPendingCanvasInsert({ + requestId: `note-${note.id}-${Date.now()}`, + document: { + id: note.id, + title: note.title || 'Untitled Note', + type: 'page', + canvasKind: 'note' + } + }) + dispatchShell({ type: 'return-home' }) + setActiveNodeId(homeCanvasId) + } catch (error) { + console.error('Failed to create canvas note', toError(error)) + } + } + + void createCanvasNote() + }, [clearTransitionTimer, create, homeCanvasId, setActiveNodeId]) + + const handleReturnHome = useCallback(() => { + clearTransitionTimer() + if (shellState.kind === 'page-focus' || shellState.kind === 'database-focus') { + if (shellState.returnViewport) { + canvasViewRef.current?.restoreViewport(shellState.returnViewport) + } + } + + dispatchShell({ type: 'return-home' }) + setActiveNodeId(homeCanvasId) + }, [clearTransitionTimer, homeCanvasId, setActiveNodeId, shellState]) + + const handleAddShared = useCallback( + async (input: AddSharedInput) => { + if (input.share) { + try { + await window.__xnetIpcSyncManager?.configureShareSession({ + signalingUrl: input.share.endpoint, + ucanToken: input.share.token, + transport: input.share.transport, + iceServers: input.share.iceServers + }) + } catch (error) { + console.error('Failed to configure shared session', toError(error)) + } + } + + if (input.docType === 'canvas') { + setHomeCanvasId(input.docId) + transitionShell({ type: 'return-home' }) + setActiveNodeId(input.docId) + return + } + + focusDocument(input.docId, input.docType, false) + }, + [focusDocument, setActiveNodeId, transitionShell] + ) + + const openDatabaseSplit = useCallback( + (docId: string) => { + transitionShell({ type: 'open-database-split', docId }) + setActiveNodeId(docId) + }, + [setActiveNodeId, transitionShell] + ) + + const handleOpenSettings = useCallback(() => { + transitionShell({ type: 'open-settings' }) + }, [transitionShell]) + + const handleOpenSocialImport = useCallback(() => { + transitionShell({ type: 'open-social-import' }) + }, [transitionShell]) + + const handleOpenDataWorkspace = useCallback(() => { + transitionShell({ type: 'open-data-workspace' }) + }, [transitionShell]) + + const handleInsertSavedLensAsCanvasFrame = useCallback( + (view: SavedViewCanvasFrameInput) => { + const inserted = + canvasViewRef.current?.createQueryFrameFromSavedView({ + viewId: view.id, + title: view.title ?? 'Saved lens', + descriptorJson: view.descriptor ?? null + }) ?? false + + if (!inserted) { + console.error('Failed to insert saved lens as a canvas query frame', view.id) + return + } + + transitionShell({ type: 'return-home' }) + setActiveNodeId(homeCanvasId) + }, + [homeCanvasId, setActiveNodeId, transitionShell] + ) + + const handleOpenStories = useCallback(() => { + if (!STORIES_ENABLED) return + + transitionShell({ type: 'open-stories' }) + }, [transitionShell]) + + const handlePendingInsertConsumed = useCallback((requestId: string) => { + setPendingCanvasInsert((current) => (current?.requestId === requestId ? null : current)) + }, []) + + const overlayTitle = useMemo(() => overlayTitleFor(shellState.kind), [shellState.kind]) + const isCanvasInteractiveShell = isCanvasInteractiveShellKind(shellState.kind) + + return { + shellState, + overlayTitle, + isCanvasInteractiveShell, + prefersReducedMotion, + homeCanvasId, + homeCanvasBootstrapError, + documents, + recentDocuments, + isLoading, + pendingCanvasInsert, + canvasCommandState, + canvasViewRef, + bootstrapHomeCanvas, + focusDocument, + handleOpenDocument, + handleCreateLinkedDocument, + handleCreateCanvasNote, + handleReturnHome, + handleAddShared, + openDatabaseSplit, + handleOpenSettings, + handleOpenSocialImport, + handleOpenDataWorkspace, + handleOpenStories, + handleInsertSavedLensAsCanvasFrame, + handleCommandStateChange: setCanvasCommandState, + handlePendingInsertConsumed + } +} diff --git a/apps/electron/src/renderer/shell/use-shell-palette-commands.ts b/apps/electron/src/renderer/shell/use-shell-palette-commands.ts new file mode 100644 index 000000000..baa3e3cff --- /dev/null +++ b/apps/electron/src/renderer/shell/use-shell-palette-commands.ts @@ -0,0 +1,549 @@ +/** + * Command-palette command table for the desktop shell, extracted from + * App.tsx. Pure declarative wiring: every command delegates to the + * document-shell handlers or the CanvasView imperative handle. + */ +import type { DocumentItem, ShellState } from './shell-state' +import type { CanvasViewCommandState, CanvasViewHandle } from '../components/CanvasView' +import type { PaletteCommand } from '@xnetjs/ui' +import type { RefObject } from 'react' +import { CANVAS_PLANNING_TEMPLATE_DEFINITIONS } from '@xnetjs/canvas' +import { useMemo } from 'react' +import { STORIES_ENABLED } from './use-document-shell' + +const MOD_ENTER_SHORTCUT = navigator.platform.includes('Mac') ? '⌘↩' : 'Ctrl+Enter' + +export interface ShellPaletteCommandsOptions { + canvasViewRef: RefObject + canvasCommandState: CanvasViewCommandState + isCanvasInteractiveShell: boolean + shellKind: ShellState['kind'] + recentDocuments: DocumentItem[] + handleCreateLinkedDocument: (type: 'page' | 'database') => Promise + handleCreateCanvasNote: () => void + handleOpenDocument: (docId: string) => void + handleOpenSettings: () => void + handleOpenSocialImport: () => void + handleOpenDataWorkspace: () => void + handleOpenStories: () => void +} + +export function useShellPaletteCommands(options: ShellPaletteCommandsOptions): PaletteCommand[] { + const { + canvasViewRef, + canvasCommandState, + isCanvasInteractiveShell, + shellKind, + recentDocuments, + handleCreateLinkedDocument, + handleCreateCanvasNote, + handleOpenDocument, + handleOpenSettings, + handleOpenSocialImport, + handleOpenDataWorkspace, + handleOpenStories + } = options + + return useMemo( + () => [ + { + id: 'create-page', + name: 'Create Page', + description: 'Create a new page and place it on the canvas', + icon: 'file-text', + shortcut: 'P', + group: 'Canvas', + keywords: ['page', 'canvas', 'create'], + execute: () => void handleCreateLinkedDocument('page') + }, + { + id: 'create-database', + name: 'Create Database', + description: 'Create a new database and place it on the canvas', + icon: 'database', + shortcut: 'D', + group: 'Canvas', + keywords: ['database', 'canvas', 'create'], + execute: () => void handleCreateLinkedDocument('database') + }, + { + id: 'create-note', + name: 'Create Canvas Note', + description: 'Create a page-backed note and place it on the canvas', + icon: 'sparkles', + shortcut: 'N', + group: 'Canvas', + keywords: ['note', 'canvas', 'create'], + execute: () => handleCreateCanvasNote() + }, + { + id: 'create-rectangle', + name: 'Create Rectangle', + description: 'Create a canvas-native rectangle on the current board', + icon: 'square', + shortcut: 'R', + group: 'Canvas', + keywords: ['shape', 'rectangle', 'canvas', 'create'], + when: () => isCanvasInteractiveShell, + execute: () => { + canvasViewRef.current?.createShape('rectangle') + } + }, + { + id: 'create-frame', + name: 'Create Frame', + description: 'Create an empty frame container on the current board', + icon: 'layout', + shortcut: 'F', + group: 'Canvas', + keywords: ['frame', 'group', 'canvas', 'create'], + when: () => isCanvasInteractiveShell, + execute: () => { + canvasViewRef.current?.createFrame() + } + }, + ...CANVAS_PLANNING_TEMPLATE_DEFINITIONS.map((template) => ({ + id: `create-canvas-template-${template.id}`, + name: `Create ${template.name}`, + description: template.description, + icon: 'layout', + group: 'Canvas', + keywords: ['template', template.category, template.name, 'canvas', 'planning'], + when: () => isCanvasInteractiveShell, + execute: () => { + canvasViewRef.current?.createPlanningTemplate(template.id) + } + })), + { + id: 'frame-selection', + name: 'Frame Selection', + description: 'Wrap the selected canvas objects in a frame container', + icon: 'layout', + shortcut: 'Mod+Shift+F', + group: 'Canvas', + keywords: ['frame', 'group', 'selection', 'canvas'], + when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 0, + execute: () => { + canvasViewRef.current?.wrapSelectionInFrame() + } + }, + { + id: 'canvas-refresh-query-frame', + name: 'Refresh Query Frame', + description: + canvasCommandState.selectedTitle && canvasCommandState.selectedIsQueryFrame + ? `Refresh ${canvasCommandState.selectedTitle}` + : 'Refresh the selected query frame', + icon: 'refresh-cw', + group: 'Canvas', + keywords: ['refresh', 'query', 'frame', 'lens', 'canvas'], + when: () => isCanvasInteractiveShell && canvasCommandState.selectedIsQueryFrame, + execute: () => { + canvasViewRef.current?.refreshSelectedQueryFrame() + } + }, + { + id: 'canvas-connect-selection', + name: 'Connect Selection', + description: 'Create a connector between the two selected canvas objects', + icon: 'link', + shortcut: 'Mod+Shift+K', + group: 'Canvas', + keywords: ['connect', 'connector', 'edge', 'selection', 'canvas'], + when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount === 2, + execute: () => { + canvasViewRef.current?.connectSelection() + } + }, + { + id: 'canvas-rename-alias', + name: 'Rename Canvas Alias', + description: + canvasCommandState.selectedTitle && canvasCommandState.selectionCount === 1 + ? `Rename the canvas copy of ${canvasCommandState.selectedTitle}` + : 'Rename the selected canvas object without changing the source title', + icon: 'pencil', + shortcut: 'Mod+Shift+A', + group: 'Canvas', + keywords: ['alias', 'rename', 'selection', 'canvas'], + when: () => + isCanvasInteractiveShell && + canvasCommandState.selectionCount === 1 && + Boolean(canvasCommandState.selectedSourceId), + execute: () => { + canvasViewRef.current?.openAliasEditor() + } + }, + { + id: 'canvas-clear-alias', + name: 'Clear Canvas Alias', + description: 'Remove the canvas-local alias from the selected object', + icon: 'x', + group: 'Canvas', + keywords: ['alias', 'clear', 'selection', 'canvas'], + when: () => + isCanvasInteractiveShell && + canvasCommandState.selectionCount === 1 && + Boolean(canvasCommandState.selectedSourceId), + execute: () => { + canvasViewRef.current?.clearSelectionAlias() + } + }, + { + id: 'canvas-comment-selection', + name: 'Comment on Selection', + description: + canvasCommandState.selectedTitle && canvasCommandState.selectionCount === 1 + ? `Add a canvas-anchored comment to ${canvasCommandState.selectedTitle}` + : 'Add a canvas-anchored comment to the selected object', + icon: 'message-square', + shortcut: 'Mod+Shift+C', + group: 'Canvas', + keywords: ['comment', 'selection', 'canvas', 'feedback'], + when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount === 1, + execute: () => { + canvasViewRef.current?.openCommentComposer() + } + }, + { + id: 'canvas-show-linked-copies', + name: 'Show Linked Copies', + description: 'Inspect other canvas objects that point at the same source node', + icon: 'copy', + group: 'Canvas', + keywords: ['references', 'copies', 'linked', 'canvas'], + when: () => + isCanvasInteractiveShell && + canvasCommandState.selectionCount === 1 && + Boolean(canvasCommandState.selectedSourceId), + execute: () => { + canvasViewRef.current?.toggleSourceReferences(true) + } + }, + { + id: 'canvas-peek-selection', + name: 'Peek Selected Object', + description: + canvasCommandState.selectedTitle && canvasCommandState.selectionCount === 1 + ? `Center and activate ${canvasCommandState.selectedTitle}` + : 'Center and activate the current canvas selection', + icon: 'eye', + shortcut: 'Enter', + group: 'Canvas', + keywords: ['peek', 'edit', 'selection', 'canvas'], + when: () => shellKind === 'canvas-home' && canvasCommandState.selectionCount === 1, + execute: () => { + canvasViewRef.current?.openSelection('peek') + } + }, + { + id: 'canvas-open-selection', + name: 'Open Selected Object', + description: + canvasCommandState.selectedTitle && canvasCommandState.selectionCount === 1 + ? `Open ${canvasCommandState.selectedTitle} in a focused surface` + : 'Open the current canvas selection in a focused surface', + icon: 'external-link', + shortcut: MOD_ENTER_SHORTCUT, + group: 'Canvas', + keywords: ['open', 'focus', 'selection', 'canvas'], + when: () => + isCanvasInteractiveShell && + canvasCommandState.selectionCount === 1 && + Boolean(canvasCommandState.selectedSourceId && canvasCommandState.selectedSourceType), + execute: () => { + canvasViewRef.current?.openSelection('focus') + } + }, + { + id: 'canvas-open-database-split', + name: 'Open Database in Split View', + description: + canvasCommandState.selectedTitle && canvasCommandState.selectionCount === 1 + ? `Keep ${canvasCommandState.selectedTitle} open beside the canvas` + : 'Open the selected database in a split view beside the canvas', + icon: 'columns', + shortcut: 'Alt+Enter', + group: 'Canvas', + keywords: ['split', 'database', 'canvas', 'preview'], + when: () => + isCanvasInteractiveShell && + canvasCommandState.selectionCount === 1 && + canvasCommandState.selectedDisplayType === 'database' && + Boolean(canvasCommandState.selectedSourceId), + execute: () => { + canvasViewRef.current?.openSelection('split') + } + }, + { + id: 'canvas-fit-selection', + name: 'Fit Selected Object', + description: 'Center the current canvas selection in view', + icon: 'layout', + group: 'Canvas', + keywords: ['fit', 'selection', 'zoom', 'canvas'], + when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 0, + execute: () => { + canvasViewRef.current?.fitSelection() + } + }, + { + id: 'canvas-toggle-lock', + name: canvasCommandState.selectionAllLocked ? 'Unlock Selection' : 'Lock Selection', + description: canvasCommandState.selectionAllLocked + ? 'Allow the current selection to move and resize again' + : 'Protect the current selection from accidental moves and nudges', + icon: 'lock', + shortcut: 'Mod+Shift+L', + group: 'Canvas', + keywords: ['lock', 'unlock', 'selection', 'canvas'], + when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 0, + execute: () => { + canvasViewRef.current?.toggleSelectionLock() + } + }, + { + id: 'canvas-align-left', + name: 'Align Selection Left', + description: 'Snap the selected objects to a shared left edge', + icon: 'align-start-horizontal', + shortcut: 'Mod+Shift+Left', + group: 'Canvas', + keywords: ['align', 'left', 'selection', 'canvas'], + when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 1, + execute: () => { + canvasViewRef.current?.alignSelection('left') + } + }, + { + id: 'canvas-align-right', + name: 'Align Selection Right', + description: 'Snap the selected objects to a shared right edge', + icon: 'align-end-horizontal', + shortcut: 'Mod+Shift+Right', + group: 'Canvas', + keywords: ['align', 'right', 'selection', 'canvas'], + when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 1, + execute: () => { + canvasViewRef.current?.alignSelection('right') + } + }, + { + id: 'canvas-align-top', + name: 'Align Selection Top', + description: 'Snap the selected objects to a shared top edge', + icon: 'align-start-vertical', + shortcut: 'Mod+Shift+Up', + group: 'Canvas', + keywords: ['align', 'top', 'selection', 'canvas'], + when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 1, + execute: () => { + canvasViewRef.current?.alignSelection('top') + } + }, + { + id: 'canvas-align-bottom', + name: 'Align Selection Bottom', + description: 'Snap the selected objects to a shared bottom edge', + icon: 'align-end-vertical', + shortcut: 'Mod+Shift+Down', + group: 'Canvas', + keywords: ['align', 'bottom', 'selection', 'canvas'], + when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 1, + execute: () => { + canvasViewRef.current?.alignSelection('bottom') + } + }, + { + id: 'canvas-distribute-horizontal', + name: 'Distribute Selection Horizontally', + description: 'Even out the horizontal spacing between selected objects', + icon: 'columns', + group: 'Canvas', + keywords: ['distribute', 'horizontal', 'selection', 'canvas'], + when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 2, + execute: () => { + canvasViewRef.current?.distributeSelection('horizontal') + } + }, + { + id: 'canvas-distribute-vertical', + name: 'Distribute Selection Vertically', + description: 'Even out the vertical spacing between selected objects', + icon: 'rows', + group: 'Canvas', + keywords: ['distribute', 'vertical', 'selection', 'canvas'], + when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 2, + execute: () => { + canvasViewRef.current?.distributeSelection('vertical') + } + }, + { + id: 'canvas-tidy-selection', + name: 'Tidy Selection', + description: 'Pack the selected objects into a clean reading grid', + icon: 'sparkles', + group: 'Canvas', + keywords: ['tidy', 'arrange', 'selection', 'canvas'], + when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 1, + execute: () => { + canvasViewRef.current?.tidySelection() + } + }, + { + id: 'canvas-cluster-selection', + name: 'Cluster Selection', + description: 'Pull selected objects into a compact planning cluster', + icon: 'sparkles', + group: 'Canvas', + keywords: ['cluster', 'arrange', 'selection', 'canvas'], + when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 1, + execute: () => { + canvasViewRef.current?.clusterSelection() + } + }, + { + id: 'canvas-stack-selection', + name: 'Stack Selection', + description: 'Stack selected objects into an offset pile', + icon: 'layers', + group: 'Canvas', + keywords: ['stack', 'pile', 'arrange', 'selection', 'canvas'], + when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 1, + execute: () => { + canvasViewRef.current?.stackSelection() + } + }, + { + id: 'canvas-convert-selection-mind-map', + name: 'Convert Selection To Mind Map', + description: 'Create a mind-map root and convert the selected objects into branches', + icon: 'git-branch', + group: 'Canvas', + keywords: ['convert', 'mind map', 'selection', 'canvas'], + when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 0, + execute: () => { + canvasViewRef.current?.convertSelectionToMindMap() + } + }, + { + id: 'canvas-send-backward', + name: 'Send Selection Backward', + description: 'Move the selected objects back one layer', + icon: 'minus', + shortcut: '[', + group: 'Canvas', + keywords: ['backward', 'z-index', 'layer', 'selection', 'canvas'], + when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 0, + execute: () => { + canvasViewRef.current?.shiftSelectionLayer('backward') + } + }, + { + id: 'canvas-bring-forward', + name: 'Bring Selection Forward', + description: 'Move the selected objects forward one layer', + icon: 'plus', + shortcut: ']', + group: 'Canvas', + keywords: ['forward', 'z-index', 'layer', 'selection', 'canvas'], + when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 0, + execute: () => { + canvasViewRef.current?.shiftSelectionLayer('forward') + } + }, + { + id: 'canvas-clear-selection', + name: 'Clear Selection', + description: 'Clear the current canvas selection', + icon: 'x', + shortcut: 'Esc', + group: 'Canvas', + keywords: ['clear', 'selection', 'canvas'], + when: () => isCanvasInteractiveShell && canvasCommandState.selectionCount > 0, + execute: () => { + canvasViewRef.current?.clearSelection() + } + }, + { + id: 'canvas-shortcut-help', + name: canvasCommandState.shortcutHelpOpen + ? 'Hide Canvas Shortcuts' + : 'Show Canvas Shortcuts', + description: 'Toggle the canvas shortcut help overlay', + icon: 'help-circle', + shortcut: '?', + group: 'Canvas', + keywords: ['help', 'shortcuts', 'canvas', 'hotkeys'], + when: () => isCanvasInteractiveShell, + execute: () => { + canvasViewRef.current?.toggleShortcutHelp() + } + }, + { + id: 'open-settings', + name: 'Open Settings', + description: 'Open the system settings overlay', + icon: 'settings', + execute: handleOpenSettings + }, + { + id: 'open-social-import', + name: 'Import Social Archive', + description: 'Open the social graph archive importer', + icon: 'upload', + group: 'Data', + keywords: ['social', 'archive', 'instagram', 'grok', 'import'], + execute: handleOpenSocialImport + }, + { + id: 'open-data-workspace', + name: 'Open Data Workspace', + description: 'Explore saved views, graph lenses, and imported data counts', + icon: 'database', + group: 'Data', + keywords: ['data', 'workspace', 'social', 'saved views', 'lenses'], + execute: handleOpenDataWorkspace + }, + ...(STORIES_ENABLED + ? [ + { + id: 'open-stories', + name: 'Open Stories', + description: 'Open the dev-only embedded Storybook surface', + icon: 'layout', + group: 'Developer', + execute: handleOpenStories + } satisfies PaletteCommand + ] + : []), + ...recentDocuments.map((document) => ({ + id: `open-${document.id}`, + name: document.title, + description: `Open ${document.type}`, + icon: + document.type === 'page' + ? 'file-text' + : document.type === 'database' + ? 'database' + : 'layout', + group: 'Recent', + execute: () => handleOpenDocument(document.id) + })) + ], + [ + canvasViewRef, + handleCreateCanvasNote, + handleCreateLinkedDocument, + handleOpenDocument, + handleOpenDataWorkspace, + handleOpenSettings, + handleOpenSocialImport, + handleOpenStories, + canvasCommandState, + isCanvasInteractiveShell, + recentDocuments, + shellKind + ] + ) +} diff --git a/apps/web/src/App.tsx b/apps/web/src/App.tsx index b52f3091d..49d832a12 100644 --- a/apps/web/src/App.tsx +++ b/apps/web/src/App.tsx @@ -3,14 +3,14 @@ * * Handles SQLite initialization, onboarding flow, and identity management. * Uses SQLite with OPFS for persistent local-first storage. + * + * The boot orchestration lives in `./boot/`: `useBootSequence` owns the + * storage-init state machine, `useStorageDurability` the durability and + * corruption watchers, and `useWebInstallPrompt` the PWA install plumbing. + * This component composes those hooks and renders per boot state. */ -import type { NodeStorageAdapter } from '@xnetjs/data' import type { Identity, KeyBundle } from '@xnetjs/identity' -import type { PersistentStorageStatus, SQLiteAdapter } from '@xnetjs/sqlite' -import type { TraceCollector } from '@xnetjs/telemetry' import { RouterProvider, createRouter, createHashHistory } from '@tanstack/react-router' -import { SQLiteNodeStorageAdapter, BlobService } from '@xnetjs/data' -import { getDefaultDataWorkerUrl } from '@xnetjs/data-bridge' import { XNetDevToolsProvider } from '@xnetjs/devtools' import { BlobProvider } from '@xnetjs/editor/react' import { @@ -18,24 +18,15 @@ import { OnboardingProvider, OnboardingFlow, ErrorBoundary, - OfflineIndicator, - type XNetRuntimeConfig + OfflineIndicator } from '@xnetjs/react' -import { - checkBrowserSupport, - checkPersistentStorage, - isSilentPersistRequestSafe, - isSQLiteCorruptionError, - recordMemoryFallbackSession, - requestPersistentStorage, - showUnsupportedBrowserMessage, - watchPersistentStoragePermission, - SCHEMA_VERSION, - SCHEMA_DDL -} from '@xnetjs/sqlite' -import { SQLiteStorageAdapter, BlobStore, ChunkManager } from '@xnetjs/storage' +import { requestPersistentStorage, showUnsupportedBrowserMessage } from '@xnetjs/sqlite' import { ThemeProvider } from '@xnetjs/ui' -import { useState, useCallback, useEffect, useRef } from 'react' +import { useState, useCallback, useEffect } from 'react' +import { updateAppStorageStatus, resolveWebRuntime } from './boot/boot-machine' +import { useBootSequence } from './boot/use-boot-sequence' +import { useWebInstallPrompt } from './boot/use-install-prompt' +import { useStorageDurability } from './boot/use-storage-durability' import { BootTimelineProbe } from './components/BootTimelineProbe' import { BundledPluginInstaller } from './components/BundledPluginInstaller' import { ConsentBanner } from './components/ConsentBanner' @@ -43,29 +34,12 @@ import { StorageOptimiseHint } from './components/StorageOptimiseHint' import { StorageWarningBanner } from './components/StorageWarningBanner' import { WarmStartSnapshots } from './components/WarmStartSnapshots' import { WorkingSetPrewarm } from './components/WorkingSetPrewarm' -import { type BootFailure, reportBootFailure } from './lib/boot-diagnostics' -import { bootMark, isBootDebugEnabled, runWhenBootSettled } from './lib/boot-timeline' import { clearXNetBrowserStorage, - clearXNetBrowserStorageResetRequest, - requestXNetBrowserStorageReset, - shouldResetXNetBrowserStorageOnLoad, - subscribeXNetStorageCorruption + requestXNetBrowserStorageReset } from './lib/browser-storage-reset' -import { scheduleChangeLogCompaction } from './lib/change-log-compaction' -import { getDataRuntime, isWorkerRuntimeEnabled } from './lib/data-runtime' -import { schedulePeriodicOptimize } from './lib/db-optimize' -import { scheduleOneTimeVacuum } from './lib/db-vacuum' -import { defaultHubUrl, persistedHubUrl, readHubParam, setPersistedHubUrl } from './lib/hub-url' -import { identityManager } from './lib/identity' -import { startMainThreadStallDetector } from './lib/main-thread-stall' -import { scheduleStalePresenceCleanup } from './lib/presence-blob-cleanup' -import { logStoreContents } from './lib/read-path-probe' -import { startRuntimeLatencyTelemetry } from './lib/runtime-latency-telemetry' import { detectBrowserFamily, getStorageBanner } from './lib/storage-banner' -import { recordDurabilityTransition, subscribeStorageStatus } from './lib/storage-durability' -import { looksEvicted, probeStoreColdStart, recordColdStartProbe } from './lib/store-cold-start' -import { createWebTraceCollector } from './lib/tracing' +import { recordDurabilityTransition } from './lib/storage-durability' import { routeTree } from './routeTree.gen' import './styles/globals.css' @@ -83,255 +57,6 @@ declare module '@tanstack/react-router' { } } -// Hub URL from env or default. -// -// In development an unset VITE_HUB_URL means "no hub" (empty string) rather than -// the production hub: dialing wss://hub.xnet.fyi by accident makes the socket -// reach `connected` against a server that won't ack this client's document -// subscriptions, which used to stall page loads (exploration 0188) and also -// leaked dev presence to production. A falsy hub URL keeps the app local-first; -// opt into a real hub by setting VITE_HUB_URL (e.g. ws://localhost:4444). -const DEFAULT_HUB_URL = defaultHubUrl() - -// A hub the user connected via Settings or the xNet Cloud claim flow (persisted in -// localStorage) wins over the build-time default — this is the read half of that -// setting, without which "connect your cloud hub" did nothing (exploration 0192). -const resolveConfiguredHubUrl = (): string => persistedHubUrl(DEFAULT_HUB_URL) - -if (typeof console !== 'undefined') { - console.info( - '[xNet] hub:', - resolveConfiguredHubUrl() || '(none — local-first; set a hub in Settings or VITE_HUB_URL)' - ) -} - -type SharedHubSession = { - endpoint: string - token: string - exp: number -} - -type BeforeInstallPromptUserChoice = { - outcome: 'accepted' | 'dismissed' - platform: string -} - -type BeforeInstallPromptEvent = Event & { - prompt: () => Promise - userChoice: Promise -} - -function resolveHubSessionFromLocation(): { hubUrl: string; authToken: string | null } { - try { - const parsed = new URL(window.location.href) - // Under hash routing the route query lives inside the fragment - // (e.g. /app/#/doc/x?shareSession=k) — check both locations. - const [hashPath, hashQuery = ''] = parsed.hash.split('?') - const hashParams = new URLSearchParams(hashQuery) - const shareSession = parsed.searchParams.get('shareSession') ?? hashParams.get('shareSession') - - const stripParams = (...names: string[]): void => { - for (const name of names) { - parsed.searchParams.delete(name) - hashParams.delete(name) - } - const hash = hashParams.size > 0 ? `${hashPath}?${hashParams.toString()}` : hashPath - window.history.replaceState({}, '', `${parsed.pathname}${parsed.search}${hash}`) - } - - if ( - parsed.searchParams.has('payload') || - parsed.searchParams.has('handle') || - hashParams.has('payload') || - hashParams.has('handle') - ) { - stripParams('payload', 'handle') - } - // A `hub` param pins a hub for this browser — the xNet Cloud dashboard's "Open - // web app" link passes the user's *personal* hub here so the app dials it - // instead of the shared default. Persist it (so it sticks across reloads) and - // strip it from the URL; an invalid value is ignored, never persisted. - const hubParam = readHubParam(parsed.search, parsed.hash) - if (hubParam.present) { - if (hubParam.hub) setPersistedHubUrl(hubParam.hub) - stripParams('hub') - } - if (!shareSession) { - return { hubUrl: resolveConfiguredHubUrl(), authToken: null } - } - - const stored = sessionStorage.getItem(`xnet:share-session:${shareSession}`) - stripParams('shareSession') - if (!stored) { - return { hubUrl: resolveConfiguredHubUrl(), authToken: null } - } - - sessionStorage.removeItem(`xnet:share-session:${shareSession}`) - const session = JSON.parse(stored) as SharedHubSession - if ( - !session || - typeof session.endpoint !== 'string' || - typeof session.token !== 'string' || - session.endpoint.length === 0 || - session.token.length === 0 || - !Number.isFinite(session.exp) || - session.exp <= Date.now() - ) { - return { hubUrl: resolveConfiguredHubUrl(), authToken: null } - } - - return { hubUrl: session.endpoint, authToken: session.token } - } catch { - return { hubUrl: DEFAULT_HUB_URL, authToken: null } - } -} - -function isStandaloneWebApp(): boolean { - if (typeof window === 'undefined' || typeof navigator === 'undefined') { - return false - } - - return ( - window.matchMedia?.('(display-mode: standalone)').matches === true || - (navigator as Navigator & { standalone?: boolean }).standalone === true - ) -} - -function useWebInstallPrompt(): { - canInstall: boolean - isInstalled: boolean - promptInstall: () => Promise -} { - const [installPrompt, setInstallPrompt] = useState(null) - const [isInstalled, setIsInstalled] = useState(() => isStandaloneWebApp()) - - useEffect(() => { - const handleBeforeInstallPrompt = (event: Event) => { - event.preventDefault() - setInstallPrompt(event as BeforeInstallPromptEvent) - } - - const handleAppInstalled = () => { - setInstallPrompt(null) - setIsInstalled(true) - } - - const mediaQuery = window.matchMedia?.('(display-mode: standalone)') - const handleDisplayModeChange = () => setIsInstalled(isStandaloneWebApp()) - - window.addEventListener('beforeinstallprompt', handleBeforeInstallPrompt) - window.addEventListener('appinstalled', handleAppInstalled) - mediaQuery?.addEventListener?.('change', handleDisplayModeChange) - - return () => { - window.removeEventListener('beforeinstallprompt', handleBeforeInstallPrompt) - window.removeEventListener('appinstalled', handleAppInstalled) - mediaQuery?.removeEventListener?.('change', handleDisplayModeChange) - } - }, []) - - const promptInstall = useCallback(async (): Promise => { - if (!installPrompt) { - return null - } - - const prompt = installPrompt - setInstallPrompt(null) - await prompt.prompt() - const userChoice = await prompt.userChoice.catch(() => null) - - if (userChoice?.outcome === 'accepted') { - setIsInstalled(true) - } - - return userChoice - }, [installPrompt]) - - return { - canInstall: Boolean(installPrompt), - isInstalled, - promptInstall - } -} - -// Cold start can legitimately take 10–20s on a slow first load (SQLite WASM -// download + OPFS), so the watchdog waits past that before declaring a hang -// (exploration 0210). A late success still replaces the timeout screen. -const BOOT_TIMEOUT_MS = 25_000 - -// ─── Types ────────────────────────────────────────────────────── -type AppState = - | { status: 'initializing' } - | { status: 'unsupported'; reason: string } - | { status: 'loading' } - | { status: 'needs-onboarding'; storageWarning?: string; storageStatus?: PersistentStorageStatus } - | { status: 'unlocking'; storageWarning?: string; storageStatus?: PersistentStorageStatus } - | { status: 'storage-corrupt'; error: Error } - | { status: 'boot-timeout'; failure: BootFailure } - | { - status: 'authenticated' - identity: Identity - keyBundle: KeyBundle - storageWarning?: string - storageStatus?: PersistentStorageStatus - } - | { status: 'error'; error: Error } - -// ─── Storage Context ──────────────────────────────────────────── -interface StorageContext { - sqliteAdapter: SQLiteAdapter - nodeStorage: NodeStorageAdapter - storageAdapter: SQLiteStorageAdapter - blobStore: BlobStore - blobService: BlobService - /** SQLite worker port for the data worker (worker runtime flag only) */ - dataWorkerStoragePort?: MessagePort -} - -/** - * With the worker runtime enabled, hand the data worker its own port into - * the SQLite worker so storage calls skip the main thread. - */ -async function createDataWorkerStoragePort(sqliteAdapter: { - createMessagePort(): Promise -}): Promise { - if (!isWorkerRuntimeEnabled()) return undefined - return sqliteAdapter.createMessagePort() -} - -function resolveWebRuntime(storage: StorageContext): XNetRuntimeConfig { - if (isWorkerRuntimeEnabled()) { - return { - mode: 'worker', - fallback: 'main-thread', - diagnostics: import.meta.env.DEV, - worker: { - url: getDefaultDataWorkerUrl(), - storagePort: storage.dataWorkerStoragePort - } - } - } - return { - mode: 'main-thread', - fallback: 'main-thread', - diagnostics: import.meta.env.DEV - } -} - -function updateAppStorageStatus( - current: AppState, - storageStatus: PersistentStorageStatus -): AppState { - switch (current.status) { - case 'needs-onboarding': - case 'unlocking': - case 'authenticated': - return { ...current, storageStatus } - default: - return current - } -} - // ─── Unsupported Browser Component ────────────────────────────── function UnsupportedBrowser({ reason }: { reason: string }): JSX.Element { useEffect(() => { @@ -342,7 +67,7 @@ function UnsupportedBrowser({ reason }: { reason: string }): JSX.Element { // ─── Main App ─────────────────────────────────────────────────── export function App(): JSX.Element { - const [appState, setAppState] = useState({ status: 'initializing' }) + const { appState, setAppState, storageRef, traceCollector, hubUrl, authToken } = useBootSequence() const [isRequestingStorage, setIsRequestingStorage] = useState(false) const [isInstallingApp, setIsInstallingApp] = useState(false) const [isResettingStorage, setIsResettingStorage] = useState(false) @@ -352,355 +77,22 @@ export function App(): JSX.Element { isInstalled: isInstalledApp, promptInstall } = useWebInstallPrompt() - const [{ hubUrl, authToken }] = useState(() => resolveHubSessionFromLocation()) - const storageRef = useRef(null) - // Opt-in performance tracing (exploration 0190): one collector shared by the - // hooks (config.tracing) and the devtools Traces panel. Off unless the user - // sets localStorage['xnet:trace'] = '1', so the hot path pays nothing. - const traceRef = useRef<{ collector?: TraceCollector } | null>(null) - if (traceRef.current === null) traceRef.current = { collector: createWebTraceCollector() } - const traceCollector = traceRef.current.collector - - // Initialize SQLite and storage on mount - useEffect(() => { - let cancelled = false - let cleanupAdapter: SQLiteAdapter | null = null - let cleanupStorageAdapter: SQLiteStorageAdapter | null = null - - async function initialize() { - try { - bootMark('init:start') - // Watch for a main-thread freeze (the ~18s cold-open stall lives in a - // post-hub:connected event-loop block no per-op timer can see; 0253). - startMainThreadStallDetector() - // Input-latency telemetry per data runtime (exploration 0264): the - // measurement the worker-runtime default flip waits on. Compare via - // the `[xNet] runtime input latency` boot log across sessions. - startRuntimeLatencyTelemetry(getDataRuntime()) - if (shouldResetXNetBrowserStorageOnLoad()) { - clearXNetBrowserStorageResetRequest() - await clearXNetBrowserStorage() - } - - // Check browser support first - const support = await checkBrowserSupport() - - if (!support.supported) { - if (cancelled) return - setAppState({ status: 'unsupported', reason: support.reason || 'Browser not supported' }) - return - } - - const storageWarning = support.warning - // Chromium/WebKit decide persist() silently and re-evaluate it on - // every call, so requesting at startup is free — returning users - // flip to granted once engagement/install/notification signals - // land. Firefox would show a modal prompt here, so it stays - // read-only until the user clicks the banner action (0172). - const storageStatus = isSilentPersistRequestSafe() - ? await requestPersistentStorage() - : await checkPersistentStorage() - recordDurabilityTransition('startup', storageStatus) - - // Dynamically import the web proxy to enable code splitting - const { WebSQLiteProxy } = await import('@xnetjs/sqlite/web-proxy') - - // Create and open SQLite adapter - const sqliteAdapter = new WebSQLiteProxy() - cleanupAdapter = sqliteAdapter - - if (cancelled) { - await sqliteAdapter.close() - return - } - - // bootDebug lets the worker emit per-op queue/exec timing + DB stats - // (exploration 0229) — workers can't read the xnet:boot:debug flag. - await sqliteAdapter.open({ path: '/xnet.db', bootDebug: isBootDebugEnabled() }) - bootMark('sqlite:open') - - // Memory-fallback telemetry (exploration 0263): multi-tab leadership - // routing should make non-durable sessions ~zero; count the ones that - // still happen so the win is measurable across sessions. - void sqliteAdapter - .getStorageMode() - .then((mode) => { - if (mode === 'memory') { - const count = recordMemoryFallbackSession() - console.warn('[xNet] sqlite memory-fallback session', { - count, - role: sqliteAdapter.getTabRole() - }) - } - }) - .catch(() => {}) - - // Graceful leadership handoff (0263): on a real page unload, close the - // worker so its OPFS handles release deterministically and the next - // tab promotes without waiting out the handle-contention backoff. - // `persisted` guards bfcache — a restorable page must keep its DB. - window.addEventListener('pagehide', (event: PageTransitionEvent) => { - if (event.persisted) return - void sqliteAdapter.close().catch(() => {}) - }) - - if (cancelled) { - await sqliteAdapter.close() - return - } - - // Apply schema - await sqliteAdapter.applySchema(SCHEMA_VERSION, SCHEMA_DDL) - bootMark('sqlite:schema') - - // Probe whether the local cache is cold/evicted so views can show a - // "restoring from hub" affordance instead of a blank screen, and so a - // silent OPFS eviction is diagnosable (exploration 0204). - // - // F1 (exploration 0249): do NOT await this. The probe is a cold - // `SELECT COUNT(*) FROM nodes` — the first read on the cold OPFS DB — and - // awaiting it serialized it ahead of identity/store/connect for nothing - // but a UI affordance with a safe default. Fire-and-forget; the affordance - // is now reactive (`useRestoringFromHub` subscribes), so it still appears - // when the probe resolves. The `sqlite:probe` mark fires immediately, so - // the boot timeline's `probe` segment ≈ 0 — proving the cold read is no - // longer on the critical path. - bootMark('sqlite:probe') - void probeStoreColdStart(sqliteAdapter, storageStatus.persisted, Boolean(hubUrl)).then( - (coldStart) => { - recordColdStartProbe(coldStart) - if (looksEvicted(coldStart)) { - console.warn( - '[xNet] Local cache is empty and this origin is not persisted — the ' + - 'browser may have evicted it. Re-syncing from the hub; enable persistent ' + - 'storage to keep data across sessions.' - ) - } - } - ) - - // Read-path diagnostic (exploration 0212): when boot debug is on, log - // the durable count matrix (nodes / changes / cursors) so the next - // capture can tell a populated-but-slow read path apart from a genuinely - // empty cache. Fire-and-forget — never blocks boot, never throws. - void logStoreContents(sqliteAdapter) - - // One-time, idle-scheduled cleanup of the stale pre-0227 presence blob - // that still bloats the OPFS DB file (exploration 0229). No-ops after - // the first run; the heavy VACUUM never touches the boot critical path. - scheduleStalePresenceCleanup(sqliteAdapter) - - // One-time, idle-scheduled VACUUM that defragments the OPFS file so the - // first cold landing query faults a smaller, denser working set — the - // ~15.8 s cold-read stall caught in exploration 0233. No-ops after the - // first run; logs file size before/after (the `db stats` measurement). - scheduleOneTimeVacuum(sqliteAdapter) - schedulePeriodicOptimize(sqliteAdapter) - - // Adaptive indexes + property-sort pushdown (exploration 0264, Wave 2) - // soak behind a local flag before any default flip; index creation - // rides the bootSettled idle cadence — no background work is free on - // the single serial SQLite worker (0260). - let adaptiveIndexingEnabled = false - try { - adaptiveIndexingEnabled = localStorage.getItem('xnet:adaptive-indexes') === 'true' - } catch { - // localStorage unavailable — keep the default off. - } - const nodeStorage = new SQLiteNodeStorageAdapter(sqliteAdapter, { - adaptiveIndexing: { enabled: adaptiveIndexingEnabled }, - scheduleMaintenance: (task) => runWhenBootSettled(() => void task()) - }) - - // Idle-scheduled change-log compaction (exploration 0254 / F3): prune - // superseded history from the local `changes` log so the OPFS file — and - // the first outbound-resync slice — shrink at the root, the durable fix - // for the recurring cold-open stall. Convergence-safe (keeps every - // live-value backer + per-node tips) and behind the - // `xnet:compact:changes=off` kill switch. - scheduleChangeLogCompaction(nodeStorage, sqliteAdapter) - - const storageAdapter = new SQLiteStorageAdapter(sqliteAdapter) - await storageAdapter.open() - // Boot-phase split (0249): storage adapter is open; what follows up to - // identity:ready is blob services + the data-worker port + identity. - bootMark('storage:open') - cleanupStorageAdapter = storageAdapter - - const blobStore = new BlobStore(storageAdapter) - const chunkManager = new ChunkManager(blobStore) - const blobService = new BlobService(chunkManager) - - if (cancelled) { - await storageAdapter.close() - await sqliteAdapter.close() - return - } - - const dataWorkerStoragePort = await createDataWorkerStoragePort(sqliteAdapter) - // Store refs for later use - storageRef.current = { - sqliteAdapter, - nodeStorage, - storageAdapter, - blobStore, - blobService, - dataWorkerStoragePort - } - - // Check for existing identity - const hasIdentity = await identityManager.hasIdentity() - // Boot-phase split (0249): everything after this mark up to - // identity:ready is the session unlock/resume crypto — so a slow - // `identityResume` segment isolates a KDF/unwrap cost from storage I/O. - bootMark('identity:checked') - if (cancelled) { - await sqliteAdapter.close() - return - } - - if (hasIdentity) { - // A persisted session from a previous unlock lets us skip the - // biometric prompt across reloads. - const resumed = await identityManager.resume().catch(() => null) - if (cancelled) return - - if (resumed) { - bootMark('identity:ready') - setAppState({ - status: 'authenticated', - identity: resumed.identity, - keyBundle: resumed, - storageWarning, - storageStatus - }) - return - } - - setAppState({ status: 'unlocking', storageWarning, storageStatus }) - try { - const keyBundle = await identityManager.unlock() - if (cancelled) return - bootMark('identity:ready') - setAppState({ - status: 'authenticated', - identity: keyBundle.identity, - keyBundle, - storageWarning, - storageStatus - }) - } catch (_err) { - if (cancelled) return - setAppState({ status: 'needs-onboarding', storageWarning, storageStatus }) - } - } else { - setAppState({ status: 'needs-onboarding', storageWarning, storageStatus }) - } - } catch (err) { - if (cancelled) return - await cleanupStorageAdapter?.close().catch(console.error) - await cleanupAdapter?.close().catch(console.error) - cleanupStorageAdapter = null - cleanupAdapter = null - - console.error('[App] Initialization failed:', err) - const error = err instanceof Error ? err : new Error(String(err)) - // Report with the furthest boot phase reached so a field failure is - // diagnosable instead of a silent blank/error screen (exploration 0210). - reportBootFailure('init', error) - if (isSQLiteCorruptionError(error)) { - setAppState({ status: 'storage-corrupt', error }) - return - } - - setAppState({ - status: 'error', - error - }) - } - } - - initialize() - - return () => { - cancelled = true - // Cleanup: close adapter immediately to prevent OPFS access handle conflicts - if (cleanupStorageAdapter) { - cleanupStorageAdapter.close().catch(console.error) - } - - if (cleanupAdapter) { - cleanupAdapter.close().catch(console.error) - } else if (storageRef.current?.sqliteAdapter) { - storageRef.current.sqliteAdapter.close().catch(console.error) - } - } - // hubUrl is resolved once from a useState initializer and never re-set, so - // this still runs a single time; it's listed to satisfy exhaustive-deps now - // that the cold-start probe reads it (exploration 0204). - }, [hubUrl]) - - // Boot watchdog (exploration 0210): a hung boot — SQLite WASM that never - // resolves, an OPFS handle that blocks, a hub socket that connects but never - // acks — throws nothing, so the init try/catch can't see it and the user is - // stuck on the "Initializing database…" spinner forever. If we're still - // initializing after the timeout, surface an actionable screen and report it. - useEffect(() => { - if (appState.status !== 'initializing') return - const timer = window.setTimeout(() => { - const failure = reportBootFailure( - 'timeout', - new Error(`Boot did not complete within ${BOOT_TIMEOUT_MS / 1000}s`) - ) - setAppState({ status: 'boot-timeout', failure }) - }, BOOT_TIMEOUT_MS) - return () => window.clearTimeout(timer) - }, [appState.status]) - - // A persistent-storage grant can land mid-session (notification opt-in, - // install, engagement crossing Chrome's threshold). Watching the - // permission is free — it never spends or triggers a request (0172). - useEffect(() => { - return watchPersistentStoragePermission((state) => { - if (state !== 'granted') return - void checkPersistentStorage().then((storageStatus) => { - recordDurabilityTransition('permission-change', storageStatus) - setAppState((current) => updateAppStorageStatus(current, storageStatus)) - }) - }) - }, []) - - // Statuses produced outside App's own handlers (the desktop-alerts - // opt-in chains a persist() request after a notification grant). - useEffect(() => { - return subscribeStorageStatus((storageStatus) => { - setAppState((current) => updateAppStorageStatus(current, storageStatus)) - }) - }, []) - - useEffect(() => { - return subscribeXNetStorageCorruption((error) => { - const storage = storageRef.current - storageRef.current = null - - void storage?.storageAdapter.close().catch(console.error) - void storage?.sqliteAdapter.close().catch(console.error) - - setAppState({ status: 'storage-corrupt', error }) - }) - }, []) + useStorageDurability(setAppState, storageRef) // Handle onboarding completion - const handleOnboardingComplete = useCallback((identity: Identity, keyBundle: KeyBundle) => { - setAppState((current) => ({ - status: 'authenticated', - identity, - keyBundle, - storageWarning: 'storageWarning' in current ? current.storageWarning : undefined, - storageStatus: 'storageStatus' in current ? current.storageStatus : undefined - })) - }, []) + const handleOnboardingComplete = useCallback( + (identity: Identity, keyBundle: KeyBundle) => { + setAppState((current) => ({ + status: 'authenticated', + identity, + keyBundle, + storageWarning: 'storageWarning' in current ? current.storageWarning : undefined, + storageStatus: 'storageStatus' in current ? current.storageStatus : undefined + })) + }, + [setAppState] + ) const handleRequestPersistentStorage = useCallback(async () => { setIsRequestingStorage(true) @@ -725,7 +117,7 @@ export function App(): JSX.Element { } finally { setIsRequestingStorage(false) } - }, []) + }, [setAppState]) const handleInstallApp = useCallback(async () => { setIsInstallingApp(true) @@ -743,7 +135,7 @@ export function App(): JSX.Element { } finally { setIsInstallingApp(false) } - }, [promptInstall]) + }, [promptInstall, setAppState]) const handleResetCorruptStorage = useCallback(async () => { setIsResettingStorage(true) @@ -758,7 +150,7 @@ export function App(): JSX.Element { } finally { setIsResettingStorage(false) } - }, []) + }, [setAppState]) // ─── Render ───────────────────────────────────────────────────── diff --git a/apps/web/src/boot/boot-machine.ts b/apps/web/src/boot/boot-machine.ts new file mode 100644 index 000000000..06be8925e --- /dev/null +++ b/apps/web/src/boot/boot-machine.ts @@ -0,0 +1,79 @@ +/** + * Web boot state machine — the app-level boot state union plus the pure + * transition helpers App.tsx and the boot hooks share. No React in here. + */ +import type { BootFailure } from '../lib/boot-diagnostics' +import type { BlobService, NodeStorageAdapter } from '@xnetjs/data' +import type { Identity, KeyBundle } from '@xnetjs/identity' +import type { XNetRuntimeConfig } from '@xnetjs/react' +import type { PersistentStorageStatus, SQLiteAdapter } from '@xnetjs/sqlite' +import type { BlobStore, SQLiteStorageAdapter } from '@xnetjs/storage' +import { getDefaultDataWorkerUrl } from '@xnetjs/data-bridge' +import { isWorkerRuntimeEnabled } from '../lib/data-runtime' + +// Cold start can legitimately take 10–20s on a slow first load (SQLite WASM +// download + OPFS), so the watchdog waits past that before declaring a hang +// (exploration 0210). A late success still replaces the timeout screen. +export const BOOT_TIMEOUT_MS = 25_000 + +// ─── Types ────────────────────────────────────────────────────── +export type AppState = + | { status: 'initializing' } + | { status: 'unsupported'; reason: string } + | { status: 'loading' } + | { status: 'needs-onboarding'; storageWarning?: string; storageStatus?: PersistentStorageStatus } + | { status: 'unlocking'; storageWarning?: string; storageStatus?: PersistentStorageStatus } + | { status: 'storage-corrupt'; error: Error } + | { status: 'boot-timeout'; failure: BootFailure } + | { + status: 'authenticated' + identity: Identity + keyBundle: KeyBundle + storageWarning?: string + storageStatus?: PersistentStorageStatus + } + | { status: 'error'; error: Error } + +// ─── Storage Context ──────────────────────────────────────────── +export interface StorageContext { + sqliteAdapter: SQLiteAdapter + nodeStorage: NodeStorageAdapter + storageAdapter: SQLiteStorageAdapter + blobStore: BlobStore + blobService: BlobService + /** SQLite worker port for the data worker (worker runtime flag only) */ + dataWorkerStoragePort?: MessagePort +} + +export function resolveWebRuntime(storage: StorageContext): XNetRuntimeConfig { + if (isWorkerRuntimeEnabled()) { + return { + mode: 'worker', + fallback: 'main-thread', + diagnostics: import.meta.env.DEV, + worker: { + url: getDefaultDataWorkerUrl(), + storagePort: storage.dataWorkerStoragePort + } + } + } + return { + mode: 'main-thread', + fallback: 'main-thread', + diagnostics: import.meta.env.DEV + } +} + +export function updateAppStorageStatus( + current: AppState, + storageStatus: PersistentStorageStatus +): AppState { + switch (current.status) { + case 'needs-onboarding': + case 'unlocking': + case 'authenticated': + return { ...current, storageStatus } + default: + return current + } +} diff --git a/apps/web/src/boot/use-boot-sequence.ts b/apps/web/src/boot/use-boot-sequence.ts new file mode 100644 index 000000000..e620b75e6 --- /dev/null +++ b/apps/web/src/boot/use-boot-sequence.ts @@ -0,0 +1,469 @@ +/** + * Web boot sequence — the storage-init effect (SQLite adapter, blob services, + * worker-vs-main runtime port, cold-start probe, identity resume/unlock) plus + * the boot watchdog, extracted from App.tsx. Owns the boot AppState, the + * storage refs, and the trace collector; App.tsx composes this hook and + * renders per state. + */ +import type { AppState, StorageContext } from './boot-machine' +import type { SQLiteAdapter } from '@xnetjs/sqlite' +import type { TraceCollector } from '@xnetjs/telemetry' +import type { Dispatch, MutableRefObject, SetStateAction } from 'react' +import { SQLiteNodeStorageAdapter, BlobService } from '@xnetjs/data' +import { + checkBrowserSupport, + checkPersistentStorage, + isSilentPersistRequestSafe, + isSQLiteCorruptionError, + recordMemoryFallbackSession, + requestPersistentStorage, + SCHEMA_VERSION, + SCHEMA_DDL +} from '@xnetjs/sqlite' +import { SQLiteStorageAdapter, BlobStore, ChunkManager } from '@xnetjs/storage' +import { useState, useEffect, useRef } from 'react' +import { reportBootFailure } from '../lib/boot-diagnostics' +import { bootMark, isBootDebugEnabled, runWhenBootSettled } from '../lib/boot-timeline' +import { + clearXNetBrowserStorage, + clearXNetBrowserStorageResetRequest, + shouldResetXNetBrowserStorageOnLoad +} from '../lib/browser-storage-reset' +import { scheduleChangeLogCompaction } from '../lib/change-log-compaction' +import { getDataRuntime, isWorkerRuntimeEnabled } from '../lib/data-runtime' +import { schedulePeriodicOptimize } from '../lib/db-optimize' +import { scheduleOneTimeVacuum } from '../lib/db-vacuum' +import { defaultHubUrl, persistedHubUrl, readHubParam, setPersistedHubUrl } from '../lib/hub-url' +import { identityManager } from '../lib/identity' +import { startMainThreadStallDetector } from '../lib/main-thread-stall' +import { scheduleStalePresenceCleanup } from '../lib/presence-blob-cleanup' +import { logStoreContents } from '../lib/read-path-probe' +import { startRuntimeLatencyTelemetry } from '../lib/runtime-latency-telemetry' +import { recordDurabilityTransition } from '../lib/storage-durability' +import { looksEvicted, probeStoreColdStart, recordColdStartProbe } from '../lib/store-cold-start' +import { createWebTraceCollector } from '../lib/tracing' +import { BOOT_TIMEOUT_MS } from './boot-machine' + +// Hub URL from env or default. +// +// In development an unset VITE_HUB_URL means "no hub" (empty string) rather than +// the production hub: dialing wss://hub.xnet.fyi by accident makes the socket +// reach `connected` against a server that won't ack this client's document +// subscriptions, which used to stall page loads (exploration 0188) and also +// leaked dev presence to production. A falsy hub URL keeps the app local-first; +// opt into a real hub by setting VITE_HUB_URL (e.g. ws://localhost:4444). +const DEFAULT_HUB_URL = defaultHubUrl() + +// A hub the user connected via Settings or the xNet Cloud claim flow (persisted in +// localStorage) wins over the build-time default — this is the read half of that +// setting, without which "connect your cloud hub" did nothing (exploration 0192). +const resolveConfiguredHubUrl = (): string => persistedHubUrl(DEFAULT_HUB_URL) + +if (typeof console !== 'undefined') { + console.info( + '[xNet] hub:', + resolveConfiguredHubUrl() || '(none — local-first; set a hub in Settings or VITE_HUB_URL)' + ) +} + +type SharedHubSession = { + endpoint: string + token: string + exp: number +} + +export function resolveHubSessionFromLocation(): { hubUrl: string; authToken: string | null } { + try { + const parsed = new URL(window.location.href) + // Under hash routing the route query lives inside the fragment + // (e.g. /app/#/doc/x?shareSession=k) — check both locations. + const [hashPath, hashQuery = ''] = parsed.hash.split('?') + const hashParams = new URLSearchParams(hashQuery) + const shareSession = parsed.searchParams.get('shareSession') ?? hashParams.get('shareSession') + + const stripParams = (...names: string[]): void => { + for (const name of names) { + parsed.searchParams.delete(name) + hashParams.delete(name) + } + const hash = hashParams.size > 0 ? `${hashPath}?${hashParams.toString()}` : hashPath + window.history.replaceState({}, '', `${parsed.pathname}${parsed.search}${hash}`) + } + + if ( + parsed.searchParams.has('payload') || + parsed.searchParams.has('handle') || + hashParams.has('payload') || + hashParams.has('handle') + ) { + stripParams('payload', 'handle') + } + // A `hub` param pins a hub for this browser — the xNet Cloud dashboard's "Open + // web app" link passes the user's *personal* hub here so the app dials it + // instead of the shared default. Persist it (so it sticks across reloads) and + // strip it from the URL; an invalid value is ignored, never persisted. + const hubParam = readHubParam(parsed.search, parsed.hash) + if (hubParam.present) { + if (hubParam.hub) setPersistedHubUrl(hubParam.hub) + stripParams('hub') + } + if (!shareSession) { + return { hubUrl: resolveConfiguredHubUrl(), authToken: null } + } + + const stored = sessionStorage.getItem(`xnet:share-session:${shareSession}`) + stripParams('shareSession') + if (!stored) { + return { hubUrl: resolveConfiguredHubUrl(), authToken: null } + } + + sessionStorage.removeItem(`xnet:share-session:${shareSession}`) + const session = JSON.parse(stored) as SharedHubSession + if ( + !session || + typeof session.endpoint !== 'string' || + typeof session.token !== 'string' || + session.endpoint.length === 0 || + session.token.length === 0 || + !Number.isFinite(session.exp) || + session.exp <= Date.now() + ) { + return { hubUrl: resolveConfiguredHubUrl(), authToken: null } + } + + return { hubUrl: session.endpoint, authToken: session.token } + } catch { + return { hubUrl: DEFAULT_HUB_URL, authToken: null } + } +} + +/** + * With the worker runtime enabled, hand the data worker its own port into + * the SQLite worker so storage calls skip the main thread. + */ +async function createDataWorkerStoragePort(sqliteAdapter: { + createMessagePort(): Promise +}): Promise { + if (!isWorkerRuntimeEnabled()) return undefined + return sqliteAdapter.createMessagePort() +} + +export interface BootSequence { + appState: AppState + setAppState: Dispatch> + storageRef: MutableRefObject + traceCollector: TraceCollector | undefined + hubUrl: string + authToken: string | null +} + +export function useBootSequence(): BootSequence { + const [appState, setAppState] = useState({ status: 'initializing' }) + const [{ hubUrl, authToken }] = useState(() => resolveHubSessionFromLocation()) + const storageRef = useRef(null) + // Opt-in performance tracing (exploration 0190): one collector shared by the + // hooks (config.tracing) and the devtools Traces panel. Off unless the user + // sets localStorage['xnet:trace'] = '1', so the hot path pays nothing. + const traceRef = useRef<{ collector?: TraceCollector } | null>(null) + if (traceRef.current === null) traceRef.current = { collector: createWebTraceCollector() } + const traceCollector = traceRef.current.collector + + // Initialize SQLite and storage on mount + useEffect(() => { + let cancelled = false + let cleanupAdapter: SQLiteAdapter | null = null + let cleanupStorageAdapter: SQLiteStorageAdapter | null = null + + async function initialize() { + try { + bootMark('init:start') + // Watch for a main-thread freeze (the ~18s cold-open stall lives in a + // post-hub:connected event-loop block no per-op timer can see; 0253). + startMainThreadStallDetector() + // Input-latency telemetry per data runtime (exploration 0264): the + // measurement the worker-runtime default flip waits on. Compare via + // the `[xNet] runtime input latency` boot log across sessions. + startRuntimeLatencyTelemetry(getDataRuntime()) + if (shouldResetXNetBrowserStorageOnLoad()) { + clearXNetBrowserStorageResetRequest() + await clearXNetBrowserStorage() + } + + // Check browser support first + const support = await checkBrowserSupport() + + if (!support.supported) { + if (cancelled) return + setAppState({ status: 'unsupported', reason: support.reason || 'Browser not supported' }) + return + } + + const storageWarning = support.warning + // Chromium/WebKit decide persist() silently and re-evaluate it on + // every call, so requesting at startup is free — returning users + // flip to granted once engagement/install/notification signals + // land. Firefox would show a modal prompt here, so it stays + // read-only until the user clicks the banner action (0172). + const storageStatus = isSilentPersistRequestSafe() + ? await requestPersistentStorage() + : await checkPersistentStorage() + recordDurabilityTransition('startup', storageStatus) + + // Dynamically import the web proxy to enable code splitting + const { WebSQLiteProxy } = await import('@xnetjs/sqlite/web-proxy') + + // Create and open SQLite adapter + const sqliteAdapter = new WebSQLiteProxy() + cleanupAdapter = sqliteAdapter + + if (cancelled) { + await sqliteAdapter.close() + return + } + + // bootDebug lets the worker emit per-op queue/exec timing + DB stats + // (exploration 0229) — workers can't read the xnet:boot:debug flag. + await sqliteAdapter.open({ path: '/xnet.db', bootDebug: isBootDebugEnabled() }) + bootMark('sqlite:open') + + // Memory-fallback telemetry (exploration 0263): multi-tab leadership + // routing should make non-durable sessions ~zero; count the ones that + // still happen so the win is measurable across sessions. + void sqliteAdapter + .getStorageMode() + .then((mode) => { + if (mode === 'memory') { + const count = recordMemoryFallbackSession() + console.warn('[xNet] sqlite memory-fallback session', { + count, + role: sqliteAdapter.getTabRole() + }) + } + }) + .catch(() => {}) + + // Graceful leadership handoff (0263): on a real page unload, close the + // worker so its OPFS handles release deterministically and the next + // tab promotes without waiting out the handle-contention backoff. + // `persisted` guards bfcache — a restorable page must keep its DB. + window.addEventListener('pagehide', (event: PageTransitionEvent) => { + if (event.persisted) return + void sqliteAdapter.close().catch(() => {}) + }) + + if (cancelled) { + await sqliteAdapter.close() + return + } + + // Apply schema + await sqliteAdapter.applySchema(SCHEMA_VERSION, SCHEMA_DDL) + bootMark('sqlite:schema') + + // Probe whether the local cache is cold/evicted so views can show a + // "restoring from hub" affordance instead of a blank screen, and so a + // silent OPFS eviction is diagnosable (exploration 0204). + // + // F1 (exploration 0249): do NOT await this. The probe is a cold + // `SELECT COUNT(*) FROM nodes` — the first read on the cold OPFS DB — and + // awaiting it serialized it ahead of identity/store/connect for nothing + // but a UI affordance with a safe default. Fire-and-forget; the affordance + // is now reactive (`useRestoringFromHub` subscribes), so it still appears + // when the probe resolves. The `sqlite:probe` mark fires immediately, so + // the boot timeline's `probe` segment ≈ 0 — proving the cold read is no + // longer on the critical path. + bootMark('sqlite:probe') + void probeStoreColdStart(sqliteAdapter, storageStatus.persisted, Boolean(hubUrl)).then( + (coldStart) => { + recordColdStartProbe(coldStart) + if (looksEvicted(coldStart)) { + console.warn( + '[xNet] Local cache is empty and this origin is not persisted — the ' + + 'browser may have evicted it. Re-syncing from the hub; enable persistent ' + + 'storage to keep data across sessions.' + ) + } + } + ) + + // Read-path diagnostic (exploration 0212): when boot debug is on, log + // the durable count matrix (nodes / changes / cursors) so the next + // capture can tell a populated-but-slow read path apart from a genuinely + // empty cache. Fire-and-forget — never blocks boot, never throws. + void logStoreContents(sqliteAdapter) + + // One-time, idle-scheduled cleanup of the stale pre-0227 presence blob + // that still bloats the OPFS DB file (exploration 0229). No-ops after + // the first run; the heavy VACUUM never touches the boot critical path. + scheduleStalePresenceCleanup(sqliteAdapter) + + // One-time, idle-scheduled VACUUM that defragments the OPFS file so the + // first cold landing query faults a smaller, denser working set — the + // ~15.8 s cold-read stall caught in exploration 0233. No-ops after the + // first run; logs file size before/after (the `db stats` measurement). + scheduleOneTimeVacuum(sqliteAdapter) + schedulePeriodicOptimize(sqliteAdapter) + + // Adaptive indexes + property-sort pushdown (exploration 0264, Wave 2) + // soak behind a local flag before any default flip; index creation + // rides the bootSettled idle cadence — no background work is free on + // the single serial SQLite worker (0260). + let adaptiveIndexingEnabled = false + try { + adaptiveIndexingEnabled = localStorage.getItem('xnet:adaptive-indexes') === 'true' + } catch { + // localStorage unavailable — keep the default off. + } + const nodeStorage = new SQLiteNodeStorageAdapter(sqliteAdapter, { + adaptiveIndexing: { enabled: adaptiveIndexingEnabled }, + scheduleMaintenance: (task) => runWhenBootSettled(() => void task()) + }) + + // Idle-scheduled change-log compaction (exploration 0254 / F3): prune + // superseded history from the local `changes` log so the OPFS file — and + // the first outbound-resync slice — shrink at the root, the durable fix + // for the recurring cold-open stall. Convergence-safe (keeps every + // live-value backer + per-node tips) and behind the + // `xnet:compact:changes=off` kill switch. + scheduleChangeLogCompaction(nodeStorage, sqliteAdapter) + + const storageAdapter = new SQLiteStorageAdapter(sqliteAdapter) + await storageAdapter.open() + // Boot-phase split (0249): storage adapter is open; what follows up to + // identity:ready is blob services + the data-worker port + identity. + bootMark('storage:open') + cleanupStorageAdapter = storageAdapter + + const blobStore = new BlobStore(storageAdapter) + const chunkManager = new ChunkManager(blobStore) + const blobService = new BlobService(chunkManager) + + if (cancelled) { + await storageAdapter.close() + await sqliteAdapter.close() + return + } + + const dataWorkerStoragePort = await createDataWorkerStoragePort(sqliteAdapter) + + // Store refs for later use + storageRef.current = { + sqliteAdapter, + nodeStorage, + storageAdapter, + blobStore, + blobService, + dataWorkerStoragePort + } + + // Check for existing identity + const hasIdentity = await identityManager.hasIdentity() + // Boot-phase split (0249): everything after this mark up to + // identity:ready is the session unlock/resume crypto — so a slow + // `identityResume` segment isolates a KDF/unwrap cost from storage I/O. + bootMark('identity:checked') + if (cancelled) { + await sqliteAdapter.close() + return + } + + if (hasIdentity) { + // A persisted session from a previous unlock lets us skip the + // biometric prompt across reloads. + const resumed = await identityManager.resume().catch(() => null) + if (cancelled) return + + if (resumed) { + bootMark('identity:ready') + setAppState({ + status: 'authenticated', + identity: resumed.identity, + keyBundle: resumed, + storageWarning, + storageStatus + }) + return + } + + setAppState({ status: 'unlocking', storageWarning, storageStatus }) + try { + const keyBundle = await identityManager.unlock() + if (cancelled) return + bootMark('identity:ready') + setAppState({ + status: 'authenticated', + identity: keyBundle.identity, + keyBundle, + storageWarning, + storageStatus + }) + } catch (_err) { + if (cancelled) return + setAppState({ status: 'needs-onboarding', storageWarning, storageStatus }) + } + } else { + setAppState({ status: 'needs-onboarding', storageWarning, storageStatus }) + } + } catch (err) { + if (cancelled) return + await cleanupStorageAdapter?.close().catch(console.error) + await cleanupAdapter?.close().catch(console.error) + cleanupStorageAdapter = null + cleanupAdapter = null + + console.error('[App] Initialization failed:', err) + const error = err instanceof Error ? err : new Error(String(err)) + // Report with the furthest boot phase reached so a field failure is + // diagnosable instead of a silent blank/error screen (exploration 0210). + reportBootFailure('init', error) + if (isSQLiteCorruptionError(error)) { + setAppState({ status: 'storage-corrupt', error }) + return + } + + setAppState({ + status: 'error', + error + }) + } + } + + initialize() + + return () => { + cancelled = true + // Cleanup: close adapter immediately to prevent OPFS access handle conflicts + if (cleanupStorageAdapter) { + cleanupStorageAdapter.close().catch(console.error) + } + + if (cleanupAdapter) { + cleanupAdapter.close().catch(console.error) + } else if (storageRef.current?.sqliteAdapter) { + storageRef.current.sqliteAdapter.close().catch(console.error) + } + } + // hubUrl is resolved once from a useState initializer and never re-set, so + // this still runs a single time; it's listed to satisfy exhaustive-deps now + // that the cold-start probe reads it (exploration 0204). + }, [hubUrl]) + + // Boot watchdog (exploration 0210): a hung boot — SQLite WASM that never + // resolves, an OPFS handle that blocks, a hub socket that connects but never + // acks — throws nothing, so the init try/catch can't see it and the user is + // stuck on the "Initializing database…" spinner forever. If we're still + // initializing after the timeout, surface an actionable screen and report it. + useEffect(() => { + if (appState.status !== 'initializing') return + const timer = window.setTimeout(() => { + const failure = reportBootFailure( + 'timeout', + new Error(`Boot did not complete within ${BOOT_TIMEOUT_MS / 1000}s`) + ) + setAppState({ status: 'boot-timeout', failure }) + }, BOOT_TIMEOUT_MS) + return () => window.clearTimeout(timer) + }, [appState.status]) + + return { appState, setAppState, storageRef, traceCollector, hubUrl, authToken } +} diff --git a/apps/web/src/boot/use-install-prompt.ts b/apps/web/src/boot/use-install-prompt.ts new file mode 100644 index 000000000..5c0696f9c --- /dev/null +++ b/apps/web/src/boot/use-install-prompt.ts @@ -0,0 +1,84 @@ +/** + * PWA install-prompt plumbing, extracted from App.tsx: captures the deferred + * `beforeinstallprompt` event, tracks standalone/installed state, and exposes + * a `promptInstall` action. + */ +import { useState, useCallback, useEffect } from 'react' + +export type BeforeInstallPromptUserChoice = { + outcome: 'accepted' | 'dismissed' + platform: string +} + +type BeforeInstallPromptEvent = Event & { + prompt: () => Promise + userChoice: Promise +} + +export function isStandaloneWebApp(): boolean { + if (typeof window === 'undefined' || typeof navigator === 'undefined') { + return false + } + + return ( + window.matchMedia?.('(display-mode: standalone)').matches === true || + (navigator as Navigator & { standalone?: boolean }).standalone === true + ) +} + +export function useWebInstallPrompt(): { + canInstall: boolean + isInstalled: boolean + promptInstall: () => Promise +} { + const [installPrompt, setInstallPrompt] = useState(null) + const [isInstalled, setIsInstalled] = useState(() => isStandaloneWebApp()) + + useEffect(() => { + const handleBeforeInstallPrompt = (event: Event) => { + event.preventDefault() + setInstallPrompt(event as BeforeInstallPromptEvent) + } + + const handleAppInstalled = () => { + setInstallPrompt(null) + setIsInstalled(true) + } + + const mediaQuery = window.matchMedia?.('(display-mode: standalone)') + const handleDisplayModeChange = () => setIsInstalled(isStandaloneWebApp()) + + window.addEventListener('beforeinstallprompt', handleBeforeInstallPrompt) + window.addEventListener('appinstalled', handleAppInstalled) + mediaQuery?.addEventListener?.('change', handleDisplayModeChange) + + return () => { + window.removeEventListener('beforeinstallprompt', handleBeforeInstallPrompt) + window.removeEventListener('appinstalled', handleAppInstalled) + mediaQuery?.removeEventListener?.('change', handleDisplayModeChange) + } + }, []) + + const promptInstall = useCallback(async (): Promise => { + if (!installPrompt) { + return null + } + + const prompt = installPrompt + setInstallPrompt(null) + await prompt.prompt() + const userChoice = await prompt.userChoice.catch(() => null) + + if (userChoice?.outcome === 'accepted') { + setIsInstalled(true) + } + + return userChoice + }, [installPrompt]) + + return { + canInstall: Boolean(installPrompt), + isInstalled, + promptInstall + } +} diff --git a/apps/web/src/boot/use-storage-durability.ts b/apps/web/src/boot/use-storage-durability.ts new file mode 100644 index 000000000..3ebc21625 --- /dev/null +++ b/apps/web/src/boot/use-storage-durability.ts @@ -0,0 +1,51 @@ +/** + * Storage durability + corruption watchers, extracted from App.tsx: the + * persistent-storage permission watcher, the out-of-band storage-status + * subscription, and the SQLite corruption subscription — unified into one + * hook that feeds status changes back into the boot state machine. + */ +import type { AppState, StorageContext } from './boot-machine' +import type { Dispatch, MutableRefObject, SetStateAction } from 'react' +import { checkPersistentStorage, watchPersistentStoragePermission } from '@xnetjs/sqlite' +import { useEffect } from 'react' +import { subscribeXNetStorageCorruption } from '../lib/browser-storage-reset' +import { recordDurabilityTransition, subscribeStorageStatus } from '../lib/storage-durability' +import { updateAppStorageStatus } from './boot-machine' + +export function useStorageDurability( + setAppState: Dispatch>, + storageRef: MutableRefObject +): void { + // A persistent-storage grant can land mid-session (notification opt-in, + // install, engagement crossing Chrome's threshold). Watching the + // permission is free — it never spends or triggers a request (0172). + useEffect(() => { + return watchPersistentStoragePermission((state) => { + if (state !== 'granted') return + void checkPersistentStorage().then((storageStatus) => { + recordDurabilityTransition('permission-change', storageStatus) + setAppState((current) => updateAppStorageStatus(current, storageStatus)) + }) + }) + }, [setAppState]) + + // Statuses produced outside App's own handlers (the desktop-alerts + // opt-in chains a persist() request after a notification grant). + useEffect(() => { + return subscribeStorageStatus((storageStatus) => { + setAppState((current) => updateAppStorageStatus(current, storageStatus)) + }) + }, [setAppState]) + + useEffect(() => { + return subscribeXNetStorageCorruption((error) => { + const storage = storageRef.current + storageRef.current = null + + void storage?.storageAdapter.close().catch(console.error) + void storage?.sqliteAdapter.close().catch(console.error) + + setAppState({ status: 'storage-corrupt', error }) + }) + }, [setAppState, storageRef]) +} diff --git a/apps/web/src/components/DataWorkspaceView.tsx b/apps/web/src/components/DataWorkspaceView.tsx index 1dfe37518..642f00f69 100644 --- a/apps/web/src/components/DataWorkspaceView.tsx +++ b/apps/web/src/components/DataWorkspaceView.tsx @@ -1,62 +1,13 @@ -import type { SavedViewDescriptor } from '@xnetjs/data' -import { SavedViewSchema, validateSavedViewDescriptor } from '@xnetjs/data' -import { - SavedViewRunner, - useMutate, - useQuery, - type MutateOp, - type SavedViewLensDraft, - type SavedViewSchemaRegistry -} from '@xnetjs/react' +/** + * DataWorkspaceView (web) — page chrome around the shared Data Workspace core + * (@xnetjs/views, exploration 0276). Web-specific concerns: OPFS store-backed + * seeding, social feed enrichment, and the moderation render gate. + */ import { useNodeStore } from '@xnetjs/react/internal' -import { - listSocialImportJobs, - subscribeSocialImportJobs, - type SocialImportJobProgress -} from '@xnetjs/social/import/core' -import { createDefaultSocialGraphAtlas, type SocialGraphAtlasEntry } from '@xnetjs/social/lenses' -import { - createSocialPatternSavedViewDraft, - detectSocialPatterns, - type SocialPatternKind, - type SocialPatternSuggestion -} from '@xnetjs/social/patterns' -import { - SocialActorSchema, - SocialCollectionSchema, - SocialContentSchema, - SocialConversationSchema, - SocialImportRunSchema, - SocialInteractionSchema, - SocialMessageSchema, - socialSchemas -} from '@xnetjs/social/schemas' -import { - recommendSocialAnalyticsCache, - type SocialAnalyticsCacheRecommendation -} from '@xnetjs/social/workspace' -import { - AlertTriangle, - BarChart3, - Database, - GitBranch, - Import, - Loader2, - MessageSquare, - Network, - Save, - Search, - Shield, - Table, - UserRound -} from 'lucide-react' -import { useEffect, useMemo, useState, type ReactNode } from 'react' +import { useDataWorkspace, DataWorkspaceBody } from '@xnetjs/views' +import { Database, Import, Loader2 } from 'lucide-react' +import { useMemo, type ReactNode } from 'react' import { useSocialFeedEnrichment } from '../hooks/useSocialFeedEnrichment' -import { - getDefaultSocialWorkspaceSeeds, - upsertDefaultSocialWorkspace, - type SocialWorkspaceSeedSummary -} from '../lib/social-workspace' import { ModeratedMedia } from './ModeratedMedia' /** @@ -68,444 +19,21 @@ const gateVisualItem = (nodeId: string, content: ReactNode): ReactNode => ( {content} ) -type SavedViewRow = { - id: string - title?: string - description?: string - descriptor?: string - scope?: string -} - -type ParsedDescriptor = { - valid: boolean - queryKind: string - queryMode: string | null - primarySchemaId: string | null -} - -type WorkspaceMetric = { - id: string - label: string - value: number | null - icon: typeof UserRound -} - -type GraphAtlasRow = { - entry: SocialGraphAtlasEntry - savedView: SavedViewRow | null -} - -const SOCIAL_SCHEMA_REGISTRY = socialSchemas as unknown as SavedViewSchemaRegistry -const PATTERN_QUERY_LIMIT = 300 -const DISMISSED_PATTERN_STORAGE_KEY = 'xnet:data-workspace:dismissed-patterns' - -function getCount(input: { totalCount: number | null; data: unknown[] }): number | null { - return input.totalCount ?? (input.data.length > 0 ? input.data.length : null) -} - -function parseSavedViewDescriptor(value: string | undefined): ParsedDescriptor { - if (!value) { - return { - valid: false, - queryKind: 'unknown', - queryMode: null, - primarySchemaId: null - } - } - - try { - const descriptor = JSON.parse(value) as SavedViewDescriptor - const validation = validateSavedViewDescriptor(descriptor) - const query = descriptor.query as Record - const queryKind = typeof query.kind === 'string' ? query.kind : 'unknown' - const queryMode = typeof query.mode === 'string' ? query.mode : null - const primarySchemaId = - queryKind === 'query-set' ? primarySchemaIdForQuerySet(query) : primarySchemaIdForQuery(query) - - return { - valid: validation.valid, - queryKind, - queryMode, - primarySchemaId - } - } catch { - return { - valid: false, - queryKind: 'invalid-json', - queryMode: null, - primarySchemaId: null - } - } -} - -function parseSavedViewDescriptorObject(value: string | undefined): SavedViewDescriptor | null { - if (!value) return null - - try { - const descriptor = JSON.parse(value) as SavedViewDescriptor - return validateSavedViewDescriptor(descriptor).valid ? descriptor : null - } catch { - return null - } -} - -function primarySchemaIdForQuery(query: Record): string | null { - const schema = query.schema as Record | undefined - return typeof schema?.id === 'string' - ? schema.id - : typeof schema?.['@id'] === 'string' - ? schema['@id'] - : typeof query.schemaId === 'string' - ? query.schemaId - : null -} - -function primarySchemaIdForQuerySet(query: Record): string | null { - const queries = query.queries as Record> | undefined - const firstQuery = queries ? Object.values(queries)[0] : null - return firstQuery ? primarySchemaIdForQuery(firstQuery) : null -} - -function metricValueLabel(value: number | null): string { - return value === null ? '-' : value.toLocaleString() -} - -function sumKnownCounts(values: readonly (number | null)[]): number { - return values.reduce((total, value) => total + (value ?? 0), 0) -} - -function descriptorKindLabel(descriptor: ParsedDescriptor): string { - if (!descriptor.valid) return 'Invalid' - if (descriptor.queryKind === 'query-set') return descriptor.queryMode ?? 'query set' - return descriptor.queryKind -} - -function isVisibleSocialImportJob(job: SocialImportJobProgress): boolean { - if (job.status !== 'completed') return true - return Date.now() - job.updatedAt < 5 * 60 * 1000 -} - -function socialImportJobPercent(job: SocialImportJobProgress): number { - if (!job.totalRecords || job.totalRecords <= 0) return job.status === 'completed' ? 100 : 0 - return Math.min(100, Math.max(0, (job.processedRecords / job.totalRecords) * 100)) -} - -function socialImportJobStatusLabel(job: SocialImportJobProgress): string { - if (job.status === 'queued') return 'Queued' - if (job.status === 'running') return 'Running' - if (job.status === 'paused') return 'Paused' - if (job.status === 'completed') return 'Complete' - if (job.status === 'failed') return 'Failed' - return 'Cancelled' -} - -function socialImportJobRecordLabel(job: SocialImportJobProgress): string { - if (!job.totalRecords) return job.processedRecords.toLocaleString() - return `${job.processedRecords.toLocaleString()} / ${job.totalRecords.toLocaleString()}` -} - -function socialImportJobRateLabel(job: SocialImportJobProgress): string { - const recordsPerSecond = job.metrics?.recordsPerSecond ?? 0 - if (!Number.isFinite(recordsPerSecond) || recordsPerSecond <= 0) return '0/s' - return `${Math.round(recordsPerSecond).toLocaleString()}/s` -} - -function readDismissedPatternIds(): string[] { - if (typeof localStorage === 'undefined') return [] - - try { - const value = JSON.parse(localStorage.getItem(DISMISSED_PATTERN_STORAGE_KEY) ?? '[]') - return Array.isArray(value) - ? value.flatMap((item) => (typeof item === 'string' ? [item] : [])) - : [] - } catch { - return [] - } -} - -function writeDismissedPatternIds(ids: readonly string[]): void { - if (typeof localStorage === 'undefined') return - - localStorage.setItem(DISMISSED_PATTERN_STORAGE_KEY, JSON.stringify([...new Set(ids)].sort())) -} - -function toPatternRows(rows: readonly unknown[]): Record[] { - return rows as unknown as Record[] -} - -function patternIconFor(kind: SocialPatternKind): typeof BarChart3 { - if (kind === 'privacy-hotspots') return Shield - if (kind === 'cross-source-overlap') return Search - if (kind === 'bridge-actors') return Network - if (kind === 'unrevisited-saves') return Import - if (kind === 'attention-bursts') return BarChart3 - return BarChart3 -} - export function DataWorkspaceView(): JSX.Element { - const { create, mutate } = useMutate() const { store, isReady: storeReady } = useNodeStore() const feedEnrichment = useSocialFeedEnrichment() - const [socialImportJobs, setSocialImportJobs] = - useState(listSocialImportJobs) - const [seedSummary, setSeedSummary] = useState(null) - const [seeding, setSeeding] = useState(false) - const [seedError, setSeedError] = useState(null) - const [saveLensMessage, setSaveLensMessage] = useState(null) - const [saveLensError, setSaveLensError] = useState(null) - const [selectedViewId, setSelectedViewId] = useState(null) - const [dismissedPatternIds, setDismissedPatternIds] = useState(readDismissedPatternIds) - const { data: savedViews, loading: savedViewsLoading } = useQuery(SavedViewSchema, { - orderBy: { title: 'asc' }, - limit: 200 - }) - const actorQuery = useQuery(SocialActorSchema, { page: { first: 1, count: 'estimate' } }) - const contentQuery = useQuery(SocialContentSchema, { - page: { first: PATTERN_QUERY_LIMIT, count: 'estimate' }, - orderBy: { importedAt: 'desc' } - }) - const interactionQuery = useQuery(SocialInteractionSchema, { - page: { first: PATTERN_QUERY_LIMIT, count: 'estimate' }, - orderBy: { importedAt: 'desc' } - }) - const messageQuery = useQuery(SocialMessageSchema, { page: { first: 1, count: 'estimate' } }) - const conversationQuery = useQuery(SocialConversationSchema, { - page: { first: 1, count: 'estimate' } - }) - const collectionQuery = useQuery(SocialCollectionSchema, { - page: { first: 1, count: 'estimate' } - }) - const importRunQuery = useQuery(SocialImportRunSchema, { - page: { first: 50, count: 'estimate' }, - orderBy: { startedAt: 'desc' } - }) - const defaultSeeds = useMemo(() => getDefaultSocialWorkspaceSeeds(), []) - const defaultSeedIds = useMemo( - () => new Set(defaultSeeds.map((seed) => seed.deterministicId)), - [defaultSeeds] - ) - const defaultSeedBySourceId = useMemo( - () => new Map(defaultSeeds.map((seed) => [seed.id, seed])), - [defaultSeeds] - ) - const graphAtlasEntries = useMemo(() => createDefaultSocialGraphAtlas({ pageSize: 100 }), []) - const socialWorkspaceViews = useMemo( - () => (savedViews as SavedViewRow[]).filter((view) => defaultSeedIds.has(view.id)), - [defaultSeedIds, savedViews] - ) - const otherSavedViews = useMemo( - () => (savedViews as SavedViewRow[]).filter((view) => !defaultSeedIds.has(view.id)), - [defaultSeedIds, savedViews] - ) - const allSavedViews = useMemo( - () => [...socialWorkspaceViews, ...otherSavedViews], - [otherSavedViews, socialWorkspaceViews] - ) - const selectedView = useMemo( - () => - allSavedViews.find((view) => view.id === selectedViewId) ?? - socialWorkspaceViews[0] ?? - allSavedViews[0] ?? - null, - [allSavedViews, selectedViewId, socialWorkspaceViews] - ) - const metrics: WorkspaceMetric[] = [ - { - id: 'actors', - label: 'People', - value: getCount(actorQuery), - icon: UserRound - }, - { - id: 'content', - label: 'Content', - value: getCount(contentQuery), - icon: Table - }, - { - id: 'interactions', - label: 'Interactions', - value: getCount(interactionQuery), - icon: Network - }, - { - id: 'messages', - label: 'Messages', - value: getCount(messageQuery), - icon: MessageSquare - }, - { - id: 'conversations', - label: 'Conversations', - value: getCount(conversationQuery), - icon: GitBranch - }, - { - id: 'collections', - label: 'Collections', - value: getCount(collectionQuery), - icon: Database - }, - { - id: 'import-runs', - label: 'Import Runs', - value: getCount(importRunQuery), - icon: Import - } - ] - const analyticsCacheRecommendation = recommendSocialAnalyticsCache({ - rowCount: sumKnownCounts(metrics.map((metric) => metric.value)), - columnCount: 12, - relationCount: getCount(interactionQuery) ?? 0 + const workspace = useDataWorkspace({ + seedReady: Boolean(store && storeReady), + getExistingNode: (id) => (store ? Promise.resolve(store.get(id)) : Promise.resolve(undefined)) }) - const dismissedPatternIdSet = useMemo(() => new Set(dismissedPatternIds), [dismissedPatternIds]) - const patternSuggestions = useMemo( - () => - detectSocialPatterns({ - content: toPatternRows(contentQuery.data), - interactions: toPatternRows(interactionQuery.data), - importRuns: toPatternRows(importRunQuery.data) - }).filter((pattern) => !dismissedPatternIdSet.has(pattern.id)), - [contentQuery.data, dismissedPatternIdSet, importRunQuery.data, interactionQuery.data] - ) - const graphAtlasRows = useMemo( - () => - graphAtlasEntries.map((entry) => { - const seed = defaultSeedBySourceId.get(entry.id) - const savedView = seed - ? (socialWorkspaceViews.find((view) => view.id === seed.deterministicId) ?? null) - : null + const { seeding, seedReady, handleSeedWorkspace } = workspace - return { entry, savedView } - }), - [defaultSeedBySourceId, graphAtlasEntries, socialWorkspaceViews] - ) - const visibleSocialImportJobs = useMemo( - () => socialImportJobs.filter(isVisibleSocialImportJob).slice(0, 3), - [socialImportJobs] + const savedViewRunnerProps = useMemo( + () => ({ feedEnrichment, wrapItem: gateVisualItem }), + [feedEnrichment] ) - useEffect(() => { - if (!selectedViewId && selectedView) { - setSelectedViewId(selectedView.id) - return - } - - if (selectedViewId && !allSavedViews.some((view) => view.id === selectedViewId)) { - setSelectedViewId(selectedView?.id ?? null) - } - }, [allSavedViews, selectedView, selectedViewId]) - - useEffect(() => subscribeSocialImportJobs(() => setSocialImportJobs(listSocialImportJobs())), []) - - async function handleSeedWorkspace() { - if (!store || !storeReady) return - - setSeeding(true) - setSeedError(null) - - try { - const summary = await upsertDefaultSocialWorkspace({ - mutate, - getExisting: (id) => store.get(id) - }) - setSeedSummary(summary) - } catch (error) { - setSeedError(error instanceof Error ? error.message : String(error)) - } finally { - setSeeding(false) - } - } - - async function handleSaveLens(draft: SavedViewLensDraft): Promise { - setSaveLensMessage(null) - setSaveLensError(null) - - try { - const savedView = await create(SavedViewSchema, { - title: draft.title, - description: draft.description, - descriptor: JSON.stringify(draft.descriptor), - scope: draft.descriptor.scope ?? 'workspace' - }) - - if (!savedView) { - throw new Error('Saved lens could not be created.') - } - - setSelectedViewId(savedView.id) - setSaveLensMessage(`Saved lens: ${draft.title}.`) - } catch (error) { - setSaveLensError(error instanceof Error ? error.message : String(error)) - throw error - } - } - - function handleOpenPattern(pattern: SocialPatternSuggestion): void { - const view = socialWorkspaceViews.find((candidate) => candidate.title === pattern.viewHint) - if (view) { - setSelectedViewId(view.id) - } - } - - async function upsertPatternSavedView( - pattern: SocialPatternSuggestion - ): Promise { - setSaveLensMessage(null) - setSaveLensError(null) - - const baseView = socialWorkspaceViews.find((candidate) => candidate.title === pattern.viewHint) - const baseDescriptor = parseSavedViewDescriptorObject(baseView?.descriptor) - - if (!baseView || !baseDescriptor) { - setSaveLensError(`Seed the ${pattern.viewHint} view before saving this pattern.`) - return null - } - - const draft = createSocialPatternSavedViewDraft({ pattern, baseDescriptor }) - if (!draft) { - setSaveLensError('Pattern lens could not be created from the base view.') - return null - } - - const existing = allSavedViews.some((view) => view.id === draft.deterministicId) - const operation: MutateOp = existing - ? { - type: 'update', - id: draft.deterministicId, - data: draft.savedViewProperties - } - : { - type: 'create', - id: draft.deterministicId, - schema: SavedViewSchema, - data: draft.savedViewProperties - } - - await mutate([operation]) - - const savedView = { - id: draft.deterministicId, - ...draft.savedViewProperties - } - setSelectedViewId(savedView.id) - setSaveLensMessage(`${existing ? 'Updated' : 'Saved'} pattern lens: ${draft.title}.`) - return savedView - } - - async function handleSavePattern(pattern: SocialPatternSuggestion): Promise { - await upsertPatternSavedView(pattern) - } - - function handleDismissPattern(patternId: string): void { - setDismissedPatternIds((current) => { - const next = [...new Set([...current, patternId])] - writeDismissedPatternIds(next) - return next - }) - } - return (
@@ -521,7 +49,7 @@ export function DataWorkspaceView(): JSX.Element {
- {seedSummary ? ( - - ) : null} - {seedError ? : null} - {saveLensMessage ? : null} - {saveLensError ? : null} - - -
- {metrics.map((metric) => { - const Icon = metric.icon - - return ( -
-
- {metric.label} - -
-
{metricValueLabel(metric.value)}
-
- ) - })} -
- - setSelectedViewId(view.id)} - /> - -
- - -
-
-
-
-

Social Starter Lenses

-

- Schema views and graph-lens query sets persisted as saved views. -

-
- {savedViewsLoading ? ( -
- - Loading -
- ) : null} -
- -
- - - -
-
-

Other Saved Views

-

- General saved views will use the same workspace surface as more importers land. -

-
- -
-
-
-
- ) -} - -function SocialImportJobsPanel({ jobs }: { jobs: SocialImportJobProgress[] }): JSX.Element | null { - if (jobs.length === 0) return null - - return ( -
- -
- {jobs.map((job) => { - const percent = socialImportJobPercent(job) - const statusLabel = socialImportJobStatusLabel(job) - - return ( -
-
-
-
- {job.status === 'running' || job.status === 'queued' ? ( - - ) : ( - - )} -
{job.archiveName}
-
-
- {job.platform} / {statusLabel} / {job.phase} -
-
-
- {Math.floor(percent)}% -
-
-
-
-
-
- - - - -
- {job.error ? ( -
- - {job.error} -
- ) : null} -
- ) - })} -
-
- ) -} - -function JobMetric({ label, value }: { label: string; value: string }): JSX.Element { - return ( -
-
{label}
-
{value}
-
- ) -} - -function SavedViewTable({ - views, - selectedViewId, - emptyLabel, - onSelect -}: { - views: SavedViewRow[] - selectedViewId: string | null - emptyLabel: string - onSelect: (viewId: string) => void -}): JSX.Element { - if (views.length === 0) { - return ( -
- {emptyLabel} -
- ) - } - - return ( -
- - - - - - - - - - - {views.map((view) => { - const descriptor = parseSavedViewDescriptor(view.descriptor) - const selected = view.id === selectedViewId - - return ( - - - - - - - ) - })} - -
ViewKindScopeSchema
- - - {descriptorKindLabel(descriptor)} - {view.scope ?? '-'} - {descriptor.primarySchemaId ?? '-'} -
-
- ) -} - -function GraphAtlasPanel({ - rows, - selectedViewId, - onOpen -}: { - rows: GraphAtlasRow[] - selectedViewId: string | null - onOpen: (view: SavedViewRow) => void -}): JSX.Element { - return ( -
-
-
-

Graph Atlas

-

- Starter graph lenses organized by node roles, relationship rules, and saved-view state. -

-
- - {rows.filter((row) => row.savedView).length}/{rows.length} seeded - -
-
- {rows.map((row) => ( - - ))} -
-
- ) -} - -function GraphAtlasCard({ - row, - selected, - onOpen -}: { - row: GraphAtlasRow - selected: boolean - onOpen: (view: SavedViewRow) => void -}): JSX.Element { - const { entry, savedView } = row - - return ( -
-
-
-
- -

{entry.title}

-
-

{entry.description}

-
- - {savedView ? 'saved' : 'seed'} - -
-
- - - -
-
- {entry.nodeRoles.slice(0, 3).map((role) => ( - - {role.role} - - ))} - {entry.relationshipKinds.slice(0, 3).map((kind) => ( - - {kind} - - ))} -
-
- -
-
- ) -} - -function GraphAtlasMetric({ label, value }: { label: string; value: number }): JSX.Element { - return ( -
-
{label}
-
{value.toLocaleString()}
-
- ) -} - -function SectionLabel({ label }: { label: string }): JSX.Element { - return ( -
{label}
- ) -} - -function SourceRow({ label, value }: { label: string; value: string }): JSX.Element { - return ( -
- {label} - {value} -
- ) -} - -function AnalyticsCacheRow({ - recommendation -}: { - recommendation: SocialAnalyticsCacheRecommendation -}): JSX.Element { - return ( -
-
- Scale cache - {recommendation.label} -
-

{recommendation.reason}

-
- - {recommendation.estimatedRows.toLocaleString()} rows - - - {recommendation.estimatedCells.toLocaleString()} cells - -
-
- ) -} - -function PatternRow({ - icon: Icon, - pattern, - onOpen, - onSave, - onDismiss -}: { - icon: typeof BarChart3 - pattern: SocialPatternSuggestion - onOpen: (pattern: SocialPatternSuggestion) => void - onSave: (pattern: SocialPatternSuggestion) => void - onDismiss: (patternId: string) => void -}): JSX.Element { - return ( -
-
- -
-
{pattern.title}
-
- {pattern.description} -
-
-
-
- - {pattern.evidenceCount.toLocaleString()} evidence - - {pattern.platforms.slice(0, 2).map((platform) => ( - - {platform} - - ))} - {pattern.privacyClasses.slice(0, 2).map((privacyClass) => ( - - {privacyClass} - - ))} - {pattern.sourceImportRunIds.length > 0 ? ( - - {pattern.sourceImportRunIds.length} runs - - ) : null} -
- {pattern.evidence.length > 0 ? ( -
- {pattern.evidence.slice(0, 2).map((item) => ( -
- {item.value} - {item.count.toLocaleString()} -
- ))} -
- ) : null} -
- - - -
-
- ) -} - -function StatusBanner({ - message, - tone -}: { - message: string - tone: 'error' | 'success' | 'warning' -}): JSX.Element { - const toneClassName = { - error: 'border-destructive/40 bg-destructive/10 text-destructive', - success: 'border-emerald-500/30 bg-emerald-500/10 text-emerald-700 dark:text-emerald-300', - warning: 'border-amber-500/30 bg-amber-500/10 text-amber-700 dark:text-amber-300' - }[tone] - - const Icon = tone === 'success' ? Shield : AlertTriangle - - return ( -
- - {message} +
) } diff --git a/apps/web/src/components/PageView.tsx b/apps/web/src/components/PageView.tsx index 2ba267c8e..d605c594c 100644 --- a/apps/web/src/components/PageView.tsx +++ b/apps/web/src/components/PageView.tsx @@ -11,13 +11,19 @@ * Features: * - Collaborative editing via Yjs * - Comment system with inline popover and sidebar + * (state machine shared with desktop via usePageComments, 0276) * - Real-time presence indicators */ import { useNavigate } from '@tanstack/react-router' import { PageSchema } from '@xnetjs/data' -import { CommentMark, CommentPlugin, restoreCommentMarks } from '@xnetjs/editor/extensions' -import { buildPersonMentionSuggestions, type Editor } from '@xnetjs/editor/react' -import { useNode, useComments, useIdentity, usePageTaskSync } from '@xnetjs/react' +import { + buildPersonMentionSuggestions, + usePageComments, + type Editor, + type PageCommentPopoverState, + type PageNewCommentState +} from '@xnetjs/editor/react' +import { useNode, useIdentity, usePageTaskSync } from '@xnetjs/react' import { CommentPopover, CommentsSidebar, @@ -26,7 +32,6 @@ import { getNodeTransfer, hasNodeTransfer, type CommentThreadData, - type OrphanedThread, type TaskPersonOption } from '@xnetjs/ui' import { MessageSquare } from 'lucide-react' @@ -49,30 +54,6 @@ import { PageTasksSection } from './PageTasksSection' import { PresenceAvatars } from './PresenceAvatars' import { ShareButton } from './ShareButton' -// ─── Comment Popover State ────────────────────────────────────────────────────── - -interface PopoverState { - visible: boolean - mode: 'preview' | 'full' - threadId: string | null - anchor: HTMLElement | null -} - -const INITIAL_POPOVER_STATE: PopoverState = { - visible: false, - mode: 'preview', - threadId: null, - anchor: null -} - -/** State for creating a new comment (before submission) */ -interface NewCommentState { - visible: boolean - anchorData: string - selectionFrom: number - selectionTo: number -} - /** Render the loading / error placeholders, or null when ready. */ function pageLoadPlaceholder( loading: boolean, @@ -135,14 +116,6 @@ function pageSyncStatusItem( } } -function lookupThread( - threadDataMap: Map, - threadId: string | null -): CommentThreadData | null { - if (!threadId) return null - return threadDataMap.get(threadId) ?? null -} - /** Place the caret at the document position nearest to a margin click. */ function focusEditorNear(editor: Editor, clientX: number, clientY: number): void { const rect = editor.view.dom.getBoundingClientRect() @@ -221,415 +194,53 @@ export function PageView({ docId }: { docId: string }) { [setNodeTags, docId] ) - // The right-panel task editor writes assignee/due-date edits through - // this live editor (the document owns those fields while it hosts the - // task — see PAGE_TASK_RECONCILIATION.md). - const taskHostEditor = useMemo( - () => ({ getEditor: () => editorRef.current, suggestions: mentionSuggestions }), - [mentionSuggestions] - ) - - // ─── Comments Integration ───────────────────────────────────────────────────── + // ─── Comments Integration (shared state machine, 0276) ─────────────────────── const { - threads, - addComment, - replyTo, - resolveThread, - reopenThread, - deleteComment, - editComment, - unresolvedCount - } = useComments({ nodeId: docId, anchorType: 'text' }) + unresolvedCount, + threadDataMap, + sidebarThreads, + currentThread, + orphanedThreads, + orphanedCollapsed, + toggleOrphanedCollapsed, + popoverState, + newCommentState, + editorRef, + handleEditorReady, + commentExtensions, + handlePopoverMouseEnter, + handlePopoverMouseLeave, + handleDismiss, + handleUpgradeToFull, + handleReply, + handleResolve, + handleReopen, + handleDelete, + handleEdit, + handleCreateComment, + handleSubmitNewComment, + handleCancelNewComment, + handleSidebarSelectThread, + handleSidebarReply, + handleSidebarResolve, + handleSidebarReopen, + handleSidebarDelete, + handleSidebarEdit, + handleDismissOrphaned, + handleReattachOrphaned + } = usePageComments({ docId }) - // Popover state for comment interactions - const [popoverState, setPopoverState] = useState(INITIAL_POPOVER_STATE) - const [newCommentState, setNewCommentState] = useState(null) - const [orphanedIds, setOrphanedIds] = useState([]) - const [orphanedCollapsed, setOrphanedCollapsed] = useState(false) - const hoverTimeoutRef = useRef | null>(null) - const dismissTimeoutRef = useRef | null>(null) - const editorRef = useRef(null) const titleInputRef = useRef(null) - const marksRestoredRef = useRef(false) - const [editorReady, setEditorReady] = useState(false) - - // Track hover state for mark and popover - const markHoveredRef = useRef(false) - const popoverHoveredRef = useRef(false) - - // Reset mark restoration state when switching documents. Skip the - // initial run: parent effects fire after the editor's ready - // notification, so an unconditional reset would null the ref the - // moment it was set. - const lastDocIdRef = useRef(docId) - useEffect(() => { - if (lastDocIdRef.current === docId) return - lastDocIdRef.current = docId - marksRestoredRef.current = false - editorRef.current = null - setEditorReady(false) - }, [docId]) - - // Handle editor ready - store ref and trigger mark restoration - const handleEditorReady = useCallback((editor: Editor) => { - editorRef.current = editor - setEditorReady(true) - }, []) - - // Restore comment marks when editor is ready and threads are loaded - useEffect(() => { - if (!editorRef.current || marksRestoredRef.current || threads.length === 0) return - - const commentsToRestore = threads.map((t) => ({ - id: t.root.id, - properties: { - anchorType: t.root.properties.anchorType, - anchorData: t.root.properties.anchorData, - resolved: t.root.properties.resolved - } - })) - - const { resolved, orphaned } = restoreCommentMarks(editorRef.current, commentsToRestore) - - if (resolved.length > 0 || orphaned.length > 0) { - marksRestoredRef.current = true - setOrphanedIds(orphaned) - console.log(`[Comments] Restored ${resolved.length} marks, ${orphaned.length} orphaned`) - } - }, [threads, editorReady]) - - // Build orphaned threads list for display - const orphanedThreads = useMemo((): OrphanedThread[] => { - const result: OrphanedThread[] = [] - - for (const id of orphanedIds) { - const thread = threads.find((t) => t.root.id === id) - if (!thread) continue - - // Parse anchor data to get context - let context: string | undefined - try { - const anchor = JSON.parse(thread.root.properties.anchorData) - context = anchor.quotedText - } catch { - // Ignore parse errors - } - - result.push({ - comment: { - id: thread.root.id, - author: thread.root.properties.createdBy, - authorDisplayName: undefined, - content: thread.root.properties.content, - createdAt: thread.root.createdAt, - replyCount: thread.replies.length - }, - reason: 'text-deleted', - context - }) - } - - return result - }, [orphanedIds, threads]) - - // Convert threads to format expected by CommentPopover/CommentsSidebar - const threadDataMap = useMemo(() => { - const map = new Map() - for (const thread of threads) { - map.set(thread.root.id, { - root: { - id: thread.root.id, - author: thread.root.properties.createdBy, - authorDisplayName: undefined, - content: thread.root.properties.content, - createdAt: thread.root.createdAt, - edited: thread.root.properties.edited, - editedAt: thread.root.properties.editedAt, - replyToUser: thread.root.properties.replyToUser, - replyToCommentId: thread.root.properties.replyToCommentId - }, - replies: thread.replies.map((r) => ({ - id: r.id, - author: r.properties.createdBy, - authorDisplayName: undefined, - content: r.properties.content, - createdAt: r.createdAt, - edited: r.properties.edited, - editedAt: r.properties.editedAt, - replyToUser: r.properties.replyToUser, - replyToCommentId: r.properties.replyToCommentId - })), - resolved: thread.root.properties.resolved - }) - } - return map - }, [threads]) - - // ─── Popover Handlers ───────────────────────────────────────────────────────── - - const isCaretInComment = useCallback((): boolean => { - const editor = editorRef.current - if (!editor) return false - const { from } = editor.state.selection - const resolved = editor.state.doc.resolve(from) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return resolved.marks().some((m: any) => m.type.name === 'comment') - }, []) - - const scheduleDismiss = useCallback(() => { - if (dismissTimeoutRef.current) clearTimeout(dismissTimeoutRef.current) - dismissTimeoutRef.current = setTimeout(() => { - if (!markHoveredRef.current && !popoverHoveredRef.current && !isCaretInComment()) { - setPopoverState(INITIAL_POPOVER_STATE) - } - }, 200) - }, [isCaretInComment]) - - const handleClickComment = useCallback((commentId: string, anchorEl: HTMLElement) => { - if (hoverTimeoutRef.current) clearTimeout(hoverTimeoutRef.current) - if (dismissTimeoutRef.current) clearTimeout(dismissTimeoutRef.current) - setPopoverState((prev) => { - if (prev.visible && prev.mode === 'full' && prev.threadId === commentId) return prev - return { visible: true, mode: 'full', threadId: commentId, anchor: anchorEl } - }) - }, []) - - const handleHoverComment = useCallback((commentId: string, anchorEl: HTMLElement) => { - markHoveredRef.current = true - if (dismissTimeoutRef.current) clearTimeout(dismissTimeoutRef.current) - if (hoverTimeoutRef.current) clearTimeout(hoverTimeoutRef.current) - hoverTimeoutRef.current = setTimeout(() => { - setPopoverState((prev) => { - if (prev.visible && prev.threadId === commentId) return prev - return { visible: true, mode: 'full', threadId: commentId, anchor: anchorEl } - }) - }, 300) - }, []) - - const handleLeaveComment = useCallback(() => { - markHoveredRef.current = false - if (hoverTimeoutRef.current) clearTimeout(hoverTimeoutRef.current) - scheduleDismiss() - }, [scheduleDismiss]) - - const handlePopoverMouseEnter = useCallback(() => { - popoverHoveredRef.current = true - if (dismissTimeoutRef.current) clearTimeout(dismissTimeoutRef.current) - }, []) - - const handlePopoverMouseLeave = useCallback(() => { - popoverHoveredRef.current = false - scheduleDismiss() - }, [scheduleDismiss]) - - const handleDismiss = useCallback(() => { - if (hoverTimeoutRef.current) clearTimeout(hoverTimeoutRef.current) - if (dismissTimeoutRef.current) clearTimeout(dismissTimeoutRef.current) - markHoveredRef.current = false - popoverHoveredRef.current = false - setPopoverState(INITIAL_POPOVER_STATE) - }, []) - - const handleUpgradeToFull = useCallback(() => { - setPopoverState((prev) => ({ ...prev, mode: 'full' })) - }, []) - - // ─── Comment Actions ────────────────────────────────────────────────────────── - - const handleReply = useCallback( - async (content: string) => { - if (!popoverState.threadId) return - await replyTo(popoverState.threadId, content) - }, - [popoverState.threadId, replyTo] - ) - - const handleResolve = useCallback(async () => { - if (!popoverState.threadId) return - await resolveThread(popoverState.threadId) - editorRef.current?.commands.setCommentResolved(popoverState.threadId, true) - }, [popoverState.threadId, resolveThread]) - - const handleReopen = useCallback(async () => { - if (!popoverState.threadId) return - await reopenThread(popoverState.threadId) - editorRef.current?.commands.setCommentResolved(popoverState.threadId, false) - }, [popoverState.threadId, reopenThread]) - - const handleDelete = useCallback( - async (commentId: string) => { - await deleteComment(commentId) - const thread = threadDataMap.get(popoverState.threadId || '') - if (thread && commentId === thread.root.id && thread.replies.length === 0) { - const editor = editorRef.current - if (editor) { - const { tr, doc: editorDoc } = editor.state - const markType = editor.schema.marks.comment - if (markType) { - editorDoc.descendants((node, pos) => { - node.marks.forEach((mark) => { - if (mark.type === markType && mark.attrs.commentId === commentId) { - tr.removeMark(pos, pos + node.nodeSize, mark) - } - }) - }) - editor.view.dispatch(tr) - } - } - handleDismiss() - } - }, - [deleteComment, threadDataMap, popoverState.threadId, handleDismiss] - ) - - const handleEdit = useCallback( - async (commentId: string, newContent: string) => { - await editComment(commentId, newContent) - }, - [editComment] - ) - - // Handler for initiating comment creation from toolbar - const handleCreateComment = useCallback(async (anchorData: string): Promise => { - if (!editorRef.current) return null - const { from, to } = editorRef.current.state.selection - if (from === to) return null - - setNewCommentState({ - visible: true, - anchorData, - selectionFrom: from, - selectionTo: to - }) - return null - }, []) - - // Handler for submitting a new comment - const handleSubmitNewComment = useCallback( - async (content: string) => { - if (!newCommentState || !content.trim() || !editorRef.current) return - - const commentId = await addComment({ - content: content.trim(), - anchorType: 'text', - anchorData: newCommentState.anchorData, - targetSchema: PageSchema.schema['@id'] - }) - - if (commentId) { - editorRef.current - .chain() - .focus() - .setTextSelection({ - from: newCommentState.selectionFrom, - to: newCommentState.selectionTo - }) - .setComment(commentId) - .run() - - const showPopover = () => { - const markEl = document.querySelector( - `[data-comment-id="${commentId}"]` - ) as HTMLElement | null - if (markEl) { - setPopoverState({ - visible: true, - mode: 'full', - threadId: commentId, - anchor: markEl - }) - } - } - setTimeout(showPopover, 50) - setTimeout(showPopover, 200) - } - - setNewCommentState(null) - }, - [newCommentState, addComment] - ) - - const handleCancelNewComment = useCallback(() => { - setNewCommentState(null) - }, []) - - // ─── Sidebar Handlers ───────────────────────────────────────────────────────── - - const handleSidebarSelectThread = useCallback((threadId: string) => { - const markEl = document.querySelector(`[data-comment-id="${threadId}"]`) as HTMLElement | null - if (markEl) { - markEl.scrollIntoView({ behavior: 'smooth', block: 'center' }) - setPopoverState({ - visible: true, - mode: 'full', - threadId, - anchor: markEl - }) - } - }, []) - - const handleSidebarReply = useCallback( - async (threadId: string, content: string) => { - await replyTo(threadId, content) - }, - [replyTo] - ) - - const handleSidebarResolve = useCallback( - async (threadId: string) => { - await resolveThread(threadId) - editorRef.current?.commands.setCommentResolved(threadId, true) - }, - [resolveThread] - ) - - const handleSidebarReopen = useCallback( - async (threadId: string) => { - await reopenThread(threadId) - editorRef.current?.commands.setCommentResolved(threadId, false) - }, - [reopenThread] - ) - - const handleSidebarDelete = useCallback( - async (commentId: string) => { - await deleteComment(commentId) - }, - [deleteComment] - ) - - const handleSidebarEdit = useCallback( - async (commentId: string, newContent: string) => { - await editComment(commentId, newContent) - }, - [editComment] - ) - - // ─── Orphaned Comment Handlers ───────────────────────────────────────────────── - const handleDismissOrphaned = useCallback( - async (commentId: string) => { - // Delete the orphaned thread entirely - const thread = threads.find((t) => t.root.id === commentId) - if (thread) { - // Delete replies first, then root - for (const reply of thread.replies) { - await deleteComment(reply.id) - } - await deleteComment(commentId) - } - // Remove from orphaned list - setOrphanedIds((prev) => prev.filter((id) => id !== commentId)) - }, - [threads, deleteComment] + // The right-panel task editor writes assignee/due-date edits through + // this live editor (the document owns those fields while it hosts the + // task — see PAGE_TASK_RECONCILIATION.md). + const taskHostEditor = useMemo( + () => ({ getEditor: () => editorRef.current, suggestions: mentionSuggestions }), + [editorRef, mentionSuggestions] ) - const handleReattachOrphaned = useCallback((commentId: string) => { - // For now, just log - reattachment requires selecting new text - console.log(`[Comments] Reattach not yet implemented for ${commentId}`) - }, []) - const handleSelectOrphaned = useCallback( (commentId: string) => { // Open the right panel for this orphaned comment @@ -643,24 +254,6 @@ export function PageView({ docId }: { docId: string }) { [threadDataMap] ) - // ─── Comment Extensions ─────────────────────────────────────────────────────── - - const commentExtensions = useMemo( - () => [ - CommentMark, - CommentPlugin.configure({ - onClickComment: handleClickComment, - onHoverComment: handleHoverComment, - onLeaveComment: handleLeaveComment - }) - ], - [handleClickComment, handleHoverComment, handleLeaveComment] - ) - - // Get the current thread for the popover - const currentThread = lookupThread(threadDataMap, popoverState.threadId) - const sidebarThreads = useMemo(() => Array.from(threadDataMap.values()), [threadDataMap]) - // ─── Context Panel Sections (0166) ────────────────────────────────────────── // Everything that is *about* the page — properties, tasks, comments // (including orphaned threads), backlinks — lives in the shared Right @@ -723,7 +316,7 @@ export function PageView({ docId }: { docId: string }) { setOrphanedCollapsed((prev) => !prev)} + onToggleCollapse={toggleOrphanedCollapsed} onDismiss={handleDismissOrphaned} onReattach={handleReattachOrphaned} onSelect={handleSelectOrphaned} @@ -765,6 +358,7 @@ export function PageView({ docId }: { docId: string }) { sidebarThreads, orphanedThreads, orphanedCollapsed, + toggleOrphanedCollapsed, popoverState.threadId, handleDismissOrphaned, handleReattachOrphaned, @@ -784,16 +378,19 @@ export function PageView({ docId }: { docId: string }) { // the body, Backspace at the top of an empty body returns to the // title, and clicks in the page margins place the caret nearby. - const handleTitleKeyDown = useCallback((event: React.KeyboardEvent) => { - if (event.key !== 'Enter' && event.key !== 'ArrowDown') return - event.preventDefault() - const editor = editorRef.current - if (!editor) return - editor.commands.focus('start') - // TipTap defers DOM focus to the next animation frame; focus the - // view directly so the caret moves immediately. - editor.view.focus() - }, []) + const handleTitleKeyDown = useCallback( + (event: React.KeyboardEvent) => { + if (event.key !== 'Enter' && event.key !== 'ArrowDown') return + event.preventDefault() + const editor = editorRef.current + if (!editor) return + editor.commands.focus('start') + // TipTap defers DOM focus to the next animation frame; focus the + // view directly so the caret moves immediately. + editor.view.focus() + }, + [editorRef] + ) const handleBackspaceAtStart = useCallback(() => { const input = titleInputRef.current @@ -804,14 +401,17 @@ export function PageView({ docId }: { docId: string }) { return true }, []) - const handleMarginMouseDown = useCallback((event: React.MouseEvent) => { - const target = event.target as HTMLElement - if (!target.hasAttribute('data-page-margin')) return - const editor = editorRef.current - if (!editor) return - event.preventDefault() - focusEditorNear(editor, event.clientX, event.clientY) - }, []) + const handleMarginMouseDown = useCallback( + (event: React.MouseEvent) => { + const target = event.target as HTMLElement + if (!target.hasAttribute('data-page-margin')) return + const editor = editorRef.current + if (!editor) return + event.preventDefault() + focusEditorNear(editor, event.clientX, event.clientY) + }, + [editorRef] + ) // Handle wikilink navigation. Reference chips created by node drops // encode non-page targets as xnet:/// (0166). @@ -917,7 +517,7 @@ export function PageView({ docId }: { docId: string }) { Promise @@ -1023,7 +623,7 @@ function PageNewCommentOverlay({ onSubmit, onCancel }: { - state: NewCommentState | null + state: PageNewCommentState | null people: TaskPersonOption[] onSubmit: (content: string) => void onCancel: () => void diff --git a/apps/web/src/lib/social-workspace.ts b/apps/web/src/lib/social-workspace.ts index d87d63a6e..79739c429 100644 --- a/apps/web/src/lib/social-workspace.ts +++ b/apps/web/src/lib/social-workspace.ts @@ -1,59 +1,9 @@ -import type { MutateOp } from '@xnetjs/react' -import { SavedViewSchema } from '@xnetjs/data' -import { createDefaultSocialWorkspaceSavedViewSeeds } from '@xnetjs/social/workspace' - -export type SocialWorkspaceSeedSummary = { - created: number - updated: number - total: number -} - -type SocialWorkspaceSeedOperationResult = { - action: 'created' | 'updated' - operation: MutateOp -} - -export function getDefaultSocialWorkspaceSeeds() { - return createDefaultSocialWorkspaceSavedViewSeeds({ pageSize: 100 }) -} - -export async function upsertDefaultSocialWorkspace(input: { - mutate: (ops: MutateOp[]) => Promise - getExisting: (id: string) => Promise -}): Promise { - const seeds = getDefaultSocialWorkspaceSeeds() - const operationResults = await Promise.all( - seeds.map(async (seed): Promise => { - const existing = await input.getExisting(seed.deterministicId) - if (existing) { - return { - action: 'updated', - operation: { - type: 'update', - id: seed.deterministicId, - data: seed.savedViewProperties - } - } - } - - return { - action: 'created', - operation: { - type: 'create', - id: seed.deterministicId, - schema: SavedViewSchema, - data: seed.savedViewProperties - } as MutateOp - } - }) - ) - - const operations = operationResults.map((result) => result.operation) - await input.mutate(operations) - - return { - created: operationResults.filter((result) => result.action === 'created').length, - updated: operationResults.filter((result) => result.action === 'updated').length, - total: seeds.length - } -} +/** + * Social workspace seeding — moved into the shared Data Workspace core + * (@xnetjs/views, exploration 0276). Re-exported here for existing app imports. + */ +export { + getDefaultSocialWorkspaceSeeds, + upsertDefaultSocialWorkspace, + type SocialWorkspaceSeedSummary +} from '@xnetjs/views' diff --git a/docs/explorations/0276_[x]_WELL_TRAVELED_CODE_PATHS_CHURN_WEIGHTED_REFACTOR_MAP.md b/docs/explorations/0276_[x]_WELL_TRAVELED_CODE_PATHS_CHURN_WEIGHTED_REFACTOR_MAP.md new file mode 100644 index 000000000..a7802ac1b --- /dev/null +++ b/docs/explorations/0276_[x]_WELL_TRAVELED_CODE_PATHS_CHURN_WEIGHTED_REFACTOR_MAP.md @@ -0,0 +1,638 @@ +# Well-Traveled Code Paths: A Churn-Weighted Refactor Map + +> Status: unimplemented (`[_]`). Numbers measured against the repo at +> `ae4c02fb` (2026-07-06); churn windows are the trailing 8 months of git +> history unless stated otherwise. Companion to exploration +> `0230_[_]_CODEBASE_REFACTORING_ATLAS_DEAD_CODE_GOD_FILES_AND_DEDUPLICATION.md` +> — 0230 ranked opportunities by _lines removable_; this doc ranks them by +> _traffic_: the files every feature PR has to walk through. + +## Problem Statement + +Where are the most impactful refactors in the codebase — specifically in the +code paths that are most well traveled, where cleaning up, simplifying, or +improving legibility pays back on every future change? + +"Impactful" here is not "most lines deleted" (0230 covered that). It is: which +files do we _edit most often_, weighted by how _hard they are to edit_? A +7,700-line file nobody touches is sleeping debt; a 2,700-line file changed 47 +times in 8 months taxes nearly every feature. The classic hotspot literature +(Tornhill's _Your Code as a Crime Scene_, CodeScene) formalizes this as +**churn × complexity**: the small fraction of code that is both complicated and +frequently changed accounts for 25–70% of defects. + +## Executive Summary + +Measuring `commits-touching-file × current LOC` over 8 months produces a clear, +somewhat surprising ranking. The **data layer** — not the canvas — is the +hottest ground in the repo, and it also happens to be the best test-protected +place to refactor. The giant `CanvasV3.tsx` (7,758 LOC) does not even crack the +top-40 churn list; its churn happens one layer up, in the app-level +`CanvasView` wrappers that are duplicated between web and electron. + +| Rank | File | Changes | LOC | Churn×LOC | What makes it hard to edit | +| ---- | ------------------------------------------------------ | ------: | ----: | --------: | ------------------------------------------------------------ | +| 1 | `packages/data/src/store/sqlite-adapter.ts` | 35 | 4,407 | 154k | 80-method god class: 8 subsystems in one file | +| 2 | `apps/electron/src/renderer/components/CanvasView.tsx` | 42 | 3,195 | 134k | drifted 74%-larger fork of the web copy | +| 3 | `packages/data/src/store/store.ts` | 47 | 2,763 | 130k | 40+ public methods; dual fast/slow txn paths | +| 4 | `packages/data/src/index.ts` | 87 | 1,193 | 104k | pure barrel — API-surface ceremony + conflicts | +| 5 | `packages/hub/src/server.ts` | 56 | 1,750 | 98k | 600-line WebSocket if/else message pump | +| 6 | `packages/react/src/index.ts` | 90 | 842 | 76k | pure barrel — highest raw churn in the repo | +| 7 | `packages/hub/src/storage/sqlite.ts` | 26 | 2,539 | 66k | reimplements LWW/hydration from #1 | +| 8 | `packages/react/src/context.ts` | 49 | 1,213 | 59k | `XNetProvider` god-component (84-line useEffect, 5 concerns) | +| 9 | `apps/web/src/App.tsx` | 56 | 1,009 | 57k | boot orchestration + state machine + telemetry interleaved | +| 10 | `apps/electron/src/renderer/components/PageView.tsx` | 44 | 993 | 44k | 93%-identical fork of web's PageView, **0 shared commits** | + +Four themes fall out, in recommended order: + +1. **Data-layer decomposition** (ranks 1, 3, 7) — highest traffic, cleanest + seams, and uniquely well protected: ~205 KB of adapter/store tests plus the + 0272 reliability lane. Extract the query compiler, the dual-mode hydration, + and a single shared LWW-merge module (currently reimplemented **3×**). +2. **Hub message router** (rank 5) — the WebSocket pump is one 600-line + dispatch chain where auth logic is copy-pasted across 4 handlers. A + handler-registry refactor is mechanical and removes the single highest-churn + editing chokepoint on the server. +3. **Stop the web/electron drift** (ranks 2, 10) — 6,574 duplicated lines + across 10 component pairs; `PageView`'s ~800-line comment state machine is + verbatim-identical yet the two copies have **zero commits in common**. 0230 + deferred whole-component convergence pending a parity audit — still right — + but the _verbatim_ hooks can be extracted now with no parity question. +4. **Barrel + provider ergonomics** (ranks 4, 6, 8) — the two biggest barrels + absorb 90 and 87 commits of pure re-export ceremony; `XNetProvider` bundles + five initialization concerns into one effect. Cheap fixes, felt weekly. + +`CanvasV3.tsx` and `ai-surface/service.ts` stay on the list (0230 Phase 4) but +are explicitly _down-weighted_: at current churn they should be split +opportunistically — when a feature already forces you in — not as standalone +projects. + +```mermaid +quadrantChart + title Churn (8 mo) vs size — where refactoring pays rent + x-axis Low churn --> High churn + y-axis Small --> Large + quadrant-1 "Refactor first (hot + big)" + quadrant-2 "Split opportunistically" + quadrant-3 "Leave alone" + quadrant-4 "Cheap ergonomics wins" + "sqlite-adapter.ts": [0.62, 0.85] + "store.ts": [0.72, 0.7] + "hub/server.ts": [0.78, 0.55] + "electron CanvasView": [0.68, 0.75] + "electron PageView": [0.7, 0.42] + "data index.ts barrel": [0.95, 0.45] + "react index.ts barrel": [0.97, 0.35] + "react context.ts": [0.75, 0.48] + "web App.tsx": [0.78, 0.44] + "CanvasV3.tsx": [0.25, 0.98] + "ai-surface service.ts": [0.2, 0.88] + "social importers": [0.1, 0.6] +``` + +## Current State In The Repository + +### How the ranking was computed + +```bash +git log --since="8 months ago" --pretty=format: --name-only \ + | grep -E '^(packages|apps)/.*\.(ts|tsx)$' \ + | grep -vE '\.(test|spec|stories|gen)\.' \ + | sort | uniq -c | sort -rn # churn +# then score = churn × current wc -l # hotspot rank +``` + +### Theme 1 — the data layer is the hottest and best-protected ground + +Every one of the 560 `@xnetjs/data` import sites funnels through two files. + +**`packages/data/src/store/sqlite-adapter.ts` (4,407 LOC, 35 changes).** One +class, 16 public + 64 private methods, spanning eight subsystems that are +separable today: + +| Lines (approx) | Subsystem | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 536–803 | change-log operations (append/get/prune) | +| 804–1188 | node CRUD + three near-duplicate list/count SQL builders | +| 1370–2170 | batch import — ~800 lines of duplicated operation builders | +| 2173–2373 | **hydration, twice**: joined-row and JSON-aggregated modes share ~200 near-identical lines incl. the LWW property merge | +| 2374–2643 | materialized views (0182 Phase 7) | +| 2644–3157 | three index families (scalar / FTS / spatial) with the same sync/rebuild/drop lifecycle each | +| 3429–3550 | adaptive-index budget management (0264) | +| 3959–4407 | **the query compiler**: `compileNodeQuery`/`compileSqlQuery` + fused candidate-and-hydrate CTE (0264 Wave 1), with feature flags (`adaptiveIndexing.enabled`, `queryVerification`) braided into codegen | + +**`packages/data/src/store/store.ts` (2,763 LOC, 47 changes).** `NodeStore` has +40+ public methods and three parallel execution paths (single-write fast path, +transaction slow path at lines ~1608–1765, transaction fast path at +~1766–1865) that each re-implement conflict tracking and listener dispatch. +`applyChange` (~2265–2488) is a 223-line method mixing LWW merge, property +reconciliation, encryption hooks, telemetry, and listener emission. + +**The LWW merge exists three times**: `store.ts` `applyChange`, the adapter's +hydration property merge (~line 2200), and `packages/hub/src/storage/sqlite.ts` +(~line 1200). This is the exact same drift class the SSRF-guard duplication in +0230 was — except this one guards _data convergence_, the protocol's core +invariant (exploration 0200 golden vectors; 0272 sim already caught one +lamport-only LWW guard bug that shipped). + +**Why refactoring here is unusually safe:** `sqlite-adapter.test.ts` (125 KB) +and `store.test.ts` (80 KB) cover all eight subsystems and both transaction +paths; `query-ast.test.ts` covers the in-memory evaluator; and the 0272 +reliability lane (`tests/reliability/` fault injection, restore drills, +adapter conformance) pins crash/recovery behavior. This is the rare god file +with characterization tests already written. + +Also verified: `packages/data/src/store/query-ast.ts` (1,409 LOC) is **not** +the SQL compiler — it is the query _type system_ plus an in-memory evaluator +used for JS fallback filtering. SQL generation lives only in the adapter. The +seam between them is already clean; the extraction below just makes it a file +boundary. + +### Theme 2 — hub `server.ts`: one function absorbs most server churn + +`packages/hub/src/server.ts` (1,750 LOC, 56 changes — the highest churn of any +non-barrel file). The HTTP side is already modular (`app.route('/backup', …)` +etc. — 11 mounted route modules). The churn magnet is the **WebSocket message +pump** (lines ~1070–1600+): 12+ message types dispatched through a chain of +type-guard if/else branches, where each handler repeats the same +auth-check → service-call → send → metrics shape, and the room-authorization +logic (`authorizeRoomAction`, lines 327–451) is invoked in four subtly +different inline forms. Every new message type or auth tweak edits this one +600-line closure. There are unit tests for capabilities/auth/query but **no +end-to-end test of the pump itself** — a handler registry would make each +message type testable in isolation. + +### Theme 3 — web/electron view duplication is drifting, measurably + +Ten component pairs exist in both `apps/web/src/components/` and +`apps/electron/src/renderer/components/` — 6,574 lines total on the two sides: + +| Pair | Web LOC | Electron LOC | Similarity | Commits touching web / electron / both (6 mo) | +| ----------------------- | ------: | -----------: | ------------------------------------------------------------------------------------------------------------ | --------------------------------------------- | +| `PageView.tsx` | 1,101 | 993 | ~93% — comment popover state machine, orphaned-thread assembly, and all comment action handlers are verbatim | 12 / 44 / **0** | +| `DataWorkspaceView.tsx` | 1,060 | 1,189 | ~92% — identical types and atlas/query logic | 13 / 12 / 9 | +| `CanvasView.tsx` | 1,843 | 3,195 | 60–70% — electron grew query frames + source references; web grew Desk + moderation gating | 23 / 42 / 11 | +| `DatabaseView.tsx` | 849 | 726 | ~85% | — | +| `ShareButton.tsx` | 40 | 320 | ~20% — effectively different components | — | +| (5 more small pairs) | | | 95–100% | — | + +The `PageView` numbers are the alarm: the copies are nearly identical _and_ +share **zero commits** — fixes land on one side only, silently. Git history +shows the drift is passive (parallel evolution, never an intentional fork). +The blocker to full convergence is real: web views integrate with the +workbench/router (`useWorkbench`, `useContextPanel`, `useStatusBarItem`), +electron with the canvas shell and IPC bridge +(`apps/electron/src/renderer/lib/ipc-sync-manager.ts`). But the _verbatim_ +logic — the comment state machine, thread assembly, action handlers, roughly +800 lines — has no platform dependency at all and can move to a shared package +today. 0230's "defer pending parity audit" stays correct for the component +shells; it should not keep hostage the hooks that are already identical. + +### Theme 4 — barrels and the provider: small files, weekly tax + +- `packages/react/src/index.ts` — **90 commits**, the most-churned file in the + repo; pure re-exports. `packages/data/src/index.ts` — 87 commits, same. + `packages/plugins/src/index.ts` — 47. Every new hook/schema/feature edits + these, so they are standing merge-conflict magnets, and `export *`-style + growth degrades tree-shaking and go-to-definition (see External Research). +- `packages/react/src/context.ts` (1,213 LOC, 49 changes) — `XNetProvider` + runs node-storage init, runtime-bridge resolution, sync-manager creation, + hub auth-token fetching, and backup orchestration; the central `useEffect` + (~lines 637–720) alone interleaves five concerns with a shared cleanup. It + has no direct tests and is very hard to simulate failure modes for. +- `apps/web/src/App.tsx` (1,009 LOC, 56 changes) — a 7-state boot state + machine, ~280-line storage-init effect, storage-durability watchers spread + over 3 effects, and PWA install handling, all inline. Its electron sibling + (1,193 LOC) churns for a different reason: each new shell view + (`social-import`, `data-workspace`, `stories`) edits a `ShellState` union + plus 4+ switch sites. +- By contrast `packages/react/src/hooks/useQuery.ts` (33 changes) was audited + and is **well-factored** — descriptor creation, subscription, transform, and + telemetry are cleanly layered. High churn alone isn't a problem; churn × + entanglement is. It also serves as the house style to refactor toward. + +### The sleeping giants (deliberately down-weighted) + +`packages/canvas/src/renderer/CanvasV3.tsx` (7,758 LOC) and +`packages/plugins/src/ai-surface/service.ts` (4,496 LOC) are the two biggest +files but neither appears in the top-40 churn list. Their seams are mapped +(input-handler modules, viewport reducer, selection capabilities, mutation +dispatcher for CanvasV3; tool registry + URI router + mutation pipeline for +the AI surface — which currently requires editing **three places** to add one +tool and has **no test file at all**). These belong on the "split when you're +already in there" list, with the AI surface's missing tests being the one item +worth doing eagerly. + +```mermaid +flowchart LR + subgraph apps["Every feature PR walks this path"] + W["apps/web + electron
views (duplicated ×2)"] + end + subgraph react["@xnetjs/react"] + B2["index.ts barrel
90 commits"] + CTX["context.ts XNetProvider
5 concerns / 1 effect"] + UQ["useQuery.ts ✓ well-factored"] + end + subgraph data["@xnetjs/data (560 import sites)"] + B1["index.ts barrel
87 commits"] + ST["store.ts NodeStore
3 txn paths, 223-line applyChange"] + AD["sqlite-adapter.ts
8 subsystems, 80 methods"] + end + subgraph hub["@xnetjs/hub"] + SV["server.ts WS pump
600-line dispatch"] + HS["storage/sqlite.ts
re-implements LWW + hydration"] + end + W --> B2 --> CTX --> UQ --> B1 --> ST --> AD + W -. sync .-> SV --> HS + ST -- "LWW merge ×3" --- AD + AD -- "LWW merge ×3" --- HS +``` + +## External Research + +- **Hotspot prioritization.** CodeScene/Tornhill's method — prioritize by + churn × complexity, because change frequency from version control is the + best available proxy for where defects and effort concentrate; top hotspots + are typically a small slice of code responsible for 25–70% of defects + ([CodeScene hotspots docs](https://docs.enterprise.codescene.io/versions/4.0.16/guides/technical/hotspots.html), + [Understand Legacy Code — hotspot analysis](https://understandlegacycode.com/blog/focus-refactoring-with-hotspots-analysis/), + [CodeScene — prioritize tech debt](https://codescene.com/blog/tech-debt-examples-prioritize-technical-debt-with-codescene)). + This doc is exactly that method applied to xNet. +- **Barrel files.** Known costs: bigger module graphs and slower builds + (Next.js measured 15–70% faster dev builds bypassing barrels), impaired + tree-shaking with `export *`, circular-dependency hiding, and constant merge + conflicts — with the standard mitigation being _scoped_ sub-barrels or + generated barrels for a stable public API + ([jsdev.space on replacing barrels](https://jsdev.space/howto/stop-using-barrel-files/), + [webpack discussion #16863](https://github.com/orgs/webpack/discussions/16863), + [barrel pros/cons](https://rahuulmiishra.medium.com/barrel-files-in-javascript-pros-cons-and-when-to-use-them-6efbeb22a8b6)). + For a published SDK like `@xnetjs/*` the barrel _is_ the public API, so the + right move is organization + generation, not deletion. +- **Decomposing god components/classes.** The consensus playbook: pin behavior + with characterization tests, extract _pure_ logic first, extract hooks with + interface-first design, and strangler-fig the rest — each phase shippable + ([Extract React Hook refactoring](https://blog.rstankov.com/extract-react-hook-refactoring/), + [CodeScene — refactoring React with custom hooks](https://codescene.com/blog/refactoring-components-in-react-with-custom-hooks), + [incremental frontend modernization](https://altersquare.io/how-teams-incrementally-modernize-large-frontend-codebases/), + [LogRocket — refactor to hooks](https://blog.logrocket.com/refactor-react-components-hooks/)). + The delegate-wrapper variant (old method forwards to the new module) keeps + every extraction diff review-sized. + +## Key Findings + +1. **Traffic and size point at different files.** The biggest file + (`CanvasV3.tsx`) is cold; the hottest files are the data-layer core, the + hub server, and the barrels. A refactor plan ranked purely by LOC (0230's + lens) would over-invest in sleeping giants and under-invest in the write + path everyone edits weekly. + +2. **The single highest-leverage refactor in the repo is decomposing + `sqlite-adapter.ts` + `store.ts` along their existing seams.** They are + rank 1 and 3 by traffic, their subsystems are aggregated rather than + tangled, and they are protected by ~205 KB of direct tests plus the + reliability lane — the cheapest risk profile a refactor of this size can + have. + +3. **The LWW merge implemented 3× is a correctness time bomb, not a style + issue.** Convergence is the product's core guarantee; 0272's simulation + already caught one shipped LWW-guard bug. One shared, golden-vector-tested + merge module eliminates the drift class. + +4. **The hub WebSocket pump is the most mechanical high-ROI extraction.** + Handler registry + one auth middleware turns a 600-line closure into ~15 + independently testable handlers; it also creates the seam where per-message + metrics and consistent error shapes become free. + +5. **`PageView`'s duplication has crossed from "debt" to "active hazard":** + 93% identical, 56 combined commits, zero shared ones. The verbatim comment + subsystem (~800 lines) is extractable _now_ without the parity audit that + rightly blocks whole-component convergence. + +6. **Barrel churn is a self-inflicted tax with a cheap fix.** Group new + surface into namespace/sub-barrel exports (and consider generating the + schema barrel per 0230 Phase 2) rather than appending to two 1,000-line + files on every feature. + +7. **`useQuery.ts` proves the codebase already knows the target shape.** The + goal is not new architecture; it is applying the layering that hook already + has to the seven files that lack it. + +## Options And Tradeoffs + +### Which theme to lead with + +| Option | Pros | Cons | +| ----------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------ | +| **A. Data layer first** (recommended) | Highest traffic; best tests; kills the 3× LWW drift; unblocks 0266 query-perf endgame work by making the compiler standalone | Touches the most-depended-on package; needs careful changesets (fixed-core lockstep) | +| B. Hub router first | Most mechanical; smallest blast radius; server churn is highest per line | Doesn't help the 560 client-side import sites; auth unification needs care | +| C. Cross-app dedup first | Stops active drift; user-visible bug class | Partially blocked by parity questions; touches both app shells at once | +| D. Sleeping giants first (CanvasV3, ai-surface) | Biggest LOC optics | Low traffic → low compounding payoff; highest characterization-test cost | + +A → B → C ordering maximizes payback-per-risk; D happens opportunistically. + +### How to extract from the god files + +| Option | Pros | Cons | +| ------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------- | +| **Delegate-wrapper extraction** (recommended): new module beside the old file; old method body becomes a one-line forward; tests migrate incrementally | Each PR is small and revertible; public API and test surface unchanged; blame stays navigable | Temporary indirection layer; a long tail of wrappers if never finished | +| Big-bang split into N files | Done in one move | Un-reviewable diff on rank-1 hotspots; conflicts with all in-flight work | +| Freeze + rewrite (v2 adapter) | Clean slate | History says no (CanvasV2→V3 left a 2.4K-LOC corpse; 0230 had to delete it) | + +### Where the shared LWW merge lives + +| Option | Pros | Cons | +| ----------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | +| **`packages/core` (e.g. `@xnetjs/core/lww`)** (recommended) | Already a dep of both `data` and `hub`; dependency-free; sits next to the 0200 protocol golden vectors conceptually | `core` is in the fixed-version release group — bump discipline needed | +| Inside `@xnetjs/data`, hub imports it | No new surface in core | Hub currently doesn't depend on `data`; would create a heavyweight edge | +| Leave 3 copies, add cross-impl conformance test | No refactor risk | Drift remains possible between test runs; 3 places to fix bugs forever | + +Even if the module lands in core, the cross-implementation conformance test +(same golden vectors run against store/adapter/hub paths) is worth writing — +it converts convergence from "reviewed" to "enforced." + +### Barrel strategy + +| Option | Pros | Cons | +| ------------------------------------------------------------------------ | ----------------------------------------------------------------------------- | ---------------------------------------------------------------------------- | +| **Scoped sub-barrels + namespace exports for new surface** (recommended) | Cuts conflict surface immediately; no build-step change; preserves public API | Existing 1,000-line files shrink only gradually | +| Generated barrels from a manifest | Eliminates hand-editing entirely; aligns with 0230's schema-SSoT plan | New codegen workflow + staleness CI check | +| Deep imports only (`@xnetjs/data/store`) | Best tree-shaking | Breaking change for SDK consumers; docs churn; not worth it pre-1.0 adoption | + +## Recommendation + +Work the four themes as **independent PR series**, highest traffic first. +Every extraction uses the delegate-wrapper pattern and ships behind the +existing public API. Full plan: + +```mermaid +flowchart TD + subgraph T1["Theme 1 — Data layer (rank 1,3,7)"] + A1["Extract query compiler
sqlite-adapter 3959–4407 → query-compiler.ts"] + A2["Extract hydration pair
2173–2373 → hydration.ts, dedup merge"] + A3["Shared LWW module in core
+ cross-impl golden-vector test"] + A4["Extract txn/batch orchestration
store.ts 932–1865"] + A1 --> A2 --> A3 --> A4 + end + subgraph T2["Theme 2 — Hub (rank 5)"] + B1["MessageRouter registry
replaces 600-line pump"] + B2["Unified room-auth middleware
one resolution path, 4 call sites"] + B1 --> B2 + end + subgraph T3["Theme 3 — Stop app drift (rank 2,10)"] + C1["Extract verbatim page-comment hooks
→ @xnetjs/views app-views"] + C2["Share DataWorkspaceView (92%)"] + C3["Parity audit → CanvasView convergence
(product decision, from 0230 Phase 5)"] + C1 --> C2 --> C3 + end + subgraph T4["Theme 4 — Ergonomics (rank 4,6,8,9)"] + D1["Sub-barrel / namespace policy
for react + data + plugins"] + D2["Decompose XNetProvider
init/bridge/sync/auth/backup"] + D3["web App.tsx boot orchestrator
electron App.tsx shell reducer"] + end + T1 --> T2 --> T3 + T4 -.-> |any time, independent| T1 +``` + +**Concrete first steps (each one PR):** + +1. **`query-compiler.ts`** — move `compileNodeQuery`/`compileSqlQuery`/ + `tryFuseCandidateWithHydrate` + telemetry hooks out of + [sqlite-adapter.ts](packages/data/src/store/sqlite-adapter.ts) behind a + `QueryCompiler` taking `(descriptor, schemaContext, flags)`. Adapter methods + become forwards. Move the relevant test blocks. +2. **`hydration.ts`** — `JoinedHydrator` + `AggregatedHydrator` sharing one + property-merge function; delete the ~200-line near-duplicate. +3. **`@xnetjs/core` LWW module** — one comparator (lamport → updated_at → + updated_by code-unit order, per the 0257 invariant), adopted by store, + adapter hydration, and hub storage; golden-vector conformance test runs + against all three call sites. +4. **Hub `MessageRouter`** — `router.on('node-sync-request', handler)` registry + with an auth-context middleware; the pump body shrinks to guard-parse + + dispatch; add the currently-missing pump-level tests handler-by-handler. +5. **`usePageComments` extraction** — the verbatim comment state machine, + orphaned-thread assembly, and action handlers from both `PageView`s into + `packages/views` (or `packages/react`); both apps' components become + consumers. No behavior change, ends the zero-shared-commit drift for that + logic. +6. **Barrel policy** — new exports land in scoped sub-barrels + (`@xnetjs/react` already has `hooks/`; mirror that shape in the barrel with + grouped `export * as` blocks); note the policy in `CLAUDE.md`. + +Changesets: extractions that keep public APIs identical are +`pnpm changeset --empty` or `patch`; the LWW unification is a real `patch` +(behavior aligns to the strictest implementation); anything that removes or +renames an export from a published barrel is `major` — bump from the diff. + +## Example Code + +**Hub message router (Theme 2):** + +```ts +// packages/hub/src/ws/message-router.ts +type Handler = (msg: T, ctx: WsContext) => Promise + +export class MessageRouter { + private handlers = new Map boolean; run: Handler }>() + + on(type: string, guard: (m: unknown) => m is T, run: Handler) { + this.handlers.set(type, { guard, run }) + return this + } + + async dispatch(raw: unknown, ctx: WsContext) { + for (const [type, h] of this.handlers) { + if (!h.guard(raw)) continue + HUB_METRICS.WS_MESSAGES_RECEIVED.inc({ type }) + try { + await h.run(raw, ctx) // auth lives in ctx.authorize(), one impl + } catch (err) { + ctx.sendError(type, err) // one error shape for every handler + } + return + } + ctx.sendError('unknown', new UnknownMessageError()) + } +} + +// server.ts shrinks to registration: +router + .on('client-handshake', isClientHandshake, handleHandshake) + .on('query-request', isQueryRequest, handleQuery) + .on('node-sync-request', isNodeSyncRequest, handleNodeSync) +// …each handler is a small, individually-tested module +``` + +**Shared LWW merge (Theme 1, step 3):** + +```ts +// packages/core/src/lww/merge.ts +export interface LwwStamp { + lamportTime: number + updatedAt: number // tie-break 1 + updatedBy: string // tie-break 2 — compare by code units (see +} // fractional-sortKey collation invariant) + +/** The ONE ordering used by store.applyChange, adapter hydration, and hub + * storage. Golden vectors from exploration 0200 run against all call sites. */ +export function lwwWins(incoming: LwwStamp, existing: LwwStamp): boolean { + if (incoming.lamportTime !== existing.lamportTime) + return incoming.lamportTime > existing.lamportTime + if (incoming.updatedAt !== existing.updatedAt) return incoming.updatedAt > existing.updatedAt + return incoming.updatedBy > existing.updatedBy +} +``` + +**Delegate-wrapper extraction shape (Theme 1, steps 1–2):** + +```ts +// packages/data/src/store/sqlite-adapter.ts — after extraction +import { QueryCompiler } from './query-compiler' + +export class SQLiteAdapterNodeStorageAdapter { + private compiler = new QueryCompiler( + () => this.schemaContext(), + () => this.flags() + ) + + /** @deprecated internal — logic lives in query-compiler.ts */ + private compileNodeQuery(d: NodeQueryDescriptor) { + return this.compiler.compile(d) // public behavior byte-identical + } +} +``` + +## Risks And Open Questions + +- **These are the worst files to have long-lived branches in.** Rank-1 hotspots + conflict with everything in flight; each extraction PR must land fast + (small, delegate-wrapper, no API change) or not start. +- **Fixed-core release coupling.** `core`, `data`, `react` version in lockstep; + the LWW module in core bumps the whole group. Coordinate with the standing + release-PR cadence (0265 policy) so staged bumps don't rot. +- **LWW unification may surface latent divergence.** If the three + implementations disagree today on some input, unifying _changes behavior_ + somewhere. That's the point — but the conformance test must run against old + fixtures first, and any divergence found is a release-noted `patch` fix, not + a silent change. +- **`fused` query paths and feature flags.** The compiler extraction must carry + the adaptive-indexing and verification flags as explicit inputs, not ambient + adapter state — otherwise the extraction just relocates the tangle. +- **Hook extraction from `PageView` can still hit subtle platform deltas** + (web renders comments in a context panel, electron in a sibling sidebar). + The extraction is scoped to _state + handlers_, not rendering; if a verbatim + block turns out to differ semantically, that's a drift bug to fix, and it + should be called out in the PR. +- **Where should the shared app-view hooks live?** `packages/views` currently + means "database view renderers"; `packages/react` means "data hooks". A new + `app-views/` subpath in `views` (agent recommendation) vs a `page/` area in + `react` is a naming decision to settle in the first PR. +- **Parity audit for CanvasView remains a product decision** (query frames and + peek on web? Desk on electron?) — inherited open question from 0230. +- **Barrel policy only helps if enforced.** Without a lint/convention note, + the next 90 commits will keep appending to `react/index.ts`. + +## Implementation Checklist + +Theme 1 — data layer decomposition + +- [x] Extract `packages/data/src/store/query-compiler.ts` (compile + count + + fuse + telemetry hooks) with adapter delegating; move matching test + blocks to `query-compiler.test.ts`. +- [x] Extract `packages/data/src/store/hydration.ts` (`JoinedHydrator`, + `AggregatedHydrator`, shared property-merge); delete duplicated logic. +- [x] Add `@xnetjs/core` LWW module; adopt in `store.ts` `applyChange`, + adapter hydration, `packages/hub/src/storage/sqlite.ts`. +- [x] Cross-implementation LWW conformance test using 0200 golden vectors. +- [x] Extract transaction/batch orchestration from `store.ts` + (`transaction-executor.ts`, `batch-write-orchestrator.ts`); unify + conflict tracking + listener dispatch between fast and slow paths. +- [x] Split the three index families out of the adapter behind an + `IndexingStrategy` interface (scalar / FTS / spatial). +- [x] Run the full reliability lane (`tests/reliability/`) after each PR. + +Theme 2 — hub server + +- [x] Introduce `MessageRouter`; migrate the 12+ WS message types one handler + per commit; delete the if/else pump. +- [x] Unify `authorizeRoomAction` / `checkRoomAuth` / inline checks into one + auth middleware with a single resolution path. +- [x] Standardize the WS error response shape; add per-message-type metrics in + the router (replacing scattered `HUB_METRICS` calls). +- [x] Add pump-level tests per handler (previously untested end-to-end). + +Theme 3 — stop cross-app drift + +- [x] Extract the verbatim page-comment subsystem from both `PageView.tsx` + files into a shared package; both apps consume it. +- [x] Share `DataWorkspaceView` core (92% identical) with platform hooks for + the canvas-insert (electron) and moderation-gate (web) deltas. +- [x] Add a drift tripwire: CI check or review convention flagging edits to one + side of a known-duplicated pair (list from this doc) without the other. +- [x] Schedule the CanvasView feature-parity audit (product decision) — gate + for full convergence, per 0230 Phase 5. + +Theme 4 — ergonomics (any time) + +- [x] Adopt sub-barrel/namespace export policy for `react`, `data`, `plugins` + barrels; document in `CLAUDE.md`. +- [x] Decompose `XNetProvider` into init / bridge / sync / auth / backup + units with tests for failure paths. +- [x] Extract web `App.tsx` boot orchestrator (storage init + durability + watchers + state machine) and electron `App.tsx` shell-state reducer. + +Opportunistic (do when already in the file) + +- [x] CanvasV3: extract pure viewport math + the three duplicate + `applyXxxUpdates` into a mutation dispatcher; input-handler modules per + tool mode. +- [x] ai-surface: tool registry + URI router; **add the missing test file + first** (mutation/rollback/audit logic currently has zero tests). + +## Validation Checklist + +- [x] `pnpm -r build && pnpm -r typecheck && pnpm -r test` green after every + PR; public exports of touched packages unchanged unless release-noted. + (Final state: build 52/52, typecheck 91/91, tests 10,305 passed.) +- [x] Reliability lane green after each Theme-1 PR (fault injection, restore + drill, adapter conformance). +- [x] LWW conformance test passes identically against store, adapter, and hub + implementations; any pre-existing divergence documented in the PR. +- [x] Query results parity: `auditQueryParity` / query-verification flag shows + no drift pre/post compiler extraction on the seeded workspace. +- [x] Hub WS behavior pinned: new per-handler pump tests pass (hub suite + 41 files / 333 tests green). Staging hub (`cloud-staging.xnet.fyi`) + smoke-test is a post-deploy follow-up once this merges to main. +- [x] Page comments work identically in web and electron after hook + extraction (create/reply/resolve/reopen/delete/orphaned threads). +- [x] Churn re-measure scheduled as a 2–3 month follow-up: re-run the script + in "How the ranking was computed" — `server.ts` and the two barrels + should drop out of the top-10 churn×LOC ranking. (Not measurable at + implementation time by construction.) +- [x] Changesets present and bumps justified from diffs for every touched + publishable package (Stop hook green): core `minor` (new lww exports), + data/plugins/react `patch` (internal decompositions, barrels unchanged). + +## References + +- Companion: `docs/explorations/0230_[_]_CODEBASE_REFACTORING_ATLAS_DEAD_CODE_GOD_FILES_AND_DEDUPLICATION.md` + (leverage-by-LOC ladder; Phases 0–1 partially shipped). +- Related explorations: 0264/0266 (query perf — motivates compiler extraction), + 0272 (reliability lane — the safety net), 0200 (protocol golden vectors — + LWW source of truth), 0257 (LWW tie-break invariant), 0182 (materialized + views), 0230 Phase 5 (parity audit precondition). +- Hotspot files: `packages/data/src/store/sqlite-adapter.ts`, + `packages/data/src/store/store.ts`, `packages/data/src/store/query-ast.ts`, + `packages/hub/src/server.ts`, `packages/hub/src/storage/sqlite.ts`, + `packages/react/src/context.ts`, `apps/web/src/App.tsx`, + `apps/electron/src/renderer/App.tsx`, and the ten duplicated + `apps/{web,electron}` component pairs. +- [CodeScene — Hotspots documentation](https://docs.enterprise.codescene.io/versions/4.0.16/guides/technical/hotspots.html) +- [Understand Legacy Code — Focus refactoring with hotspot analysis](https://understandlegacycode.com/blog/focus-refactoring-with-hotspots-analysis/) +- [CodeScene — Prioritize technical debt](https://codescene.com/blog/tech-debt-examples-prioritize-technical-debt-with-codescene) +- [jsdev.space — Replace barrel files with better import strategies](https://jsdev.space/howto/stop-using-barrel-files/) +- [webpack discussion #16863 — barrels, tree-shaking, monorepos](https://github.com/orgs/webpack/discussions/16863) +- [Barrel files: pros, cons, when to use](https://rahuulmiishra.medium.com/barrel-files-in-javascript-pros-cons-and-when-to-use-them-6efbeb22a8b6) +- [Rado Stankov — Extract React Hook refactoring](https://blog.rstankov.com/extract-react-hook-refactoring/) +- [CodeScene — Refactoring React components with custom hooks](https://codescene.com/blog/refactoring-components-in-react-with-custom-hooks) +- [AlterSquare — Incrementally modernizing large frontend codebases](https://altersquare.io/how-teams-incrementally-modernize-large-frontend-codebases/) +- [LogRocket — Refactor React components to hooks](https://blog.logrocket.com/refactor-react-components-hooks/) diff --git a/package.json b/package.json index ea1abdbda..615a69497 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "check:humane-patterns": "node scripts/check-humane-patterns.mjs", "check:publish-closure": "node scripts/check-publish-closure.mjs", "check:electron-parity": "node scripts/check-electron-parity.mjs", + "check:view-drift": "node scripts/check-view-drift.mjs", "test:editor": "pnpm --filter @xnetjs/editor test", "test:stories": "storybook test --url http://127.0.0.1:6006", "test:watch": "vitest", diff --git a/packages/canvas/src/__tests__/scene-mutations.test.ts b/packages/canvas/src/__tests__/scene-mutations.test.ts new file mode 100644 index 000000000..f546e9607 --- /dev/null +++ b/packages/canvas/src/__tests__/scene-mutations.test.ts @@ -0,0 +1,145 @@ +/** + * Scene-mutation dispatcher tests for the canvas v3 renderer. + */ + +import type { CanvasNode } from '../types' +import { describe, expect, it, vi } from 'vitest' +import * as Y from 'yjs' +import { + applyCanvasSceneUpdates, + mergeCanvasNodeLockUpdate, + mergeCanvasNodePositionUpdate, + mergeCanvasNodePropertiesUpdate +} from '../renderer/scene-mutations' +import { getCanvasObjectsMap } from '../scene/doc-layout' + +function createNode(id: string, overrides: Partial = {}): CanvasNode { + return { + id, + type: 'page', + position: { x: 10, y: 20, width: 120, height: 80, zIndex: 0 }, + properties: { title: id }, + ...overrides + } +} + +function createDocWithNodes(nodes: readonly CanvasNode[]): Y.Doc { + const doc = new Y.Doc() + const objects = getCanvasObjectsMap(doc) + + doc.transact(() => { + for (const node of nodes) { + objects.set(node.id, node) + } + }) + + return doc +} + +describe('applyCanvasSceneUpdates', () => { + it('returns false without touching the doc for an empty batch', () => { + const doc = createDocWithNodes([createNode('a')]) + const onSceneMutation = vi.fn() + const merge = vi.fn(mergeCanvasNodeLockUpdate) + + const changed = applyCanvasSceneUpdates({ doc, updates: [], merge, onSceneMutation }) + + expect(changed).toBe(false) + expect(merge).not.toHaveBeenCalled() + expect(onSceneMutation).not.toHaveBeenCalled() + }) + + it('skips updates whose node is missing and stays silent when nothing changed', () => { + const doc = createDocWithNodes([createNode('a')]) + const onSceneMutation = vi.fn() + + const changed = applyCanvasSceneUpdates({ + doc, + updates: [{ id: 'ghost', locked: true }], + merge: mergeCanvasNodeLockUpdate, + onSceneMutation + }) + + expect(changed).toBe(false) + expect(onSceneMutation).not.toHaveBeenCalled() + expect(getCanvasObjectsMap(doc).get('a')?.locked).toBeUndefined() + }) + + it('applies every matched update and notifies onSceneMutation once', () => { + const doc = createDocWithNodes([createNode('a'), createNode('b')]) + const onSceneMutation = vi.fn() + + const changed = applyCanvasSceneUpdates({ + doc, + updates: [ + { id: 'a', locked: true }, + { id: 'ghost', locked: true }, + { id: 'b', locked: true } + ], + merge: mergeCanvasNodeLockUpdate, + onSceneMutation + }) + + const objects = getCanvasObjectsMap(doc) + expect(changed).toBe(true) + expect(onSceneMutation).toHaveBeenCalledTimes(1) + expect(objects.get('a')?.locked).toBe(true) + expect(objects.get('b')?.locked).toBe(true) + }) + + it('batches the whole update set into a single doc transaction', () => { + const doc = createDocWithNodes([createNode('a'), createNode('b')]) + let updateEvents = 0 + doc.on('update', () => { + updateEvents += 1 + }) + + applyCanvasSceneUpdates({ + doc, + updates: [ + { id: 'a', locked: true }, + { id: 'b', locked: false } + ], + merge: mergeCanvasNodeLockUpdate + }) + + expect(updateEvents).toBe(1) + }) +}) + +describe('mergeCanvasNodePositionUpdate', () => { + it('merges partial positions over the existing position', () => { + const node = createNode('a') + + const next = mergeCanvasNodePositionUpdate(node, { id: 'a', position: { x: 300 } }) + + expect(next.position).toEqual({ x: 300, y: 20, width: 120, height: 80, zIndex: 0 }) + expect(node.position.x).toBe(10) + }) +}) + +describe('mergeCanvasNodeLockUpdate', () => { + it('replaces the lock flag and keeps the rest of the node', () => { + const node = createNode('a') + + const next = mergeCanvasNodeLockUpdate(node, { id: 'a', locked: true }) + + expect(next.locked).toBe(true) + expect(next.position).toEqual(node.position) + expect(next.properties).toEqual(node.properties) + }) +}) + +describe('mergeCanvasNodePropertiesUpdate', () => { + it('merges new properties over the existing properties', () => { + const node = createNode('a', { properties: { title: 'a', label: 'Label' } }) + + const next = mergeCanvasNodePropertiesUpdate(node, { + id: 'a', + properties: { title: 'renamed' } + }) + + expect(next.properties).toEqual({ title: 'renamed', label: 'Label' }) + expect(node.properties.title).toBe('a') + }) +}) diff --git a/packages/canvas/src/__tests__/viewport-math.test.ts b/packages/canvas/src/__tests__/viewport-math.test.ts new file mode 100644 index 000000000..6362ac5d5 --- /dev/null +++ b/packages/canvas/src/__tests__/viewport-math.test.ts @@ -0,0 +1,315 @@ +/** + * Pure viewport and geometry math tests for the canvas v3 renderer. + */ + +import type { CanvasNode, Rect, ViewportState } from '../types' +import { describe, expect, it } from 'vitest' +import { + createCanvasCameraForViewport, + getActiveSnapGridSize, + getBoundsForRects, + getCanvasObjectHitTargetRect, + getFitViewport, + getNodePositionRect, + getRectAnchorPointForPlacement, + getScreenLineForSnapGuide, + getScreenPointForCanvasPoint, + getScreenRectForCanvasRect, + getViewportWorldTopLeft, + intersectsViewport, + pickConnectorPlacementForScreenPoint, + snapCanvasValue +} from '../renderer/viewport-math' + +const VIEWPORT_SIZE = { width: 960, height: 640 } + +function createNode(id: string, position: Rect): CanvasNode { + return { + id, + type: 'page', + position: { + ...position, + zIndex: 0 + }, + properties: { + title: id + } + } +} + +describe('createCanvasCameraForViewport', () => { + it('centres the camera on the viewport at the requested zoom', () => { + const camera = createCanvasCameraForViewport({ x: 120, y: -40, zoom: 1.5 }, VIEWPORT_SIZE) + + expect(camera.localCenter).toEqual({ x: 120, y: -40 }) + expect(camera.zoom).toBe(1.5) + expect(camera.viewportPx).toEqual(VIEWPORT_SIZE) + }) +}) + +describe('getScreenPointForCanvasPoint', () => { + it('maps the viewport centre to the screen centre', () => { + const viewport: ViewportState = { x: 120, y: -40, zoom: 2 } + + expect(getScreenPointForCanvasPoint({ x: 120, y: -40 }, viewport, VIEWPORT_SIZE)).toEqual({ + x: VIEWPORT_SIZE.width / 2, + y: VIEWPORT_SIZE.height / 2 + }) + }) + + it('scales offsets from the viewport centre by zoom', () => { + const point = { x: 130, y: -20 } + + const atZoom1 = getScreenPointForCanvasPoint(point, { x: 120, y: -40, zoom: 1 }, VIEWPORT_SIZE) + const atZoom2 = getScreenPointForCanvasPoint(point, { x: 120, y: -40, zoom: 2 }, VIEWPORT_SIZE) + + expect(atZoom1).toEqual({ x: 490, y: 340 }) + expect(atZoom2).toEqual({ x: 500, y: 360 }) + }) + + it('round-trips with the world top-left at any zoom', () => { + for (const zoom of [0.25, 1, 3]) { + const viewport: ViewportState = { x: 57, y: -213, zoom } + const topLeft = getViewportWorldTopLeft(viewport, VIEWPORT_SIZE) + + const screen = getScreenPointForCanvasPoint(topLeft, viewport, VIEWPORT_SIZE) + + expect(screen.x).toBeCloseTo(0) + expect(screen.y).toBeCloseTo(0) + } + }) +}) + +describe('getScreenRectForCanvasRect', () => { + it('projects a canvas rect into screen space', () => { + const viewport: ViewportState = { x: 0, y: 0, zoom: 1 } + + expect( + getScreenRectForCanvasRect( + { x: -50, y: -25, width: 100, height: 50 }, + viewport, + VIEWPORT_SIZE + ) + ).toEqual({ + x: VIEWPORT_SIZE.width / 2 - 50, + y: VIEWPORT_SIZE.height / 2 - 25, + width: 100, + height: 50 + }) + }) + + it('scales rect dimensions by the viewport zoom', () => { + const rect = { x: 10, y: 20, width: 100, height: 50 } + + for (const zoom of [0.5, 2]) { + const screenRect = getScreenRectForCanvasRect(rect, { x: 0, y: 0, zoom }, VIEWPORT_SIZE) + + expect(screenRect.width).toBeCloseTo(rect.width * zoom) + expect(screenRect.height).toBeCloseTo(rect.height * zoom) + } + }) +}) + +describe('getViewportWorldTopLeft', () => { + it('offsets the centre by half the viewport in world units', () => { + expect(getViewportWorldTopLeft({ x: 100, y: 50, zoom: 2 }, VIEWPORT_SIZE)).toEqual({ + x: 100 - VIEWPORT_SIZE.width / 4, + y: 50 - VIEWPORT_SIZE.height / 4 + }) + }) +}) + +describe('getBoundsForRects', () => { + it('returns null for an empty list', () => { + expect(getBoundsForRects([])).toBeNull() + }) + + it('unions rects into a single bounding box', () => { + expect( + getBoundsForRects([ + { x: 40, y: 30, width: 100, height: 80 }, + { x: 220, y: 160, width: 140, height: 120 } + ]) + ).toEqual({ x: 40, y: 30, width: 320, height: 250 }) + }) +}) + +describe('getNodePositionRect', () => { + it('reads the node position as a plain rect', () => { + const node = createNode('a', { x: 12, y: 34, width: 120, height: 80 }) + + expect(getNodePositionRect(node)).toEqual({ x: 12, y: 34, width: 120, height: 80 }) + }) +}) + +describe('getScreenLineForSnapGuide', () => { + const viewport: ViewportState = { x: 0, y: 0, zoom: 1 } + + it('maps a vertical guide onto a screen line', () => { + const line = getScreenLineForSnapGuide( + { + id: 'v', + source: 'object', + orientation: 'vertical', + position: 10, + start: -20, + end: 40, + relatedNodeIds: [] + }, + viewport, + VIEWPORT_SIZE + ) + + expect(line).toEqual({ + x1: VIEWPORT_SIZE.width / 2 + 10, + y1: VIEWPORT_SIZE.height / 2 - 20, + x2: VIEWPORT_SIZE.width / 2 + 10, + y2: VIEWPORT_SIZE.height / 2 + 40 + }) + }) + + it('maps a horizontal guide onto a screen line', () => { + const line = getScreenLineForSnapGuide( + { + id: 'h', + source: 'object', + orientation: 'horizontal', + position: -30, + start: 0, + end: 100, + relatedNodeIds: [] + }, + viewport, + VIEWPORT_SIZE + ) + + expect(line).toEqual({ + x1: VIEWPORT_SIZE.width / 2, + y1: VIEWPORT_SIZE.height / 2 - 30, + x2: VIEWPORT_SIZE.width / 2 + 100, + y2: VIEWPORT_SIZE.height / 2 - 30 + }) + }) +}) + +describe('intersectsViewport', () => { + it('accepts rects inside or near the viewport margin', () => { + expect(intersectsViewport({ x: 10, y: 10, width: 50, height: 50 }, VIEWPORT_SIZE)).toBe(true) + expect(intersectsViewport({ x: -360, y: 0, width: 50, height: 50 }, VIEWPORT_SIZE)).toBe(true) + }) + + it('rejects rects beyond the margin', () => { + expect(intersectsViewport({ x: -420, y: 0, width: 50, height: 50 }, VIEWPORT_SIZE)).toBe(false) + expect( + intersectsViewport( + { x: VIEWPORT_SIZE.width + 321, y: 0, width: 50, height: 50 }, + VIEWPORT_SIZE + ) + ).toBe(false) + }) + + it('honours a custom margin', () => { + expect(intersectsViewport({ x: -100, y: 0, width: 50, height: 50 }, VIEWPORT_SIZE, 0)).toBe( + false + ) + }) +}) + +describe('getFitViewport', () => { + it('centres on the rect and fits the constraining axis', () => { + const next = getFitViewport({ + rect: { x: 0, y: 0, width: 400, height: 100 }, + viewportSize: VIEWPORT_SIZE, + minZoom: 0.1, + maxZoom: 4, + padding: 80 + }) + + expect(next.x).toBe(200) + expect(next.y).toBe(50) + // Width is the constraining axis: (960 - 160) / 400. + expect(next.zoom).toBeCloseTo(2) + }) + + it('clamps the zoom to the configured range', () => { + const zoomedOut = getFitViewport({ + rect: { x: 0, y: 0, width: 100_000, height: 100 }, + viewportSize: VIEWPORT_SIZE, + minZoom: 0.5, + maxZoom: 4, + padding: 80 + }) + const zoomedIn = getFitViewport({ + rect: { x: 0, y: 0, width: 10, height: 10 }, + viewportSize: VIEWPORT_SIZE, + minZoom: 0.5, + maxZoom: 4, + padding: 80 + }) + + expect(zoomedOut.zoom).toBe(0.5) + expect(zoomedIn.zoom).toBe(4) + }) +}) + +describe('snapCanvasValue', () => { + it('snaps values to the nearest grid step', () => { + expect(snapCanvasValue(29, 20)).toBe(20) + expect(snapCanvasValue(31, 20)).toBe(40) + expect(snapCanvasValue(-29, 20)).toBe(-20) + }) +}) + +describe('getActiveSnapGridSize', () => { + it('defaults to 20 when the config omits a grid size', () => { + expect(getActiveSnapGridSize({})).toBe(20) + }) + + it('returns null for non-positive or non-finite grid sizes', () => { + expect(getActiveSnapGridSize({ gridSize: 0 })).toBeNull() + expect(getActiveSnapGridSize({ gridSize: -4 })).toBeNull() + expect(getActiveSnapGridSize({ gridSize: Number.NaN })).toBeNull() + }) +}) + +describe('getCanvasObjectHitTargetRect', () => { + it('pads large rects by the hit-target padding', () => { + expect(getCanvasObjectHitTargetRect({ x: 100, y: 100, width: 200, height: 120 })).toEqual({ + x: 92, + y: 92, + width: 216, + height: 136 + }) + }) + + it('expands tiny rects to the minimum hit-target size', () => { + expect(getCanvasObjectHitTargetRect({ x: 100, y: 100, width: 10, height: 10 })).toEqual({ + x: 87, + y: 87, + width: 36, + height: 36 + }) + }) +}) + +describe('pickConnectorPlacementForScreenPoint', () => { + const rect = { x: 0, y: 0, width: 100, height: 100 } + + it('picks the side facing the pointer', () => { + expect(pickConnectorPlacementForScreenPoint(rect, { x: 95, y: 50 })).toBe('right') + expect(pickConnectorPlacementForScreenPoint(rect, { x: 5, y: 50 })).toBe('left') + expect(pickConnectorPlacementForScreenPoint(rect, { x: 50, y: 95 })).toBe('bottom') + expect(pickConnectorPlacementForScreenPoint(rect, { x: 50, y: 5 })).toBe('top') + }) +}) + +describe('getRectAnchorPointForPlacement', () => { + const rect = { x: 10, y: 20, width: 100, height: 60 } + + it('returns the midpoint of each edge', () => { + expect(getRectAnchorPointForPlacement(rect, 'top')).toEqual({ x: 60, y: 20 }) + expect(getRectAnchorPointForPlacement(rect, 'right')).toEqual({ x: 110, y: 50 }) + expect(getRectAnchorPointForPlacement(rect, 'bottom')).toEqual({ x: 60, y: 80 }) + expect(getRectAnchorPointForPlacement(rect, 'left')).toEqual({ x: 10, y: 50 }) + }) +}) diff --git a/packages/canvas/src/renderer/CanvasV3.tsx b/packages/canvas/src/renderer/CanvasV3.tsx index a025ea34f..8071d22e7 100644 --- a/packages/canvas/src/renderer/CanvasV3.tsx +++ b/packages/canvas/src/renderer/CanvasV3.tsx @@ -13,20 +13,14 @@ import type { CanvasLayerDirection, CanvasNode, CanvasNodeProperties, - CanvasObjectAnchorPlacement, Point, Rect, ResizeHandle, - ShapeType + ShapeType, + ViewportState } from '../types' import type { CanvasObjectRecord, CanvasTileSummary } from '@xnetjs/canvas-core' -import { - createCanvasCamera, - createWorldPointFromCanvasPoint, - screenToWorldPoint, - worldPointToAnchorLocal, - worldToScreenPoint -} from '@xnetjs/canvas-core' +import { screenToWorldPoint, worldPointToAnchorLocal } from '@xnetjs/canvas-core' import { clamp } from '@xnetjs/core' import React, { forwardRef, @@ -116,6 +110,31 @@ import { import { type CanvasThemeTokens, useCanvasThemeTokens } from '../theme/canvas-theme' import { planDomIslandPool } from './dom-island-pool' import { computePinchViewport, measureTouchPinch, type PinchGestureState } from './pinch-zoom' +import { + applyCanvasSceneUpdates, + mergeCanvasNodeLockUpdate, + mergeCanvasNodePositionUpdate, + mergeCanvasNodePropertiesUpdate, + type CanvasNodePropertiesUpdate +} from './scene-mutations' +import { + createCanvasCameraForViewport, + getActiveSnapGridSize, + getBoundsForRects, + getCanvasObjectHitTargetRect, + getFitViewport, + getNodePositionRect, + getRectAnchorPointForPlacement, + getScreenLineForSnapGuide, + getScreenRectForCanvasRect, + getScreenRectForObject, + getViewportWorldTopLeft, + intersectsViewport, + pickConnectorPlacementForScreenPoint, + snapCanvasValue, + type ConnectorHandlePlacement, + type Size +} from './viewport-math' const EMPTY_FRAME_STATS: FrameStats = { frameCount: 0, @@ -287,17 +306,6 @@ export type CanvasProps = { navigationToolsStyle?: React.CSSProperties } -type ViewportState = { - x: number - y: number - zoom: number -} - -type Size = { - width: number - height: number -} - type ScreenObject = { object: CanvasObjectRecord node: CanvasNode @@ -359,15 +367,6 @@ const SELECTION_POPOVER_CAPABILITY: Record - type ConnectorStart = { nodeId: string placement: ConnectorHandlePlacement @@ -542,69 +541,13 @@ const EDGE_TYPE_OPTIONS: readonly { ] const MIN_SELECTION_DIMENSION_WIDTH = 96 const MIN_SELECTION_DIMENSION_HEIGHT = 72 -const CANVAS_OBJECT_HIT_TARGET_PADDING = 8 -const CANVAS_OBJECT_MIN_HIT_TARGET_SIZE = 36 const CANVAS_DRAG_START_THRESHOLD_PX = 3 const SMART_GUIDE_SCREEN_THRESHOLD = 8 -function snapCanvasValue(value: number, gridSize: number): number { - return Math.round(value / gridSize) * gridSize -} - -function getActiveSnapGridSize(config: CanvasConfig): number | null { - const gridSize = config.gridSize ?? 20 - - return Number.isFinite(gridSize) && gridSize > 0 ? gridSize : null -} - -function getCanvasObjectHitTargetRect(rect: Rect): Rect { - const extraWidth = Math.max( - CANVAS_OBJECT_HIT_TARGET_PADDING * 2, - CANVAS_OBJECT_MIN_HIT_TARGET_SIZE - rect.width - ) - const extraHeight = Math.max( - CANVAS_OBJECT_HIT_TARGET_PADDING * 2, - CANVAS_OBJECT_MIN_HIT_TARGET_SIZE - rect.height - ) - - return { - x: rect.x - extraWidth / 2, - y: rect.y - extraHeight / 2, - width: rect.width + extraWidth, - height: rect.height + extraHeight - } -} - function getObjectTitle(object: CanvasObjectRecord): string { return object.preview.title ?? object.kind.replace('-', ' ') } -function pickConnectorPlacementForScreenPoint(rect: Rect, point: Point): ConnectorHandlePlacement { - const centerX = rect.x + rect.width / 2 - const centerY = rect.y + rect.height / 2 - const dx = rect.width > 0 ? (point.x - centerX) / (rect.width / 2) : 0 - const dy = rect.height > 0 ? (point.y - centerY) / (rect.height / 2) : 0 - - if (Math.abs(dx) >= Math.abs(dy)) { - return dx >= 0 ? 'right' : 'left' - } - - return dy >= 0 ? 'bottom' : 'top' -} - -function getRectAnchorPointForPlacement(rect: Rect, placement: ConnectorHandlePlacement): Point { - switch (placement) { - case 'top': - return { x: rect.x + rect.width / 2, y: rect.y } - case 'right': - return { x: rect.x + rect.width, y: rect.y + rect.height / 2 } - case 'bottom': - return { x: rect.x + rect.width / 2, y: rect.y + rect.height } - case 'left': - return { x: rect.x, y: rect.y + rect.height / 2 } - } -} - function getNodeTitle(node: CanvasNode, fallback: string): string { const title = typeof node.alias === 'string' @@ -1055,88 +998,6 @@ function applyMindMapInheritedStyle( } } -function createCanvasCameraForViewport(viewport: ViewportState, viewportSize: Size) { - return createCanvasCamera({ - localCenter: { x: viewport.x, y: viewport.y }, - zoom: viewport.zoom, - viewportPx: viewportSize - }) -} - -function getViewportWorldTopLeft(viewport: ViewportState, viewportSize: Size): Point { - return { - x: viewport.x - viewportSize.width / 2 / viewport.zoom, - y: viewport.y - viewportSize.height / 2 / viewport.zoom - } -} - -function getScreenRectForObject( - object: CanvasObjectRecord, - viewport: ViewportState, - viewportSize: Size -): Rect { - return getScreenRectForCanvasRect(object.position, viewport, viewportSize) -} - -function getScreenPointForCanvasPoint( - point: Point, - viewport: ViewportState, - viewportSize: Size -): Point { - const camera = createCanvasCameraForViewport(viewport, viewportSize) - - return worldToScreenPoint(camera, createWorldPointFromCanvasPoint(point)) -} - -function getScreenRectForCanvasRect(rect: Rect, viewport: ViewportState, viewportSize: Size): Rect { - const camera = createCanvasCameraForViewport(viewport, viewportSize) - const topLeft = worldToScreenPoint( - camera, - createWorldPointFromCanvasPoint({ x: rect.x, y: rect.y }) - ) - const bottomRight = worldToScreenPoint( - camera, - createWorldPointFromCanvasPoint({ - x: rect.x + rect.width, - y: rect.y + rect.height - }) - ) - - return { - x: Math.min(topLeft.x, bottomRight.x), - y: Math.min(topLeft.y, bottomRight.y), - width: Math.abs(bottomRight.x - topLeft.x), - height: Math.abs(bottomRight.y - topLeft.y) - } -} - -function getBoundsForRects(rects: readonly Rect[]): Rect | null { - if (rects.length === 0) { - return null - } - - const minX = Math.min(...rects.map((rect) => rect.x)) - const minY = Math.min(...rects.map((rect) => rect.y)) - const maxX = Math.max(...rects.map((rect) => rect.x + rect.width)) - const maxY = Math.max(...rects.map((rect) => rect.y + rect.height)) - - return { - x: minX, - y: minY, - width: maxX - minX, - height: maxY - minY - } -} - -function getNodePositionRect(node: CanvasNode): Rect { - return { - x: node.position.x, - y: node.position.y, - width: node.position.width, - height: node.position.height - } -} - function createResizeUpdatesFromOriginals(input: { nodes: readonly CanvasNode[] handle: ResizeHandle @@ -1196,64 +1057,6 @@ function createResizePreviewState(input: { } } -function getScreenLineForSnapGuide( - guide: CanvasSnapGuideSegment, - viewport: ViewportState, - viewportSize: Size -): { x1: number; y1: number; x2: number; y2: number } { - const startPoint = - guide.orientation === 'vertical' - ? { x: guide.position, y: guide.start } - : { x: guide.start, y: guide.position } - const endPoint = - guide.orientation === 'vertical' - ? { x: guide.position, y: guide.end } - : { x: guide.end, y: guide.position } - const start = getScreenPointForCanvasPoint(startPoint, viewport, viewportSize) - const end = getScreenPointForCanvasPoint(endPoint, viewport, viewportSize) - - return { - x1: start.x, - y1: start.y, - x2: end.x, - y2: end.y - } -} - -function intersectsViewport(rect: Rect, viewportSize: Size, marginPx = 320): boolean { - return ( - rect.x + rect.width >= -marginPx && - rect.y + rect.height >= -marginPx && - rect.x <= viewportSize.width + marginPx && - rect.y <= viewportSize.height + marginPx - ) -} - -function getFitViewport(input: { - rect: Rect - viewportSize: Size - minZoom: number - maxZoom: number - padding: number -}): ViewportState { - const availableWidth = Math.max(1, input.viewportSize.width - input.padding * 2) - const availableHeight = Math.max(1, input.viewportSize.height - input.padding * 2) - const zoom = clamp( - Math.min( - availableWidth / Math.max(input.rect.width, 1), - availableHeight / Math.max(input.rect.height, 1) - ), - input.minZoom, - input.maxZoom - ) - - return { - x: input.rect.x + input.rect.width / 2, - y: input.rect.y + input.rect.height / 2, - zoom - } -} - function isTextInputLikeElement(target: EventTarget | null): boolean { if (!(target instanceof HTMLElement)) { return false @@ -3169,107 +2972,35 @@ export const Canvas = forwardRef(function CanvasV3( }, [doc, selectedNodeIds]) const applyPositionUpdates = useCallback( - (updates: CanvasPositionUpdate[]): boolean => { - if (updates.length === 0) { - return false - } - - const objects = getCanvasObjectsMap(doc) - let changed = false - - doc.transact(() => { - for (const update of updates) { - const node = objects.get(update.id) - if (!node) { - continue - } - - objects.set(update.id, { - ...node, - position: { - ...node.position, - ...update.position - } - }) - changed = true - } - }) - - if (changed) { - onSceneMutation?.() - } - - return changed - }, + (updates: CanvasPositionUpdate[]): boolean => + applyCanvasSceneUpdates({ + doc, + updates, + merge: mergeCanvasNodePositionUpdate, + onSceneMutation + }), [doc, onSceneMutation] ) const applyLockUpdates = useCallback( - (updates: CanvasLockUpdate[]): boolean => { - if (updates.length === 0) { - return false - } - - const objects = getCanvasObjectsMap(doc) - let changed = false - - doc.transact(() => { - for (const update of updates) { - const node = objects.get(update.id) - if (!node) { - continue - } - - objects.set(update.id, { - ...node, - locked: update.locked - }) - changed = true - } - }) - - if (changed) { - onSceneMutation?.() - } - - return changed - }, + (updates: CanvasLockUpdate[]): boolean => + applyCanvasSceneUpdates({ + doc, + updates, + merge: mergeCanvasNodeLockUpdate, + onSceneMutation + }), [doc, onSceneMutation] ) const applyNodePropertiesUpdates = useCallback( - (updates: CanvasNodePropertiesUpdate[]): boolean => { - if (updates.length === 0) { - return false - } - - const objects = getCanvasObjectsMap(doc) - let changed = false - - doc.transact(() => { - for (const update of updates) { - const node = objects.get(update.id) - if (!node) { - continue - } - - objects.set(update.id, { - ...node, - properties: { - ...node.properties, - ...update.properties - } - }) - changed = true - } - }) - - if (changed) { - onSceneMutation?.() - } - - return changed - }, + (updates: CanvasNodePropertiesUpdate[]): boolean => + applyCanvasSceneUpdates({ + doc, + updates, + merge: mergeCanvasNodePropertiesUpdate, + onSceneMutation + }), [doc, onSceneMutation] ) diff --git a/packages/canvas/src/renderer/scene-mutations.ts b/packages/canvas/src/renderer/scene-mutations.ts new file mode 100644 index 000000000..a68df0df3 --- /dev/null +++ b/packages/canvas/src/renderer/scene-mutations.ts @@ -0,0 +1,95 @@ +/** + * Shared scene-mutation dispatcher for the canvas v3 renderer. + * + * The renderer applies many flavours of node updates (position, lock, + * properties) that all follow the same shape: run one Y.Doc transaction, + * merge each update into the existing node, and notify `onSceneMutation` + * only when something actually changed. + */ + +import type { CanvasLockUpdate, CanvasPositionUpdate } from '../selection/scene-operations' +import type { CanvasNode, CanvasNodeProperties } from '../types' +import type * as Y from 'yjs' +import { getCanvasObjectsMap } from '../scene/doc-layout' + +export type CanvasNodePropertiesUpdate = { + id: string + properties: CanvasNodeProperties +} + +export type ApplyCanvasSceneUpdatesInput = { + doc: Y.Doc + updates: readonly TUpdate[] + /** Returns the next node for an update; must not mutate the current node. */ + merge: (node: CanvasNode, update: TUpdate) => CanvasNode + onSceneMutation?: () => void +} + +/** + * Applies a batch of node updates in a single transaction. Updates whose id + * has no matching node are skipped. Returns true when at least one node + * changed, in which case `onSceneMutation` has been notified. + */ +export function applyCanvasSceneUpdates( + input: ApplyCanvasSceneUpdatesInput +): boolean { + const { doc, updates, merge, onSceneMutation } = input + + if (updates.length === 0) { + return false + } + + const objects = getCanvasObjectsMap(doc) + let changed = false + + doc.transact(() => { + for (const update of updates) { + const node = objects.get(update.id) + if (!node) { + continue + } + + objects.set(update.id, merge(node, update)) + changed = true + } + }) + + if (changed) { + onSceneMutation?.() + } + + return changed +} + +export function mergeCanvasNodePositionUpdate( + node: CanvasNode, + update: CanvasPositionUpdate +): CanvasNode { + return { + ...node, + position: { + ...node.position, + ...update.position + } + } +} + +export function mergeCanvasNodeLockUpdate(node: CanvasNode, update: CanvasLockUpdate): CanvasNode { + return { + ...node, + locked: update.locked + } +} + +export function mergeCanvasNodePropertiesUpdate( + node: CanvasNode, + update: CanvasNodePropertiesUpdate +): CanvasNode { + return { + ...node, + properties: { + ...node.properties, + ...update.properties + } + } +} diff --git a/packages/canvas/src/renderer/viewport-math.ts b/packages/canvas/src/renderer/viewport-math.ts new file mode 100644 index 000000000..1a2ce5cfe --- /dev/null +++ b/packages/canvas/src/renderer/viewport-math.ts @@ -0,0 +1,240 @@ +/** + * Pure viewport and geometry math for the canvas v3 renderer. + * + * Everything in this module is deterministic input→output with no React, DOM, + * or Y.Doc dependencies, so it can be unit tested directly. + */ + +import type { CanvasSnapGuideSegment } from '../selection/snap-guides' +import type { + CanvasConfig, + CanvasNode, + CanvasObjectAnchorPlacement, + Point, + Rect, + ViewportState +} from '../types' +import type { CanvasObjectRecord } from '@xnetjs/canvas-core' +import { + createCanvasCamera, + createWorldPointFromCanvasPoint, + worldToScreenPoint +} from '@xnetjs/canvas-core' +import { clamp } from '@xnetjs/core' + +export type Size = { + width: number + height: number +} + +export type ConnectorHandlePlacement = Extract< + CanvasObjectAnchorPlacement, + 'top' | 'right' | 'bottom' | 'left' +> + +const CANVAS_OBJECT_HIT_TARGET_PADDING = 8 +const CANVAS_OBJECT_MIN_HIT_TARGET_SIZE = 36 + +export function snapCanvasValue(value: number, gridSize: number): number { + return Math.round(value / gridSize) * gridSize +} + +export function getActiveSnapGridSize(config: CanvasConfig): number | null { + const gridSize = config.gridSize ?? 20 + + return Number.isFinite(gridSize) && gridSize > 0 ? gridSize : null +} + +export function getCanvasObjectHitTargetRect(rect: Rect): Rect { + const extraWidth = Math.max( + CANVAS_OBJECT_HIT_TARGET_PADDING * 2, + CANVAS_OBJECT_MIN_HIT_TARGET_SIZE - rect.width + ) + const extraHeight = Math.max( + CANVAS_OBJECT_HIT_TARGET_PADDING * 2, + CANVAS_OBJECT_MIN_HIT_TARGET_SIZE - rect.height + ) + + return { + x: rect.x - extraWidth / 2, + y: rect.y - extraHeight / 2, + width: rect.width + extraWidth, + height: rect.height + extraHeight + } +} + +export function pickConnectorPlacementForScreenPoint( + rect: Rect, + point: Point +): ConnectorHandlePlacement { + const centerX = rect.x + rect.width / 2 + const centerY = rect.y + rect.height / 2 + const dx = rect.width > 0 ? (point.x - centerX) / (rect.width / 2) : 0 + const dy = rect.height > 0 ? (point.y - centerY) / (rect.height / 2) : 0 + + if (Math.abs(dx) >= Math.abs(dy)) { + return dx >= 0 ? 'right' : 'left' + } + + return dy >= 0 ? 'bottom' : 'top' +} + +export function getRectAnchorPointForPlacement( + rect: Rect, + placement: ConnectorHandlePlacement +): Point { + switch (placement) { + case 'top': + return { x: rect.x + rect.width / 2, y: rect.y } + case 'right': + return { x: rect.x + rect.width, y: rect.y + rect.height / 2 } + case 'bottom': + return { x: rect.x + rect.width / 2, y: rect.y + rect.height } + case 'left': + return { x: rect.x, y: rect.y + rect.height / 2 } + } +} + +export function createCanvasCameraForViewport(viewport: ViewportState, viewportSize: Size) { + return createCanvasCamera({ + localCenter: { x: viewport.x, y: viewport.y }, + zoom: viewport.zoom, + viewportPx: viewportSize + }) +} + +export function getViewportWorldTopLeft(viewport: ViewportState, viewportSize: Size): Point { + return { + x: viewport.x - viewportSize.width / 2 / viewport.zoom, + y: viewport.y - viewportSize.height / 2 / viewport.zoom + } +} + +export function getScreenRectForObject( + object: CanvasObjectRecord, + viewport: ViewportState, + viewportSize: Size +): Rect { + return getScreenRectForCanvasRect(object.position, viewport, viewportSize) +} + +export function getScreenPointForCanvasPoint( + point: Point, + viewport: ViewportState, + viewportSize: Size +): Point { + const camera = createCanvasCameraForViewport(viewport, viewportSize) + + return worldToScreenPoint(camera, createWorldPointFromCanvasPoint(point)) +} + +export function getScreenRectForCanvasRect( + rect: Rect, + viewport: ViewportState, + viewportSize: Size +): Rect { + const camera = createCanvasCameraForViewport(viewport, viewportSize) + const topLeft = worldToScreenPoint( + camera, + createWorldPointFromCanvasPoint({ x: rect.x, y: rect.y }) + ) + const bottomRight = worldToScreenPoint( + camera, + createWorldPointFromCanvasPoint({ + x: rect.x + rect.width, + y: rect.y + rect.height + }) + ) + + return { + x: Math.min(topLeft.x, bottomRight.x), + y: Math.min(topLeft.y, bottomRight.y), + width: Math.abs(bottomRight.x - topLeft.x), + height: Math.abs(bottomRight.y - topLeft.y) + } +} + +export function getBoundsForRects(rects: readonly Rect[]): Rect | null { + if (rects.length === 0) { + return null + } + + const minX = Math.min(...rects.map((rect) => rect.x)) + const minY = Math.min(...rects.map((rect) => rect.y)) + const maxX = Math.max(...rects.map((rect) => rect.x + rect.width)) + const maxY = Math.max(...rects.map((rect) => rect.y + rect.height)) + + return { + x: minX, + y: minY, + width: maxX - minX, + height: maxY - minY + } +} + +export function getNodePositionRect(node: CanvasNode): Rect { + return { + x: node.position.x, + y: node.position.y, + width: node.position.width, + height: node.position.height + } +} + +export function getScreenLineForSnapGuide( + guide: CanvasSnapGuideSegment, + viewport: ViewportState, + viewportSize: Size +): { x1: number; y1: number; x2: number; y2: number } { + const startPoint = + guide.orientation === 'vertical' + ? { x: guide.position, y: guide.start } + : { x: guide.start, y: guide.position } + const endPoint = + guide.orientation === 'vertical' + ? { x: guide.position, y: guide.end } + : { x: guide.end, y: guide.position } + const start = getScreenPointForCanvasPoint(startPoint, viewport, viewportSize) + const end = getScreenPointForCanvasPoint(endPoint, viewport, viewportSize) + + return { + x1: start.x, + y1: start.y, + x2: end.x, + y2: end.y + } +} + +export function intersectsViewport(rect: Rect, viewportSize: Size, marginPx = 320): boolean { + return ( + rect.x + rect.width >= -marginPx && + rect.y + rect.height >= -marginPx && + rect.x <= viewportSize.width + marginPx && + rect.y <= viewportSize.height + marginPx + ) +} + +export function getFitViewport(input: { + rect: Rect + viewportSize: Size + minZoom: number + maxZoom: number + padding: number +}): ViewportState { + const availableWidth = Math.max(1, input.viewportSize.width - input.padding * 2) + const availableHeight = Math.max(1, input.viewportSize.height - input.padding * 2) + const zoom = clamp( + Math.min( + availableWidth / Math.max(input.rect.width, 1), + availableHeight / Math.max(input.rect.height, 1) + ), + input.minZoom, + input.maxZoom + ) + + return { + x: input.rect.x + input.rect.width / 2, + y: input.rect.y + input.rect.height / 2, + zoom + } +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index ef7b8c799..ae77fe08b 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -114,6 +114,15 @@ export { AUTH_ACTIONS } from './auth-types' // Shared utility helpers (dependency-free) export { clamp, clamp01, formatBytes } from './utils' + +// The ONE Last-Write-Wins ordering (protocol §L1.7; exploration 0276) +export { + compareChangeApplicationOrder, + compareLwwStamps, + lwwUpdateGuardSql, + lwwWins, + type LwwStamp +} from './lww' export { SsrfError, assertPublicUrl, validateExternalUrl } from './utils' // Core types diff --git a/packages/core/src/lww.test.ts b/packages/core/src/lww.test.ts new file mode 100644 index 000000000..cd541e644 --- /dev/null +++ b/packages/core/src/lww.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from 'vitest' +import { + compareChangeApplicationOrder, + compareLwwStamps, + lwwUpdateGuardSql, + lwwWins, + type LwwStamp +} from './lww' + +const stamp = (lamport: number, wallTime: number, author: string): LwwStamp => ({ + lamport, + wallTime, + author +}) + +/** + * The four golden LWW scenarios from the protocol conformance corpus + * (exploration 0200; `packages/runtime/src/conformance.test.ts` L1 suite). + * If these move, the committed vectors under `conformance/vectors/` must be + * regenerated — do not change the ordering rule here without a protocol bump. + */ +describe('compareLwwStamps (spec §L1.7)', () => { + it('higher lamport wins regardless of wall time', () => { + expect(lwwWins(stamp(2, 100, 'did:key:zB'), stamp(1, 999, 'did:key:zA'))).toBe(true) + expect(lwwWins(stamp(1, 999, 'did:key:zA'), stamp(2, 100, 'did:key:zB'))).toBe(false) + }) + + it('lamport tie falls back to wall time', () => { + expect(lwwWins(stamp(5, 501, 'did:key:zA'), stamp(5, 500, 'did:key:zB'))).toBe(true) + }) + + it('full tie resolved by higher author DID', () => { + expect(lwwWins(stamp(5, 500, 'did:key:zB'), stamp(5, 500, 'did:key:zA'))).toBe(true) + expect(lwwWins(stamp(5, 500, 'did:key:zA'), stamp(5, 500, 'did:key:zB'))).toBe(false) + }) + + it('author tiebreak is UTF-16 code-unit order, not locale collation', () => { + // 'A' (U+0041) < 'a' (U+0061): the lowercase DID must win. localeCompare + // would order these the other way in many locales. + expect(lwwWins(stamp(1, 1, 'did:key:zaaa'), stamp(1, 1, 'did:key:zAAA'))).toBe(true) + expect(compareLwwStamps(stamp(1, 1, 'did:key:zAAA'), stamp(1, 1, 'did:key:zaaa'))).toBeLessThan( + 0 + ) + }) + + it('identical stamps compare equal (incoming does not replace)', () => { + expect(compareLwwStamps(stamp(3, 3, 'did:key:zX'), stamp(3, 3, 'did:key:zX'))).toBe(0) + expect(lwwWins(stamp(3, 3, 'did:key:zX'), stamp(3, 3, 'did:key:zX'))).toBe(false) + }) +}) + +describe('compareChangeApplicationOrder', () => { + it('orders by lamport then author code units', () => { + const changes = [ + { lamport: 2, author: 'did:key:zA' }, + { lamport: 1, author: 'did:key:zb' }, + { lamport: 1, author: 'did:key:zB' } + ] + const sorted = [...changes].sort(compareChangeApplicationOrder) + expect(sorted).toEqual([ + { lamport: 1, author: 'did:key:zB' }, + { lamport: 1, author: 'did:key:zb' }, + { lamport: 2, author: 'did:key:zA' } + ]) + }) +}) + +describe('lwwUpdateGuardSql', () => { + it('emits the nested lamport/wallTime/author guard', () => { + const sql = lwwUpdateGuardSql({ + table: 'node_properties', + lamportColumn: 'lamport_time', + wallTimeColumn: 'updated_at', + authorColumn: 'updated_by' + }) + expect(sql).toContain('excluded.lamport_time > node_properties.lamport_time') + expect(sql).toContain('excluded.updated_at = node_properties.updated_at') + expect(sql).toContain('excluded.updated_by > node_properties.updated_by') + }) +}) diff --git a/packages/core/src/lww.ts b/packages/core/src/lww.ts new file mode 100644 index 000000000..21c893ae6 --- /dev/null +++ b/packages/core/src/lww.ts @@ -0,0 +1,75 @@ +/** + * The ONE Last-Write-Wins ordering for xNet (docs/specs/protocol §L1.7, + * exploration 0276). + * + * Per-property conflict resolution and change-log application ordering were + * previously re-implemented in three places (`NodeStore.applyChange`, the + * SQLite adapter's ON CONFLICT guards, and the hub storages' change + * ordering) — a drift class on the protocol's core convergence invariant. + * Every implementation now derives from this module, and the golden-vector + * conformance suite (exploration 0200) pins them equal. + * + * Ordering: lamport time, then wall time, then author DID compared by UTF-16 + * code units. NEVER `localeCompare` — locale collation is non-deterministic + * across ICU versions and would break CRDT convergence (see the + * `0004-tie-author-case-codeunit` golden vector). + */ + +/** The timestamp triple every LWW comparison runs on. */ +export interface LwwStamp { + lamport: number + wallTime: number + author: string +} + +/** + * Spec comparator (§L1.7): negative when `a` loses to `b`, positive when `a` + * beats `b`, zero only for identical stamps. + */ +export function compareLwwStamps(a: LwwStamp, b: LwwStamp): number { + if (a.lamport !== b.lamport) return a.lamport - b.lamport + if (a.wallTime !== b.wallTime) return a.wallTime - b.wallTime + // UTF-16 code-unit order (not localeCompare) for deterministic convergence. + return a.author < b.author ? -1 : a.author > b.author ? 1 : 0 +} + +/** Whether an incoming write replaces the existing one under LWW. */ +export function lwwWins(incoming: LwwStamp, existing: LwwStamp): boolean { + return compareLwwStamps(incoming, existing) > 0 +} + +/** + * Deterministic application order for change logs: lamport time, then author + * by code units. Used when replaying/relaying batches so every peer folds + * changes in the same sequence (matches the hub's + * `ORDER BY lamport_time ASC, lamport_author ASC`). + */ +export function compareChangeApplicationOrder( + a: { lamport: number; author: string }, + b: { lamport: number; author: string } +): number { + if (a.lamport !== b.lamport) return a.lamport - b.lamport + return a.author < b.author ? -1 : a.author > b.author ? 1 : 0 +} + +/** + * SQL `ON CONFLICT … DO UPDATE … WHERE` guard implementing {@link lwwWins} + * inside SQLite (the `excluded.` pseudo-table is the incoming row). Column + * text comparison (`>`) is byte order under SQLite's default BINARY + * collation, which matches the code-unit rule for our ASCII DID strings. + */ +export function lwwUpdateGuardSql(input: { + table: string + lamportColumn: string + wallTimeColumn: string + authorColumn: string +}): string { + const { table, lamportColumn, wallTimeColumn, authorColumn } = input + return ( + `excluded.${lamportColumn} > ${table}.${lamportColumn}\n` + + ` OR (excluded.${lamportColumn} = ${table}.${lamportColumn}\n` + + ` AND (excluded.${wallTimeColumn} > ${table}.${wallTimeColumn}\n` + + ` OR (excluded.${wallTimeColumn} = ${table}.${wallTimeColumn}\n` + + ` AND excluded.${authorColumn} > ${table}.${authorColumn})))` + ) +} diff --git a/packages/data/src/store/batch-write-orchestrator.ts b/packages/data/src/store/batch-write-orchestrator.ts new file mode 100644 index 000000000..09103ea72 --- /dev/null +++ b/packages/data/src/store/batch-write-orchestrator.ts @@ -0,0 +1,158 @@ +/** + * Deterministic-import planning and application for `NodeStore` + * (exploration 0276). + * + * Importers with stable node IDs get one Lamport timestamp and batch ID for + * the whole import: `planDeterministicNodeImport` preflights + materializes + + * signs in memory, and either the adapter applies the plan in one + * `applyNodeBatch` (fast path, chosen by the store) or + * `executeDeterministicNodeImport` persists it through the legacy + * per-collection writes inside a storage transaction. + * + * Uses the same `WriteExecutionHost` capability set as the transaction + * executors, so all write strategies share one seam into `NodeStore`. + */ + +import type { + ApplyNodeBatchResult, + DeterministicNodeImportDraft, + NodeBatchWriteTimings, + NodeChange, + NodeId, + NodePayload, + NodeState, + NodeStorageAdapter +} from './types' +import type { SchemaIRI } from '../schema/node' +import type { PendingTransactionEvent, WriteExecutionHost } from './transaction-executor' + +export type DeterministicNodeImportPlan = { + created: number + updated: number + nodes: NodeState[] + changes: NodeChange[] + events: PendingTransactionEvent[] + affectedSchemaIds: SchemaIRI[] + timings: Pick +} + +export type DeterministicNodeImportAppliedPlan = DeterministicNodeImportPlan & { + applyMs: number + storage?: ApplyNodeBatchResult +} + +const elapsedMs = (startedAt: number): number => Math.max(0, Date.now() - startedAt) + +export type DeterministicNodeImportInput = { + drafts: readonly DeterministicNodeImportDraft[] + storage: NodeStorageAdapter + lamport: number + now: number + batchId: string + batchSize: number +} + +export async function planDeterministicNodeImport( + host: WriteExecutionHost, + input: DeterministicNodeImportInput +): Promise { + const ids = input.drafts.map((draft) => draft.id) + const preflightStartedAt = Date.now() + const preflight = await host.getBatchPreflight(ids, input.storage) + const preflightMs = elapsedMs(preflightStartedAt) + const materializeStartedAt = Date.now() + const existingNodes = host.cloneNodeMap(preflight.nodesById) + const lastChanges = new Map(preflight.lastChangesByNodeId) + const nodesById = new Map(existingNodes) + const changedIds: NodeId[] = [] + const seenChangedIds = new Set() + const changes: NodeChange[] = [] + const events: PendingTransactionEvent[] = [] + let created = 0 + let updated = 0 + + for (let index = 0; index < input.drafts.length; index++) { + const draft = input.drafts[index] + const currentNode = nodesById.get(draft.id) ?? null + const previousNode = host.cloneNodeState(currentNode) + const isCreate = currentNode === null + const payload: NodePayload = { + nodeId: draft.id, + ...(isCreate ? { schemaId: draft.schemaId } : {}), + properties: draft.properties + } + const change = await host.createBatchedChangeWithParentHash( + 'node-change', + payload, + lastChanges.get(draft.id)?.hash ?? null, + input.lamport, + input.now, + input.batchId, + index, + input.batchSize + ) + const node = host.materializeNodeChange( + change, + currentNode ?? host.createInitialNodeFromChange(change, draft.schemaId) + ) + + nodesById.set(draft.id, node) + lastChanges.set(draft.id, change) + changes.push(change) + events.push({ change, result: host.cloneNodeState(node), previousNode }) + + if (!seenChangedIds.has(draft.id)) { + changedIds.push(draft.id) + seenChangedIds.add(draft.id) + } + + if (isCreate) { + created += 1 + } else { + updated += 1 + } + } + + const nodes = changedIds.flatMap((id) => { + const node = nodesById.get(id) + return node ? [node] : [] + }) + const affectedSchemaIds = Array.from(new Set(nodes.map((node) => node.schemaId))) + + return { + created, + updated, + nodes, + changes, + events, + affectedSchemaIds, + timings: { + preflightMs, + materializeMs: elapsedMs(materializeStartedAt) + } + } +} + +/** Legacy application path: per-collection writes inside a transaction. */ +export async function executeDeterministicNodeImport( + host: WriteExecutionHost, + input: DeterministicNodeImportInput & { deferIndexes: boolean } +): Promise { + const plan = await planDeterministicNodeImport(host, input) + + const applyStartedAt = Date.now() + await host.importMaterializedNodes(input.storage, plan.nodes, { + deferIndexes: input.deferIndexes + }) + await host.appendImportedChanges(input.storage, plan.changes) + await input.storage.setLastLamportTime(host.clockTime()) + + for (const node of plan.nodes) { + await host.persistEncryptedNodeSnapshot(node, input.storage) + } + + return { + ...plan, + applyMs: elapsedMs(applyStartedAt) + } +} diff --git a/packages/data/src/store/hydration.ts b/packages/data/src/store/hydration.ts new file mode 100644 index 000000000..ada713fab --- /dev/null +++ b/packages/data/src/store/hydration.ts @@ -0,0 +1,264 @@ +/** + * Node hydration for the SQLite storage adapter (exploration 0276). + * + * Two modes reconstruct `NodeState`s from SQL rows: + * + * - **Joined** — one row per (node × property), the classic EAV join shape. + * - **Aggregated** — ONE row per node via `json_group_object` (exploration + * 0264, Wave 2), collapsing the boundary payload before it leaves SQLite. + * + * Both modes share the node-shell construction and the "latest property + * write wins `updatedBy`" rule, previously duplicated inside + * `sqlite-adapter.ts`. Correctness is pinned equal across modes by the + * hydration test suite. + */ + +import type { DID } from '@xnetjs/core' +import type { SQLiteAdapter, SQLValue } from '@xnetjs/sqlite' +import type { SchemaIRI } from '../schema/node' +import type { NodeState, PropertyTimestamp } from './types' +import { + SQL_HYDRATE_ARITY_BUCKETS, + SQLITE_HYDRATE_NODE_BATCH_SIZE, + chunkItems, + padToArityBucket +} from './sql-batching' + +// ─── Row shapes ────────────────────────────────────────────────────────────── + +export interface JoinedNodePropertyRow { + id: string + schema_id: string + created_at: number + updated_at: number + created_by: string + deleted_at: number | null + property_key: string | null + value: Uint8Array | null + lamport_time: number | null + updated_by: string | null + prop_updated_at: number | null + ordinal: number | null + [key: string]: SQLValue +} + +/** One-row-per-node aggregated hydrate result (exploration 0264, Wave 2). */ +export interface AggregatedNodeRow { + id: string + schema_id: string + created_at: number + updated_at: number + created_by: string + deleted_at: number | null + ordinal: number | null + props_json: string | null + meta_json: string | null + [key: string]: SQLValue +} + +interface NodeRowShell { + id: string + schema_id: string + created_at: number + updated_at: number + created_by: string + deleted_at: number | null +} + +// ─── Row parsing ───────────────────────────────────────────────────────────── + +/** The node scaffold both hydrate modes build before merging properties in. */ +function baseNodeState(row: NodeRowShell): NodeState { + return { + id: row.id, + schemaId: row.schema_id as SchemaIRI, + properties: {}, + timestamps: {}, + deleted: row.deleted_at !== null, + deletedAt: row.deleted_at + ? { lamport: 0, author: '' as DID, wallTime: row.deleted_at } + : undefined, + createdAt: row.created_at, + createdBy: row.created_by as DID, + updatedAt: row.updated_at, + updatedBy: row.created_by as DID + } +} + +function deserializeValue(data: Uint8Array | null): unknown { + if (!data) return null + return JSON.parse(new TextDecoder().decode(data)) +} + +export function hydrateJoinedRows(rows: JoinedNodePropertyRow[]): NodeState[] { + const nodeMap = new Map() + + for (const row of rows) { + let node = nodeMap.get(row.id) + + if (!node) { + node = baseNodeState(row) + nodeMap.set(row.id, node) + } + + if (row.property_key && row.value !== null) { + node.properties[row.property_key] = deserializeValue(row.value) + node.timestamps[row.property_key] = { + lamport: row.lamport_time ?? 0, + author: (row.updated_by ?? '') as DID, + wallTime: row.prop_updated_at ?? 0 + } + if ((row.prop_updated_at ?? 0) >= node.updatedAt) { + node.updatedBy = (row.updated_by ?? node.createdBy) as DID + } + } + } + + return Array.from(nodeMap.values()) +} + +/** Parse one-row-per-node aggregated hydrate results into NodeStates. */ +export function hydrateAggregatedRows(rows: AggregatedNodeRow[]): NodeState[] { + const nodes: NodeState[] = [] + for (const row of rows) { + const node = baseNodeState(row) + node.properties = row.props_json ? (JSON.parse(row.props_json) as Record) : {} + const meta = row.meta_json + ? (JSON.parse(row.meta_json) as Record) + : {} + + const timestamps: Record = {} + for (const [key, entry] of Object.entries(meta)) { + timestamps[key] = { + lamport: entry.l ?? 0, + author: (entry.b ?? '') as DID, + wallTime: entry.w ?? 0 + } + if ((entry.w ?? 0) >= row.updated_at) { + node.updatedBy = (entry.b ?? row.created_by) as DID + } + } + node.timestamps = timestamps + + nodes.push(node) + } + return nodes +} + +// ─── Chunk queries ─────────────────────────────────────────────────────────── + +export function buildHydrateChunkQuery(ids: string[]): { sql: string; params: SQLValue[] } { + // Pad to a fixed arity bucket so repeated hydrates share ONE SQL string + // and hit the worker's prepared-statement cache (exploration 0264). NULL + // ids never satisfy the JOIN, so padding rows vanish from the result. + const padded = padToArityBucket(ids, SQL_HYDRATE_ARITY_BUCKETS) + const values = padded.map(() => '(?, ?)').join(', ') + const params: SQLValue[] = padded.flatMap((id, ordinal) => [id, ordinal]) + const sql = ` + WITH wanted(id, ordinal) AS ( + VALUES ${values} + ) + SELECT + n.id, + n.schema_id, + n.created_at, + n.updated_at, + n.created_by, + n.deleted_at, + p.property_key, + p.value, + p.lamport_time, + p.updated_by, + p.updated_at AS prop_updated_at, + wanted.ordinal + FROM wanted + JOIN nodes n ON n.id = wanted.id + LEFT JOIN node_properties p ON p.node_id = n.id + ORDER BY wanted.ordinal ASC, p.property_key ASC + ` + return { sql, params } +} + +/** + * Aggregated hydrate (exploration 0264, Wave 2): collapse the EAV rows to + * ONE row per node inside SQL via `json_group_object`, so the boundary + * ships N rows instead of N × properties. `value` is stored as JSON text + * in a BLOB — `json(CAST(… AS TEXT))` re-emits it as real JSON inside the + * aggregate (without the cast/wrap it would double-encode as a string). + */ +export function buildAggregatedHydrateChunkQuery(ids: string[]): { + sql: string + params: SQLValue[] +} { + const padded = padToArityBucket(ids, SQL_HYDRATE_ARITY_BUCKETS) + const values = padded.map(() => '(?, ?)').join(', ') + const params: SQLValue[] = padded.flatMap((id, ordinal) => [id, ordinal]) + const sql = ` + WITH wanted(id, ordinal) AS ( + VALUES ${values} + ) + SELECT + n.id, + n.schema_id, + n.created_at, + n.updated_at, + n.created_by, + n.deleted_at, + wanted.ordinal, + json_group_object(p.property_key, json(CAST(p.value AS TEXT))) + FILTER (WHERE p.property_key IS NOT NULL) AS props_json, + json_group_object( + p.property_key, + json_object('l', p.lamport_time, 'b', p.updated_by, 'w', p.updated_at) + ) FILTER (WHERE p.property_key IS NOT NULL) AS meta_json + FROM wanted + JOIN nodes n ON n.id = wanted.id + LEFT JOIN node_properties p ON p.node_id = n.id + GROUP BY n.id + ORDER BY wanted.ordinal ASC + ` + return { sql, params } +} + +// ─── Batched hydrate ───────────────────────────────────────────────────────── + +export async function hydrateNodesByIds( + db: SQLiteAdapter, + ids: string[], + aggregated: boolean +): Promise { + if (ids.length === 0) { + return [] + } + + const chunks = + ids.length > SQLITE_HYDRATE_NODE_BATCH_SIZE + ? chunkItems(ids, SQLITE_HYDRATE_NODE_BATCH_SIZE) + : [ids] + const reads = chunks.map((chunk) => + aggregated ? buildAggregatedHydrateChunkQuery(chunk) : buildHydrateChunkQuery(chunk) + ) + const parse = (rows: unknown[]): NodeState[] => + aggregated + ? hydrateAggregatedRows(rows as AggregatedNodeRow[]) + : hydrateJoinedRows(rows as JoinedNodePropertyRow[]) + + // Multi-chunk hydrates previously paid one worker round-trip per chunk; + // queryBatch sends the whole hydrate as ONE RPC and one scheduler slot + // (exploration 0263). Single chunks keep query()'s coalescing. + if (reads.length > 1 && typeof db.queryBatch === 'function') { + const results = await db.queryBatch(reads) + const nodes: NodeState[] = [] + for (const rows of results) { + nodes.push(...parse(rows)) + } + return nodes + } + + const nodes: NodeState[] = [] + for (const read of reads) { + const rows = await db.query(read.sql, read.params) + nodes.push(...parse(rows)) + } + return nodes +} diff --git a/packages/data/src/store/indexing/full-text.ts b/packages/data/src/store/indexing/full-text.ts new file mode 100644 index 000000000..6ba086461 --- /dev/null +++ b/packages/data/src/store/indexing/full-text.ts @@ -0,0 +1,140 @@ +/** + * Full-text index family (`nodes_fts` / FTS5) for the SQLite node storage + * adapter (exploration 0276). + * + * Row writes go through `updateNodeFTS`/`deleteNodeFTS` from `@xnetjs/sqlite` + * (which no-op when FTS5 is unavailable, e.g. sql.js); table existence is + * probed once per session and memoized here. + */ + +import type { SchemaIRI } from '../../schema/node' +import type { FullTextSearchQueryPlan } from '../query-compiler' +import type { NodeId, NodeState } from '../types' +import type { IndexingContext, IndexingStrategy } from './index' +import type { SQLValue } from '@xnetjs/sqlite' +import { deleteNodeFTS, extractSearchableContent, updateNodeFTS } from '@xnetjs/sqlite' +import { getNodeQuerySearchTokens, type NodeQueryDescriptor } from '../query' + +type FullTextSearchTablesState = 'unknown' | 'absent' | 'ready' + +export class FullTextIndexing implements IndexingStrategy { + private tablesState: FullTextSearchTablesState = 'unknown' + + constructor(private readonly ctx: IndexingContext) {} + + async hasTable(): Promise { + if (this.tablesState === 'ready') { + return true + } + + if (this.tablesState === 'absent') { + return false + } + + const table = await this.ctx.db.queryOne<{ name: string }>( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'nodes_fts'" + ) + + this.tablesState = table ? 'ready' : 'absent' + return table !== null + } + + async prepareQueryPlan(descriptor: NodeQueryDescriptor): Promise { + if (!descriptor.search) { + return null + } + + const tokens = getNodeQuerySearchTokens(descriptor.search) + if (tokens.length === 0) { + return null + } + + const capabilities = await this.ctx.getStorageCapabilities() + if (!capabilities.fullTextSearch || !(await this.hasTable())) { + return null + } + + return { + matchExpression: tokens.map((token) => `${token}*`).join(' AND ') + } + } + + /** + * Touched-batch refresh for one node: deleted nodes leave the index, + * live nodes re-index their searchable content. `indexProperties` is a + * scalar/spatial concern; FTS always tracks the node's content. + */ + async syncNode(node: NodeState, _indexProperties: boolean): Promise { + if (!(await this.hasTable())) { + return 0 + } + + if (node.deleted) { + await deleteNodeFTS(this.ctx.db, node.id) + return 0 + } + + const title = typeof node.properties.title === 'string' ? node.properties.title : null + const content = extractSearchableContent(node.properties) + await updateNodeFTS(this.ctx.db, node.id, title, content) + return title || content ? 1 : 0 + } + + /** + * setNode-path refresh: unconditional update from the given properties + * (updateNodeFTS itself no-ops when FTS5 is not supported). + */ + async updateNode(nodeId: NodeId, properties: NodeState['properties']): Promise { + const title = typeof properties.title === 'string' ? properties.title : null + const content = extractSearchableContent(properties) + await updateNodeFTS(this.ctx.db, nodeId, title, content) + } + + async deleteNode(nodeId: NodeId): Promise { + await deleteNodeFTS(this.ctx.db, nodeId) + } + + async rebuildForSchemas( + schemaIds: readonly SchemaIRI[], + nodesBySchemaId: ReadonlyMap, + _indexProperties: boolean + ): Promise { + if (!(await this.hasTable())) { + return + } + + for (const schemaId of schemaIds) { + const nodes = nodesBySchemaId.get(schemaId) ?? [] + for (const node of nodes) { + await this.syncNode(node, true) + } + } + } + + /** The FTS delete+insert for one node, as batch operations (exact SQL). */ + createNodeOperations(node: NodeState): Array<{ sql: string; params?: SQLValue[] }> { + const title = typeof node.properties.title === 'string' ? node.properties.title : null + const content = extractSearchableContent(node.properties) + const operations: Array<{ sql: string; params?: SQLValue[] }> = [ + { + sql: 'DELETE FROM nodes_fts WHERE node_id = ?', + params: [node.id] + } + ] + + if (title || content) { + operations.push({ + sql: 'INSERT INTO nodes_fts (node_id, title, content) VALUES (?, ?, ?)', + params: [node.id, title ?? '', content ?? ''] + }) + } + + return operations + } + + hasSearchableContent(node: NodeState): boolean { + const title = typeof node.properties.title === 'string' ? node.properties.title : null + const content = extractSearchableContent(node.properties) + return Boolean(title || content) + } +} diff --git a/packages/data/src/store/indexing/index.ts b/packages/data/src/store/indexing/index.ts new file mode 100644 index 000000000..47da9065f --- /dev/null +++ b/packages/data/src/store/indexing/index.ts @@ -0,0 +1,64 @@ +/** + * Indexing strategies for the SQLite node storage adapter (exploration 0276). + * + * Three sidecar index families accelerate `queryNodes` — scalar + * (`node_property_scalars`), full-text (`nodes_fts` / FTS5), and spatial + * (`node_spatial_*` / R*Tree). Each family follows the same lifecycle: sync + * rows for one node on write, rebuild whole schemas from materialized state, + * prepare a query plan for a descriptor, and drop/clear. The families were + * previously interleaved through `sqlite-adapter.ts`; each now lives in its + * own module behind the shared {@link IndexingStrategy} shape, taking an + * explicit {@link IndexingContext} instead of reaching into the adapter. + * + * SQL emitted by the strategies is byte-identical to the pre-split adapter: + * prepared-statement and worker statement caches key on the SQL string, so + * even whitespace-only reformatting would repartition those caches. + */ + +import type { SchemaIRI } from '../../schema/node' +import type { NodeQueryDescriptor, NodeQueryStorageCapabilitiesMetadata } from '../query' +import type { NodeId, NodeState } from '../types' +import type { SQLiteAdapter } from '@xnetjs/sqlite' + +/** The adapter capabilities an index family needs — nothing more. */ +export interface IndexingContext { + readonly db: SQLiteAdapter + /** Memoized FTS5/R*Tree capability probe (owned by the adapter). */ + getStorageCapabilities(): Promise + /** Serialize a write onto the adapter's single write lane. */ + enqueueWrite(write: () => Promise): Promise + /** List every node of a schema (deleted included) for index (re)builds. */ + listNodesForSchema(schemaId: SchemaIRI): Promise + /** Hydrate one node from materialized state. */ + getNode(id: NodeId): Promise +} + +/** + * The write-path lifecycle every index family shares. Families expose + * additional methods for their specifics (batch operation builders, table + * probes, clears, query-plan preparation) — this is the common core the + * adapter drives on every node write and schema rebuild. + */ +export interface IndexingStrategy { + /** + * Refresh this family's rows for ONE node — the hot `setNode` / touched + * batch-import path. Returns the number of index rows written. + */ + syncNode(node: NodeState, indexProperties: boolean): Promise + /** Rebuild this family's rows for whole schemas from materialized state. */ + rebuildForSchemas( + schemaIds: readonly SchemaIRI[], + nodesBySchemaId: ReadonlyMap, + indexProperties: boolean + ): Promise + /** Prepare this family's query plan for a descriptor (null: not applicable). */ + prepareQueryPlan?(descriptor: NodeQueryDescriptor): Promise +} + +export { + ScalarIndexing, + createDeleteRemovedPropertiesOperation, + deleteRemovedProperties +} from './scalar' +export { FullTextIndexing } from './full-text' +export { SpatialIndexing } from './spatial' diff --git a/packages/data/src/store/indexing/scalar.ts b/packages/data/src/store/indexing/scalar.ts new file mode 100644 index 000000000..d19ac664d --- /dev/null +++ b/packages/data/src/store/indexing/scalar.ts @@ -0,0 +1,204 @@ +/** + * Scalar index family (`node_property_scalars`) for the SQLite node storage + * adapter (exploration 0276). + * + * The scalar sidecar denormalizes every indexable property value into typed + * columns (`value_text`/`value_number`/`value_boolean` + hash) so the query + * compiler can push predicates and sorts into SQL. Rows are replaced + * wholesale per node (delete + insert), keeping the sidecar in lockstep with + * the LWW-materialized `node_properties` state. + */ + +import type { SchemaIRI } from '../../schema/node' +import type { NodeState } from '../types' +import type { IndexingContext, IndexingStrategy } from './index' +import type { SQLiteAdapter, SQLValue } from '@xnetjs/sqlite' +import { toScalarIndexValue } from '../query-compiler' + +/** + * Delete `node_properties` rows for keys no longer present on the node. + * Not an index write — it maintains the canonical EAV table — but it fans + * out from the same per-node write step as the scalar sidecar sync, so it + * lives beside it. + */ +export async function deleteRemovedProperties(db: SQLiteAdapter, node: NodeState): Promise { + const keys = Object.keys(node.properties) + + if (keys.length === 0) { + await db.run('DELETE FROM node_properties WHERE node_id = ?', [node.id]) + return + } + + const placeholders = keys.map(() => '?').join(', ') + await db.run( + `DELETE FROM node_properties WHERE node_id = ? AND property_key NOT IN (${placeholders})`, + [node.id, ...keys] + ) +} + +/** Batch-operation form of {@link deleteRemovedProperties}. */ +export function createDeleteRemovedPropertiesOperation(node: NodeState): { + sql: string + params?: SQLValue[] +} { + const keys = Object.keys(node.properties) + + if (keys.length === 0) { + return { + sql: 'DELETE FROM node_properties WHERE node_id = ?', + params: [node.id] + } + } + + return { + // Indentation is part of the SQL string — kept byte-identical to the + // pre-split adapter so batch statement-cache keys do not repartition. + sql: `DELETE FROM node_properties + WHERE node_id = ? AND property_key NOT IN (${keys.map(() => '?').join(', ')})`, + params: [node.id, ...keys] + } +} + +export class ScalarIndexing implements IndexingStrategy { + constructor(private readonly ctx: IndexingContext) {} + + async syncNode(node: NodeState, indexProperties: boolean): Promise { + await this.ctx.db.run('DELETE FROM node_property_scalars WHERE node_id = ?', [node.id]) + + if (!indexProperties) { + return 0 + } + + let rowsWritten = 0 + for (const [key, value] of Object.entries(node.properties)) { + const timestamp = node.timestamps[key] + const scalar = toScalarIndexValue(value) + if (!timestamp || !scalar) continue + + await this.ctx.db.run( + `INSERT INTO node_property_scalars + ( + node_id, + schema_id, + property_key, + value_type, + value_text, + value_number, + value_boolean, + value_hash, + updated_at, + lamport_time + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + node.id, + node.schemaId, + key, + scalar.valueType, + scalar.valueText, + scalar.valueNumber, + scalar.valueBoolean, + scalar.valueHash, + timestamp.wallTime, + timestamp.lamport + ] + ) + rowsWritten += 1 + } + + return rowsWritten + } + + async rebuildForSchemas( + schemaIds: readonly SchemaIRI[], + nodesBySchemaId: ReadonlyMap, + indexProperties: boolean + ): Promise { + for (const schemaId of schemaIds) { + await this.ctx.db.run('DELETE FROM node_property_scalars WHERE schema_id = ?', [schemaId]) + + if (!indexProperties) { + continue + } + + const nodes = nodesBySchemaId.get(schemaId) ?? [] + for (const node of nodes) { + await this.syncNode(node, true) + } + } + } + + /** + * Rebuild the whole scalar sidecar from materialized `node_properties`. + * Scan/write loop only — the adapter wraps it in its write lane and a + * transaction. + */ + async rebuildAll(): Promise<{ nodesScanned: number; scalarRowsWritten: number }> { + await this.ctx.db.run('DELETE FROM node_property_scalars') + const rows = await this.ctx.db.query<{ id: string }>('SELECT id FROM nodes ORDER BY id ASC') + let scalarRowsWritten = 0 + + for (const row of rows) { + const node = await this.ctx.getNode(row.id) + if (!node) continue + + scalarRowsWritten += await this.syncNode(node, true) + } + + return { + nodesScanned: rows.length, + scalarRowsWritten + } + } + + /** The scalar-row inserts for one node, as batch operations (exact SQL). */ + createNodeOperations(node: NodeState): Array<{ sql: string; params?: SQLValue[] }> { + return Object.entries(node.properties).flatMap(([key, value]) => { + const timestamp = node.timestamps[key] + const scalar = toScalarIndexValue(value) + if (!timestamp || !scalar) return [] + + return [ + { + sql: `INSERT INTO node_property_scalars + ( + node_id, + schema_id, + property_key, + value_type, + value_text, + value_number, + value_boolean, + value_hash, + updated_at, + lamport_time + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + params: [ + node.id, + node.schemaId, + key, + scalar.valueType, + scalar.valueText, + scalar.valueNumber, + scalar.valueBoolean, + scalar.valueHash, + timestamp.wallTime, + timestamp.lamport + ] + } + ] + }) + } + + /** How many scalar rows {@link syncNode}/{@link createNodeOperations} would write. */ + countIndexRowsForNode(node: NodeState): number { + return Object.values(node.properties).filter((value) => toScalarIndexValue(value) !== null) + .length + } + + /** Drop every scalar index row (the adapter's `clear()`). */ + async clear(): Promise { + await this.ctx.db.run('DELETE FROM node_property_scalars') + } +} diff --git a/packages/data/src/store/indexing/spatial.ts b/packages/data/src/store/indexing/spatial.ts new file mode 100644 index 000000000..917d259c1 --- /dev/null +++ b/packages/data/src/store/indexing/spatial.ts @@ -0,0 +1,408 @@ +/** + * Spatial index family (`node_spatial_indexes` / `node_spatial_ids` / + * `node_spatial_rtree`) for the SQLite node storage adapter (exploration + * 0276). + * + * Spatial indexes are created lazily, per (schema, field-mapping) config: + * the first spatial query for a shape registers a config row and back-fills + * the R*Tree from materialized state. Tables are created on demand (the DDL + * is idempotent) and existence is memoized per session. + */ + +import type { SchemaIRI } from '../../schema/node' +import type { NodeQueryDescriptor, NodeQuerySpatialFilter } from '../query' +import type { SpatialBoundingBox, SpatialQueryPlan } from '../query-compiler' +import type { NodeId, NodeState } from '../types' +import type { IndexingContext, IndexingStrategy } from './index' +import type { SQLValue } from '@xnetjs/sqlite' +import { hashScalarValue, stringifyStable } from '../query-compiler' +import { SQLITE_BIND_PARAMETER_BATCH_SIZE, chunkItems } from '../sql-batching' + +type SpatialTablesState = 'unknown' | 'absent' | 'ready' + +export interface SpatialIndexConfigRow { + spatial_key: string + schema_id: string + x_key: string + y_key: string + width_key: string | null + height_key: string | null + [key: string]: SQLValue +} + +export class SpatialIndexing implements IndexingStrategy { + private tablesState: SpatialTablesState = 'unknown' + + constructor(private readonly ctx: IndexingContext) {} + + async prepareQueryPlan(descriptor: NodeQueryDescriptor): Promise { + if (!descriptor.spatial) { + return null + } + + const capabilities = await this.ctx.getStorageCapabilities() + if (!capabilities.rtree) { + return null + } + + await this.ensureTables() + const spatialKey = this.buildIndexKey(descriptor.schemaId, descriptor.spatial) + const existing = await this.ctx.db.queryOne( + `SELECT spatial_key, schema_id, x_key, y_key, width_key, height_key + FROM node_spatial_indexes + WHERE spatial_key = ?`, + [spatialKey] + ) + + if (!existing) { + await this.createIndexConfig(descriptor.schemaId, descriptor.spatial, spatialKey) + } + + return { + spatialKey, + bounds: this.getSearchBounds(descriptor.spatial) + } + } + + private async ensureTables(): Promise { + const capabilities = await this.ctx.getStorageCapabilities() + if (!capabilities.rtree) { + this.tablesState = 'absent' + return + } + + await this.ctx.db.exec(` +CREATE TABLE IF NOT EXISTS node_spatial_indexes ( + spatial_key TEXT PRIMARY KEY, + schema_id TEXT NOT NULL, + x_key TEXT NOT NULL, + y_key TEXT NOT NULL, + width_key TEXT, + height_key TEXT, + created_at INTEGER NOT NULL, + last_built_at INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS node_spatial_ids ( + spatial_id INTEGER PRIMARY KEY, + spatial_key TEXT NOT NULL, + node_id TEXT NOT NULL, + schema_id TEXT NOT NULL, + UNIQUE(spatial_key, node_id), + FOREIGN KEY (spatial_key) REFERENCES node_spatial_indexes(spatial_key) ON DELETE CASCADE, + FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE +); + +CREATE VIRTUAL TABLE IF NOT EXISTS node_spatial_rtree USING rtree( + spatial_id, + min_x, + max_x, + min_y, + max_y +); + +CREATE INDEX IF NOT EXISTS idx_node_spatial_ids_schema + ON node_spatial_ids(schema_id, spatial_key, node_id); +`) + this.tablesState = 'ready' + } + + async hasTables(): Promise { + if (this.tablesState === 'ready') { + return true + } + + if (this.tablesState === 'absent') { + return false + } + + const table = await this.ctx.db.queryOne<{ count: number }>( + `SELECT COUNT(*) as count + FROM sqlite_master + WHERE type IN ('table', 'virtual table') + AND name IN ('node_spatial_ids', 'node_spatial_rtree')` + ) + const ready = Number(table?.count ?? 0) === 2 + this.tablesState = ready ? 'ready' : 'absent' + + return ready + } + + private async createIndexConfig( + schemaId: SchemaIRI, + spatial: NodeQuerySpatialFilter, + spatialKey: string + ): Promise { + const fields = this.getFieldConfig(spatial) + const now = Date.now() + + await this.ctx.enqueueWrite(async () => { + await this.ctx.db.beginTransaction() + try { + await this.ctx.db.run( + `INSERT INTO node_spatial_indexes + (spatial_key, schema_id, x_key, y_key, width_key, height_key, created_at, last_built_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + [ + spatialKey, + schemaId, + fields.xKey, + fields.yKey, + fields.widthKey, + fields.heightKey, + now, + now + ] + ) + + const nodes = await this.ctx.listNodesForSchema(schemaId) + const config: SpatialIndexConfigRow = { + spatial_key: spatialKey, + schema_id: schemaId, + x_key: fields.xKey, + y_key: fields.yKey, + width_key: fields.widthKey, + height_key: fields.heightKey + } + + for (const node of nodes) { + await this.replaceRowForConfig(node, config, true) + } + + await this.ctx.db.commit() + } catch (err) { + await this.ctx.db.rollback() + throw err + } + }) + } + + async syncNode(node: NodeState, indexProperties: boolean): Promise { + if (!(await this.hasTables())) { + return 0 + } + + const configs = await this.ctx.db.query( + `SELECT spatial_key, schema_id, x_key, y_key, width_key, height_key + FROM node_spatial_indexes + WHERE schema_id = ?`, + [node.schemaId] + ) + + let rowsWritten = 0 + for (const config of configs) { + rowsWritten += await this.replaceRowForConfig(node, config, indexProperties) + } + + return rowsWritten + } + + private async replaceRowForConfig( + node: NodeState, + config: SpatialIndexConfigRow, + indexProperties: boolean + ): Promise { + await this.deleteRow(config.spatial_key, node.id) + + if (!indexProperties) { + return 0 + } + + const bounds = this.getNodeBounds(node, config) + if (!bounds) { + return 0 + } + + const result = await this.ctx.db.run( + `INSERT INTO node_spatial_ids (spatial_key, node_id, schema_id) + VALUES (?, ?, ?)`, + [config.spatial_key, node.id, node.schemaId] + ) + const spatialId = Number(result.lastInsertRowid) + await this.ctx.db.run( + `INSERT INTO node_spatial_rtree (spatial_id, min_x, max_x, min_y, max_y) + VALUES (?, ?, ?, ?, ?)`, + [spatialId, bounds.minX, bounds.maxX, bounds.minY, bounds.maxY] + ) + return 1 + } + + async deleteNode(nodeId: NodeId): Promise { + if (!(await this.hasTables())) { + return + } + + const rows = await this.ctx.db.query<{ spatial_key: string }>( + `SELECT spatial_key + FROM node_spatial_ids + WHERE node_id = ?`, + [nodeId] + ) + + for (const row of rows) { + await this.deleteRow(row.spatial_key, nodeId) + } + } + + private async deleteRow(spatialKey: string, nodeId: NodeId): Promise { + const existing = await this.ctx.db.queryOne<{ spatial_id: number }>( + `SELECT spatial_id + FROM node_spatial_ids + WHERE spatial_key = ? AND node_id = ?`, + [spatialKey, nodeId] + ) + + if (!existing) { + return + } + + await this.ctx.db.run('DELETE FROM node_spatial_rtree WHERE spatial_id = ?', [ + existing.spatial_id + ]) + await this.ctx.db.run('DELETE FROM node_spatial_ids WHERE spatial_id = ?', [ + existing.spatial_id + ]) + } + + private async clearRowsForConfig(spatialKey: string): Promise { + const rows = await this.ctx.db.query<{ spatial_id: number }>( + `SELECT spatial_id + FROM node_spatial_ids + WHERE spatial_key = ?`, + [spatialKey] + ) + + for (const batch of chunkItems(rows, SQLITE_BIND_PARAMETER_BATCH_SIZE)) { + const placeholders = batch.map(() => '?').join(', ') + await this.ctx.db.run( + `DELETE FROM node_spatial_rtree WHERE spatial_id IN (${placeholders})`, + [...batch.map((row) => row.spatial_id)] + ) + } + + await this.ctx.db.run('DELETE FROM node_spatial_ids WHERE spatial_key = ?', [spatialKey]) + } + + /** Drop every spatial row AND config (the adapter's `clear()`). */ + async clear(): Promise { + if (!(await this.hasTables())) { + return + } + + await this.ctx.db.run('DELETE FROM node_spatial_rtree') + await this.ctx.db.run('DELETE FROM node_spatial_ids') + await this.ctx.db.run('DELETE FROM node_spatial_indexes') + } + + async rebuildForSchemas( + schemaIds: readonly SchemaIRI[], + nodesBySchemaId: ReadonlyMap, + indexProperties: boolean + ): Promise { + if (!(await this.hasTables())) { + return + } + + for (const schemaId of schemaIds) { + const configs = await this.ctx.db.query( + `SELECT spatial_key, schema_id, x_key, y_key, width_key, height_key + FROM node_spatial_indexes + WHERE schema_id = ?`, + [schemaId] + ) + + for (const config of configs) { + await this.clearRowsForConfig(config.spatial_key) + + if (!indexProperties) { + continue + } + + const nodes = nodesBySchemaId.get(schemaId) ?? [] + for (const node of nodes) { + await this.replaceRowForConfig(node, config, true) + } + } + } + } + + private buildIndexKey(schemaId: SchemaIRI, spatial: NodeQuerySpatialFilter): string { + const fields = this.getFieldConfig(spatial) + return hashScalarValue( + stringifyStable({ + schemaId, + x: fields.xKey, + y: fields.yKey, + width: fields.widthKey, + height: fields.heightKey + }) + ) + } + + private getFieldConfig(spatial: NodeQuerySpatialFilter): { + xKey: string + yKey: string + widthKey: string | null + heightKey: string | null + } { + return { + xKey: spatial.fields.x, + yKey: spatial.fields.y, + widthKey: spatial.kind === 'window' ? (spatial.fields.width ?? null) : null, + heightKey: spatial.kind === 'window' ? (spatial.fields.height ?? null) : null + } + } + + private getSearchBounds(spatial: NodeQuerySpatialFilter): SpatialBoundingBox { + if (spatial.kind === 'radius') { + const radius = Math.abs(spatial.radius) + return { + minX: spatial.center.x - radius, + maxX: spatial.center.x + radius, + minY: spatial.center.y - radius, + maxY: spatial.center.y + radius + } + } + + const overscan = spatial.overscan ?? 0 + const left = spatial.rect.x - overscan + const right = spatial.rect.x + spatial.rect.width + overscan + const top = spatial.rect.y - overscan + const bottom = spatial.rect.y + spatial.rect.height + overscan + + return { + minX: Math.min(left, right), + maxX: Math.max(left, right), + minY: Math.min(top, bottom), + maxY: Math.max(top, bottom) + } + } + + private getNodeBounds(node: NodeState, config: SpatialIndexConfigRow): SpatialBoundingBox | null { + const x = this.getFiniteNumberProperty(node, config.x_key) + const y = this.getFiniteNumberProperty(node, config.y_key) + + if (x === null || y === null) { + return null + } + + const width = this.getFiniteNumberProperty(node, config.width_key) ?? 0 + const height = this.getFiniteNumberProperty(node, config.height_key) ?? 0 + + return { + minX: Math.min(x, x + width), + maxX: Math.max(x, x + width), + minY: Math.min(y, y + height), + maxY: Math.max(y, y + height) + } + } + + private getFiniteNumberProperty(node: NodeState, key: string | null): number | null { + if (!key) { + return null + } + + const value = node.properties[key] + return typeof value === 'number' && Number.isFinite(value) ? value : null + } +} diff --git a/packages/data/src/store/lww-conformance.test.ts b/packages/data/src/store/lww-conformance.test.ts new file mode 100644 index 000000000..c0e8e7834 --- /dev/null +++ b/packages/data/src/store/lww-conformance.test.ts @@ -0,0 +1,168 @@ +/** + * Cross-implementation LWW conformance (explorations 0200/0276). + * + * The protocol's Last-Write-Wins ordering now lives in ONE place — + * `@xnetjs/core`'s `compareLwwStamps` — and every materializer must fold a + * change set to exactly the state that ordering predicts. This suite replays + * concurrent, signed change sets in shuffled orders through BOTH storage + * implementations (the in-memory adapter and the SQLite adapter, whose LWW + * lives in SQL `ON CONFLICT` guards) and asserts each result equals the + * shared-comparator oracle. The golden-vector corpus derives the same rule in + * `packages/runtime/src/conformance.test.ts`; the hub's change ordering is + * pinned in `packages/hub/test/lww-order.test.ts`. + */ +import type { DID } from '@xnetjs/core' +import { compareLwwStamps, type LwwStamp } from '@xnetjs/core' +import { generateSigningKeyPair } from '@xnetjs/crypto' +import { createDID } from '@xnetjs/identity' +import { createMemorySQLiteAdapter } from '@xnetjs/sqlite/memory' +import { describe, expect, it } from 'vitest' +import type { SchemaIRI } from '../schema/node' +import type { NodeChange, NodeStorageAdapter } from './types' +import { MemoryNodeStorageAdapter } from './memory-adapter' +import { SQLiteNodeStorageAdapter } from './sqlite-adapter' +import { NodeStore } from './store' + +const SCHEMA: SchemaIRI = 'xnet://xnet.fyi/Task' +const NODE_ID = 'conformance-node' +const SHUFFLE_SEEDS = [1, 42, 99, 314159] + +function makeStore(storage: NodeStorageAdapter = new MemoryNodeStorageAdapter()): NodeStore { + const keyPair = generateSigningKeyPair() + const did = createDID(keyPair.publicKey) as DID + return new NodeStore({ + storage, + authorDID: did, + signingKey: keyPair.privateKey + }) +} + +// Deterministic PRNG (mulberry32) so a failing shuffle reproduces from its seed. +function mulberry32(seed: number): () => number { + let a = seed >>> 0 + return () => { + a = (a + 0x6d2b79f5) | 0 + let t = Math.imul(a ^ (a >>> 15), 1 | a) + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t + return ((t ^ (t >>> 14)) >>> 0) / 4294967296 + } +} + +function shuffle(items: readonly T[], seed: number): T[] { + const rng = mulberry32(seed) + const out = items.slice() + for (let i = out.length - 1; i > 0; i -= 1) { + const j = Math.floor(rng() * (i + 1)) + ;[out[i], out[j]] = [out[j], out[i]] + } + return out +} + +function dedupeByHash(changes: readonly NodeChange[]): NodeChange[] { + const seen = new Map() + for (const c of changes) seen.set(c.hash, c) + return [...seen.values()] +} + +/** + * The oracle: fold properties using the SHARED comparator from + * `@xnetjs/core`. Any implementation that converges somewhere else disagrees + * with the protocol ordering itself, not merely with a sibling. + */ +function oracleFold(changes: readonly NodeChange[]): Record { + const properties: Record = {} + const stamps: Record = {} + for (const change of changes) { + const stamp: LwwStamp = { + lamport: change.lamport, + wallTime: change.wallTime, + author: change.authorDID + } + for (const [key, value] of Object.entries(change.payload.properties ?? {})) { + const current = stamps[key] + if (!current || compareLwwStamps(stamp, current) > 0) { + properties[key] = value + stamps[key] = stamp + } + } + } + return properties +} + +/** Concurrent writers: same Lamport time, one shared key + disjoint keys. */ +async function buildConcurrentChangeSet(peers: number): Promise { + const creator = makeStore() + await creator.create({ + id: NODE_ID, + schemaId: SCHEMA, + properties: { title: 'origin', status: 'open' } + }) + const createChanges = await creator.getAllChanges() + + const writers = [creator] + for (let i = 0; i < peers; i += 1) { + const peer = makeStore() + await peer.applyRemoteChanges(createChanges.slice()) + writers.push(peer) + } + + await creator.update(NODE_ID, { properties: { title: 'from-0', only0: 0 } }) + for (let i = 1; i <= peers; i += 1) { + await writers[i].update(NODE_ID, { + properties: { title: `from-${i}`, [`only${i}`]: i } + }) + } + + const all: NodeChange[] = [] + for (const w of writers) all.push(...(await w.getAllChanges())) + return dedupeByHash(all) +} + +async function materializeVia( + makeStorage: () => Promise | NodeStorageAdapter, + changes: readonly NodeChange[] +): Promise> { + const store = makeStore(await makeStorage()) + await store.applyRemoteChanges(changes.slice()) + const node = await store.get(NODE_ID) + if (!node) throw new Error('node did not materialize') + return node.properties +} + +describe('LWW conformance across implementations (0200 golden ordering)', () => { + it('memory adapter, SQLite adapter, and the shared oracle agree for every order', async () => { + const changes = await buildConcurrentChangeSet(2) + const oracle = oracleFold(changes) + + for (const seed of SHUFFLE_SEEDS) { + const order = shuffle(changes, seed) + + const viaMemory = await materializeVia(() => new MemoryNodeStorageAdapter(), order) + expect(viaMemory, `memory adapter diverged from oracle (seed ${seed})`).toEqual(oracle) + + const viaSqlite = await materializeVia(async () => { + const db = await createMemorySQLiteAdapter() + return new SQLiteNodeStorageAdapter(db) + }, order) + expect(viaSqlite, `sqlite adapter diverged from oracle (seed ${seed})`).toEqual(oracle) + } + }) + + it('author tiebreak converges by code units in both adapters (golden vector 0004)', async () => { + // Two writers with identical lamport+wallTime force the author tiebreak. + // We can't choose DIDs (they're derived from keys), so instead assert the + // adapters agree with the oracle — which resolves the tie via the shared + // code-unit comparator — over many independent identity draws. + for (let round = 0; round < 5; round += 1) { + const changes = await buildConcurrentChangeSet(1) + const oracle = oracleFold(changes) + const viaMemory = await materializeVia(() => new MemoryNodeStorageAdapter(), changes) + const viaSqlite = await materializeVia(async () => { + const db = await createMemorySQLiteAdapter() + return new SQLiteNodeStorageAdapter(db) + }, changes) + expect(viaMemory).toEqual(oracle) + expect(viaSqlite).toEqual(oracle) + } + }) +}) diff --git a/packages/data/src/store/query-compiler.test.ts b/packages/data/src/store/query-compiler.test.ts new file mode 100644 index 000000000..22eee16d9 --- /dev/null +++ b/packages/data/src/store/query-compiler.test.ts @@ -0,0 +1,149 @@ +import type { NodeQueryDescriptor } from './query' +import type { SchemaIRI } from '../schema/node' +import { describe, expect, it } from 'vitest' +import { + QueryCompiler, + buildSqlOrderBy, + hashScalarValue, + quoteSqlLiteral, + stringifyStable, + toScalarIndexValue, + type QueryCompilerFlags +} from './query-compiler' + +const SCHEMA = 'xnet://xnet.fyi/Task@1.0.0' as SchemaIRI + +function compiler(flags: Partial = {}) { + return new QueryCompiler(() => ({ + adaptiveIndexingEnabled: false, + aggregatedHydration: true, + ...flags + })) +} + +function descriptor(overrides: Partial = {}): NodeQueryDescriptor { + return { schemaId: SCHEMA, includeDeleted: false, ...overrides } +} + +describe('toScalarIndexValue', () => { + it('maps primitives to typed scalar values', () => { + expect(toScalarIndexValue('todo')).toMatchObject({ valueType: 'text', valueText: 'todo' }) + expect(toScalarIndexValue(3)).toMatchObject({ valueType: 'number', valueNumber: 3 }) + expect(toScalarIndexValue(true)).toMatchObject({ valueType: 'boolean', valueBoolean: 1 }) + expect(toScalarIndexValue(null)).toMatchObject({ valueType: 'null', valueHash: 'null' }) + }) + + it('rejects non-scalar values', () => { + expect(toScalarIndexValue({ nested: true })).toBeNull() + expect(toScalarIndexValue([1, 2])).toBeNull() + expect(toScalarIndexValue(Number.NaN)).toBeNull() + expect(toScalarIndexValue(undefined)).toBeNull() + }) +}) + +describe('pure SQL helpers', () => { + it('quoteSqlLiteral escapes single quotes', () => { + expect(quoteSqlLiteral("it's")).toBe("'it''s'") + }) + + it('hashScalarValue is stable', () => { + expect(hashScalarValue('abc')).toBe(hashScalarValue('abc')) + expect(hashScalarValue('abc')).not.toBe(hashScalarValue('abd')) + }) + + it('stringifyStable sorts keys and drops undefined', () => { + expect(stringifyStable({ b: 1, a: 2, c: undefined })).toBe('{"a":2,"b":1}') + }) + + it('buildSqlOrderBy defaults and honours system keys only', () => { + expect(buildSqlOrderBy()).toBe('n.updated_at DESC, n.id ASC') + expect(buildSqlOrderBy({ createdAt: 'asc' })).toBe('n.created_at ASC, n.id ASC') + expect(buildSqlOrderBy({ priority: 'asc' })).toBe('n.updated_at DESC, n.id ASC') + }) +}) + +describe('QueryCompiler.compile', () => { + it('compiles a nodeId lookup with pushed-down pagination semantics', () => { + const compiled = compiler().compile(descriptor({ nodeId: 'node-1' })) + expect(compiled).not.toBeNull() + expect(compiled?.sql).toContain('n.id = ?') + expect(compiled?.params).toEqual([SCHEMA, 'node-1']) + expect(compiled?.postFilterReason).toBe('verified-in-js') + }) + + it('returns null when a where value cannot be scalar-indexed', () => { + expect(compiler().compile(descriptor({ where: { tags: ['a'] } }))).toBeNull() + }) + + it('joins node_property_scalars per scalar where entry', () => { + const compiled = compiler().compile(descriptor({ where: { status: 'todo', priority: 2 } })) + expect(compiled?.sql).toContain('JOIN node_property_scalars p0') + expect(compiled?.sql).toContain('JOIN node_property_scalars p1') + expect(compiled?.sql).toContain('p0.value_text = ?') + expect(compiled?.sql).toContain('p1.value_number = ?') + expect(compiled?.adaptiveIndexHints.map((hint) => hint.propertyKey)).toEqual([ + 'status', + 'priority' + ]) + }) + + it('pushes pagination down and emits a fused query with exact count', () => { + const compiled = compiler().compile( + descriptor({ where: { status: 'todo' }, limit: 10, offset: 5, count: 'exact' }) + ) + expect(compiled?.sqlPagination).toBe(true) + expect(compiled?.sql).toContain('LIMIT ? OFFSET ?') + expect(compiled?.postFilterReason).toBe('pagination-pushed-down') + expect(compiled?.postFilterDescriptor.limit).toBeUndefined() + expect(compiled?.fused?.includesExactCount).toBe(true) + expect(compiled?.fused?.sql).toContain('COUNT(*) OVER () AS total_count') + expect(compiled?.fused?.sql).toContain('json_group_object') + }) + + it('emits row-multiplied fused SQL when aggregated hydration is off', () => { + const compiled = compiler({ aggregatedHydration: false }).compile( + descriptor({ where: { status: 'todo' }, limit: 10 }) + ) + expect(compiled?.fused?.sql).not.toContain('json_group_object') + expect(compiled?.fused?.sql).toContain('p.property_key, p.value, p.lamport_time') + }) + + it('does not push pagination down for cursor reads', () => { + const compiled = compiler().compile( + descriptor({ where: { status: 'todo' }, limit: 10, after: 'node-9' }) + ) + expect(compiled?.sqlPagination).toBe(false) + expect(compiled?.fused).toBeUndefined() + }) + + it('gates property-sort pushdown behind the adaptive-indexing flag', () => { + const sorted = descriptor({ orderBy: { priority: 'asc' }, limit: 10 }) + const gated = compiler({ adaptiveIndexingEnabled: false }).compile(sorted) + expect(gated?.sqlPagination).toBe(false) + expect(gated?.sql).not.toContain('sortp') + + const pushed = compiler({ adaptiveIndexingEnabled: true }).compile(sorted) + expect(pushed?.sqlPagination).toBe(true) + expect(pushed?.sql).toContain('LEFT JOIN node_property_scalars sortp') + expect(pushed?.sql).toContain('(sortp.node_id IS NULL) ASC') + }) + + it('labels FTS and spatial candidate plans as JS-verified', () => { + const fts = compiler().compile(descriptor({ search: { text: 'hello' } }), null, { + matchExpression: 'hello' + }) + expect(fts?.sql).toContain('nodes_fts MATCH ?') + expect(fts?.postFilterReason).toBe('fts-verified-in-js') + + const spatial = compiler().compile( + descriptor({ + spatial: { viewport: { minX: 0, maxX: 1, minY: 0, maxY: 1 } } as never + }), + { spatialKey: 'position', bounds: { minX: 0, maxX: 1, minY: 0, maxY: 1 } }, + null + ) + expect(spatial?.sql).toContain('node_spatial_rtree') + expect(spatial?.postFilterReason).toBe('spatial-rtree-verified-in-js') + expect(spatial?.spatialIndexKey).toBe('position') + }) +}) diff --git a/packages/data/src/store/query-compiler.ts b/packages/data/src/store/query-compiler.ts new file mode 100644 index 000000000..215a5d5a3 --- /dev/null +++ b/packages/data/src/store/query-compiler.ts @@ -0,0 +1,515 @@ +/** + * Query→SQL compiler for the SQLite node storage adapter. + * + * Turns a `NodeQueryDescriptor` into candidate-select SQL (and, for fully + * pushed-down descriptors, a fused single-statement candidate+hydrate query — + * exploration 0264, Wave 1). Extracted from `sqlite-adapter.ts` so compilation + * is testable and evolvable independently of storage (exploration 0276). + * + * The compiler is pure: behaviour flags (adaptive indexing, aggregated + * hydration) are explicit inputs read per-compile via the flags getter, never + * ambient adapter state. + */ + +import type { SQLValue } from '@xnetjs/sqlite' +import { withoutNodeQueryPagination, type NodeQueryDescriptor, type SortDirection } from './query' + +// ─── Scalar index values ───────────────────────────────────────────────────── + +export type ScalarValueType = 'text' | 'number' | 'boolean' | 'null' + +export interface ScalarIndexValue { + valueType: ScalarValueType + valueText: string | null + valueNumber: number | null + valueBoolean: number | null + valueHash: string +} + +export interface AdaptiveIndexHint { + propertyKey: string + scalar: ScalarIndexValue +} + +// ─── Accelerator plans ─────────────────────────────────────────────────────── + +export interface SpatialBoundingBox { + minX: number + maxX: number + minY: number + maxY: number +} + +export interface SpatialQueryPlan { + spatialKey: string + bounds: SpatialBoundingBox +} + +export interface FullTextSearchQueryPlan { + matchExpression: string +} + +// ─── Compiled output ───────────────────────────────────────────────────────── + +export interface CompiledNodeQuery { + sql: string + params: SQLValue[] + postFilterDescriptor: NodeQueryDescriptor + postFilterReason: string + sqlPagination: boolean + adaptiveIndexHints: AdaptiveIndexHint[] + spatialIndexKey?: string + fullTextSearchQuery?: string + /** + * Single-statement candidate+hydrate query (exploration 0264, Wave 1). + * Present only for fully-pushed-down descriptors (`sqlPagination`): the + * candidate select becomes a CTE feeding the property hydrate join, so a + * cold query costs ONE worker round-trip instead of id-select + hydrate. + * When the descriptor asks for `count: 'exact'`, a `COUNT(*) OVER ()` + * window inside the CTE folds the total in (no separate COUNT RPC). + */ + fused?: { + sql: string + params: SQLValue[] + includesExactCount: boolean + } +} + +/** Behaviour flags the compiler must not read from ambient adapter state. */ +export interface QueryCompilerFlags { + adaptiveIndexingEnabled: boolean + aggregatedHydration: boolean +} + +// ─── Pure helpers (shared with the adapter's indexing/telemetry paths) ────── + +export function toScalarIndexValue(value: unknown): ScalarIndexValue | null { + if (value === null) { + return { + valueType: 'null', + valueText: null, + valueNumber: null, + valueBoolean: null, + valueHash: 'null' + } + } + + if (typeof value === 'string') { + return { + valueType: 'text', + valueText: value, + valueNumber: null, + valueBoolean: null, + valueHash: hashScalarValue(value) + } + } + + if (typeof value === 'number' && Number.isFinite(value)) { + return { + valueType: 'number', + valueText: null, + valueNumber: value, + valueBoolean: null, + valueHash: hashScalarValue(String(value)) + } + } + + if (typeof value === 'boolean') { + return { + valueType: 'boolean', + valueText: null, + valueNumber: null, + valueBoolean: value ? 1 : 0, + valueHash: value ? 'true' : 'false' + } + } + + return null +} + +export function hashScalarValue(value: string): string { + let hash = 2166136261 + for (let i = 0; i < value.length; i++) { + hash ^= value.charCodeAt(i) + hash = Math.imul(hash, 16777619) + } + return (hash >>> 0).toString(16).padStart(8, '0') +} + +export function stringifyStable(value: unknown): string { + if (value === undefined) { + return 'null' + } + + if (value === null || typeof value !== 'object') { + return JSON.stringify(value) + } + + if (Array.isArray(value)) { + return `[${value.map((item) => stringifyStable(item)).join(',')}]` + } + + const entries = Object.entries(value as Record) + .filter(([, entryValue]) => entryValue !== undefined) + .sort(([left], [right]) => left.localeCompare(right)) + + return `{${entries + .map(([key, entryValue]) => `${JSON.stringify(key)}:${stringifyStable(entryValue)}`) + .join(',')}}` +} + +export function quoteSqlLiteral(value: string): string { + return `'${value.replaceAll("'", "''")}'` +} + +export function buildSqlOrderBy(orderBy?: Partial>): string { + const entries = Object.entries(orderBy ?? {}).filter( + (entry): entry is [string, SortDirection] => entry[1] === 'asc' || entry[1] === 'desc' + ) + if (entries.length === 0) { + return 'n.updated_at DESC, n.id ASC' + } + + const clauses = entries + .filter(([key]) => key === 'createdAt' || key === 'updatedAt') + .map(([key, direction]) => { + const column = key === 'createdAt' ? 'n.created_at' : 'n.updated_at' + return `${column} ${direction.toUpperCase()}` + }) + + return clauses.length > 0 ? [...clauses, 'n.id ASC'].join(', ') : 'n.updated_at DESC, n.id ASC' +} + +function hasOnlySystemOrdering(orderBy?: Record): boolean { + return Object.keys(orderBy ?? {}).every((key) => key === 'createdAt' || key === 'updatedAt') +} + +function appendScalarPredicate( + where: string[], + params: SQLValue[], + alias: string, + scalar: ScalarIndexValue +): void { + switch (scalar.valueType) { + case 'text': + where.push(`${alias}.value_text = ?`) + params.push(scalar.valueText) + return + case 'number': + where.push(`${alias}.value_number = ?`) + params.push(scalar.valueNumber) + return + case 'boolean': + where.push(`${alias}.value_boolean = ?`) + params.push(scalar.valueBoolean) + return + case 'null': + return + } +} + +/** + * ORDER BY for a pushed-down custom-property sort (0264). Mirrors the JS + * comparator in `applyNodeQueryDescriptor`: missing properties sort last + * ascending / first descending; typed value columns carry the order (for a + * homogeneous property exactly one is non-NULL). `n.id` breaks ties so the + * page boundary is a total order. + */ +function buildPropertySortOrderBy(direction: SortDirection): string { + const dir = direction.toUpperCase() + const nullsDir = direction === 'asc' ? 'ASC' : 'DESC' + return ( + `(sortp.node_id IS NULL) ${nullsDir}, ` + + `sortp.value_number ${dir}, sortp.value_boolean ${dir}, sortp.value_text ${dir}, ` + + 'n.id ASC' + ) +} + +function getCompiledPostFilterReason(input: { + useSqlPagination: boolean + hasFullTextSearchPlan: boolean + hasSpatialPlan: boolean +}): string { + if (input.useSqlPagination) { + return 'pagination-pushed-down' + } + + if (input.hasFullTextSearchPlan && input.hasSpatialPlan) { + return 'fts-rtree-verified-in-js' + } + + if (input.hasFullTextSearchPlan) { + return 'fts-verified-in-js' + } + + if (input.hasSpatialPlan) { + return 'spatial-rtree-verified-in-js' + } + + return 'verified-in-js' +} + +// ─── Compiler ──────────────────────────────────────────────────────────────── + +export class QueryCompiler { + constructor(private readonly flags: () => QueryCompilerFlags) {} + + compile( + descriptor: NodeQueryDescriptor, + spatialPlan: SpatialQueryPlan | null = null, + fullTextSearchPlan: FullTextSearchQueryPlan | null = null + ): CompiledNodeQuery | null { + if (descriptor.nodeId) { + return this.compileSqlQuery(descriptor, { + whereEntries: [], + canUseSqlPagination: true, + spatialPlan, + fullTextSearchPlan + }) + } + + const whereEntries = Object.entries(descriptor.where ?? {}) + const scalarWhere = whereEntries.map(([key, value]) => ({ + key, + scalar: toScalarIndexValue(value) + })) + + if (scalarWhere.some((entry) => entry.scalar === null)) { + return null + } + + const hasPropertySort = Object.keys(descriptor.orderBy ?? {}).some( + (key) => key !== 'createdAt' && key !== 'updatedAt' + ) + // Property-sort pushdown (exploration 0264, Wave 2; gated behind the + // adaptive-indexing flag with the typed scalar indexes that serve it): + // a SINGLE custom-property sort orders via a LEFT JOIN on the scalar + // index instead of falling back to a full schema scan + JS sort. + const propertySort = this.resolvePropertySortPushdown(descriptor, hasPropertySort) + const hasSqlCandidateBenefit = + scalarWhere.length > 0 || + spatialPlan !== null || + fullTextSearchPlan !== null || + !descriptor.spatial || + hasOnlySystemOrdering(descriptor.orderBy) + + if (!hasSqlCandidateBenefit && !propertySort) { + return null + } + + return this.compileSqlQuery(descriptor, { + whereEntries: scalarWhere as Array<{ key: string; scalar: ScalarIndexValue }>, + canUseSqlPagination: + (!hasPropertySort || propertySort !== null) && !descriptor.spatial && !descriptor.search, + spatialPlan, + fullTextSearchPlan, + propertySort + }) + } + + /** + * A custom-property sort can push down when it is the descriptor's ONLY + * order key and no cursor/spatial/search accelerator is in play. Gated on + * the adaptive-indexing flag: the typed partial indexes on + * `node_property_scalars` make the join+order cheap, and the flag keeps + * the behavioural change opt-in while it soaks (exploration 0264). + */ + private resolvePropertySortPushdown( + descriptor: NodeQueryDescriptor, + hasPropertySort: boolean + ): { key: string; direction: SortDirection } | null { + if (!hasPropertySort || !this.flags().adaptiveIndexingEnabled) return null + if (descriptor.spatial || descriptor.search || descriptor.after !== undefined) return null + + const entries = Object.entries(descriptor.orderBy ?? {}).filter( + (entry): entry is [string, SortDirection] => entry[1] === 'asc' || entry[1] === 'desc' + ) + if (entries.length !== 1) return null + const [key, direction] = entries[0] + if (key === 'createdAt' || key === 'updatedAt') return null + return { key, direction } + } + + private compileSqlQuery( + descriptor: NodeQueryDescriptor, + options: { + whereEntries: Array<{ key: string; scalar: ScalarIndexValue }> + canUseSqlPagination: boolean + spatialPlan?: SpatialQueryPlan | null + fullTextSearchPlan?: FullTextSearchQueryPlan | null + propertySort?: { key: string; direction: SortDirection } | null + } + ): CompiledNodeQuery { + const joins: string[] = [] + const where: string[] = ['n.schema_id = ?'] + const whereParams: SQLValue[] = [descriptor.schemaId] + + if (descriptor.nodeId) { + where.push('n.id = ?') + whereParams.push(descriptor.nodeId) + } + + if (!descriptor.includeDeleted) { + where.push('n.deleted_at IS NULL') + } + + options.whereEntries.forEach((entry, index) => { + const alias = `p${index}` + const schemaId = quoteSqlLiteral(descriptor.schemaId) + const propertyKey = quoteSqlLiteral(entry.key) + const valueType = quoteSqlLiteral(entry.scalar.valueType) + joins.push( + `JOIN node_property_scalars ${alias} + ON ${alias}.node_id = n.id + AND ${alias}.schema_id = ${schemaId} + AND ${alias}.property_key = ${propertyKey} + AND ${alias}.value_type = ${valueType}` + ) + appendScalarPredicate(where, whereParams, alias, entry.scalar) + }) + + if (options.fullTextSearchPlan) { + joins.push('JOIN nodes_fts ON nodes_fts.node_id = n.id') + where.push('nodes_fts MATCH ?') + whereParams.push(options.fullTextSearchPlan.matchExpression) + } + + if (options.spatialPlan) { + joins.push( + `JOIN node_spatial_ids spatial_ids + ON spatial_ids.node_id = n.id + AND spatial_ids.schema_id = n.schema_id + AND spatial_ids.spatial_key = ${quoteSqlLiteral(options.spatialPlan.spatialKey)}` + ) + joins.push( + `JOIN node_spatial_rtree spatial_rtree + ON spatial_rtree.spatial_id = spatial_ids.spatial_id` + ) + where.push( + `spatial_rtree.max_x >= ?`, + `spatial_rtree.min_x <= ?`, + `spatial_rtree.max_y >= ?`, + `spatial_rtree.min_y <= ?` + ) + whereParams.push( + options.spatialPlan.bounds.minX, + options.spatialPlan.bounds.maxX, + options.spatialPlan.bounds.minY, + options.spatialPlan.bounds.maxY + ) + } + + // Property-sort pushdown (0264): LEFT JOIN the scalar row for the sort + // key (nodes without the property must still appear) and order by the + // typed value columns — for a homogeneous key exactly one column varies, + // the others stay constant-NULL and are inert. Null placement mirrors the + // JS comparator: nulls LAST ascending, FIRST descending. + if (options.propertySort) { + const schemaId = quoteSqlLiteral(descriptor.schemaId) + const propertyKey = quoteSqlLiteral(options.propertySort.key) + joins.push( + `LEFT JOIN node_property_scalars sortp + ON sortp.node_id = n.id + AND sortp.schema_id = ${schemaId} + AND sortp.property_key = ${propertyKey}` + ) + } + + const orderBy = options.propertySort + ? buildPropertySortOrderBy(options.propertySort.direction) + : buildSqlOrderBy(descriptor.orderBy) + const useSqlPagination = + options.canUseSqlPagination && + descriptor.after === undefined && + (descriptor.limit !== undefined || (descriptor.offset ?? 0) > 0) + + let sql = ` + SELECT n.id + FROM nodes n + ${joins.join('\n')} + WHERE ${where.join(' AND ')} + ORDER BY ${orderBy} + ` + + let fused: CompiledNodeQuery['fused'] + if (useSqlPagination) { + sql += '\nLIMIT ? OFFSET ?' + whereParams.push(descriptor.limit ?? -1, descriptor.offset ?? 0) + + // One-RPC fusion (exploration 0264): candidate select as a CTE feeding + // the hydrate join. ROW_NUMBER preserves candidate order through the + // property join; the optional COUNT window folds `count: 'exact'` in + // (window functions evaluate before LIMIT, so it sees the full match + // set). Only built for pushed-down descriptors — JS-verified FTS/ + // spatial paths keep the two-step shape. + const includesExactCount = descriptor.count === 'exact' + const countColumn = includesExactCount ? ',\n COUNT(*) OVER () AS total_count' : '' + const countSelect = includesExactCount ? 'c.total_count,' : '' + const candidatesCte = ` + WITH candidates AS ( + SELECT + n.id, n.schema_id, n.created_at, n.updated_at, n.created_by, n.deleted_at, + ROW_NUMBER() OVER (ORDER BY ${orderBy}) AS ordinal${countColumn} + FROM nodes n + ${joins.join('\n')} + WHERE ${where.join(' AND ')} + ORDER BY ${orderBy} + LIMIT ? OFFSET ? + )` + // Aggregated fusion ships ONE row per node (0264 Wave 2 benchmark: + // ~5× faster SQL + ~10× cheaper boundary clone than row-multiplied). + const fusedSql = this.flags().aggregatedHydration + ? `${candidatesCte} + SELECT + c.id, c.schema_id, c.created_at, c.updated_at, c.created_by, c.deleted_at, + ${countSelect} + c.ordinal, + json_group_object(p.property_key, json(CAST(p.value AS TEXT))) + FILTER (WHERE p.property_key IS NOT NULL) AS props_json, + json_group_object( + p.property_key, + json_object('l', p.lamport_time, 'b', p.updated_by, 'w', p.updated_at) + ) FILTER (WHERE p.property_key IS NOT NULL) AS meta_json + FROM candidates c + LEFT JOIN node_properties p ON p.node_id = c.id + GROUP BY c.id + ORDER BY c.ordinal ASC + ` + : `${candidatesCte} + SELECT + c.id, c.schema_id, c.created_at, c.updated_at, c.created_by, c.deleted_at, + ${countSelect} + p.property_key, p.value, p.lamport_time, p.updated_by, p.updated_at AS prop_updated_at, + c.ordinal + FROM candidates c + LEFT JOIN node_properties p ON p.node_id = c.id + ORDER BY c.ordinal ASC, p.property_key ASC + ` + fused = { + sql: fusedSql, + params: [...whereParams], + includesExactCount + } + } + + return { + sql, + params: whereParams, + fused, + postFilterDescriptor: useSqlPagination ? withoutNodeQueryPagination(descriptor) : descriptor, + postFilterReason: getCompiledPostFilterReason({ + useSqlPagination, + hasFullTextSearchPlan: + options.fullTextSearchPlan !== null && options.fullTextSearchPlan !== undefined, + hasSpatialPlan: options.spatialPlan !== null && options.spatialPlan !== undefined + }), + sqlPagination: useSqlPagination, + adaptiveIndexHints: options.whereEntries.map((entry) => ({ + propertyKey: entry.key, + scalar: entry.scalar + })), + spatialIndexKey: options.spatialPlan?.spatialKey, + fullTextSearchQuery: options.fullTextSearchPlan?.matchExpression + } + } +} diff --git a/packages/data/src/store/sql-batching.ts b/packages/data/src/store/sql-batching.ts new file mode 100644 index 000000000..15a45bd30 --- /dev/null +++ b/packages/data/src/store/sql-batching.ts @@ -0,0 +1,38 @@ +/** + * Shared SQL batching helpers for the SQLite storage layer (exploration 0276). + * + * SQLite binds are capped per statement, and every distinct `VALUES (?,?),…`/ + * `IN (?,…)` arity is a distinct SQL string — a guaranteed miss in the + * worker's prepared-statement cache (explorations 0263/0264). These helpers + * keep id-list SQL inside the bind budget and collapse the statement-shape + * space to a handful of cacheable arities. + */ + +export const SQLITE_BIND_PARAMETER_BATCH_SIZE = 900 +export const SQLITE_HYDRATE_NODE_BATCH_SIZE = Math.floor(SQLITE_BIND_PARAMETER_BATCH_SIZE / 2) + +/** + * Fixed arity buckets for id-list SQL (exploration 0264). Padding id lists up + * to the nearest bucket with NULLs (which never join/match) collapses the + * shape space to a handful of cacheable statements. + */ +export const SQL_HYDRATE_ARITY_BUCKETS = [1, 10, 50, 150, SQLITE_HYDRATE_NODE_BATCH_SIZE] as const +export const SQL_IN_ARITY_BUCKETS = [10, 50, 300, SQLITE_BIND_PARAMETER_BATCH_SIZE] as const + +/** Pad `items` with NULLs up to the nearest arity bucket (see above). */ +export function padToArityBucket( + items: readonly T[], + buckets: readonly number[] +): ReadonlyArray { + const size = buckets.find((bucket) => bucket >= items.length) ?? items.length + if (size === items.length) return items + return [...items, ...Array(size - items.length).fill(null)] +} + +export function chunkItems(items: readonly T[], size: number): T[][] { + const chunks: T[][] = [] + for (let index = 0; index < items.length; index += size) { + chunks.push(items.slice(index, index + size)) + } + return chunks +} diff --git a/packages/data/src/store/sqlite-adapter.ts b/packages/data/src/store/sqlite-adapter.ts index cc1941f87..3cd81e16d 100644 --- a/packages/data/src/store/sqlite-adapter.ts +++ b/packages/data/src/store/sqlite-adapter.ts @@ -15,7 +15,6 @@ import type { AuthorizationStateVersion, ListNodesOptions, CountNodesOptions, - PropertyTimestamp, SetNodeOptions, ImportNodesOptions, RebuildNodeIndexesOptions, @@ -25,6 +24,7 @@ import type { } from './types' import type { SchemaIRI } from '../schema/node' import type { ContentId, DID } from '@xnetjs/core' +import { lwwUpdateGuardSql } from '@xnetjs/core' import type { SQLiteAdapter, SQLValue, @@ -33,8 +33,6 @@ import type { SQLiteNodeBatchApplyInput } from '@xnetjs/sqlite' import { - updateNodeFTS, - deleteNodeFTS, extractSearchableContent, analyzeQuery, detectSQLiteCapabilities, @@ -45,16 +43,58 @@ import { import { SYSTEM_SCHEMA_BASE_IRIS } from '../schema/schemas/system' import { applyNodeQueryDescriptor, - getNodeQuerySearchTokens, withoutNodeQueryMaterializedView, withoutNodeQueryPagination, type NodeQueryDescriptor, type NodeQueryParityCheckMetadata, type NodeQueryResult, - type NodeQuerySpatialFilter, - type NodeQueryStorageCapabilitiesMetadata, - type SortDirection + type NodeQueryStorageCapabilitiesMetadata } from './query' +import { + hydrateAggregatedRows, + hydrateJoinedRows, + hydrateNodesByIds, + type AggregatedNodeRow, + type JoinedNodePropertyRow +} from './hydration' +import { + SQL_IN_ARITY_BUCKETS, + SQLITE_BIND_PARAMETER_BATCH_SIZE, + chunkItems, + padToArityBucket +} from './sql-batching' +import { + QueryCompiler, + buildSqlOrderBy, + hashScalarValue, + quoteSqlLiteral, + stringifyStable, + toScalarIndexValue, + type AdaptiveIndexHint, + type CompiledNodeQuery, + type FullTextSearchQueryPlan, + type ScalarValueType, + type SpatialQueryPlan +} from './query-compiler' +import { + FullTextIndexing, + ScalarIndexing, + SpatialIndexing, + createDeleteRemovedPropertiesOperation, + deleteRemovedProperties, + type IndexingContext +} from './indexing' + +/** + * The shared LWW upsert guard for `node_properties` (protocol §L1.7 via + * `@xnetjs/core`); keep every property write on this ONE ordering (0272/0276). + */ +const NODE_PROPERTIES_LWW_GUARD = lwwUpdateGuardSql({ + table: 'node_properties', + lamportColumn: 'lamport_time', + wallTimeColumn: 'updated_at', + authorColumn: 'updated_by' +}) // ─── Row Types ────────────────────────────────────────────────────────────── @@ -73,46 +113,6 @@ interface ChangeRow { [key: string]: SQLValue } -interface JoinedNodePropertyRow { - id: string - schema_id: string - created_at: number - updated_at: number - created_by: string - deleted_at: number | null - property_key: string | null - value: Uint8Array | null - lamport_time: number | null - updated_by: string | null - prop_updated_at: number | null - ordinal: number | null - [key: string]: SQLValue -} - -/** One-row-per-node aggregated hydrate result (exploration 0264, Wave 2). */ -interface AggregatedNodeRow { - id: string - schema_id: string - created_at: number - updated_at: number - created_by: string - deleted_at: number | null - ordinal: number | null - props_json: string | null - meta_json: string | null - [key: string]: SQLValue -} - -type ScalarValueType = 'text' | 'number' | 'boolean' | 'null' - -interface ScalarIndexValue { - valueType: ScalarValueType - valueText: string | null - valueNumber: number | null - valueBoolean: number | null - valueHash: string -} - interface AdaptiveIndexingConfig { enabled: boolean minHits: number @@ -177,35 +177,6 @@ export interface SQLiteNodeStorageAdapterOptions { scheduleMaintenance?: (task: () => Promise | void) => void } -interface AdaptiveIndexHint { - propertyKey: string - scalar: ScalarIndexValue -} - -interface CompiledNodeQuery { - sql: string - params: SQLValue[] - postFilterDescriptor: NodeQueryDescriptor - postFilterReason: string - sqlPagination: boolean - adaptiveIndexHints: AdaptiveIndexHint[] - spatialIndexKey?: string - fullTextSearchQuery?: string - /** - * Single-statement candidate+hydrate query (exploration 0264, Wave 1). - * Present only for fully-pushed-down descriptors (`sqlPagination`): the - * candidate select becomes a CTE feeding the property hydrate join, so a - * cold query costs ONE worker round-trip instead of id-select + hydrate. - * When the descriptor asks for `count: 'exact'`, a `COUNT(*) OVER ()` - * window inside the CTE folds the total in (no separate COUNT RPC). - */ - fused?: { - sql: string - params: SQLValue[] - includesExactCount: boolean - } -} - interface QueryTelemetry { descriptorHash: string adaptiveIndexNames: string[] @@ -229,35 +200,6 @@ interface AdaptiveIndexBudgetUsage { indexedRows: number } -type SpatialTablesState = 'unknown' | 'absent' | 'ready' -type FullTextSearchTablesState = 'unknown' | 'absent' | 'ready' - -interface SpatialIndexConfigRow { - spatial_key: string - schema_id: string - x_key: string - y_key: string - width_key: string | null - height_key: string | null - [key: string]: SQLValue -} - -interface SpatialBoundingBox { - minX: number - maxX: number - minY: number - maxY: number -} - -interface SpatialQueryPlan { - spatialKey: string - bounds: SpatialBoundingBox -} - -interface FullTextSearchQueryPlan { - matchExpression: string -} - interface MaterializedQueryRow { view_id: string descriptor_hash: string @@ -339,29 +281,6 @@ interface PendingQueryTelemetry { totalCandidates: number lastSeenAt: number } -const SQLITE_BIND_PARAMETER_BATCH_SIZE = 900 -const SQLITE_HYDRATE_NODE_BATCH_SIZE = Math.floor(SQLITE_BIND_PARAMETER_BATCH_SIZE / 2) - -/** - * Fixed arity buckets for id-list SQL (exploration 0264). Every distinct - * `VALUES (?,?),…`/`IN (?,…)` arity is a distinct SQL string — a guaranteed - * miss in the worker's prepared-statement cache (0263). Padding id lists up - * to the nearest bucket with NULLs (which never join/match) collapses the - * shape space to a handful of cacheable statements. - */ -const SQL_HYDRATE_ARITY_BUCKETS = [1, 10, 50, 150, SQLITE_HYDRATE_NODE_BATCH_SIZE] as const -const SQL_IN_ARITY_BUCKETS = [10, 50, 300, SQLITE_BIND_PARAMETER_BATCH_SIZE] as const - -/** Pad `items` with NULLs up to the nearest arity bucket (see above). */ -function padToArityBucket( - items: readonly T[], - buckets: readonly number[] -): ReadonlyArray { - const size = buckets.find((bucket) => bucket >= items.length) ?? items.length - if (size === items.length) return items - return [...items, ...Array(size - items.length).fill(null)] -} - function getMaterializedQueryRefreshReason(input: { cached: MaterializedQueryRow | null descriptorHash: string @@ -453,10 +372,6 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { */ private compiledQueryDiagnosticsMemo = new Map>() - private spatialTablesState: SpatialTablesState = 'unknown' - - private fullTextSearchTablesState: FullTextSearchTablesState = 'unknown' - private writeQueue: Promise = Promise.resolve() /** @@ -470,6 +385,28 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { /** One-time guard: ensure the `auth_fingerprint` column exists on upgraded DBs. */ private materializationColumnsReady = false + /** + * Descriptor→SQL compilation lives in `query-compiler.ts` (exploration + * 0276). Flags are read per-compile so the compiler never captures stale + * adapter state. + */ + private readonly queryCompiler = new QueryCompiler(() => ({ + adaptiveIndexingEnabled: this.adaptiveIndexing.enabled, + aggregatedHydration: this.aggregatedHydration + })) + + /** + * The three sidecar index families — scalar / full-text / spatial — live + * behind `IndexingStrategy` in ./indexing (exploration 0276). Each takes + * the narrow `IndexingContext` capability set; table-existence memos live + * inside the family instances. + */ + private readonly scalarIndexing: ScalarIndexing + + private readonly fullTextIndexing: FullTextIndexing + + private readonly spatialIndexing: SpatialIndexing + constructor( private db: SQLiteAdapter, options: SQLiteNodeStorageAdapterOptions = {} @@ -485,6 +422,17 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { this.queryDiagnostics = options.queryDiagnostics ?? false this.aggregatedHydration = options.aggregatedHydration ?? true this.scheduleMaintenance = options.scheduleMaintenance + + const indexingContext: IndexingContext = { + db, + getStorageCapabilities: () => this.getStorageCapabilities(), + enqueueWrite: (write) => this.enqueueWrite(write), + listNodesForSchema: (schemaId) => this.listNodesOptimized({ schemaId, includeDeleted: true }), + getNode: (id) => this.getNode(id) + } + this.scalarIndexing = new ScalarIndexing(indexingContext) + this.fullTextIndexing = new FullTextIndexing(indexingContext) + this.spatialIndexing = new SpatialIndexing(indexingContext) } getSQLiteAdapter(): SQLiteAdapter { @@ -805,7 +753,7 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { // One joined read via the shared hydrate path. The previous shape — a // node-metadata queryOne followed by a properties query — cost a // worker-backed adapter two RPC round-trips per node (exploration 0263). - const nodes = await this.hydrateNodesByIds([id]) + const nodes = await hydrateNodesByIds(this.db, [id], this.aggregatedHydration) return nodes[0] ?? null } @@ -816,7 +764,7 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { // hydrateNodesByIds chunks internally and batches multi-chunk reads into // one queryBatch RPC (exploration 0263) — don't pre-chunk here or every // chunk pays its own worker round-trip again. - return this.hydrateNodesByIds(uniqueIds) + return hydrateNodesByIds(this.db, uniqueIds, this.aggregatedHydration) } async getExistingNodeIds(ids: readonly NodeId[]): Promise { @@ -890,7 +838,7 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { ] ) - await this.deleteRemovedProperties(node) + await deleteRemovedProperties(this.db, node) // Upsert properties for (const [key, value] of Object.entries(node.properties)) { @@ -914,11 +862,7 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { lamport_time = excluded.lamport_time, updated_by = excluded.updated_by, updated_at = excluded.updated_at - WHERE excluded.lamport_time > node_properties.lamport_time - OR (excluded.lamport_time = node_properties.lamport_time - AND (excluded.updated_at > node_properties.updated_at - OR (excluded.updated_at = node_properties.updated_at - AND excluded.updated_by > node_properties.updated_by)))`, + WHERE ${NODE_PROPERTIES_LWW_GUARD}`, [ node.id, key, @@ -941,16 +885,14 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { const indexedNode = trustMaterializedState ? node : await this.getNode(node.id) if (indexedNode) { const indexProperties = options?.indexProperties ?? true - await this.syncScalarRowsForNode(indexedNode, indexProperties) - await this.syncSpatialRowsForNode(indexedNode, indexProperties) + await this.scalarIndexing.syncNode(indexedNode, indexProperties) + await this.spatialIndexing.syncNode(indexedNode, indexProperties) } // Update FTS index for searchable content // This is a no-op if FTS5 is not supported (e.g., sql.js) const searchableProperties = indexedNode?.properties ?? node.properties - const title = typeof searchableProperties.title === 'string' ? searchableProperties.title : null - const content = extractSearchableContent(searchableProperties) - await updateNodeFTS(this.db, node.id, title, content) + await this.fullTextIndexing.updateNode(node.id, searchableProperties) await this.invalidateMaterializedViewsForSchema(node.schemaId) } @@ -962,8 +904,8 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { private async deleteNodeInternal(id: NodeId): Promise { const existing = await this.getNode(id) // Delete from FTS index first (no-op if FTS5 is not supported) - await deleteNodeFTS(this.db, id) - await this.deleteSpatialRowsForNode(id) + await this.fullTextIndexing.deleteNode(id) + await this.spatialIndexing.deleteNode(id) // Delete node (cascades to properties via FK) await this.db.run(`DELETE FROM nodes WHERE id = ?`, [id]) if (existing) { @@ -993,7 +935,7 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { whereClause += ` AND n.deleted_at IS NULL` } - const orderBy = this.buildSqlOrderBy(options?.orderBy) + const orderBy = buildSqlOrderBy(options?.orderBy) const outerOrderBy = orderBy.replaceAll('n.', 'ln.') // Use CTE for pagination, then join properties @@ -1031,7 +973,7 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { } const rows = await this.db.query(sql, params) - return this.hydrateJoinedRows(rows) + return hydrateJoinedRows(rows) } async countNodes(options?: CountNodesOptions): Promise { @@ -1057,9 +999,9 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { return this.queryMaterializedView(descriptor, start) } - const spatialPlan = await this.prepareSpatialQueryPlan(descriptor) - const fullTextSearchPlan = await this.prepareFullTextSearchQueryPlan(descriptor) - const compiled = this.compileNodeQuery(descriptor, spatialPlan, fullTextSearchPlan) + const spatialPlan = await this.spatialIndexing.prepareQueryPlan(descriptor) + const fullTextSearchPlan = await this.fullTextIndexing.prepareQueryPlan(descriptor) + const compiled = this.queryCompiler.compile(descriptor, spatialPlan, fullTextSearchPlan) if (!compiled) { const storageCapabilities = await this.getStorageCapabilities() @@ -1110,15 +1052,15 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { if (compiled.fused) { const rows = mainQuery.result as Array candidates = this.aggregatedHydration - ? this.hydrateAggregatedRows(rows as AggregatedNodeRow[]) - : this.hydrateJoinedRows(rows as JoinedNodePropertyRow[]) + ? hydrateAggregatedRows(rows as AggregatedNodeRow[]) + : hydrateJoinedRows(rows as JoinedNodePropertyRow[]) candidateCount = candidates.length if (compiled.fused.includesExactCount && rows.length > 0) { fusedExactCount = Number(rows[0].total_count ?? 0) } } else { const ids = (mainQuery.result as Array<{ id: string }>).map((row) => row.id) - candidates = await this.hydrateNodesByIds(ids) + candidates = await hydrateNodesByIds(this.db, ids, this.aggregatedHydration) candidateCount = ids.length } const nodes = applyNodeQueryDescriptor(candidates, compiled.postFilterDescriptor) @@ -1442,7 +1384,7 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { // Spatial indexes still need their existing eager per-node path until they // get a touched-node batch writer. Social import schemas do not use spatial // indexes, so this preserves correctness without blocking the common path. - if (await this.hasSpatialTables()) { + if (await this.spatialIndexing.hasTables()) { return { ...input, indexMode: 'eager' } } @@ -1454,7 +1396,7 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { ): Promise { if (!this.db.transactionBatch) return false if (input.indexMode === 'eager') return false - if (await this.hasSpatialTables()) return false + if (await this.spatialIndexing.hasTables()) return false return true } @@ -1463,7 +1405,7 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { ): Promise { if (!this.db.applyNodeBatch) return false if (input.indexMode === 'eager') return false - if (await this.hasSpatialTables()) return false + if (await this.spatialIndexing.hasTables()) return false return true } @@ -1482,7 +1424,7 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { ): Promise { const indexProperties = input.indexProperties ?? true const hasFullTextSearch = - input.indexMode !== 'defer-schema' && (await this.hasFullTextSearchTable()) + input.indexMode !== 'defer-schema' && (await this.fullTextIndexing.hasTable()) const scalarIndexRows: SQLiteNodeBatchApplyInput['scalarIndexRows'] = [] const ftsNodeIds: string[] = [] const ftsRows: SQLiteNodeBatchApplyInput['ftsRows'] = [] @@ -1491,7 +1433,7 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { for (const node of input.nodes) { for (const [key, value] of Object.entries(node.properties)) { const timestamp = node.timestamps[key] - const scalar = this.toScalarIndexValue(value) + const scalar = toScalarIndexValue(value) if (!timestamp || !scalar) continue scalarIndexRows.push({ @@ -1584,7 +1526,8 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { const indexProperties = input.indexProperties ?? true const operations: Array<{ sql: string; params?: SQLValue[] }> = [] - const hasFullTextSearch = input.indexMode === 'touched' && (await this.hasFullTextSearchTable()) + const hasFullTextSearch = + input.indexMode === 'touched' && (await this.fullTextIndexing.hasTable()) const affectedSchemaIds = new Set(input.affectedSchemaIds) let scalarRowsWritten = 0 let ftsRowsWritten = 0 @@ -1600,12 +1543,12 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { ) ) if (input.indexMode === 'touched' && indexProperties) { - scalarRowsWritten += this.countScalarIndexRowsForNode(node) + scalarRowsWritten += this.scalarIndexing.countIndexRowsForNode(node) } if ( input.indexMode === 'touched' && hasFullTextSearch && - this.hasSearchableNodeContent(node) + this.fullTextIndexing.hasSearchableContent(node) ) { ftsRowsWritten += 1 } @@ -1678,7 +1621,7 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { if (options?.trustMaterializedState !== true) return false if (options?.deferIndexes === true) return false - return !(await this.hasSpatialTables()) + return !(await this.spatialIndexing.hasTables()) } private async importNodesWithTransactionBatch( @@ -1691,7 +1634,7 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { const operations: Array<{ sql: string; params?: SQLValue[] }> = [] const indexProperties = options?.indexProperties ?? true - const hasFullTextSearch = await this.hasFullTextSearchTable() + const hasFullTextSearch = await this.fullTextIndexing.hasTable() const affectedSchemaIds = new Set() for (const node of nodes) { @@ -1747,7 +1690,7 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { node.deleted && node.deletedAt ? node.deletedAt.wallTime : null ] }, - this.createDeleteRemovedPropertiesOperation(node) + createDeleteRemovedPropertiesOperation(node) ] for (const [key, value] of Object.entries(node.properties)) { @@ -1765,11 +1708,7 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { lamport_time = excluded.lamport_time, updated_by = excluded.updated_by, updated_at = excluded.updated_at - WHERE excluded.lamport_time > node_properties.lamport_time - OR (excluded.lamport_time = node_properties.lamport_time - AND (excluded.updated_at > node_properties.updated_at - OR (excluded.updated_at = node_properties.updated_at - AND excluded.updated_by > node_properties.updated_by)))`, + WHERE ${NODE_PROPERTIES_LWW_GUARD}`, params: [ node.id, key, @@ -1787,10 +1726,10 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { params: [node.id] }) if (indexProperties) { - operations.push(...this.createScalarIndexOperations(node)) + operations.push(...this.scalarIndexing.createNodeOperations(node)) } if (hasFullTextSearch) { - operations.push(...this.createFullTextIndexOperations(node)) + operations.push(...this.fullTextIndexing.createNodeOperations(node)) } } @@ -1837,126 +1776,16 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { } } - private createDeleteRemovedPropertiesOperation(node: NodeState): { - sql: string - params?: SQLValue[] - } { - const keys = Object.keys(node.properties) - - if (keys.length === 0) { - return { - sql: 'DELETE FROM node_properties WHERE node_id = ?', - params: [node.id] - } - } - - return { - sql: `DELETE FROM node_properties - WHERE node_id = ? AND property_key NOT IN (${keys.map(() => '?').join(', ')})`, - params: [node.id, ...keys] - } - } - - private createScalarIndexOperations( - node: NodeState - ): Array<{ sql: string; params?: SQLValue[] }> { - return Object.entries(node.properties).flatMap(([key, value]) => { - const timestamp = node.timestamps[key] - const scalar = this.toScalarIndexValue(value) - if (!timestamp || !scalar) return [] - - return [ - { - sql: `INSERT INTO node_property_scalars - ( - node_id, - schema_id, - property_key, - value_type, - value_text, - value_number, - value_boolean, - value_hash, - updated_at, - lamport_time - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - params: [ - node.id, - node.schemaId, - key, - scalar.valueType, - scalar.valueText, - scalar.valueNumber, - scalar.valueBoolean, - scalar.valueHash, - timestamp.wallTime, - timestamp.lamport - ] - } - ] - }) - } - - private createFullTextIndexOperations( - node: NodeState - ): Array<{ sql: string; params?: SQLValue[] }> { - const title = typeof node.properties.title === 'string' ? node.properties.title : null - const content = extractSearchableContent(node.properties) - const operations: Array<{ sql: string; params?: SQLValue[] }> = [ - { - sql: 'DELETE FROM nodes_fts WHERE node_id = ?', - params: [node.id] - } - ] - - if (title || content) { - operations.push({ - sql: 'INSERT INTO nodes_fts (node_id, title, content) VALUES (?, ?, ?)', - params: [node.id, title ?? '', content ?? ''] - }) - } - - return operations - } - - private countScalarIndexRowsForNode(node: NodeState): number { - return Object.values(node.properties).filter((value) => this.toScalarIndexValue(value) !== null) - .length - } - - private hasSearchableNodeContent(node: NodeState): boolean { - const title = typeof node.properties.title === 'string' ? node.properties.title : null - const content = extractSearchableContent(node.properties) - return Boolean(title || content) - } - private async syncTouchedIndexesForNodes( nodes: readonly NodeState[], indexProperties: boolean ): Promise<{ scalarRowsWritten: number; ftsRowsWritten: number }> { let scalarRowsWritten = 0 let ftsRowsWritten = 0 - const hasFullTextSearch = await this.hasFullTextSearchTable() for (const node of nodes) { - scalarRowsWritten += await this.syncScalarRowsForNode(node, indexProperties) - - if (!hasFullTextSearch) { - continue - } - - if (node.deleted) { - await deleteNodeFTS(this.db, node.id) - continue - } - - const title = typeof node.properties.title === 'string' ? node.properties.title : null - const content = extractSearchableContent(node.properties) - await updateNodeFTS(this.db, node.id, title, content) - if (title || content) { - ftsRowsWritten += 1 - } + scalarRowsWritten += await this.scalarIndexing.syncNode(node, indexProperties) + ftsRowsWritten += await this.fullTextIndexing.syncNode(node, indexProperties) } return { scalarRowsWritten, ftsRowsWritten } @@ -2022,89 +1851,15 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { } const indexProperties = options?.indexProperties ?? true - await this.rebuildScalarIndexesForSchemas(uniqueSchemaIds, nodesBySchemaId, indexProperties) - await this.rebuildSpatialIndexesForSchemas(uniqueSchemaIds, nodesBySchemaId, indexProperties) - await this.rebuildFullTextIndexesForSchemas(uniqueSchemaIds, nodesBySchemaId) + await this.scalarIndexing.rebuildForSchemas(uniqueSchemaIds, nodesBySchemaId, indexProperties) + await this.spatialIndexing.rebuildForSchemas(uniqueSchemaIds, nodesBySchemaId, indexProperties) + await this.fullTextIndexing.rebuildForSchemas(uniqueSchemaIds, nodesBySchemaId, indexProperties) for (const schemaId of uniqueSchemaIds) { await this.invalidateMaterializedViewsForSchema(schemaId) } } - private async rebuildScalarIndexesForSchemas( - schemaIds: readonly SchemaIRI[], - nodesBySchemaId: ReadonlyMap, - indexProperties: boolean - ): Promise { - for (const schemaId of schemaIds) { - await this.db.run('DELETE FROM node_property_scalars WHERE schema_id = ?', [schemaId]) - - if (!indexProperties) { - continue - } - - const nodes = nodesBySchemaId.get(schemaId) ?? [] - for (const node of nodes) { - await this.syncScalarRowsForNode(node, true) - } - } - } - - private async rebuildSpatialIndexesForSchemas( - schemaIds: readonly SchemaIRI[], - nodesBySchemaId: ReadonlyMap, - indexProperties: boolean - ): Promise { - if (!(await this.hasSpatialTables())) { - return - } - - for (const schemaId of schemaIds) { - const configs = await this.db.query( - `SELECT spatial_key, schema_id, x_key, y_key, width_key, height_key - FROM node_spatial_indexes - WHERE schema_id = ?`, - [schemaId] - ) - - for (const config of configs) { - await this.clearSpatialRowsForConfig(config.spatial_key) - - if (!indexProperties) { - continue - } - - const nodes = nodesBySchemaId.get(schemaId) ?? [] - for (const node of nodes) { - await this.replaceSpatialRowForConfig(node, config, true) - } - } - } - } - - private async rebuildFullTextIndexesForSchemas( - schemaIds: readonly SchemaIRI[], - nodesBySchemaId: ReadonlyMap - ): Promise { - if (!(await this.hasFullTextSearchTable())) { - return - } - - for (const schemaId of schemaIds) { - const nodes = nodesBySchemaId.get(schemaId) ?? [] - for (const node of nodes) { - if (node.deleted) { - await deleteNodeFTS(this.db, node.id) - continue - } - - const title = typeof node.properties.title === 'string' ? node.properties.title : null - const content = extractSearchableContent(node.properties) - await updateNodeFTS(this.db, node.id, title, content) - } - } - } - /** * Rebuild the scalar sidecar from materialized node_properties. */ @@ -2112,22 +1867,9 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { return this.enqueueWrite(async () => { await this.db.beginTransaction() try { - await this.db.run('DELETE FROM node_property_scalars') - const rows = await this.db.query<{ id: string }>('SELECT id FROM nodes ORDER BY id ASC') - let scalarRowsWritten = 0 - - for (const row of rows) { - const node = await this.getNode(row.id) - if (!node) continue - - scalarRowsWritten += await this.syncScalarRowsForNode(node, true) - } - + const result = await this.scalarIndexing.rebuildAll() await this.db.commit() - return { - nodesScanned: rows.length, - scalarRowsWritten - } + return result } catch (err) { await this.db.rollback() throw err @@ -2154,9 +1896,9 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { await this.db.run('DELETE FROM yjs_updates') await this.db.run('DELETE FROM yjs_state') await this.db.run('DELETE FROM changes') - await this.clearSpatialRows() + await this.spatialIndexing.clear() await this.clearMaterializedViewRows() - await this.db.run('DELETE FROM node_property_scalars') + await this.scalarIndexing.clear() await this.db.run('DELETE FROM node_properties') await this.db.run('DELETE FROM nodes') await this.db.run("DELETE FROM sync_state WHERE key = 'lastLamportTime'") @@ -2170,196 +1912,6 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { // ─── Private Helpers ────────────────────────────────────────────────────── - private hydrateJoinedRows(rows: JoinedNodePropertyRow[]): NodeState[] { - const nodeMap = new Map() - - for (const row of rows) { - let node = nodeMap.get(row.id) - - if (!node) { - node = { - id: row.id, - schemaId: row.schema_id as SchemaIRI, - properties: {}, - timestamps: {}, - deleted: row.deleted_at !== null, - deletedAt: row.deleted_at - ? { lamport: 0, author: '' as DID, wallTime: row.deleted_at } - : undefined, - createdAt: row.created_at, - createdBy: row.created_by as DID, - updatedAt: row.updated_at, - updatedBy: row.created_by as DID - } - nodeMap.set(row.id, node) - } - - if (row.property_key && row.value !== null) { - node.properties[row.property_key] = this.deserializeValue(row.value) - node.timestamps[row.property_key] = { - lamport: row.lamport_time ?? 0, - author: (row.updated_by ?? '') as DID, - wallTime: row.prop_updated_at ?? 0 - } - if ((row.prop_updated_at ?? 0) >= node.updatedAt) { - node.updatedBy = (row.updated_by ?? node.createdBy) as DID - } - } - } - - return Array.from(nodeMap.values()) - } - - private buildHydrateChunkQuery(ids: string[]): { sql: string; params: SQLValue[] } { - // Pad to a fixed arity bucket so repeated hydrates share ONE SQL string - // and hit the worker's prepared-statement cache (exploration 0264). NULL - // ids never satisfy the JOIN, so padding rows vanish from the result. - const padded = padToArityBucket(ids, SQL_HYDRATE_ARITY_BUCKETS) - const values = padded.map(() => '(?, ?)').join(', ') - const params: SQLValue[] = padded.flatMap((id, ordinal) => [id, ordinal]) - const sql = ` - WITH wanted(id, ordinal) AS ( - VALUES ${values} - ) - SELECT - n.id, - n.schema_id, - n.created_at, - n.updated_at, - n.created_by, - n.deleted_at, - p.property_key, - p.value, - p.lamport_time, - p.updated_by, - p.updated_at AS prop_updated_at, - wanted.ordinal - FROM wanted - JOIN nodes n ON n.id = wanted.id - LEFT JOIN node_properties p ON p.node_id = n.id - ORDER BY wanted.ordinal ASC, p.property_key ASC - ` - return { sql, params } - } - - private async hydrateNodesByIds(ids: string[]): Promise { - if (ids.length === 0) { - return [] - } - - const aggregated = this.aggregatedHydration - const chunks = - ids.length > SQLITE_HYDRATE_NODE_BATCH_SIZE - ? chunkItems(ids, SQLITE_HYDRATE_NODE_BATCH_SIZE) - : [ids] - const reads = chunks.map((chunk) => - aggregated ? this.buildAggregatedHydrateChunkQuery(chunk) : this.buildHydrateChunkQuery(chunk) - ) - const parse = (rows: unknown[]): NodeState[] => - aggregated - ? this.hydrateAggregatedRows(rows as AggregatedNodeRow[]) - : this.hydrateJoinedRows(rows as JoinedNodePropertyRow[]) - - // Multi-chunk hydrates previously paid one worker round-trip per chunk; - // queryBatch sends the whole hydrate as ONE RPC and one scheduler slot - // (exploration 0263). Single chunks keep query()'s coalescing. - if (reads.length > 1 && typeof this.db.queryBatch === 'function') { - const results = await this.db.queryBatch(reads) - const nodes: NodeState[] = [] - for (const rows of results) { - nodes.push(...parse(rows)) - } - return nodes - } - - const nodes: NodeState[] = [] - for (const read of reads) { - const rows = await this.db.query(read.sql, read.params) - nodes.push(...parse(rows)) - } - return nodes - } - - /** - * Aggregated hydrate (exploration 0264, Wave 2): collapse the EAV rows to - * ONE row per node inside SQL via `json_group_object`, so the boundary - * ships N rows instead of N × properties. `value` is stored as JSON text - * in a BLOB — `json(CAST(… AS TEXT))` re-emits it as real JSON inside the - * aggregate (without the cast/wrap it would double-encode as a string). - */ - private buildAggregatedHydrateChunkQuery(ids: string[]): { sql: string; params: SQLValue[] } { - const padded = padToArityBucket(ids, SQL_HYDRATE_ARITY_BUCKETS) - const values = padded.map(() => '(?, ?)').join(', ') - const params: SQLValue[] = padded.flatMap((id, ordinal) => [id, ordinal]) - const sql = ` - WITH wanted(id, ordinal) AS ( - VALUES ${values} - ) - SELECT - n.id, - n.schema_id, - n.created_at, - n.updated_at, - n.created_by, - n.deleted_at, - wanted.ordinal, - json_group_object(p.property_key, json(CAST(p.value AS TEXT))) - FILTER (WHERE p.property_key IS NOT NULL) AS props_json, - json_group_object( - p.property_key, - json_object('l', p.lamport_time, 'b', p.updated_by, 'w', p.updated_at) - ) FILTER (WHERE p.property_key IS NOT NULL) AS meta_json - FROM wanted - JOIN nodes n ON n.id = wanted.id - LEFT JOIN node_properties p ON p.node_id = n.id - GROUP BY n.id - ORDER BY wanted.ordinal ASC - ` - return { sql, params } - } - - /** Parse one-row-per-node aggregated hydrate results into NodeStates. */ - private hydrateAggregatedRows(rows: AggregatedNodeRow[]): NodeState[] { - const nodes: NodeState[] = [] - for (const row of rows) { - const properties = row.props_json - ? (JSON.parse(row.props_json) as Record) - : {} - const meta = row.meta_json - ? (JSON.parse(row.meta_json) as Record) - : {} - - const timestamps: Record = {} - let updatedBy: DID = row.created_by as DID - for (const [key, entry] of Object.entries(meta)) { - timestamps[key] = { - lamport: entry.l ?? 0, - author: (entry.b ?? '') as DID, - wallTime: entry.w ?? 0 - } - if ((entry.w ?? 0) >= row.updated_at) { - updatedBy = (entry.b ?? row.created_by) as DID - } - } - - nodes.push({ - id: row.id, - schemaId: row.schema_id as SchemaIRI, - properties, - timestamps, - deleted: row.deleted_at !== null, - deletedAt: row.deleted_at - ? { lamport: 0, author: '' as DID, wallTime: row.deleted_at } - : undefined, - createdAt: row.created_at, - createdBy: row.created_by as DID, - updatedAt: row.updated_at, - updatedBy - }) - } - return nodes - } - /** * Read (or refresh) a materialized view. * @@ -2386,8 +1938,8 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { const authFingerprint = descriptor.authFingerprint ?? null const baseDescriptor = withoutNodeQueryMaterializedView(withoutNodeQueryPagination(descriptor)) - const descriptorJson = this.stringifyStable(baseDescriptor) - const descriptorHash = this.hashScalarValue(descriptorJson) + const descriptorJson = stringifyStable(baseDescriptor) + const descriptorHash = hashScalarValue(descriptorJson) const cached = await this.getMaterializedView(materializedView.viewId) const cacheExpired = materializedView.maxAgeMs !== undefined && @@ -2601,7 +2153,7 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { const offset = usesCursor ? 0 : (descriptor.offset ?? 0) const idRows = await this.db.query<{ node_id: string }>(sql, [readPlan.viewId, limit, offset]) const ids = idRows.map((row) => row.node_id) - const hydrated = await this.hydrateNodesByIds(ids) + const hydrated = await hydrateNodesByIds(this.db, ids, this.aggregatedHydration) const nodes = usesCursor ? applyNodeQueryDescriptor(hydrated, descriptor) : hydrated return { @@ -2645,520 +2197,6 @@ export class SQLiteNodeStorageAdapter implements NodeStorageAdapter { await this.db.run('DELETE FROM node_query_materializations') } - private async deleteRemovedProperties(node: NodeState): Promise { - const keys = Object.keys(node.properties) - - if (keys.length === 0) { - await this.db.run('DELETE FROM node_properties WHERE node_id = ?', [node.id]) - return - } - - const placeholders = keys.map(() => '?').join(', ') - await this.db.run( - `DELETE FROM node_properties WHERE node_id = ? AND property_key NOT IN (${placeholders})`, - [node.id, ...keys] - ) - } - - private async syncScalarRowsForNode(node: NodeState, indexProperties: boolean): Promise { - await this.db.run('DELETE FROM node_property_scalars WHERE node_id = ?', [node.id]) - - if (!indexProperties) { - return 0 - } - - let rowsWritten = 0 - for (const [key, value] of Object.entries(node.properties)) { - const timestamp = node.timestamps[key] - const scalar = this.toScalarIndexValue(value) - if (!timestamp || !scalar) continue - - await this.db.run( - `INSERT INTO node_property_scalars - ( - node_id, - schema_id, - property_key, - value_type, - value_text, - value_number, - value_boolean, - value_hash, - updated_at, - lamport_time - ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - [ - node.id, - node.schemaId, - key, - scalar.valueType, - scalar.valueText, - scalar.valueNumber, - scalar.valueBoolean, - scalar.valueHash, - timestamp.wallTime, - timestamp.lamport - ] - ) - rowsWritten += 1 - } - - return rowsWritten - } - - private async prepareFullTextSearchQueryPlan( - descriptor: NodeQueryDescriptor - ): Promise { - if (!descriptor.search) { - return null - } - - const tokens = getNodeQuerySearchTokens(descriptor.search) - if (tokens.length === 0) { - return null - } - - const capabilities = await this.getStorageCapabilities() - if (!capabilities.fullTextSearch || !(await this.hasFullTextSearchTable())) { - return null - } - - return { - matchExpression: tokens.map((token) => `${token}*`).join(' AND ') - } - } - - private async hasFullTextSearchTable(): Promise { - if (this.fullTextSearchTablesState === 'ready') { - return true - } - - if (this.fullTextSearchTablesState === 'absent') { - return false - } - - const table = await this.db.queryOne<{ name: string }>( - "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'nodes_fts'" - ) - - this.fullTextSearchTablesState = table ? 'ready' : 'absent' - return table !== null - } - - private async prepareSpatialQueryPlan( - descriptor: NodeQueryDescriptor - ): Promise { - if (!descriptor.spatial) { - return null - } - - const capabilities = await this.getStorageCapabilities() - if (!capabilities.rtree) { - return null - } - - await this.ensureSpatialTables() - const spatialKey = this.buildSpatialIndexKey(descriptor.schemaId, descriptor.spatial) - const existing = await this.db.queryOne( - `SELECT spatial_key, schema_id, x_key, y_key, width_key, height_key - FROM node_spatial_indexes - WHERE spatial_key = ?`, - [spatialKey] - ) - - if (!existing) { - await this.createSpatialIndexConfig(descriptor.schemaId, descriptor.spatial, spatialKey) - } - - return { - spatialKey, - bounds: this.getSpatialSearchBounds(descriptor.spatial) - } - } - - private async ensureSpatialTables(): Promise { - const capabilities = await this.getStorageCapabilities() - if (!capabilities.rtree) { - this.spatialTablesState = 'absent' - return - } - - await this.db.exec(` -CREATE TABLE IF NOT EXISTS node_spatial_indexes ( - spatial_key TEXT PRIMARY KEY, - schema_id TEXT NOT NULL, - x_key TEXT NOT NULL, - y_key TEXT NOT NULL, - width_key TEXT, - height_key TEXT, - created_at INTEGER NOT NULL, - last_built_at INTEGER NOT NULL -); - -CREATE TABLE IF NOT EXISTS node_spatial_ids ( - spatial_id INTEGER PRIMARY KEY, - spatial_key TEXT NOT NULL, - node_id TEXT NOT NULL, - schema_id TEXT NOT NULL, - UNIQUE(spatial_key, node_id), - FOREIGN KEY (spatial_key) REFERENCES node_spatial_indexes(spatial_key) ON DELETE CASCADE, - FOREIGN KEY (node_id) REFERENCES nodes(id) ON DELETE CASCADE -); - -CREATE VIRTUAL TABLE IF NOT EXISTS node_spatial_rtree USING rtree( - spatial_id, - min_x, - max_x, - min_y, - max_y -); - -CREATE INDEX IF NOT EXISTS idx_node_spatial_ids_schema - ON node_spatial_ids(schema_id, spatial_key, node_id); -`) - this.spatialTablesState = 'ready' - } - - private async hasSpatialTables(): Promise { - if (this.spatialTablesState === 'ready') { - return true - } - - if (this.spatialTablesState === 'absent') { - return false - } - - const table = await this.db.queryOne<{ count: number }>( - `SELECT COUNT(*) as count - FROM sqlite_master - WHERE type IN ('table', 'virtual table') - AND name IN ('node_spatial_ids', 'node_spatial_rtree')` - ) - const ready = Number(table?.count ?? 0) === 2 - this.spatialTablesState = ready ? 'ready' : 'absent' - - return ready - } - - private async createSpatialIndexConfig( - schemaId: SchemaIRI, - spatial: NodeQuerySpatialFilter, - spatialKey: string - ): Promise { - const fields = this.getSpatialFieldConfig(spatial) - const now = Date.now() - - await this.enqueueWrite(async () => { - await this.db.beginTransaction() - try { - await this.db.run( - `INSERT INTO node_spatial_indexes - (spatial_key, schema_id, x_key, y_key, width_key, height_key, created_at, last_built_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, - [ - spatialKey, - schemaId, - fields.xKey, - fields.yKey, - fields.widthKey, - fields.heightKey, - now, - now - ] - ) - - const nodes = await this.listNodesOptimized({ schemaId, includeDeleted: true }) - const config: SpatialIndexConfigRow = { - spatial_key: spatialKey, - schema_id: schemaId, - x_key: fields.xKey, - y_key: fields.yKey, - width_key: fields.widthKey, - height_key: fields.heightKey - } - - for (const node of nodes) { - await this.replaceSpatialRowForConfig(node, config, true) - } - - await this.db.commit() - } catch (err) { - await this.db.rollback() - throw err - } - }) - } - - private async syncSpatialRowsForNode(node: NodeState, indexProperties: boolean): Promise { - if (!(await this.hasSpatialTables())) { - return - } - - const configs = await this.db.query( - `SELECT spatial_key, schema_id, x_key, y_key, width_key, height_key - FROM node_spatial_indexes - WHERE schema_id = ?`, - [node.schemaId] - ) - - for (const config of configs) { - await this.replaceSpatialRowForConfig(node, config, indexProperties) - } - } - - private async replaceSpatialRowForConfig( - node: NodeState, - config: SpatialIndexConfigRow, - indexProperties: boolean - ): Promise { - await this.deleteSpatialRow(config.spatial_key, node.id) - - if (!indexProperties) { - return - } - - const bounds = this.getNodeSpatialBounds(node, config) - if (!bounds) { - return - } - - const result = await this.db.run( - `INSERT INTO node_spatial_ids (spatial_key, node_id, schema_id) - VALUES (?, ?, ?)`, - [config.spatial_key, node.id, node.schemaId] - ) - const spatialId = Number(result.lastInsertRowid) - await this.db.run( - `INSERT INTO node_spatial_rtree (spatial_id, min_x, max_x, min_y, max_y) - VALUES (?, ?, ?, ?, ?)`, - [spatialId, bounds.minX, bounds.maxX, bounds.minY, bounds.maxY] - ) - } - - private async deleteSpatialRowsForNode(nodeId: NodeId): Promise { - if (!(await this.hasSpatialTables())) { - return - } - - const rows = await this.db.query<{ spatial_key: string }>( - `SELECT spatial_key - FROM node_spatial_ids - WHERE node_id = ?`, - [nodeId] - ) - - for (const row of rows) { - await this.deleteSpatialRow(row.spatial_key, nodeId) - } - } - - private async deleteSpatialRow(spatialKey: string, nodeId: NodeId): Promise { - const existing = await this.db.queryOne<{ spatial_id: number }>( - `SELECT spatial_id - FROM node_spatial_ids - WHERE spatial_key = ? AND node_id = ?`, - [spatialKey, nodeId] - ) - - if (!existing) { - return - } - - await this.db.run('DELETE FROM node_spatial_rtree WHERE spatial_id = ?', [existing.spatial_id]) - await this.db.run('DELETE FROM node_spatial_ids WHERE spatial_id = ?', [existing.spatial_id]) - } - - private async clearSpatialRowsForConfig(spatialKey: string): Promise { - const rows = await this.db.query<{ spatial_id: number }>( - `SELECT spatial_id - FROM node_spatial_ids - WHERE spatial_key = ?`, - [spatialKey] - ) - - for (const batch of chunkItems(rows, SQLITE_BIND_PARAMETER_BATCH_SIZE)) { - const placeholders = batch.map(() => '?').join(', ') - await this.db.run(`DELETE FROM node_spatial_rtree WHERE spatial_id IN (${placeholders})`, [ - ...batch.map((row) => row.spatial_id) - ]) - } - - await this.db.run('DELETE FROM node_spatial_ids WHERE spatial_key = ?', [spatialKey]) - } - - private async clearSpatialRows(): Promise { - if (!(await this.hasSpatialTables())) { - return - } - - await this.db.run('DELETE FROM node_spatial_rtree') - await this.db.run('DELETE FROM node_spatial_ids') - await this.db.run('DELETE FROM node_spatial_indexes') - } - - private buildSpatialIndexKey(schemaId: SchemaIRI, spatial: NodeQuerySpatialFilter): string { - const fields = this.getSpatialFieldConfig(spatial) - return this.hashScalarValue( - this.stringifyStable({ - schemaId, - x: fields.xKey, - y: fields.yKey, - width: fields.widthKey, - height: fields.heightKey - }) - ) - } - - private getSpatialFieldConfig(spatial: NodeQuerySpatialFilter): { - xKey: string - yKey: string - widthKey: string | null - heightKey: string | null - } { - return { - xKey: spatial.fields.x, - yKey: spatial.fields.y, - widthKey: spatial.kind === 'window' ? (spatial.fields.width ?? null) : null, - heightKey: spatial.kind === 'window' ? (spatial.fields.height ?? null) : null - } - } - - private getSpatialSearchBounds(spatial: NodeQuerySpatialFilter): SpatialBoundingBox { - if (spatial.kind === 'radius') { - const radius = Math.abs(spatial.radius) - return { - minX: spatial.center.x - radius, - maxX: spatial.center.x + radius, - minY: spatial.center.y - radius, - maxY: spatial.center.y + radius - } - } - - const overscan = spatial.overscan ?? 0 - const left = spatial.rect.x - overscan - const right = spatial.rect.x + spatial.rect.width + overscan - const top = spatial.rect.y - overscan - const bottom = spatial.rect.y + spatial.rect.height + overscan - - return { - minX: Math.min(left, right), - maxX: Math.max(left, right), - minY: Math.min(top, bottom), - maxY: Math.max(top, bottom) - } - } - - private getNodeSpatialBounds( - node: NodeState, - config: SpatialIndexConfigRow - ): SpatialBoundingBox | null { - const x = this.getFiniteNumberProperty(node, config.x_key) - const y = this.getFiniteNumberProperty(node, config.y_key) - - if (x === null || y === null) { - return null - } - - const width = this.getFiniteNumberProperty(node, config.width_key) ?? 0 - const height = this.getFiniteNumberProperty(node, config.height_key) ?? 0 - - return { - minX: Math.min(x, x + width), - maxX: Math.max(x, x + width), - minY: Math.min(y, y + height), - maxY: Math.max(y, y + height) - } - } - - private getFiniteNumberProperty(node: NodeState, key: string | null): number | null { - if (!key) { - return null - } - - const value = node.properties[key] - return typeof value === 'number' && Number.isFinite(value) ? value : null - } - - private toScalarIndexValue(value: unknown): ScalarIndexValue | null { - if (value === null) { - return { - valueType: 'null', - valueText: null, - valueNumber: null, - valueBoolean: null, - valueHash: 'null' - } - } - - if (typeof value === 'string') { - return { - valueType: 'text', - valueText: value, - valueNumber: null, - valueBoolean: null, - valueHash: this.hashScalarValue(value) - } - } - - if (typeof value === 'number' && Number.isFinite(value)) { - return { - valueType: 'number', - valueText: null, - valueNumber: value, - valueBoolean: null, - valueHash: this.hashScalarValue(String(value)) - } - } - - if (typeof value === 'boolean') { - return { - valueType: 'boolean', - valueText: null, - valueNumber: null, - valueBoolean: value ? 1 : 0, - valueHash: value ? 'true' : 'false' - } - } - - return null - } - - private hashScalarValue(value: string): string { - let hash = 2166136261 - for (let i = 0; i < value.length; i++) { - hash ^= value.charCodeAt(i) - hash = Math.imul(hash, 16777619) - } - return (hash >>> 0).toString(16).padStart(8, '0') - } - - private stringifyStable(value: unknown): string { - if (value === undefined) { - return 'null' - } - - if (value === null || typeof value !== 'object') { - return JSON.stringify(value) - } - - if (Array.isArray(value)) { - return `[${value.map((item) => this.stringifyStable(item)).join(',')}]` - } - - const entries = Object.entries(value as Record) - .filter(([, entryValue]) => entryValue !== undefined) - .sort(([left], [right]) => left.localeCompare(right)) - - return `{${entries - .map(([key, entryValue]) => `${JSON.stringify(key)}:${this.stringifyStable(entryValue)}`) - .join(',')}}` - } - - private quoteSqlLiteral(value: string): string { - return `'${value.replaceAll("'", "''")}'` - } - private quoteSqlIdentifier(identifier: string): string { if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(identifier)) { throw new Error(`Unsafe SQLite identifier: ${identifier}`) @@ -3172,8 +2210,8 @@ CREATE INDEX IF NOT EXISTS idx_node_spatial_ids_schema result: NodeQueryResult, adaptiveIndexHints: AdaptiveIndexHint[] ): Promise { - const descriptorJson = this.stringifyStable(descriptor) - const descriptorHash = this.hashScalarValue(descriptorJson) + const descriptorJson = stringifyStable(descriptor) + const descriptorHash = hashScalarValue(descriptorJson) const now = Date.now() const sample: PendingQueryTelemetry = { schemaId: descriptor.schemaId, @@ -3901,8 +2939,8 @@ CREATE INDEX IF NOT EXISTS idx_node_spatial_ids_schema ): string { return [ 'idx_auto_prop', - this.hashScalarValue(schemaId), - this.hashScalarValue(propertyKey), + hashScalarValue(schemaId), + hashScalarValue(propertyKey), valueType ].join('_') } @@ -3918,9 +2956,9 @@ CREATE INDEX IF NOT EXISTS idx_node_spatial_ids_schema return `CREATE INDEX IF NOT EXISTS ${indexIdentifier} ON node_property_scalars(${columns}) -WHERE schema_id = ${this.quoteSqlLiteral(schemaId)} - AND property_key = ${this.quoteSqlLiteral(propertyKey)} - AND value_type = ${this.quoteSqlLiteral(valueType)}` +WHERE schema_id = ${quoteSqlLiteral(schemaId)} + AND property_key = ${quoteSqlLiteral(propertyKey)} + AND value_type = ${quoteSqlLiteral(valueType)}` } private getAdaptiveIndexColumns(valueType: ScalarValueType): string { @@ -3941,7 +2979,7 @@ WHERE schema_id = ${this.quoteSqlLiteral(schemaId)} spatialPlan: SpatialQueryPlan | null, fullTextSearchPlan: FullTextSearchQueryPlan | null ): Promise { - const compiled = this.compileNodeQuery( + const compiled = this.queryCompiler.compile( withoutNodeQueryPagination(descriptor), spatialPlan, fullTextSearchPlan @@ -3956,352 +2994,6 @@ WHERE schema_id = ${this.quoteSqlLiteral(schemaId)} return Number(row?.count ?? 0) } - private compileNodeQuery( - descriptor: NodeQueryDescriptor, - spatialPlan: SpatialQueryPlan | null = null, - fullTextSearchPlan: FullTextSearchQueryPlan | null = null - ): CompiledNodeQuery | null { - if (descriptor.nodeId) { - return this.compileSqlQuery(descriptor, { - whereEntries: [], - canUseSqlPagination: true, - spatialPlan, - fullTextSearchPlan - }) - } - - const whereEntries = Object.entries(descriptor.where ?? {}) - const scalarWhere = whereEntries.map(([key, value]) => ({ - key, - scalar: this.toScalarIndexValue(value) - })) - - if (scalarWhere.some((entry) => entry.scalar === null)) { - return null - } - - const hasPropertySort = Object.keys(descriptor.orderBy ?? {}).some( - (key) => key !== 'createdAt' && key !== 'updatedAt' - ) - // Property-sort pushdown (exploration 0264, Wave 2; gated behind the - // adaptive-indexing flag with the typed scalar indexes that serve it): - // a SINGLE custom-property sort orders via a LEFT JOIN on the scalar - // index instead of falling back to a full schema scan + JS sort. - const propertySort = this.resolvePropertySortPushdown(descriptor, hasPropertySort) - const hasSqlCandidateBenefit = - scalarWhere.length > 0 || - spatialPlan !== null || - fullTextSearchPlan !== null || - !descriptor.spatial || - this.hasOnlySystemOrdering(descriptor.orderBy) - - if (!hasSqlCandidateBenefit && !propertySort) { - return null - } - - return this.compileSqlQuery(descriptor, { - whereEntries: scalarWhere as Array<{ key: string; scalar: ScalarIndexValue }>, - canUseSqlPagination: - (!hasPropertySort || propertySort !== null) && !descriptor.spatial && !descriptor.search, - spatialPlan, - fullTextSearchPlan, - propertySort - }) - } - - /** - * A custom-property sort can push down when it is the descriptor's ONLY - * order key and no cursor/spatial/search accelerator is in play. Gated on - * the adaptive-indexing flag: the typed partial indexes on - * `node_property_scalars` make the join+order cheap, and the flag keeps - * the behavioural change opt-in while it soaks (exploration 0264). - */ - private resolvePropertySortPushdown( - descriptor: NodeQueryDescriptor, - hasPropertySort: boolean - ): { key: string; direction: SortDirection } | null { - if (!hasPropertySort || !this.adaptiveIndexing.enabled) return null - if (descriptor.spatial || descriptor.search || descriptor.after !== undefined) return null - - const entries = Object.entries(descriptor.orderBy ?? {}).filter( - (entry): entry is [string, SortDirection] => entry[1] === 'asc' || entry[1] === 'desc' - ) - if (entries.length !== 1) return null - const [key, direction] = entries[0] - if (key === 'createdAt' || key === 'updatedAt') return null - return { key, direction } - } - - private compileSqlQuery( - descriptor: NodeQueryDescriptor, - options: { - whereEntries: Array<{ key: string; scalar: ScalarIndexValue }> - canUseSqlPagination: boolean - spatialPlan?: SpatialQueryPlan | null - fullTextSearchPlan?: FullTextSearchQueryPlan | null - propertySort?: { key: string; direction: SortDirection } | null - } - ): CompiledNodeQuery { - const joins: string[] = [] - const where: string[] = ['n.schema_id = ?'] - const whereParams: SQLValue[] = [descriptor.schemaId] - - if (descriptor.nodeId) { - where.push('n.id = ?') - whereParams.push(descriptor.nodeId) - } - - if (!descriptor.includeDeleted) { - where.push('n.deleted_at IS NULL') - } - - options.whereEntries.forEach((entry, index) => { - const alias = `p${index}` - const schemaId = this.quoteSqlLiteral(descriptor.schemaId) - const propertyKey = this.quoteSqlLiteral(entry.key) - const valueType = this.quoteSqlLiteral(entry.scalar.valueType) - joins.push( - `JOIN node_property_scalars ${alias} - ON ${alias}.node_id = n.id - AND ${alias}.schema_id = ${schemaId} - AND ${alias}.property_key = ${propertyKey} - AND ${alias}.value_type = ${valueType}` - ) - this.appendScalarPredicate(where, whereParams, alias, entry.scalar) - }) - - if (options.fullTextSearchPlan) { - joins.push('JOIN nodes_fts ON nodes_fts.node_id = n.id') - where.push('nodes_fts MATCH ?') - whereParams.push(options.fullTextSearchPlan.matchExpression) - } - - if (options.spatialPlan) { - joins.push( - `JOIN node_spatial_ids spatial_ids - ON spatial_ids.node_id = n.id - AND spatial_ids.schema_id = n.schema_id - AND spatial_ids.spatial_key = ${this.quoteSqlLiteral(options.spatialPlan.spatialKey)}` - ) - joins.push( - `JOIN node_spatial_rtree spatial_rtree - ON spatial_rtree.spatial_id = spatial_ids.spatial_id` - ) - where.push( - `spatial_rtree.max_x >= ?`, - `spatial_rtree.min_x <= ?`, - `spatial_rtree.max_y >= ?`, - `spatial_rtree.min_y <= ?` - ) - whereParams.push( - options.spatialPlan.bounds.minX, - options.spatialPlan.bounds.maxX, - options.spatialPlan.bounds.minY, - options.spatialPlan.bounds.maxY - ) - } - - // Property-sort pushdown (0264): LEFT JOIN the scalar row for the sort - // key (nodes without the property must still appear) and order by the - // typed value columns — for a homogeneous key exactly one column varies, - // the others stay constant-NULL and are inert. Null placement mirrors the - // JS comparator: nulls LAST ascending, FIRST descending. - if (options.propertySort) { - const schemaId = this.quoteSqlLiteral(descriptor.schemaId) - const propertyKey = this.quoteSqlLiteral(options.propertySort.key) - joins.push( - `LEFT JOIN node_property_scalars sortp - ON sortp.node_id = n.id - AND sortp.schema_id = ${schemaId} - AND sortp.property_key = ${propertyKey}` - ) - } - - const orderBy = options.propertySort - ? this.buildPropertySortOrderBy(options.propertySort.direction) - : this.buildSqlOrderBy(descriptor.orderBy) - const useSqlPagination = - options.canUseSqlPagination && - descriptor.after === undefined && - (descriptor.limit !== undefined || (descriptor.offset ?? 0) > 0) - - let sql = ` - SELECT n.id - FROM nodes n - ${joins.join('\n')} - WHERE ${where.join(' AND ')} - ORDER BY ${orderBy} - ` - - let fused: CompiledNodeQuery['fused'] - if (useSqlPagination) { - sql += '\nLIMIT ? OFFSET ?' - whereParams.push(descriptor.limit ?? -1, descriptor.offset ?? 0) - - // One-RPC fusion (exploration 0264): candidate select as a CTE feeding - // the hydrate join. ROW_NUMBER preserves candidate order through the - // property join; the optional COUNT window folds `count: 'exact'` in - // (window functions evaluate before LIMIT, so it sees the full match - // set). Only built for pushed-down descriptors — JS-verified FTS/ - // spatial paths keep the two-step shape. - const includesExactCount = descriptor.count === 'exact' - const countColumn = includesExactCount ? ',\n COUNT(*) OVER () AS total_count' : '' - const countSelect = includesExactCount ? 'c.total_count,' : '' - const candidatesCte = ` - WITH candidates AS ( - SELECT - n.id, n.schema_id, n.created_at, n.updated_at, n.created_by, n.deleted_at, - ROW_NUMBER() OVER (ORDER BY ${orderBy}) AS ordinal${countColumn} - FROM nodes n - ${joins.join('\n')} - WHERE ${where.join(' AND ')} - ORDER BY ${orderBy} - LIMIT ? OFFSET ? - )` - // Aggregated fusion ships ONE row per node (0264 Wave 2 benchmark: - // ~5× faster SQL + ~10× cheaper boundary clone than row-multiplied). - const fusedSql = this.aggregatedHydration - ? `${candidatesCte} - SELECT - c.id, c.schema_id, c.created_at, c.updated_at, c.created_by, c.deleted_at, - ${countSelect} - c.ordinal, - json_group_object(p.property_key, json(CAST(p.value AS TEXT))) - FILTER (WHERE p.property_key IS NOT NULL) AS props_json, - json_group_object( - p.property_key, - json_object('l', p.lamport_time, 'b', p.updated_by, 'w', p.updated_at) - ) FILTER (WHERE p.property_key IS NOT NULL) AS meta_json - FROM candidates c - LEFT JOIN node_properties p ON p.node_id = c.id - GROUP BY c.id - ORDER BY c.ordinal ASC - ` - : `${candidatesCte} - SELECT - c.id, c.schema_id, c.created_at, c.updated_at, c.created_by, c.deleted_at, - ${countSelect} - p.property_key, p.value, p.lamport_time, p.updated_by, p.updated_at AS prop_updated_at, - c.ordinal - FROM candidates c - LEFT JOIN node_properties p ON p.node_id = c.id - ORDER BY c.ordinal ASC, p.property_key ASC - ` - fused = { - sql: fusedSql, - params: [...whereParams], - includesExactCount - } - } - - return { - sql, - params: whereParams, - fused, - postFilterDescriptor: useSqlPagination ? withoutNodeQueryPagination(descriptor) : descriptor, - postFilterReason: this.getCompiledPostFilterReason({ - useSqlPagination, - hasFullTextSearchPlan: - options.fullTextSearchPlan !== null && options.fullTextSearchPlan !== undefined, - hasSpatialPlan: options.spatialPlan !== null && options.spatialPlan !== undefined - }), - sqlPagination: useSqlPagination, - adaptiveIndexHints: options.whereEntries.map((entry) => ({ - propertyKey: entry.key, - scalar: entry.scalar - })), - spatialIndexKey: options.spatialPlan?.spatialKey, - fullTextSearchQuery: options.fullTextSearchPlan?.matchExpression - } - } - - private getCompiledPostFilterReason(input: { - useSqlPagination: boolean - hasFullTextSearchPlan: boolean - hasSpatialPlan: boolean - }): string { - if (input.useSqlPagination) { - return 'pagination-pushed-down' - } - - if (input.hasFullTextSearchPlan && input.hasSpatialPlan) { - return 'fts-rtree-verified-in-js' - } - - if (input.hasFullTextSearchPlan) { - return 'fts-verified-in-js' - } - - if (input.hasSpatialPlan) { - return 'spatial-rtree-verified-in-js' - } - - return 'verified-in-js' - } - - private appendScalarPredicate( - where: string[], - params: SQLValue[], - alias: string, - scalar: ScalarIndexValue - ): void { - switch (scalar.valueType) { - case 'text': - where.push(`${alias}.value_text = ?`) - params.push(scalar.valueText) - return - case 'number': - where.push(`${alias}.value_number = ?`) - params.push(scalar.valueNumber) - return - case 'boolean': - where.push(`${alias}.value_boolean = ?`) - params.push(scalar.valueBoolean) - return - case 'null': - return - } - } - - /** - * ORDER BY for a pushed-down custom-property sort (0264). Mirrors the JS - * comparator in `applyNodeQueryDescriptor`: missing properties sort last - * ascending / first descending; typed value columns carry the order (for a - * homogeneous property exactly one is non-NULL). `n.id` breaks ties so the - * page boundary is a total order. - */ - private buildPropertySortOrderBy(direction: SortDirection): string { - const dir = direction.toUpperCase() - const nullsDir = direction === 'asc' ? 'ASC' : 'DESC' - return ( - `(sortp.node_id IS NULL) ${nullsDir}, ` + - `sortp.value_number ${dir}, sortp.value_boolean ${dir}, sortp.value_text ${dir}, ` + - 'n.id ASC' - ) - } - - private buildSqlOrderBy(orderBy?: Partial>): string { - const entries = Object.entries(orderBy ?? {}).filter( - (entry): entry is [string, SortDirection] => entry[1] === 'asc' || entry[1] === 'desc' - ) - if (entries.length === 0) { - return 'n.updated_at DESC, n.id ASC' - } - - const clauses = entries - .filter(([key]) => key === 'createdAt' || key === 'updatedAt') - .map(([key, direction]) => { - const column = key === 'createdAt' ? 'n.created_at' : 'n.updated_at' - return `${column} ${direction.toUpperCase()}` - }) - - return clauses.length > 0 ? [...clauses, 'n.id ASC'].join(', ') : 'n.updated_at DESC, n.id ASC' - } - - private hasOnlySystemOrdering(orderBy?: Record): boolean { - return Object.keys(orderBy ?? {}).every((key) => key === 'createdAt' || key === 'updatedAt') - } - private serializePayload(payload: NodePayload): Uint8Array { return new TextEncoder().encode(JSON.stringify(payload)) } @@ -4334,11 +3026,6 @@ WHERE schema_id = ${this.quoteSqlLiteral(schemaId)} return new TextEncoder().encode(JSON.stringify(value)) } - private deserializeValue(data: Uint8Array | null): unknown { - if (!data) return null - return JSON.parse(new TextDecoder().decode(data)) - } - private deserializeChange(row: ChangeRow): NodeChange { const parsed: unknown = JSON.parse(new TextDecoder().decode(row.payload)) const envelope = @@ -4376,14 +3063,6 @@ WHERE schema_id = ${this.quoteSqlLiteral(schemaId)} } } -function chunkItems(items: readonly T[], size: number): T[][] { - const chunks: T[][] = [] - for (let index = 0; index < items.length; index += size) { - chunks.push(items.slice(index, index + size)) - } - return chunks -} - // ─── Factory Functions ─────────────────────────────────────────────────────── /** diff --git a/packages/data/src/store/store.ts b/packages/data/src/store/store.ts index 0c7b76b9b..b4b712b57 100644 --- a/packages/data/src/store/store.ts +++ b/packages/data/src/store/store.ts @@ -47,6 +47,19 @@ import type { import type { StoreAuthAPI } from '../auth/store-auth' import type { LensRegistry } from '../schema/lens' import type { AuthAction, AuthDecision, DID, ContentId, PolicyEvaluator } from '@xnetjs/core' +import { compareChangeApplicationOrder, lwwWins } from '@xnetjs/core' +import { + executeTransactionOperations, + executeTransactionOperationsFast, + type PendingTransactionEvent, + type WriteExecutionHost +} from './transaction-executor' +import { + executeDeterministicNodeImport, + planDeterministicNodeImport, + type DeterministicNodeImportAppliedPlan, + type DeterministicNodeImportPlan +} from './batch-write-orchestrator' import { base64ToBytes, bytesToBase64 } from '@xnetjs/crypto' import { parseDID } from '@xnetjs/identity' import { @@ -100,27 +113,6 @@ type SerializedNodeSnapshot = { unknown?: Record } -type PendingTransactionEvent = { - change: NodeChange - result: NodeState | null - previousNode: NodeState | null -} - -type DeterministicNodeImportPlan = { - created: number - updated: number - nodes: NodeState[] - changes: NodeChange[] - events: PendingTransactionEvent[] - affectedSchemaIds: SchemaIRI[] - timings: Pick -} - -type DeterministicNodeImportAppliedPlan = DeterministicNodeImportPlan & { - applyMs: number - storage?: ApplyNodeBatchResult -} - type DeterministicNodeImportExecution = ImportDeterministicNodesResult & { storage?: ApplyNodeBatchResult timings: NodeBatchWriteTimings @@ -136,6 +128,7 @@ export class NodeStore { private changeSigner?: ChangeSigner private clock: LamportClock private conflicts: MergeConflict[] = [] + private writeHost?: WriteExecutionHost private listeners: Set = new Set() private nodeListeners: Map> = new Map() private batchListeners: Set = new Set() @@ -1380,81 +1373,7 @@ export class NodeStore { batchId: string batchSize: number }): Promise { - const ids = input.drafts.map((draft) => draft.id) - const preflightStartedAt = Date.now() - const preflight = await this.getBatchPreflight(ids, input.storage) - const preflightMs = elapsedMs(preflightStartedAt) - const materializeStartedAt = Date.now() - const existingNodes = this.cloneNodeMap(preflight.nodesById) - const lastChanges = new Map(preflight.lastChangesByNodeId) - const nodesById = new Map(existingNodes) - const changedIds: NodeId[] = [] - const seenChangedIds = new Set() - const changes: NodeChange[] = [] - const events: PendingTransactionEvent[] = [] - let created = 0 - let updated = 0 - - for (let index = 0; index < input.drafts.length; index++) { - const draft = input.drafts[index] - const currentNode = nodesById.get(draft.id) ?? null - const previousNode = this.cloneNodeState(currentNode) - const isCreate = currentNode === null - const payload: NodePayload = { - nodeId: draft.id, - ...(isCreate ? { schemaId: draft.schemaId } : {}), - properties: draft.properties - } - const change = await this.createBatchedChangeWithParentHash( - 'node-change', - payload, - lastChanges.get(draft.id)?.hash ?? null, - input.lamport, - input.now, - input.batchId, - index, - input.batchSize - ) - const node = this.materializeNodeChange( - change, - currentNode ?? this.createInitialNodeFromChange(change, draft.schemaId) - ) - - nodesById.set(draft.id, node) - lastChanges.set(draft.id, change) - changes.push(change) - events.push({ change, result: this.cloneNodeState(node), previousNode }) - - if (!seenChangedIds.has(draft.id)) { - changedIds.push(draft.id) - seenChangedIds.add(draft.id) - } - - if (isCreate) { - created += 1 - } else { - updated += 1 - } - } - - const nodes = changedIds.flatMap((id) => { - const node = nodesById.get(id) - return node ? [node] : [] - }) - const affectedSchemaIds = Array.from(new Set(nodes.map((node) => node.schemaId))) - - return { - created, - updated, - nodes, - changes, - events, - affectedSchemaIds, - timings: { - preflightMs, - materializeMs: elapsedMs(materializeStartedAt) - } - } + return planDeterministicNodeImport(this.writeExecutionHost(), input) } private async executeDeterministicNodeImport(input: { @@ -1466,23 +1385,7 @@ export class NodeStore { batchSize: number deferIndexes: boolean }): Promise { - const plan = await this.planDeterministicNodeImport(input) - - const applyStartedAt = Date.now() - await this.importMaterializedNodes(input.storage, plan.nodes, { - deferIndexes: input.deferIndexes - }) - await this.appendImportedChanges(input.storage, plan.changes) - await input.storage.setLastLamportTime(this.clock.time) - - for (const node of plan.nodes) { - await this.persistEncryptedNodeSnapshot(node, input.storage) - } - - return { - ...plan, - applyMs: elapsedMs(applyStartedAt) - } + return executeDeterministicNodeImport(this.writeExecutionHost(), input) } private async getNodesById( @@ -1633,135 +1536,11 @@ export class NodeStore { changes: NodeChange[] events: PendingTransactionEvent[] }> { - const results: (NodeState | null)[] = [] - const changes: NodeChange[] = [] - const events: PendingTransactionEvent[] = [] - - for (let i = 0; i < input.operations.length; i++) { - const op = input.operations[i] - let change: NodeChange - let result: NodeState | null = null - let previousNode: NodeState | null = null - - switch (op.type) { - case 'create': { - const id = op.options.id ?? createNodeId() - const payload: NodePayload = { - nodeId: id, - schemaId: op.options.schemaId, - properties: op.options.properties - } - change = await this.createBatchedChange( - 'node-change', - payload, - input.lamport, - input.now, - input.batchId, - i, - input.batchSize, - input.storage - ) - await this.applyChange(change, input.storage) - result = await input.storage.getNode(id) - await this.persistEncryptedNodeSnapshot(result, input.storage) - break - } - - case 'update': { - const existing = this.cloneNodeState(await input.storage.getNode(op.nodeId)) - if (!existing) { - throw new Error(`Node not found: ${op.nodeId}`) - } - previousNode = existing - const payload: NodePayload = { - nodeId: op.nodeId, - properties: op.options.properties - } - change = await this.createBatchedChange( - 'node-change', - payload, - input.lamport, - input.now, - input.batchId, - i, - input.batchSize, - input.storage - ) - await this.applyChange(change, input.storage) - result = await input.storage.getNode(op.nodeId) - await this.persistEncryptedNodeSnapshot(result, input.storage) - break - } - - case 'delete': { - const existing = this.cloneNodeState(await input.storage.getNode(op.nodeId)) - if (!existing) { - throw new Error(`Node not found: ${op.nodeId}`) - } - previousNode = existing - const payload: NodePayload = { - nodeId: op.nodeId, - properties: {}, - deleted: true - } - change = await this.createBatchedChange( - 'node-change', - payload, - input.lamport, - input.now, - input.batchId, - i, - input.batchSize, - input.storage - ) - await this.applyChange(change, input.storage) - result = null - break - } - - case 'restore': { - const existing = this.cloneNodeState(await input.storage.getNode(op.nodeId)) - if (!existing) { - throw new Error(`Node not found: ${op.nodeId}`) - } - previousNode = existing - const payload: NodePayload = { - nodeId: op.nodeId, - properties: {}, - deleted: false - } - change = await this.createBatchedChange( - 'node-change', - payload, - input.lamport, - input.now, - input.batchId, - i, - input.batchSize, - input.storage - ) - await this.applyChange(change, input.storage) - result = await input.storage.getNode(op.nodeId) - await this.persistEncryptedNodeSnapshot(result, input.storage) - break - } - } - - changes.push(change) - results.push(result) - events.push({ change, result, previousNode }) - } - - return { results, changes, events } + return executeTransactionOperations(this.writeExecutionHost(), input) } /** - * Transaction fast path: one preflight for all touched nodes, in-memory - * materialization and signing, then one transactional applyNodeBatch. - * Avoids the per-operation storage round trips of - * {@link executeTransactionOperations} and the adapter-level - * withTransaction snapshotting that dominates small interactive - * transactions. + * Transaction fast path — see `transaction-executor.ts` (0263/0264/0276). */ private async executeTransactionOperationsFast(input: { operations: TransactionOperation[] @@ -1774,83 +1553,7 @@ export class NodeStore { changes: NodeChange[] events: PendingTransactionEvent[] }> { - const planned = input.operations.map((op) => ({ - op, - id: op.type === 'create' ? (op.options.id ?? createNodeId()) : op.nodeId - })) - - const preflight = await this.getBatchPreflight( - planned.map((entry) => entry.id), - this.storage - ) - const nodesById = this.cloneNodeMap(preflight.nodesById) - const lastChanges = new Map(preflight.lastChangesByNodeId) - - const results: (NodeState | null)[] = [] - const changes: NodeChange[] = [] - const events: PendingTransactionEvent[] = [] - const affectedSchemaIds = new Set() - - for (let i = 0; i < planned.length; i++) { - const { op, id } = planned[i] - const existing = nodesById.get(id) ?? null - - if (op.type !== 'create' && !existing) { - throw new Error(`Node not found: ${id}`) - } - - const payload: NodePayload = - op.type === 'create' - ? { nodeId: id, schemaId: op.options.schemaId, properties: op.options.properties } - : op.type === 'update' - ? { nodeId: id, properties: op.options.properties } - : { nodeId: id, properties: {}, deleted: op.type === 'delete' } - - const change = await this.createBatchedChangeWithParentHash( - 'node-change', - payload, - lastChanges.get(id)?.hash ?? null, - input.lamport, - input.now, - input.batchId, - i, - input.batchSize - ) - - const schemaId = existing?.schemaId ?? (op.type === 'create' ? op.options.schemaId : null) - if (!schemaId) { - throw new Error(`First change for node ${id} must include schemaId`) - } - - const node = this.materializeNodeChange( - change, - existing ?? this.createInitialNodeFromChange(change, schemaId) - ) - - nodesById.set(id, node) - lastChanges.set(id, change) - affectedSchemaIds.add(schemaId) - changes.push(change) - const result = op.type === 'delete' ? null : node - results.push(result) - events.push({ change, result, previousNode: existing }) - } - - const touchedIds = Array.from(new Set(planned.map((entry) => entry.id))) - await this.storage.applyNodeBatch!({ - batchId: input.batchId, - nodes: touchedIds.flatMap((id) => { - const node = nodesById.get(id) - return node ? [node] : [] - }), - changes, - lastLamportTime: this.clock.time, - affectedSchemaIds: Array.from(affectedSchemaIds), - indexMode: 'touched', - indexProperties: true - }) - - return { results, changes, events } + return executeTransactionOperationsFast(this.writeExecutionHost(), input) } // ========================================================================== @@ -1945,12 +1648,13 @@ export class NodeStore { * Apply multiple remote changes (from sync). */ async applyRemoteChanges(changes: NodeChange[]): Promise { - // Sort by Lamport timestamp for causal ordering - const sorted = [...changes].sort( - (a, b) => - a.lamport - b.lamport || - // UTF-16 code-unit order (not localeCompare) for deterministic convergence. - (a.authorDID < b.authorDID ? -1 : a.authorDID > b.authorDID ? 1 : 0) + // Sort by Lamport timestamp for causal ordering (the shared protocol + // application order — code-unit author tiebreak, never localeCompare). + const sorted = [...changes].sort((a, b) => + compareChangeApplicationOrder( + { lamport: a.lamport, author: a.authorDID }, + { lamport: b.lamport, author: b.authorDID } + ) ) for (const change of sorted) { @@ -2342,6 +2046,72 @@ export class NodeStore { /** * Apply a change to storage and update materialized state. */ + /** + * The narrow capability set the write orchestration modules + * (transaction-executor.ts, batch-write-orchestrator.ts) run on — one seam + * for every write strategy (exploration 0276). + */ + private writeExecutionHost(): WriteExecutionHost { + this.writeHost ??= { + storage: this.storage, + clockTime: () => this.clock.time, + cloneNodeState: (node) => this.cloneNodeState(node), + cloneNodeMap: (nodesById) => this.cloneNodeMap(nodesById), + getBatchPreflight: (ids, storage) => this.getBatchPreflight(ids, storage), + createBatchedChange: ( + type, + payload, + lamport, + wallTime, + batchId, + batchIndex, + batchSize, + storage + ) => + this.createBatchedChange( + type, + payload, + lamport, + wallTime, + batchId, + batchIndex, + batchSize, + storage + ), + createBatchedChangeWithParentHash: ( + type, + payload, + parentHash, + lamport, + wallTime, + batchId, + batchIndex, + batchSize + ) => + this.createBatchedChangeWithParentHash( + type, + payload, + parentHash, + lamport, + wallTime, + batchId, + batchIndex, + batchSize + ), + applyChange: (change, storage) => this.applyChange(change, storage), + materializeNodeChange: (change, currentNode) => + this.materializeNodeChange(change, currentNode), + createInitialNodeFromChange: (change, schemaId) => + this.createInitialNodeFromChange(change, schemaId), + persistEncryptedNodeSnapshot: (node, storage) => + this.persistEncryptedNodeSnapshot(node, storage), + importMaterializedNodes: (storage, nodes, options) => + this.importMaterializedNodes(storage, nodes, options), + appendImportedChanges: (storage, changes) => this.appendImportedChanges(storage, changes) + } + return this.writeHost + } + private async applyChange( change: NodeChange, storage: NodeStorageAdapter = this.storage @@ -2467,13 +2237,11 @@ export class NodeStore { } /** - * Determine if newTs should replace existingTs (LWW). + * Determine if newTs should replace existingTs (LWW). Delegates to the ONE + * protocol ordering in `@xnetjs/core` (§L1.7; exploration 0276). */ private shouldReplace(existing: PropertyTimestamp, incoming: PropertyTimestamp): boolean { - if (incoming.lamport !== existing.lamport) return incoming.lamport > existing.lamport - if (incoming.wallTime !== existing.wallTime) return incoming.wallTime > existing.wallTime - // UTF-16 code-unit order (not localeCompare) for deterministic convergence. - return incoming.author > existing.author + return lwwWins(incoming, existing) } /** diff --git a/packages/data/src/store/transaction-executor.ts b/packages/data/src/store/transaction-executor.ts new file mode 100644 index 000000000..e41464343 --- /dev/null +++ b/packages/data/src/store/transaction-executor.ts @@ -0,0 +1,314 @@ +/** + * Transaction execution paths for `NodeStore` (exploration 0276). + * + * Both paths take the SAME narrow `WriteExecutionHost` capability set instead + * of reaching into `NodeStore` privates, so the two strategies read + * side-by-side as pure orchestration: + * + * - **Slow path** (`executeTransactionOperations`): one storage round-trip + * per operation inside an adapter transaction — required whenever content + * encryption or adapters without `applyNodeBatch` are in play. + * - **Fast path** (`executeTransactionOperationsFast`): one preflight for all + * touched nodes, in-memory materialization and signing, then ONE + * transactional `applyNodeBatch` (exploration 0263/0264). + * + * Conflict tracking is unified by construction: both paths materialize + * through `host.materializeNodeChange` (→ the shared LWW reducer), and both + * return `PendingTransactionEvent`s for the caller to dispatch, so listener + * behavior cannot drift between strategies. + */ + +import type { SchemaIRI } from '../schema/node' +import type { + NodeBatchPreflightResult, + NodeChange, + NodeId, + NodePayload, + NodeState, + NodeStorageAdapter, + TransactionOperation +} from './types' +import { createNodeId } from '../schema/node' + +/** One executed operation, queued for post-commit listener dispatch. */ +export type PendingTransactionEvent = { + change: NodeChange + result: NodeState | null + previousNode: NodeState | null +} + +export type TransactionExecutionResult = { + results: (NodeState | null)[] + changes: NodeChange[] + events: PendingTransactionEvent[] +} + +/** + * The `NodeStore` capabilities the write orchestration needs — nothing more. + * Implemented by `NodeStore` as bound privates; tests can stub it directly. + */ +export interface WriteExecutionHost { + readonly storage: NodeStorageAdapter + clockTime(): number + cloneNodeState(node: NodeState | null): NodeState | null + cloneNodeMap(nodesById: ReadonlyMap): Map + getBatchPreflight( + ids: readonly NodeId[], + storage: NodeStorageAdapter + ): Promise + createBatchedChange( + type: string, + payload: NodePayload, + lamport: number, + wallTime: number, + batchId: string, + batchIndex: number, + batchSize: number, + storage: NodeStorageAdapter + ): Promise + createBatchedChangeWithParentHash( + type: string, + payload: NodePayload, + parentHash: NodeChange['parentHash'], + lamport: number, + wallTime: number, + batchId: string, + batchIndex: number, + batchSize: number + ): Promise + applyChange(change: NodeChange, storage: NodeStorageAdapter): Promise + materializeNodeChange(change: NodeChange, currentNode: NodeState): NodeState + createInitialNodeFromChange(change: NodeChange, schemaId: SchemaIRI): NodeState + persistEncryptedNodeSnapshot(node: NodeState | null, storage: NodeStorageAdapter): Promise + importMaterializedNodes( + storage: NodeStorageAdapter, + nodes: readonly NodeState[], + options?: { deferIndexes?: boolean } + ): Promise + appendImportedChanges(storage: NodeStorageAdapter, changes: readonly NodeChange[]): Promise +} + +export type TransactionExecutionInput = { + operations: TransactionOperation[] + lamport: number + now: number + batchId: string + batchSize: number +} + +/** Legacy per-operation path (runs inside `storage.withTransaction`). */ +export async function executeTransactionOperations( + host: WriteExecutionHost, + input: TransactionExecutionInput & { storage: NodeStorageAdapter } +): Promise { + const results: (NodeState | null)[] = [] + const changes: NodeChange[] = [] + const events: PendingTransactionEvent[] = [] + + for (let i = 0; i < input.operations.length; i++) { + const op = input.operations[i] + let change: NodeChange + let result: NodeState | null = null + let previousNode: NodeState | null = null + + switch (op.type) { + case 'create': { + const id = op.options.id ?? createNodeId() + const payload: NodePayload = { + nodeId: id, + schemaId: op.options.schemaId, + properties: op.options.properties + } + change = await host.createBatchedChange( + 'node-change', + payload, + input.lamport, + input.now, + input.batchId, + i, + input.batchSize, + input.storage + ) + await host.applyChange(change, input.storage) + result = await input.storage.getNode(id) + await host.persistEncryptedNodeSnapshot(result, input.storage) + break + } + + case 'update': { + const existing = host.cloneNodeState(await input.storage.getNode(op.nodeId)) + if (!existing) { + throw new Error(`Node not found: ${op.nodeId}`) + } + previousNode = existing + const payload: NodePayload = { + nodeId: op.nodeId, + properties: op.options.properties + } + change = await host.createBatchedChange( + 'node-change', + payload, + input.lamport, + input.now, + input.batchId, + i, + input.batchSize, + input.storage + ) + await host.applyChange(change, input.storage) + result = await input.storage.getNode(op.nodeId) + await host.persistEncryptedNodeSnapshot(result, input.storage) + break + } + + case 'delete': { + const existing = host.cloneNodeState(await input.storage.getNode(op.nodeId)) + if (!existing) { + throw new Error(`Node not found: ${op.nodeId}`) + } + previousNode = existing + const payload: NodePayload = { + nodeId: op.nodeId, + properties: {}, + deleted: true + } + change = await host.createBatchedChange( + 'node-change', + payload, + input.lamport, + input.now, + input.batchId, + i, + input.batchSize, + input.storage + ) + await host.applyChange(change, input.storage) + result = null + break + } + + case 'restore': { + const existing = host.cloneNodeState(await input.storage.getNode(op.nodeId)) + if (!existing) { + throw new Error(`Node not found: ${op.nodeId}`) + } + previousNode = existing + const payload: NodePayload = { + nodeId: op.nodeId, + properties: {}, + deleted: false + } + change = await host.createBatchedChange( + 'node-change', + payload, + input.lamport, + input.now, + input.batchId, + i, + input.batchSize, + input.storage + ) + await host.applyChange(change, input.storage) + result = await input.storage.getNode(op.nodeId) + await host.persistEncryptedNodeSnapshot(result, input.storage) + break + } + } + + changes.push(change) + results.push(result) + events.push({ change, result, previousNode }) + } + + return { results, changes, events } +} + +/** + * Transaction fast path: one preflight for all touched nodes, in-memory + * materialization and signing, then one transactional applyNodeBatch. + * Avoids the per-operation storage round trips of + * {@link executeTransactionOperations} and the adapter-level withTransaction + * snapshotting that dominates small interactive transactions. + */ +export async function executeTransactionOperationsFast( + host: WriteExecutionHost, + input: TransactionExecutionInput +): Promise { + const planned = input.operations.map((op) => ({ + op, + id: op.type === 'create' ? (op.options.id ?? createNodeId()) : op.nodeId + })) + + const preflight = await host.getBatchPreflight( + planned.map((entry) => entry.id), + host.storage + ) + const nodesById = host.cloneNodeMap(preflight.nodesById) + const lastChanges = new Map(preflight.lastChangesByNodeId) + + const results: (NodeState | null)[] = [] + const changes: NodeChange[] = [] + const events: PendingTransactionEvent[] = [] + const affectedSchemaIds = new Set() + + for (let i = 0; i < planned.length; i++) { + const { op, id } = planned[i] + const existing = nodesById.get(id) ?? null + + if (op.type !== 'create' && !existing) { + throw new Error(`Node not found: ${id}`) + } + + const payload: NodePayload = + op.type === 'create' + ? { nodeId: id, schemaId: op.options.schemaId, properties: op.options.properties } + : op.type === 'update' + ? { nodeId: id, properties: op.options.properties } + : { nodeId: id, properties: {}, deleted: op.type === 'delete' } + + const change = await host.createBatchedChangeWithParentHash( + 'node-change', + payload, + lastChanges.get(id)?.hash ?? null, + input.lamport, + input.now, + input.batchId, + i, + input.batchSize + ) + + const schemaId = existing?.schemaId ?? (op.type === 'create' ? op.options.schemaId : null) + if (!schemaId) { + throw new Error(`First change for node ${id} must include schemaId`) + } + + const node = host.materializeNodeChange( + change, + existing ?? host.createInitialNodeFromChange(change, schemaId) + ) + + nodesById.set(id, node) + lastChanges.set(id, change) + affectedSchemaIds.add(schemaId) + changes.push(change) + const result = op.type === 'delete' ? null : node + results.push(result) + events.push({ change, result, previousNode: existing }) + } + + const touchedIds = Array.from(new Set(planned.map((entry) => entry.id))) + await host.storage.applyNodeBatch!({ + batchId: input.batchId, + nodes: touchedIds.flatMap((id) => { + const node = nodesById.get(id) + return node ? [node] : [] + }), + changes, + lastLamportTime: host.clockTime(), + affectedSchemaIds: Array.from(affectedSchemaIds), + indexMode: 'touched', + indexProperties: true + }) + + return { results, changes, events } +} diff --git a/packages/editor/package.json b/packages/editor/package.json index d703a2f2e..418da3145 100644 --- a/packages/editor/package.json +++ b/packages/editor/package.json @@ -40,6 +40,7 @@ "@tiptap/suggestion": "^3.15.3", "@tiptap/y-tiptap": "^3.0.1", "@xnetjs/data": "workspace:*", + "@xnetjs/react": "workspace:*", "@xnetjs/ui": "workspace:*", "lucide-react": "^0.563.0", "mermaid": "^11.12.2", diff --git a/packages/editor/src/hooks/usePageComments.ts b/packages/editor/src/hooks/usePageComments.ts new file mode 100644 index 000000000..c591deafa --- /dev/null +++ b/packages/editor/src/hooks/usePageComments.ts @@ -0,0 +1,634 @@ +/** + * usePageComments - the shared page-comment subsystem behind PageView + * (exploration 0276, Theme 3: well-traveled code paths). + * + * The web and desktop PageViews carried ~800-line verbatim copies of the same + * comment state machine: popover show/hide with hover grace timers, text-anchor + * mark restoration, orphaned-thread assembly, thread-data conversion, and the + * reply / resolve / reopen / delete / edit actions (inline popover + sidebar + * variants). This hook owns all of that; the per-app PageViews keep only their + * platform rendering (context panel vs. inline sidebar, editor surface wiring). + * + * Platform deltas preserved as options: + * - `dismissPopoverOnCaretExit` (desktop): dismiss the popover when the caret + * moves outside the popover thread's comment mark. + */ +import type { AnyExtension } from '@tiptap/core' +import type { Editor } from '@tiptap/react' +import type { CommentThreadData, OrphanedThread } from '@xnetjs/ui' +import { PageSchema } from '@xnetjs/data' +import { useComments, type CommentThread } from '@xnetjs/react' +import { useCallback, useEffect, useMemo, useRef, useState, type MutableRefObject } from 'react' +import { CommentMark, CommentPlugin, restoreCommentMarks } from '../extensions/comment' + +// ─── Types ───────────────────────────────────────────────────────────────────── + +/** Comment popover visibility / anchoring state. */ +export interface PageCommentPopoverState { + visible: boolean + mode: 'preview' | 'full' + threadId: string | null + anchor: HTMLElement | null +} + +const INITIAL_POPOVER_STATE: PageCommentPopoverState = { + visible: false, + mode: 'preview', + threadId: null, + anchor: null +} + +/** State for creating a new comment (before submission). */ +export interface PageNewCommentState { + visible: boolean + anchorData: string + /** Selection range to restore when applying the mark */ + selectionFrom: number + selectionTo: number +} + +export interface UsePageCommentsOptions { + /** The page Node ID comments target. */ + docId: string + /** + * Dismiss the popover when the caret moves outside the popover thread's + * comment mark (TipTap `selectionUpdate`). Desktop PageView behavior. + */ + dismissPopoverOnCaretExit?: boolean +} + +export interface UsePageCommentsResult { + // Data + threads: CommentThread[] + unresolvedCount: number + threadDataMap: Map + sidebarThreads: CommentThreadData[] + /** Thread backing the popover, or null when it is not (yet) loaded. */ + currentThread: CommentThreadData | null + orphanedThreads: OrphanedThread[] + orphanedCollapsed: boolean + toggleOrphanedCollapsed: () => void + popoverState: PageCommentPopoverState + newCommentState: PageNewCommentState | null + + // Editor wiring + editorRef: MutableRefObject + editorReady: boolean + handleEditorReady: (editor: Editor) => void + /** CommentMark + CommentPlugin wired to the popover handlers. */ + commentExtensions: AnyExtension[] + + // Popover + showThreadPopover: (threadId: string, anchor: HTMLElement | null) => void + handlePopoverMouseEnter: () => void + handlePopoverMouseLeave: () => void + handleDismiss: () => void + handleUpgradeToFull: () => void + + // Comment actions (popover-scoped) + handleReply: (content: string) => Promise + handleResolve: () => Promise + handleReopen: () => Promise + handleDelete: (commentId: string) => Promise + handleEdit: (commentId: string, newContent: string) => Promise + + // New-comment flow + handleCreateComment: (anchorData: string) => Promise + handleSubmitNewComment: (content: string) => Promise + handleCancelNewComment: () => void + + // Sidebar actions (thread-id scoped) + handleSidebarSelectThread: (threadId: string) => void + handleSidebarReply: (threadId: string, content: string) => Promise + handleSidebarResolve: (threadId: string) => Promise + handleSidebarReopen: (threadId: string) => Promise + handleSidebarDelete: (commentId: string) => Promise + handleSidebarEdit: (commentId: string, newContent: string) => Promise + + // Orphaned threads + handleDismissOrphaned: (commentId: string) => Promise + handleReattachOrphaned: (commentId: string) => void +} + +// ─── Hook ────────────────────────────────────────────────────────────────────── + +export function usePageComments({ + docId, + dismissPopoverOnCaretExit = false +}: UsePageCommentsOptions): UsePageCommentsResult { + // Load comments for this page, filtered to text anchors only + const { + threads, + addComment, + replyTo, + resolveThread, + reopenThread, + deleteComment, + editComment, + unresolvedCount + } = useComments({ nodeId: docId, anchorType: 'text' }) + + // Popover state for comment interactions + const [popoverState, setPopoverState] = useState(INITIAL_POPOVER_STATE) + const [newCommentState, setNewCommentState] = useState(null) + const [orphanedIds, setOrphanedIds] = useState([]) + const [orphanedCollapsed, setOrphanedCollapsed] = useState(false) + const hoverTimeoutRef = useRef | null>(null) + const dismissTimeoutRef = useRef | null>(null) + const editorRef = useRef(null) + const marksRestoredRef = useRef(false) + const [editorReady, setEditorReady] = useState(false) + + // Track hover state for mark and popover; the popover stays open as + // long as either is hovered. + const markHoveredRef = useRef(false) + const popoverHoveredRef = useRef(false) + + // Reset mark restoration state when switching documents. Skip the + // initial run: parent effects fire after the editor's ready + // notification, so an unconditional reset would null the ref the + // moment it was set. + const lastDocIdRef = useRef(docId) + useEffect(() => { + if (lastDocIdRef.current === docId) return + lastDocIdRef.current = docId + marksRestoredRef.current = false + editorRef.current = null + setEditorReady(false) + }, [docId]) + + // Handle editor ready - store ref and trigger mark restoration + const handleEditorReady = useCallback((editor: Editor) => { + editorRef.current = editor + setEditorReady(true) + }, []) + + // Restore comment marks when editor is ready and threads are loaded. + // Both editorReady and threads are in the dependency array so the effect + // fires regardless of which one becomes available first. + useEffect(() => { + if (!editorRef.current || marksRestoredRef.current || threads.length === 0) return + + const commentsToRestore = threads.map((t) => ({ + id: t.root.id, + properties: { + anchorType: t.root.properties.anchorType, + anchorData: t.root.properties.anchorData, + resolved: t.root.properties.resolved + } + })) + + const { resolved, orphaned } = restoreCommentMarks(editorRef.current, commentsToRestore) + + if (resolved.length > 0 || orphaned.length > 0) { + marksRestoredRef.current = true + setOrphanedIds(orphaned) + console.log(`[Comments] Restored ${resolved.length} marks, ${orphaned.length} orphaned`) + } + }, [threads, editorReady]) + + // Dismiss the comment popover when the caret moves out of comment marks + // (opt-in; TipTap's onSelectionUpdate fires after every cursor movement). + useEffect(() => { + if (!dismissPopoverOnCaretExit) return + const editor = editorRef.current + if (!editor) return + + const onSelectionUpdate = () => { + setPopoverState((prev) => { + if (!prev.visible || !prev.threadId) return prev + const { from } = editor.state.selection + const resolved = editor.state.doc.resolve(from) + const inComment = resolved.marks().some((mark) => { + const typedMark = mark as { + type?: { name?: string } + attrs?: { commentId?: string } + } + return typedMark.type?.name === 'comment' && typedMark.attrs?.commentId === prev.threadId + }) + if (!inComment && !markHoveredRef.current && !popoverHoveredRef.current) { + return INITIAL_POPOVER_STATE + } + return prev + }) + } + + editor.on('selectionUpdate', onSelectionUpdate) + return () => { + editor.off('selectionUpdate', onSelectionUpdate) + } + }, [dismissPopoverOnCaretExit, editorReady]) + + // Build orphaned threads list for display + const orphanedThreads = useMemo((): OrphanedThread[] => { + const result: OrphanedThread[] = [] + + for (const id of orphanedIds) { + const thread = threads.find((t) => t.root.id === id) + if (!thread) continue + + // Parse anchor data to get context + let context: string | undefined + try { + const anchor = JSON.parse(thread.root.properties.anchorData) + context = anchor.quotedText + } catch { + // Ignore parse errors + } + + result.push({ + comment: { + id: thread.root.id, + author: thread.root.properties.createdBy, + authorDisplayName: undefined, + content: thread.root.properties.content, + createdAt: thread.root.createdAt, + replyCount: thread.replies.length + }, + reason: 'text-deleted', + context + }) + } + + return result + }, [orphanedIds, threads]) + + // Convert threads to format expected by CommentPopover/CommentsSidebar + const threadDataMap = useMemo(() => { + const map = new Map() + for (const thread of threads) { + map.set(thread.root.id, { + root: { + id: thread.root.id, + author: thread.root.properties.createdBy, + authorDisplayName: undefined, + content: thread.root.properties.content, + createdAt: thread.root.createdAt, + edited: thread.root.properties.edited, + editedAt: thread.root.properties.editedAt, + replyToUser: thread.root.properties.replyToUser, + replyToCommentId: thread.root.properties.replyToCommentId + }, + replies: thread.replies.map((r) => ({ + id: r.id, + author: r.properties.createdBy, + authorDisplayName: undefined, + content: r.properties.content, + createdAt: r.createdAt, + edited: r.properties.edited, + editedAt: r.properties.editedAt, + replyToUser: r.properties.replyToUser, + replyToCommentId: r.properties.replyToCommentId + })), + resolved: thread.root.properties.resolved + }) + } + return map + }, [threads]) + + // ─── Popover Handlers ───────────────────────────────────────────────────────── + + /** Check if the editor caret is currently inside a comment mark. */ + const isCaretInComment = useCallback((): boolean => { + const editor = editorRef.current + if (!editor) return false + const { from } = editor.state.selection + const resolved = editor.state.doc.resolve(from) + return resolved.marks().some((mark) => { + const typedMark = mark as { type?: { name?: string } } + return typedMark.type?.name === 'comment' + }) + }, []) + + /** Schedule a dismiss after a short delay, unless mark/popover is hovered or caret is in comment. */ + const scheduleDismiss = useCallback(() => { + if (dismissTimeoutRef.current) clearTimeout(dismissTimeoutRef.current) + dismissTimeoutRef.current = setTimeout(() => { + if (!markHoveredRef.current && !popoverHoveredRef.current && !isCaretInComment()) { + setPopoverState(INITIAL_POPOVER_STATE) + } + }, 200) + }, [isCaretInComment]) + + const handleClickComment = useCallback((commentId: string, anchorEl: HTMLElement) => { + if (hoverTimeoutRef.current) clearTimeout(hoverTimeoutRef.current) + if (dismissTimeoutRef.current) clearTimeout(dismissTimeoutRef.current) + setPopoverState((prev) => { + // Already showing for this comment — keep as-is to avoid flicker + if (prev.visible && prev.mode === 'full' && prev.threadId === commentId) return prev + return { visible: true, mode: 'full', threadId: commentId, anchor: anchorEl } + }) + }, []) + + const handleHoverComment = useCallback((commentId: string, anchorEl: HTMLElement) => { + markHoveredRef.current = true + // Cancel any pending dismiss; delay showing to avoid flicker on quick passes + if (dismissTimeoutRef.current) clearTimeout(dismissTimeoutRef.current) + if (hoverTimeoutRef.current) clearTimeout(hoverTimeoutRef.current) + hoverTimeoutRef.current = setTimeout(() => { + setPopoverState((prev) => { + if (prev.visible && prev.threadId === commentId) return prev + return { visible: true, mode: 'full', threadId: commentId, anchor: anchorEl } + }) + }, 300) + }, []) + + const handleLeaveComment = useCallback(() => { + markHoveredRef.current = false + if (hoverTimeoutRef.current) clearTimeout(hoverTimeoutRef.current) + scheduleDismiss() + }, [scheduleDismiss]) + + const handlePopoverMouseEnter = useCallback(() => { + popoverHoveredRef.current = true + if (dismissTimeoutRef.current) clearTimeout(dismissTimeoutRef.current) + }, []) + + const handlePopoverMouseLeave = useCallback(() => { + popoverHoveredRef.current = false + scheduleDismiss() + }, [scheduleDismiss]) + + const handleDismiss = useCallback(() => { + if (hoverTimeoutRef.current) clearTimeout(hoverTimeoutRef.current) + if (dismissTimeoutRef.current) clearTimeout(dismissTimeoutRef.current) + markHoveredRef.current = false + popoverHoveredRef.current = false + setPopoverState(INITIAL_POPOVER_STATE) + }, []) + + const handleUpgradeToFull = useCallback(() => { + setPopoverState((prev) => ({ ...prev, mode: 'full' })) + }, []) + + /** Show the popover for a thread (e.g. sidebar selection, orphaned threads). */ + const showThreadPopover = useCallback((threadId: string, anchor: HTMLElement | null) => { + setPopoverState({ visible: true, mode: 'full', threadId, anchor }) + }, []) + + // ─── Comment Actions ────────────────────────────────────────────────────────── + + const handleReply = useCallback( + async (content: string) => { + if (!popoverState.threadId) return + await replyTo(popoverState.threadId, content) + }, + [popoverState.threadId, replyTo] + ) + + const handleResolve = useCallback(async () => { + if (!popoverState.threadId) return + await resolveThread(popoverState.threadId) + // Update the mark visual state to resolved (amber -> green) + editorRef.current?.commands.setCommentResolved(popoverState.threadId, true) + }, [popoverState.threadId, resolveThread]) + + const handleReopen = useCallback(async () => { + if (!popoverState.threadId) return + await reopenThread(popoverState.threadId) + // Update the mark visual state back to active (green -> amber) + editorRef.current?.commands.setCommentResolved(popoverState.threadId, false) + }, [popoverState.threadId, reopenThread]) + + const handleDelete = useCallback( + async (commentId: string) => { + await deleteComment(commentId) + // If deleting root with no replies, remove the mark from the document and close popover + const thread = threadDataMap.get(popoverState.threadId || '') + if (thread && commentId === thread.root.id && thread.replies.length === 0) { + const editor = editorRef.current + if (editor) { + const { tr, doc: editorDoc } = editor.state + const markType = editor.schema.marks.comment + if (markType) { + editorDoc.descendants((node, pos) => { + node.marks.forEach((mark) => { + if (mark.type === markType && mark.attrs.commentId === commentId) { + tr.removeMark(pos, pos + node.nodeSize, mark) + } + }) + }) + editor.view.dispatch(tr) + } + } + handleDismiss() + } + }, + [deleteComment, threadDataMap, popoverState.threadId, handleDismiss] + ) + + const handleEdit = useCallback( + async (commentId: string, newContent: string) => { + await editComment(commentId, newContent) + }, + [editComment] + ) + + // Handler for initiating comment creation from toolbar selection. + // This shows the input UI; actual comment creation happens on submit. + const handleCreateComment = useCallback(async (anchorData: string): Promise => { + if (!editorRef.current) return null + // Capture the current selection range so we can apply the mark later + const { from, to } = editorRef.current.state.selection + if (from === to) return null + + setNewCommentState({ + visible: true, + anchorData, + selectionFrom: from, + selectionTo: to + }) + return null + }, []) + + // Handler for submitting a new comment + const handleSubmitNewComment = useCallback( + async (content: string) => { + if (!newCommentState || !content.trim() || !editorRef.current) return + + const commentId = await addComment({ + content: content.trim(), + anchorType: 'text', + anchorData: newCommentState.anchorData, + targetSchema: PageSchema.schema['@id'] + }) + + if (commentId) { + // Apply the mark: set selection to the original range, then mark it + editorRef.current + .chain() + .focus() + .setTextSelection({ + from: newCommentState.selectionFrom, + to: newCommentState.selectionTo + }) + .setComment(commentId) + .run() + + // After a short delay, find the mark element and show the popover. + // This gives time for the DOM and threads state to update. + const showPopover = () => { + const markEl = document.querySelector( + `[data-comment-id="${commentId}"]` + ) as HTMLElement | null + if (markEl) { + setPopoverState({ + visible: true, + mode: 'full', + threadId: commentId, + anchor: markEl + }) + } + } + // Try immediately, then retry after a delay if needed + setTimeout(showPopover, 50) + setTimeout(showPopover, 200) + } + + setNewCommentState(null) + }, + [newCommentState, addComment] + ) + + const handleCancelNewComment = useCallback(() => { + setNewCommentState(null) + }, []) + + // ─── Sidebar Handlers ───────────────────────────────────────────────────────── + + const handleSidebarSelectThread = useCallback( + (threadId: string) => { + // Find and scroll to the comment mark in the editor + const markEl = document.querySelector(`[data-comment-id="${threadId}"]`) as HTMLElement | null + if (markEl) { + markEl.scrollIntoView({ behavior: 'smooth', block: 'center' }) + showThreadPopover(threadId, markEl) + } + }, + [showThreadPopover] + ) + + const handleSidebarReply = useCallback( + async (threadId: string, content: string) => { + await replyTo(threadId, content) + }, + [replyTo] + ) + + const handleSidebarResolve = useCallback( + async (threadId: string) => { + await resolveThread(threadId) + editorRef.current?.commands.setCommentResolved(threadId, true) + }, + [resolveThread] + ) + + const handleSidebarReopen = useCallback( + async (threadId: string) => { + await reopenThread(threadId) + editorRef.current?.commands.setCommentResolved(threadId, false) + }, + [reopenThread] + ) + + const handleSidebarDelete = useCallback( + async (commentId: string) => { + await deleteComment(commentId) + }, + [deleteComment] + ) + + const handleSidebarEdit = useCallback( + async (commentId: string, newContent: string) => { + await editComment(commentId, newContent) + }, + [editComment] + ) + + // ─── Orphaned Comment Handlers ───────────────────────────────────────────────── + + const handleDismissOrphaned = useCallback( + async (commentId: string) => { + // Delete the orphaned thread entirely + const thread = threads.find((t) => t.root.id === commentId) + if (thread) { + // Delete replies first, then root + for (const reply of thread.replies) { + await deleteComment(reply.id) + } + await deleteComment(commentId) + } + // Remove from orphaned list + setOrphanedIds((prev) => prev.filter((id) => id !== commentId)) + }, + [threads, deleteComment] + ) + + const handleReattachOrphaned = useCallback((commentId: string) => { + // For now, just log - reattachment requires selecting new text + console.log(`[Comments] Reattach not yet implemented for ${commentId}`) + }, []) + + const toggleOrphanedCollapsed = useCallback(() => { + setOrphanedCollapsed((prev) => !prev) + }, []) + + // ─── Comment Extensions ─────────────────────────────────────────────────────── + + const commentExtensions = useMemo( + () => [ + CommentMark, + CommentPlugin.configure({ + onClickComment: handleClickComment, + onHoverComment: handleHoverComment, + onLeaveComment: handleLeaveComment + }) + ], + [handleClickComment, handleHoverComment, handleLeaveComment] + ) + + // Get the current thread for the popover. If the thread is not in the map + // yet (newly created), it will show once threads update. + const currentThread = popoverState.threadId + ? (threadDataMap.get(popoverState.threadId) ?? null) + : null + const sidebarThreads = useMemo(() => Array.from(threadDataMap.values()), [threadDataMap]) + + return { + threads, + unresolvedCount, + threadDataMap, + sidebarThreads, + currentThread, + orphanedThreads, + orphanedCollapsed, + toggleOrphanedCollapsed, + popoverState, + newCommentState, + editorRef, + editorReady, + handleEditorReady, + commentExtensions, + showThreadPopover, + handlePopoverMouseEnter, + handlePopoverMouseLeave, + handleDismiss, + handleUpgradeToFull, + handleReply, + handleResolve, + handleReopen, + handleDelete, + handleEdit, + handleCreateComment, + handleSubmitNewComment, + handleCancelNewComment, + handleSidebarSelectThread, + handleSidebarReply, + handleSidebarResolve, + handleSidebarReopen, + handleSidebarDelete, + handleSidebarEdit, + handleDismissOrphaned, + handleReattachOrphaned + } +} diff --git a/packages/editor/src/react.ts b/packages/editor/src/react.ts index ca218b007..14431a8a4 100644 --- a/packages/editor/src/react.ts +++ b/packages/editor/src/react.ts @@ -195,3 +195,13 @@ export { // Re-export hooks from @tiptap/react for convenience export { useEditor, EditorContent } from '@tiptap/react' export type { Editor } from '@tiptap/react' + +// Shared page-comment subsystem (exploration 0276): the comment state +// machine both the web and desktop PageViews consume. +export { usePageComments } from './hooks/usePageComments' +export type { + PageCommentPopoverState, + PageNewCommentState, + UsePageCommentsOptions, + UsePageCommentsResult +} from './hooks/usePageComments' diff --git a/packages/hub/src/server.ts b/packages/hub/src/server.ts index 1e772ea08..b3cd25515 100644 --- a/packages/hub/src/server.ts +++ b/packages/hub/src/server.ts @@ -3,7 +3,6 @@ */ import type { AuthSession } from './auth/ucan' -import type { HubStorage, SerializedNodeChange } from './storage/interface' import type { HubConfig, HubInstance } from './types' import type { MiddlewareHandler } from 'hono' import type { IncomingMessage } from 'http' @@ -15,7 +14,6 @@ import { generateIdentity } from '@xnetjs/identity' import { Hono } from 'hono' import { cors } from 'hono/cors' import { WebSocketServer } from 'ws' -import { hasHubCapability } from './auth/capabilities' import { createHubAuthError } from './auth/errors' import { authenticateConnection, @@ -54,10 +52,9 @@ import { FederationHealthChecker } from './services/federation-health' import { FileService } from './services/files' import { ShardRegistry } from './services/index-shards' import { KeyRegistryService } from './services/key-registry' -import { NodeRelayError, NodeRelayService } from './services/node-relay' +import { NodeRelayService } from './services/node-relay' import { QueryService } from './services/query' import { RelayService } from './services/relay' -import { reportUnauthorizedRemoteWrite } from './services/remote-mutation-telemetry' import { SchemaRegistryService } from './services/schemas' import { ShardIngestRouter } from './services/shard-ingest' import { ShardRebalancer } from './services/shard-rebalancer' @@ -67,6 +64,10 @@ import { createSignalingService } from './services/signaling' import { TaskIdentifierService } from './services/task-identifiers' import { createStorage } from './storage' import { setupHubTelemetry } from './telemetry/bridge' +import { authorizeRoomAction, denyAndCloseSocket } from './ws/authorize' +import { buildWsError } from './ws/errors' +import { isRecord } from './ws/guards' +import { createWsMessageRouter } from './ws/register' const getMessageSize = (data: RawData): number => { if (typeof data === 'string') { @@ -96,381 +97,6 @@ const safeParseJson = (payload: string): unknown | null => { } } -const parseTopics = (value: unknown): string[] => - Array.isArray(value) ? value.filter((entry): entry is string => typeof entry === 'string') : [] - -const isSubscribeMessage = (value: unknown): value is { type: 'subscribe'; topics?: unknown } => { - if (!value || typeof value !== 'object') return false - const candidate = value as { type?: unknown } - return candidate.type === 'subscribe' -} - -const isUnsubscribeMessage = ( - value: unknown -): value is { type: 'unsubscribe'; topics?: unknown } => { - if (!value || typeof value !== 'object') return false - const candidate = value as { type?: unknown } - return candidate.type === 'unsubscribe' -} - -const isPublishMessage = ( - value: unknown -): value is { type: 'publish'; topic?: unknown; data?: unknown } => { - if (!value || typeof value !== 'object') return false - const candidate = value as { type?: unknown } - return candidate.type === 'publish' -} - -const isRecord = (value: unknown): value is Record => - Boolean(value && typeof value === 'object' && !Array.isArray(value)) - -const isQueryRequest = ( - value: unknown -): value is { type: 'query-request'; id: string; query: string; federate?: boolean } => { - if (!isRecord(value)) return false - return ( - value.type === 'query-request' && - typeof value.id === 'string' && - typeof value.query === 'string' - ) -} - -const isIndexUpdate = ( - value: unknown -): value is { type: 'index-update'; docId: string; meta: { schemaIri: string; title: string } } => { - if (!isRecord(value)) return false - if (value.type !== 'index-update') return false - if (typeof value.docId !== 'string') return false - if (!isRecord(value.meta)) return false - return typeof value.meta.schemaIri === 'string' && typeof value.meta.title === 'string' -} - -const isIndexRemove = (value: unknown): value is { type: 'index-remove'; docId: string } => { - if (!isRecord(value)) return false - return value.type === 'index-remove' && typeof value.docId === 'string' -} - -const isNodeSyncRequest = ( - value: unknown -): value is { type: 'node-sync-request'; room: string; sinceLamport: number } => { - if (!isRecord(value)) return false - return ( - value.type === 'node-sync-request' && - typeof value.room === 'string' && - typeof value.sinceLamport === 'number' - ) -} - -const isNodeClearRequest = (value: unknown): value is { type: 'node-clear'; room: string } => { - if (!isRecord(value)) return false - return value.type === 'node-clear' && typeof value.room === 'string' -} - -const isNodeChangePayload = ( - value: unknown -): value is { type: 'node-change'; room: string; change: SerializedNodeChange } => { - if (!isRecord(value)) return false - if (value.type !== 'node-change' || typeof value.room !== 'string') return false - if (!isRecord(value.change)) return false - const change = value.change as Record - return typeof change.hash === 'string' && typeof change.signatureB64 === 'string' -} - -const isAwarenessMessage = ( - value: unknown -): value is { type: 'awareness'; update?: string; state?: unknown } => { - if (!isRecord(value)) return false - if (value.type !== 'awareness') return false - const candidate = value as { update?: unknown; state?: unknown } - return ( - (typeof candidate.update === 'string' && candidate.update.length > 0) || - typeof candidate.state !== 'undefined' - ) -} - -const isSyncRelayMessage = ( - value: unknown -): value is { type: 'sync-step1' | 'sync-step2' | 'sync-update'; from?: unknown } => { - if (!isRecord(value)) return false - return value.type === 'sync-step1' || value.type === 'sync-step2' || value.type === 'sync-update' -} - -const isClientHandshake = ( - value: unknown -): value is { - type: 'client-handshake' - did: string - protocolVersion: number - minProtocolVersion: number - features: string[] - packageVersion: string -} => { - if (!isRecord(value)) return false - if (value.type !== 'client-handshake') return false - return ( - typeof value.did === 'string' && - typeof value.protocolVersion === 'number' && - typeof value.minProtocolVersion === 'number' && - Array.isArray(value.features) && - typeof value.packageVersion === 'string' - ) -} - -const topicToResource = (topic: string): string => - topic.startsWith('xnet-doc-') ? topic.slice('xnet-doc-'.length) : topic - -// ─── Space containment maintenance (exploration 0179) ───────────────────────── -// Schemas that carry a `space` relation (their canonical security home). -const SPACEABLE_SCHEMA_PREFIXES = [ - 'xnet://xnet.fyi/Page', - 'xnet://xnet.fyi/Database', - 'xnet://xnet.fyi/Canvas', - 'xnet://xnet.fyi/Dashboard', - 'xnet://xnet.fyi/Project', - 'xnet://xnet.fyi/Channel', - 'xnet://xnet.fyi/Task' -] -const SPACE_SCHEMA_PREFIX = 'xnet://xnet.fyi/Space' - -const firstRelationId = (value: unknown): string | null => { - if (typeof value === 'string') return value.trim() || null - if (Array.isArray(value)) return value.length > 0 ? firstRelationId(value[0]) : null - if (value && typeof value === 'object' && 'id' in value) { - const id = (value as { id?: unknown }).id - return typeof id === 'string' ? id.trim() || null : null - } - return null -} - -type ContainmentChange = { - nodeId?: string - schemaId?: string - payload?: { - nodeId?: string - schemaId?: string - properties?: Record - deleted?: boolean - } -} - -/** - * Keep the hub's node→container index fresh from relayed node-changes so - * container (Space) grants resolve. A content node's container is its `space`; - * a Space's container is its `parent`. Only updates when the relevant property - * is actually present in the change (partial CRDT updates never clobber it). - */ -const maintainSpaceContainment = async ( - storage: HubStorage, - change: ContainmentChange -): Promise => { - const nodeId = change.payload?.nodeId ?? change.nodeId - const schemaId = change.schemaId ?? change.payload?.schemaId - const properties = change.payload?.properties - if (!nodeId || !schemaId || !properties || change.payload?.deleted) return - const hasKey = (k: string): boolean => Object.prototype.hasOwnProperty.call(properties, k) - const recordVisibility = async (): Promise => { - if (!hasKey('visibility')) return - const value = properties.visibility - await storage.setNodeVisibility(nodeId, typeof value === 'string' ? value : null) - } - if (schemaId.startsWith(SPACE_SCHEMA_PREFIX)) { - if (hasKey('parent')) await storage.setNodeContainer(nodeId, firstRelationId(properties.parent)) - await recordVisibility() - return - } - if (SPACEABLE_SCHEMA_PREFIXES.some((prefix) => schemaId.startsWith(prefix))) { - if (hasKey('space')) await storage.setNodeContainer(nodeId, firstRelationId(properties.space)) - await recordVisibility() - } -} - -const getPublishPeerId = (payload: { data?: unknown }): string | null => { - if (!isRecord(payload.data)) return null - return typeof payload.data.from === 'string' ? payload.data.from : null -} - -type AuthzCode = 'UNAUTHORIZED' | 'TOKEN_EXPIRED' | 'TOKEN_REVOKED' - -type AuthzDecision = { - allowed: boolean - code?: AuthzCode - message?: string - source?: 'capability' | 'grant-index' | 'space-grant' -} - -const isTokenExpired = (session: AuthSession): boolean => { - const exp = session.token?.exp - if (typeof exp !== 'number') { - return false - } - return exp <= Math.floor(Date.now() / 1000) -} - -const logAuthDecision = (input: { - allowed: boolean - did: string - action: string - resource: string - source?: 'capability' | 'grant-index' | 'space-grant' - code?: AuthzCode - reason?: string -}): void => { - const base = `[AuthZ] ${input.allowed ? 'allow' : 'deny'} ${input.action} resource=${input.resource} did=${input.did}` - if (input.allowed) { - console.log(`${base} source=${input.source ?? 'capability'}`) - return - } - console.warn( - `${base} code=${input.code ?? 'UNAUTHORIZED'} reason=${input.reason ?? 'unauthorized'}` - ) -} - -const authorizeRoomAction = async (input: { - storage: HubStorage - session: AuthSession - action: 'hub/relay' | 'hub/signal' - topic: string - shareAccess?: ShareAccessService -}): Promise => { - const resource = topicToResource(input.topic) - - if (isTokenExpired(input.session)) { - const decision: AuthzDecision = { - allowed: false, - code: 'TOKEN_EXPIRED', - message: 'Authentication token has expired' - } - logAuthDecision({ - allowed: false, - did: input.session.did, - action: input.action, - resource, - code: decision.code, - reason: decision.message - }) - return decision - } - - // A DID whose share grants were all revoked ("remove access") is denied - // outright — wildcard self-issued capabilities do not restore access. - if ( - input.shareAccess && - input.session.did !== 'did:key:anonymous' && - (await input.shareAccess.isDenied(input.session.did, resource)) - ) { - const decision: AuthzDecision = { - allowed: false, - code: 'TOKEN_REVOKED', - message: 'Access to this resource has been revoked' - } - logAuthDecision({ - allowed: false, - did: input.session.did, - action: input.action, - resource, - code: decision.code, - reason: decision.message - }) - return decision - } - - if ( - hasHubCapability(input.session.capabilities, input.action, resource) || - hasHubCapability(input.session.capabilities, 'hub/signal', resource) - ) { - logAuthDecision({ - allowed: true, - did: input.session.did, - action: input.action, - resource, - source: 'capability' - }) - return { allowed: true, source: 'capability' } - } - - const grantedDocIds = await input.storage.listGrantedDocIds(input.session.did) - if (grantedDocIds.includes(resource)) { - logAuthDecision({ - allowed: true, - did: input.session.did, - action: input.action, - resource, - source: 'grant-index' - }) - return { allowed: true, source: 'grant-index' } - } - - // Container (Space) membership: a member of an ancestor Space may access nodes - // beneath it even without a direct per-doc grant (exploration 0179). - if ( - input.shareAccess && - input.session.did !== 'did:key:anonymous' && - (await input.shareAccess.canAccessNode(input.session.did, resource)) - ) { - logAuthDecision({ - allowed: true, - did: input.session.did, - action: input.action, - resource, - source: 'space-grant' - }) - return { allowed: true, source: 'space-grant' } - } - - if (Array.isArray(input.session.token?.prf) && input.session.token.prf.length > 0) { - const decision: AuthzDecision = { - allowed: false, - code: 'TOKEN_REVOKED', - message: 'Grant token is no longer active for this resource' - } - logAuthDecision({ - allowed: false, - did: input.session.did, - action: input.action, - resource, - code: decision.code, - reason: decision.message - }) - return decision - } - - const decision: AuthzDecision = { - allowed: false, - code: 'UNAUTHORIZED', - message: 'Capability and grant index checks denied access' - } - logAuthDecision({ - allowed: false, - did: input.session.did, - action: input.action, - resource, - code: decision.code, - reason: decision.message - }) - return decision -} - -const checkRoomAuth = async ( - storage: HubStorage, - session: AuthSession, - topics: string[], - shareAccess?: ShareAccessService -): Promise<{ ok: true } | { ok: false; topic: string; decision: AuthzDecision }> => { - for (const topic of topics) { - const decision = await authorizeRoomAction({ - storage, - session, - action: 'hub/signal', - topic, - shareAccess - }) - if (!decision.allowed) { - return { ok: false, topic, decision } - } - } - return { ok: true } -} - type ShareHandleDocType = 'page' | 'database' | 'canvas' type ShareHandleRecord = { @@ -682,25 +308,8 @@ export const createServer = async (config: HubConfig): Promise => { } } - const denyAndCloseSocket = ( - ws: WebSocket, - decision: AuthzDecision, - action: 'hub/relay' | 'hub/signal', - topic: string - ): void => { - ws.send( - JSON.stringify({ - type: 'auth-denied', - code: decision.code ?? 'UNAUTHORIZED', - action, - resource: topicToResource(topic), - error: decision.message ?? 'Insufficient capabilities for room' - }) - ) - metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) - ws.close(4403, decision.message ?? 'Insufficient capabilities for room') - } - + // Periodic re-check of live subscriptions (token expiry / revocation) — + // resolves through the same unified room-auth path as the message handlers. const enforceSocketTopicAuth = async (ws: WebSocket): Promise => { const session = socketSessions.get(ws) if (!session) return @@ -716,12 +325,32 @@ export const createServer = async (config: HubConfig): Promise => { shareAccess }) if (!decision.allowed) { - denyAndCloseSocket(ws, decision, 'hub/signal', topic) + denyAndCloseSocket(ws, decision, 'hub/signal', topic, metrics) return } } } + // WebSocket message router (exploration 0276 Theme 2): every message-type + // handler lives under src/ws/handlers/, registered in the pump's original + // branch order by createWsMessageRouter. + const messageRouter = createWsMessageRouter({ + config, + storage, + metrics, + query, + federation, + federationEnabled: federationConfig.enabled, + nodeRelay, + shareAccess, + awareness, + relay, + signaling, + remoteMutationTelemetry, + socketTopics, + socketPeers + }) + // On-disk usage is cached (a recursive size walk shouldn't run on every poll — // /health is hit by the control-plane probe + the dashboard). 30s is plenty. let usageCache: { at: number; usage: DataUsage } | null = null @@ -1208,505 +837,14 @@ export const createServer = async (config: HubConfig): Promise => { ws.close(1008, 'Rate limit exceeded') return } - ws.send(JSON.stringify({ type: 'error', message: check.reason })) + ws.send(JSON.stringify(buildWsError({ kind: 'error', message: check.reason }))) return } const payload = safeParseJson(dataToString(data)) if (!payload) return - metrics.increment(HUB_METRICS.WS_MESSAGES_RECEIVED) - - // Handle client handshake (version negotiation) - if (isClientHandshake(payload)) { - const hubProtocolVersion = 1 - const hubMinProtocolVersion = 1 - - // Check version compatibility - const clientMax = payload.protocolVersion - const clientMin = payload.minProtocolVersion - - // Find compatible version range - const agreedVersion = Math.min(hubProtocolVersion, clientMax) - const minRequired = Math.max(hubMinProtocolVersion, clientMin) - - if (agreedVersion < minRequired) { - // Versions are incompatible - const suggestion = - clientMax < hubMinProtocolVersion - ? 'upgrade-client' - : hubProtocolVersion < clientMin - ? 'upgrade-hub' - : 'incompatible' - - ws.send( - JSON.stringify({ - type: 'version-mismatch', - hubVersion: hubProtocolVersion, - clientVersion: clientMax, - suggestion, - message: - suggestion === 'upgrade-client' - ? `Client protocol v${clientMax} is too old. Please upgrade to at least v${hubMinProtocolVersion}.` - : suggestion === 'upgrade-hub' - ? `Hub protocol v${hubProtocolVersion} is too old for client v${clientMin}.` - : 'Protocol versions are incompatible.' - }) - ) - metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) - // Don't close the connection - just warn - } else if (clientMax < hubProtocolVersion) { - // Client is using older version - log for metrics - console.log( - `Client ${payload.did} using older protocol v${clientMax} (hub is v${hubProtocolVersion})` - ) - } - return - } - - if (isQueryRequest(payload)) { - if (!authContext.can('query/read', '*')) { - const authError = createHubAuthError({ - code: 'FORBIDDEN', - message: 'Capability does not allow querying', - action: 'hub/query' - }) - ws.send( - JSON.stringify({ - type: 'query-error', - id: payload.id, - error: authError.message, - code: authError.code, - action: authError.action - }) - ) - return - } - const response = - payload.federate && federationConfig.enabled - ? await federation.search(payload) - : await query.handleQuery(payload, authContext.did) - metrics.increment(HUB_METRICS.QUERY_REQUESTS_TOTAL) - metrics.observe(HUB_METRICS.QUERY_DURATION_MS, response.took) - ws.send(JSON.stringify(response)) - metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) - return - } - - if (isIndexUpdate(payload)) { - if (!authContext.can('index/write', payload.docId)) { - const authError = createHubAuthError({ - code: 'FORBIDDEN', - message: 'Capability does not allow index update', - action: 'hub/relay', - resource: payload.docId - }) - ws.send( - JSON.stringify({ - type: 'index-error', - docId: payload.docId, - error: authError.message, - code: authError.code, - action: authError.action - }) - ) - return - } - const ack = await query.handleIndexUpdate(payload.docId, authContext.did, payload) - ws.send(JSON.stringify(ack)) - metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) - return - } - - if (isIndexRemove(payload)) { - if (!authContext.can('index/write', payload.docId)) { - const authError = createHubAuthError({ - code: 'FORBIDDEN', - message: 'Capability does not allow index removal', - action: 'hub/relay', - resource: payload.docId - }) - ws.send( - JSON.stringify({ - type: 'index-error', - docId: payload.docId, - error: authError.message, - code: authError.code, - action: authError.action - }) - ) - return - } - await query.removeFromIndex(payload.docId) - ws.send(JSON.stringify({ type: 'index-ack', docId: payload.docId, indexed: false })) - metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) - return - } - - if (isNodeSyncRequest(payload)) { - const roomDecision = await authorizeRoomAction({ - storage, - session, - action: 'hub/relay', - topic: payload.room, - shareAccess - }) - if (!roomDecision.allowed) { - ws.send( - JSON.stringify({ - type: 'node-error', - code: roomDecision.code ?? 'UNAUTHORIZED', - error: roomDecision.message ?? 'Unauthorized', - action: 'hub/relay', - resource: topicToResource(payload.room) - }) - ) - metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) - return - } - - try { - const response = await nodeRelay.handleSyncRequest(payload, authContext) - ws.send(JSON.stringify(response)) - metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) - } catch (err) { - if (err instanceof NodeRelayError) { - ws.send( - JSON.stringify({ - type: 'node-error', - code: err.code, - error: err.message, - action: err.action, - resource: err.resource - }) - ) - metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) - return - } - throw err - } - return - } - if (isNodeClearRequest(payload)) { - const roomDecision = await authorizeRoomAction({ - storage, - session, - action: 'hub/relay', - topic: payload.room, - shareAccess - }) - if (!roomDecision.allowed) { - reportUnauthorizedRemoteWrite(remoteMutationTelemetry, session.did) - ws.send( - JSON.stringify({ - type: 'node-error', - code: roomDecision.code ?? 'UNAUTHORIZED', - error: roomDecision.message ?? 'Unauthorized', - action: 'hub/relay', - resource: topicToResource(payload.room) - }) - ) - metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) - return - } - - try { - const response = await nodeRelay.handleClear(payload, authContext) - ws.send(JSON.stringify(response)) - metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) - } catch (err) { - if (err instanceof NodeRelayError) { - ws.send( - JSON.stringify({ - type: 'node-error', - code: err.code, - error: err.message, - action: err.action, - resource: err.resource - }) - ) - metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) - return - } - throw err - } - return - } - - if (isPublishMessage(payload) && isNodeSyncRequest(payload.data)) { - const roomDecision = await authorizeRoomAction({ - storage, - session, - action: 'hub/relay', - topic: payload.data.room, - shareAccess - }) - if (!roomDecision.allowed) { - ws.send( - JSON.stringify({ - type: 'node-error', - code: roomDecision.code ?? 'UNAUTHORIZED', - error: roomDecision.message ?? 'Unauthorized', - action: 'hub/relay', - resource: topicToResource(payload.data.room) - }) - ) - metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) - return - } - - try { - const response = await nodeRelay.handleSyncRequest(payload.data, authContext) - ws.send(JSON.stringify(response)) - metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) - } catch (err) { - if (err instanceof NodeRelayError) { - ws.send( - JSON.stringify({ - type: 'node-error', - code: err.code, - error: err.message, - action: err.action, - resource: err.resource - }) - ) - metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) - return - } - throw err - } - return - } - - if (config.auth && isSubscribeMessage(payload)) { - const topics = parseTopics(payload.topics) - const auth = await checkRoomAuth(storage, session, topics, shareAccess) - if (!auth.ok) { - const resource = topicToResource(auth.topic) - ws.send( - JSON.stringify({ - type: 'auth-denied', - code: auth.decision.code ?? 'UNAUTHORIZED', - action: 'hub/signal', - resource, - error: auth.decision.message ?? 'Insufficient capabilities for room' - }) - ) - metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) - ws.close(4403, auth.decision.message ?? 'Insufficient capabilities for room') - return - } - } - - if ( - config.auth && - isPublishMessage(payload) && - typeof payload.topic === 'string' && - payload.topic.startsWith('xnet-doc-') - ) { - const publishDecision = await authorizeRoomAction({ - storage, - session, - action: 'hub/signal', - topic: payload.topic, - shareAccess - }) - if (!publishDecision.allowed) { - reportUnauthorizedRemoteWrite(remoteMutationTelemetry, session.did) - denyAndCloseSocket(ws, publishDecision, 'hub/signal', payload.topic) - return - } - } - - if (isPublishMessage(payload) && isNodeChangePayload(payload.data)) { - const roomDecision = await authorizeRoomAction({ - storage, - session, - action: 'hub/relay', - topic: payload.data.room, - shareAccess - }) - if (!roomDecision.allowed) { - reportUnauthorizedRemoteWrite(remoteMutationTelemetry, session.did) - ws.send( - JSON.stringify({ - type: 'node-error', - code: roomDecision.code ?? 'UNAUTHORIZED', - error: roomDecision.message ?? 'Unauthorized', - action: 'hub/relay', - resource: topicToResource(payload.data.room) - }) - ) - metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) - return - } - - // Share-grant role enforcement: read grantees cannot relay - // node-changes; comment grantees only comment-kind schemas. - // Checked for the session DID and the change author DID. - const changeResource = topicToResource(payload.data.room) - const changeSchemaId = - payload.data.change.schemaId ?? payload.data.change.payload?.schemaId - const writerDids = new Set([session.did, payload.data.change.authorDid]) - for (const writerDid of writerDids) { - if (!writerDid || writerDid === 'did:key:anonymous') continue - const allowed = await shareAccess.canWriteNodeChange( - writerDid, - changeResource, - changeSchemaId - ) - if (!allowed) { - reportUnauthorizedRemoteWrite(remoteMutationTelemetry, writerDid) - ws.send( - JSON.stringify({ - type: 'node-error', - code: 'WRITE_FORBIDDEN', - error: 'Share grant does not allow writing to this document', - action: 'hub/relay', - resource: changeResource - }) - ) - metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) - return - } - } - - try { - const isNew = await nodeRelay.handleNodeChange(payload.data, authContext) - // Maintain the Space containment index (best-effort, never blocks relay). - try { - await maintainSpaceContainment(storage, payload.data.change) - } catch { - /* containment is advisory; a failure must not drop the change */ - } - if (!isNew) return - } catch (err) { - if (err instanceof NodeRelayError) { - ws.send( - JSON.stringify({ - type: 'node-error', - code: err.code, - error: err.message, - action: err.action, - resource: err.resource - }) - ) - metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) - return - } - throw err - } - } - - if ( - isPublishMessage(payload) && - typeof payload.topic === 'string' && - isAwarenessMessage(payload.data) - ) { - const accepted = await awareness.handleAwarenessMessage( - payload.topic, - authContext.did, - payload.data - ) - if (!accepted) { - metrics.increment(HUB_METRICS.WS_MESSAGES_REJECTED) - return - } - } - - if (isPublishMessage(payload) && typeof payload.topic === 'string') { - const peerId = getPublishPeerId(payload) - if (peerId) { - const peers = socketPeers.get(ws) ?? new Set() - peers.add(peerId) - socketPeers.set(ws, peers) - } - - if (payload.topic.startsWith('xnet-doc-') && isSyncRelayMessage(payload.data)) { - // sync-step2 / sync-update carry Yjs document updates; - // share grantees below `write` may not relay them - // (sync-step1 is a state request and stays readable). - if (payload.data.type !== 'sync-step1' && session.did !== 'did:key:anonymous') { - const yjsResource = topicToResource(payload.topic) - const allowed = await shareAccess.canWriteYjs(session.did, yjsResource) - if (!allowed) { - reportUnauthorizedRemoteWrite(remoteMutationTelemetry, session.did) - ws.send( - JSON.stringify({ - type: 'auth-denied', - code: 'WRITE_FORBIDDEN', - action: 'hub/relay', - resource: yjsResource, - error: 'Share grant does not allow editing this document' - }) - ) - metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) - metrics.increment(HUB_METRICS.WS_MESSAGES_REJECTED) - return - } - } - const accepted = await relay.handleSyncMessage( - payload.topic, - payload.data, - signaling.publishFromHub - ) - if (!accepted) { - metrics.increment(HUB_METRICS.WS_MESSAGES_REJECTED) - return - } - } - } - - signaling.handleMessage(ws, payload) - - if (isSubscribeMessage(payload)) { - const topics = parseTopics(payload.topics) - if (topics.length > 0) { - const existing = socketTopics.get(ws) ?? new Set() - for (const topic of topics) { - if (!existing.has(topic)) { - existing.add(topic) - void relay.handleRoomJoin(topic, signaling.publishFromHub) - const snapshot = await awareness.getSnapshot(topic) - if (snapshot.length > 0 && ws.readyState === 1) { - ws.send( - JSON.stringify({ - type: 'publish', - topic, - data: { - type: 'awareness-snapshot', - from: 'hub-relay', - users: snapshot.map((entry) => ({ - did: entry.userDid, - state: entry.state, - lastSeen: entry.lastSeen, - isStale: Date.now() - entry.lastSeen > 5 * 60 * 1000 - })) - } - }) - ) - } - } - } - socketTopics.set(ws, existing) - } - } - - if (isUnsubscribeMessage(payload)) { - const topics = parseTopics(payload.topics) - const existing = socketTopics.get(ws) - if (existing && topics.length > 0) { - for (const topic of topics) { - if (existing.delete(topic)) { - relay.handleRoomLeave(topic) - await awareness.handleDisconnect(topic, authContext.did) - } - } - if (existing.size === 0) { - socketTopics.delete(ws) - } - } - } + await messageRouter.dispatch(payload, { ws, session, authContext }) })() }) diff --git a/packages/hub/src/storage/memory.ts b/packages/hub/src/storage/memory.ts index 1df841f48..084b20b4e 100644 --- a/packages/hub/src/storage/memory.ts +++ b/packages/hub/src/storage/memory.ts @@ -2,6 +2,7 @@ * @xnetjs/hub - In-memory storage adapter. */ +import { compareChangeApplicationOrder } from '@xnetjs/core' import type { AwarenessEntry, BlobMeta, @@ -649,13 +650,20 @@ export const createMemoryStorage = (): HubStorage => { sinceLamport: number ): Promise => { const changes = nodeChangesByRoom.get(room) ?? [] - return changes - .filter((change) => change.lamportTime > sinceLamport) - .sort((a, b) => - a.lamportTime === b.lamportTime - ? a.lamportAuthor.localeCompare(b.lamportAuthor) - : a.lamportTime - b.lamportTime - ) + return ( + changes + .filter((change) => change.lamportTime > sinceLamport) + // The shared protocol application order (code-unit author tiebreak). + // localeCompare here diverged from the SQLite storage's BINARY-collation + // `ORDER BY lamport_time, lamport_author` and from the client sort — + // locale collation is non-deterministic across ICU versions (0276). + .sort((a, b) => + compareChangeApplicationOrder( + { lamport: a.lamportTime, author: a.lamportAuthor }, + { lamport: b.lamportTime, author: b.lamportAuthor } + ) + ) + ) } const getNodeChangesForNode = async ( diff --git a/packages/hub/src/ws/authorize.ts b/packages/hub/src/ws/authorize.ts new file mode 100644 index 000000000..271f49c6b --- /dev/null +++ b/packages/hub/src/ws/authorize.ts @@ -0,0 +1,247 @@ +/** + * @xnetjs/hub - Unified room authorization for the WebSocket message pump. + * + * All WS handlers resolve room access through ONE path: `authorizeRoomAction` + * (single topic) or `requireRoomAuth` (topic list). The POLICY here is moved + * verbatim from server.ts (exploration 0276 Theme 2) — only the call sites + * were unified. + * + * The handlers historically invoked this in four subtly different inline + * forms. The differences are in the DENY handling, not the decision itself, + * and each is preserved explicitly at its call site: + * 1. node-sync-request / publish-wrapped sync-request (`hub/relay`): + * deny → `node-error` response, connection stays open, no abuse telemetry. + * 2. node-clear + publish node-change (`hub/relay`): deny → abuse telemetry + * (`reportUnauthorizedRemoteWrite`) THEN `node-error`, connection stays open. + * 3. subscribe under `config.auth` (`hub/signal`, every topic via + * `requireRoomAuth`): deny → `auth-denied` response + close(4403). + * 4. publish to an `xnet-doc-*` topic under `config.auth` (`hub/signal`): + * deny → abuse telemetry + `auth-denied` + close(4403) + * (`denyAndCloseSocket`). + */ + +import type { AuthSession } from '../auth/ucan' +import type { Metrics } from '../middleware/metrics' +import type { ShareAccessService } from '../services/share-access' +import type { HubStorage } from '../storage/interface' +import type { WebSocket } from 'ws' +import { hasHubCapability } from '../auth/capabilities' +import { HUB_METRICS } from '../middleware/metrics' +import { buildWsError } from './errors' + +export const topicToResource = (topic: string): string => + topic.startsWith('xnet-doc-') ? topic.slice('xnet-doc-'.length) : topic + +export type AuthzCode = 'UNAUTHORIZED' | 'TOKEN_EXPIRED' | 'TOKEN_REVOKED' + +export type AuthzDecision = { + allowed: boolean + code?: AuthzCode + message?: string + source?: 'capability' | 'grant-index' | 'space-grant' +} + +export type RoomAuthAction = 'hub/relay' | 'hub/signal' + +const isTokenExpired = (session: AuthSession): boolean => { + const exp = session.token?.exp + if (typeof exp !== 'number') { + return false + } + return exp <= Math.floor(Date.now() / 1000) +} + +const logAuthDecision = (input: { + allowed: boolean + did: string + action: string + resource: string + source?: 'capability' | 'grant-index' | 'space-grant' + code?: AuthzCode + reason?: string +}): void => { + const base = `[AuthZ] ${input.allowed ? 'allow' : 'deny'} ${input.action} resource=${input.resource} did=${input.did}` + if (input.allowed) { + console.log(`${base} source=${input.source ?? 'capability'}`) + return + } + console.warn( + `${base} code=${input.code ?? 'UNAUTHORIZED'} reason=${input.reason ?? 'unauthorized'}` + ) +} + +export const authorizeRoomAction = async (input: { + storage: HubStorage + session: AuthSession + action: RoomAuthAction + topic: string + shareAccess?: ShareAccessService +}): Promise => { + const resource = topicToResource(input.topic) + + if (isTokenExpired(input.session)) { + const decision: AuthzDecision = { + allowed: false, + code: 'TOKEN_EXPIRED', + message: 'Authentication token has expired' + } + logAuthDecision({ + allowed: false, + did: input.session.did, + action: input.action, + resource, + code: decision.code, + reason: decision.message + }) + return decision + } + + // A DID whose share grants were all revoked ("remove access") is denied + // outright — wildcard self-issued capabilities do not restore access. + if ( + input.shareAccess && + input.session.did !== 'did:key:anonymous' && + (await input.shareAccess.isDenied(input.session.did, resource)) + ) { + const decision: AuthzDecision = { + allowed: false, + code: 'TOKEN_REVOKED', + message: 'Access to this resource has been revoked' + } + logAuthDecision({ + allowed: false, + did: input.session.did, + action: input.action, + resource, + code: decision.code, + reason: decision.message + }) + return decision + } + + if ( + hasHubCapability(input.session.capabilities, input.action, resource) || + hasHubCapability(input.session.capabilities, 'hub/signal', resource) + ) { + logAuthDecision({ + allowed: true, + did: input.session.did, + action: input.action, + resource, + source: 'capability' + }) + return { allowed: true, source: 'capability' } + } + + const grantedDocIds = await input.storage.listGrantedDocIds(input.session.did) + if (grantedDocIds.includes(resource)) { + logAuthDecision({ + allowed: true, + did: input.session.did, + action: input.action, + resource, + source: 'grant-index' + }) + return { allowed: true, source: 'grant-index' } + } + + // Container (Space) membership: a member of an ancestor Space may access nodes + // beneath it even without a direct per-doc grant (exploration 0179). + if ( + input.shareAccess && + input.session.did !== 'did:key:anonymous' && + (await input.shareAccess.canAccessNode(input.session.did, resource)) + ) { + logAuthDecision({ + allowed: true, + did: input.session.did, + action: input.action, + resource, + source: 'space-grant' + }) + return { allowed: true, source: 'space-grant' } + } + + if (Array.isArray(input.session.token?.prf) && input.session.token.prf.length > 0) { + const decision: AuthzDecision = { + allowed: false, + code: 'TOKEN_REVOKED', + message: 'Grant token is no longer active for this resource' + } + logAuthDecision({ + allowed: false, + did: input.session.did, + action: input.action, + resource, + code: decision.code, + reason: decision.message + }) + return decision + } + + const decision: AuthzDecision = { + allowed: false, + code: 'UNAUTHORIZED', + message: 'Capability and grant index checks denied access' + } + logAuthDecision({ + allowed: false, + did: input.session.did, + action: input.action, + resource, + code: decision.code, + reason: decision.message + }) + return decision +} + +export type RoomAuthResult = { ok: true } | { ok: false; topic: string; decision: AuthzDecision } + +/** + * Authorize an action against every topic in a list; stops at the first denial + * (formerly `checkRoomAuth`, which was hardwired to `hub/signal`). + */ +export const requireRoomAuth = async (input: { + storage: HubStorage + session: AuthSession + action: RoomAuthAction + topics: string[] + shareAccess?: ShareAccessService +}): Promise => { + for (const topic of input.topics) { + const decision = await authorizeRoomAction({ + storage: input.storage, + session: input.session, + action: input.action, + topic, + shareAccess: input.shareAccess + }) + if (!decision.allowed) { + return { ok: false, topic, decision } + } + } + return { ok: true } +} + +/** Send an `auth-denied` error and close the socket with the room-auth code. */ +export const denyAndCloseSocket = ( + ws: WebSocket, + decision: AuthzDecision, + action: RoomAuthAction, + topic: string, + metrics: Metrics +): void => { + ws.send( + JSON.stringify( + buildWsError({ + kind: 'auth-denied', + code: decision.code ?? 'UNAUTHORIZED', + action, + resource: topicToResource(topic), + error: decision.message ?? 'Insufficient capabilities for room' + }) + ) + ) + metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) + ws.close(4403, decision.message ?? 'Insufficient capabilities for room') +} diff --git a/packages/hub/src/ws/errors.ts b/packages/hub/src/ws/errors.ts new file mode 100644 index 000000000..463339f89 --- /dev/null +++ b/packages/hub/src/ws/errors.ts @@ -0,0 +1,79 @@ +/** + * @xnetjs/hub - WebSocket error-response builder. + * + * The hub speaks several error families over the wire (`query-error`, + * `index-error`, `node-error`, `auth-denied`, plain `error`). Clients parse + * these exact shapes (`useHubSearch` reads `query-error`, the runtime + * node-store sync provider reads `node-error`), so this builder centralizes + * construction WITHOUT changing any shape: each family keeps its historical + * field set and field order. + */ + +export type WsErrorMessage = + | { type: 'error'; message: string | undefined } + | { type: 'query-error'; id: string; error: string; code: string; action: string } + | { type: 'index-error'; docId: string; error: string; code: string; action: string } + | { + type: 'node-error' + code: string + error: string + action: string | undefined + resource: string | undefined + } + | { type: 'auth-denied'; code: string; action: string; resource: string; error: string } + +export type WsErrorInput = + | { kind: 'error'; message: string | undefined } + | { kind: 'query-error'; id: string; error: string; code: string; action: string } + | { kind: 'index-error'; docId: string; error: string; code: string; action: string } + | { + kind: 'node-error' + code: string + error: string + action: string | undefined + resource: string | undefined + } + | { kind: 'auth-denied'; code: string; action: string; resource: string; error: string } + +/** + * Build a WS error response. One entry point, one compat shape per family — + * the wire format is frozen, so add new fields here only behind a new `kind`. + */ +export const buildWsError = (input: WsErrorInput): WsErrorMessage => { + switch (input.kind) { + case 'error': + return { type: 'error', message: input.message } + case 'query-error': + return { + type: 'query-error', + id: input.id, + error: input.error, + code: input.code, + action: input.action + } + case 'index-error': + return { + type: 'index-error', + docId: input.docId, + error: input.error, + code: input.code, + action: input.action + } + case 'node-error': + return { + type: 'node-error', + code: input.code, + error: input.error, + action: input.action, + resource: input.resource + } + case 'auth-denied': + return { + type: 'auth-denied', + code: input.code, + action: input.action, + resource: input.resource, + error: input.error + } + } +} diff --git a/packages/hub/src/ws/guards.ts b/packages/hub/src/ws/guards.ts new file mode 100644 index 000000000..ada229c31 --- /dev/null +++ b/packages/hub/src/ws/guards.ts @@ -0,0 +1,159 @@ +/** + * @xnetjs/hub - Type guards for inbound WebSocket messages. + * + * Moved verbatim from server.ts (exploration 0276 Theme 2). Each guard both + * narrows the parsed JSON payload and doubles as the router's match predicate. + */ + +import type { SerializedNodeChange } from '../storage/interface' + +export const isRecord = (value: unknown): value is Record => + Boolean(value && typeof value === 'object' && !Array.isArray(value)) + +export const parseTopics = (value: unknown): string[] => + Array.isArray(value) ? value.filter((entry): entry is string => typeof entry === 'string') : [] + +export type SubscribeMessage = { type: 'subscribe'; topics?: unknown } + +export const isSubscribeMessage = (value: unknown): value is SubscribeMessage => { + if (!value || typeof value !== 'object') return false + const candidate = value as { type?: unknown } + return candidate.type === 'subscribe' +} + +export type UnsubscribeMessage = { type: 'unsubscribe'; topics?: unknown } + +export const isUnsubscribeMessage = (value: unknown): value is UnsubscribeMessage => { + if (!value || typeof value !== 'object') return false + const candidate = value as { type?: unknown } + return candidate.type === 'unsubscribe' +} + +export type PublishMessage = { type: 'publish'; topic?: unknown; data?: unknown } + +export const isPublishMessage = (value: unknown): value is PublishMessage => { + if (!value || typeof value !== 'object') return false + const candidate = value as { type?: unknown } + return candidate.type === 'publish' +} + +export type QueryRequestMessage = { + type: 'query-request' + id: string + query: string + federate?: boolean +} + +export const isQueryRequest = (value: unknown): value is QueryRequestMessage => { + if (!isRecord(value)) return false + return ( + value.type === 'query-request' && + typeof value.id === 'string' && + typeof value.query === 'string' + ) +} + +export type IndexUpdateMessage = { + type: 'index-update' + docId: string + meta: { schemaIri: string; title: string } +} + +export const isIndexUpdate = (value: unknown): value is IndexUpdateMessage => { + if (!isRecord(value)) return false + if (value.type !== 'index-update') return false + if (typeof value.docId !== 'string') return false + if (!isRecord(value.meta)) return false + return typeof value.meta.schemaIri === 'string' && typeof value.meta.title === 'string' +} + +export type IndexRemoveMessage = { type: 'index-remove'; docId: string } + +export const isIndexRemove = (value: unknown): value is IndexRemoveMessage => { + if (!isRecord(value)) return false + return value.type === 'index-remove' && typeof value.docId === 'string' +} + +export type NodeSyncRequestMessage = { + type: 'node-sync-request' + room: string + sinceLamport: number +} + +export const isNodeSyncRequest = (value: unknown): value is NodeSyncRequestMessage => { + if (!isRecord(value)) return false + return ( + value.type === 'node-sync-request' && + typeof value.room === 'string' && + typeof value.sinceLamport === 'number' + ) +} + +export type NodeClearMessage = { type: 'node-clear'; room: string } + +export const isNodeClearRequest = (value: unknown): value is NodeClearMessage => { + if (!isRecord(value)) return false + return value.type === 'node-clear' && typeof value.room === 'string' +} + +export type NodeChangeMessage = { + type: 'node-change' + room: string + change: SerializedNodeChange +} + +export const isNodeChangePayload = (value: unknown): value is NodeChangeMessage => { + if (!isRecord(value)) return false + if (value.type !== 'node-change' || typeof value.room !== 'string') return false + if (!isRecord(value.change)) return false + const change = value.change as Record + return typeof change.hash === 'string' && typeof change.signatureB64 === 'string' +} + +export type AwarenessMessage = { type: 'awareness'; update?: string; state?: unknown } + +export const isAwarenessMessage = (value: unknown): value is AwarenessMessage => { + if (!isRecord(value)) return false + if (value.type !== 'awareness') return false + const candidate = value as { update?: unknown; state?: unknown } + return ( + (typeof candidate.update === 'string' && candidate.update.length > 0) || + typeof candidate.state !== 'undefined' + ) +} + +export type SyncRelayMessage = { + type: 'sync-step1' | 'sync-step2' | 'sync-update' + from?: unknown +} + +export const isSyncRelayMessage = (value: unknown): value is SyncRelayMessage => { + if (!isRecord(value)) return false + return value.type === 'sync-step1' || value.type === 'sync-step2' || value.type === 'sync-update' +} + +export type ClientHandshakeMessage = { + type: 'client-handshake' + did: string + protocolVersion: number + minProtocolVersion: number + features: string[] + packageVersion: string +} + +export const isClientHandshake = (value: unknown): value is ClientHandshakeMessage => { + if (!isRecord(value)) return false + if (value.type !== 'client-handshake') return false + return ( + typeof value.did === 'string' && + typeof value.protocolVersion === 'number' && + typeof value.minProtocolVersion === 'number' && + Array.isArray(value.features) && + typeof value.packageVersion === 'string' + ) +} + +export const getPublishPeerId = (payload: { data?: unknown }): string | null => { + if (!isRecord(payload.data)) return null + return typeof payload.data.from === 'string' ? payload.data.from : null +} diff --git a/packages/hub/src/ws/handlers/client-handshake.ts b/packages/hub/src/ws/handlers/client-handshake.ts new file mode 100644 index 000000000..d0aec3904 --- /dev/null +++ b/packages/hub/src/ws/handlers/client-handshake.ts @@ -0,0 +1,63 @@ +/** + * @xnetjs/hub - `client-handshake` handler (protocol version negotiation). + * + * The hub's own `handshake` message is sent on connection (before the pump) + * in server.ts; this handles the client's reply. A client-handshake may + * arrive at any point — it was never required to be the first message — and + * an incompatible version only warns, it never closes the connection. + */ + +import type { Metrics } from '../../middleware/metrics' +import type { ClientHandshakeMessage } from '../guards' +import type { WsHandler } from '../message-router' +import { HUB_METRICS } from '../../middleware/metrics' + +export const createClientHandshakeHandler = (deps: { + metrics: Metrics +}): WsHandler => { + return (payload, ctx) => { + const hubProtocolVersion = 1 + const hubMinProtocolVersion = 1 + + // Check version compatibility + const clientMax = payload.protocolVersion + const clientMin = payload.minProtocolVersion + + // Find compatible version range + const agreedVersion = Math.min(hubProtocolVersion, clientMax) + const minRequired = Math.max(hubMinProtocolVersion, clientMin) + + if (agreedVersion < minRequired) { + // Versions are incompatible + const suggestion = + clientMax < hubMinProtocolVersion + ? 'upgrade-client' + : hubProtocolVersion < clientMin + ? 'upgrade-hub' + : 'incompatible' + + ctx.ws.send( + JSON.stringify({ + type: 'version-mismatch', + hubVersion: hubProtocolVersion, + clientVersion: clientMax, + suggestion, + message: + suggestion === 'upgrade-client' + ? `Client protocol v${clientMax} is too old. Please upgrade to at least v${hubMinProtocolVersion}.` + : suggestion === 'upgrade-hub' + ? `Hub protocol v${hubProtocolVersion} is too old for client v${clientMin}.` + : 'Protocol versions are incompatible.' + }) + ) + deps.metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) + // Don't close the connection - just warn + } else if (clientMax < hubProtocolVersion) { + // Client is using older version - log for metrics + console.log( + `Client ${payload.did} using older protocol v${clientMax} (hub is v${hubProtocolVersion})` + ) + } + return 'handled' + } +} diff --git a/packages/hub/src/ws/handlers/node-change.ts b/packages/hub/src/ws/handlers/node-change.ts new file mode 100644 index 000000000..d337bbf44 --- /dev/null +++ b/packages/hub/src/ws/handlers/node-change.ts @@ -0,0 +1,181 @@ +/** + * @xnetjs/hub - Published `node-change` handler: authorize, enforce share-grant + * write roles, persist via the node relay, maintain Space containment, then + * FALL THROUGH so a new change still broadcasts to room subscribers via + * signaling. + */ + +import type { Metrics } from '../../middleware/metrics' +import type { NodeRelayService } from '../../services/node-relay' +import type { RemoteMutationTelemetryOptions } from '../../services/remote-mutation-telemetry' +import type { ShareAccessService } from '../../services/share-access' +import type { HubStorage } from '../../storage/interface' +import type { NodeChangeMessage, PublishMessage } from '../guards' +import type { WsHandler, WsHandlerResult } from '../message-router' +import { HUB_METRICS } from '../../middleware/metrics' +import { NodeRelayError } from '../../services/node-relay' +import { reportUnauthorizedRemoteWrite } from '../../services/remote-mutation-telemetry' +import { authorizeRoomAction, topicToResource } from '../authorize' +import { buildWsError } from '../errors' + +// ─── Space containment maintenance (exploration 0179) ───────────────────────── +// Schemas that carry a `space` relation (their canonical security home). +const SPACEABLE_SCHEMA_PREFIXES = [ + 'xnet://xnet.fyi/Page', + 'xnet://xnet.fyi/Database', + 'xnet://xnet.fyi/Canvas', + 'xnet://xnet.fyi/Dashboard', + 'xnet://xnet.fyi/Project', + 'xnet://xnet.fyi/Channel', + 'xnet://xnet.fyi/Task' +] +const SPACE_SCHEMA_PREFIX = 'xnet://xnet.fyi/Space' + +const firstRelationId = (value: unknown): string | null => { + if (typeof value === 'string') return value.trim() || null + if (Array.isArray(value)) return value.length > 0 ? firstRelationId(value[0]) : null + if (value && typeof value === 'object' && 'id' in value) { + const id = (value as { id?: unknown }).id + return typeof id === 'string' ? id.trim() || null : null + } + return null +} + +type ContainmentChange = { + nodeId?: string + schemaId?: string + payload?: { + nodeId?: string + schemaId?: string + properties?: Record + deleted?: boolean + } +} + +/** + * Keep the hub's node→container index fresh from relayed node-changes so + * container (Space) grants resolve. A content node's container is its `space`; + * a Space's container is its `parent`. Only updates when the relevant property + * is actually present in the change (partial CRDT updates never clobber it). + */ +export const maintainSpaceContainment = async ( + storage: HubStorage, + change: ContainmentChange +): Promise => { + const nodeId = change.payload?.nodeId ?? change.nodeId + const schemaId = change.schemaId ?? change.payload?.schemaId + const properties = change.payload?.properties + if (!nodeId || !schemaId || !properties || change.payload?.deleted) return + const hasKey = (k: string): boolean => Object.prototype.hasOwnProperty.call(properties, k) + const recordVisibility = async (): Promise => { + if (!hasKey('visibility')) return + const value = properties.visibility + await storage.setNodeVisibility(nodeId, typeof value === 'string' ? value : null) + } + if (schemaId.startsWith(SPACE_SCHEMA_PREFIX)) { + if (hasKey('parent')) await storage.setNodeContainer(nodeId, firstRelationId(properties.parent)) + await recordVisibility() + return + } + if (SPACEABLE_SCHEMA_PREFIXES.some((prefix) => schemaId.startsWith(prefix))) { + if (hasKey('space')) await storage.setNodeContainer(nodeId, firstRelationId(properties.space)) + await recordVisibility() + } +} + +export const createNodeChangeHandler = (deps: { + storage: HubStorage + nodeRelay: NodeRelayService + shareAccess: ShareAccessService + metrics: Metrics + remoteMutationTelemetry: RemoteMutationTelemetryOptions +}): WsHandler => { + return async (payload, ctx): Promise => { + const roomDecision = await authorizeRoomAction({ + storage: deps.storage, + session: ctx.session, + action: 'hub/relay', + topic: payload.data.room, + shareAccess: deps.shareAccess + }) + if (!roomDecision.allowed) { + // Deny form 2 (see ws/authorize.ts): write attempt → abuse telemetry, + // then node-error; the socket stays open. + reportUnauthorizedRemoteWrite(deps.remoteMutationTelemetry, ctx.session.did) + ctx.ws.send( + JSON.stringify( + buildWsError({ + kind: 'node-error', + code: roomDecision.code ?? 'UNAUTHORIZED', + error: roomDecision.message ?? 'Unauthorized', + action: 'hub/relay', + resource: topicToResource(payload.data.room) + }) + ) + ) + deps.metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) + return 'handled' + } + + // Share-grant role enforcement: read grantees cannot relay + // node-changes; comment grantees only comment-kind schemas. + // Checked for the session DID and the change author DID. + const changeResource = topicToResource(payload.data.room) + const changeSchemaId = payload.data.change.schemaId ?? payload.data.change.payload?.schemaId + const writerDids = new Set([ctx.session.did, payload.data.change.authorDid]) + for (const writerDid of writerDids) { + if (!writerDid || writerDid === 'did:key:anonymous') continue + const allowed = await deps.shareAccess.canWriteNodeChange( + writerDid, + changeResource, + changeSchemaId + ) + if (!allowed) { + reportUnauthorizedRemoteWrite(deps.remoteMutationTelemetry, writerDid) + ctx.ws.send( + JSON.stringify( + buildWsError({ + kind: 'node-error', + code: 'WRITE_FORBIDDEN', + error: 'Share grant does not allow writing to this document', + action: 'hub/relay', + resource: changeResource + }) + ) + ) + deps.metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) + return 'handled' + } + } + + try { + const isNew = await deps.nodeRelay.handleNodeChange(payload.data, ctx.authContext) + // Maintain the Space containment index (best-effort, never blocks relay). + try { + await maintainSpaceContainment(deps.storage, payload.data.change) + } catch { + /* containment is advisory; a failure must not drop the change */ + } + if (!isNew) return 'handled' + } catch (err) { + if (err instanceof NodeRelayError) { + ctx.ws.send( + JSON.stringify( + buildWsError({ + kind: 'node-error', + code: err.code, + error: err.message, + action: err.action, + resource: err.resource + }) + ) + ) + deps.metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) + return 'handled' + } + throw err + } + // New change: fall through so signaling broadcasts the publish. + return 'continue' + } +} diff --git a/packages/hub/src/ws/handlers/node-sync.ts b/packages/hub/src/ws/handlers/node-sync.ts new file mode 100644 index 000000000..800385a4f --- /dev/null +++ b/packages/hub/src/ws/handlers/node-sync.ts @@ -0,0 +1,141 @@ +/** + * @xnetjs/hub - Node-log sync handlers: `node-sync-request`, `node-clear`, + * and the publish-wrapped `node-sync-request` variant. + */ + +import type { AuthSession } from '../../auth/ucan' +import type { Metrics } from '../../middleware/metrics' +import type { NodeRelayService } from '../../services/node-relay' +import type { RemoteMutationTelemetryOptions } from '../../services/remote-mutation-telemetry' +import type { ShareAccessService } from '../../services/share-access' +import type { HubStorage } from '../../storage/interface' +import type { NodeClearMessage, NodeSyncRequestMessage, PublishMessage } from '../guards' +import type { WsConnectionContext, WsHandler, WsHandlerResult } from '../message-router' +import type { WebSocket } from 'ws' +import { HUB_METRICS } from '../../middleware/metrics' +import { NodeRelayError } from '../../services/node-relay' +import { reportUnauthorizedRemoteWrite } from '../../services/remote-mutation-telemetry' +import { authorizeRoomAction, topicToResource, type AuthzDecision } from '../authorize' +import { buildWsError } from '../errors' + +type NodeSyncDeps = { + storage: HubStorage + nodeRelay: NodeRelayService + shareAccess: ShareAccessService + metrics: Metrics + remoteMutationTelemetry: RemoteMutationTelemetryOptions +} + +const sendNodeAuthDenied = ( + ws: WebSocket, + decision: AuthzDecision, + room: string, + metrics: Metrics +): void => { + ws.send( + JSON.stringify( + buildWsError({ + kind: 'node-error', + code: decision.code ?? 'UNAUTHORIZED', + error: decision.message ?? 'Unauthorized', + action: 'hub/relay', + resource: topicToResource(room) + }) + ) + ) + metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) +} + +const sendNodeRelayError = (ws: WebSocket, err: NodeRelayError, metrics: Metrics): void => { + ws.send( + JSON.stringify( + buildWsError({ + kind: 'node-error', + code: err.code, + error: err.message, + action: err.action, + resource: err.resource + }) + ) + ) + metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) +} + +const authorizeRelayRoom = async ( + deps: NodeSyncDeps, + session: AuthSession, + room: string +): Promise => + authorizeRoomAction({ + storage: deps.storage, + session, + action: 'hub/relay', + topic: room, + shareAccess: deps.shareAccess + }) + +const handleSyncRequest = async ( + deps: NodeSyncDeps, + payload: NodeSyncRequestMessage, + ctx: WsConnectionContext +): Promise => { + const roomDecision = await authorizeRelayRoom(deps, ctx.session, payload.room) + if (!roomDecision.allowed) { + // Deny form 1 (see ws/authorize.ts): node-error, keep the socket open, + // no abuse telemetry — a sync request is a read, not a write attempt. + sendNodeAuthDenied(ctx.ws, roomDecision, payload.room, deps.metrics) + return 'handled' + } + + try { + const response = await deps.nodeRelay.handleSyncRequest(payload, ctx.authContext) + ctx.ws.send(JSON.stringify(response)) + deps.metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) + } catch (err) { + if (err instanceof NodeRelayError) { + sendNodeRelayError(ctx.ws, err, deps.metrics) + return 'handled' + } + throw err + } + return 'handled' +} + +export const createNodeSyncRequestHandler = ( + deps: NodeSyncDeps +): WsHandler => { + return (payload, ctx) => handleSyncRequest(deps, payload, ctx) +} + +/** `publish`-wrapped `node-sync-request` (data carries the request). */ +export const createPublishedNodeSyncRequestHandler = ( + deps: NodeSyncDeps +): WsHandler => { + return (payload, ctx) => handleSyncRequest(deps, payload.data, ctx) +} + +export const createNodeClearHandler = (deps: NodeSyncDeps): WsHandler => { + return async (payload, ctx): Promise => { + const roomDecision = await authorizeRelayRoom(deps, ctx.session, payload.room) + if (!roomDecision.allowed) { + // Deny form 2 (see ws/authorize.ts): a clear is a destructive write, so + // the denial ALSO reports abuse telemetry before the node-error. + reportUnauthorizedRemoteWrite(deps.remoteMutationTelemetry, ctx.session.did) + sendNodeAuthDenied(ctx.ws, roomDecision, payload.room, deps.metrics) + return 'handled' + } + + try { + const response = await deps.nodeRelay.handleClear(payload, ctx.authContext) + ctx.ws.send(JSON.stringify(response)) + deps.metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) + } catch (err) { + if (err instanceof NodeRelayError) { + sendNodeRelayError(ctx.ws, err, deps.metrics) + return 'handled' + } + throw err + } + return 'handled' + } +} diff --git a/packages/hub/src/ws/handlers/publish-relay.ts b/packages/hub/src/ws/handlers/publish-relay.ts new file mode 100644 index 000000000..e694d399a --- /dev/null +++ b/packages/hub/src/ws/handlers/publish-relay.ts @@ -0,0 +1,132 @@ +/** + * @xnetjs/hub - Publish pipeline stages that precede the generic signaling + * broadcast: the doc-topic auth gate, awareness ingestion, and the Yjs sync + * relay (with per-connection peer tracking). All stages fall through on + * success so the message still reaches signaling. + */ + +import type { Metrics } from '../../middleware/metrics' +import type { AwarenessService } from '../../services/awareness' +import type { RelayService } from '../../services/relay' +import type { RemoteMutationTelemetryOptions } from '../../services/remote-mutation-telemetry' +import type { ShareAccessService } from '../../services/share-access' +import type { createSignalingService } from '../../services/signaling' +import type { HubStorage } from '../../storage/interface' +import type { AwarenessMessage, PublishMessage } from '../guards' +import type { WsHandler, WsHandlerResult } from '../message-router' +import type { WebSocket } from 'ws' +import { HUB_METRICS } from '../../middleware/metrics' +import { reportUnauthorizedRemoteWrite } from '../../services/remote-mutation-telemetry' +import { authorizeRoomAction, denyAndCloseSocket, topicToResource } from '../authorize' +import { buildWsError } from '../errors' +import { getPublishPeerId, isSyncRelayMessage } from '../guards' + +type SignalingService = ReturnType + +/** + * Auth gate for publishes into `xnet-doc-*` rooms (only registered when + * `config.auth` is on). Falls through on success. + */ +export const createDocPublishAuthHandler = (deps: { + storage: HubStorage + shareAccess: ShareAccessService + metrics: Metrics + remoteMutationTelemetry: RemoteMutationTelemetryOptions +}): WsHandler => { + return async (payload, ctx): Promise => { + const publishDecision = await authorizeRoomAction({ + storage: deps.storage, + session: ctx.session, + action: 'hub/signal', + topic: payload.topic, + shareAccess: deps.shareAccess + }) + if (!publishDecision.allowed) { + // Deny form 4 (see ws/authorize.ts): a doc publish is a write attempt, + // so unlike the subscribe gate this reports abuse telemetry, then + // auth-denied + close(4403). + reportUnauthorizedRemoteWrite(deps.remoteMutationTelemetry, ctx.session.did) + denyAndCloseSocket(ctx.ws, publishDecision, 'hub/signal', payload.topic, deps.metrics) + return 'handled' + } + return 'continue' + } +} + +/** Awareness ingestion for published awareness updates. Falls through when accepted. */ +export const createAwarenessPublishHandler = (deps: { + awareness: AwarenessService + metrics: Metrics +}): WsHandler => { + return async (payload, ctx): Promise => { + const accepted = await deps.awareness.handleAwarenessMessage( + payload.topic, + ctx.authContext.did, + payload.data + ) + if (!accepted) { + deps.metrics.increment(HUB_METRICS.WS_MESSAGES_REJECTED) + return 'handled' + } + return 'continue' + } +} + +/** + * Peer tracking for every topic publish, plus the Yjs sync relay (with + * share-grant write enforcement) for `xnet-doc-*` sync messages. + */ +export const createSyncRelayPublishHandler = (deps: { + relay: RelayService + shareAccess: ShareAccessService + signaling: SignalingService + metrics: Metrics + remoteMutationTelemetry: RemoteMutationTelemetryOptions + socketPeers: Map> +}): WsHandler => { + return async (payload, ctx): Promise => { + const peerId = getPublishPeerId(payload) + if (peerId) { + const peers = deps.socketPeers.get(ctx.ws) ?? new Set() + peers.add(peerId) + deps.socketPeers.set(ctx.ws, peers) + } + + if (payload.topic.startsWith('xnet-doc-') && isSyncRelayMessage(payload.data)) { + // sync-step2 / sync-update carry Yjs document updates; + // share grantees below `write` may not relay them + // (sync-step1 is a state request and stays readable). + if (payload.data.type !== 'sync-step1' && ctx.session.did !== 'did:key:anonymous') { + const yjsResource = topicToResource(payload.topic) + const allowed = await deps.shareAccess.canWriteYjs(ctx.session.did, yjsResource) + if (!allowed) { + reportUnauthorizedRemoteWrite(deps.remoteMutationTelemetry, ctx.session.did) + ctx.ws.send( + JSON.stringify( + buildWsError({ + kind: 'auth-denied', + code: 'WRITE_FORBIDDEN', + action: 'hub/relay', + resource: yjsResource, + error: 'Share grant does not allow editing this document' + }) + ) + ) + deps.metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) + deps.metrics.increment(HUB_METRICS.WS_MESSAGES_REJECTED) + return 'handled' + } + } + const accepted = await deps.relay.handleSyncMessage( + payload.topic, + payload.data, + deps.signaling.publishFromHub + ) + if (!accepted) { + deps.metrics.increment(HUB_METRICS.WS_MESSAGES_REJECTED) + return 'handled' + } + } + return 'continue' + } +} diff --git a/packages/hub/src/ws/handlers/query-request.ts b/packages/hub/src/ws/handlers/query-request.ts new file mode 100644 index 000000000..0b37c887b --- /dev/null +++ b/packages/hub/src/ws/handlers/query-request.ts @@ -0,0 +1,52 @@ +/** + * @xnetjs/hub - `query-request` handler (local search + optional federation). + */ + +import type { Metrics } from '../../middleware/metrics' +import type { FederationService } from '../../services/federation' +import type { QueryService } from '../../services/query' +import type { QueryRequestMessage } from '../guards' +import type { WsHandler, WsHandlerResult } from '../message-router' +import { createHubAuthError } from '../../auth/errors' +import { HUB_METRICS } from '../../middleware/metrics' +import { buildWsError } from '../errors' + +export const createQueryRequestHandler = (deps: { + query: QueryService + federation: FederationService + federationEnabled: boolean + metrics: Metrics +}): WsHandler => { + return async (payload, ctx): Promise => { + if (!ctx.authContext.can('query/read', '*')) { + const authError = createHubAuthError({ + code: 'FORBIDDEN', + message: 'Capability does not allow querying', + action: 'hub/query' + }) + // NOTE: historically this send does NOT increment WS_MESSAGES_SENT + // (unlike the success path) — preserved as-is. + ctx.ws.send( + JSON.stringify( + buildWsError({ + kind: 'query-error', + id: payload.id, + error: authError.message, + code: authError.code, + action: authError.action + }) + ) + ) + return 'handled' + } + const response = + payload.federate && deps.federationEnabled + ? await deps.federation.search(payload) + : await deps.query.handleQuery(payload, ctx.authContext.did) + deps.metrics.increment(HUB_METRICS.QUERY_REQUESTS_TOTAL) + deps.metrics.observe(HUB_METRICS.QUERY_DURATION_MS, response.took) + ctx.ws.send(JSON.stringify(response)) + deps.metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) + return 'handled' + } +} diff --git a/packages/hub/src/ws/handlers/search-index.ts b/packages/hub/src/ws/handlers/search-index.ts new file mode 100644 index 000000000..b30e7de80 --- /dev/null +++ b/packages/hub/src/ws/handlers/search-index.ts @@ -0,0 +1,79 @@ +/** + * @xnetjs/hub - `index-update` / `index-remove` handlers (search index writes). + */ + +import type { Metrics } from '../../middleware/metrics' +import type { QueryService } from '../../services/query' +import type { IndexRemoveMessage, IndexUpdateMessage } from '../guards' +import type { WsHandler, WsHandlerResult } from '../message-router' +import { createHubAuthError } from '../../auth/errors' +import { HUB_METRICS } from '../../middleware/metrics' +import { buildWsError } from '../errors' + +export const createIndexUpdateHandler = (deps: { + query: QueryService + metrics: Metrics +}): WsHandler => { + return async (payload, ctx): Promise => { + if (!ctx.authContext.can('index/write', payload.docId)) { + const authError = createHubAuthError({ + code: 'FORBIDDEN', + message: 'Capability does not allow index update', + action: 'hub/relay', + resource: payload.docId + }) + // NOTE: historically this send does NOT increment WS_MESSAGES_SENT + // (unlike the ack path) — preserved as-is. + ctx.ws.send( + JSON.stringify( + buildWsError({ + kind: 'index-error', + docId: payload.docId, + error: authError.message, + code: authError.code, + action: authError.action + }) + ) + ) + return 'handled' + } + const ack = await deps.query.handleIndexUpdate(payload.docId, ctx.authContext.did, payload) + ctx.ws.send(JSON.stringify(ack)) + deps.metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) + return 'handled' + } +} + +export const createIndexRemoveHandler = (deps: { + query: QueryService + metrics: Metrics +}): WsHandler => { + return async (payload, ctx): Promise => { + if (!ctx.authContext.can('index/write', payload.docId)) { + const authError = createHubAuthError({ + code: 'FORBIDDEN', + message: 'Capability does not allow index removal', + action: 'hub/relay', + resource: payload.docId + }) + // NOTE: historically this send does NOT increment WS_MESSAGES_SENT + // (unlike the ack path) — preserved as-is. + ctx.ws.send( + JSON.stringify( + buildWsError({ + kind: 'index-error', + docId: payload.docId, + error: authError.message, + code: authError.code, + action: authError.action + }) + ) + ) + return 'handled' + } + await deps.query.removeFromIndex(payload.docId) + ctx.ws.send(JSON.stringify({ type: 'index-ack', docId: payload.docId, indexed: false })) + deps.metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) + return 'handled' + } +} diff --git a/packages/hub/src/ws/handlers/subscribe.ts b/packages/hub/src/ws/handlers/subscribe.ts new file mode 100644 index 000000000..46bc21ed9 --- /dev/null +++ b/packages/hub/src/ws/handlers/subscribe.ts @@ -0,0 +1,130 @@ +/** + * @xnetjs/hub - Subscribe/unsubscribe stages. + * + * Two-phase like the original pump: the auth gate runs BEFORE signaling sees + * the subscribe (deny → close), while the bookkeeping stage runs AFTER + * signaling has registered the subscription (room join, awareness snapshot). + */ + +import type { Metrics } from '../../middleware/metrics' +import type { AwarenessService } from '../../services/awareness' +import type { RelayService } from '../../services/relay' +import type { ShareAccessService } from '../../services/share-access' +import type { createSignalingService } from '../../services/signaling' +import type { HubStorage } from '../../storage/interface' +import type { SubscribeMessage, UnsubscribeMessage } from '../guards' +import type { WsHandler, WsHandlerResult } from '../message-router' +import type { WebSocket } from 'ws' +import { HUB_METRICS } from '../../middleware/metrics' +import { requireRoomAuth, topicToResource } from '../authorize' +import { buildWsError } from '../errors' +import { parseTopics } from '../guards' + +type SignalingService = ReturnType + +/** + * Auth gate for subscribes (only registered when `config.auth` is on). + * Falls through on success so signaling + bookkeeping still run. + */ +export const createSubscribeAuthHandler = (deps: { + storage: HubStorage + shareAccess: ShareAccessService + metrics: Metrics +}): WsHandler => { + return async (payload, ctx): Promise => { + const topics = parseTopics(payload.topics) + const auth = await requireRoomAuth({ + storage: deps.storage, + session: ctx.session, + action: 'hub/signal', + topics, + shareAccess: deps.shareAccess + }) + if (!auth.ok) { + // Deny form 3 (see ws/authorize.ts): auth-denied + close(4403); unlike + // the doc-publish gate this one does NOT report abuse telemetry. + const resource = topicToResource(auth.topic) + ctx.ws.send( + JSON.stringify( + buildWsError({ + kind: 'auth-denied', + code: auth.decision.code ?? 'UNAUTHORIZED', + action: 'hub/signal', + resource, + error: auth.decision.message ?? 'Insufficient capabilities for room' + }) + ) + ) + deps.metrics.increment(HUB_METRICS.WS_MESSAGES_SENT) + ctx.ws.close(4403, auth.decision.message ?? 'Insufficient capabilities for room') + return 'handled' + } + return 'continue' + } +} + +/** Post-signaling subscribe bookkeeping: relay room join + awareness snapshot. */ +export const createSubscribeBookkeepingHandler = (deps: { + relay: RelayService + awareness: AwarenessService + signaling: SignalingService + socketTopics: Map> +}): WsHandler => { + return async (payload, ctx): Promise => { + const topics = parseTopics(payload.topics) + if (topics.length > 0) { + const existing = deps.socketTopics.get(ctx.ws) ?? new Set() + for (const topic of topics) { + if (!existing.has(topic)) { + existing.add(topic) + void deps.relay.handleRoomJoin(topic, deps.signaling.publishFromHub) + const snapshot = await deps.awareness.getSnapshot(topic) + if (snapshot.length > 0 && ctx.ws.readyState === 1) { + ctx.ws.send( + JSON.stringify({ + type: 'publish', + topic, + data: { + type: 'awareness-snapshot', + from: 'hub-relay', + users: snapshot.map((entry) => ({ + did: entry.userDid, + state: entry.state, + lastSeen: entry.lastSeen, + isStale: Date.now() - entry.lastSeen > 5 * 60 * 1000 + })) + } + }) + ) + } + } + } + deps.socketTopics.set(ctx.ws, existing) + } + return 'continue' + } +} + +/** Post-signaling unsubscribe bookkeeping: relay room leave + awareness exit. */ +export const createUnsubscribeBookkeepingHandler = (deps: { + relay: RelayService + awareness: AwarenessService + socketTopics: Map> +}): WsHandler => { + return async (payload, ctx): Promise => { + const topics = parseTopics(payload.topics) + const existing = deps.socketTopics.get(ctx.ws) + if (existing && topics.length > 0) { + for (const topic of topics) { + if (existing.delete(topic)) { + deps.relay.handleRoomLeave(topic) + await deps.awareness.handleDisconnect(topic, ctx.authContext.did) + } + } + if (existing.size === 0) { + deps.socketTopics.delete(ctx.ws) + } + } + return 'continue' + } +} diff --git a/packages/hub/src/ws/message-router.ts b/packages/hub/src/ws/message-router.ts new file mode 100644 index 000000000..ad82a3d32 --- /dev/null +++ b/packages/hub/src/ws/message-router.ts @@ -0,0 +1,82 @@ +/** + * @xnetjs/hub - WebSocket message router (exploration 0276 Theme 2). + * + * Replaces the inline if/else pump in server.ts with an ordered registry. + * Routes are evaluated in REGISTRATION order — the pump's original branch + * order is load-bearing (several stages match the same `publish` type and + * fall through to the next stage), so registration order is the contract. + * + * A handler returns: + * - `'handled'` → stop dispatching (the pump's `return`) + * - `'continue'` → keep evaluating later routes (the pump's fall-through) + */ + +import type { AuthContext, AuthSession } from '../auth/ucan' +import type { Metrics } from '../middleware/metrics' +import type { WebSocket } from 'ws' +import { HUB_METRICS } from '../middleware/metrics' + +export type WsConnectionContext = { + ws: WebSocket + session: AuthSession + authContext: AuthContext +} + +export type WsHandlerResult = 'handled' | 'continue' + +export type WsHandler = ( + payload: T, + ctx: WsConnectionContext +) => Promise | WsHandlerResult + +type WsRoute = { + type: string + matches: (value: unknown) => boolean + handle: WsHandler +} + +export type MessageRouter = { + on: (type: string, guard: (value: unknown) => value is T, handler: WsHandler) => void + dispatch: (payload: unknown, ctx: WsConnectionContext) => Promise +} + +/** + * Per-type received counter. Only types with a registered route are counted + * by name (anything else buckets to `unknown`) so a malicious client cannot + * inflate metric cardinality with arbitrary `type` strings. + */ +const messageTypeMetric = (type: string): string => + `hub_ws_messages_received_${type.replace(/[^a-zA-Z0-9]/g, '_')}_total` + +export const createMessageRouter = (metrics: Metrics): MessageRouter => { + const routes: WsRoute[] = [] + const knownTypes = new Set() + + const on = ( + type: string, + guard: (value: unknown) => value is T, + handler: WsHandler + ): void => { + knownTypes.add(type) + routes.push({ type, matches: guard, handle: handler as WsHandler }) + } + + const dispatch = async (payload: unknown, ctx: WsConnectionContext): Promise => { + metrics.increment(HUB_METRICS.WS_MESSAGES_RECEIVED) + const rawType = + payload && + typeof payload === 'object' && + typeof (payload as { type?: unknown }).type === 'string' + ? (payload as { type: string }).type + : 'unknown' + metrics.increment(messageTypeMetric(knownTypes.has(rawType) ? rawType : 'unknown')) + + for (const route of routes) { + if (!route.matches(payload)) continue + const result = await route.handle(payload as never, ctx) + if (result === 'handled') return + } + } + + return { on, dispatch } +} diff --git a/packages/hub/src/ws/register.ts b/packages/hub/src/ws/register.ts new file mode 100644 index 000000000..2853b0ab5 --- /dev/null +++ b/packages/hub/src/ws/register.ts @@ -0,0 +1,147 @@ +/** + * @xnetjs/hub - WS route registration (exploration 0276 Theme 2). + * + * Builds the message router with every stage of the old inline pump, in the + * pump's EXACT original branch order. Order is load-bearing: several stages + * match the same `publish` type and fall through to later stages (auth gate → + * node-change persist → awareness → Yjs relay → signaling broadcast). + */ + +import type { Metrics } from '../middleware/metrics' +import type { AwarenessService } from '../services/awareness' +import type { FederationService } from '../services/federation' +import type { NodeRelayService } from '../services/node-relay' +import type { QueryService } from '../services/query' +import type { RelayService } from '../services/relay' +import type { RemoteMutationTelemetryOptions } from '../services/remote-mutation-telemetry' +import type { ShareAccessService } from '../services/share-access' +import type { createSignalingService } from '../services/signaling' +import type { HubStorage } from '../storage/interface' +import type { HubConfig } from '../types' +import type { WebSocket } from 'ws' +import { + isAwarenessMessage, + isClientHandshake, + isIndexRemove, + isIndexUpdate, + isNodeChangePayload, + isNodeClearRequest, + isNodeSyncRequest, + isPublishMessage, + isQueryRequest, + isSubscribeMessage, + isUnsubscribeMessage, + type AwarenessMessage, + type NodeChangeMessage, + type NodeSyncRequestMessage, + type PublishMessage +} from './guards' +import { createClientHandshakeHandler } from './handlers/client-handshake' +import { createNodeChangeHandler } from './handlers/node-change' +import { + createNodeClearHandler, + createNodeSyncRequestHandler, + createPublishedNodeSyncRequestHandler +} from './handlers/node-sync' +import { + createAwarenessPublishHandler, + createDocPublishAuthHandler, + createSyncRelayPublishHandler +} from './handlers/publish-relay' +import { createQueryRequestHandler } from './handlers/query-request' +import { createIndexRemoveHandler, createIndexUpdateHandler } from './handlers/search-index' +import { + createSubscribeAuthHandler, + createSubscribeBookkeepingHandler, + createUnsubscribeBookkeepingHandler +} from './handlers/subscribe' +import { createMessageRouter, type MessageRouter } from './message-router' + +type SignalingService = ReturnType + +export type WsRouterDeps = { + config: HubConfig + storage: HubStorage + metrics: Metrics + query: QueryService + federation: FederationService + federationEnabled: boolean + nodeRelay: NodeRelayService + shareAccess: ShareAccessService + awareness: AwarenessService + relay: RelayService + signaling: SignalingService + remoteMutationTelemetry: RemoteMutationTelemetryOptions + socketTopics: Map> + socketPeers: Map> +} + +export const createWsMessageRouter = (deps: WsRouterDeps): MessageRouter => { + const router = createMessageRouter(deps.metrics) + + // Terminal typed handlers (each was an early-`return` branch in the pump). + router.on('client-handshake', isClientHandshake, createClientHandshakeHandler(deps)) + router.on('query-request', isQueryRequest, createQueryRequestHandler(deps)) + router.on('index-update', isIndexUpdate, createIndexUpdateHandler(deps)) + router.on('index-remove', isIndexRemove, createIndexRemoveHandler(deps)) + router.on('node-sync-request', isNodeSyncRequest, createNodeSyncRequestHandler(deps)) + router.on('node-clear', isNodeClearRequest, createNodeClearHandler(deps)) + router.on( + 'publish', + (value): value is PublishMessage & { data: NodeSyncRequestMessage } => + isPublishMessage(value) && isNodeSyncRequest(value.data), + createPublishedNodeSyncRequestHandler(deps) + ) + + // Auth gates (only when the hub enforces auth; `config.auth` is fixed for + // the server's lifetime). Both fall through on success. + if (deps.config.auth) { + router.on('subscribe', isSubscribeMessage, createSubscribeAuthHandler(deps)) + router.on( + 'publish', + (value): value is PublishMessage & { topic: string } => + isPublishMessage(value) && + typeof value.topic === 'string' && + value.topic.startsWith('xnet-doc-'), + createDocPublishAuthHandler(deps) + ) + } + + // Publish pipeline: persist node-changes, ingest awareness, relay Yjs sync. + router.on( + 'publish', + (value): value is PublishMessage & { data: NodeChangeMessage } => + isPublishMessage(value) && isNodeChangePayload(value.data), + createNodeChangeHandler(deps) + ) + router.on( + 'publish', + (value): value is PublishMessage & { topic: string; data: AwarenessMessage } => + isPublishMessage(value) && typeof value.topic === 'string' && isAwarenessMessage(value.data), + createAwarenessPublishHandler(deps) + ) + router.on( + 'publish', + (value): value is PublishMessage & { topic: string } => + isPublishMessage(value) && typeof value.topic === 'string', + createSyncRelayPublishHandler(deps) + ) + + // Generic signaling pass-through (subscribe/unsubscribe/publish/ping) — the + // pump handed every remaining message to the signaling service. Registered + // under `ping` so ping traffic is counted; the guard matches everything. + router.on( + 'ping', + (value): value is unknown => true, + (payload, ctx) => { + deps.signaling.handleMessage(ctx.ws, payload) + return 'continue' + } + ) + + // Post-signaling bookkeeping (room join/leave, awareness snapshots). + router.on('subscribe', isSubscribeMessage, createSubscribeBookkeepingHandler(deps)) + router.on('unsubscribe', isUnsubscribeMessage, createUnsubscribeBookkeepingHandler(deps)) + + return router +} diff --git a/packages/hub/test/lww-order.test.ts b/packages/hub/test/lww-order.test.ts new file mode 100644 index 000000000..d013bd6f4 --- /dev/null +++ b/packages/hub/test/lww-order.test.ts @@ -0,0 +1,68 @@ +/** + * Change-application ordering conformance for hub storages (0276). + * + * Replicas fold node changes in `compareChangeApplicationOrder` (lamport → + * author by UTF-16 code units, protocol §L1.7). The hub relays history via + * `getNodeChangesSince`, so its storages must hand changes back in that same + * order — the in-memory storage previously used `localeCompare`, which + * disagrees with the SQLite storage's BINARY collation on case-mixed authors + * and is non-deterministic across ICU versions. + */ +import { compareChangeApplicationOrder } from '@xnetjs/core' +import { describe, expect, it } from 'vitest' +import type { SerializedNodeChange } from '../src/storage/interface' +import { createMemoryStorage } from '../src/storage/memory' + +const ROOM = 'room-lww-order' + +function change(input: { + hash: string + lamportTime: number + lamportAuthor: string +}): SerializedNodeChange { + return { + id: `change-${input.hash}`, + type: 'node', + hash: input.hash, + room: ROOM, + nodeId: 'node-1', + lamportTime: input.lamportTime, + lamportAuthor: input.lamportAuthor, + authorDid: input.lamportAuthor, + wallTime: 1, + parentHash: null, + payload: { nodeId: 'node-1', properties: {} }, + signatureBase64: 'sig' + } as SerializedNodeChange +} + +describe('hub storage change ordering (protocol §L1.7)', () => { + it('memory storage returns changes in shared application order', async () => { + const storage = createMemoryStorage() + + // Includes the golden-vector case pair: 'did:key:zAAA' (uppercase) must + // sort BEFORE 'did:key:zaaa' by code units — many locales collate the + // other way, which is exactly the drift this test pins down. + const inserted = [ + change({ hash: 'h1', lamportTime: 2, lamportAuthor: 'did:key:zAAA' }), + change({ hash: 'h2', lamportTime: 1, lamportAuthor: 'did:key:zaaa' }), + change({ hash: 'h3', lamportTime: 1, lamportAuthor: 'did:key:zAAA' }), + change({ hash: 'h4', lamportTime: 3, lamportAuthor: 'did:key:zbbb' }) + ] + for (const c of inserted) { + await storage.appendNodeChange(ROOM, c) + } + + const returned = await storage.getNodeChangesSince(ROOM, 0) + const expected = [...inserted].sort((a, b) => + compareChangeApplicationOrder( + { lamport: a.lamportTime, author: a.lamportAuthor }, + { lamport: b.lamportTime, author: b.lamportAuthor } + ) + ) + + expect(returned.map((c) => c.hash)).toEqual(expected.map((c) => c.hash)) + // The case-mixed lamport tie resolves uppercase-first (code units). + expect(returned.map((c) => c.hash).slice(0, 2)).toEqual(['h3', 'h2']) + }) +}) diff --git a/packages/hub/test/security-hardening-regressions.test.ts b/packages/hub/test/security-hardening-regressions.test.ts index c3ac5fab4..e0b2e341a 100644 --- a/packages/hub/test/security-hardening-regressions.test.ts +++ b/packages/hub/test/security-hardening-regressions.test.ts @@ -15,24 +15,28 @@ const readSource = (relativePath: string): string => describe('Security hardening regressions', () => { it('keeps bearer tokens out of URL construction paths', () => { + // Hub-session/URL handling moved from App.tsx to the boot orchestrator + // (exploration 0276) — check both the shell and the boot unit. const webApp = readSource('apps/web/src/App.tsx') + const webBoot = readSource('apps/web/src/boot/use-boot-sequence.ts') const webShareRoute = readSource('apps/web/src/routes/share.tsx') const dataService = readSource('apps/electron/src/data-process/data-service.ts') expect(webApp).not.toMatch(/[?&]token=/) + expect(webBoot).not.toMatch(/[?&]token=/) expect(webShareRoute).not.toMatch(/[?&]token=/) expect(dataService).not.toMatch(/searchParams\.set\(\s*['"]token['"]/) }) it('strips secret-bearing params from browser URLs', () => { - const webApp = readSource('apps/web/src/App.tsx') + const webBoot = readSource('apps/web/src/boot/use-boot-sequence.ts') const webShareRoute = readSource('apps/web/src/routes/share.tsx') // stripParams removes the named params from both the search string and // the hash query (hash routing) before rewriting history. - expect(webApp).toContain("stripParams('payload', 'handle')") - expect(webApp).toContain("stripParams('shareSession')") - expect(webApp).toContain('window.history.replaceState') + expect(webBoot).toContain("stripParams('payload', 'handle')") + expect(webBoot).toContain("stripParams('shareSession')") + expect(webBoot).toContain('window.history.replaceState') expect(webShareRoute).toContain('window.history.replaceState') }) diff --git a/packages/hub/test/ws-message-pump.test.ts b/packages/hub/test/ws-message-pump.test.ts new file mode 100644 index 000000000..165c02ae3 --- /dev/null +++ b/packages/hub/test/ws-message-pump.test.ts @@ -0,0 +1,281 @@ +/** + * @xnetjs/hub - WebSocket message-pump tests (exploration 0276 Theme 2). + * + * The pump was decomposed into src/ws/ (router + handlers); these tests pin + * the wire behavior the old inline pump had: handshake/version negotiation, + * unknown-type tolerance, query dispatch, sync-request authorization, and the + * exact error-response shapes clients parse. + */ + +import { createUCAN, generateKeyBundle } from '@xnetjs/identity' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { WebSocket } from 'ws' +import { createHub, type HubInstance } from '../src' + +const PORT = 14476 +const AUTH_PORT = 14576 + +const connectAndWaitHandshake = (port: number, protocols?: string[]): Promise => + new Promise((resolve) => { + const ws = new WebSocket(`ws://localhost:${port}`, protocols) + ws.on('open', () => { + // Consume the hub handshake before resolving + ws.once('message', () => resolve(ws)) + }) + }) + +const waitForMessage = (ws: WebSocket): Promise> => + new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error('timeout')), 2000) + ws.once('message', (data) => { + clearTimeout(timeout) + resolve(JSON.parse(data.toString()) as Record) + }) + }) + +const expectNoMessage = (ws: WebSocket, ms: number): Promise => + new Promise((resolve, reject) => { + const onMessage = (data: import('ws').RawData): void => { + clearTimeout(timer) + reject(new Error(`unexpected message: ${data.toString()}`)) + } + const timer = setTimeout(() => { + ws.off('message', onMessage) + resolve() + }, ms) + ws.once('message', onMessage) + }) + +describe('WS message pump (no auth)', () => { + let hub: HubInstance + + beforeAll(async () => { + hub = await createHub({ port: PORT, auth: false, storage: 'memory' }) + await hub.start() + }) + + afterAll(async () => { + await hub.stop() + }) + + it('sends the hub handshake as the first message on connect', async () => { + const msg = await new Promise>((resolve) => { + const ws = new WebSocket(`ws://localhost:${PORT}`) + ws.once('message', (data) => { + resolve(JSON.parse(data.toString()) as Record) + ws.close() + }) + }) + + expect(msg.type).toBe('handshake') + expect(msg.protocolVersion).toBe(1) + expect(msg.minProtocolVersion).toBe(1) + expect(msg.features).toContain('node-changes') + }) + + it('ignores unknown message types and keeps the connection alive', async () => { + const ws = await connectAndWaitHandshake(PORT) + + ws.send(JSON.stringify({ type: 'totally-bogus', anything: true })) + await expectNoMessage(ws, 100) + + // The connection still works afterwards. + ws.send(JSON.stringify({ type: 'ping' })) + const pong = await waitForMessage(ws) + expect(pong.type).toBe('pong') + ws.close() + }) + + it('counts messages by type (unknown types bucket to `unknown`)', async () => { + const res = await fetch(`http://localhost:${PORT}/metrics`) + const body = await res.text() + expect(body).toContain('hub_ws_messages_received_total') + expect(body).toContain('hub_ws_messages_received_unknown_total') + }) + + it('accepts a compatible client-handshake without replying', async () => { + const ws = await connectAndWaitHandshake(PORT) + + ws.send( + JSON.stringify({ + type: 'client-handshake', + did: 'did:key:test-client', + protocolVersion: 1, + minProtocolVersion: 1, + features: [], + packageVersion: '0.0.1' + }) + ) + await expectNoMessage(ws, 100) + ws.close() + }) + + it('answers an incompatible client-handshake with version-mismatch (and stays open)', async () => { + const ws = await connectAndWaitHandshake(PORT) + + ws.send( + JSON.stringify({ + type: 'client-handshake', + did: 'did:key:test-client', + protocolVersion: 0, + minProtocolVersion: 0, + features: [], + packageVersion: '0.0.1' + }) + ) + + const msg = await waitForMessage(ws) + expect(msg).toMatchObject({ + type: 'version-mismatch', + hubVersion: 1, + clientVersion: 0, + suggestion: 'upgrade-client' + }) + expect(typeof msg.message).toBe('string') + + // The mismatch only warns; the connection is not closed. + ws.send(JSON.stringify({ type: 'ping' })) + const pong = await waitForMessage(ws) + expect(pong.type).toBe('pong') + ws.close() + }) + + it('suggests upgrade-hub when the client requires a newer protocol', async () => { + const ws = await connectAndWaitHandshake(PORT) + + ws.send( + JSON.stringify({ + type: 'client-handshake', + did: 'did:key:test-client', + protocolVersion: 5, + minProtocolVersion: 5, + features: [], + packageVersion: '0.0.1' + }) + ) + + const msg = await waitForMessage(ws) + expect(msg).toMatchObject({ + type: 'version-mismatch', + hubVersion: 1, + clientVersion: 5, + suggestion: 'upgrade-hub' + }) + ws.close() + }) + + it('dispatches query-request through the index round-trip', async () => { + const ws = await connectAndWaitHandshake(PORT) + + ws.send( + JSON.stringify({ + type: 'index-update', + docId: 'doc-pump-1', + meta: { schemaIri: 'xnet://xnet.dev/Page', title: 'Pump Alpha' }, + text: 'alpha pump content' + }) + ) + const ack = await waitForMessage(ws) + expect(ack).toMatchObject({ type: 'index-ack', docId: 'doc-pump-1', indexed: true }) + + ws.send(JSON.stringify({ type: 'query-request', id: 'q-pump-1', query: 'alpha' })) + const response = (await waitForMessage(ws)) as { + type: string + id: string + results?: Array<{ docId: string }> + } + expect(response.type).toBe('query-response') + expect(response.id).toBe('q-pump-1') + expect(response.results?.map((r) => r.docId)).toContain('doc-pump-1') + ws.close() + }) +}) + +describe('WS message pump (auth)', () => { + let hub: HubInstance + const ROOM = 'workspace-pump-auth' + + const createToken = (capabilities: Array<{ with: string; can: string }>): string => { + const keys = generateKeyBundle() + return createUCAN({ + issuer: keys.identity.did, + issuerKey: keys.signingKey, + audience: 'did:key:hub', + capabilities + }) + } + + const connectWithToken = (token: string): Promise => + connectAndWaitHandshake(AUTH_PORT, ['xnet-sync.v1', `xnet-auth.${token}`]) + + beforeAll(async () => { + hub = await createHub({ port: AUTH_PORT, auth: true, storage: 'memory' }) + await hub.start() + }) + + afterAll(async () => { + await hub.stop() + }) + + it('serves node-sync-request for an authorized room capability', async () => { + const ws = await connectWithToken(createToken([{ with: ROOM, can: 'hub/relay' }])) + + ws.send(JSON.stringify({ type: 'node-sync-request', room: ROOM, sinceLamport: 0 })) + const response = await waitForMessage(ws) + + expect(response.type).toBe('node-sync-response') + expect(response.changes).toEqual([]) + ws.close() + }) + + it('denies node-sync-request without room capability, preserving the node-error shape', async () => { + const ws = await connectWithToken(createToken([{ with: '*', can: 'hub/query' }])) + + ws.send(JSON.stringify({ type: 'node-sync-request', room: ROOM, sinceLamport: 0 })) + const error = await waitForMessage(ws) + + // Exact legacy wire shape — clients parse these fields. + expect(error).toEqual({ + type: 'node-error', + code: 'UNAUTHORIZED', + error: 'Capability and grant index checks denied access', + action: 'hub/relay', + resource: ROOM + }) + ws.close() + }) + + it('denies query-request without query capability, preserving the query-error shape', async () => { + const ws = await connectWithToken(createToken([{ with: ROOM, can: 'hub/relay' }])) + + ws.send(JSON.stringify({ type: 'query-request', id: 'q-denied', query: 'alpha' })) + const error = await waitForMessage(ws) + + // Exact legacy wire shape — useHubSearch parses these fields. + expect(error).toEqual({ + type: 'query-error', + id: 'q-denied', + error: 'Capability does not allow querying', + code: 'FORBIDDEN', + action: 'hub/query' + }) + ws.close() + }) + + it('denies subscribe to an unauthorized doc room with auth-denied and closes 4403', async () => { + const ws = await connectWithToken(createToken([{ with: '*', can: 'hub/query' }])) + + const closed = new Promise((resolve) => ws.once('close', (code) => resolve(code))) + ws.send(JSON.stringify({ type: 'subscribe', topics: ['xnet-doc-secret'] })) + + const error = await waitForMessage(ws) + expect(error).toEqual({ + type: 'auth-denied', + code: 'UNAUTHORIZED', + action: 'hub/signal', + resource: 'secret', + error: 'Capability and grant index checks denied access' + }) + expect(await closed).toBe(4403) + }) +}) diff --git a/packages/plugins/src/ai-surface/args.ts b/packages/plugins/src/ai-surface/args.ts new file mode 100644 index 000000000..e00d833b6 --- /dev/null +++ b/packages/plugins/src/ai-surface/args.ts @@ -0,0 +1,137 @@ +/** + * Shared argument and record readers for the AI surface. + * + * Used by the service, the built-in tool registry (`tools/`), and the + * resource URI routes (`resources/`) so every entry point coerces untrusted + * agent-supplied arguments the same way — including the exact error messages. + */ + +import type { AiContextSeed, AiTargetKind } from './types' + +// ─── Required Readers ─────────────────────────────────────────────────────── + +export function readRequiredString(record: Record, key: string): string { + const value = record[key] + if (typeof value !== 'string' || !value.trim()) { + throw new Error(`${key} must be a non-empty string`) + } + return value +} + +export function readRequiredRecord( + record: Record, + key: string +): Record { + const value = record[key] + if (!isRecord(value)) { + throw new Error(`${key} must be an object`) + } + return value +} + +export function readRequiredStringArray(value: unknown, key: string): string[] { + const result = readStringArray(value) + if (result.length === 0) { + throw new Error(`${key} must contain at least one string`) + } + return result +} + +// ─── Optional Readers ─────────────────────────────────────────────────────── + +export function readStringArray(value: unknown): string[] { + if (!Array.isArray(value)) return [] + return value.filter((item): item is string => typeof item === 'string' && item.trim() !== '') +} + +export function readCsvStringArray(value: string | null): string[] { + if (!value) return [] + return value + .split(',') + .map((item) => item.trim()) + .filter(Boolean) +} + +export function readOptionalString( + record: Record, + key: string +): string | undefined { + const value = record[key] + return typeof value === 'string' && value.trim() ? value : undefined +} + +export function readOptionalRecord( + record: Record, + key: string +): Record | undefined { + return readRecord(record, key) +} + +export function readOptionalNumber( + record: Record, + key: string +): number | undefined { + const value = record[key] + return typeof value === 'number' && Number.isFinite(value) ? value : undefined +} + +export function readOptionalBoolean( + record: Record, + key: string +): boolean | undefined { + const value = record[key] + return typeof value === 'boolean' ? value : undefined +} + +export function readUrlNumber(params: URLSearchParams, key: string): number | undefined { + const value = params.get(key) + if (value === null) return undefined + const parsed = Number(value) + return Number.isFinite(parsed) ? parsed : undefined +} + +// ─── Record Readers ───────────────────────────────────────────────────────── + +export function readRecordString(record: Record, key: string): string | undefined { + const value = record[key] + return typeof value === 'string' && value.trim() ? value : undefined +} + +export function readRecord( + record: Record, + key: string +): Record | undefined { + const value = record[key] + return isRecord(value) ? value : undefined +} + +export function readRecordNumber(record: Record, key: string): number | undefined { + const value = record[key] + return typeof value === 'number' && Number.isFinite(value) ? value : undefined +} + +export function readRecordBoolean( + record: Record, + key: string +): boolean | undefined { + const value = record[key] + return typeof value === 'boolean' ? value : undefined +} + +export function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +// ─── Domain Readers ───────────────────────────────────────────────────────── + +export function readContextSeeds(value: unknown): AiContextSeed[] { + if (!Array.isArray(value)) return [] + return value + .map((seed) => { + if (!isRecord(seed)) return null + const kind = typeof seed.kind === 'string' ? (seed.kind as AiTargetKind) : null + const id = typeof seed.id === 'string' ? seed.id : null + return kind && id ? { kind, id } : null + }) + .filter((seed): seed is AiContextSeed => seed !== null) +} diff --git a/packages/plugins/src/ai-surface/host.ts b/packages/plugins/src/ai-surface/host.ts new file mode 100644 index 000000000..62e242f7a --- /dev/null +++ b/packages/plugins/src/ai-surface/host.ts @@ -0,0 +1,140 @@ +/** + * The narrow surface `AiSurfaceService` hands to its built-in tool handlers + * (`tools/`) and resource URI routes (`resources/`). + * + * The service stays the facade: it implements this interface with bound + * closures over its private methods, so extracted modules depend on this + * contract instead of the service class and the dependency points one way. + */ + +import type { + AiDatabaseMutationApplyResult, + AiPageMarkdownApplyResult, + AiPageMarkdownRollbackResult, + AiResourceContent, + AiSearchOptions +} from './service' +import type { + AiAuditEvent, + AiContextPack, + AiContextPackResource, + AiContextSeed, + AiMutationPlan +} from './types' +import type { NodeData } from '../services/local-api' + +export type AiSurfaceHost = { + // ─── Workspace And Search ───────────────────────────────────────────────── + search(options: AiSearchOptions): Promise> + expandGraph(options: { + nodeId: string + hops?: number + limit?: number + }): Promise> + createContextPack(options: { + query?: string + seeds?: AiContextSeed[] + limit?: number + }): Promise + createExternalContextResource(options: { + url: string + text: string + mimeType?: string + }): AiContextPackResource + getWorkspaceSummary(): Promise> + getRecentNodes(): Promise> + listNodes(): Promise> + listSchemas(): Promise> + getNodeOrThrow(id: string): Promise + getNodeProjection(id: string): Promise> + + // ─── Pages ──────────────────────────────────────────────────────────────── + readPageMarkdown( + pageId: string, + includeFrontmatter: boolean, + uri?: string + ): Promise + readPageOutline(pageId: string): Promise> + planPagePatch(args: Record): Promise + applyPageMarkdown(args: Record): Promise + rollbackPageMarkdown(args: Record): Promise + + // ─── Audit ──────────────────────────────────────────────────────────────── + getAuditLog(options: { planId?: string; limit?: number }): { + events: AiAuditEvent[] + count: number + limit: number + } + + // ─── Databases ──────────────────────────────────────────────────────────── + describeDatabase( + databaseId: string, + options?: { includeSample?: boolean } + ): Promise> + readDatabaseViews(databaseId: string): Promise> + queryDatabase(options: { + databaseId: string + schemaId?: string + descriptor?: Record + where?: Record + search?: unknown + orderBy?: Record + materializedView?: unknown + count?: string + limit?: number + offset?: number + }): Promise> + sampleDatabase(options: { + databaseId: string + schemaId?: string + descriptor?: Record + sampleSize?: number + }): Promise> + explainDatabaseQuery(options: { + databaseId: string + schemaId?: string + descriptor?: Record + limit?: number + offset?: number + }): Promise> + planDatabaseMutation(args: Record): Promise + applyDatabaseMutation(args: Record): Promise + + // ─── Canvases ───────────────────────────────────────────────────────────── + listCanvases(options: { limit?: number; offset?: number }): Promise> + readCanvasViewport(options: { + canvasId: string + x?: number + y?: number + w?: number + h?: number + tileSize?: number + tileIds?: string[] + includeSourcePreviews: boolean + }): Promise> + readCanvasObjects(canvasId: string): Promise> + readCanvasSelection(options: { + canvasId: string + objectIds: string[] + includeSourcePreviews: boolean + }): Promise> + searchCanvas(options: { + canvasId: string + query: string + limit?: number + }): Promise> + exportCanvasJsonCanvas(options: { + canvasId: string + includeXNetMetadata: boolean + x?: number + y?: number + w?: number + h?: number + }): Promise> + readCanvasObject(canvasId: string, objectId: string): Promise> + planCanvasJsonCanvasImport(args: Record): Promise + planCanvasMutation(args: Record): Promise + + // ─── Serialization ──────────────────────────────────────────────────────── + jsonResource(uri: string, value: unknown): AiResourceContent +} diff --git a/packages/plugins/src/ai-surface/resources/router.ts b/packages/plugins/src/ai-surface/resources/router.ts new file mode 100644 index 000000000..008cb3b8e --- /dev/null +++ b/packages/plugins/src/ai-surface/resources/router.ts @@ -0,0 +1,140 @@ +/** + * Declarative matcher for xnet:// resource URIs. + * + * Routes are registered as URI templates — `xnet://page/{pageId}.md`, + * `xnet://database/{databaseId}/schema` — and resolved in registration order. + * A template segment is either a literal or a `{param}` placeholder with an + * optional literal suffix (the `.md` case); query strings never participate + * in matching and are handed to the handler as `URLSearchParams`. + */ + +import type { AiResourceContent } from '../service' + +export type AiResourceRouteMatch = { + /** The original URI as requested (echoed into responses). */ + uri: string + /** Values captured by `{param}` template placeholders, URI-decoded. */ + params: Record + searchParams: URLSearchParams +} + +export type AiResourceRouteHandler = ( + host: THost, + match: AiResourceRouteMatch +) => Promise + +export type AiResourceRouter = { + register(template: string, handler: AiResourceRouteHandler): AiResourceRouter + /** + * Resolve a URI against the registered routes. Throws + * `Invalid xNet resource URI: …` for non-`xnet:` URIs and + * `Resource not found: …` when no route matches. + */ + resolve(host: THost, uri: string): Promise +} + +type CompiledSegment = + | { kind: 'literal'; value: string } + | { kind: 'param'; name: string; suffix: string } + +type CompiledRoute = { + host: string + segments: CompiledSegment[] + handler: AiResourceRouteHandler +} + +const XNET_URI_PREFIX = 'xnet://' +const PARAM_SEGMENT_PATTERN = /^\{([a-zA-Z][a-zA-Z0-9]*)\}(.*)$/ + +export function createAiResourceRouter(): AiResourceRouter { + const routes: CompiledRoute[] = [] + + const router: AiResourceRouter = { + register(template, handler) { + routes.push(compileRoute(template, handler)) + return router + }, + async resolve(host, uri) { + const parsed = parseXNetUri(uri) + for (const route of routes) { + const params = matchRoute(route, parsed) + if (params) { + return await route.handler(host, { uri, params, searchParams: parsed.searchParams }) + } + } + throw new Error(`Resource not found: ${uri}`) + } + } + + return router +} + +export function parseXNetUri(uri: string): { + host: string + parts: string[] + searchParams: URLSearchParams +} { + let parsed: URL + try { + parsed = new URL(uri) + } catch { + throw new Error(`Invalid xNet resource URI: ${uri}`) + } + if (parsed.protocol !== 'xnet:') { + throw new Error(`Invalid xNet resource URI: ${uri}`) + } + return { + host: parsed.hostname, + parts: parsed.pathname + .split('/') + .filter(Boolean) + .map((part) => decodeURIComponent(part)), + searchParams: parsed.searchParams + } +} + +function compileRoute( + template: string, + handler: AiResourceRouteHandler +): CompiledRoute { + if (!template.startsWith(XNET_URI_PREFIX)) { + throw new Error(`Resource route template must start with ${XNET_URI_PREFIX}: ${template}`) + } + const [path] = template.slice(XNET_URI_PREFIX.length).split('?') + const [host, ...rawSegments] = path.split('/').filter(Boolean) + if (!host) { + throw new Error(`Resource route template must include a host: ${template}`) + } + + return { + host, + segments: rawSegments.map((segment) => { + const param = PARAM_SEGMENT_PATTERN.exec(segment) + return param + ? { kind: 'param' as const, name: param[1], suffix: param[2] } + : { kind: 'literal' as const, value: segment } + }), + handler + } +} + +function matchRoute( + route: CompiledRoute, + parsed: { host: string; parts: string[] } +): Record | null { + if (parsed.host !== route.host) return null + if (parsed.parts.length !== route.segments.length) return null + + const params: Record = {} + for (const [index, segment] of route.segments.entries()) { + const part = parsed.parts[index] + if (segment.kind === 'literal') { + if (part !== segment.value) return null + continue + } + if (segment.suffix && !part.endsWith(segment.suffix)) return null + params[segment.name] = segment.suffix ? part.slice(0, -segment.suffix.length) : part + } + + return params +} diff --git a/packages/plugins/src/ai-surface/resources/routes.ts b/packages/plugins/src/ai-surface/resources/routes.ts new file mode 100644 index 000000000..e7c08a13a --- /dev/null +++ b/packages/plugins/src/ai-surface/resources/routes.ts @@ -0,0 +1,139 @@ +/** + * Built-in xnet:// resource routes. + * + * One `register` call per URI family, in the same precedence order the + * original hand-rolled matcher used. Handlers delegate to the service through + * the narrow {@link AiSurfaceHost}, so `readResource()` is a single + * `resolve()` call. + */ + +import type { AiSurfaceHost } from '../host' +import { readCsvStringArray, readUrlNumber } from '../args' +import { createAiResourceRouter, type AiResourceRouter } from './router' + +export function createBuiltInResourceRouter(): AiResourceRouter { + return ( + createAiResourceRouter() + // ─── Workspace ──────────────────────────────────────────────────────── + .register('xnet://nodes', async (host, { uri }) => + host.jsonResource(uri, await host.listNodes()) + ) + .register('xnet://schemas', async (host, { uri }) => + host.jsonResource(uri, await host.listSchemas()) + ) + .register('xnet://workspace/summary', async (host, { uri }) => + host.jsonResource(uri, await host.getWorkspaceSummary()) + ) + .register('xnet://workspace/recent', async (host, { uri }) => + host.jsonResource(uri, await host.getRecentNodes()) + ) + .register('xnet://workspace/search', async (host, { uri, searchParams }) => + host.jsonResource( + uri, + await host.search({ + query: searchParams.get('q') ?? '', + schemaId: searchParams.get('schema') ?? undefined, + limit: readUrlNumber(searchParams, 'limit'), + offset: readUrlNumber(searchParams, 'offset') + }) + ) + ) + // ─── Nodes And Pages ────────────────────────────────────────────────── + .register('xnet://node/{nodeId}', async (host, { uri, params }) => + host.jsonResource(uri, await host.getNodeProjection(params.nodeId)) + ) + .register( + 'xnet://page/{pageId}.md', + async (host, { uri, params }) => await host.readPageMarkdown(params.pageId, true, uri) + ) + .register( + 'xnet://page/{pageId}', + async (host, { uri, params }) => await host.readPageMarkdown(params.pageId, true, uri) + ) + .register('xnet://page/{pageId}/outline', async (host, { uri, params }) => + host.jsonResource(uri, await host.readPageOutline(params.pageId)) + ) + .register('xnet://page/{pageId}/context-pack', async (host, { uri, params }) => + host.jsonResource( + uri, + await host.createContextPack({ seeds: [{ kind: 'page', id: params.pageId }] }) + ) + ) + // ─── Databases ──────────────────────────────────────────────────────── + .register('xnet://database/{databaseId}/schema', async (host, { uri, params }) => + host.jsonResource(uri, await host.describeDatabase(params.databaseId)) + ) + .register('xnet://database/{databaseId}/views', async (host, { uri, params }) => + host.jsonResource(uri, await host.readDatabaseViews(params.databaseId)) + ) + .register( + 'xnet://database/{databaseId}/sample', + async (host, { uri, params, searchParams }) => + host.jsonResource( + uri, + await host.sampleDatabase({ + databaseId: params.databaseId, + sampleSize: readUrlNumber(searchParams, 'limit') + }) + ) + ) + .register('xnet://database/{databaseId}/query', async (host, { uri, params, searchParams }) => + host.jsonResource( + uri, + await host.queryDatabase({ + databaseId: params.databaseId, + schemaId: searchParams.get('schema') ?? undefined, + search: searchParams.get('q') ?? undefined, + materializedView: searchParams.get('view') + ? { viewId: searchParams.get('view') ?? '' } + : undefined, + limit: readUrlNumber(searchParams, 'limit'), + offset: readUrlNumber(searchParams, 'offset') + }) + ) + ) + // ─── Canvases ───────────────────────────────────────────────────────── + .register('xnet://canvas/{canvasId}/viewport', async (host, { uri, params, searchParams }) => + host.jsonResource( + uri, + await host.readCanvasViewport({ + canvasId: params.canvasId, + x: readUrlNumber(searchParams, 'x'), + y: readUrlNumber(searchParams, 'y'), + w: readUrlNumber(searchParams, 'w'), + h: readUrlNumber(searchParams, 'h'), + tileSize: readUrlNumber(searchParams, 'tileSize'), + tileIds: readCsvStringArray(searchParams.get('tileIds')), + includeSourcePreviews: searchParams.get('includeSourcePreviews') === 'true' + }) + ) + ) + .register('xnet://canvas/{canvasId}/objects', async (host, { uri, params }) => + host.jsonResource(uri, await host.readCanvasObjects(params.canvasId)) + ) + .register('xnet://canvas/{canvasId}/selection', async (host, { uri, params, searchParams }) => + host.jsonResource( + uri, + await host.readCanvasSelection({ + canvasId: params.canvasId, + objectIds: readCsvStringArray(searchParams.get('ids')), + includeSourcePreviews: searchParams.get('includeSourcePreviews') !== 'false' + }) + ) + ) + .register( + 'xnet://canvas/{canvasId}/json-canvas', + async (host, { uri, params, searchParams }) => + host.jsonResource( + uri, + await host.exportCanvasJsonCanvas({ + canvasId: params.canvasId, + includeXNetMetadata: searchParams.get('includeXNetMetadata') !== 'false' + }) + ) + ) + .register('xnet://canvas/{canvasId}/object/{objectId}', async (host, { uri, params }) => + host.jsonResource(uri, await host.readCanvasObject(params.canvasId, params.objectId)) + ) + ) +} diff --git a/packages/plugins/src/ai-surface/service.test.ts b/packages/plugins/src/ai-surface/service.test.ts new file mode 100644 index 000000000..0a730447b --- /dev/null +++ b/packages/plugins/src/ai-surface/service.test.ts @@ -0,0 +1,678 @@ +/** + * Characterization tests for AiSurfaceService (exploration 0276). + * + * Pins the externally observable surface — tool names/risk/scopes, callTool + * dispatch, the page/database mutation plan → apply → rollback round-trips, + * audit-log recording, and the xnet:// resource URI families — so the tool + * registry and resource URI router refactors must preserve behavior exactly. + */ + +import type { + AiDatabaseMutationApplyResult, + AiPageMarkdownApplyResult, + AiPageMarkdownRollbackResult, + AiSurfaceService +} from './service' +import type { AiAuditEvent, AiMutationPlan } from './types' +import type { NodeData, NodeStoreAPI, SchemaRegistryAPI } from '../services/local-api' +import { beforeEach, describe, expect, it } from 'vitest' +import { createAiSurfaceService } from './service' + +// ─── Fixtures ─────────────────────────────────────────────────────────────── + +const PAGE_SCHEMA = 'xnet://xnet.fyi/Page@1.0.0' +const DATABASE_SCHEMA = 'xnet://xnet.fyi/Database@1.0.0' +const ROW_SCHEMA = 'xnet://xnet.fyi/DatabaseRow@2.0.0' +const CANVAS_SCHEMA = 'xnet://xnet.fyi/Canvas@1.0.0' + +const PAGE_MARKDOWN = '# Meeting Notes\n\nDiscussed roadmap milestones.' + +function createFixtureNodes(): NodeData[] { + return [ + { + id: 'page-1', + schemaId: PAGE_SCHEMA, + properties: { title: 'Meeting Notes', markdown: PAGE_MARKDOWN }, + deleted: false, + createdAt: 1, + updatedAt: 100 + }, + { + id: 'db-1', + schemaId: DATABASE_SCHEMA, + properties: { + title: 'Tasks', + rowSchemaId: ROW_SCHEMA, + columns: [{ id: 'col-title', name: 'Title', type: 'text' }], + views: [{ id: 'view-1', name: 'All tasks' }] + }, + deleted: false, + createdAt: 2, + updatedAt: 90 + }, + { + id: 'row-1', + schemaId: ROW_SCHEMA, + properties: { title: 'Ship the refactor', database: 'db-1', status: 'todo' }, + deleted: false, + createdAt: 3, + updatedAt: 80 + }, + { + id: 'canvas-1', + schemaId: CANVAS_SCHEMA, + properties: { + title: 'Planning Board', + objects: [ + { + id: 'obj-1', + type: 'note', + x: 0, + y: 0, + width: 240, + height: 160, + properties: { title: 'Sticky note' } + } + ], + edges: [] + }, + deleted: false, + createdAt: 4, + updatedAt: 70 + } + ] +} + +type MemoryStore = NodeStoreAPI & { readonly nodes: Map } + +function createMemoryStore(seed: NodeData[]): MemoryStore { + const nodes = new Map( + seed.map((node) => [node.id, { ...node, properties: { ...node.properties } }]) + ) + let created = 0 + let tick = 1000 + + return { + nodes, + get: async (id) => nodes.get(id) ?? null, + list: async (options) => { + let result = Array.from(nodes.values()) + if (options?.schemaId) result = result.filter((node) => node.schemaId === options.schemaId) + if (options?.offset) result = result.slice(options.offset) + if (options?.limit) result = result.slice(0, options.limit) + return result + }, + create: async (options) => { + created += 1 + tick += 1 + const node: NodeData = { + id: `created-${created}`, + schemaId: options.schemaId, + properties: { ...options.properties }, + deleted: false, + createdAt: tick, + updatedAt: tick + } + nodes.set(node.id, node) + return node + }, + update: async (id, options) => { + const existing = nodes.get(id) + if (!existing) throw new Error(`Node not found: ${id}`) + const node: NodeData = { + ...existing, + properties: { ...existing.properties, ...options.properties }, + updatedAt: existing.updatedAt + 1 + } + nodes.set(id, node) + return node + }, + delete: async (id) => { + const existing = nodes.get(id) + if (existing) { + nodes.set(id, { ...existing, deleted: true, updatedAt: existing.updatedAt + 1 }) + } + }, + subscribe: () => () => {} + } +} + +const schemas: SchemaRegistryAPI = { + getAllIRIs: () => [PAGE_SCHEMA, DATABASE_SCHEMA, ROW_SCHEMA, CANVAS_SCHEMA], + get: async (iri) => { + if (iri === PAGE_SCHEMA) { + return { iri, name: 'Page', properties: { title: { type: 'text' } } } + } + if (iri === DATABASE_SCHEMA) { + return { iri, name: 'Database', properties: { title: { type: 'text' } } } + } + if (iri === ROW_SCHEMA) { + return { iri, name: 'DatabaseRow', properties: { title: { type: 'text' } } } + } + if (iri === CANVAS_SCHEMA) { + return { iri, name: 'Canvas', properties: { title: { type: 'text' } } } + } + return null + } +} + +// ─── Expected Tool Surface ────────────────────────────────────────────────── + +// The full built-in tool surface, in registration order. Names, risk levels, +// and required scopes are load-bearing: agents, the MCP server, and scope +// gating all key off them. +const EXPECTED_BUILT_IN_TOOLS: Record = { + xnet_search: { risk: 'low', requiredScopes: ['workspace.search'] }, + xnet_graph_expand: { risk: 'low', requiredScopes: ['workspace.read'] }, + xnet_create_context_pack: { risk: 'low', requiredScopes: ['workspace.read', 'workspace.search'] }, + xnet_create_external_context_resource: { risk: 'medium', requiredScopes: ['network.fetch'] }, + xnet_read_page_markdown: { risk: 'low', requiredScopes: ['page.read'] }, + xnet_validate_page_markdown: { risk: 'low', requiredScopes: ['page.read'] }, + xnet_plan_page_patch: { risk: 'medium', requiredScopes: ['page.read', 'page.propose'] }, + xnet_apply_page_markdown: { risk: 'high', requiredScopes: ['page.read', 'page.write'] }, + xnet_get_audit_log: { risk: 'low', requiredScopes: ['workspace.read'] }, + xnet_rollback_page_markdown: { risk: 'high', requiredScopes: ['page.write'] }, + xnet_database_describe: { risk: 'low', requiredScopes: ['database.read'] }, + xnet_database_query: { risk: 'low', requiredScopes: ['database.read', 'database.query'] }, + xnet_database_sample: { risk: 'low', requiredScopes: ['database.read', 'database.query'] }, + xnet_database_explain_query: { + risk: 'low', + requiredScopes: ['database.read', 'database.query', 'storage.diagnostics'] + }, + xnet_plan_database_mutation: { + risk: 'medium', + requiredScopes: ['database.read', 'database.propose'] + }, + xnet_apply_database_mutation: { + risk: 'high', + requiredScopes: ['database.read', 'database.write.rows', 'database.write.schema'] + }, + xnet_canvas_list: { risk: 'low', requiredScopes: ['canvas.read'] }, + xnet_canvas_read_viewport: { risk: 'low', requiredScopes: ['canvas.read'] }, + xnet_canvas_read_selection: { risk: 'low', requiredScopes: ['canvas.read'] }, + xnet_canvas_search: { risk: 'low', requiredScopes: ['canvas.read'] }, + xnet_canvas_export_json_canvas: { risk: 'low', requiredScopes: ['canvas.read'] }, + xnet_canvas_plan_json_canvas_import: { + risk: 'medium', + requiredScopes: ['canvas.read', 'canvas.propose'] + }, + xnet_plan_canvas_mutation: { risk: 'medium', requiredScopes: ['canvas.read', 'canvas.propose'] }, + xnet_validate_mutation_plan: { risk: 'medium', requiredScopes: ['workspace.read'] } +} + +// ─── Tests ────────────────────────────────────────────────────────────────── + +describe('AiSurfaceService characterization (0276)', () => { + let store: MemoryStore + let service: AiSurfaceService + + beforeEach(() => { + store = createMemoryStore(createFixtureNodes()) + service = createAiSurfaceService({ store, schemas }) + }) + + describe('getTools', () => { + it('returns the built-in tool names in registration order', () => { + expect(service.getTools().map((tool) => tool.name)).toEqual( + Object.keys(EXPECTED_BUILT_IN_TOOLS) + ) + }) + + it('pins risk levels and required scopes for every built-in tool', () => { + for (const tool of service.getTools()) { + expect( + { risk: tool.risk, requiredScopes: tool.requiredScopes }, + `tool surface for ${tool.name}` + ).toEqual(EXPECTED_BUILT_IN_TOOLS[tool.name]) + } + }) + + it('every tool carries a title, description, and object input schema', () => { + for (const tool of service.getTools()) { + expect(tool.title, `title for ${tool.name}`).toBeTruthy() + expect(tool.description, `description for ${tool.name}`).toBeTruthy() + expect(tool.inputSchema.type, `inputSchema for ${tool.name}`).toBe('object') + } + }) + + it('appends contributed extra tools without their invoke implementation', async () => { + const extras = createAiSurfaceService({ + store, + schemas, + extraTools: [ + { + name: 'my_extra_tool', + title: 'My extra tool', + description: 'A contributed tool.', + risk: 'low', + requiredScopes: ['workspace.read'], + inputSchema: { type: 'object', properties: {} }, + invoke: (args) => ({ echo: args }) + }, + { + // Collides with a built-in name: the built-in must win everywhere. + name: 'xnet_search', + title: 'Shadowing search', + description: 'Must be dropped.', + risk: 'low', + requiredScopes: ['workspace.read'], + inputSchema: { type: 'object', properties: {} }, + invoke: () => 'shadowed' + } + ] + }) + + const tools = extras.getTools() + expect(tools.map((tool) => tool.name)).toEqual([ + ...Object.keys(EXPECTED_BUILT_IN_TOOLS), + 'my_extra_tool' + ]) + expect(tools.some((tool) => 'invoke' in tool)).toBe(false) + + await expect(extras.callTool('my_extra_tool', { a: 1 })).resolves.toEqual({ + echo: { a: 1 } + }) + const search = (await extras.callTool('xnet_search', { query: 'roadmap' })) as Record< + string, + unknown + > + expect(search).not.toBe('shadowed') + expect(search.count).toBe(1) + }) + }) + + describe('callTool dispatch', () => { + it('xnet_search finds nodes by property text and validates required args', async () => { + const result = (await service.callTool('xnet_search', { query: 'roadmap' })) as { + count: number + results: Array<{ id: string; schemaId: string }> + } + expect(result.count).toBe(1) + expect(result.results[0].id).toBe('page-1') + expect(result.results[0].schemaId).toBe(PAGE_SCHEMA) + + await expect(service.callTool('xnet_search', {})).rejects.toThrow( + 'query must be a non-empty string' + ) + }) + + it('xnet_database_describe reports schema, columns, views, and row schema', async () => { + const result = (await service.callTool('xnet_database_describe', { + databaseId: 'db-1' + })) as Record + + expect(result.rowSchemaId).toBe(ROW_SCHEMA) + expect(result.columns).toEqual([{ id: 'col-title', name: 'Title', type: 'text' }]) + expect(result.views).toEqual([{ id: 'view-1', name: 'All tasks' }]) + expect((result.database as Record).id).toBe('db-1') + expect(result.revision).toBe('updatedAt:90') + }) + + it('xnet_database_query filters rows by database membership via list fallback', async () => { + const result = (await service.callTool('xnet_database_query', { + databaseId: 'db-1' + })) as { + rows: Array<{ id: string }> + count: number + queryPlan: { strategy: string } + } + + expect(result.count).toBe(1) + expect(result.rows[0].id).toBe('row-1') + expect(result.queryPlan.strategy).toBe('list-fallback') + }) + + it('xnet_canvas_list returns only canvas nodes', async () => { + const result = (await service.callTool('xnet_canvas_list', {})) as { + count: number + canvases: Array<{ id: string }> + } + expect(result.count).toBe(1) + expect(result.canvases[0].id).toBe('canvas-1') + }) + + it('throws the unknown-tool error for unregistered names', async () => { + await expect(service.callTool('xnet_nonexistent')).rejects.toThrow( + 'Unknown AI surface tool: xnet_nonexistent' + ) + }) + }) + + describe('page markdown mutation round-trip (plan → apply → rollback → audit)', () => { + const editedMarkdown = '# Meeting Notes\n\nDiscussed roadmap milestones.\n\nAdded a decision.' + + async function planPagePatch(): Promise { + return (await service.callTool('xnet_plan_page_patch', { + pageId: 'page-1', + markdown: editedMarkdown, + intent: 'Add a decision', + actor: 'characterization-test' + })) as AiMutationPlan + } + + it('plans a validated replaceMarkdown mutation with a review diff', async () => { + const plan = await planPagePatch() + + expect(plan.validation.valid).toBe(true) + expect(plan.status).toBe('validated') + expect(plan.actor).toBe('characterization-test') + expect(plan.risk).toBe('medium') + expect(plan.requiredScopes).toEqual(['page.read', 'page.propose']) + expect(plan.changes).toHaveLength(1) + expect(plan.changes[0].targetKind).toBe('page') + expect(plan.changes[0].targetId).toBe('page-1') + expect(plan.changes[0].baseRevision).toBe('updatedAt:100') + expect(plan.changes[0].operations[0].op).toBe('replaceMarkdown') + expect(plan.changes[0].operations[0].args.markdown).toBe(editedMarkdown) + expect(plan.changes[0].operations[0].args.diff).toContain('+Added a decision.') + // Body markdown without frontmatter validates with a warning, not an error. + expect(plan.validation.warnings).toContain('Markdown is missing xNet frontmatter identity') + }) + + it('applies, records an audit event, and rolls back to the previous markdown', async () => { + const plan = await planPagePatch() + + const applied = (await service.callTool('xnet_apply_page_markdown', { + plan, + confirmApply: true + })) as AiPageMarkdownApplyResult + + expect(applied.applied).toBe(true) + expect(applied.pageId).toBe('page-1') + expect(applied.planId).toBe(plan.id) + expect(applied.mode).toBe('node-property') + expect(applied.validation.valid).toBe(true) + expect(applied.rollbackHandle).toMatch(/^rollback_/) + expect(applied.auditEventId).toBeTruthy() + + const afterApply = store.nodes.get('page-1') + expect(afterApply?.properties.markdown).toBe(editedMarkdown) + expect(afterApply?.properties.aiLastAppliedPlanId).toBe(plan.id) + + // The audit log recorded the apply, retrievable by plan id. + const audit = (await service.callTool('xnet_get_audit_log', { planId: plan.id })) as { + events: AiAuditEvent[] + count: number + } + expect(audit.count).toBe(1) + expect(audit.events[0].planId).toBe(plan.id) + expect(audit.events[0].actor).toBe('characterization-test') + expect(audit.events[0].appliedChangeIds).toEqual(['page-1']) + expect(audit.events[0].rollbackHandle).toBe(applied.rollbackHandle) + + const rolledBack = (await service.callTool('xnet_rollback_page_markdown', { + rollbackHandle: applied.rollbackHandle, + confirmRollback: true + })) as AiPageMarkdownRollbackResult + + expect(rolledBack.rolledBack).toBe(true) + expect(rolledBack.pageId).toBe('page-1') + expect(rolledBack.planId).toBe(plan.id) + expect(rolledBack.auditEventId).toBeTruthy() + + const afterRollback = store.nodes.get('page-1') + expect(afterRollback?.properties.markdown).toBe(PAGE_MARKDOWN) + expect(afterRollback?.properties.aiRolledBackPlanId).toBe(plan.id) + + // The rollback landed as a second audit event under the same plan id. + const auditAfterRollback = (await service.callTool('xnet_get_audit_log', { + planId: plan.id + })) as { events: AiAuditEvent[]; count: number } + expect(auditAfterRollback.count).toBe(2) + expect(auditAfterRollback.events[1].actor).toBe('xnet-rollback') + expect(auditAfterRollback.events[1].appliedChangeIds).toEqual(['rollback:page-1']) + }) + + it('requires explicit confirmation flags for apply and rollback', async () => { + const plan = await planPagePatch() + + await expect(service.callTool('xnet_apply_page_markdown', { plan })).rejects.toThrow( + 'confirmApply must be true to apply a page Markdown plan' + ) + await expect( + service.callTool('xnet_rollback_page_markdown', { rollbackHandle: 'rollback_x' }) + ).rejects.toThrow('confirmRollback must be true to rollback a page Markdown apply') + }) + + it('reports unknown rollback handles without throwing', async () => { + const result = (await service.callTool('xnet_rollback_page_markdown', { + rollbackHandle: 'rollback_missing', + confirmRollback: true + })) as AiPageMarkdownRollbackResult + + expect(result.rolledBack).toBe(false) + expect(result.pageId).toBe('unknown') + expect(result.validation.errors).toEqual(['Unknown rollback handle: rollback_missing']) + }) + + it('rejects stale plans unless allowStale is set', async () => { + const plan = await planPagePatch() + // Move the live node past the plan's base revision. + await store.update('page-1', { properties: { title: 'Meeting Notes (renamed)' } }) + + const rejected = (await service.callTool('xnet_apply_page_markdown', { + plan, + confirmApply: true + })) as AiPageMarkdownApplyResult + expect(rejected.applied).toBe(false) + expect(rejected.validation.errors[0]).toMatch( + /baseRevision updatedAt:100 does not match live revision/ + ) + + const allowed = (await service.callTool('xnet_apply_page_markdown', { + plan, + confirmApply: true, + allowStale: true + })) as AiPageMarkdownApplyResult + expect(allowed.applied).toBe(true) + expect( + allowed.validation.warnings.some((w) => w.includes('does not match live revision')) + ).toBe(true) + }) + + it('rejects structurally invalid plans instead of applying them', async () => { + const result = (await service.callTool('xnet_apply_page_markdown', { + plan: { id: 'plan_bogus' }, + confirmApply: true + })) as AiPageMarkdownApplyResult + + expect(result.applied).toBe(false) + expect(result.planId).toBe('plan_bogus') + expect(result.validation.valid).toBe(false) + expect(result.validation.errors.length).toBeGreaterThan(0) + }) + }) + + describe('database mutation round-trip (plan → apply, transactional rollback, audit)', () => { + it('plans and applies a row create, recording an audit event', async () => { + const plan = (await service.callTool('xnet_plan_database_mutation', { + databaseId: 'db-1', + operations: [{ op: 'createRow', args: { properties: { title: 'New row' } } }], + actor: 'characterization-test' + })) as AiMutationPlan + + expect(plan.validation.valid).toBe(true) + expect(plan.risk).toBe('medium') + expect(plan.requiredScopes).toEqual([ + 'database.read', + 'database.propose', + 'database.write.rows' + ]) + expect(plan.changes).toHaveLength(1) + expect(plan.changes[0].targetKind).toBe('databaseRows') + + const applied = (await service.callTool('xnet_apply_database_mutation', { + plan, + confirmApply: true + })) as AiDatabaseMutationApplyResult + + expect(applied.applied).toBe(true) + expect(applied.appliedChangeIds).toEqual(['row:create:created-1']) + expect(applied.rolledBackChangeIds).toEqual([]) + expect(applied.auditEventId).toBeTruthy() + + const createdRow = store.nodes.get('created-1') + expect(createdRow?.schemaId).toBe(ROW_SCHEMA) + expect(createdRow?.properties).toEqual({ database: 'db-1', title: 'New row' }) + + const audit = (await service.callTool('xnet_get_audit_log', { planId: plan.id })) as { + events: AiAuditEvent[] + count: number + } + expect(audit.count).toBe(1) + expect(audit.events[0].appliedChangeIds).toEqual(['row:create:created-1']) + }) + + it('rolls back already-applied row mutations when a later operation fails', async () => { + const plan = (await service.callTool('xnet_plan_database_mutation', { + databaseId: 'db-1', + operations: [ + { op: 'createRow', args: { properties: { title: 'Will be rolled back' } } }, + { op: 'updateRow', args: { rowId: 'missing-row', properties: { title: 'Nope' } } } + ] + })) as AiMutationPlan + expect(plan.validation.valid).toBe(true) + + const result = (await service.callTool('xnet_apply_database_mutation', { + plan, + confirmApply: true + })) as AiDatabaseMutationApplyResult + + expect(result.applied).toBe(false) + expect(result.validation.errors).toEqual(['Node not found: missing-row']) + expect(result.appliedChangeIds).toEqual(['row:create:created-1']) + expect(result.rolledBackChangeIds).toEqual(['row:rollback-delete:created-1']) + expect( + result.validation.warnings.some((warning) => + warning.includes('Previously applied row mutations in this plan were rolled back.') + ) + ).toBe(true) + + // The created row was deleted again by the rollback. + expect(store.nodes.get('created-1')?.deleted).toBe(true) + + // Failed applies do not append audit events. + const audit = (await service.callTool('xnet_get_audit_log', { planId: plan.id })) as { + count: number + } + expect(audit.count).toBe(0) + }) + + it('requires confirmDelete for destructive row operations at plan time', async () => { + const plan = (await service.callTool('xnet_plan_database_mutation', { + databaseId: 'db-1', + operations: [{ op: 'deleteRow', args: { rowId: 'row-1' } }] + })) as AiMutationPlan + + expect(plan.validation.valid).toBe(false) + expect(plan.risk).toBe('high') + expect(plan.validation.errors).toEqual([ + 'operations[0] delete/drop/remove operations require confirmDelete true or deletionMarker "DELETE"' + ]) + }) + }) + + describe('readResource URI families', () => { + it('serves the workspace summary as compact JSON', async () => { + const content = await service.readResource('xnet://workspace/summary') + expect(content.uri).toBe('xnet://workspace/summary') + expect(content.mimeType).toBe('application/json') + + const summary = JSON.parse(content.text) as Record + expect(summary.nodeSampleCount).toBe(4) + expect(summary.schemaCount).toBe(4) + expect(summary.schemaCounts).toEqual({ + [PAGE_SCHEMA]: 1, + [DATABASE_SCHEMA]: 1, + [ROW_SCHEMA]: 1, + [CANVAS_SCHEMA]: 1 + }) + const tools = summary.tools as Array<{ name: string }> + expect(tools.map((tool) => tool.name)).toEqual(Object.keys(EXPECTED_BUILT_IN_TOOLS)) + }) + + it('serves page markdown with xNet frontmatter identity', async () => { + const content = await service.readResource('xnet://page/page-1.md') + expect(content.mimeType).toBe('text/markdown') + expect(content.text).toContain('id: "page-1"') + expect(content.text).toContain(`schemaId: "${PAGE_SCHEMA}"`) + expect(content.text).toContain('revision: "updatedAt:100"') + expect(content.text).toContain('# Meeting Notes') + expect(content.text.startsWith('---\nxnet:\n')).toBe(true) + }) + + it('serves the page outline extracted from the markdown projection', async () => { + const content = await service.readResource('xnet://page/page-1/outline') + const outline = JSON.parse(content.text) as { + pageId: string + headings: Array<{ level: number; title: string }> + } + expect(outline.pageId).toBe('page-1') + expect(outline.headings).toEqual([{ level: 1, title: 'Meeting Notes', lineNumber: 1 }]) + }) + + it('serves the database schema projection', async () => { + const content = await service.readResource('xnet://database/db-1/schema') + expect(content.mimeType).toBe('application/json') + + const described = JSON.parse(content.text) as Record + expect(described.rowSchemaId).toBe(ROW_SCHEMA) + expect(described.columns).toEqual([{ id: 'col-title', name: 'Title', type: 'text' }]) + expect((described.database as Record).id).toBe('db-1') + }) + + it('serves canvas objects with normalized geometry', async () => { + const content = await service.readResource('xnet://canvas/canvas-1/objects') + const canvas = JSON.parse(content.text) as { + canvasId: string + count: number + objects: Array> + } + expect(canvas.canvasId).toBe('canvas-1') + expect(canvas.count).toBe(1) + expect(canvas.objects[0]).toMatchObject({ + id: 'obj-1', + type: 'note', + x: 0, + y: 0, + width: 240, + height: 160 + }) + }) + + it('rejects malformed and unknown resource URIs with the exact error messages', async () => { + await expect(service.readResource('not-a-uri')).rejects.toThrow( + 'Invalid xNet resource URI: not-a-uri' + ) + await expect(service.readResource('https://example.com/x')).rejects.toThrow( + 'Invalid xNet resource URI: https://example.com/x' + ) + await expect(service.readResource('xnet://bogus/whatever')).rejects.toThrow( + 'Resource not found: xnet://bogus/whatever' + ) + await expect(service.readResource('xnet://page/nope.md')).rejects.toThrow( + 'Node not found: nope' + ) + }) + }) + + describe('getResources', () => { + it('lists the advertised resource URI templates', () => { + expect(service.getResources().map((resource) => resource.uri)).toEqual([ + 'xnet://workspace/summary', + 'xnet://workspace/recent', + 'xnet://nodes', + 'xnet://schemas', + 'xnet://page/{pageId}.md', + 'xnet://page/{pageId}/outline', + 'xnet://database/{databaseId}/schema', + 'xnet://database/{databaseId}/views', + 'xnet://database/{databaseId}/sample?limit=10', + 'xnet://canvas/{canvasId}/viewport?x=0&y=0&w=1000&h=800', + 'xnet://canvas/{canvasId}/objects', + 'xnet://canvas/{canvasId}/selection?ids=object-1,object-2', + 'xnet://canvas/{canvasId}/json-canvas' + ]) + }) + }) +}) diff --git a/packages/plugins/src/ai-surface/service.ts b/packages/plugins/src/ai-surface/service.ts index 0df64b146..866cd55dc 100644 --- a/packages/plugins/src/ai-surface/service.ts +++ b/packages/plugins/src/ai-surface/service.ts @@ -2,6 +2,7 @@ * AI surface service for focused resources, context packs, and plan-only tools. */ +import type { AiSurfaceHost } from './host' import type { AiAuditEvent, AiChangeSet, @@ -14,7 +15,6 @@ import type { AiResource, AiRiskLevel, AiScope, - AiTargetKind, AiToolDefinition } from './types' import type { NodeData, NodeStoreAPI, SchemaData, SchemaRegistryAPI } from '../services/local-api' @@ -26,11 +26,25 @@ import type { NodeQuerySearchFilter, SortDirection } from '@xnetjs/data' +import { + isRecord, + readOptionalBoolean, + readOptionalString, + readRecord, + readRecordBoolean, + readRecordNumber, + readRecordString, + readRequiredRecord, + readRequiredString, + readStringArray +} from './args' import { renderMarkdownReviewDiff, stripXNetPageFrontmatter, validateXNetPageMarkdown } from './page-markdown' +import { createBuiltInResourceRouter } from './resources/routes' +import { BUILT_IN_TOOL_ENTRIES, BUILT_IN_TOOLS_BY_NAME } from './tools' import { attachAiPlanValidation, createAiOperation, validateAiMutationPlan } from './validation' // ─── Types ───────────────────────────────────────────────────────────────── @@ -198,6 +212,9 @@ const DEFAULT_LIMITS: AiSurfaceLimits = { maxDatabaseRows: 100 } +/** Stateless route table for `readResource` — shared across service instances. */ +const BUILT_IN_RESOURCE_ROUTES = createBuiltInResourceRouter() + // ─── Service ──────────────────────────────────────────────────────────────── export class AiSurfaceService { @@ -209,6 +226,52 @@ export class AiSurfaceService { /** Contributed tools by name (exploration 0196), de-duped at construction. */ private readonly extraTools = new Map() + /** + * The narrow surface handed to built-in tool handlers and resource routes + * (`tools/`, `resources/`): closures over the private methods, so the class + * stays the facade and its public type is unchanged. The closures are lazy — + * nothing here runs until a tool call or resource read arrives. + */ + private readonly host: AiSurfaceHost = { + search: (options) => this.search(options), + expandGraph: (options) => this.expandGraph(options), + createContextPack: (options) => this.createContextPack(options), + createExternalContextResource: (options) => this.createExternalContextResource(options), + getWorkspaceSummary: () => this.getWorkspaceSummary(), + getRecentNodes: () => this.getRecentNodes(), + listNodes: async () => { + const nodes = await this.config.store.list({ limit: this.limits.maxListLimit }) + return { nodes, count: nodes.length, limit: this.limits.maxListLimit } + }, + listSchemas: async () => ({ schemas: await this.getSchemaSummaries(true) }), + getNodeOrThrow: (id) => this.getNodeOrThrow(id), + getNodeProjection: (id) => this.getNodeProjection(id), + readPageMarkdown: (pageId, includeFrontmatter, uri) => + this.readPageMarkdown(pageId, includeFrontmatter, uri), + readPageOutline: (pageId) => this.readPageOutline(pageId), + planPagePatch: (args) => this.planPagePatch(args), + applyPageMarkdown: (args) => this.applyPageMarkdown(args), + rollbackPageMarkdown: (args) => this.rollbackPageMarkdown(args), + getAuditLog: (options) => this.getAuditLog(options), + describeDatabase: (databaseId, options) => this.describeDatabase(databaseId, options), + readDatabaseViews: (databaseId) => this.readDatabaseViews(databaseId), + queryDatabase: (options) => this.queryDatabase(options), + sampleDatabase: (options) => this.sampleDatabase(options), + explainDatabaseQuery: (options) => this.explainDatabaseQuery(options), + planDatabaseMutation: (args) => this.planDatabaseMutation(args), + applyDatabaseMutation: (args) => this.applyDatabaseMutation(args), + listCanvases: (options) => this.listCanvases(options), + readCanvasViewport: (options) => this.readCanvasViewport(options), + readCanvasObjects: (canvasId) => this.readCanvasObjects(canvasId), + readCanvasSelection: (options) => this.readCanvasSelection(options), + searchCanvas: (options) => this.searchCanvas(options), + exportCanvasJsonCanvas: (options) => this.exportCanvasJsonCanvas(options), + readCanvasObject: (canvasId, objectId) => this.readCanvasObject(canvasId, objectId), + planCanvasJsonCanvasImport: (args) => this.planCanvasJsonCanvasImport(args), + planCanvasMutation: (args) => this.planCanvasMutation(args), + jsonResource: (uri, value) => this.jsonResource(uri, value) + } + constructor(private readonly config: AiSurfaceServiceConfig) { this.limits = { ...DEFAULT_LIMITS, ...config.limits } this.clock = config.clock ?? (() => new Date()) @@ -352,792 +415,25 @@ export class AiSurfaceService { } private builtInTools(): AiToolDefinition[] { - return [ - { - name: 'xnet_search', - title: 'Search xNet workspace', - description: 'Search node titles and searchable properties with pagination and limits.', - risk: 'low', - requiredScopes: ['workspace.search'], - inputSchema: { - type: 'object', - properties: { - query: { type: 'string', description: 'Search text.' }, - schemaId: { type: 'string', description: 'Optional schema IRI filter.' }, - limit: { type: 'number', description: 'Maximum result count.' }, - offset: { type: 'number', description: 'Result offset for pagination.' } - }, - required: ['query'] - } - }, - { - name: 'xnet_graph_expand', - title: 'Expand a node along its relations', - description: - 'Walk typed relation edges out from a node to its connected neighbors (bounded by hops and a result limit). Use for just-in-time expansion: fetch a specific node’s connections only when you need them, instead of pulling the whole graph into context.', - risk: 'low', - requiredScopes: ['workspace.read'], - inputSchema: { - type: 'object', - properties: { - nodeId: { type: 'string', description: 'The node to expand from.' }, - hops: { - type: 'number', - description: 'How many relation hops to walk (1–2, default 1).' - }, - limit: { type: 'number', description: 'Maximum neighbors to return.' } - }, - required: ['nodeId'] - } - }, - { - name: 'xnet_create_context_pack', - title: 'Create context pack', - description: 'Create a bounded context pack from seeds and optional search results.', - risk: 'low', - requiredScopes: ['workspace.read', 'workspace.search'], - inputSchema: { - type: 'object', - properties: { - query: { type: 'string', description: 'Optional search query.' }, - seeds: { - type: 'array', - description: 'Seed resources such as pages, databases, canvases, or nodes.', - items: { - type: 'object', - properties: { - kind: { type: 'string', description: 'Seed kind.' }, - id: { type: 'string', description: 'Seed id.' } - } - } - }, - limit: { type: 'number', description: 'Maximum resources to include.' } - } - } - }, - { - name: 'xnet_create_external_context_resource', - title: 'Create untrusted external context resource', - description: - 'Wrap externally fetched content as an untrusted context-pack resource with an explicit instruction boundary.', - risk: 'medium', - requiredScopes: ['network.fetch'], - inputSchema: { - type: 'object', - properties: { - url: { type: 'string', description: 'External source URL.' }, - text: { type: 'string', description: 'Fetched external text content.' }, - mimeType: { type: 'string', description: 'Source MIME type. Defaults to text/plain.' } - }, - required: ['url', 'text'] - } - }, - { - name: 'xnet_read_page_markdown', - title: 'Read page Markdown', - description: 'Read a page as Markdown with optional xNet frontmatter.', - risk: 'low', - requiredScopes: ['page.read'], - inputSchema: { - type: 'object', - properties: { - pageId: { type: 'string', description: 'Page node id.' }, - includeFrontmatter: { - type: 'boolean', - description: 'Include xNet identity frontmatter. Defaults to true.' - } - }, - required: ['pageId'] - } - }, - { - name: 'xnet_validate_page_markdown', - title: 'Validate page Markdown', - description: 'Validate xNet page frontmatter and supported xNet Markdown directives.', - risk: 'low', - requiredScopes: ['page.read'], - inputSchema: { - type: 'object', - properties: { - pageId: { type: 'string', description: 'Optional target page node id.' }, - baseRevision: { type: 'string', description: 'Optional expected base revision.' }, - markdown: { type: 'string', description: 'Markdown to validate.' } - }, - required: ['markdown'] - } - }, - { - name: 'xnet_plan_page_patch', - title: 'Plan page Markdown patch', - description: - 'Validate an edited Markdown page and return a mutation plan without applying it.', - risk: 'medium', - requiredScopes: ['page.read', 'page.propose'], - inputSchema: { - type: 'object', - properties: { - pageId: { type: 'string', description: 'Page node id.' }, - baseRevision: { type: 'string', description: 'Revision the patch was based on.' }, - markdown: { type: 'string', description: 'Proposed full Markdown replacement.' }, - intent: { type: 'string', description: 'User or agent intent for the patch.' }, - actor: { type: 'string', description: 'Agent or user creating the plan.' } - }, - required: ['pageId', 'markdown'] - } - }, - { - name: 'xnet_apply_page_markdown', - title: 'Apply page Markdown plan', - description: - 'Apply a validated page Markdown mutation plan through the configured TipTap/Yjs document adapter, with a node-property fallback.', - risk: 'high', - requiredScopes: ['page.read', 'page.write'], - inputSchema: { - type: 'object', - properties: { - plan: { type: 'object', description: 'Validated page Markdown mutation plan.' }, - confirmApply: { - type: 'boolean', - description: 'Must be true to apply the page Markdown plan.' - }, - allowStale: { - type: 'boolean', - description: 'Allow applying when the plan base revision differs from the live node.' - } - }, - required: ['plan', 'confirmApply'] - } - }, - { - name: 'xnet_get_audit_log', - title: 'Read AI audit log', - description: 'Read recent AI mutation audit events with optional plan filtering.', - risk: 'low', - requiredScopes: ['workspace.read'], - inputSchema: { - type: 'object', - properties: { - planId: { type: 'string', description: 'Optional mutation plan id filter.' }, - limit: { type: 'number', description: 'Maximum audit events to return.' } - } - } - }, - { - name: 'xnet_rollback_page_markdown', - title: 'Rollback page Markdown apply', - description: 'Rollback a previously applied page Markdown plan by rollback handle.', - risk: 'high', - requiredScopes: ['page.write'], - inputSchema: { - type: 'object', - properties: { - rollbackHandle: { type: 'string', description: 'Rollback handle from apply result.' }, - confirmRollback: { - type: 'boolean', - description: 'Must be true to perform the rollback.' - } - }, - required: ['rollbackHandle', 'confirmRollback'] - } - }, - { - name: 'xnet_database_describe', - title: 'Describe database', - description: 'Describe database schema, columns, views, row schema, and row counts.', - risk: 'low', - requiredScopes: ['database.read'], - inputSchema: { - type: 'object', - properties: { - databaseId: { type: 'string', description: 'Database node id.' }, - includeSample: { - type: 'boolean', - description: 'Include a small descriptor-backed row sample.' - } - }, - required: ['databaseId'] - } - }, - { - name: 'xnet_database_query', - title: 'Query database rows', - description: - 'Read a bounded page of database rows using NodeQueryDescriptor-compatible options.', - risk: 'low', - requiredScopes: ['database.read', 'database.query'], - inputSchema: { - type: 'object', - properties: { - databaseId: { type: 'string', description: 'Database node id.' }, - schemaId: { type: 'string', description: 'Optional row schema IRI.' }, - descriptor: { - type: 'object', - description: 'Optional NodeQueryDescriptor-compatible query shape.' - }, - where: { - type: 'object', - description: 'Optional exact property filters for row nodes.' - }, - search: { - type: 'object', - description: 'Optional NodeQueryDescriptor search filter.' - }, - orderBy: { - type: 'object', - description: 'Optional NodeQueryDescriptor order map.' - }, - materializedView: { - type: 'object', - description: 'Optional materialized view query options.' - }, - count: { type: 'string', description: 'Page count mode: exact, estimate, or none.' }, - limit: { type: 'number', description: 'Maximum row count.' }, - offset: { type: 'number', description: 'Row offset.' } - }, - required: ['databaseId'] - } - }, - { - name: 'xnet_database_sample', - title: 'Sample database rows', - description: 'Return a small deterministic sample for schema and content inspection.', - risk: 'low', - requiredScopes: ['database.read', 'database.query'], - inputSchema: { - type: 'object', - properties: { - databaseId: { type: 'string', description: 'Database node id.' }, - schemaId: { type: 'string', description: 'Optional row schema IRI.' }, - sampleSize: { type: 'number', description: 'Sample row count.' }, - descriptor: { - type: 'object', - description: 'Optional NodeQueryDescriptor-compatible query shape.' - } - }, - required: ['databaseId'] - } - }, - { - name: 'xnet_database_explain_query', - title: 'Explain database query', - description: - 'Explain descriptor, pagination, materialized view, and storage plan metadata.', - risk: 'low', - requiredScopes: ['database.read', 'database.query', 'storage.diagnostics'], - inputSchema: { - type: 'object', - properties: { - databaseId: { type: 'string', description: 'Database node id.' }, - schemaId: { type: 'string', description: 'Optional row schema IRI.' }, - descriptor: { - type: 'object', - description: 'Optional NodeQueryDescriptor-compatible query shape.' - }, - limit: { type: 'number', description: 'Maximum row count for the dry-run query.' }, - offset: { type: 'number', description: 'Row offset.' } - }, - required: ['databaseId'] - } - }, - { - name: 'xnet_plan_database_mutation', - title: 'Plan database mutation', - description: 'Create a database mutation plan for later review without applying it.', - risk: 'medium', - requiredScopes: ['database.read', 'database.propose'], - inputSchema: { - type: 'object', - properties: { - databaseId: { type: 'string', description: 'Database node id.' }, - baseRevision: { type: 'string', description: 'Revision the mutation was based on.' }, - operations: { type: 'array', description: 'Database operations to validate.' }, - intent: { type: 'string', description: 'User or agent intent for the mutation.' }, - actor: { type: 'string', description: 'Agent or user creating the plan.' } - }, - required: ['databaseId', 'operations'] - } - }, - { - name: 'xnet_apply_database_mutation', - title: 'Apply database mutation plan', - description: - 'Apply a validated database row/schema mutation plan with transactional row rollback and audit logging.', - risk: 'high', - requiredScopes: ['database.read', 'database.write.rows', 'database.write.schema'], - inputSchema: { - type: 'object', - properties: { - plan: { type: 'object', description: 'Validated database mutation plan.' }, - confirmApply: { - type: 'boolean', - description: 'Must be true to apply the database mutation plan.' - }, - allowStale: { - type: 'boolean', - description: - 'Allow applying when the plan base revision differs from the live database node.' - } - }, - required: ['plan', 'confirmApply'] - } - }, - { - name: 'xnet_canvas_list', - title: 'List canvases', - description: 'List canvas nodes visible to the AI surface.', - risk: 'low', - requiredScopes: ['canvas.read'], - inputSchema: { - type: 'object', - properties: { - limit: { type: 'number', description: 'Maximum canvas count.' }, - offset: { type: 'number', description: 'Canvas offset.' } - } - } - }, - { - name: 'xnet_canvas_read_viewport', - title: 'Read canvas viewport', - description: 'Read canvas objects and edges intersecting a viewport.', - risk: 'low', - requiredScopes: ['canvas.read'], - inputSchema: { - type: 'object', - properties: { - canvasId: { type: 'string', description: 'Canvas node id.' }, - x: { type: 'number', description: 'Viewport x.' }, - y: { type: 'number', description: 'Viewport y.' }, - w: { type: 'number', description: 'Viewport width.' }, - h: { type: 'number', description: 'Viewport height.' }, - includeSourcePreviews: { - type: 'boolean', - description: 'Include previews for source-backed objects.' - }, - tileSize: { type: 'number', description: 'Optional tile size for tile scoping.' }, - tileIds: { - type: 'array', - description: 'Optional tile ids such as 0/1/-2 to constrain the read.', - items: { type: 'string' } - } - }, - required: ['canvasId'] - } - }, - { - name: 'xnet_canvas_read_selection', - title: 'Read canvas selection', - description: 'Read selected canvas objects, connected edges, and optional source previews.', - risk: 'low', - requiredScopes: ['canvas.read'], - inputSchema: { - type: 'object', - properties: { - canvasId: { type: 'string', description: 'Canvas node id.' }, - objectIds: { - type: 'array', - description: 'Selected object ids.', - items: { type: 'string' } - }, - includeSourcePreviews: { - type: 'boolean', - description: 'Include previews for source-backed objects.' - } - }, - required: ['canvasId', 'objectIds'] - } - }, - { - name: 'xnet_canvas_search', - title: 'Search canvas', - description: 'Search canvas object text, labels, ids, and source metadata.', - risk: 'low', - requiredScopes: ['canvas.read'], - inputSchema: { - type: 'object', - properties: { - canvasId: { type: 'string', description: 'Canvas node id.' }, - query: { type: 'string', description: 'Search text.' }, - limit: { type: 'number', description: 'Maximum result count.' } - }, - required: ['canvasId', 'query'] - } - }, - { - name: 'xnet_canvas_export_json_canvas', - title: 'Export canvas as JSON Canvas', - description: 'Export a canvas or viewport as JSON Canvas with xNet source metadata.', - risk: 'low', - requiredScopes: ['canvas.read'], - inputSchema: { - type: 'object', - properties: { - canvasId: { type: 'string', description: 'Canvas node id.' }, - includeXNetMetadata: { - type: 'boolean', - description: 'Include xNet source metadata. Defaults to true.' - }, - x: { type: 'number', description: 'Optional viewport x.' }, - y: { type: 'number', description: 'Optional viewport y.' }, - w: { type: 'number', description: 'Optional viewport width.' }, - h: { type: 'number', description: 'Optional viewport height.' } - }, - required: ['canvasId'] - } - }, - { - name: 'xnet_canvas_plan_json_canvas_import', - title: 'Plan JSON Canvas import', - description: 'Convert a JSON Canvas document into a plan-only canvas mutation.', - risk: 'medium', - requiredScopes: ['canvas.read', 'canvas.propose'], - inputSchema: { - type: 'object', - properties: { - canvasId: { type: 'string', description: 'Canvas node id.' }, - document: { type: 'object', description: 'JSON Canvas document.' }, - baseRevision: { type: 'string', description: 'Revision the import was based on.' }, - actor: { type: 'string', description: 'Agent or user creating the plan.' }, - intent: { type: 'string', description: 'User or agent intent for the import.' } - }, - required: ['canvasId', 'document'] - } - }, - { - name: 'xnet_plan_canvas_mutation', - title: 'Plan canvas mutation', - description: 'Create a canvas mutation plan for later review without applying it.', - risk: 'medium', - requiredScopes: ['canvas.read', 'canvas.propose'], - inputSchema: { - type: 'object', - properties: { - canvasId: { type: 'string', description: 'Canvas node id.' }, - baseRevision: { type: 'string', description: 'Revision the mutation was based on.' }, - operations: { type: 'array', description: 'Canvas operations to validate.' }, - intent: { type: 'string', description: 'User or agent intent for the mutation.' }, - actor: { type: 'string', description: 'Agent or user creating the plan.' } - }, - required: ['canvasId', 'operations'] - } - }, - { - name: 'xnet_validate_mutation_plan', - title: 'Validate mutation plan', - description: 'Validate a serialized mutation plan and return errors or warnings.', - risk: 'medium', - requiredScopes: ['workspace.read'], - inputSchema: { - type: 'object', - properties: { - plan: { type: 'object', description: 'Mutation plan object to validate.' } - }, - required: ['plan'] - } - } - ] + return BUILT_IN_TOOL_ENTRIES.map((entry) => entry.definition) } + /** + * Dispatch a tool call: the built-in registry first (a built-in `xnet_*` + * name always wins), then contributed tools (plugin/connector `agentTools`, + * exploration 0196). + */ async callTool(name: string, args: Record = {}): Promise { - switch (name) { - case 'xnet_search': - return await this.search({ - query: readRequiredString(args, 'query'), - schemaId: readOptionalString(args, 'schemaId') ?? readOptionalString(args, 'schema'), - limit: readOptionalNumber(args, 'limit'), - offset: readOptionalNumber(args, 'offset') - }) - - case 'xnet_graph_expand': - return await this.expandGraph({ - nodeId: readRequiredString(args, 'nodeId'), - hops: readOptionalNumber(args, 'hops'), - limit: readOptionalNumber(args, 'limit') - }) - - case 'xnet_create_context_pack': - return await this.createContextPack({ - query: readOptionalString(args, 'query'), - seeds: readContextSeeds(args.seeds), - limit: readOptionalNumber(args, 'limit') - }) - - case 'xnet_create_external_context_resource': - return this.createExternalContextResource({ - url: readRequiredString(args, 'url'), - text: readRequiredString(args, 'text'), - mimeType: readOptionalString(args, 'mimeType') - }) - - case 'xnet_read_page_markdown': { - const content = await this.readPageMarkdown( - readRequiredString(args, 'pageId'), - readOptionalBoolean(args, 'includeFrontmatter') ?? true - ) - return { markdown: content.text, mimeType: content.mimeType, uri: content.uri } - } + const builtIn = BUILT_IN_TOOLS_BY_NAME.get(name) + if (builtIn) return await builtIn.execute(this.host, args) - case 'xnet_validate_page_markdown': { - const pageId = readOptionalString(args, 'pageId') - const node = pageId ? await this.getNodeOrThrow(pageId) : null - return validateXNetPageMarkdown(readRequiredString(args, 'markdown'), { - pageId, - schemaId: node?.schemaId, - baseRevision: readOptionalString(args, 'baseRevision') - }) - } - - case 'xnet_plan_page_patch': - return await this.planPagePatch(args) - - case 'xnet_apply_page_markdown': - return await this.applyPageMarkdown(args) - - case 'xnet_get_audit_log': - return this.getAuditLog({ - planId: readOptionalString(args, 'planId'), - limit: readOptionalNumber(args, 'limit') - }) - - case 'xnet_rollback_page_markdown': - return await this.rollbackPageMarkdown(args) - - case 'xnet_database_describe': - return await this.describeDatabase(readRequiredString(args, 'databaseId'), { - includeSample: readOptionalBoolean(args, 'includeSample') ?? false - }) - - case 'xnet_database_query': - return await this.queryDatabase({ - databaseId: readRequiredString(args, 'databaseId'), - schemaId: readOptionalString(args, 'schemaId'), - descriptor: readOptionalRecord(args, 'descriptor'), - where: readOptionalRecord(args, 'where'), - search: args.search, - orderBy: readOptionalRecord(args, 'orderBy'), - materializedView: args.materializedView, - count: readOptionalString(args, 'count'), - limit: readOptionalNumber(args, 'limit'), - offset: readOptionalNumber(args, 'offset') - }) - - case 'xnet_database_sample': - return await this.sampleDatabase({ - databaseId: readRequiredString(args, 'databaseId'), - schemaId: readOptionalString(args, 'schemaId'), - descriptor: readOptionalRecord(args, 'descriptor'), - sampleSize: readOptionalNumber(args, 'sampleSize') - }) - - case 'xnet_database_explain_query': - return await this.explainDatabaseQuery({ - databaseId: readRequiredString(args, 'databaseId'), - schemaId: readOptionalString(args, 'schemaId'), - descriptor: readOptionalRecord(args, 'descriptor'), - limit: readOptionalNumber(args, 'limit'), - offset: readOptionalNumber(args, 'offset') - }) - - case 'xnet_plan_database_mutation': - return await this.planDatabaseMutation(args) - - case 'xnet_apply_database_mutation': - return await this.applyDatabaseMutation(args) - - case 'xnet_canvas_list': - return await this.listCanvases({ - limit: readOptionalNumber(args, 'limit'), - offset: readOptionalNumber(args, 'offset') - }) - - case 'xnet_canvas_read_viewport': - return await this.readCanvasViewport({ - canvasId: readRequiredString(args, 'canvasId'), - x: readOptionalNumber(args, 'x'), - y: readOptionalNumber(args, 'y'), - w: readOptionalNumber(args, 'w'), - h: readOptionalNumber(args, 'h'), - tileSize: readOptionalNumber(args, 'tileSize'), - tileIds: readStringArray(args.tileIds), - includeSourcePreviews: readOptionalBoolean(args, 'includeSourcePreviews') ?? false - }) - - case 'xnet_canvas_read_selection': - return await this.readCanvasSelection({ - canvasId: readRequiredString(args, 'canvasId'), - objectIds: readRequiredStringArray(args.objectIds, 'objectIds'), - includeSourcePreviews: readOptionalBoolean(args, 'includeSourcePreviews') ?? false - }) - - case 'xnet_canvas_search': - return await this.searchCanvas({ - canvasId: readRequiredString(args, 'canvasId'), - query: readRequiredString(args, 'query'), - limit: readOptionalNumber(args, 'limit') - }) - - case 'xnet_canvas_export_json_canvas': - return await this.exportCanvasJsonCanvas({ - canvasId: readRequiredString(args, 'canvasId'), - includeXNetMetadata: readOptionalBoolean(args, 'includeXNetMetadata') ?? true, - x: readOptionalNumber(args, 'x'), - y: readOptionalNumber(args, 'y'), - w: readOptionalNumber(args, 'w'), - h: readOptionalNumber(args, 'h') - }) - - case 'xnet_canvas_plan_json_canvas_import': - return await this.planCanvasJsonCanvasImport(args) - - case 'xnet_plan_canvas_mutation': - return await this.planCanvasMutation(args) - - case 'xnet_validate_mutation_plan': { - const validation = validateAiMutationPlan(args.plan) - return { validation } - } - - default: { - // Contributed tools (plugin/connector `agentTools`, exploration 0196). - const extra = this.extraTools.get(name) - if (extra) return await extra.invoke(args) - throw new Error(`Unknown AI surface tool: ${name}`) - } - } + const extra = this.extraTools.get(name) + if (extra) return await extra.invoke(args) + throw new Error(`Unknown AI surface tool: ${name}`) } async readResource(uri: string): Promise { - const parsed = parseXNetUri(uri) - - if (uri === 'xnet://nodes') { - const nodes = await this.config.store.list({ limit: this.limits.maxListLimit }) - return this.jsonResource(uri, { nodes, count: nodes.length, limit: this.limits.maxListLimit }) - } - - if (uri === 'xnet://schemas') { - return this.jsonResource(uri, { schemas: await this.getSchemaSummaries(true) }) - } - - if (parsed.host === 'workspace' && parsed.parts[0] === 'summary') { - return this.jsonResource(uri, await this.getWorkspaceSummary()) - } - - if (parsed.host === 'workspace' && parsed.parts[0] === 'recent') { - return this.jsonResource(uri, await this.getRecentNodes()) - } - - if (parsed.host === 'workspace' && parsed.parts[0] === 'search') { - return this.jsonResource( - uri, - await this.search({ - query: parsed.searchParams.get('q') ?? '', - schemaId: parsed.searchParams.get('schema') ?? undefined, - limit: readUrlNumber(parsed.searchParams, 'limit'), - offset: readUrlNumber(parsed.searchParams, 'offset') - }) - ) - } - - if (parsed.host === 'node' && parsed.parts[0]) { - return this.jsonResource(uri, await this.getNodeProjection(parsed.parts[0])) - } - - if (parsed.host === 'page' && parsed.parts[0]) { - const pageId = parsed.parts[0].endsWith('.md') - ? parsed.parts[0].slice(0, -'.md'.length) - : parsed.parts[0] - if (parsed.parts.length === 1 || parsed.parts[0].endsWith('.md')) { - return await this.readPageMarkdown(pageId, true, uri) - } - if (parsed.parts[1] === 'outline') { - return this.jsonResource(uri, await this.readPageOutline(pageId)) - } - if (parsed.parts[1] === 'context-pack') { - return this.jsonResource( - uri, - await this.createContextPack({ seeds: [{ kind: 'page', id: pageId }] }) - ) - } - } - - if (parsed.host === 'database' && parsed.parts[0]) { - const databaseId = parsed.parts[0] - if (parsed.parts[1] === 'schema') { - return this.jsonResource(uri, await this.describeDatabase(databaseId)) - } - if (parsed.parts[1] === 'views') { - return this.jsonResource(uri, await this.readDatabaseViews(databaseId)) - } - if (parsed.parts[1] === 'sample') { - return this.jsonResource( - uri, - await this.sampleDatabase({ - databaseId, - sampleSize: readUrlNumber(parsed.searchParams, 'limit') - }) - ) - } - if (parsed.parts[1] === 'query') { - return this.jsonResource( - uri, - await this.queryDatabase({ - databaseId, - schemaId: parsed.searchParams.get('schema') ?? undefined, - search: parsed.searchParams.get('q') ?? undefined, - materializedView: parsed.searchParams.get('view') - ? { viewId: parsed.searchParams.get('view') ?? '' } - : undefined, - limit: readUrlNumber(parsed.searchParams, 'limit'), - offset: readUrlNumber(parsed.searchParams, 'offset') - }) - ) - } - } - - if (parsed.host === 'canvas' && parsed.parts[0]) { - const canvasId = parsed.parts[0] - if (parsed.parts[1] === 'viewport') { - return this.jsonResource( - uri, - await this.readCanvasViewport({ - canvasId, - x: readUrlNumber(parsed.searchParams, 'x'), - y: readUrlNumber(parsed.searchParams, 'y'), - w: readUrlNumber(parsed.searchParams, 'w'), - h: readUrlNumber(parsed.searchParams, 'h'), - tileSize: readUrlNumber(parsed.searchParams, 'tileSize'), - tileIds: readCsvStringArray(parsed.searchParams.get('tileIds')), - includeSourcePreviews: parsed.searchParams.get('includeSourcePreviews') === 'true' - }) - ) - } - if (parsed.parts[1] === 'objects') { - return this.jsonResource(uri, await this.readCanvasObjects(canvasId)) - } - if (parsed.parts[1] === 'selection') { - return this.jsonResource( - uri, - await this.readCanvasSelection({ - canvasId, - objectIds: readCsvStringArray(parsed.searchParams.get('ids')), - includeSourcePreviews: parsed.searchParams.get('includeSourcePreviews') !== 'false' - }) - ) - } - if (parsed.parts[1] === 'json-canvas') { - return this.jsonResource( - uri, - await this.exportCanvasJsonCanvas({ - canvasId, - includeXNetMetadata: parsed.searchParams.get('includeXNetMetadata') !== 'false' - }) - ) - } - if (parsed.parts[1] === 'object' && parsed.parts[2]) { - return this.jsonResource(uri, await this.readCanvasObject(canvasId, parsed.parts[2])) - } - } - - throw new Error(`Resource not found: ${uri}`) + return await BUILT_IN_RESOURCE_ROUTES.resolve(this.host, uri) } toJsonText(value: unknown, format: 'concise' | 'detailed' = 'concise'): string { @@ -2815,30 +2111,6 @@ function createResource( } } -function parseXNetUri(uri: string): { - host: string - parts: string[] - searchParams: URLSearchParams -} { - let parsed: URL - try { - parsed = new URL(uri) - } catch { - throw new Error(`Invalid xNet resource URI: ${uri}`) - } - if (parsed.protocol !== 'xnet:') { - throw new Error(`Invalid xNet resource URI: ${uri}`) - } - return { - host: parsed.hostname, - parts: parsed.pathname - .split('/') - .filter(Boolean) - .map((part) => decodeURIComponent(part)), - searchParams: parsed.searchParams - } -} - function renderPageMarkdown( node: NodeData, includeFrontmatter: boolean, @@ -4324,18 +3596,6 @@ function readOperations(value: unknown): AiOperation[] { }) } -function readContextSeeds(value: unknown): AiContextSeed[] { - if (!Array.isArray(value)) return [] - return value - .map((seed) => { - if (!isRecord(seed)) return null - const kind = typeof seed.kind === 'string' ? (seed.kind as AiTargetKind) : null - const id = typeof seed.id === 'string' ? seed.id : null - return kind && id ? { kind, id } : null - }) - .filter((seed): seed is AiContextSeed => seed !== null) -} - function uriForSeed(seed: AiContextSeed): string | null { switch (seed.kind) { case 'node': @@ -4401,96 +3661,3 @@ function stringifyTruncatedJson(text: string, maxCharacters: number): string { function quoteYaml(value: string): string { return JSON.stringify(value) } - -function readRequiredString(record: Record, key: string): string { - const value = record[key] - if (typeof value !== 'string' || !value.trim()) { - throw new Error(`${key} must be a non-empty string`) - } - return value -} - -function readRequiredRecord(record: Record, key: string): Record { - const value = record[key] - if (!isRecord(value)) { - throw new Error(`${key} must be an object`) - } - return value -} - -function readRequiredStringArray(value: unknown, key: string): string[] { - const result = readStringArray(value) - if (result.length === 0) { - throw new Error(`${key} must contain at least one string`) - } - return result -} - -function readStringArray(value: unknown): string[] { - if (!Array.isArray(value)) return [] - return value.filter((item): item is string => typeof item === 'string' && item.trim() !== '') -} - -function readCsvStringArray(value: string | null): string[] { - if (!value) return [] - return value - .split(',') - .map((item) => item.trim()) - .filter(Boolean) -} - -function readOptionalString(record: Record, key: string): string | undefined { - const value = record[key] - return typeof value === 'string' && value.trim() ? value : undefined -} - -function readOptionalRecord( - record: Record, - key: string -): Record | undefined { - return readRecord(record, key) -} - -function readOptionalNumber(record: Record, key: string): number | undefined { - const value = record[key] - return typeof value === 'number' && Number.isFinite(value) ? value : undefined -} - -function readOptionalBoolean(record: Record, key: string): boolean | undefined { - const value = record[key] - return typeof value === 'boolean' ? value : undefined -} - -function readUrlNumber(params: URLSearchParams, key: string): number | undefined { - const value = params.get(key) - if (value === null) return undefined - const parsed = Number(value) - return Number.isFinite(parsed) ? parsed : undefined -} - -function readRecordString(record: Record, key: string): string | undefined { - const value = record[key] - return typeof value === 'string' && value.trim() ? value : undefined -} - -function readRecord( - record: Record, - key: string -): Record | undefined { - const value = record[key] - return isRecord(value) ? value : undefined -} - -function readRecordNumber(record: Record, key: string): number | undefined { - const value = record[key] - return typeof value === 'number' && Number.isFinite(value) ? value : undefined -} - -function readRecordBoolean(record: Record, key: string): boolean | undefined { - const value = record[key] - return typeof value === 'boolean' ? value : undefined -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value) -} diff --git a/packages/plugins/src/ai-surface/tools/audit.ts b/packages/plugins/src/ai-surface/tools/audit.ts new file mode 100644 index 000000000..c3c836ca3 --- /dev/null +++ b/packages/plugins/src/ai-surface/tools/audit.ts @@ -0,0 +1,51 @@ +/** + * Audit and plan-validation tools: the AI mutation audit log and standalone + * mutation-plan validation. + */ + +import type { AiToolEntry } from './entry' +import { readOptionalNumber, readOptionalString } from '../args' +import { validateAiMutationPlan } from '../validation' + +export const getAuditLogTool: AiToolEntry = { + definition: { + name: 'xnet_get_audit_log', + title: 'Read AI audit log', + description: 'Read recent AI mutation audit events with optional plan filtering.', + risk: 'low', + requiredScopes: ['workspace.read'], + inputSchema: { + type: 'object', + properties: { + planId: { type: 'string', description: 'Optional mutation plan id filter.' }, + limit: { type: 'number', description: 'Maximum audit events to return.' } + } + } + }, + execute: (host, args) => + host.getAuditLog({ + planId: readOptionalString(args, 'planId'), + limit: readOptionalNumber(args, 'limit') + }) +} + +export const validateMutationPlanTool: AiToolEntry = { + definition: { + name: 'xnet_validate_mutation_plan', + title: 'Validate mutation plan', + description: 'Validate a serialized mutation plan and return errors or warnings.', + risk: 'medium', + requiredScopes: ['workspace.read'], + inputSchema: { + type: 'object', + properties: { + plan: { type: 'object', description: 'Mutation plan object to validate.' } + }, + required: ['plan'] + } + }, + execute: (_host, args) => { + const validation = validateAiMutationPlan(args.plan) + return { validation } + } +} diff --git a/packages/plugins/src/ai-surface/tools/canvas.ts b/packages/plugins/src/ai-surface/tools/canvas.ts new file mode 100644 index 000000000..d5a3131e1 --- /dev/null +++ b/packages/plugins/src/ai-surface/tools/canvas.ts @@ -0,0 +1,222 @@ +/** + * Canvas tools: bounded scene reads (list/viewport/selection/search), JSON + * Canvas import/export, and plan-only canvas mutations. + */ + +import type { AiToolEntry } from './entry' +import { + readOptionalBoolean, + readOptionalNumber, + readRequiredString, + readRequiredStringArray, + readStringArray +} from '../args' + +export const canvasListTool: AiToolEntry = { + definition: { + name: 'xnet_canvas_list', + title: 'List canvases', + description: 'List canvas nodes visible to the AI surface.', + risk: 'low', + requiredScopes: ['canvas.read'], + inputSchema: { + type: 'object', + properties: { + limit: { type: 'number', description: 'Maximum canvas count.' }, + offset: { type: 'number', description: 'Canvas offset.' } + } + } + }, + execute: async (host, args) => + await host.listCanvases({ + limit: readOptionalNumber(args, 'limit'), + offset: readOptionalNumber(args, 'offset') + }) +} + +export const canvasReadViewportTool: AiToolEntry = { + definition: { + name: 'xnet_canvas_read_viewport', + title: 'Read canvas viewport', + description: 'Read canvas objects and edges intersecting a viewport.', + risk: 'low', + requiredScopes: ['canvas.read'], + inputSchema: { + type: 'object', + properties: { + canvasId: { type: 'string', description: 'Canvas node id.' }, + x: { type: 'number', description: 'Viewport x.' }, + y: { type: 'number', description: 'Viewport y.' }, + w: { type: 'number', description: 'Viewport width.' }, + h: { type: 'number', description: 'Viewport height.' }, + includeSourcePreviews: { + type: 'boolean', + description: 'Include previews for source-backed objects.' + }, + tileSize: { type: 'number', description: 'Optional tile size for tile scoping.' }, + tileIds: { + type: 'array', + description: 'Optional tile ids such as 0/1/-2 to constrain the read.', + items: { type: 'string' } + } + }, + required: ['canvasId'] + } + }, + execute: async (host, args) => + await host.readCanvasViewport({ + canvasId: readRequiredString(args, 'canvasId'), + x: readOptionalNumber(args, 'x'), + y: readOptionalNumber(args, 'y'), + w: readOptionalNumber(args, 'w'), + h: readOptionalNumber(args, 'h'), + tileSize: readOptionalNumber(args, 'tileSize'), + tileIds: readStringArray(args.tileIds), + includeSourcePreviews: readOptionalBoolean(args, 'includeSourcePreviews') ?? false + }) +} + +export const canvasReadSelectionTool: AiToolEntry = { + definition: { + name: 'xnet_canvas_read_selection', + title: 'Read canvas selection', + description: 'Read selected canvas objects, connected edges, and optional source previews.', + risk: 'low', + requiredScopes: ['canvas.read'], + inputSchema: { + type: 'object', + properties: { + canvasId: { type: 'string', description: 'Canvas node id.' }, + objectIds: { + type: 'array', + description: 'Selected object ids.', + items: { type: 'string' } + }, + includeSourcePreviews: { + type: 'boolean', + description: 'Include previews for source-backed objects.' + } + }, + required: ['canvasId', 'objectIds'] + } + }, + execute: async (host, args) => + await host.readCanvasSelection({ + canvasId: readRequiredString(args, 'canvasId'), + objectIds: readRequiredStringArray(args.objectIds, 'objectIds'), + includeSourcePreviews: readOptionalBoolean(args, 'includeSourcePreviews') ?? false + }) +} + +export const canvasSearchTool: AiToolEntry = { + definition: { + name: 'xnet_canvas_search', + title: 'Search canvas', + description: 'Search canvas object text, labels, ids, and source metadata.', + risk: 'low', + requiredScopes: ['canvas.read'], + inputSchema: { + type: 'object', + properties: { + canvasId: { type: 'string', description: 'Canvas node id.' }, + query: { type: 'string', description: 'Search text.' }, + limit: { type: 'number', description: 'Maximum result count.' } + }, + required: ['canvasId', 'query'] + } + }, + execute: async (host, args) => + await host.searchCanvas({ + canvasId: readRequiredString(args, 'canvasId'), + query: readRequiredString(args, 'query'), + limit: readOptionalNumber(args, 'limit') + }) +} + +export const canvasExportJsonCanvasTool: AiToolEntry = { + definition: { + name: 'xnet_canvas_export_json_canvas', + title: 'Export canvas as JSON Canvas', + description: 'Export a canvas or viewport as JSON Canvas with xNet source metadata.', + risk: 'low', + requiredScopes: ['canvas.read'], + inputSchema: { + type: 'object', + properties: { + canvasId: { type: 'string', description: 'Canvas node id.' }, + includeXNetMetadata: { + type: 'boolean', + description: 'Include xNet source metadata. Defaults to true.' + }, + x: { type: 'number', description: 'Optional viewport x.' }, + y: { type: 'number', description: 'Optional viewport y.' }, + w: { type: 'number', description: 'Optional viewport width.' }, + h: { type: 'number', description: 'Optional viewport height.' } + }, + required: ['canvasId'] + } + }, + execute: async (host, args) => + await host.exportCanvasJsonCanvas({ + canvasId: readRequiredString(args, 'canvasId'), + includeXNetMetadata: readOptionalBoolean(args, 'includeXNetMetadata') ?? true, + x: readOptionalNumber(args, 'x'), + y: readOptionalNumber(args, 'y'), + w: readOptionalNumber(args, 'w'), + h: readOptionalNumber(args, 'h') + }) +} + +export const canvasPlanJsonCanvasImportTool: AiToolEntry = { + definition: { + name: 'xnet_canvas_plan_json_canvas_import', + title: 'Plan JSON Canvas import', + description: 'Convert a JSON Canvas document into a plan-only canvas mutation.', + risk: 'medium', + requiredScopes: ['canvas.read', 'canvas.propose'], + inputSchema: { + type: 'object', + properties: { + canvasId: { type: 'string', description: 'Canvas node id.' }, + document: { type: 'object', description: 'JSON Canvas document.' }, + baseRevision: { type: 'string', description: 'Revision the import was based on.' }, + actor: { type: 'string', description: 'Agent or user creating the plan.' }, + intent: { type: 'string', description: 'User or agent intent for the import.' } + }, + required: ['canvasId', 'document'] + } + }, + execute: async (host, args) => await host.planCanvasJsonCanvasImport(args) +} + +export const planCanvasMutationTool: AiToolEntry = { + definition: { + name: 'xnet_plan_canvas_mutation', + title: 'Plan canvas mutation', + description: 'Create a canvas mutation plan for later review without applying it.', + risk: 'medium', + requiredScopes: ['canvas.read', 'canvas.propose'], + inputSchema: { + type: 'object', + properties: { + canvasId: { type: 'string', description: 'Canvas node id.' }, + baseRevision: { type: 'string', description: 'Revision the mutation was based on.' }, + operations: { type: 'array', description: 'Canvas operations to validate.' }, + intent: { type: 'string', description: 'User or agent intent for the mutation.' }, + actor: { type: 'string', description: 'Agent or user creating the plan.' } + }, + required: ['canvasId', 'operations'] + } + }, + execute: async (host, args) => await host.planCanvasMutation(args) +} + +export const canvasToolEntries: readonly AiToolEntry[] = [ + canvasListTool, + canvasReadViewportTool, + canvasReadSelectionTool, + canvasSearchTool, + canvasExportJsonCanvasTool, + canvasPlanJsonCanvasImportTool, + planCanvasMutationTool +] diff --git a/packages/plugins/src/ai-surface/tools/database.ts b/packages/plugins/src/ai-surface/tools/database.ts new file mode 100644 index 000000000..fb1df688c --- /dev/null +++ b/packages/plugins/src/ai-surface/tools/database.ts @@ -0,0 +1,214 @@ +/** + * Database tools: descriptor-backed reads (describe/query/sample/explain) and + * the plan → confirmed-apply mutation pair with transactional row rollback. + */ + +import type { AiToolEntry } from './entry' +import { + readOptionalBoolean, + readOptionalNumber, + readOptionalRecord, + readOptionalString, + readRequiredString +} from '../args' + +export const databaseDescribeTool: AiToolEntry = { + definition: { + name: 'xnet_database_describe', + title: 'Describe database', + description: 'Describe database schema, columns, views, row schema, and row counts.', + risk: 'low', + requiredScopes: ['database.read'], + inputSchema: { + type: 'object', + properties: { + databaseId: { type: 'string', description: 'Database node id.' }, + includeSample: { + type: 'boolean', + description: 'Include a small descriptor-backed row sample.' + } + }, + required: ['databaseId'] + } + }, + execute: async (host, args) => + await host.describeDatabase(readRequiredString(args, 'databaseId'), { + includeSample: readOptionalBoolean(args, 'includeSample') ?? false + }) +} + +export const databaseQueryTool: AiToolEntry = { + definition: { + name: 'xnet_database_query', + title: 'Query database rows', + description: + 'Read a bounded page of database rows using NodeQueryDescriptor-compatible options.', + risk: 'low', + requiredScopes: ['database.read', 'database.query'], + inputSchema: { + type: 'object', + properties: { + databaseId: { type: 'string', description: 'Database node id.' }, + schemaId: { type: 'string', description: 'Optional row schema IRI.' }, + descriptor: { + type: 'object', + description: 'Optional NodeQueryDescriptor-compatible query shape.' + }, + where: { + type: 'object', + description: 'Optional exact property filters for row nodes.' + }, + search: { + type: 'object', + description: 'Optional NodeQueryDescriptor search filter.' + }, + orderBy: { + type: 'object', + description: 'Optional NodeQueryDescriptor order map.' + }, + materializedView: { + type: 'object', + description: 'Optional materialized view query options.' + }, + count: { type: 'string', description: 'Page count mode: exact, estimate, or none.' }, + limit: { type: 'number', description: 'Maximum row count.' }, + offset: { type: 'number', description: 'Row offset.' } + }, + required: ['databaseId'] + } + }, + execute: async (host, args) => + await host.queryDatabase({ + databaseId: readRequiredString(args, 'databaseId'), + schemaId: readOptionalString(args, 'schemaId'), + descriptor: readOptionalRecord(args, 'descriptor'), + where: readOptionalRecord(args, 'where'), + search: args.search, + orderBy: readOptionalRecord(args, 'orderBy'), + materializedView: args.materializedView, + count: readOptionalString(args, 'count'), + limit: readOptionalNumber(args, 'limit'), + offset: readOptionalNumber(args, 'offset') + }) +} + +export const databaseSampleTool: AiToolEntry = { + definition: { + name: 'xnet_database_sample', + title: 'Sample database rows', + description: 'Return a small deterministic sample for schema and content inspection.', + risk: 'low', + requiredScopes: ['database.read', 'database.query'], + inputSchema: { + type: 'object', + properties: { + databaseId: { type: 'string', description: 'Database node id.' }, + schemaId: { type: 'string', description: 'Optional row schema IRI.' }, + sampleSize: { type: 'number', description: 'Sample row count.' }, + descriptor: { + type: 'object', + description: 'Optional NodeQueryDescriptor-compatible query shape.' + } + }, + required: ['databaseId'] + } + }, + execute: async (host, args) => + await host.sampleDatabase({ + databaseId: readRequiredString(args, 'databaseId'), + schemaId: readOptionalString(args, 'schemaId'), + descriptor: readOptionalRecord(args, 'descriptor'), + sampleSize: readOptionalNumber(args, 'sampleSize') + }) +} + +export const databaseExplainQueryTool: AiToolEntry = { + definition: { + name: 'xnet_database_explain_query', + title: 'Explain database query', + description: 'Explain descriptor, pagination, materialized view, and storage plan metadata.', + risk: 'low', + requiredScopes: ['database.read', 'database.query', 'storage.diagnostics'], + inputSchema: { + type: 'object', + properties: { + databaseId: { type: 'string', description: 'Database node id.' }, + schemaId: { type: 'string', description: 'Optional row schema IRI.' }, + descriptor: { + type: 'object', + description: 'Optional NodeQueryDescriptor-compatible query shape.' + }, + limit: { type: 'number', description: 'Maximum row count for the dry-run query.' }, + offset: { type: 'number', description: 'Row offset.' } + }, + required: ['databaseId'] + } + }, + execute: async (host, args) => + await host.explainDatabaseQuery({ + databaseId: readRequiredString(args, 'databaseId'), + schemaId: readOptionalString(args, 'schemaId'), + descriptor: readOptionalRecord(args, 'descriptor'), + limit: readOptionalNumber(args, 'limit'), + offset: readOptionalNumber(args, 'offset') + }) +} + +export const planDatabaseMutationTool: AiToolEntry = { + definition: { + name: 'xnet_plan_database_mutation', + title: 'Plan database mutation', + description: 'Create a database mutation plan for later review without applying it.', + risk: 'medium', + requiredScopes: ['database.read', 'database.propose'], + inputSchema: { + type: 'object', + properties: { + databaseId: { type: 'string', description: 'Database node id.' }, + baseRevision: { type: 'string', description: 'Revision the mutation was based on.' }, + operations: { type: 'array', description: 'Database operations to validate.' }, + intent: { type: 'string', description: 'User or agent intent for the mutation.' }, + actor: { type: 'string', description: 'Agent or user creating the plan.' } + }, + required: ['databaseId', 'operations'] + } + }, + execute: async (host, args) => await host.planDatabaseMutation(args) +} + +export const applyDatabaseMutationTool: AiToolEntry = { + definition: { + name: 'xnet_apply_database_mutation', + title: 'Apply database mutation plan', + description: + 'Apply a validated database row/schema mutation plan with transactional row rollback and audit logging.', + risk: 'high', + requiredScopes: ['database.read', 'database.write.rows', 'database.write.schema'], + inputSchema: { + type: 'object', + properties: { + plan: { type: 'object', description: 'Validated database mutation plan.' }, + confirmApply: { + type: 'boolean', + description: 'Must be true to apply the database mutation plan.' + }, + allowStale: { + type: 'boolean', + description: + 'Allow applying when the plan base revision differs from the live database node.' + } + }, + required: ['plan', 'confirmApply'] + } + }, + execute: async (host, args) => await host.applyDatabaseMutation(args) +} + +export const databaseToolEntries: readonly AiToolEntry[] = [ + databaseDescribeTool, + databaseQueryTool, + databaseSampleTool, + databaseExplainQueryTool, + planDatabaseMutationTool, + applyDatabaseMutationTool +] diff --git a/packages/plugins/src/ai-surface/tools/entry.ts b/packages/plugins/src/ai-surface/tools/entry.ts new file mode 100644 index 000000000..a7569b6dd --- /dev/null +++ b/packages/plugins/src/ai-surface/tools/entry.ts @@ -0,0 +1,16 @@ +/** + * Registry entry contract for built-in AI surface tools. + * + * Each tool is one self-contained entry: its MCP-visible definition plus the + * handler that coerces raw agent arguments and delegates to the service via + * the narrow {@link AiSurfaceHost}. Adding a tool means adding one entry to + * one group file — `getTools()` and `callTool()` pick it up from the registry. + */ + +import type { AiSurfaceHost } from '../host' +import type { AiToolDefinition } from '../types' + +export type AiToolEntry = { + definition: AiToolDefinition + execute(host: AiSurfaceHost, args: Record): Promise | unknown +} diff --git a/packages/plugins/src/ai-surface/tools/index.ts b/packages/plugins/src/ai-surface/tools/index.ts new file mode 100644 index 000000000..efd9c2841 --- /dev/null +++ b/packages/plugins/src/ai-surface/tools/index.ts @@ -0,0 +1,45 @@ +/** + * Built-in AI surface tool registry. + * + * `BUILT_IN_TOOL_ENTRIES` is the single registration point: `getTools()` + * derives the definition list from it and `callTool()` dispatches through it, + * so adding a tool means adding one entry to one group file and listing it + * here — no switch statement to extend. + */ + +import type { AiToolEntry } from './entry' +import { getAuditLogTool, validateMutationPlanTool } from './audit' +import { canvasToolEntries } from './canvas' +import { databaseToolEntries } from './database' +import { + applyPageMarkdownTool, + planPagePatchTool, + readPageMarkdownTool, + rollbackPageMarkdownTool, + validatePageMarkdownTool +} from './page-mutation' +import { searchToolEntries } from './search' + +export type { AiToolEntry } from './entry' + +/** + * All built-in tools in wire-visible registration order. The order is part of + * the surface (agents and snapshots key off it) — append new tools to the + * group that fits, keeping existing positions stable. + */ +export const BUILT_IN_TOOL_ENTRIES: readonly AiToolEntry[] = [ + ...searchToolEntries, + readPageMarkdownTool, + validatePageMarkdownTool, + planPagePatchTool, + applyPageMarkdownTool, + getAuditLogTool, + rollbackPageMarkdownTool, + ...databaseToolEntries, + ...canvasToolEntries, + validateMutationPlanTool +] + +export const BUILT_IN_TOOLS_BY_NAME: ReadonlyMap = new Map( + BUILT_IN_TOOL_ENTRIES.map((entry) => [entry.definition.name, entry]) +) diff --git a/packages/plugins/src/ai-surface/tools/page-mutation.ts b/packages/plugins/src/ai-surface/tools/page-mutation.ts new file mode 100644 index 000000000..a12d41ec7 --- /dev/null +++ b/packages/plugins/src/ai-surface/tools/page-mutation.ts @@ -0,0 +1,135 @@ +/** + * Page Markdown tools: read/validate projections, plan-only patches, and the + * confirmed apply/rollback pair (plan → apply → rollback, audit-logged). + */ + +import type { AiToolEntry } from './entry' +import { readOptionalBoolean, readOptionalString, readRequiredString } from '../args' +import { validateXNetPageMarkdown } from '../page-markdown' + +export const readPageMarkdownTool: AiToolEntry = { + definition: { + name: 'xnet_read_page_markdown', + title: 'Read page Markdown', + description: 'Read a page as Markdown with optional xNet frontmatter.', + risk: 'low', + requiredScopes: ['page.read'], + inputSchema: { + type: 'object', + properties: { + pageId: { type: 'string', description: 'Page node id.' }, + includeFrontmatter: { + type: 'boolean', + description: 'Include xNet identity frontmatter. Defaults to true.' + } + }, + required: ['pageId'] + } + }, + execute: async (host, args) => { + const content = await host.readPageMarkdown( + readRequiredString(args, 'pageId'), + readOptionalBoolean(args, 'includeFrontmatter') ?? true + ) + return { markdown: content.text, mimeType: content.mimeType, uri: content.uri } + } +} + +export const validatePageMarkdownTool: AiToolEntry = { + definition: { + name: 'xnet_validate_page_markdown', + title: 'Validate page Markdown', + description: 'Validate xNet page frontmatter and supported xNet Markdown directives.', + risk: 'low', + requiredScopes: ['page.read'], + inputSchema: { + type: 'object', + properties: { + pageId: { type: 'string', description: 'Optional target page node id.' }, + baseRevision: { type: 'string', description: 'Optional expected base revision.' }, + markdown: { type: 'string', description: 'Markdown to validate.' } + }, + required: ['markdown'] + } + }, + execute: async (host, args) => { + const pageId = readOptionalString(args, 'pageId') + const node = pageId ? await host.getNodeOrThrow(pageId) : null + return validateXNetPageMarkdown(readRequiredString(args, 'markdown'), { + pageId, + schemaId: node?.schemaId, + baseRevision: readOptionalString(args, 'baseRevision') + }) + } +} + +export const planPagePatchTool: AiToolEntry = { + definition: { + name: 'xnet_plan_page_patch', + title: 'Plan page Markdown patch', + description: 'Validate an edited Markdown page and return a mutation plan without applying it.', + risk: 'medium', + requiredScopes: ['page.read', 'page.propose'], + inputSchema: { + type: 'object', + properties: { + pageId: { type: 'string', description: 'Page node id.' }, + baseRevision: { type: 'string', description: 'Revision the patch was based on.' }, + markdown: { type: 'string', description: 'Proposed full Markdown replacement.' }, + intent: { type: 'string', description: 'User or agent intent for the patch.' }, + actor: { type: 'string', description: 'Agent or user creating the plan.' } + }, + required: ['pageId', 'markdown'] + } + }, + execute: async (host, args) => await host.planPagePatch(args) +} + +export const applyPageMarkdownTool: AiToolEntry = { + definition: { + name: 'xnet_apply_page_markdown', + title: 'Apply page Markdown plan', + description: + 'Apply a validated page Markdown mutation plan through the configured TipTap/Yjs document adapter, with a node-property fallback.', + risk: 'high', + requiredScopes: ['page.read', 'page.write'], + inputSchema: { + type: 'object', + properties: { + plan: { type: 'object', description: 'Validated page Markdown mutation plan.' }, + confirmApply: { + type: 'boolean', + description: 'Must be true to apply the page Markdown plan.' + }, + allowStale: { + type: 'boolean', + description: 'Allow applying when the plan base revision differs from the live node.' + } + }, + required: ['plan', 'confirmApply'] + } + }, + execute: async (host, args) => await host.applyPageMarkdown(args) +} + +export const rollbackPageMarkdownTool: AiToolEntry = { + definition: { + name: 'xnet_rollback_page_markdown', + title: 'Rollback page Markdown apply', + description: 'Rollback a previously applied page Markdown plan by rollback handle.', + risk: 'high', + requiredScopes: ['page.write'], + inputSchema: { + type: 'object', + properties: { + rollbackHandle: { type: 'string', description: 'Rollback handle from apply result.' }, + confirmRollback: { + type: 'boolean', + description: 'Must be true to perform the rollback.' + } + }, + required: ['rollbackHandle', 'confirmRollback'] + } + }, + execute: async (host, args) => await host.rollbackPageMarkdown(args) +} diff --git a/packages/plugins/src/ai-surface/tools/search.ts b/packages/plugins/src/ai-surface/tools/search.ts new file mode 100644 index 000000000..8922e2fa9 --- /dev/null +++ b/packages/plugins/src/ai-surface/tools/search.ts @@ -0,0 +1,135 @@ +/** + * Workspace search and context tools: keyword search, graph expansion, and + * context-pack assembly (explorations 0196/0211). + */ + +import type { AiToolEntry } from './entry' +import { + readContextSeeds, + readOptionalNumber, + readOptionalString, + readRequiredString +} from '../args' + +export const searchTool: AiToolEntry = { + definition: { + name: 'xnet_search', + title: 'Search xNet workspace', + description: 'Search node titles and searchable properties with pagination and limits.', + risk: 'low', + requiredScopes: ['workspace.search'], + inputSchema: { + type: 'object', + properties: { + query: { type: 'string', description: 'Search text.' }, + schemaId: { type: 'string', description: 'Optional schema IRI filter.' }, + limit: { type: 'number', description: 'Maximum result count.' }, + offset: { type: 'number', description: 'Result offset for pagination.' } + }, + required: ['query'] + } + }, + execute: async (host, args) => + await host.search({ + query: readRequiredString(args, 'query'), + schemaId: readOptionalString(args, 'schemaId') ?? readOptionalString(args, 'schema'), + limit: readOptionalNumber(args, 'limit'), + offset: readOptionalNumber(args, 'offset') + }) +} + +export const graphExpandTool: AiToolEntry = { + definition: { + name: 'xnet_graph_expand', + title: 'Expand a node along its relations', + description: + 'Walk typed relation edges out from a node to its connected neighbors (bounded by hops and a result limit). Use for just-in-time expansion: fetch a specific node’s connections only when you need them, instead of pulling the whole graph into context.', + risk: 'low', + requiredScopes: ['workspace.read'], + inputSchema: { + type: 'object', + properties: { + nodeId: { type: 'string', description: 'The node to expand from.' }, + hops: { + type: 'number', + description: 'How many relation hops to walk (1–2, default 1).' + }, + limit: { type: 'number', description: 'Maximum neighbors to return.' } + }, + required: ['nodeId'] + } + }, + execute: async (host, args) => + await host.expandGraph({ + nodeId: readRequiredString(args, 'nodeId'), + hops: readOptionalNumber(args, 'hops'), + limit: readOptionalNumber(args, 'limit') + }) +} + +export const createContextPackTool: AiToolEntry = { + definition: { + name: 'xnet_create_context_pack', + title: 'Create context pack', + description: 'Create a bounded context pack from seeds and optional search results.', + risk: 'low', + requiredScopes: ['workspace.read', 'workspace.search'], + inputSchema: { + type: 'object', + properties: { + query: { type: 'string', description: 'Optional search query.' }, + seeds: { + type: 'array', + description: 'Seed resources such as pages, databases, canvases, or nodes.', + items: { + type: 'object', + properties: { + kind: { type: 'string', description: 'Seed kind.' }, + id: { type: 'string', description: 'Seed id.' } + } + } + }, + limit: { type: 'number', description: 'Maximum resources to include.' } + } + } + }, + execute: async (host, args) => + await host.createContextPack({ + query: readOptionalString(args, 'query'), + seeds: readContextSeeds(args.seeds), + limit: readOptionalNumber(args, 'limit') + }) +} + +export const createExternalContextResourceTool: AiToolEntry = { + definition: { + name: 'xnet_create_external_context_resource', + title: 'Create untrusted external context resource', + description: + 'Wrap externally fetched content as an untrusted context-pack resource with an explicit instruction boundary.', + risk: 'medium', + requiredScopes: ['network.fetch'], + inputSchema: { + type: 'object', + properties: { + url: { type: 'string', description: 'External source URL.' }, + text: { type: 'string', description: 'Fetched external text content.' }, + mimeType: { type: 'string', description: 'Source MIME type. Defaults to text/plain.' } + }, + required: ['url', 'text'] + } + }, + execute: (host, args) => + host.createExternalContextResource({ + url: readRequiredString(args, 'url'), + text: readRequiredString(args, 'text'), + mimeType: readOptionalString(args, 'mimeType') + }) +} + +export const searchToolEntries: readonly AiToolEntry[] = [ + searchTool, + graphExpandTool, + createContextPackTool, + createExternalContextResourceTool +] diff --git a/packages/react/src/context.ts b/packages/react/src/context.ts index 892956b85..587dde241 100644 --- a/packages/react/src/context.ts +++ b/packages/react/src/context.ts @@ -4,63 +4,37 @@ * Provides NodeStore and optional identity to the React tree. * All data access happens through useQuery/useMutate/useNode hooks. */ -import type { XNetRuntimeConfig, XNetRuntimeStatus, XNetRuntimeMode } from './runtime' -import type { BlobStoreForSync } from '@xnetjs/runtime' -import type { ConnectionManager } from '@xnetjs/runtime' +import type { XNetRuntimeConfig, XNetRuntimeStatus } from './runtime' import type { DID } from '@xnetjs/core' import type { SecurityLevel } from '@xnetjs/crypto' -import type { NodeChangeEvent, NodeStorageAdapter } from '@xnetjs/data' +import type { NodeStorageAdapter } from '@xnetjs/data' import type { Identity, PQKeyRegistry, HybridKeyBundle } from '@xnetjs/identity' +import type { BlobStoreForSync, ConnectionManager, SyncManager, SyncStatus } from '@xnetjs/runtime' import type { SyncReplicationConfig } from '@xnetjs/sync' import type { ReactNode } from 'react' -import { MemoryNodeStorageAdapter, NodeStore } from '@xnetjs/data' +import { NodeStore } from '@xnetjs/data' import { - createDataBridge, - createMainThreadBridge, - MainThreadBridge, - WorkerBridge, type DataBridge, - type MainThreadBridgeOptions, type NodeQueryRouterThresholds, - type RemoteNodeQueryClient, - type SyncManagerLike + type RemoteNodeQueryClient } from '@xnetjs/data-bridge' import { UndoManager } from '@xnetjs/history' -import { createUCAN } from '@xnetjs/identity' import { PluginRegistry, type Platform } from '@xnetjs/plugins' -import React, { - createContext, - useCallback, - useContext, - useEffect, - useMemo, - useRef, - useState -} from 'react' +import React, { createContext, useContext, useEffect, useMemo, useState } from 'react' import { SecurityProvider } from './context/security-context' import { TelemetryContext, type TelemetryReporter } from './context/telemetry-context' import { TracingContext, type TracingReporter } from './context/tracing-context' import { PluginRegistryContext } from './hooks/usePlugins' -import { AutoBackup } from './hub/auto-backup' -import { uploadBackup } from './hub/backup' -import { createRuntimeStatus, resolveRuntimeConfig } from './runtime' -import { createSyncManager, type SyncManager, type SyncStatus } from '@xnetjs/runtime' - -// Debug logging - enable via localStorage.setItem('xnet:sync:debug', 'true') -function log(...args: unknown[]): void { - if (typeof localStorage !== 'undefined' && localStorage.getItem('xnet:sync:debug') === 'true') { - console.log('[XNetProvider]', ...args) - } -} - -/** Run `fn` when the main thread is idle, falling back to a timer. */ -function scheduleIdle(fn: () => void): void { - if (typeof window === 'undefined') return - const ric = (window as Window & { requestIdleCallback?: (cb: () => void) => void }) - .requestIdleCallback - if (typeof ric === 'function') ric(fn) - else setTimeout(fn, 1000) -} +import { log } from './provider/debug' +import { useHubAuthToken } from './provider/use-hub-auth-token' +import { useHubSearchIndex } from './provider/use-hub-search-index' +import { useNodeStoreRuntime } from './provider/use-node-store-runtime' +import { + useBridgeSyncWiring, + useHubStatus, + useSyncManagerLifecycle +} from './provider/use-sync-manager' +import { resolveRuntimeConfig } from './runtime' function resolveConfiguredSignalingUrls( hubUrl: string | null, @@ -83,242 +57,6 @@ function resolveConfiguredSignalingUrls( return urls } -const HUB_CAPABILITIES = [ - { with: '*', can: 'hub/*' }, - { with: '*', can: 'backup/*' }, - { with: '*', can: 'files/*' }, - { with: '*', can: 'query/*' }, - { with: '*', can: 'index/*' } -] as const - -const HUB_TOKEN_TTL_SECONDS = 60 * 60 * 24 -const HUB_INDEX_DEBOUNCE_MS = 2000 - -type RuntimeResolution = { - bridge: DataBridge | null - createdInternally: boolean - status: XNetRuntimeStatus -} - -type SyncManagedBridge = DataBridge & { - setSyncManager?: (syncManager: SyncManagerLike | null) => void -} - -function getRuntimeErrorMessage(err: unknown): string { - return err instanceof Error ? err.message : String(err) -} - -function inferBridgeMode(bridge: DataBridge): XNetRuntimeMode | null { - if (bridge instanceof WorkerBridge) return 'worker' - if (bridge instanceof MainThreadBridge) return 'main-thread' - return null -} - -function reportRuntimeStatus( - telemetry: TelemetryReporter | undefined, - status: XNetRuntimeStatus -): void { - telemetry?.reportUsage(`react.runtime.request.${status.requestedMode}`, 1) - - if (status.activeMode) { - telemetry?.reportUsage(`react.runtime.active.${status.activeMode}`, 1) - } - - if (status.usedFallback && status.fallbackMode) { - telemetry?.reportUsage(`react.runtime.fallback.${status.fallbackMode}`, 1) - } -} - -function logRuntimeStatus(runtime: XNetRuntimeConfig, status: XNetRuntimeStatus): void { - if (!runtime.diagnostics) return - - if (status.phase === 'error') { - console.error('[XNetProvider] Runtime initialization failed:', status) - return - } - - if (status.usedFallback) { - console.warn('[XNetProvider] Runtime fallback activated:', status) - return - } - - console.info('[XNetProvider] Runtime ready:', status) -} - -/** - * "#design #perf" search text for a node's tag ids, so searching a tag - * name finds tagged nodes (exploration 0169). Unresolvable ids are - * skipped — an archived or not-yet-synced tag never blocks indexing. - */ -async function resolveTagSearchText( - store: NodeStore, - tagIds: string[] -): Promise { - const names = await Promise.all( - tagIds.map(async (id) => { - const tag = await store.get(id).catch(() => null) - const name = tag?.properties?.name - return typeof name === 'string' && name ? `#${name}` : null - }) - ) - const present = names.filter((entry): entry is string => entry !== null) - return present.length > 0 ? present.join(' ') : undefined -} - -function resolveRuntimeFailure( - runtime: XNetRuntimeConfig, - nodeStore: NodeStore, - reason: string, - bridgeOptions?: MainThreadBridgeOptions -): RuntimeResolution { - if (runtime.fallback === 'main-thread') { - return { - bridge: createMainThreadBridge(nodeStore, bridgeOptions), - createdInternally: true, - status: createRuntimeStatus(runtime, { - activeMode: 'main-thread', - fallbackMode: 'main-thread', - usedFallback: true, - phase: 'ready', - reason - }) - } - } - - return { - bridge: null, - createdInternally: false, - status: createRuntimeStatus(runtime, { - phase: 'error', - reason - }) - } -} - -async function resolveRuntimeBridge(input: { - runtime: XNetRuntimeConfig - nodeStore: NodeStore - authorDID: DID - signingKey: Uint8Array - signalingUrl?: string - dataBridge?: DataBridge - remoteNodeQueryClient?: RemoteNodeQueryClient - remoteNodeQueryRouting?: Partial - syncManager?: SyncManager -}): Promise { - const { - runtime, - nodeStore, - authorDID, - signingKey, - signalingUrl, - dataBridge, - remoteNodeQueryClient, - remoteNodeQueryRouting, - syncManager - } = input - - if (runtime.mode === 'ipc') { - if (!syncManager) { - return resolveRuntimeFailure( - runtime, - nodeStore, - 'IPC runtime requires config.syncManager to be provided explicitly.', - { remoteNodeQueryClient, remoteNodeQueryRouting } - ) - } - - return { - bridge: - dataBridge ?? - createMainThreadBridge(nodeStore, { - remoteNodeQueryClient, - remoteNodeQueryRouting - }), - createdInternally: !dataBridge, - status: createRuntimeStatus(runtime, { - activeMode: 'ipc', - phase: 'ready' - }) - } - } - - if (dataBridge) { - const activeMode = inferBridgeMode(dataBridge) ?? runtime.mode - const usedFallback = activeMode !== runtime.mode - - return { - bridge: dataBridge, - createdInternally: false, - status: createRuntimeStatus(runtime, { - activeMode, - fallbackMode: usedFallback ? activeMode : null, - usedFallback, - phase: 'ready', - reason: usedFallback - ? `Configured runtime "${runtime.mode}" resolved to "${activeMode}" through the supplied dataBridge.` - : null - }) - } - } - - if (runtime.mode === 'worker') { - try { - const bridge = await createDataBridge({ - nodeStore, - config: { - dbName: runtime.worker?.dbName, - authorDID, - signingKey, - signalingUrl: runtime.worker?.signalingUrl ?? signalingUrl, - storagePort: runtime.worker?.storagePort, - remoteNodeQueryClient, - remoteNodeQueryRouting - }, - workerUrl: runtime.worker?.url, - mode: 'worker' - }) - - return { - bridge, - createdInternally: true, - status: createRuntimeStatus(runtime, { - activeMode: 'worker', - phase: 'ready' - }) - } - } catch (err) { - return resolveRuntimeFailure( - runtime, - nodeStore, - `Worker runtime unavailable: ${getRuntimeErrorMessage(err)}`, - { remoteNodeQueryClient, remoteNodeQueryRouting } - ) - } - } - - const bridge = await createDataBridge({ - nodeStore, - config: { - authorDID, - signingKey, - signalingUrl, - remoteNodeQueryClient, - remoteNodeQueryRouting - }, - mode: 'main-thread' - }) - - return { - bridge, - createdInternally: true, - status: createRuntimeStatus(runtime, { - activeMode: 'main-thread', - phase: 'ready' - }) - } -} - /** * XNet configuration */ @@ -572,14 +310,8 @@ export interface XNetProviderProps { * Initializes NodeStore and provides it to the React tree. */ export function XNetProvider({ config, children }: XNetProviderProps): JSX.Element { - const [nodeStore, setNodeStore] = useState(null) - const [nodeStoreReady, setNodeStoreReady] = useState(false) const [undoManager, setUndoManager] = useState(null) - const [dataBridge, setDataBridge] = useState(null) - const [syncManager, setSyncManager] = useState(null) - const [hubStatus, setHubStatus] = useState('disconnected') const [pluginRegistry, setPluginRegistry] = useState(null) - const nodeStorageRef = useRef(null) const platform = config.platform ?? 'web' const authorDID = config.authorDID ?? (config.identity?.did as string | undefined) @@ -614,449 +346,55 @@ export function XNetProvider({ config, children }: XNetProviderProps): JSX.Eleme platform ] ) - const [runtimeStatus, setRuntimeStatus] = useState(() => - createRuntimeStatus(runtimeConfig) - ) - - const getHubAuthToken = useCallback(async (): Promise => { - if (staticHubAuthToken) return staticHubAuthToken - if (!hubUrl || !autoAuth) return '' - if (!authorDID || !config.signingKey) { - throw new Error('Missing authorDID/signingKey for hub auth') - } - - return createUCAN({ - issuer: authorDID, - issuerKey: config.signingKey, - audience: hubUrl, - capabilities: HUB_CAPABILITIES as unknown as Array<{ with: string; can: string }>, - expiration: Math.floor(Date.now() / 1000) + HUB_TOKEN_TTL_SECONDS - }) - }, [authorDID, autoAuth, config.signingKey, hubUrl, staticHubAuthToken]) - - useEffect(() => { - const nodeStorageAdapter = config.nodeStorage ?? new MemoryNodeStorageAdapter() - nodeStorageRef.current = nodeStorageAdapter - const signingKey = config.signingKey - setRuntimeStatus(createRuntimeStatus(runtimeConfig)) - - // Skip NodeStore initialization if credentials not provided - if (!authorDID || !signingKey) { - console.warn( - 'XNetProvider: authorDID and signingKey not provided. NodeStore will not be initialized. ' + - 'Provide these via config.authorDID/config.signingKey or config.identity.' - ) - setRuntimeStatus( - createRuntimeStatus(runtimeConfig, { - phase: 'error', - reason: 'authorDID and signingKey are required to initialize the runtime.' - }) - ) - return - } - - // Track whether this effect instance is still active (handles StrictMode double-mount) - let cancelled = false - - // Initialize the node storage adapter if it has an open() method - const initializeNodeStore = async () => { - if ('open' in nodeStorageAdapter && typeof nodeStorageAdapter.open === 'function') { - await nodeStorageAdapter.open() - } - - // Check if effect was cleaned up while we were awaiting - if (cancelled) return - - const ns = new NodeStore({ - storage: nodeStorageAdapter, - authorDID: authorDID as DID, - signingKey - }) - - await ns.initialize() - - // Check again after second await - if (cancelled) return - - const resolvedRuntime = await resolveRuntimeBridge({ - runtime: runtimeConfig, - nodeStore: ns, - authorDID: authorDID as DID, - signingKey, - signalingUrl: signalingUrls[0], - dataBridge: config.dataBridge, - remoteNodeQueryClient: config.remoteNodeQueryClient, - remoteNodeQueryRouting: config.remoteNodeQueryRouting, - syncManager: config.syncManager - }) - - if (cancelled) { - if (resolvedRuntime.createdInternally && resolvedRuntime.bridge) { - resolvedRuntime.bridge.destroy() - } - return - } - - setRuntimeStatus(resolvedRuntime.status) - reportRuntimeStatus(config.telemetry, resolvedRuntime.status) - logRuntimeStatus(runtimeConfig, resolvedRuntime.status) - - if (resolvedRuntime.status.phase !== 'ready' || !resolvedRuntime.bridge) { - config.telemetry?.reportCrash( - new Error(resolvedRuntime.status.reason ?? 'Runtime failed'), - { - codeNamespace: 'react.runtime.initialize', - requestedMode: resolvedRuntime.status.requestedMode - } - ) - setNodeStore(null) - setNodeStoreReady(false) - setDataBridge(null) - bridgeRef = null - return - } - - setNodeStore(ns) - setNodeStoreReady(true) - setDataBridge(resolvedRuntime.bridge) - - // Store bridge ref for cleanup (only if we created it) - bridgeRef = resolvedRuntime.createdInternally ? resolvedRuntime.bridge : null - - // Expose NodeStore to window for main process access (Electron Local API) - if (typeof window !== 'undefined') { - const win = window as Window & { __xnetNodeStore?: NodeStore } - win.__xnetNodeStore = ns - } - - // Refresh query-planner statistics at idle, after first paint, so the - // planner stays in sync as the database grows (exploration 0184). Cheap - // (`PRAGMA optimize` only ANALYZEs drifted tables) and never blocks the - // initial render. - scheduleIdle(() => { - if (!cancelled) void ns.optimize() - }) - } - - let bridgeRef: DataBridge | null = null - initializeNodeStore() - - return () => { - cancelled = true - // Clean up DataBridge first - if (bridgeRef) { - bridgeRef.destroy() - } - setDataBridge(null) - setNodeStore(null) - setNodeStoreReady(false) - - // Clean up window reference - if (typeof window !== 'undefined') { - delete (window as Window & { __xnetNodeStore?: NodeStore }).__xnetNodeStore - } - - if ('close' in nodeStorageAdapter && typeof nodeStorageAdapter.close === 'function') { - nodeStorageAdapter.close() - } - } - }, [ + const getHubAuthToken = useHubAuthToken({ authorDID, - config.nodeStorage, - config.signingKey, - config.dataBridge, - signalingUrls, - config.syncManager, - config.remoteNodeQueryClient, - config.remoteNodeQueryRouting, - config.telemetry, + signingKey: config.signingKey, hubUrl, - runtimeConfig, - runtimeWorkerUrlKey - ]) - - // Create SyncManager when NodeStore is ready - useEffect(() => { - // If an external SyncManager is provided (e.g., IPC-based for Electron), use it directly - if (config.syncManager) { - // Set the syncManager immediately so components can subscribe to status updates - setSyncManager(config.syncManager) - - // If the external SyncManager supports setIdentity (e.g., IPCSyncManager for Electron), - // set the identity before starting so updates can be signed - const sm = config.syncManager as SyncManager & { - setIdentity?: (authorDID: string, signingKey: Uint8Array) => void - configureReplication?: (config: SyncReplicationConfig | undefined) => void - } - if (sm.setIdentity && authorDID && config.signingKey) { - sm.setIdentity(authorDID, config.signingKey) - } - if (sm.configureReplication) { - sm.configureReplication(config.sync) - } - - config.syncManager.start().catch((err) => { - console.warn('[XNetProvider] External SyncManager failed to start:', err) - // SyncManager is still usable for local-only operation - }) - - return () => { - config.syncManager!.stop().catch((err) => { - console.warn('[XNetProvider] External SyncManager failed to stop:', err) - }) - setSyncManager(null) - } - } - - if (!nodeStore || !nodeStoreReady || config.disableSyncManager) { - log('SyncManager disabled or NodeStore not ready', { - nodeStore: !!nodeStore, - nodeStoreReady, - disableSyncManager: config.disableSyncManager - }) - setSyncManager(null) - return - } - - const storage = nodeStorageRef.current - if (!storage) { - log('No storage adapter available') - return - } - - // No hub and no signaling servers → empty URL. The connection manager treats - // that as "stay offline" (no socket, no browser connection error) instead of - // dialing a hardcoded localhost hub that nothing is serving (exploration - // 0188). A real hub is opted into via hubUrl / signalingServers. - const signalingUrl = signalingUrls[0] ?? '' - - if (autoAuth && hubUrl && (!authorDID || !config.signingKey)) { - console.warn('[XNetProvider] Hub auth enabled but authorDID/signingKey missing') - } - - if (autoBackup && (!hubUrl || !encryptionKey)) { - console.warn('[XNetProvider] Auto-backup requires hubUrl and encryptionKey') - } - - console.log('[XNetProvider] Creating SyncManager with signalingUrls:', signalingUrls) - log('Creating SyncManager with signalingUrls:', signalingUrls) - let autoBackupManager: AutoBackup | null = null - const enableAutoBackup = Boolean(autoBackup && hubUrl && encryptionKey) + autoAuth, + staticHubAuthToken + }) - const sm = createSyncManager({ - nodeStore, - storage, - signalingUrl, - signalingUrls, + // Initialization: storage → NodeStore → runtime bridge (provider/ unit, 0276) + const { nodeStore, nodeStoreReady, dataBridge, runtimeStatus, nodeStorageRef } = + useNodeStoreRuntime({ authorDID, signingKey: config.signingKey, - replication: config.sync, - blobStore: config.blobStore, - nodeSyncRoom: hubUrl ? nodeSyncRoom : undefined, - getUCANToken: hubUrl ? getHubAuthToken : undefined, - onDocUpdate: enableAutoBackup - ? (nodeId, doc) => { - autoBackupManager?.handleDocUpdate(nodeId, doc) - } - : undefined, - onDocEvict: enableAutoBackup - ? (nodeId, doc) => { - autoBackupManager?.handleDocEvict(nodeId, doc) - } - : undefined + nodeStorage: config.nodeStorage, + dataBridge: config.dataBridge, + remoteNodeQueryClient: config.remoteNodeQueryClient, + remoteNodeQueryRouting: config.remoteNodeQueryRouting, + syncManager: config.syncManager, + telemetry: config.telemetry, + hubUrl, + signalingUrls, + runtimeConfig, + runtimeWorkerUrlKey }) - if (enableAutoBackup && hubUrl && encryptionKey) { - autoBackupManager = new AutoBackup( - async (docId, plaintext) => { - await uploadBackup( - { - hubUrl, - encryptionKey, - getAuthToken: autoAuth ? getHubAuthToken : undefined - }, - docId, - plaintext - ) - }, - { - debounceMs: backupDebounceMs, - isEnabled: () => sm.connection?.status === 'connected' - } - ) - } - - // Set SyncManager immediately so hooks can use it - // (it will connect in the background) - setSyncManager(sm) - console.log('[XNetProvider] SyncManager created and set in context') - log('SyncManager created, starting...') - - sm.start() - .then(() => { - log('SyncManager started successfully') - }) - .catch((err) => { - console.warn('[XNetProvider] SyncManager failed to start:', err) - log('SyncManager start failed:', err) - }) - - return () => { - sm.stop().catch((err) => { - console.warn('[XNetProvider] SyncManager failed to stop:', err) - }) - autoBackupManager?.destroy() - setSyncManager(null) - } - }, [ + // Sync + backup lifecycle, bridge wiring, hub status, search indexing + // (provider/ units, 0276) + const syncManager = useSyncManagerLifecycle({ nodeStore, nodeStoreReady, - config.disableSyncManager, - config.syncManager, + nodeStorageRef, + externalSyncManager: config.syncManager, + disableSyncManager: config.disableSyncManager, signalingUrls, - config.blobStore, - config.sync, authorDID, + signingKey: config.signingKey, + sync: config.sync, + blobStore: config.blobStore, + hubUrl, + nodeSyncRoom, autoAuth, autoBackup, backupDebounceMs, encryptionKey, - getHubAuthToken, - hubUrl, - nodeSyncRoom - ]) - - // Connect SyncManager to DataBridge for Y.Doc acquisition - // This allows useNode to use bridge.acquireDoc() instead of direct SyncManager access - useEffect(() => { - if (!dataBridge || !syncManager) return - - const bridge = dataBridge as SyncManagedBridge - - if (typeof bridge.setSyncManager === 'function') { - bridge.setSyncManager(syncManager) - log('Connected SyncManager to DataBridge') - } - - return () => { - if (typeof bridge.setSyncManager === 'function') { - bridge.setSyncManager(null) - } - } - }, [dataBridge, syncManager]) - - // Track hub connection status from SyncManager - useEffect(() => { - if (!syncManager) { - setHubStatus('disconnected') - return - } - - setHubStatus(syncManager.status) - return syncManager.on('status', (status) => { - setHubStatus(status) - }) - }, [syncManager]) - - // Hub search index updates (NodeStore -> hub index) - useEffect(() => { - if (!nodeStore || !syncManager || !hubUrl || !enableSearchIndex) return - const connection = syncManager.connection - if (!connection) return - - const timers = new Map>() - const pending = new Map< - string, - | { - type: 'update' - meta: { schemaIri: string; title: string; properties: Record } - /** Extra searchable text (e.g. resolved #tag names — 0169) */ - text?: string - } - | { type: 'remove' } - >() - - const schedule = ( - docId: string, - payload: - | { - type: 'update' - meta: { schemaIri: string; title: string; properties: Record } - text?: string - } - | { type: 'remove' } - ): void => { - pending.set(docId, payload) - const existing = timers.get(docId) - if (existing) clearTimeout(existing) - - timers.set( - docId, - setTimeout(() => { - timers.delete(docId) - const next = pending.get(docId) - pending.delete(docId) - if (!next) return - - if (connection.status !== 'connected') return - - if (next.type === 'remove') { - connection.sendRaw({ type: 'index-remove', docId }) - return - } - - connection.sendRaw({ - type: 'index-update', - docId, - meta: next.meta, - ...(next.text !== undefined ? { text: next.text } : {}) - }) - }, HUB_INDEX_DEBOUNCE_MS) - ) - } - - const handleChange = (event: NodeChangeEvent) => { - const node = event.node - if (!node || node.deleted) { - schedule(event.change.payload.nodeId, { type: 'remove' }) - return - } - - if (!node.schemaId) return - - // `name`-titled nodes (Tag, Folder, Project, Channel) index their name. - const title = - typeof node.properties.title === 'string' - ? node.properties.title - : typeof node.properties.name === 'string' - ? node.properties.name - : '' - const meta = { schemaIri: node.schemaId, title, properties: node.properties } - - // Resolve tag ids to names so searching "design" finds tagged nodes (0169). - const tagIds = Array.isArray(node.properties.tags) - ? node.properties.tags.filter((id): id is string => typeof id === 'string') - : [] - if (tagIds.length === 0) { - schedule(node.id, { type: 'update', meta }) - return - } - void resolveTagSearchText(nodeStore, tagIds).then((text) => { - schedule(node.id, { type: 'update', meta, ...(text ? { text } : {}) }) - }) - } - - const unsubscribe = nodeStore.subscribe(handleChange) - - return () => { - unsubscribe() - for (const timer of timers.values()) { - clearTimeout(timer) - } - timers.clear() - pending.clear() - } - }, [enableSearchIndex, hubUrl, nodeStore, syncManager]) + getHubAuthToken + }) + useBridgeSyncWiring(dataBridge, syncManager) + const hubStatus = useHubStatus(syncManager) + useHubSearchIndex({ nodeStore, syncManager, hubUrl, enableSearchIndex }) // Create PluginRegistry when NodeStore is ready useEffect(() => { diff --git a/packages/react/src/provider/debug.ts b/packages/react/src/provider/debug.ts new file mode 100644 index 000000000..6f068a944 --- /dev/null +++ b/packages/react/src/provider/debug.ts @@ -0,0 +1,17 @@ +/** Shared debug/scheduling helpers for the XNetProvider units (0276). */ + +// Debug logging - enable via localStorage.setItem('xnet:sync:debug', 'true') +export function log(...args: unknown[]): void { + if (typeof localStorage !== 'undefined' && localStorage.getItem('xnet:sync:debug') === 'true') { + console.log('[XNetProvider]', ...args) + } +} + +/** Run `fn` when the main thread is idle, falling back to a timer. */ +export function scheduleIdle(fn: () => void): void { + if (typeof window === 'undefined') return + const ric = (window as Window & { requestIdleCallback?: (cb: () => void) => void }) + .requestIdleCallback + if (typeof ric === 'function') ric(fn) + else setTimeout(fn, 1000) +} diff --git a/packages/react/src/provider/runtime-resolution.ts b/packages/react/src/provider/runtime-resolution.ts new file mode 100644 index 000000000..29777477d --- /dev/null +++ b/packages/react/src/provider/runtime-resolution.ts @@ -0,0 +1,228 @@ +/** + * Runtime-bridge resolution for `XNetProvider` (0276): picks worker / main + * thread / IPC, applies the configured fallback policy, and reports the + * resulting `XNetRuntimeStatus` to telemetry/diagnostics. + */ + +import type { TelemetryReporter } from '../context/telemetry-context' +import type { XNetRuntimeConfig, XNetRuntimeStatus, XNetRuntimeMode } from '../runtime' +import type { DID } from '@xnetjs/core' +import type { NodeStore } from '@xnetjs/data' +import type { SyncManager } from '@xnetjs/runtime' +import { + createDataBridge, + createMainThreadBridge, + MainThreadBridge, + WorkerBridge, + type DataBridge, + type MainThreadBridgeOptions, + type NodeQueryRouterThresholds, + type RemoteNodeQueryClient, + type SyncManagerLike +} from '@xnetjs/data-bridge' +import { createRuntimeStatus } from '../runtime' + +export type RuntimeResolution = { + bridge: DataBridge | null + createdInternally: boolean + status: XNetRuntimeStatus +} + +export type SyncManagedBridge = DataBridge & { + setSyncManager?: (syncManager: SyncManagerLike | null) => void +} + +function getRuntimeErrorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err) +} + +function inferBridgeMode(bridge: DataBridge): XNetRuntimeMode | null { + if (bridge instanceof WorkerBridge) return 'worker' + if (bridge instanceof MainThreadBridge) return 'main-thread' + return null +} + +export function reportRuntimeStatus( + telemetry: TelemetryReporter | undefined, + status: XNetRuntimeStatus +): void { + telemetry?.reportUsage(`react.runtime.request.${status.requestedMode}`, 1) + + if (status.activeMode) { + telemetry?.reportUsage(`react.runtime.active.${status.activeMode}`, 1) + } + + if (status.usedFallback && status.fallbackMode) { + telemetry?.reportUsage(`react.runtime.fallback.${status.fallbackMode}`, 1) + } +} + +export function logRuntimeStatus(runtime: XNetRuntimeConfig, status: XNetRuntimeStatus): void { + if (!runtime.diagnostics) return + + if (status.phase === 'error') { + console.error('[XNetProvider] Runtime initialization failed:', status) + return + } + + if (status.usedFallback) { + console.warn('[XNetProvider] Runtime fallback activated:', status) + return + } + + console.info('[XNetProvider] Runtime ready:', status) +} + +function resolveRuntimeFailure( + runtime: XNetRuntimeConfig, + nodeStore: NodeStore, + reason: string, + bridgeOptions?: MainThreadBridgeOptions +): RuntimeResolution { + if (runtime.fallback === 'main-thread') { + return { + bridge: createMainThreadBridge(nodeStore, bridgeOptions), + createdInternally: true, + status: createRuntimeStatus(runtime, { + activeMode: 'main-thread', + fallbackMode: 'main-thread', + usedFallback: true, + phase: 'ready', + reason + }) + } + } + + return { + bridge: null, + createdInternally: false, + status: createRuntimeStatus(runtime, { + phase: 'error', + reason + }) + } +} + +export async function resolveRuntimeBridge(input: { + runtime: XNetRuntimeConfig + nodeStore: NodeStore + authorDID: DID + signingKey: Uint8Array + signalingUrl?: string + dataBridge?: DataBridge + remoteNodeQueryClient?: RemoteNodeQueryClient + remoteNodeQueryRouting?: Partial + syncManager?: SyncManager +}): Promise { + const { + runtime, + nodeStore, + authorDID, + signingKey, + signalingUrl, + dataBridge, + remoteNodeQueryClient, + remoteNodeQueryRouting, + syncManager + } = input + + if (runtime.mode === 'ipc') { + if (!syncManager) { + return resolveRuntimeFailure( + runtime, + nodeStore, + 'IPC runtime requires config.syncManager to be provided explicitly.', + { remoteNodeQueryClient, remoteNodeQueryRouting } + ) + } + + return { + bridge: + dataBridge ?? + createMainThreadBridge(nodeStore, { + remoteNodeQueryClient, + remoteNodeQueryRouting + }), + createdInternally: !dataBridge, + status: createRuntimeStatus(runtime, { + activeMode: 'ipc', + phase: 'ready' + }) + } + } + + if (dataBridge) { + const activeMode = inferBridgeMode(dataBridge) ?? runtime.mode + const usedFallback = activeMode !== runtime.mode + + return { + bridge: dataBridge, + createdInternally: false, + status: createRuntimeStatus(runtime, { + activeMode, + fallbackMode: usedFallback ? activeMode : null, + usedFallback, + phase: 'ready', + reason: usedFallback + ? `Configured runtime "${runtime.mode}" resolved to "${activeMode}" through the supplied dataBridge.` + : null + }) + } + } + + if (runtime.mode === 'worker') { + try { + const bridge = await createDataBridge({ + nodeStore, + config: { + dbName: runtime.worker?.dbName, + authorDID, + signingKey, + signalingUrl: runtime.worker?.signalingUrl ?? signalingUrl, + storagePort: runtime.worker?.storagePort, + remoteNodeQueryClient, + remoteNodeQueryRouting + }, + workerUrl: runtime.worker?.url, + mode: 'worker' + }) + + return { + bridge, + createdInternally: true, + status: createRuntimeStatus(runtime, { + activeMode: 'worker', + phase: 'ready' + }) + } + } catch (err) { + return resolveRuntimeFailure( + runtime, + nodeStore, + `Worker runtime unavailable: ${getRuntimeErrorMessage(err)}`, + { remoteNodeQueryClient, remoteNodeQueryRouting } + ) + } + } + + const bridge = await createDataBridge({ + nodeStore, + config: { + authorDID, + signingKey, + signalingUrl, + remoteNodeQueryClient, + remoteNodeQueryRouting + }, + mode: 'main-thread' + }) + + return { + bridge, + createdInternally: true, + status: createRuntimeStatus(runtime, { + activeMode: 'main-thread', + phase: 'ready' + }) + } +} diff --git a/packages/react/src/provider/use-hub-auth-token.test.tsx b/packages/react/src/provider/use-hub-auth-token.test.tsx new file mode 100644 index 000000000..7c3fcc8a0 --- /dev/null +++ b/packages/react/src/provider/use-hub-auth-token.test.tsx @@ -0,0 +1,55 @@ +import { renderHook } from '@testing-library/react' +import { describe, expect, it } from 'vitest' +import { useHubAuthToken } from './use-hub-auth-token' + +describe('useHubAuthToken (provider auth unit, 0276)', () => { + it('returns the static token without touching the signing key', async () => { + const { result } = renderHook(() => + useHubAuthToken({ + authorDID: undefined, + signingKey: undefined, + hubUrl: 'https://hub.example', + autoAuth: true, + staticHubAuthToken: 'static-token' + }) + ) + await expect(result.current()).resolves.toBe('static-token') + }) + + it('returns empty when no hub is configured or auto-auth is off', async () => { + const noHub = renderHook(() => + useHubAuthToken({ + authorDID: 'did:key:zTest', + signingKey: new Uint8Array(32), + hubUrl: null, + autoAuth: true, + staticHubAuthToken: '' + }) + ) + await expect(noHub.result.current()).resolves.toBe('') + + const authOff = renderHook(() => + useHubAuthToken({ + authorDID: 'did:key:zTest', + signingKey: new Uint8Array(32), + hubUrl: 'https://hub.example', + autoAuth: false, + staticHubAuthToken: '' + }) + ) + await expect(authOff.result.current()).resolves.toBe('') + }) + + it('fails loudly when hub auth is on but credentials are missing', async () => { + const { result } = renderHook(() => + useHubAuthToken({ + authorDID: undefined, + signingKey: undefined, + hubUrl: 'https://hub.example', + autoAuth: true, + staticHubAuthToken: '' + }) + ) + await expect(result.current()).rejects.toThrow('Missing authorDID/signingKey for hub auth') + }) +}) diff --git a/packages/react/src/provider/use-hub-auth-token.ts b/packages/react/src/provider/use-hub-auth-token.ts new file mode 100644 index 000000000..90182dbb2 --- /dev/null +++ b/packages/react/src/provider/use-hub-auth-token.ts @@ -0,0 +1,44 @@ +/** + * Hub UCAN authentication for `XNetProvider` (0276): mints short-lived + * capability tokens from the local signing key, or passes through a static + * token when one is configured. + */ + +import { createUCAN } from '@xnetjs/identity' +import { useCallback } from 'react' + +const HUB_CAPABILITIES = [ + { with: '*', can: 'hub/*' }, + { with: '*', can: 'backup/*' }, + { with: '*', can: 'files/*' }, + { with: '*', can: 'query/*' }, + { with: '*', can: 'index/*' } +] as const + +const HUB_TOKEN_TTL_SECONDS = 60 * 60 * 24 + +export function useHubAuthToken(input: { + authorDID: string | undefined + signingKey: Uint8Array | undefined + hubUrl: string | null + autoAuth: boolean + staticHubAuthToken: string +}): () => Promise { + const { authorDID, signingKey, hubUrl, autoAuth, staticHubAuthToken } = input + + return useCallback(async (): Promise => { + if (staticHubAuthToken) return staticHubAuthToken + if (!hubUrl || !autoAuth) return '' + if (!authorDID || !signingKey) { + throw new Error('Missing authorDID/signingKey for hub auth') + } + + return createUCAN({ + issuer: authorDID, + issuerKey: signingKey, + audience: hubUrl, + capabilities: HUB_CAPABILITIES as unknown as Array<{ with: string; can: string }>, + expiration: Math.floor(Date.now() / 1000) + HUB_TOKEN_TTL_SECONDS + }) + }, [authorDID, autoAuth, signingKey, hubUrl, staticHubAuthToken]) +} diff --git a/packages/react/src/provider/use-hub-search-index.ts b/packages/react/src/provider/use-hub-search-index.ts new file mode 100644 index 000000000..83ebeecc5 --- /dev/null +++ b/packages/react/src/provider/use-hub-search-index.ts @@ -0,0 +1,139 @@ +/** + * Hub search-index updates for `XNetProvider` (0276): mirrors NodeStore + * changes into the hub's search index (debounced per doc), resolving tag ids + * to names so searching a tag name finds tagged nodes (exploration 0169). + */ + +import type { NodeChangeEvent, NodeStore } from '@xnetjs/data' +import type { SyncManager } from '@xnetjs/runtime' +import { useEffect } from 'react' + +const HUB_INDEX_DEBOUNCE_MS = 2000 + +/** + * "#design #perf" search text for a node's tag ids, so searching a tag + * name finds tagged nodes (exploration 0169). Unresolvable ids are + * skipped — an archived or not-yet-synced tag never blocks indexing. + */ +async function resolveTagSearchText( + store: NodeStore, + tagIds: string[] +): Promise { + const names = await Promise.all( + tagIds.map(async (id) => { + const tag = await store.get(id).catch(() => null) + const name = tag?.properties?.name + return typeof name === 'string' && name ? `#${name}` : null + }) + ) + const present = names.filter((entry): entry is string => entry !== null) + return present.length > 0 ? present.join(' ') : undefined +} + +export function useHubSearchIndex(input: { + nodeStore: NodeStore | null + syncManager: SyncManager | null + hubUrl: string | null + enableSearchIndex: boolean +}): void { + const { nodeStore, syncManager, hubUrl, enableSearchIndex } = input + + useEffect(() => { + if (!nodeStore || !syncManager || !hubUrl || !enableSearchIndex) return + const connection = syncManager.connection + if (!connection) return + + const timers = new Map>() + const pending = new Map< + string, + | { + type: 'update' + meta: { schemaIri: string; title: string; properties: Record } + /** Extra searchable text (e.g. resolved #tag names — 0169) */ + text?: string + } + | { type: 'remove' } + >() + + const schedule = ( + docId: string, + payload: + | { + type: 'update' + meta: { schemaIri: string; title: string; properties: Record } + text?: string + } + | { type: 'remove' } + ): void => { + pending.set(docId, payload) + const existing = timers.get(docId) + if (existing) clearTimeout(existing) + + timers.set( + docId, + setTimeout(() => { + timers.delete(docId) + const next = pending.get(docId) + pending.delete(docId) + if (!next) return + + if (connection.status !== 'connected') return + + if (next.type === 'remove') { + connection.sendRaw({ type: 'index-remove', docId }) + return + } + + connection.sendRaw({ + type: 'index-update', + docId, + meta: next.meta, + ...(next.text !== undefined ? { text: next.text } : {}) + }) + }, HUB_INDEX_DEBOUNCE_MS) + ) + } + + const handleChange = (event: NodeChangeEvent) => { + const node = event.node + if (!node || node.deleted) { + schedule(event.change.payload.nodeId, { type: 'remove' }) + return + } + + if (!node.schemaId) return + + // `name`-titled nodes (Tag, Folder, Project, Channel) index their name. + const title = + typeof node.properties.title === 'string' + ? node.properties.title + : typeof node.properties.name === 'string' + ? node.properties.name + : '' + const meta = { schemaIri: node.schemaId, title, properties: node.properties } + + // Resolve tag ids to names so searching "design" finds tagged nodes (0169). + const tagIds = Array.isArray(node.properties.tags) + ? node.properties.tags.filter((id): id is string => typeof id === 'string') + : [] + if (tagIds.length === 0) { + schedule(node.id, { type: 'update', meta }) + return + } + void resolveTagSearchText(nodeStore, tagIds).then((text) => { + schedule(node.id, { type: 'update', meta, ...(text ? { text } : {}) }) + }) + } + + const unsubscribe = nodeStore.subscribe(handleChange) + + return () => { + unsubscribe() + for (const timer of timers.values()) { + clearTimeout(timer) + } + timers.clear() + pending.clear() + } + }, [enableSearchIndex, hubUrl, nodeStore, syncManager]) +} diff --git a/packages/react/src/provider/use-node-store-runtime.ts b/packages/react/src/provider/use-node-store-runtime.ts new file mode 100644 index 000000000..a5ffa6665 --- /dev/null +++ b/packages/react/src/provider/use-node-store-runtime.ts @@ -0,0 +1,212 @@ +/** + * NodeStore + runtime-bridge initialization for `XNetProvider` (0276). + * + * Owns the init lifecycle: open the storage adapter, create + initialize the + * `NodeStore`, resolve the data bridge (worker / main-thread / IPC with the + * configured fallback), report status, and tear everything down on unmount — + * including the StrictMode double-mount `cancelled` protocol. + */ + +import type { TelemetryReporter } from '../context/telemetry-context' +import type { XNetRuntimeConfig, XNetRuntimeStatus } from '../runtime' +import type { DID } from '@xnetjs/core' +import type { NodeStorageAdapter } from '@xnetjs/data' +import type { + DataBridge, + NodeQueryRouterThresholds, + RemoteNodeQueryClient +} from '@xnetjs/data-bridge' +import type { SyncManager } from '@xnetjs/runtime' +import type { MutableRefObject } from 'react' +import { MemoryNodeStorageAdapter, NodeStore } from '@xnetjs/data' +import { useEffect, useRef, useState } from 'react' +import { createRuntimeStatus } from '../runtime' +import { scheduleIdle } from './debug' +import { logRuntimeStatus, reportRuntimeStatus, resolveRuntimeBridge } from './runtime-resolution' + +export type NodeStoreRuntimeInput = { + authorDID: string | undefined + signingKey: Uint8Array | undefined + nodeStorage: NodeStorageAdapter | undefined + dataBridge: DataBridge | undefined + remoteNodeQueryClient: RemoteNodeQueryClient | undefined + remoteNodeQueryRouting: Partial | undefined + syncManager: SyncManager | undefined + telemetry: TelemetryReporter | undefined + hubUrl: string | null + signalingUrls: string[] + runtimeConfig: XNetRuntimeConfig + runtimeWorkerUrlKey: string +} + +export type NodeStoreRuntime = { + nodeStore: NodeStore | null + nodeStoreReady: boolean + dataBridge: DataBridge | null + runtimeStatus: XNetRuntimeStatus + nodeStorageRef: MutableRefObject +} + +export function useNodeStoreRuntime(input: NodeStoreRuntimeInput): NodeStoreRuntime { + const { + authorDID, + signingKey, + nodeStorage, + dataBridge: configDataBridge, + remoteNodeQueryClient, + remoteNodeQueryRouting, + syncManager: configSyncManager, + telemetry, + hubUrl, + signalingUrls, + runtimeConfig, + runtimeWorkerUrlKey + } = input + + const [nodeStore, setNodeStore] = useState(null) + const [nodeStoreReady, setNodeStoreReady] = useState(false) + const [dataBridge, setDataBridge] = useState(null) + const nodeStorageRef = useRef(null) + const [runtimeStatus, setRuntimeStatus] = useState(() => + createRuntimeStatus(runtimeConfig) + ) + + useEffect(() => { + const nodeStorageAdapter = nodeStorage ?? new MemoryNodeStorageAdapter() + nodeStorageRef.current = nodeStorageAdapter + setRuntimeStatus(createRuntimeStatus(runtimeConfig)) + + // Skip NodeStore initialization if credentials not provided + if (!authorDID || !signingKey) { + console.warn( + 'XNetProvider: authorDID and signingKey not provided. NodeStore will not be initialized. ' + + 'Provide these via config.authorDID/config.signingKey or config.identity.' + ) + setRuntimeStatus( + createRuntimeStatus(runtimeConfig, { + phase: 'error', + reason: 'authorDID and signingKey are required to initialize the runtime.' + }) + ) + return + } + + // Track whether this effect instance is still active (handles StrictMode double-mount) + let cancelled = false + + // Initialize the node storage adapter if it has an open() method + const initializeNodeStore = async () => { + if ('open' in nodeStorageAdapter && typeof nodeStorageAdapter.open === 'function') { + await nodeStorageAdapter.open() + } + + // Check if effect was cleaned up while we were awaiting + if (cancelled) return + + const ns = new NodeStore({ + storage: nodeStorageAdapter, + authorDID: authorDID as DID, + signingKey + }) + + await ns.initialize() + + // Check again after second await + if (cancelled) return + + const resolvedRuntime = await resolveRuntimeBridge({ + runtime: runtimeConfig, + nodeStore: ns, + authorDID: authorDID as DID, + signingKey, + signalingUrl: signalingUrls[0], + dataBridge: configDataBridge, + remoteNodeQueryClient, + remoteNodeQueryRouting, + syncManager: configSyncManager + }) + + if (cancelled) { + if (resolvedRuntime.createdInternally && resolvedRuntime.bridge) { + resolvedRuntime.bridge.destroy() + } + return + } + + setRuntimeStatus(resolvedRuntime.status) + reportRuntimeStatus(telemetry, resolvedRuntime.status) + logRuntimeStatus(runtimeConfig, resolvedRuntime.status) + + if (resolvedRuntime.status.phase !== 'ready' || !resolvedRuntime.bridge) { + telemetry?.reportCrash(new Error(resolvedRuntime.status.reason ?? 'Runtime failed'), { + codeNamespace: 'react.runtime.initialize', + requestedMode: resolvedRuntime.status.requestedMode + }) + setNodeStore(null) + setNodeStoreReady(false) + setDataBridge(null) + bridgeRef = null + return + } + + setNodeStore(ns) + setNodeStoreReady(true) + setDataBridge(resolvedRuntime.bridge) + + // Store bridge ref for cleanup (only if we created it) + bridgeRef = resolvedRuntime.createdInternally ? resolvedRuntime.bridge : null + + // Expose NodeStore to window for main process access (Electron Local API) + if (typeof window !== 'undefined') { + const win = window as Window & { __xnetNodeStore?: NodeStore } + win.__xnetNodeStore = ns + } + + // Refresh query-planner statistics at idle, after first paint, so the + // planner stays in sync as the database grows (exploration 0184). Cheap + // (`PRAGMA optimize` only ANALYZEs drifted tables) and never blocks the + // initial render. + scheduleIdle(() => { + if (!cancelled) void ns.optimize() + }) + } + + let bridgeRef: DataBridge | null = null + initializeNodeStore() + + return () => { + cancelled = true + // Clean up DataBridge first + if (bridgeRef) { + bridgeRef.destroy() + } + setDataBridge(null) + setNodeStore(null) + setNodeStoreReady(false) + + // Clean up window reference + if (typeof window !== 'undefined') { + delete (window as Window & { __xnetNodeStore?: NodeStore }).__xnetNodeStore + } + + if ('close' in nodeStorageAdapter && typeof nodeStorageAdapter.close === 'function') { + nodeStorageAdapter.close() + } + } + }, [ + authorDID, + nodeStorage, + signingKey, + configDataBridge, + signalingUrls, + configSyncManager, + remoteNodeQueryClient, + remoteNodeQueryRouting, + telemetry, + hubUrl, + runtimeConfig, + runtimeWorkerUrlKey + ]) + + return { nodeStore, nodeStoreReady, dataBridge, runtimeStatus, nodeStorageRef } +} diff --git a/packages/react/src/provider/use-sync-manager.ts b/packages/react/src/provider/use-sync-manager.ts new file mode 100644 index 000000000..11893f5b0 --- /dev/null +++ b/packages/react/src/provider/use-sync-manager.ts @@ -0,0 +1,262 @@ +/** + * Sync + backup lifecycle for `XNetProvider` (0276). + * + * Owns the `SyncManager` (external IPC-provided or internally created), the + * optional encrypted `AutoBackup` pipeline hanging off doc updates/evictions, + * the bridge↔sync wiring, and the hub connection status stream. + */ + +import type { SyncManagedBridge } from './runtime-resolution' +import type { NodeStorageAdapter, NodeStore } from '@xnetjs/data' +import type { DataBridge } from '@xnetjs/data-bridge' +import type { BlobStoreForSync, SyncManager, SyncStatus } from '@xnetjs/runtime' +import type { SyncReplicationConfig } from '@xnetjs/sync' +import type { MutableRefObject } from 'react' +import { createSyncManager } from '@xnetjs/runtime' +import { useEffect, useState } from 'react' +import { AutoBackup } from '../hub/auto-backup' +import { uploadBackup } from '../hub/backup' +import { log } from './debug' + +export type SyncManagerLifecycleInput = { + nodeStore: NodeStore | null + nodeStoreReady: boolean + nodeStorageRef: MutableRefObject + externalSyncManager: SyncManager | undefined + disableSyncManager: boolean | undefined + signalingUrls: string[] + authorDID: string | undefined + signingKey: Uint8Array | undefined + sync: SyncReplicationConfig | undefined + blobStore: BlobStoreForSync | undefined + hubUrl: string | null + nodeSyncRoom: string + autoAuth: boolean + autoBackup: boolean + backupDebounceMs: number + encryptionKey: Uint8Array | null + getHubAuthToken: () => Promise +} + +export function useSyncManagerLifecycle(input: SyncManagerLifecycleInput): SyncManager | null { + const { + nodeStore, + nodeStoreReady, + nodeStorageRef, + externalSyncManager, + disableSyncManager, + signalingUrls, + authorDID, + signingKey, + sync, + blobStore, + hubUrl, + nodeSyncRoom, + autoAuth, + autoBackup, + backupDebounceMs, + encryptionKey, + getHubAuthToken + } = input + + const [syncManager, setSyncManager] = useState(null) + + useEffect(() => { + // If an external SyncManager is provided (e.g., IPC-based for Electron), use it directly + if (externalSyncManager) { + // Set the syncManager immediately so components can subscribe to status updates + setSyncManager(externalSyncManager) + + // If the external SyncManager supports setIdentity (e.g., IPCSyncManager for Electron), + // set the identity before starting so updates can be signed + const sm = externalSyncManager as SyncManager & { + setIdentity?: (authorDID: string, signingKey: Uint8Array) => void + configureReplication?: (config: SyncReplicationConfig | undefined) => void + } + if (sm.setIdentity && authorDID && signingKey) { + sm.setIdentity(authorDID, signingKey) + } + if (sm.configureReplication) { + sm.configureReplication(sync) + } + + externalSyncManager.start().catch((err) => { + console.warn('[XNetProvider] External SyncManager failed to start:', err) + // SyncManager is still usable for local-only operation + }) + + return () => { + externalSyncManager.stop().catch((err) => { + console.warn('[XNetProvider] External SyncManager failed to stop:', err) + }) + setSyncManager(null) + } + } + + if (!nodeStore || !nodeStoreReady || disableSyncManager) { + log('SyncManager disabled or NodeStore not ready', { + nodeStore: !!nodeStore, + nodeStoreReady, + disableSyncManager + }) + setSyncManager(null) + return + } + + const storage = nodeStorageRef.current + if (!storage) { + log('No storage adapter available') + return + } + + // No hub and no signaling servers → empty URL. The connection manager treats + // that as "stay offline" (no socket, no browser connection error) instead of + // dialing a hardcoded localhost hub that nothing is serving (exploration + // 0188). A real hub is opted into via hubUrl / signalingServers. + const signalingUrl = signalingUrls[0] ?? '' + + if (autoAuth && hubUrl && (!authorDID || !signingKey)) { + console.warn('[XNetProvider] Hub auth enabled but authorDID/signingKey missing') + } + + if (autoBackup && (!hubUrl || !encryptionKey)) { + console.warn('[XNetProvider] Auto-backup requires hubUrl and encryptionKey') + } + + console.log('[XNetProvider] Creating SyncManager with signalingUrls:', signalingUrls) + log('Creating SyncManager with signalingUrls:', signalingUrls) + let autoBackupManager: AutoBackup | null = null + const enableAutoBackup = Boolean(autoBackup && hubUrl && encryptionKey) + + const sm = createSyncManager({ + nodeStore, + storage, + signalingUrl, + signalingUrls, + authorDID, + signingKey, + replication: sync, + blobStore, + nodeSyncRoom: hubUrl ? nodeSyncRoom : undefined, + getUCANToken: hubUrl ? getHubAuthToken : undefined, + onDocUpdate: enableAutoBackup + ? (nodeId, doc) => { + autoBackupManager?.handleDocUpdate(nodeId, doc) + } + : undefined, + onDocEvict: enableAutoBackup + ? (nodeId, doc) => { + autoBackupManager?.handleDocEvict(nodeId, doc) + } + : undefined + }) + + if (enableAutoBackup && hubUrl && encryptionKey) { + autoBackupManager = new AutoBackup( + async (docId, plaintext) => { + await uploadBackup( + { + hubUrl, + encryptionKey, + getAuthToken: autoAuth ? getHubAuthToken : undefined + }, + docId, + plaintext + ) + }, + { + debounceMs: backupDebounceMs, + isEnabled: () => sm.connection?.status === 'connected' + } + ) + } + + // Set SyncManager immediately so hooks can use it + // (it will connect in the background) + setSyncManager(sm) + console.log('[XNetProvider] SyncManager created and set in context') + log('SyncManager created, starting...') + + sm.start() + .then(() => { + log('SyncManager started successfully') + }) + .catch((err) => { + console.warn('[XNetProvider] SyncManager failed to start:', err) + log('SyncManager start failed:', err) + }) + + return () => { + sm.stop().catch((err) => { + console.warn('[XNetProvider] SyncManager failed to stop:', err) + }) + autoBackupManager?.destroy() + setSyncManager(null) + } + }, [ + nodeStore, + nodeStoreReady, + disableSyncManager, + externalSyncManager, + signalingUrls, + blobStore, + sync, + authorDID, + autoAuth, + autoBackup, + backupDebounceMs, + encryptionKey, + getHubAuthToken, + hubUrl, + nodeSyncRoom, + nodeStorageRef, + signingKey + ]) + + return syncManager +} + +/** + * Connect SyncManager to DataBridge for Y.Doc acquisition. + * This allows useNode to use bridge.acquireDoc() instead of direct SyncManager access. + */ +export function useBridgeSyncWiring( + dataBridge: DataBridge | null, + syncManager: SyncManager | null +): void { + useEffect(() => { + if (!dataBridge || !syncManager) return + + const bridge = dataBridge as SyncManagedBridge + + if (typeof bridge.setSyncManager === 'function') { + bridge.setSyncManager(syncManager) + log('Connected SyncManager to DataBridge') + } + + return () => { + if (typeof bridge.setSyncManager === 'function') { + bridge.setSyncManager(null) + } + } + }, [dataBridge, syncManager]) +} + +/** Track hub connection status from SyncManager. */ +export function useHubStatus(syncManager: SyncManager | null): SyncStatus { + const [hubStatus, setHubStatus] = useState('disconnected') + + useEffect(() => { + if (!syncManager) { + setHubStatus('disconnected') + return + } + + setHubStatus(syncManager.status) + return syncManager.on('status', (status) => { + setHubStatus(status) + }) + }, [syncManager]) + + return hubStatus +} diff --git a/packages/views/package.json b/packages/views/package.json index 016fbb850..5e2e2ea36 100644 --- a/packages/views/package.json +++ b/packages/views/package.json @@ -25,6 +25,7 @@ "@xnetjs/core": "workspace:*", "@xnetjs/data": "workspace:*", "@xnetjs/react": "workspace:*", + "@xnetjs/social": "workspace:*", "@xnetjs/ui": "workspace:*", "lucide-react": "^0.563.0", "nanoid": "^5.1.6" diff --git a/packages/views/src/data-workspace/DataWorkspaceCore.tsx b/packages/views/src/data-workspace/DataWorkspaceCore.tsx new file mode 100644 index 000000000..65b055c1d --- /dev/null +++ b/packages/views/src/data-workspace/DataWorkspaceCore.tsx @@ -0,0 +1,1284 @@ +/** + * DataWorkspaceCore — the shared Data Workspace surface (exploration 0276, + * Theme 3: well-traveled code paths). + * + * The web and desktop DataWorkspaceViews were ~92%-identical drifted copies: + * the same saved-view/descriptor parsing, workspace metrics, graph atlas, + * pattern detection, seeding and lens-saving logic. This module owns: + * + * - `useDataWorkspace` — all queries, derived data, and handlers. Platform + * deltas arrive as options: `getExistingNode` (OPFS store vs. IPC bridge), + * `seedReady` (web gates on the store), and the desktop-only + * `onInsertSavedLensAsCanvasFrame` canvas integration. + * - `DataWorkspaceBody` — the shared render (banners, import jobs, metrics, + * graph atlas, sources/patterns rail, saved-view tables, SavedViewRunner). + * Canvas-frame affordances (Frame / Pin buttons, visual-canvas projection) + * render only when the canvas callback is provided. Web-only SavedViewRunner + * extras (feed enrichment, moderation `wrapItem`) pass through + * `savedViewRunnerProps`. + * + * The app components keep only their own chrome: header, seed button + * placement, close affordances, and scroll containers. + */ +import type { SavedViewDescriptor } from '@xnetjs/data' +import { SavedViewSchema, validateSavedViewDescriptor } from '@xnetjs/data' +import { + SavedViewRunner, + useMutate, + useQuery, + type MutateOp, + type SavedViewLensDraft, + type SavedViewRunnerProps, + type SavedViewSchemaRegistry, + type SavedViewVisualCanvasProjectionRequest +} from '@xnetjs/react' +import { + listSocialImportJobs, + subscribeSocialImportJobs, + type SocialImportJobProgress +} from '@xnetjs/social/import/core' +import { createDefaultSocialGraphAtlas, type SocialGraphAtlasEntry } from '@xnetjs/social/lenses' +import { + createSocialPatternSavedViewDraft, + detectSocialPatterns, + type SocialPatternKind, + type SocialPatternSuggestion +} from '@xnetjs/social/patterns' +import { + SocialActorSchema, + SocialCollectionSchema, + SocialContentSchema, + SocialConversationSchema, + SocialImportRunSchema, + SocialInteractionSchema, + SocialMessageSchema, + socialSchemas +} from '@xnetjs/social/schemas' +import { + recommendSocialAnalyticsCache, + type SocialAnalyticsCacheRecommendation +} from '@xnetjs/social/workspace' +import { + AlertTriangle, + BarChart3, + Database, + GitBranch, + Import, + Layout, + Loader2, + MessageSquare, + Network, + Save, + Search, + Shield, + Table, + UserRound +} from 'lucide-react' +import { useCallback, useEffect, useMemo, useState, type ReactElement } from 'react' +import { + getDefaultSocialWorkspaceSeeds, + upsertDefaultSocialWorkspace, + type SocialWorkspaceSeedSummary +} from './social-workspace.js' + +// ─── Types ───────────────────────────────────────────────────────────────────── + +/** Saved lens payload for the desktop canvas-frame integration. */ +export type SavedViewCanvasFrameInput = { + id: string + title?: string + description?: string + descriptor?: string +} + +export type SavedViewRow = { + id: string + title?: string + description?: string + descriptor?: string + scope?: string +} + +type ParsedDescriptor = { + valid: boolean + queryKind: string + queryMode: string | null + primarySchemaId: string | null +} + +export type WorkspaceMetric = { + id: string + label: string + value: number | null + icon: typeof UserRound +} + +export type GraphAtlasRow = { + entry: SocialGraphAtlasEntry + savedView: SavedViewRow | null +} + +const SOCIAL_SCHEMA_REGISTRY = socialSchemas as unknown as SavedViewSchemaRegistry +const PATTERN_QUERY_LIMIT = 300 +const DISMISSED_PATTERN_STORAGE_KEY = 'xnet:data-workspace:dismissed-patterns' + +// ─── Descriptor / metric helpers ─────────────────────────────────────────────── + +function getCount(input: { totalCount: number | null; data: unknown[] }): number | null { + return input.totalCount ?? (input.data.length > 0 ? input.data.length : null) +} + +function parseSavedViewDescriptor(value: string | undefined): ParsedDescriptor { + if (!value) { + return { + valid: false, + queryKind: 'unknown', + queryMode: null, + primarySchemaId: null + } + } + + try { + const descriptor = JSON.parse(value) as SavedViewDescriptor + const validation = validateSavedViewDescriptor(descriptor) + const query = descriptor.query as Record + const queryKind = typeof query.kind === 'string' ? query.kind : 'unknown' + const queryMode = typeof query.mode === 'string' ? query.mode : null + const primarySchemaId = + queryKind === 'query-set' ? primarySchemaIdForQuerySet(query) : primarySchemaIdForQuery(query) + + return { + valid: validation.valid, + queryKind, + queryMode, + primarySchemaId + } + } catch { + return { + valid: false, + queryKind: 'invalid-json', + queryMode: null, + primarySchemaId: null + } + } +} + +function parseSavedViewDescriptorObject(value: string | undefined): SavedViewDescriptor | null { + if (!value) return null + + try { + const descriptor = JSON.parse(value) as SavedViewDescriptor + return validateSavedViewDescriptor(descriptor).valid ? descriptor : null + } catch { + return null + } +} + +function primarySchemaIdForQuery(query: Record): string | null { + const schema = query.schema as Record | undefined + return typeof schema?.id === 'string' + ? schema.id + : typeof schema?.['@id'] === 'string' + ? schema['@id'] + : typeof query.schemaId === 'string' + ? query.schemaId + : null +} + +function primarySchemaIdForQuerySet(query: Record): string | null { + const queries = query.queries as Record> | undefined + const firstQuery = queries ? Object.values(queries)[0] : null + return firstQuery ? primarySchemaIdForQuery(firstQuery) : null +} + +function metricValueLabel(value: number | null): string { + return value === null ? '-' : value.toLocaleString() +} + +function sumKnownCounts(values: readonly (number | null)[]): number { + return values.reduce((total, value) => total + (value ?? 0), 0) +} + +function descriptorKindLabel(descriptor: ParsedDescriptor): string { + if (!descriptor.valid) return 'Invalid' + if (descriptor.queryKind === 'query-set') return descriptor.queryMode ?? 'query set' + return descriptor.queryKind +} + +// ─── Social import job helpers ───────────────────────────────────────────────── + +function isVisibleSocialImportJob(job: SocialImportJobProgress): boolean { + if (job.status !== 'completed') return true + return Date.now() - job.updatedAt < 5 * 60 * 1000 +} + +function socialImportJobPercent(job: SocialImportJobProgress): number { + if (!job.totalRecords || job.totalRecords <= 0) return job.status === 'completed' ? 100 : 0 + return Math.min(100, Math.max(0, (job.processedRecords / job.totalRecords) * 100)) +} + +function socialImportJobStatusLabel(job: SocialImportJobProgress): string { + if (job.status === 'queued') return 'Queued' + if (job.status === 'running') return 'Running' + if (job.status === 'paused') return 'Paused' + if (job.status === 'completed') return 'Complete' + if (job.status === 'failed') return 'Failed' + return 'Cancelled' +} + +function socialImportJobRecordLabel(job: SocialImportJobProgress): string { + if (!job.totalRecords) return job.processedRecords.toLocaleString() + return `${job.processedRecords.toLocaleString()} / ${job.totalRecords.toLocaleString()}` +} + +function socialImportJobRateLabel(job: SocialImportJobProgress): string { + const recordsPerSecond = job.metrics?.recordsPerSecond ?? 0 + if (!Number.isFinite(recordsPerSecond) || recordsPerSecond <= 0) return '0/s' + return `${Math.round(recordsPerSecond).toLocaleString()}/s` +} + +// ─── Dismissed patterns (localStorage) ───────────────────────────────────────── + +function readDismissedPatternIds(): string[] { + if (typeof localStorage === 'undefined') return [] + + try { + const value = JSON.parse(localStorage.getItem(DISMISSED_PATTERN_STORAGE_KEY) ?? '[]') + return Array.isArray(value) + ? value.flatMap((item) => (typeof item === 'string' ? [item] : [])) + : [] + } catch { + return [] + } +} + +function writeDismissedPatternIds(ids: readonly string[]): void { + if (typeof localStorage === 'undefined') return + + localStorage.setItem(DISMISSED_PATTERN_STORAGE_KEY, JSON.stringify([...new Set(ids)].sort())) +} + +function toPatternRows(rows: readonly unknown[]): Record[] { + return rows as unknown as Record[] +} + +function patternIconFor(kind: SocialPatternKind): typeof BarChart3 { + if (kind === 'privacy-hotspots') return Shield + if (kind === 'cross-source-overlap') return Search + if (kind === 'bridge-actors') return Network + if (kind === 'unrevisited-saves') return Import + if (kind === 'attention-bursts') return BarChart3 + return BarChart3 +} + +// ─── Hook ────────────────────────────────────────────────────────────────────── + +export type UseDataWorkspaceOptions = { + /** Resolve an existing node by deterministic id during seeding (platform storage differs). */ + getExistingNode: (id: string) => Promise + /** Gate seeding until the platform store is ready (web OPFS store). Default true. */ + seedReady?: boolean + /** Desktop-only: insert a saved lens onto the canvas as a frame. */ + onInsertSavedLensAsCanvasFrame?: (input: SavedViewCanvasFrameInput) => void +} + +export type UseDataWorkspaceResult = { + // Seeding / status + seedReady: boolean + seeding: boolean + seedSummary: SocialWorkspaceSeedSummary | null + seedError: string | null + saveLensMessage: string | null + saveLensError: string | null + handleSeedWorkspace: () => Promise + + // Saved views + savedViewsLoading: boolean + savedViewCount: number + defaultSeedCount: number + importRunCount: number | null + socialWorkspaceViews: SavedViewRow[] + otherSavedViews: SavedViewRow[] + selectedView: SavedViewRow | null + setSelectedViewId: (viewId: string | null) => void + + // Derived data + metrics: WorkspaceMetric[] + analyticsCacheRecommendation: SocialAnalyticsCacheRecommendation + patternSuggestions: SocialPatternSuggestion[] + graphAtlasRows: GraphAtlasRow[] + visibleSocialImportJobs: SocialImportJobProgress[] + /** Re-read the module-level import-job store (e.g. after an IPC upsert). */ + refreshSocialImportJobs: () => void + + // Lens / pattern actions + handleSaveLens: (draft: SavedViewLensDraft) => Promise + handleOpenPattern: (pattern: SocialPatternSuggestion) => void + handleSavePattern: (pattern: SocialPatternSuggestion) => Promise + handlePinPattern: (pattern: SocialPatternSuggestion) => Promise + handleOpenVisualCanvasProjection: (request: SavedViewVisualCanvasProjectionRequest) => void + handleDismissPattern: (patternId: string) => void + + /** Passed through so the body can render canvas-frame affordances. */ + onInsertSavedLensAsCanvasFrame?: (input: SavedViewCanvasFrameInput) => void +} + +export function useDataWorkspace({ + getExistingNode, + seedReady = true, + onInsertSavedLensAsCanvasFrame +}: UseDataWorkspaceOptions): UseDataWorkspaceResult { + const { create, mutate } = useMutate() + const [socialImportJobs, setSocialImportJobs] = + useState(listSocialImportJobs) + const [seedSummary, setSeedSummary] = useState(null) + const [seeding, setSeeding] = useState(false) + const [seedError, setSeedError] = useState(null) + const [saveLensMessage, setSaveLensMessage] = useState(null) + const [saveLensError, setSaveLensError] = useState(null) + const [selectedViewId, setSelectedViewId] = useState(null) + const [dismissedPatternIds, setDismissedPatternIds] = useState(readDismissedPatternIds) + const { data: savedViews, loading: savedViewsLoading } = useQuery(SavedViewSchema, { + orderBy: { title: 'asc' }, + limit: 200 + }) + const actorQuery = useQuery(SocialActorSchema, { page: { first: 1, count: 'estimate' } }) + const contentQuery = useQuery(SocialContentSchema, { + page: { first: PATTERN_QUERY_LIMIT, count: 'estimate' }, + orderBy: { importedAt: 'desc' } + }) + const interactionQuery = useQuery(SocialInteractionSchema, { + page: { first: PATTERN_QUERY_LIMIT, count: 'estimate' }, + orderBy: { importedAt: 'desc' } + }) + const messageQuery = useQuery(SocialMessageSchema, { page: { first: 1, count: 'estimate' } }) + const conversationQuery = useQuery(SocialConversationSchema, { + page: { first: 1, count: 'estimate' } + }) + const collectionQuery = useQuery(SocialCollectionSchema, { + page: { first: 1, count: 'estimate' } + }) + const importRunQuery = useQuery(SocialImportRunSchema, { + page: { first: 50, count: 'estimate' }, + orderBy: { startedAt: 'desc' } + }) + + const defaultSeeds = useMemo(() => getDefaultSocialWorkspaceSeeds(), []) + const defaultSeedIds = useMemo( + () => new Set(defaultSeeds.map((seed) => seed.deterministicId)), + [defaultSeeds] + ) + const defaultSeedBySourceId = useMemo( + () => new Map(defaultSeeds.map((seed) => [seed.id, seed])), + [defaultSeeds] + ) + const graphAtlasEntries = useMemo(() => createDefaultSocialGraphAtlas({ pageSize: 100 }), []) + const socialWorkspaceViews = useMemo( + () => (savedViews as SavedViewRow[]).filter((view) => defaultSeedIds.has(view.id)), + [defaultSeedIds, savedViews] + ) + const otherSavedViews = useMemo( + () => (savedViews as SavedViewRow[]).filter((view) => !defaultSeedIds.has(view.id)), + [defaultSeedIds, savedViews] + ) + const allSavedViews = useMemo( + () => [...socialWorkspaceViews, ...otherSavedViews], + [otherSavedViews, socialWorkspaceViews] + ) + const selectedView = useMemo( + () => + allSavedViews.find((view) => view.id === selectedViewId) ?? + socialWorkspaceViews[0] ?? + allSavedViews[0] ?? + null, + [allSavedViews, selectedViewId, socialWorkspaceViews] + ) + const metrics: WorkspaceMetric[] = [ + { + id: 'actors', + label: 'People', + value: getCount(actorQuery), + icon: UserRound + }, + { + id: 'content', + label: 'Content', + value: getCount(contentQuery), + icon: Table + }, + { + id: 'interactions', + label: 'Interactions', + value: getCount(interactionQuery), + icon: Network + }, + { + id: 'messages', + label: 'Messages', + value: getCount(messageQuery), + icon: MessageSquare + }, + { + id: 'conversations', + label: 'Conversations', + value: getCount(conversationQuery), + icon: GitBranch + }, + { + id: 'collections', + label: 'Collections', + value: getCount(collectionQuery), + icon: Database + }, + { + id: 'import-runs', + label: 'Import Runs', + value: getCount(importRunQuery), + icon: Import + } + ] + const analyticsCacheRecommendation = recommendSocialAnalyticsCache({ + rowCount: sumKnownCounts(metrics.map((metric) => metric.value)), + columnCount: 12, + relationCount: getCount(interactionQuery) ?? 0 + }) + const dismissedPatternIdSet = useMemo(() => new Set(dismissedPatternIds), [dismissedPatternIds]) + const patternSuggestions = useMemo( + () => + detectSocialPatterns({ + content: toPatternRows(contentQuery.data), + interactions: toPatternRows(interactionQuery.data), + importRuns: toPatternRows(importRunQuery.data) + }).filter((pattern) => !dismissedPatternIdSet.has(pattern.id)), + [contentQuery.data, dismissedPatternIdSet, importRunQuery.data, interactionQuery.data] + ) + const graphAtlasRows = useMemo( + () => + graphAtlasEntries.map((entry) => { + const seed = defaultSeedBySourceId.get(entry.id) + const savedView = seed + ? (socialWorkspaceViews.find((view) => view.id === seed.deterministicId) ?? null) + : null + + return { entry, savedView } + }), + [defaultSeedBySourceId, graphAtlasEntries, socialWorkspaceViews] + ) + const visibleSocialImportJobs = useMemo( + () => socialImportJobs.filter(isVisibleSocialImportJob).slice(0, 3), + [socialImportJobs] + ) + + useEffect(() => { + if (!selectedViewId && selectedView) { + setSelectedViewId(selectedView.id) + return + } + + if (selectedViewId && !allSavedViews.some((view) => view.id === selectedViewId)) { + setSelectedViewId(selectedView?.id ?? null) + } + }, [allSavedViews, selectedView, selectedViewId]) + + const refreshSocialImportJobs = useCallback(() => setSocialImportJobs(listSocialImportJobs()), []) + + useEffect(() => subscribeSocialImportJobs(refreshSocialImportJobs), [refreshSocialImportJobs]) + + async function handleSeedWorkspace(): Promise { + if (!seedReady) return + + setSeeding(true) + setSeedError(null) + + try { + const summary = await upsertDefaultSocialWorkspace({ + mutate, + getExisting: getExistingNode + }) + setSeedSummary(summary) + } catch (error) { + setSeedError(error instanceof Error ? error.message : String(error)) + } finally { + setSeeding(false) + } + } + + async function handleSaveLens(draft: SavedViewLensDraft): Promise { + setSaveLensMessage(null) + setSaveLensError(null) + + try { + const savedView = await create(SavedViewSchema, { + title: draft.title, + description: draft.description, + descriptor: JSON.stringify(draft.descriptor), + scope: draft.descriptor.scope ?? 'workspace' + }) + + if (!savedView) { + throw new Error('Saved lens could not be created.') + } + + setSelectedViewId(savedView.id) + setSaveLensMessage(`Saved lens: ${draft.title}.`) + } catch (error) { + setSaveLensError(error instanceof Error ? error.message : String(error)) + throw error + } + } + + function handleOpenPattern(pattern: SocialPatternSuggestion): void { + const view = socialWorkspaceViews.find((candidate) => candidate.title === pattern.viewHint) + if (view) { + setSelectedViewId(view.id) + } + } + + async function upsertPatternSavedView( + pattern: SocialPatternSuggestion + ): Promise { + setSaveLensMessage(null) + setSaveLensError(null) + + const baseView = socialWorkspaceViews.find((candidate) => candidate.title === pattern.viewHint) + const baseDescriptor = parseSavedViewDescriptorObject(baseView?.descriptor) + + if (!baseView || !baseDescriptor) { + setSaveLensError(`Seed the ${pattern.viewHint} view before saving this pattern.`) + return null + } + + const draft = createSocialPatternSavedViewDraft({ pattern, baseDescriptor }) + if (!draft) { + setSaveLensError('Pattern lens could not be created from the base view.') + return null + } + + const existing = allSavedViews.some((view) => view.id === draft.deterministicId) + const operation: MutateOp = existing + ? { + type: 'update', + id: draft.deterministicId, + data: draft.savedViewProperties + } + : { + type: 'create', + id: draft.deterministicId, + schema: SavedViewSchema, + data: draft.savedViewProperties + } + + await mutate([operation]) + + const savedView = { + id: draft.deterministicId, + ...draft.savedViewProperties + } + setSelectedViewId(savedView.id) + setSaveLensMessage(`${existing ? 'Updated' : 'Saved'} pattern lens: ${draft.title}.`) + return savedView + } + + async function handleSavePattern(pattern: SocialPatternSuggestion): Promise { + await upsertPatternSavedView(pattern) + } + + async function handlePinPattern(pattern: SocialPatternSuggestion): Promise { + if (!onInsertSavedLensAsCanvasFrame) return + + const savedView = await upsertPatternSavedView(pattern) + if (!savedView) return + + onInsertSavedLensAsCanvasFrame(savedView) + } + + function handleOpenVisualCanvasProjection(request: SavedViewVisualCanvasProjectionRequest): void { + if (!onInsertSavedLensAsCanvasFrame) return + + const descriptorJson = + typeof request.descriptor === 'string' + ? request.descriptor + : request.descriptor + ? JSON.stringify(request.descriptor) + : selectedView?.descriptor + + onInsertSavedLensAsCanvasFrame({ + id: selectedView?.id ?? request.id, + title: request.title, + ...(request.description ? { description: request.description } : {}), + ...(descriptorJson ? { descriptor: descriptorJson } : {}) + }) + } + + function handleDismissPattern(patternId: string): void { + setDismissedPatternIds((current) => { + const next = [...new Set([...current, patternId])] + writeDismissedPatternIds(next) + return next + }) + } + + return { + seedReady, + seeding, + seedSummary, + seedError, + saveLensMessage, + saveLensError, + handleSeedWorkspace, + savedViewsLoading, + savedViewCount: savedViews.length, + defaultSeedCount: defaultSeeds.length, + importRunCount: getCount(importRunQuery), + socialWorkspaceViews, + otherSavedViews, + selectedView, + setSelectedViewId, + metrics, + analyticsCacheRecommendation, + patternSuggestions, + graphAtlasRows, + visibleSocialImportJobs, + refreshSocialImportJobs, + handleSaveLens, + handleOpenPattern, + handleSavePattern, + handlePinPattern, + handleOpenVisualCanvasProjection, + handleDismissPattern, + onInsertSavedLensAsCanvasFrame + } +} + +// ─── Body ────────────────────────────────────────────────────────────────────── + +export type DataWorkspaceBodyProps = { + workspace: UseDataWorkspaceResult + /** Web-only SavedViewRunner extras: feed enrichment + moderation gate. */ + savedViewRunnerProps?: Pick +} + +/** + * The shared Data Workspace surface: status banners, import jobs, metric + * cards, graph atlas, and the sources/patterns + saved-views grid. The host + * component provides page chrome (header, seed button, scroll container). + */ +export function DataWorkspaceBody({ + workspace, + savedViewRunnerProps +}: DataWorkspaceBodyProps): ReactElement { + const { + seedSummary, + seedError, + saveLensMessage, + saveLensError, + visibleSocialImportJobs, + metrics, + graphAtlasRows, + selectedView, + setSelectedViewId, + importRunCount, + savedViewCount, + defaultSeedCount, + socialWorkspaceViews, + otherSavedViews, + analyticsCacheRecommendation, + patternSuggestions, + savedViewsLoading, + handleSaveLens, + handleOpenPattern, + handleSavePattern, + handlePinPattern, + handleOpenVisualCanvasProjection, + handleDismissPattern, + onInsertSavedLensAsCanvasFrame + } = workspace + + return ( + <> + {seedSummary ? ( + + ) : null} + {seedError ? : null} + {saveLensMessage ? : null} + {saveLensError ? : null} + + +
+ {metrics.map((metric) => { + const Icon = metric.icon + + return ( +
+
+ {metric.label} + +
+
{metricValueLabel(metric.value)}
+
+ ) + })} +
+ + setSelectedViewId(view.id)} + onInsertCanvasFrame={onInsertSavedLensAsCanvasFrame} + /> + +
+ + +
+
+
+
+

Social Starter Lenses

+

+ Schema views and graph-lens query sets persisted as saved views. +

+
+ {savedViewsLoading ? ( +
+ + Loading +
+ ) : null} +
+ +
+ + + +
+
+

Other Saved Views

+

+ General saved views will use the same workspace surface as more importers land. +

+
+ +
+
+
+ + ) +} + +// ─── Subcomponents ───────────────────────────────────────────────────────────── + +function SocialImportJobsPanel({ jobs }: { jobs: SocialImportJobProgress[] }): ReactElement | null { + if (jobs.length === 0) return null + + return ( +
+ +
+ {jobs.map((job) => { + const percent = socialImportJobPercent(job) + const statusLabel = socialImportJobStatusLabel(job) + + return ( +
+
+
+
+ {job.status === 'running' || job.status === 'queued' ? ( + + ) : ( + + )} +
{job.archiveName}
+
+
+ {job.platform} / {statusLabel} / {job.phase} +
+
+
+ {Math.floor(percent)}% +
+
+
+
+
+
+ + + + +
+ {job.error ? ( +
+ + {job.error} +
+ ) : null} +
+ ) + })} +
+
+ ) +} + +function JobMetric({ label, value }: { label: string; value: string }): ReactElement { + return ( +
+
{label}
+
{value}
+
+ ) +} + +function SavedViewTable({ + views, + selectedViewId, + emptyLabel, + onSelect, + onInsertCanvasFrame +}: { + views: SavedViewRow[] + selectedViewId: string | null + emptyLabel: string + onSelect: (viewId: string) => void + onInsertCanvasFrame?: (input: SavedViewCanvasFrameInput) => void +}): ReactElement { + if (views.length === 0) { + return ( +
+ {emptyLabel} +
+ ) + } + + return ( +
+ + + + + + + + {onInsertCanvasFrame ? : null} + + + + {views.map((view) => { + const descriptor = parseSavedViewDescriptor(view.descriptor) + const selected = view.id === selectedViewId + + return ( + + + + + + {onInsertCanvasFrame ? ( + + ) : null} + + ) + })} + +
ViewKindScopeSchemaCanvas
+ + + {descriptorKindLabel(descriptor)} + {view.scope ?? '-'} + {descriptor.primarySchemaId ?? '-'} + + +
+
+ ) +} + +function GraphAtlasPanel({ + rows, + selectedViewId, + onOpen, + onInsertCanvasFrame +}: { + rows: GraphAtlasRow[] + selectedViewId: string | null + onOpen: (view: SavedViewRow) => void + onInsertCanvasFrame?: (input: SavedViewCanvasFrameInput) => void +}): ReactElement { + return ( +
+
+
+

Graph Atlas

+

+ Starter graph lenses organized by node roles, relationship rules, and saved-view state. +

+
+ + {rows.filter((row) => row.savedView).length}/{rows.length} seeded + +
+
+ {rows.map((row) => ( + + ))} +
+
+ ) +} + +function GraphAtlasCard({ + row, + selected, + onOpen, + onInsertCanvasFrame +}: { + row: GraphAtlasRow + selected: boolean + onOpen: (view: SavedViewRow) => void + onInsertCanvasFrame?: (input: SavedViewCanvasFrameInput) => void +}): ReactElement { + const { entry, savedView } = row + + return ( +
+
+
+
+ +

{entry.title}

+
+

{entry.description}

+
+ + {savedView ? 'saved' : 'seed'} + +
+
+ + + +
+
+ {entry.nodeRoles.slice(0, 3).map((role) => ( + + {role.role} + + ))} + {entry.relationshipKinds.slice(0, 3).map((kind) => ( + + {kind} + + ))} +
+
+ + {onInsertCanvasFrame ? ( + + ) : null} +
+
+ ) +} + +function GraphAtlasMetric({ label, value }: { label: string; value: number }): ReactElement { + return ( +
+
{label}
+
{value.toLocaleString()}
+
+ ) +} + +function SectionLabel({ label }: { label: string }): ReactElement { + return ( +
{label}
+ ) +} + +function SourceRow({ label, value }: { label: string; value: string }): ReactElement { + return ( +
+ {label} + {value} +
+ ) +} + +function AnalyticsCacheRow({ + recommendation +}: { + recommendation: SocialAnalyticsCacheRecommendation +}): ReactElement { + return ( +
+
+ Scale cache + {recommendation.label} +
+

{recommendation.reason}

+
+ + {recommendation.estimatedRows.toLocaleString()} rows + + + {recommendation.estimatedCells.toLocaleString()} cells + +
+
+ ) +} + +function PatternRow({ + icon: Icon, + pattern, + onOpen, + onSave, + onPin, + onDismiss +}: { + icon: typeof BarChart3 + pattern: SocialPatternSuggestion + onOpen: (pattern: SocialPatternSuggestion) => void + onSave: (pattern: SocialPatternSuggestion) => void + onPin?: (pattern: SocialPatternSuggestion) => void + onDismiss: (patternId: string) => void +}): ReactElement { + return ( +
+
+ +
+
{pattern.title}
+
+ {pattern.description} +
+
+
+
+ + {pattern.evidenceCount.toLocaleString()} evidence + + {pattern.platforms.slice(0, 2).map((platform) => ( + + {platform} + + ))} + {pattern.privacyClasses.slice(0, 2).map((privacyClass) => ( + + {privacyClass} + + ))} + {pattern.sourceImportRunIds.length > 0 ? ( + + {pattern.sourceImportRunIds.length} runs + + ) : null} +
+ {pattern.evidence.length > 0 ? ( +
+ {pattern.evidence.slice(0, 2).map((item) => ( +
+ {item.value} + {item.count.toLocaleString()} +
+ ))} +
+ ) : null} +
+ + + {onPin ? ( + + ) : null} + +
+
+ ) +} + +function StatusBanner({ + message, + tone +}: { + message: string + tone: 'error' | 'success' | 'warning' +}): ReactElement { + const toneClassName = { + error: 'border-destructive/40 bg-destructive/10 text-destructive', + success: 'border-emerald-500/30 bg-emerald-500/10 text-emerald-700 dark:text-emerald-300', + warning: 'border-amber-500/30 bg-amber-500/10 text-amber-700 dark:text-amber-300' + }[tone] + + const Icon = tone === 'success' ? Shield : AlertTriangle + + return ( +
+ + {message} +
+ ) +} diff --git a/packages/views/src/data-workspace/index.ts b/packages/views/src/data-workspace/index.ts new file mode 100644 index 000000000..0f4b75bfc --- /dev/null +++ b/packages/views/src/data-workspace/index.ts @@ -0,0 +1,16 @@ +export { + DataWorkspaceBody, + useDataWorkspace, + type DataWorkspaceBodyProps, + type GraphAtlasRow, + type SavedViewCanvasFrameInput, + type SavedViewRow, + type UseDataWorkspaceOptions, + type UseDataWorkspaceResult, + type WorkspaceMetric +} from './DataWorkspaceCore.js' +export { + getDefaultSocialWorkspaceSeeds, + upsertDefaultSocialWorkspace, + type SocialWorkspaceSeedSummary +} from './social-workspace.js' diff --git a/packages/views/src/data-workspace/social-workspace.ts b/packages/views/src/data-workspace/social-workspace.ts new file mode 100644 index 000000000..0fb2674c6 --- /dev/null +++ b/packages/views/src/data-workspace/social-workspace.ts @@ -0,0 +1,64 @@ +/** + * Social workspace seeding — deterministic saved-view seeds over the social + * import schemas. Extracted verbatim from the (identical) web and desktop + * `lib/social-workspace.ts` copies (exploration 0276, Theme 3). + */ +import type { MutateOp } from '@xnetjs/react' +import { SavedViewSchema } from '@xnetjs/data' +import { createDefaultSocialWorkspaceSavedViewSeeds } from '@xnetjs/social/workspace' + +export type SocialWorkspaceSeedSummary = { + created: number + updated: number + total: number +} + +type SocialWorkspaceSeedOperationResult = { + action: 'created' | 'updated' + operation: MutateOp +} + +export function getDefaultSocialWorkspaceSeeds() { + return createDefaultSocialWorkspaceSavedViewSeeds({ pageSize: 100 }) +} + +export async function upsertDefaultSocialWorkspace(input: { + mutate: (ops: MutateOp[]) => Promise + getExisting: (id: string) => Promise +}): Promise { + const seeds = getDefaultSocialWorkspaceSeeds() + const operationResults = await Promise.all( + seeds.map(async (seed): Promise => { + const existing = await input.getExisting(seed.deterministicId) + if (existing) { + return { + action: 'updated', + operation: { + type: 'update', + id: seed.deterministicId, + data: seed.savedViewProperties + } + } + } + + return { + action: 'created', + operation: { + type: 'create', + id: seed.deterministicId, + schema: SavedViewSchema, + data: seed.savedViewProperties + } as MutateOp + } + }) + ) + + const operations = operationResults.map((result) => result.operation) + await input.mutate(operations) + + return { + created: operationResults.filter((result) => result.action === 'created').length, + updated: operationResults.filter((result) => result.action === 'updated').length, + total: seeds.length + } +} diff --git a/packages/views/src/index.ts b/packages/views/src/index.ts index fc3747007..7d9af9a6b 100644 --- a/packages/views/src/index.ts +++ b/packages/views/src/index.ts @@ -299,3 +299,20 @@ export { type SchemaToFormOptions, type SchemaFormProps } from './form/index.js' + +// Shared Data Workspace core (exploration 0276): the saved-view / graph-atlas +// workspace surface both the web and desktop DataWorkspaceViews consume. +export { + DataWorkspaceBody, + useDataWorkspace, + getDefaultSocialWorkspaceSeeds, + upsertDefaultSocialWorkspace, + type DataWorkspaceBodyProps, + type GraphAtlasRow, + type SavedViewCanvasFrameInput, + type SavedViewRow, + type SocialWorkspaceSeedSummary, + type UseDataWorkspaceOptions, + type UseDataWorkspaceResult, + type WorkspaceMetric +} from './data-workspace/index.js' diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0cadf2e0f..5ccb98b32 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1162,6 +1162,9 @@ importers: '@xnetjs/data': specifier: workspace:* version: link:../data + '@xnetjs/react': + specifier: workspace:* + version: link:../react '@xnetjs/ui': specifier: workspace:* version: link:../ui @@ -2132,6 +2135,9 @@ importers: '@xnetjs/react': specifier: workspace:* version: link:../react + '@xnetjs/social': + specifier: workspace:* + version: link:../social '@xnetjs/ui': specifier: workspace:* version: link:../ui diff --git a/scripts/check-view-drift.mjs b/scripts/check-view-drift.mjs new file mode 100644 index 000000000..cdd5f5307 --- /dev/null +++ b/scripts/check-view-drift.mjs @@ -0,0 +1,104 @@ +#!/usr/bin/env node +/** + * Warn when one side of a known web ↔ desktop component fork changes without + * its twin (exploration 0276, Theme 3). + * + * A handful of components exist as deliberate forks in BOTH + * apps/web/src/components and apps/electron/src/renderer/components. Their + * shared logic now lives in packages (usePageComments, DataWorkspaceBody, …), + * but the per-app chrome still comes in pairs — and history shows the pairs + * drift silently: PageView reached ~93% identical copies with ZERO shared + * commits before 0276 extracted the common core. + * + * This tripwire looks at a git diff and, when it touches one side of a known + * pair but not the other, prints a WARNING naming the untouched twin. It is + * advisory by default (exit 0) so intentionally one-sided changes stay cheap; + * `--strict` turns warnings into failures for use as a gate. + * + * Usage: + * node scripts/check-view-drift.mjs # staged changes (git diff --cached) + * node scripts/check-view-drift.mjs --base main # changes vs. a base ref + * node scripts/check-view-drift.mjs --strict # exit 1 on warnings + * + * (or `pnpm check:view-drift`) + */ +import { execFileSync } from 'node:child_process' + +const WEB_COMPONENTS = 'apps/web/src/components' +const ELECTRON_COMPONENTS = 'apps/electron/src/renderer/components' + +// Known duplicated pairs: .tsx exists (on purpose) in both component +// trees. Add here when a new deliberate fork lands; remove when a fork is +// dissolved into a shared package. +const PAIRED_COMPONENTS = [ + 'PageView', + 'DataWorkspaceView', + 'CanvasView', + 'DatabaseView', + 'PluginManager', + 'AddSharedDialog', + 'ShareButton', + 'PresenceAvatars', + 'BundledPluginInstaller' +] + +// ── CLI args ───────────────────────────────────────────────────────────────── +const args = process.argv.slice(2) +const strict = args.includes('--strict') +const baseIndex = args.indexOf('--base') +const baseRef = baseIndex !== -1 ? args[baseIndex + 1] : null +if (baseIndex !== -1 && !baseRef) { + console.error('✗ view drift: --base requires a git ref argument') + process.exit(2) +} + +// ── Collect changed files ──────────────────────────────────────────────────── +function changedFiles() { + const diffArgs = baseRef + ? ['diff', '--name-only', `${baseRef}...HEAD`] + : ['diff', '--name-only', '--cached'] + try { + return execFileSync('git', diffArgs, { encoding: 'utf8' }) + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + } catch (error) { + console.error(`✗ view drift: git ${diffArgs.join(' ')} failed: ${error.message}`) + process.exit(2) + } +} + +const changed = new Set(changedFiles()) + +// ── Check pairs ────────────────────────────────────────────────────────────── +const warnings = [] +for (const name of PAIRED_COMPONENTS) { + const webPath = `${WEB_COMPONENTS}/${name}.tsx` + const electronPath = `${ELECTRON_COMPONENTS}/${name}.tsx` + const webChanged = changed.has(webPath) + const electronChanged = changed.has(electronPath) + + if (webChanged && !electronChanged) { + warnings.push(`${name}: ${webPath} changed but its desktop twin (${electronPath}) did not`) + } else if (electronChanged && !webChanged) { + warnings.push(`${name}: ${electronPath} changed but its web twin (${webPath}) did not`) + } +} + +// ── Report ─────────────────────────────────────────────────────────────────── +const source = baseRef ? `vs ${baseRef}` : 'staged' +if (warnings.length > 0) { + for (const w of warnings) console.warn(`⚠ view drift (${source}): ${w}`) + console.warn( + ' → These components are known web/desktop forks (0276). If the change is\n' + + ' shared behavior, port it to the twin (or better: lift it into the\n' + + ' shared core in packages/editor or packages/views). If it is truly\n' + + ' platform-specific, ignore this warning.' + ) + process.exit(strict ? 1 : 0) +} + +console.log( + `✓ view drift OK (${source}) — ${PAIRED_COMPONENTS.length} paired components checked` +) +process.exit(0) diff --git a/site/public/llms-full.txt b/site/public/llms-full.txt index acc35eb5a..83006b127 100644 --- a/site/public/llms-full.txt +++ b/site/public/llms-full.txt @@ -39,6 +39,7 @@ Before reading this documentation, understand that xNet works differently: - Implement XNet in Your Language - Conformance - The Workbench + - The Quiet Shell & the Desk - Tasks - Dashboards & Widgets - Chat, Presence & Calls @@ -3326,6 +3327,85 @@ See the [Plugins guide](/docs/guides/plugins/) for the contribution API. --- +## The Quiet Shell & the Desk + +**You will learn** + +- What the quiet chrome posture is and how to switch into it +- The disclosure ladder: corners, edges, chords, and Esc +- The Desk — your bounded home canvas — and how pinning works +- The SurfaceDock, and how it maps to the pinned shell's tray + + +## Overview + +The quiet shell inverts the calm shell's composition: at rest the screen is +just your work — a document, a database, or the Desk canvas — and the chrome +is *summoned*, never pinned. Glyph clusters sit dimmed in the corners; the +navigator and the contextual canvas slide in from the edges; the dock expands +from the bottom-right the way devtools always has. Nothing is removed — every +drawer is reachable three ways: pointer (hover an edge, or swipe on a phone), +keyboard chord, or the ⌘K palette. + +Switch postures any time with **⌘K → "View: Quiet chrome"** (and back with +"View: Pinned chrome"). The setting is per-device and changes nothing about +your data or routes. + +## The disclosure ladder + +| Level | What you see | How you got here | +| ----- | --------------------------- | -------------------------------------------- | +| L0 | Bare surface, dimmed glyphs | Rest state | +| L1 | Glyphs lit | Pointer near an edge, or a touch | +| L2 | One overlay open | Edge hover-dwell, swipe, ⌘B / ⌘\ / ⌘J / ⌘K | +| L3 | Pinned chrome | "View: Pinned chrome" or the workbench grid | + +**Esc always walks down one rung.** Dismissing an overlay returns you to the +bare surface; it never dead-ends. + +| Surface | Pointer / touch | Chord | Palette | +| --------- | ------------------------ | ----- | ------------------- | +| Navigator | left edge hover / swipe | ⌘B | Toggle left panel | +| Context | right edge hover / swipe | ⌘\ | Toggle right panel | +| Dock | corner launcher / FAB | ⌘J | `Dock: ` | +| Palette | corner search glyph | ⌘K | — | + +## The Desk + +The Desk is a personal home canvas with a deterministic, identity-derived id — +it is created the first time you visit it, and the same identity gets the same +Desk on every device. It is **bounded but growable**: panning clamps near your +content (you can't strand yourself in empty space), and the board grows as +cards land outside it. **Fit to content** (Ctrl/Cmd 1) is the home anchor. + +Build your workspace by pinning: + +- **Drag** anything from the navigator onto the Desk. +- **Pin to Desk** from a document via ⌘K, or the desk icon that appears when + you hover a navigator row. Pins queue up and land the next time the Desk is + on screen — you don't have to be looking at it. +- **Go to Desk** (`g k`) from anywhere. + +Pinned cards are live views of the real nodes — edit the source and the card +follows. On a phone the Desk renders as an ordered list (reading order of the +board), which is also how screen readers traverse it. + +## The SurfaceDock + +The bottom-right launcher is the devtools grammar for everyone: a single +glyph that expands into the *hero* panels (Shelf, Capture, Notifications) with +the rest behind **More** and the palette. It reuses the pinned shell's bottom +tray state, so ⌘J toggles the same panels in either posture. Plugins add +panels through the `surfaceDock` contribution point in `@xnetjs/plugins`. + +## Rollout + +Quiet chrome ships opt-in. New identities boot straight to the Desk under +quiet chrome once the staged-rollout flag flips; existing workspaces keep +whatever posture they chose. + +--- + ## Tasks **You will learn** diff --git a/site/src/data/changelog/2026-07-06-faster-safer-foundations-the-hottest-cod.json b/site/src/data/changelog/2026-07-06-faster-safer-foundations-the-hottest-cod.json new file mode 100644 index 000000000..215cd6f36 --- /dev/null +++ b/site/src/data/changelog/2026-07-06-faster-safer-foundations-the-hottest-cod.json @@ -0,0 +1,8 @@ +{ + "id": "2026-07-06-faster-safer-foundations-the-hottest-cod", + "date": "July 6, 2026", + "title": "Faster, safer foundations: the hottest code paths got a deep clean", + "summary": "The storage engine, sync server, and app shells were reorganized into smaller, independently tested modules — same behavior, one shared conflict-resolution rule everywhere, and page comments now share one implementation across web and desktop.", + "highlights": [], + "tags": ["platform", "performance"] +}