diff --git a/src/components/menu/Inspector.svelte b/src/components/menu/Inspector.svelte
index 2337729d..ca9425f4 100644
--- a/src/components/menu/Inspector.svelte
+++ b/src/components/menu/Inspector.svelte
@@ -32,7 +32,7 @@
import { LIGHT_PARAMS, SHADOW_TYPES, SHADOW_SIZES, setShadowMapSize, cappedShadowSize } from '$lib/lightParams';
import { animatedObjects, setAnimationState } from '$lib/animatedImports';
import { moveObjectToGroup, selectObject } from '$lib/objectActions';
- import { listPhysicsObjects, enablePhysicsOnSelection, PHYSICS_MATERIALS, physicsShapeChanged } from '$lib/physics';
+ import { listPhysicsObjects, enablePhysicsOnSelection, setPhysicsFor, PHYSICS_MATERIALS } from '$lib/physics';
import { sceneGravity, setSceneGravity, resetSceneGravity, DEFAULT_GRAVITY } from '$lib/scenePhysics';
import { showColliders, colliderVizObjects, setColliderViz } from '$lib/colliderHelpers';
import { enterColliderEdit } from '$lib/colliderEdit';
@@ -364,13 +364,9 @@
/** @param {any} patch */
function setPhysics(patch) {
- const before = $selectedObject.userData.physics ? { ...$selectedObject.userData.physics } : null;
- const next = { mode: 'auto', ...($selectedObject.userData.physics ?? {}), ...patch };
- $selectedObject.userData.physics = next;
- recordEntry({ kind: 'props', uuid: $selectedObject.uuid, before: { physics: before }, after: { physics: next } });
- $peers.send({ type: 'objectParameters', parameter: 'physics', uuid: $selectedObject.uuid, physics: next });
- objectsGroup.update((v) => v); // collider viz re-syncs from the poke
- physicsShapeChanged($selectedObject.uuid); // CL-A A2: live mid-sim rebuild
+ // shared write path — replicates, records props undo, pokes the collider
+ // viz and live-rebuilds mid-sim colliders (CL-A A2) for EVERY caller
+ setPhysicsFor($selectedObject.uuid, patch);
selectedObject.update((v) => v);
}
diff --git a/src/components/menu/Settings.svelte b/src/components/menu/Settings.svelte
index 69cc6be1..45675750 100644
--- a/src/components/menu/Settings.svelte
+++ b/src/components/menu/Settings.svelte
@@ -81,6 +81,7 @@
let aiFormKey = '';
let aiFormModel = '';
let aiFormStream = true;
+ let aiFormPhysics = false;
let aiFormTemp = '';
let aiTesting = false;
let aiTestResult: { ok: boolean; detail: string; modelOk?: boolean | null; model?: string } | null = null;
@@ -141,6 +142,7 @@
aiApplyPreset();
aiFormKey = '';
aiFormStream = true;
+ aiFormPhysics = false;
aiFormTemp = '';
aiTestResult = null;
aiFormModels = [];
@@ -157,6 +159,7 @@
aiFormKey = p.apiKey;
aiFormModel = p.model;
aiFormStream = p.stream !== false;
+ aiFormPhysics = p.physicsTools === true;
aiFormTemp = typeof p.temperature === 'number' ? String(p.temperature) : '';
aiTestResult = null;
aiFormModels = Array.isArray(p.models) ? p.models : [];
@@ -179,6 +182,7 @@
apiKey: aiFormKey,
model: aiFormModel,
stream: aiFormStream,
+ physicsTools: aiFormPhysics,
temperature: Number.isFinite(temp) ? temp : undefined,
models: aiFormModels
};
@@ -862,6 +866,19 @@
Stream responses
+
+
+ Lets the assistant set physics bodies, attach joints and start the simulation.
+ Multi-step physics is hard for small local models (4B) — recommended for 14B+
+ or hosted models.
+
+
Turn streaming OFF for a self-hosted server whose tool calls only work unstreamed —
diff --git a/src/lib/ai/assistant.js b/src/lib/ai/assistant.js
index 80dbf213..63490a09 100644
--- a/src/lib/ai/assistant.js
+++ b/src/lib/ai/assistant.js
@@ -92,6 +92,16 @@ function toolStatusLabel(name, args) {
if (name === 'group_objects') return 'Grouping objects';
if (name === 'clear_scene') return 'Clearing the scene';
if (name === 'list_scene') return 'Reading the scene';
+ if (name === 'create_flow_nodes') return 'Adding ' + (args?.nodes?.length ?? 0) + ' behavior node(s)';
+ if (name === 'update_flow_nodes') {
+ const removing = args?.remove?.length ?? 0;
+ const updating = args?.updates?.length ?? 0;
+ if (removing && !updating) return 'Removing ' + removing + ' behavior node(s)';
+ return 'Updating ' + updating + ' behavior node(s)';
+ }
+ if (name === 'set_physics') return 'Setting physics on ' + (args?.updates?.length ?? 0) + ' object(s)';
+ if (name === 'create_joints') return 'Attaching ' + (args?.joints?.length ?? 0) + ' joint(s)';
+ if (name === 'control_simulation') return 'Simulation: ' + (args?.action ?? '…');
return name;
}
@@ -105,16 +115,23 @@ function toolStatusLabel(name, args) {
*/
function tallyChanges(name, result, tally) {
if (name === 'list_scene' || !result || typeof result !== 'object') return;
+ // read-only-style: starting/stopping the sim changes no scene content
+ if (name === 'control_simulation') return;
/** @param {any[]} list */
const addAll = (list) => {
for (const entry of list) {
if (!entry || entry.error) continue;
if (typeof entry.uuid === 'string') tally.uuids.add(entry.uuid);
- else tally.other += 1;
+ else tally.other += 1; // flow nodes / joints have no object uuid
}
};
if (Array.isArray(result.created)) return addAll(result.created);
- if (Array.isArray(result.updated)) return addAll(result.updated);
+ if (Array.isArray(result.updated)) {
+ addAll(result.updated);
+ if (typeof result.removed === 'number') tally.other += result.removed;
+ return;
+ }
+ if (Array.isArray(result.joints)) return addAll(result.joints);
if (typeof result.deleted === 'number') {
tally.other += result.deleted;
return;
diff --git a/src/lib/ai/flowTools.js b/src/lib/ai/flowTools.js
new file mode 100644
index 00000000..badeeec4
--- /dev/null
+++ b/src/lib/ai/flowTools.js
@@ -0,0 +1,496 @@
+import { get } from 'svelte/store';
+import { graphOf, graphExists, SCENE_GRAPH } from '../../stores/flowStore';
+import { objectsGroup, lockedObjects } from '../../stores/sceneStore.js';
+import { peers } from '../../stores/appStore.js';
+import {
+ createFlowNode,
+ createFlowEdge,
+ deleteFlowNodes,
+ serializeNode,
+ serializeEdge,
+ setNodeData
+} from '$lib/nodesHandler';
+import { createObjectGraph, recordFlowNodesEntry } from '$lib/flowGraphs';
+import { nodeCatalog, findNodeSpec } from '$lib/nodeCatalog';
+import {
+ setPhysicsFor,
+ toggleSimulation,
+ stopSimulation,
+ pauseSimulation,
+ resetSimulation,
+ simulating,
+ remoteSimulating
+} from '$lib/physics';
+import { createJoint } from '$lib/joints';
+import { activeAiConfig } from './providers.js';
+
+// AI flow + physics executors (assistant v3). The behavior counterpart to
+// tools.js: create/update flow nodes (the @xyflow editor graphs driving
+// per-frame animation) and — gated per provider — physics params, joints and
+// simulation control. Same conventions as tools.js: apply locally + broadcast
+// + record undo history exactly like a human edit; executors never throw
+// (errors come back as {error} so the model can self-correct).
+
+/** Physics is hard for small local models — a per-provider opt-in checkbox
+ * (Settings → AI) gates these tools. @returns {boolean} */
+export function physicsToolsEnabled() {
+ return !!activeAiConfig()?.physicsTools;
+}
+
+/** Node types only usable when physics tools are on — the sim-driving set
+ * (mirrors physics.js PHYSICS_TYPES incl. the CL-C collider override) plus the
+ * physics-DEPENDENT trigger/readout nodes (inert without a running sim, so
+ * offering them while the sim tools are gated off would only mislead). */
+export const PHYSICS_NODE_TYPES = [
+ 'mass',
+ 'bounciness',
+ 'friction',
+ 'angularvelocity',
+ 'motor',
+ 'collider',
+ 'onimpact',
+ 'onenter',
+ 'onexit',
+ 'velocity'
+];
+
+// Editor-only node types the AI must not create: Object Flow composition needs
+// declared sockets picked in the editor, sound needs an Explorer asset hash.
+const EXCLUDED_NODE_TYPES = ['objectflow', 'flowinput', 'flowoutput', 'sound', 'customnode'];
+
+const ALL_CATALOG_TYPES = nodeCatalog.flatMap((group) => group.items.map((item) => item.type));
+
+/** Small local models invent near-miss node names — map them home. */
+const NODE_TYPE_ALIASES = /** @type {Record} */ ({
+ rotate: 'spin',
+ rotation: 'spin',
+ rotator: 'spin',
+ patrol: 'pathpatrol',
+ path: 'pathpatrol',
+ waypoints: 'pathpatrol',
+ walk: 'pathpatrol',
+ move: 'pathpatrol',
+ color: 'setcolor',
+ colorchange: 'setcolor',
+ jump: 'bounce',
+ hop: 'bounce',
+ flash: 'blink',
+ wobble: 'shake',
+ click: 'onclick',
+ key: 'keypress',
+ keyboard: 'keypress',
+ particles: 'particle',
+ emitter: 'particle',
+ weight: 'mass',
+ restitution: 'bounciness',
+ torque: 'angularvelocity'
+});
+
+/**
+ * The curated node vocabulary offered to the model (catalog minus editor-only;
+ * physics node types only when the provider checkbox is on).
+ * @param {boolean} physics
+ * @returns {string[]}
+ */
+export function aiNodeTypes(physics) {
+ return ALL_CATALOG_TYPES.filter(
+ (type) =>
+ !EXCLUDED_NODE_TYPES.includes(type) && (physics || !PHYSICS_NODE_TYPES.includes(type))
+ );
+}
+
+/** @param {any} raw @returns {string|null} a real catalog type or null */
+function normalizeNodeType(raw) {
+ const flat = String(raw ?? '')
+ .trim()
+ .toLowerCase()
+ .replace(/[\s_-]+/g, '');
+ if (ALL_CATALOG_TYPES.includes(flat)) return flat;
+ return NODE_TYPE_ALIASES[flat] ?? null;
+}
+
+/** A fresh uuid (crypto — same helper shape as tools.js). @returns {string} */
+function genUuid() {
+ if (typeof crypto !== 'undefined' && crypto.randomUUID) return crypto.randomUUID();
+ return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
+ const r = (Math.random() * 16) | 0;
+ return (c === 'x' ? r : (r & 0x3) | 0x8).toString(16);
+ });
+}
+
+/** @param {string} uuid */
+function objectOf(uuid) {
+ return get(objectsGroup)?.getObjectByProperty('uuid', uuid) ?? null;
+}
+
+/** @param {string} uuid */
+function lockedByPeer(uuid) {
+ return get(lockedObjects).some((/** @type {any} */ l) => l[1] === uuid);
+}
+
+/**
+ * Resolve the `graph` argument: 'scene' (or the literal SCENE_GRAPH id) → the
+ * scene graph; an existing object uuid → that object's graph. Null = invalid.
+ * @param {any} raw @returns {string|null}
+ */
+function resolveGraph(raw) {
+ const id = String(raw ?? '').trim();
+ if (!id || id === 'scene' || id === SCENE_GRAPH) return SCENE_GRAPH;
+ return objectOf(id) ? id : null;
+}
+
+/** Editor-like grid auto-layout, offset by how many nodes the graph already has.
+ * @param {number} index @returns {{x: number, y: number}} */
+function gridPosition(index) {
+ return { x: 60 + (index % 4) * 190, y: 60 + Math.floor(index / 4) * 130 };
+}
+
+/** ≥2 triples of finite numbers, else null. @param {any} raw @returns {number[][]|null} */
+function validatePoints(raw) {
+ if (!Array.isArray(raw) || raw.length < 2) return null;
+ const points = [];
+ for (const entry of raw) {
+ if (!Array.isArray(entry) || entry.length !== 3 || entry.some((n) => !Number.isFinite(n)))
+ return null;
+ points.push([entry[0], entry[1], entry[2]]);
+ }
+ return points;
+}
+
+/**
+ * Node data exactly like the editor builds it (Nodes.svelte addNode): label +
+ * type + spec defaults, overlaid with the model's data for KNOWN keys only
+ * (plus validated pathpatrol.points and script.code).
+ * @param {string} type @param {any} spec @param {any} raw @param {string[]} errors
+ */
+function buildNodeData(type, spec, raw, errors) {
+ /** @type {any} */
+ const data = { label: spec?.label ?? type, type, ...(spec?.defaults ?? {}) };
+ if (!raw || typeof raw !== 'object') return data;
+ for (const [key, value] of Object.entries(raw)) {
+ if (key === 'type') continue;
+ if (key === 'label') {
+ if (typeof value === 'string' && value) data.label = value;
+ continue;
+ }
+ if (key === 'points' && type === 'pathpatrol') {
+ const points = validatePoints(value);
+ if (points) data.points = points;
+ else errors.push('pathpatrol points must be >=2 [x,y,z] triples — kept the default');
+ continue;
+ }
+ if (key === 'code' && type === 'script') {
+ if (typeof value === 'string' && value) data.code = value;
+ continue;
+ }
+ if (key in (spec?.defaults ?? {})) data[key] = value;
+ // unknown keys are dropped silently — defaults render fine without them
+ }
+ return data;
+}
+
+/**
+ * create_flow_nodes: add behavior nodes (and optional edges) to ONE graph.
+ * Local `ref` keys let edges point at freshly created nodes. Records ONE
+ * 'flownodes' history entry for the whole call.
+ * @param {any} args {graph, nodes: [{ref?, type, data?}], edges?: [{from, to, fromHandle?, toHandle?}]}
+ * @returns {any}
+ */
+export function createFlowNodesTool(args) {
+ const graphId = resolveGraph(args?.graph);
+ if (graphId === null)
+ return { error: 'unknown graph "' + args?.graph + '" — pass "scene" or an existing object uuid' };
+ const specs = Array.isArray(args?.nodes) ? args.nodes : [];
+ if (!specs.length)
+ return { error: 'no nodes provided — pass nodes: [{ type, data? }] (one graph per call)' };
+
+ const physics = physicsToolsEnabled();
+ const allowed = aiNodeTypes(physics);
+ /** @type {any} */
+ const peer = get(peers);
+
+ // missing object graph → create it first (records 'flowgraph' + graphcreate)
+ if (graphId !== SCENE_GRAPH && !graphExists(graphId)) createObjectGraph(graphId);
+
+ const existing = graphOf(graphId)?.nodes.length ?? 0;
+ /** @type {any[]} */ const created = [];
+ /** @type {any[]} */ const createdNodes = [];
+ /** @type {string[]} */ const errors = [];
+ /** @type {Record} */ const refIds = {};
+
+ for (const rawSpec of specs) {
+ const type = normalizeNodeType(rawSpec?.type);
+ if (!type || !ALL_CATALOG_TYPES.includes(type)) {
+ errors.push('unknown node type "' + rawSpec?.type + '"');
+ continue;
+ }
+ if (!allowed.includes(type)) {
+ errors.push(
+ PHYSICS_NODE_TYPES.includes(type)
+ ? 'node "' + type + '" needs Physics tools enabled (Settings -> AI)'
+ : 'node type "' + type + '" cannot be created by the assistant'
+ );
+ continue;
+ }
+ const spec = findNodeSpec(type);
+ const node = {
+ id: genUuid(),
+ type,
+ position: gridPosition(existing + createdNodes.length),
+ data: buildNodeData(type, spec, rawSpec?.data, errors),
+ class: 'w-[150px]'
+ };
+ createFlowNode(node, graphId);
+ const serialized = serializeNode(node);
+ if (peer) peer.send({ type: 'nodecreate', node: serialized, graphId });
+ createdNodes.push(serialized);
+ const ref = typeof rawSpec?.ref === 'string' && rawSpec.ref ? rawSpec.ref : null;
+ if (ref) refIds[ref] = node.id;
+ created.push({ ...(ref ? { ref } : {}), id: node.id, type });
+ }
+
+ /** @type {any[]} */ const createdEdges = [];
+ const edgeSpecs = Array.isArray(args?.edges) ? args.edges : [];
+ if (edgeSpecs.length) {
+ const inGraph = new Set((graphOf(graphId)?.nodes ?? []).map((/** @type {any} */ n) => n.id));
+ for (const e of edgeSpecs) {
+ const source = refIds[e?.from] ?? (inGraph.has(e?.from) ? e.from : null);
+ const target = refIds[e?.to] ?? (inGraph.has(e?.to) ? e.to : null);
+ if (!source || !target) {
+ errors.push('edge "' + e?.from + '" -> "' + e?.to + '": unknown node ref/id');
+ continue;
+ }
+ const sourceHandle = typeof e?.fromHandle === 'string' && e.fromHandle ? e.fromHandle : undefined;
+ const targetHandle = typeof e?.toHandle === 'string' && e.toHandle ? e.toHandle : undefined;
+ // id format MUST match the editor's (peer dedupe diverges otherwise)
+ const edge = {
+ id:
+ 'e-' + source + (sourceHandle ? '.' + sourceHandle : '') +
+ '-' + target + (targetHandle ? '.' + targetHandle : ''),
+ source,
+ target,
+ ...(sourceHandle ? { sourceHandle } : {}),
+ ...(targetHandle ? { targetHandle } : {})
+ };
+ createFlowEdge(edge, graphId);
+ const serialized = serializeEdge(edge);
+ if (peer) peer.send({ type: 'edgecreate', edge: serialized, graphId });
+ createdEdges.push(serialized);
+ }
+ }
+
+ if (createdNodes.length || createdEdges.length)
+ recordFlowNodesEntry({ op: 'create', graphId, nodes: createdNodes, edges: createdEdges });
+ if (!createdNodes.length && !createdEdges.length)
+ return { error: 'nothing was created: ' + (errors[0] ?? 'no valid nodes'), errors };
+ return {
+ graph: graphId,
+ created,
+ edges: createdEdges.length,
+ ...(errors.length ? { errors } : {})
+ };
+}
+
+/**
+ * update_flow_nodes: tune node data and/or remove nodes in ONE graph.
+ * @param {any} args {graph, updates?: [{id, data}], remove?: [ids]}
+ * @returns {any}
+ */
+export function updateFlowNodesTool(args) {
+ const graphId = resolveGraph(args?.graph);
+ if (graphId === null)
+ return { error: 'unknown graph "' + args?.graph + '" — pass "scene" or an existing object uuid' };
+ const graph = graphOf(graphId);
+ if (!graph || !graph.nodes.length) return { error: 'that graph has no nodes yet' };
+ /** @type {any} */
+ const peer = get(peers);
+ /** @type {string[]} */ const errors = [];
+ /** @type {any[]} */ const updated = [];
+ /** @type {{id: string, before: any, after: any}[]} */ const items = [];
+
+ for (const u of Array.isArray(args?.updates) ? args.updates : []) {
+ const node = graph.nodes.find((/** @type {any} */ n) => n.id === u?.id);
+ if (!node) {
+ errors.push('no node "' + u?.id + '" in that graph');
+ continue;
+ }
+ const spec = findNodeSpec(node.type);
+ /** @type {any} */ const patch = {};
+ /** @type {any} */ const before = {};
+ const raw = u?.data && typeof u.data === 'object' ? u.data : {};
+ for (const [key, value] of Object.entries(raw)) {
+ if (key === 'type') continue;
+ if (key === 'points' && node.type === 'pathpatrol') {
+ const points = validatePoints(value);
+ if (!points) {
+ errors.push('pathpatrol points must be >=2 [x,y,z] triples');
+ continue;
+ }
+ before.points = node.data.points;
+ patch.points = points;
+ continue;
+ }
+ const known = key === 'label' || key in (spec?.defaults ?? {}) || key in node.data;
+ if (!known) continue;
+ before[key] = node.data[key];
+ patch[key] = value;
+ }
+ if (!Object.keys(patch).length) {
+ errors.push('no valid data keys for node "' + u?.id + '"');
+ continue;
+ }
+ // ALWAYS pass the graph — setNodeData defaults to the active editor graph
+ setNodeData(node.id, patch, graphId);
+ items.push({ id: node.id, before, after: patch });
+ updated.push({ id: node.id, keys: Object.keys(patch) });
+ }
+ if (items.length) recordFlowNodesEntry({ op: 'data', graphId, items });
+
+ const wantRemove = Array.isArray(args?.remove) ? args.remove : [];
+ const removeIds = wantRemove.filter((/** @type {any} */ id) =>
+ graph.nodes.some((/** @type {any} */ n) => n.id === id)
+ );
+ wantRemove
+ .filter((/** @type {any} */ id) => !removeIds.includes(id))
+ .forEach((/** @type {any} */ id) => errors.push('no node "' + id + '" to remove'));
+ if (removeIds.length) {
+ // capture serialized nodes + touching edges BEFORE deleting (undo restore)
+ const nodes = graph.nodes
+ .filter((/** @type {any} */ n) => removeIds.includes(n.id))
+ .map(serializeNode);
+ const edges = graph.edges
+ .filter((/** @type {any} */ e) => removeIds.includes(e.source) || removeIds.includes(e.target))
+ .map(serializeEdge);
+ deleteFlowNodes(removeIds, graphId); // also drops touching edges (applier-identical)
+ if (peer) peer.send({ type: 'nodedelete', ids: removeIds, graphId });
+ recordFlowNodesEntry({ op: 'delete', graphId, nodes, edges });
+ }
+
+ if (!updated.length && !removeIds.length)
+ return { error: errors[0] ?? 'nothing to do — pass updates and/or remove', errors };
+ return {
+ graph: graphId,
+ updated,
+ removed: removeIds.length,
+ ...(errors.length ? { errors } : {})
+ };
+}
+
+const PHYSICS_MODES = ['auto', 'static', 'dynamic'];
+const COLLIDER_KINDS = ['box', 'sphere', 'capsule', 'cylinder', 'hull'];
+
+/**
+ * set_physics: merge body params onto objects' userData.physics via the shared
+ * setPhysicsFor path (replicated + 'props' undo entries). Skips peer-locked.
+ * @param {any} args {updates: [{uuid, mode?, mass?, restitution?, friction?, collider?}]}
+ * @returns {any}
+ */
+export function setPhysicsTool(args) {
+ const updates = Array.isArray(args?.updates) ? args.updates : [];
+ if (!updates.length)
+ return { error: 'no updates provided — pass updates: [{ uuid, mode?, mass?, ... }]' };
+ const results = updates.map((/** @type {any} */ u) => {
+ const uuid = u?.uuid;
+ if (!uuid || !objectOf(uuid)) return { uuid, error: 'no object with that uuid' };
+ if (lockedByPeer(uuid)) return { uuid, skipped: 'locked by another peer' };
+ /** @type {any} */ const patch = {};
+ if (typeof u.mode === 'string' && PHYSICS_MODES.includes(u.mode)) patch.mode = u.mode;
+ if (Number.isFinite(u.mass)) {
+ patch.mass = Math.max(0.01, u.mass);
+ if (!patch.mode) patch.mode = 'dynamic'; // mass only means anything on a dynamic body
+ }
+ const restitution = Number.isFinite(u.restitution) ? u.restitution : u.bounciness;
+ if (Number.isFinite(restitution)) patch.restitution = Math.min(Math.max(restitution, 0), 1);
+ if (Number.isFinite(u.friction)) patch.friction = Math.min(Math.max(u.friction, 0), 2);
+ if (typeof u.collider === 'string' && COLLIDER_KINDS.includes(u.collider))
+ patch.collider = u.collider;
+ if (typeof u.sensor === 'boolean') patch.sensor = u.sensor; // CL-A A3 trigger volume
+ if (!Object.keys(patch).length)
+ return { uuid, error: 'no physics keys — pass mode/mass/restitution/friction/collider' };
+ setPhysicsFor(uuid, patch);
+ return { uuid, ok: true, physics: patch };
+ });
+ if (results.every((/** @type {any} */ r) => r.error))
+ return { error: 'nothing was updated: ' + results[0].error, updated: results };
+ return { updated: results };
+}
+
+/**
+ * create_joints: attach object pairs (fixed weld / revolute hinge + optional
+ * motor). Pre-validates uuids/locks — createJoint toasts + returns null on bad
+ * input, which must come back to the model as {error}, not a silent no-op.
+ * @param {any} args {joints: [{kind, a, b, axis?, motor?: {vel, maxForce}}]}
+ * @returns {any}
+ */
+export function createJointsTool(args) {
+ const specs = Array.isArray(args?.joints) ? args.joints : [];
+ if (!specs.length)
+ return { error: 'no joints provided — pass joints: [{ kind: "fixed"|"revolute", a, b }]' };
+ const results = specs.map((/** @type {any} */ j) => {
+ const rawKind = String(j?.kind ?? '').toLowerCase();
+ const kind =
+ rawKind === 'fixed' || rawKind === 'weld'
+ ? 'fixed'
+ : rawKind === 'revolute' || rawKind === 'hinge'
+ ? 'revolute'
+ : null;
+ if (!kind)
+ return { error: 'unknown joint kind "' + j?.kind + '" — use fixed (weld) or revolute (hinge)' };
+ const a = j?.a;
+ const b = j?.b;
+ if (!a || !objectOf(a)) return { error: 'joint "a" object not found: ' + a };
+ if (!b || !objectOf(b)) return { error: 'joint "b" object not found: ' + b };
+ if (a === b) return { error: 'cannot joint an object to itself' };
+ if (lockedByPeer(a) || lockedByPeer(b)) return { skipped: 'locked by another peer', a, b };
+ const axis = ['x', 'y', 'z'].includes(j?.axis) ? j.axis : undefined;
+ const motor =
+ kind === 'revolute' && j?.motor && Number.isFinite(j.motor.vel)
+ ? { vel: j.motor.vel, maxForce: Number.isFinite(j.motor.maxForce) ? j.motor.maxForce : 100 }
+ : undefined;
+ const def = createJoint(/** @type {'fixed'|'revolute'} */ (kind), a, b, axis, motor);
+ if (!def) return { error: 'joint creation failed', a, b };
+ return { id: def.id, kind, a, b };
+ });
+ if (results.every((/** @type {any} */ r) => r.error))
+ return { error: 'nothing was attached: ' + results[0].error, joints: results };
+ return { joints: results };
+}
+
+/**
+ * control_simulation: start/stop/pause/resume/reset the physics sim. Start
+ * guards the remote-initiator rule; sim start/stop deliberately records NO
+ * history entry of its own here (stopSimulation's transformSet matches manual
+ * behavior — undoing the AI batch never stops a running sim).
+ * @param {any} args {action: 'start'|'stop'|'pause'|'resume'|'reset'}
+ * @returns {Promise}
+ */
+export async function controlSimulationTool(args) {
+ const action = String(args?.action ?? '').toLowerCase();
+ if (action === 'start') {
+ if (get(simulating)) return { simulating: true, note: 'already running' };
+ if (get(remoteSimulating))
+ return { error: 'another peer is already simulating — one run at a time' };
+ await toggleSimulation(); // handles the rapier wasm warmup
+ const on = get(simulating);
+ return on
+ ? { simulating: true }
+ : { error: 'simulation did not start — no object has physics yet (use set_physics or a mass node first)' };
+ }
+ if (action === 'stop') {
+ stopSimulation();
+ return { simulating: get(simulating) };
+ }
+ if (action === 'pause') {
+ pauseSimulation(true);
+ return { simulating: get(simulating), paused: true };
+ }
+ if (action === 'resume') {
+ pauseSimulation(false);
+ return { simulating: get(simulating), paused: false };
+ }
+ if (action === 'reset') {
+ resetSimulation();
+ return { simulating: get(simulating) };
+ }
+ return { error: 'unknown action "' + args?.action + '" — start | stop | pause | resume | reset' };
+}
diff --git a/src/lib/ai/providers.js b/src/lib/ai/providers.js
index 9044140a..de2adda2 100644
--- a/src/lib/ai/providers.js
+++ b/src/lib/ai/providers.js
@@ -20,6 +20,9 @@ import { writable, get } from 'svelte/store';
* (vLLM 0.26 + Qwen3.5 swallows the call and streams an invented tool name);
* ai/client.js also detects that at runtime and falls back for the session.
* @property {number} [temperature] sampling temperature; omitted = server default
+ * @property {boolean} [physicsTools] offer the physics tool set (set_physics /
+ * create_joints / control_simulation) to this provider. Off by default —
+ * multi-step physics is hard for small local models (see the docs page).
* @property {string[]} [models] model ids the endpoint reported on the last
* successful Test connection (GET /models) — Settings' model-picker suggestions.
* Persisted so the picker still works after a reload without re-fetching.
@@ -155,6 +158,7 @@ export function addAiProvider(config) {
model: normalizeModel(config.model || preset.defaultModel)
};
if (config.stream === false) entry.stream = false;
+ if (config.physicsTools === true) entry.physicsTools = true;
if (typeof config.temperature === 'number') entry.temperature = config.temperature;
if (Array.isArray(config.models) && config.models.length) {
entry.models = config.models.map(String).slice(0, 500);
diff --git a/src/lib/ai/tools.js b/src/lib/ai/tools.js
index a923dad1..8dd2c4bd 100644
--- a/src/lib/ai/tools.js
+++ b/src/lib/ai/tools.js
@@ -12,6 +12,17 @@ import {
import { setObjectColor, switchMaterialType, setMaterialParam } from '$lib/materialsHandler';
import { notifyExternalMove } from '$lib/flowRuntime';
import { meshGenReady } from './meshProviders.js';
+import { graphOf, SCENE_GRAPH } from '../../stores/flowStore';
+import { sceneJoints } from '$lib/joints';
+import {
+ physicsToolsEnabled,
+ aiNodeTypes,
+ createFlowNodesTool,
+ updateFlowNodesTool,
+ setPhysicsTool,
+ createJointsTool,
+ controlSimulationTool
+} from './flowTools.js';
// AI tool layer (roadmap #10, A4). Maps OpenAI-style function calls onto the
// existing REPLICATED mutation surface — every tool applies locally AND broadcasts
@@ -110,6 +121,56 @@ function describeObject(object) {
out.materialType = material.type;
if (material.color) out.color = '#' + material.color.getHexString();
}
+ if (object.userData?.physics) out.physics = { ...object.userData.physics };
+ const flow = summarizeGraph(object.uuid);
+ if (flow) out.flow = flow;
+ return out;
+}
+
+/** Node data compacted for the model: label/type dupes stripped, big point
+ * lists reduced to a count, long strings clipped. @param {any} node */
+function compactNodeData(node) {
+ /** @type {any} */
+ const out = {};
+ for (const [key, value] of Object.entries(node.data ?? {})) {
+ if (key === 'label' || key === 'type') continue;
+ if (key === 'points' && Array.isArray(value) && value.length > 6) {
+ out.points = value.length + ' points';
+ continue;
+ }
+ if (typeof value === 'string' && value.length > 80) {
+ out[key] = value.slice(0, 77) + '…';
+ continue;
+ }
+ out[key] = value;
+ }
+ return out;
+}
+
+/**
+ * Compact one graph document for the scene summary ("make the spider faster"
+ * needs the node ids + data). Capped per graph so a node-heavy scene can't
+ * blow the context.
+ * @param {string} graphId @param {number} [cap]
+ * @returns {{nodes: any[], edges?: any[], truncatedNodes?: number}|null}
+ */
+function summarizeGraph(graphId, cap = 12) {
+ const graph = graphOf(graphId);
+ if (!graph || (!graph.nodes.length && !graph.edges.length)) return null;
+ /** @type {any} */
+ const out = {
+ nodes: graph.nodes
+ .slice(0, cap)
+ .map((/** @type {any} */ n) => ({ id: n.id, type: n.type, ...compactNodeData(n) }))
+ };
+ if (graph.nodes.length > cap) out.truncatedNodes = graph.nodes.length - cap;
+ if (graph.edges.length)
+ out.edges = graph.edges.map((/** @type {any} */ e) => ({
+ from: e.source,
+ to: e.target,
+ ...(e.sourceHandle ? { fromHandle: e.sourceHandle } : {}),
+ ...(e.targetHandle ? { toHandle: e.targetHandle } : {})
+ }));
return out;
}
@@ -135,7 +196,14 @@ export function summarizeScene(cap = 200) {
objects.push(describeObject(object));
});
}
- return truncated ? { objects, truncated } : { objects };
+ /** @type {any} */
+ const out = truncated ? { objects, truncated } : { objects };
+ const sceneFlow = summarizeGraph(SCENE_GRAPH);
+ if (sceneFlow) out.sceneFlow = sceneFlow;
+ const joints = get(sceneJoints);
+ if (joints.length)
+ out.joints = joints.map((/** @type {any} */ j) => ({ id: j.id, kind: j.kind, a: j.a, b: j.b }));
+ return out;
}
/** @param {string} uuid */
@@ -272,9 +340,16 @@ const TOOL_NAMES = [
'delete_objects',
'group_objects',
'clear_scene',
- 'generate_mesh'
+ 'generate_mesh',
+ 'create_flow_nodes',
+ 'update_flow_nodes',
+ 'set_physics',
+ 'create_joints',
+ 'control_simulation'
];
+const SIM_ACTIONS = ['start', 'stop', 'pause', 'resume', 'reset'];
+
/** Common near-misses. Small local models invent singular/verb variants. */
const NAME_ALIASES = /** @type {Record} */ ({
create_object: 'create_objects',
@@ -304,9 +379,65 @@ const NAME_ALIASES = /** @type {Record} */ ({
list: 'list_scene',
clear: 'clear_scene',
generate_model: 'generate_mesh',
- create_mesh: 'generate_mesh'
+ create_mesh: 'generate_mesh',
+ // flow / behavior invention family
+ add_behavior: 'create_flow_nodes',
+ add_behaviour: 'create_flow_nodes',
+ create_behavior: 'create_flow_nodes',
+ animate: 'create_flow_nodes',
+ animate_object: 'create_flow_nodes',
+ add_animation: 'create_flow_nodes',
+ add_node: 'create_flow_nodes',
+ add_nodes: 'create_flow_nodes',
+ add_flow_node: 'create_flow_nodes',
+ add_flow_nodes: 'create_flow_nodes',
+ create_flow_node: 'create_flow_nodes',
+ create_nodes: 'create_flow_nodes',
+ update_flow_node: 'update_flow_nodes',
+ update_node: 'update_flow_nodes',
+ update_nodes: 'update_flow_nodes',
+ edit_node: 'update_flow_nodes',
+ set_node_data: 'update_flow_nodes',
+ remove_node: 'update_flow_nodes',
+ remove_nodes: 'update_flow_nodes',
+ delete_node: 'update_flow_nodes',
+ delete_nodes: 'update_flow_nodes',
+ // physics family
+ enable_physics: 'set_physics',
+ set_physic: 'set_physics',
+ update_physics: 'set_physics',
+ physics: 'set_physics',
+ make_static: 'set_physics',
+ make_dynamic: 'set_physics',
+ set_mass: 'set_physics',
+ create_joint: 'create_joints',
+ add_joint: 'create_joints',
+ add_joints: 'create_joints',
+ hinge: 'create_joints',
+ weld: 'create_joints',
+ attach: 'create_joints',
+ attach_objects: 'create_joints',
+ connect_objects: 'create_joints',
+ start_simulation: 'control_simulation',
+ stop_simulation: 'control_simulation',
+ pause_simulation: 'control_simulation',
+ reset_simulation: 'control_simulation',
+ run_simulation: 'control_simulation',
+ simulate: 'control_simulation',
+ simulation: 'control_simulation'
});
+/** Alias hits landing on control_simulation carry the action IN THE NAME
+ * ("start_simulation") more often than in args — fill it in.
+ * @param {string} name @param {string} sourceKey @param {any} args */
+function withActionFill(name, sourceKey, args) {
+ if (name === 'control_simulation' && !args.action) {
+ const action = SIM_ACTIONS.find((a) => sourceKey.includes(a)) ?? 'start';
+ return { name, args: { ...args, action }, repaired: true };
+ }
+ return { name, args, repaired: true };
+}
+
/**
* Best-effort repair of a tool call from a weaker model: fix the NAME (case, aliases,
* `functions.` prefixes) and, when the name is pure invention, infer the tool from the
@@ -328,12 +459,27 @@ export function repairToolCall(rawName, rawArgs) {
const snake = key.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase();
for (const candidate of [key, snake]) {
if (TOOL_NAMES.includes(candidate)) return { name: candidate, args, repaired: true };
- if (NAME_ALIASES[candidate]) return { name: NAME_ALIASES[candidate], args, repaired: true };
+ if (NAME_ALIASES[candidate]) return withActionFill(NAME_ALIASES[candidate], candidate, args);
}
// name is invention — infer from the argument shape
if (Array.isArray(args.objects)) return { name: 'create_objects', args, repaired: true };
- if (Array.isArray(args.updates)) return { name: 'update_objects', args, repaired: true };
+ if (Array.isArray(args.nodes)) return { name: 'create_flow_nodes', args, repaired: true };
+ if (Array.isArray(args.joints)) return { name: 'create_joints', args, repaired: true };
+ if (typeof args.action === 'string' && SIM_ACTIONS.includes(args.action.toLowerCase()))
+ return { name: 'control_simulation', args, repaired: true };
+ if (Array.isArray(args.updates)) {
+ // updates whose items carry ONLY physics keys route to set_physics
+ const PHYS_KEYS = ['mass', 'restitution', 'bounciness', 'friction', 'mode', 'collider'];
+ const OBJ_KEYS = ['position', 'rotation', 'scale', 'color', 'name', 'visible', 'materialType', 'materialParams', 'parentUuid'];
+ const items = args.updates.filter((/** @type {any} */ u) => u && typeof u === 'object');
+ const physicsOnly =
+ items.length > 0 &&
+ items.every(
+ (/** @type {any} */ u) => PHYS_KEYS.some((k) => k in u) && !OBJ_KEYS.some((k) => k in u)
+ );
+ return { name: physicsOnly ? 'set_physics' : 'update_objects', args, repaired: true };
+ }
if (Array.isArray(args.uuids)) return { name: 'delete_objects', args, repaired: true };
if (Array.isArray(args.memberUuids)) return { name: 'group_objects', args, repaired: true };
// a single object spec passed directly (kind/primitive/light present)
@@ -352,6 +498,11 @@ export function repairToolCall(rawName, rawArgs) {
return { name: original, args, repaired: false };
}
+/** Names stay in TOOL_NAMES even while gated OFF so repair still normalizes
+ * them — the executor answers with this instead. */
+const PHYSICS_DISABLED =
+ 'physics tools are disabled — enable "Physics tools" for this provider in Settings → AI';
+
/**
* Execute one tool call. Never throws — returns a JSON-serializable result.
* @param {string} rawName
@@ -420,6 +571,27 @@ export async function executeAiTool(rawName, rawArgs) {
return { cleared: count };
}
+ case 'create_flow_nodes':
+ return createFlowNodesTool(args);
+
+ case 'update_flow_nodes':
+ return updateFlowNodesTool(args);
+
+ case 'set_physics': {
+ if (!physicsToolsEnabled()) return { error: PHYSICS_DISABLED };
+ return setPhysicsTool(args);
+ }
+
+ case 'create_joints': {
+ if (!physicsToolsEnabled()) return { error: PHYSICS_DISABLED };
+ return createJointsTool(args);
+ }
+
+ case 'control_simulation': {
+ if (!physicsToolsEnabled()) return { error: PHYSICS_DISABLED };
+ return await controlSimulationTool(args);
+ }
+
case 'generate_mesh': {
// LONG tool: generation takes minutes, so we DON'T block the chat loop.
// Kick off the job and return immediately; the job runner places +
@@ -616,10 +788,186 @@ export const MESH_TOOL = {
}
};
-/** Toolset for the assistant — includes generate_mesh only when a mesh provider is
- * ready. Call this per turn (readiness can change). @returns {any[]} */
+/** Flow-node tool schemas. Built per call — the node-type enum tracks the
+ * physics gate (physics node types absent when gated off).
+ * @param {boolean} physics @returns {any[]} */
+function flowToolSchemas(physics) {
+ const types = aiNodeTypes(physics);
+ return [
+ {
+ type: 'function',
+ function: {
+ name: 'create_flow_nodes',
+ description:
+ 'Add behavior (flow) nodes to a node graph — this is how objects get MOTION and interactivity: spin, bounce, patrol a path, pulse, blink, react to clicks. graph is "scene" or an object uuid (ONE graph per call). KEY RULE: a behavior node placed in an OBJECT\'s graph with no edges automatically drives that object — most behaviors are one node, zero edges, so usually OMIT edges.',
+ parameters: {
+ type: 'object',
+ properties: {
+ graph: {
+ type: 'string',
+ description: '"scene" or an existing object uuid (nodes in an object\'s graph drive that object)'
+ },
+ nodes: {
+ type: 'array',
+ items: {
+ type: 'object',
+ properties: {
+ ref: {
+ type: 'string',
+ description: 'local key so edges in THIS call can reference the new node'
+ },
+ type: { type: 'string', enum: types },
+ data: {
+ type: 'object',
+ description:
+ 'node params, e.g. spin {axis:"y",speed:2}; bounce {amplitude:0.5,speed:2}; pathpatrol {points:[[x,y,z],…] (>=2 world waypoints), speed:1, mode:"loop"|"pingpong"}; setcolor {color:"#ff0000"}. Omit for defaults.'
+ }
+ },
+ required: ['type']
+ }
+ },
+ edges: {
+ type: 'array',
+ description:
+ 'optional wires between nodes (by ref or existing node id). Usually OMIT — unwired behavior nodes already drive their graph\'s object.',
+ items: {
+ type: 'object',
+ properties: {
+ from: { type: 'string' },
+ to: { type: 'string' },
+ fromHandle: { type: 'string' },
+ toHandle: { type: 'string' }
+ },
+ required: ['from', 'to']
+ }
+ }
+ },
+ required: ['graph', 'nodes']
+ }
+ }
+ },
+ {
+ type: 'function',
+ function: {
+ name: 'update_flow_nodes',
+ description:
+ 'Tune or remove existing behavior nodes in one graph. Node ids and current data are in the scene summary under each object\'s "flow" (and "sceneFlow").',
+ parameters: {
+ type: 'object',
+ properties: {
+ graph: { type: 'string', description: '"scene" or an object uuid' },
+ updates: {
+ type: 'array',
+ items: {
+ type: 'object',
+ properties: {
+ id: { type: 'string' },
+ data: { type: 'object', description: 'data keys to change, e.g. {speed: 3}' }
+ },
+ required: ['id', 'data']
+ }
+ },
+ remove: { type: 'array', items: { type: 'string' }, description: 'node ids to delete' }
+ },
+ required: ['graph']
+ }
+ }
+ }
+ ];
+}
+
+/** Physics tool schemas — only offered when the provider's "Physics tools"
+ * checkbox is on (hard for small local models). */
+const PHYSICS_AI_TOOLS = [
+ {
+ type: 'function',
+ function: {
+ name: 'set_physics',
+ description:
+ 'Set physics body params on objects (batch). NOTE: new primitives already spawn dynamic with mass 1 — use this mostly to TUNE bodies or make scenery immovable (mode "static" for ground/walls). mode "auto" reverts to scenery defaults.',
+ parameters: {
+ type: 'object',
+ properties: {
+ updates: {
+ type: 'array',
+ items: {
+ type: 'object',
+ properties: {
+ uuid: { type: 'string' },
+ mode: { type: 'string', enum: ['auto', 'static', 'dynamic'] },
+ mass: { type: 'number', description: 'kg; implies mode dynamic' },
+ restitution: { type: 'number', description: 'bounciness 0..1' },
+ friction: { type: 'number', description: '0..2' },
+ collider: { type: 'string', enum: ['box', 'sphere', 'capsule', 'cylinder', 'hull'] },
+ sensor: {
+ type: 'boolean',
+ description: 'true = a trigger volume: no collision response, fires On Enter/On Exit nodes'
+ }
+ },
+ required: ['uuid']
+ }
+ }
+ },
+ required: ['updates']
+ }
+ }
+ },
+ {
+ type: 'function',
+ function: {
+ name: 'create_joints',
+ description:
+ 'Attach object pairs with physics joints: "fixed" welds them rigidly, "revolute" hinges around an axis (optional motor spins it). Move parts to their final pose WORLD-ALIGNED (no rotation) before jointing.',
+ parameters: {
+ type: 'object',
+ properties: {
+ joints: {
+ type: 'array',
+ items: {
+ type: 'object',
+ properties: {
+ kind: { type: 'string', enum: ['fixed', 'revolute'] },
+ a: { type: 'string', description: 'uuid of the base object' },
+ b: { type: 'string', description: 'uuid of the attached object (hinge anchors at its origin)' },
+ axis: { type: 'string', enum: ['x', 'y', 'z'], description: 'hinge axis (revolute only, default y)' },
+ motor: {
+ type: 'object',
+ description: 'revolute only: {vel: rad/s, maxForce}',
+ properties: { vel: { type: 'number' }, maxForce: { type: 'number' } }
+ }
+ },
+ required: ['kind', 'a', 'b']
+ }
+ }
+ },
+ required: ['joints']
+ }
+ }
+ },
+ {
+ type: 'function',
+ function: {
+ name: 'control_simulation',
+ description:
+ 'Start, stop, pause, resume or reset the physics simulation. After building a physics assembly, start the simulation so the user sees it move.',
+ parameters: {
+ type: 'object',
+ properties: { action: { type: 'string', enum: ['start', 'stop', 'pause', 'resume', 'reset'] } },
+ required: ['action']
+ }
+ }
+ }
+];
+
+/** Toolset for the assistant — flow tools always; physics tools when the
+ * provider checkbox is on; generate_mesh only when a mesh provider is ready.
+ * Call this per turn (readiness can change). @returns {any[]} */
export function getAiTools() {
- return meshGenReady() ? [...AI_TOOLS, MESH_TOOL] : AI_TOOLS;
+ const physics = physicsToolsEnabled();
+ const tools = [...AI_TOOLS, ...flowToolSchemas(physics)];
+ if (physics) tools.push(...PHYSICS_AI_TOOLS);
+ if (meshGenReady()) tools.push(MESH_TOOL);
+ return tools;
}
/** Build the system prompt with scene-building guidance. @returns {string} */
@@ -627,6 +975,31 @@ export function buildSystemPrompt() {
const meshLine = meshGenReady()
? '\nCustom meshes: for objects no primitive can approximate, call generate_mesh with a text\ndescription (slow, async — it appears shortly). Prefer primitives for simple shapes.'
: '';
+ const physics = physicsToolsEnabled();
+ const flowBlock = [
+ '',
+ 'Behaviors (flow nodes): objects MOVE via behavior nodes. create_flow_nodes adds them;',
+ "the graph argument picks whose: \"scene\" or an object's uuid. KEY RULE: a behavior node",
+ "in an OBJECT's graph with no edges drives that object — one node, zero edges is the",
+ 'normal case, so usually OMIT edges. Node types: ' + aiNodeTypes(physics).join(', ') + '.',
+ 'pathpatrol walks world waypoints: data.points = [[x,y,z],…] (at least 2).',
+ 'Moving-creature recipe (e.g. "a moving spider"): create the body parts, group them, put',
+ "ONE pathpatrol node on the GROUP's graph, and a bounce node on each leg's graph.",
+ 'Existing node ids/data appear in the scene summary ("flow"/"sceneFlow") — tune or remove',
+ 'them with update_flow_nodes.'
+ ].join('\n');
+ const physicsBlock = physics
+ ? [
+ '',
+ 'Physics: new primitives already spawn DYNAMIC (mass 1) — they fall and collide once a',
+ 'simulation runs. Use set_physics to tune bodies or pin scenery (mode "static" for',
+ 'ground/walls). create_joints welds (fixed) or hinges (revolute, optional motor) pairs:',
+ 'assemble parts at their final pose WORLD-ALIGNED (no rotation) BEFORE jointing, or the',
+ 'hinge axis is wrong and the solver launches the assembly. A door = static frame +',
+ 'revolute joint; a wheel = revolute + motor. When a physics build is done, call',
+ 'control_simulation action "start" so it comes alive (undo will not stop a running sim).'
+ ].join('\n')
+ : '';
return [
'You are a 3D scene-building assistant embedded in a collaborative prototyping app.',
'You build scenes by calling tools that create and arrange objects. Everything you do is',
@@ -651,6 +1024,6 @@ export function buildSystemPrompt() {
"an object's name as the tool name — the name of a thing you create belongs in that object's",
'`name` field inside create_objects. Every call takes effect immediately, so never repeat a',
'call that already came back without an error — once the scene matches the request, stop',
- 'calling tools and write the summary.' + meshLine
+ 'calling tools and write the summary.' + meshLine + flowBlock + physicsBlock
].join('\n');
}
diff --git a/src/lib/flowGraphs.js b/src/lib/flowGraphs.js
index 4da1d649..e38e0957 100644
--- a/src/lib/flowGraphs.js
+++ b/src/lib/flowGraphs.js
@@ -10,6 +10,13 @@ import {
import { peers, showToast } from '../stores/appStore';
import { registerHistoryKind, recordEntry } from './history';
import { removeEmbedsOf } from './objectFlow';
+import {
+ createFlowNode,
+ createFlowEdge,
+ deleteFlowNodes,
+ deleteFlowEdges,
+ updateFlowNodeData
+} from './nodesHandler';
// H1 (flow v2): object-flow lifecycle -- create/delete graph documents,
// replicated (graphcreate/graphdelete) and undoable (the 'flowgraph' history
@@ -97,6 +104,64 @@ registerHistoryKind('flowgraph', (entry, state) => {
return true;
});
+// 'flownodes' history kind (AI flow tools): node/edge creation, data edits and
+// removals INSIDE one graph as a single undoable entry. Entries carry
+// SERIALIZED node/edge copies (serializeNode/serializeEdge shapes) so replayed
+// re-broadcasts hash identically on every peer (nodesync drift guard).
+
+/**
+ * Record an undoable flow-node mutation.
+ * op 'create'/'delete' take {nodes, edges} (serialized); op 'data' takes
+ * {items: [{id, before, after}]} of node-data patches.
+ * @param {{op: 'create'|'delete'|'data', graphId: string, nodes?: any[],
+ * edges?: any[], items?: {id: string, before: any, after: any}[]}} info
+ */
+export function recordFlowNodesEntry(info) {
+ recordEntry({ kind: 'flownodes', ...info, before: 'before', after: 'after' });
+}
+
+registerHistoryKind('flownodes', (entry, state) => {
+ const undoing = state === entry.before;
+ /** @type {any} */
+ const peer = get(peers);
+ const graphId = entry.graphId;
+ if (entry.op === 'data') {
+ for (const item of entry.items ?? []) {
+ const data = undoing ? item.before : item.after;
+ updateFlowNodeData(item.id, data, graphId);
+ if (peer) peer.send({ type: 'nodedata', id: item.id, data, graphId });
+ }
+ return true;
+ }
+ const removing = entry.op === 'create' ? undoing : !undoing;
+ if (removing) {
+ const edgeIds = (entry.edges ?? []).map((/** @type {any} */ e) => e.id);
+ const nodeIds = (entry.nodes ?? []).map((/** @type {any} */ n) => n.id);
+ if (edgeIds.length) deleteFlowEdges(edgeIds, graphId);
+ if (nodeIds.length) deleteFlowNodes(nodeIds, graphId);
+ if (peer) {
+ if (edgeIds.length) peer.send({ type: 'edgedelete', ids: edgeIds, graphId });
+ if (nodeIds.length) peer.send({ type: 'nodedelete', ids: nodeIds, graphId });
+ }
+ } else {
+ // resurrect the graph document defensively (redo after its owner graph
+ // was deleted, or undo of a delete in a fresh session)
+ if (graphId !== SCENE_GRAPH && !graphExists(graphId)) {
+ updateGraph(graphId, () => ({ nodes: [], edges: [] }));
+ if (peer) peer.send({ type: 'graphcreate', uuid: graphId });
+ }
+ for (const node of entry.nodes ?? []) {
+ createFlowNode({ ...node }, graphId);
+ if (peer) peer.send({ type: 'nodecreate', node, graphId });
+ }
+ for (const edge of entry.edges ?? []) {
+ createFlowEdge({ ...edge }, graphId);
+ if (peer) peer.send({ type: 'edgecreate', edge, graphId });
+ }
+ }
+ return true;
+});
+
// --- serialization helpers -----------------------------------------------------
/**
diff --git a/src/lib/physics.js b/src/lib/physics.js
index 15e55c49..00028475 100644
--- a/src/lib/physics.js
+++ b/src/lib/physics.js
@@ -385,6 +385,33 @@ export function listPhysicsObjects() {
});
}
+/**
+ * Set/merge physics body params on ONE object's userData.physics — the shared
+ * write path for the Inspector, quick actions and the AI set_physics tool.
+ * Rides the existing 'props' history kind (undo/redo replays + re-broadcasts
+ * free) and replicates via objectParameters. Returns the new params, or null
+ * when the object doesn't exist.
+ * @param {string} uuid
+ * @param {{mode?: 'auto'|'static'|'dynamic', mass?: number, restitution?: number,
+ * friction?: number, collider?: string}} patch
+ * @returns {any|null}
+ */
+export function setPhysicsFor(uuid, patch) {
+ const group = get(objectsGroup);
+ const object = group?.getObjectByProperty('uuid', uuid);
+ if (!object) return null;
+ const before = object.userData.physics ? { ...object.userData.physics } : null;
+ const next = { mode: 'auto', ...(object.userData.physics ?? {}), ...patch };
+ object.userData.physics = next;
+ recordEntry({ kind: 'props', uuid, before: { physics: before }, after: { physics: next } });
+ /** @type {any} */
+ const peer = get(peers);
+ peer?.send({ type: 'objectParameters', parameter: 'physics', uuid, physics: next });
+ objectsGroup.update((v) => v); // collider viz re-syncs from the poke
+ physicsShapeChanged(uuid); // CL-A A2: live mid-sim collider rebuild
+ return next;
+}
+
/**
* C1 quick action: make the current selection dynamic (userData.physics mode
* 'dynamic', mass 1 unless already set) — replicates via the existing
@@ -393,8 +420,6 @@ export function listPhysicsObjects() {
*/
export function enablePhysicsOnSelection() {
const group = get(objectsGroup);
- /** @type {any} */
- const peer = get(peers);
const multi = get(selectedObjects);
const primary = /** @type {any} */ (get(selectedObject));
const uuids = multi?.length ? multi : primary?.uuid ? [primary.uuid] : [];
@@ -402,12 +427,11 @@ export function enablePhysicsOnSelection() {
uuids.forEach((/** @type {string} */ uuid) => {
const object = group?.getObjectByProperty('uuid', uuid);
if (!object) return;
- const before = object.userData.physics ? { ...object.userData.physics } : null;
- const next = { ...(object.userData.physics ?? {}), mode: 'dynamic', mass: object.userData.physics?.mass ?? 1 };
- object.userData.physics = next;
- recordEntry({ kind: 'props', uuid, before: { physics: before }, after: { physics: next } });
- peer?.send({ type: 'objectParameters', parameter: 'physics', uuid, physics: next });
- count++;
+ const next = setPhysicsFor(uuid, {
+ mode: 'dynamic',
+ mass: object.userData.physics?.mass ?? 1
+ });
+ if (next) count++;
});
if (count === 0) {
showToast('Select an object first — then Enable physics makes it fall and collide');
diff --git a/tests/e2e/ai-flow-physics.test.cjs b/tests/e2e/ai-flow-physics.test.cjs
new file mode 100644
index 00000000..94961b26
--- /dev/null
+++ b/tests/e2e/ai-flow-physics.test.cjs
@@ -0,0 +1,339 @@
+// AI assistant v3: flow-node behaviors + physics tools. A scripted mock endpoint
+// walks the full "create a moving spider" scenario — create_objects → group →
+// create_flow_nodes (pathpatrol on the GROUP graph + bounce on a leg, with local
+// refs) → set_physics (static ground) → create_joints (one hinge) →
+// control_simulation start → text. Asserts the graph documents + nodes exist,
+// the group is flow-animated, physics userData is set, the joint def landed,
+// the sim runs, all mutations were broadcast, and ONE undo (after reset)
+// reverts nodes+physics+joints+objects while redo restores them WITHOUT
+// restarting the sim. Plus: the physics gate (provider checkbox off → disabled
+// error, mass node rejected per-item) and repair (invented `add_behavior` name
+// with nodes[] args still creates).
+const h = require('./helpers.cjs');
+
+// same-origin with the app under test (worktree lanes serve on their own port)
+const BASE = h.URL.replace(/\/$/, '') + '/mock-flow-ai/v1';
+
+const simulating = (peer) =>
+ peer.page.evaluate(
+ () => new Promise((r) => window.__stores.physics.simulating.subscribe((s) => r(s))())
+ );
+const count = (peer) =>
+ peer.page.evaluate(
+ () => new Promise((r) => window.__stores.objectsGroup.subscribe((g) => r(g ? g.children.length : 0))())
+ );
+const msgs = (peer) =>
+ peer.page.evaluate(
+ () => new Promise((r) => window.__stores.aiAssistant.aiMessages.subscribe((m) => r(m))())
+ );
+const busy = (peer) =>
+ peer.page.evaluate(
+ () => new Promise((r) => window.__stores.aiAssistant.aiBusy.subscribe((b) => r(b))())
+ );
+/** {name -> uuid} for every object in the replicated group (recursive). */
+const uuidsByName = (peer) =>
+ peer.page.evaluate(
+ () =>
+ new Promise((r) =>
+ window.__stores.objectsGroup.subscribe((g) => {
+ const map = {};
+ g?.traverse((o) => {
+ if (o !== g && o.name) map[o.name] = o.uuid;
+ });
+ r(map);
+ })()
+ )
+ );
+const graphsSnapshot = (peer) =>
+ peer.page.evaluate(
+ () =>
+ new Promise((r) =>
+ window.__stores.flowGraphs.subscribe((all) => {
+ const out = {};
+ for (const [id, g] of Object.entries(all))
+ out[id] = { nodes: g.nodes.map((n) => ({ id: n.id, type: n.type, data: { ...n.data } })), edges: g.edges.length };
+ r(out);
+ })()
+ )
+ );
+
+h.run(async () => {
+ const browser = await h.launch();
+
+ // throwaway page warms the vite dep-optimizer for the lazy rapier import
+ {
+ const warm = await h.setupPage(browser, 'warm');
+ await warm.page.evaluate(() => window.__stores.physics.warmup().catch(() => {}));
+ await warm.page.waitForTimeout(4000);
+ await warm.ctx.close();
+ }
+
+ const A = await h.setupPage(browser, 'A');
+
+ // ---- scripted endpoint: one scenario step per tool round-trip -------------------
+ /** Track what the client executed so the mock can address real uuids. */
+ const answer = (route, message) =>
+ route.fulfill({
+ status: 200,
+ contentType: 'application/json',
+ body: JSON.stringify({
+ choices: [{ message, finish_reason: message.tool_calls ? 'tool_calls' : 'stop' }]
+ })
+ });
+ const call = (name, args) => ({
+ role: 'assistant',
+ content: null,
+ tool_calls: [{ id: 'c-' + name, type: 'function', function: { name, arguments: JSON.stringify(args) } }]
+ });
+
+ let scenario = 'spider';
+ await A.page.route('**/mock-flow-ai/v1/chat/completions', (route) => {
+ /** @type {any} */
+ let body = {};
+ try {
+ body = JSON.parse(route.request().postData() || '{}');
+ } catch {}
+ const toolMsgs = (body.messages || []).filter((m) => m.role === 'tool');
+ const results = toolMsgs.map((m) => {
+ try {
+ return JSON.parse(m.content);
+ } catch {
+ return {};
+ }
+ });
+
+ if (scenario === 'gated') {
+ // physics tools OFF: try set_physics (must come back disabled), then a mass
+ // node (must be rejected per-item), then stop.
+ if (toolMsgs.length === 0) return answer(route, call('set_physics', { updates: [{ uuid: 'whatever', mode: 'static' }] }));
+ if (toolMsgs.length === 1)
+ return answer(route, call('create_flow_nodes', { graph: 'scene', nodes: [{ type: 'mass' }, { type: 'spin' }] }));
+ return answer(route, { role: 'assistant', content: 'Gating checked.' });
+ }
+ if (scenario === 'repair') {
+ // invented tool name carrying nodes[] args — repair must route it
+ if (toolMsgs.length === 0)
+ return answer(route, call('add_behavior', { graph: 'scene', nodes: [{ type: 'rotate', data: { speed: 3 } }] }));
+ return answer(route, { role: 'assistant', content: 'Behavior added.' });
+ }
+
+ // ---- spider scenario ----
+ const step = toolMsgs.length;
+ if (step === 0)
+ return answer(
+ route,
+ call('create_objects', {
+ objects: [
+ { kind: 'primitive', primitive: 'Box', params: [8, 0.2, 8], position: [0, -0.1, 0], name: 'Ground' },
+ { kind: 'primitive', primitive: 'Box', params: [1, 0.5, 1.4], position: [0, 0.6, 0], color: '#442200', name: 'Body' },
+ { kind: 'primitive', primitive: 'Box', params: [0.15, 0.5, 0.15], position: [0.7, 0.25, 0], name: 'LegL' },
+ { kind: 'primitive', primitive: 'Box', params: [0.15, 0.5, 0.15], position: [-0.7, 0.25, 0], name: 'LegR' },
+ { kind: 'primitive', primitive: 'Box', params: [0.4, 0.4, 0.4], position: [3, 2, 0], name: 'Pebble' }
+ ]
+ })
+ );
+ if (step === 1) {
+ const created = results[0]?.created ?? [];
+ const uuidOf = (n) => created.find((c) => c.name === n)?.uuid;
+ return answer(
+ route,
+ call('group_objects', { name: 'Spider', memberUuids: [uuidOf('Body'), uuidOf('LegL'), uuidOf('LegR')] })
+ );
+ }
+ if (step === 2) {
+ const groupUuid = results[1]?.groupUuid;
+ return answer(
+ route,
+ call('create_flow_nodes', {
+ graph: groupUuid,
+ nodes: [
+ {
+ ref: 'walk',
+ type: 'pathpatrol',
+ data: { points: [[2, 0.6, 2], [-2, 0.6, 2], [-2, 0.6, -2], [2, 0.6, -2]], speed: 1.5 }
+ }
+ ]
+ })
+ );
+ }
+ if (step === 3) {
+ const legUuid = results[0]?.created?.find((c) => c.name === 'LegL')?.uuid;
+ return answer(
+ route,
+ call('create_flow_nodes', { graph: legUuid, nodes: [{ ref: 'b', type: 'bounce', data: { amplitude: 0.3, speed: 4 } }] })
+ );
+ }
+ if (step === 4)
+ // wired pair on the scene graph — exercises refs → edge id + edgecreate
+ return answer(
+ route,
+ call('create_flow_nodes', {
+ graph: 'scene',
+ nodes: [
+ { ref: 'n', type: 'number', data: { value: 2 } },
+ { ref: 'm', type: 'math', data: { op: 'add' } }
+ ],
+ edges: [{ from: 'n', to: 'm', toHandle: 'a' }]
+ })
+ );
+ if (step === 5) {
+ // "make it faster" — tune the pathpatrol node by its id (nodedata path)
+ const groupUuid = results[1]?.groupUuid;
+ const walkId = results[2]?.created?.[0]?.id;
+ return answer(route, call('update_flow_nodes', { graph: groupUuid, updates: [{ id: walkId, data: { speed: 2.5 } }] }));
+ }
+ if (step === 6) {
+ const groundUuid = results[0]?.created?.find((c) => c.name === 'Ground')?.uuid;
+ return answer(route, call('set_physics', { updates: [{ uuid: groundUuid, mode: 'static' }] }));
+ }
+ if (step === 7) {
+ const created = results[0]?.created ?? [];
+ const groundUuid = created.find((c) => c.name === 'Ground')?.uuid;
+ const pebbleUuid = created.find((c) => c.name === 'Pebble')?.uuid;
+ return answer(route, call('create_joints', { joints: [{ kind: 'revolute', a: groundUuid, b: pebbleUuid, axis: 'y' }] }));
+ }
+ if (step === 8) return answer(route, call('control_simulation', { action: 'start' }));
+ return answer(route, { role: 'assistant', content: 'Spider built and walking.' });
+ });
+
+ // provider: unstreamed + physics tools ON
+ await A.page.evaluate((base) => {
+ window.__stores.aiProviders.addAiProvider({
+ preset: 'custom',
+ label: 'FlowMock',
+ baseUrl: base,
+ apiKey: 'k',
+ model: 'mock',
+ stream: false,
+ physicsTools: true
+ });
+ window.__stores.aiProviders.setAiEnabled(true);
+ }, BASE);
+
+ // capture every peer broadcast type (no peers connected — wrap send)
+ await A.page.evaluate(() => {
+ window.__sent = [];
+ let peerRef;
+ window.__stores.peers.subscribe((p) => (peerRef = p))();
+ const orig = peerRef.send.bind(peerRef);
+ peerRef.send = (data) => {
+ window.__sent.push(data?.type);
+ return orig(data);
+ };
+ });
+
+ // ---- the spider scenario ---------------------------------------------------------
+ const base0 = await count(A);
+ await A.page.evaluate(() => window.__stores.aiAssistant.runPrompt('create a moving spider'));
+ await h.eventually(() => busy(A), (b) => b === false, 'spider prompt finished', 90000);
+
+ await h.eventually(() => count(A), (n) => n === base0 + 3, 'scene has ground + pebble + spider group (3 top-level)');
+ const names = await uuidsByName(A);
+ const spiderUuid = names['Spider Group']; // createGroup suffixes the /group name
+ h.check(!!(names.Ground && names.Body && names.LegL && spiderUuid), 'named objects + group exist');
+
+ const graphs = await graphsSnapshot(A);
+ const groupGraph = graphs[spiderUuid];
+ const walkNode = groupGraph?.nodes.find((n) => n.type === 'pathpatrol');
+ h.check(!!walkNode, 'group graph has the pathpatrol node');
+ h.check(walkNode?.data.points?.length === 4, 'pathpatrol points survived validation');
+ h.check(walkNode?.data.speed === 2.5, 'update_flow_nodes tuned the patrol speed');
+ h.check(!!graphs[names.LegL]?.nodes.some((n) => n.type === 'bounce'), 'leg graph has the bounce node');
+ h.check(
+ graphs.scene?.nodes.some((n) => n.type === 'number') && graphs.scene?.edges === 1,
+ 'scene graph got the wired number→math pair (1 edge)'
+ );
+
+ await h.eventually(
+ () => A.page.evaluate((u) => window.__stores.flowRuntime.isAnimatedTarget(u), spiderUuid),
+ (v) => v === true,
+ 'group is a live animated target (implicit owner)'
+ );
+
+ const groundPhysics = await A.page.evaluate(
+ (u) =>
+ new Promise((r) =>
+ window.__stores.objectsGroup.subscribe((g) => r(g.getObjectByProperty('uuid', u)?.userData?.physics ?? null))()
+ ),
+ names.Ground
+ );
+ h.check(groundPhysics?.mode === 'static', 'ground userData.physics is static');
+
+ const joints0 = await A.page.evaluate(
+ () => new Promise((r) => window.__stores.joints.sceneJoints.subscribe((j) => r(j))())
+ );
+ h.check(joints0.length === 1 && joints0[0].kind === 'revolute', 'one revolute joint def in sceneJoints');
+
+ await h.eventually(() => simulating(A), (s) => s === true, 'simulation is running', 30000);
+
+ const sent = await A.page.evaluate(() => window.__sent);
+ for (const type of ['create', 'group', 'graphcreate', 'nodecreate', 'edgecreate', 'nodedata', 'objectParameters', 'jointcreate', 'simulate']) {
+ h.check(sent.includes(type), 'broadcast captured: ' + type);
+ }
+
+ let list = await msgs(A);
+ h.check(!list.some((m) => m.role === 'error'), 'no tool errors in the spider run');
+ h.check(
+ list.some((m) => m.role === 'tool-status' && /behavior node/.test(m.content)),
+ 'transcript labels the flow-node calls'
+ );
+
+ // ---- one undo reverts everything (after reset), redo restores without sim -------
+ await A.page.evaluate(() => window.__stores.physics.resetSimulation());
+ await h.eventually(() => simulating(A), (s) => s === false, 'simulation reset/stopped');
+ await A.page.evaluate(() => window.__stores.history.undo());
+ await h.eventually(() => count(A), (n) => n === base0, 'one undo removed all spider objects');
+ await h.eventually(
+ () => A.page.evaluate(() => new Promise((r) => window.__stores.joints.sceneJoints.subscribe((j) => r(j.length))())),
+ (n) => n === 0,
+ 'undo removed the joint def'
+ );
+ const graphsAfterUndo = await graphsSnapshot(A);
+ h.check(!graphsAfterUndo[spiderUuid], 'undo removed the group graph document');
+
+ await A.page.evaluate(() => window.__stores.history.redo());
+ await h.eventually(() => count(A), (n) => n === base0 + 3, 'redo restored the spider');
+ const graphsAfterRedo = await graphsSnapshot(A);
+ h.check(
+ !!graphsAfterRedo[spiderUuid]?.nodes.some((n) => n.type === 'pathpatrol'),
+ 'redo restored the pathpatrol node'
+ );
+ h.check((await simulating(A)) === false, 'redo did NOT restart the simulation');
+
+ // ---- gating: physics tools OFF ---------------------------------------------------
+ await A.page.evaluate(() => window.__stores.aiAssistant.resetAiConversation());
+ await A.page.evaluate(() => {
+ let providers;
+ window.__stores.aiProviders.aiProviders.subscribe((p) => (providers = p))();
+ window.__stores.aiProviders.updateAiProvider(providers[0].id, { physicsTools: false });
+ window.__sent.length = 0;
+ });
+ scenario = 'gated';
+ await A.page.evaluate(() => window.__stores.aiAssistant.runPrompt('make the ground static'));
+ await h.eventually(() => busy(A), (b) => b === false, 'gated prompt finished', 60000);
+ list = await msgs(A);
+ h.check(
+ list.some((m) => m.role === 'error' && /physics tools are disabled/i.test(m.content)),
+ 'set_physics returned the disabled error'
+ );
+ const sentGated = await A.page.evaluate(() => window.__sent);
+ h.check(!sentGated.includes('objectParameters'), 'no physics broadcast while gated');
+ const sceneGraph = (await graphsSnapshot(A)).scene;
+ h.check(!(sceneGraph?.nodes ?? []).some((n) => n.type === 'mass'), 'mass node rejected while gated');
+ h.check((sceneGraph?.nodes ?? []).some((n) => n.type === 'spin'), 'non-physics node in the same call still created');
+
+ // ---- repair: invented add_behavior name with nodes[] args -------------------------
+ await A.page.evaluate(() => window.__stores.aiAssistant.resetAiConversation());
+ scenario = 'repair';
+ await A.page.evaluate(() => window.__stores.aiAssistant.runPrompt('spin something'));
+ await h.eventually(() => busy(A), (b) => b === false, 'repair prompt finished', 60000);
+ const sceneAfter = (await graphsSnapshot(A)).scene;
+ h.check(
+ (sceneAfter?.nodes ?? []).some((n) => n.type === 'spin' && n.data.speed === 3),
+ 'invented add_behavior (alias) with rotate→spin node created'
+ );
+ list = await msgs(A);
+ h.check(!list.some((m) => m.role === 'error' && /unknown tool/.test(m.content)), 'no unknown-tool error for the repaired call');
+
+ await h.finish(browser);
+});