Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 9 additions & 33 deletions dashboard/src/v2/components/chat/cinematic/CinematicStage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,6 @@ import { getChatWidgetData } from "../../../lib/chat-widget-view-models.js";
import { PlanningRequestWidget } from "../widgets/PlanningRequestWidget.js";
import { ExternalReferenceWidget } from "../widgets/ExternalReferenceWidget.js";
import { LazyAgentAvatarScene } from "../../agents/LazyAgentAvatarScene.js";
import {
AGENT_SCENE_TOOL_IDS,
isAgentSceneTool,
type AgentSceneTool,
} from "../../../lib/agent-scene-tools.js";
import { DEFAULT_AGENT_AVATAR_CONFIG } from "../../../lib/agent-avatar.js";
import { useReducedMotion } from "../../../hooks/use-reduced-motion.js";
import { resolveDisplayDeliveryStatus } from "../../../hooks/use-chat-thread-data.js";
Expand All @@ -35,6 +30,7 @@ import {
import { STAGE_ACTIVITY_MESSAGE_MIN_INTERVAL_MS } from "../../../lib/agent-humor-messages.js";
import { resolveCinematicActivityDisplayState } from "../../../lib/cinematic-activity.js";
import { StageActivityStrip } from "./StageActivityStrip.js";
import { useCinematicWorkTool } from "./use-cinematic-work-tool.js";

/* ════════════════════════════════════════════════════════════════════════
* CinematicStage — the default "3D Chat" view of the chat page.
Expand Down Expand Up @@ -80,10 +76,6 @@ export interface CinematicStageProps {
onOpenThreads: () => void;
}

/** The bot cycles through its toolbox while the runtime is executing. */
const WORK_TOOLS: readonly AgentSceneTool[] = AGENT_SCENE_TOOL_IDS;
const TOOL_SWAP_MS = 7_000;

const CREATE_APP_ACTION_ICONS: Record<DashboardCreateAppQuickactionKind, typeof Monitor> = {
web_app: Globe2,
desktop_app: Monitor,
Expand Down Expand Up @@ -141,13 +133,6 @@ const QUICK_ACTION_GROUPS = [
{ zone: "workflow", label: "Workflows" },
] as const;

/** Debug override: /chat?stageTool=wrench pins a specific tool on the stage. */
const readForcedTool = (): AgentSceneTool | null => {
if (typeof window === "undefined") return null;
const value = new URLSearchParams(window.location.search).get("stageTool");
return isAgentSceneTool(value) ? value : null;
};

const canSpeakAgentMessage = (message: ChatMessageRecord): boolean => (
!getChatWidgetData(message).suppressBodyMarkdown
&& speechTextFromMarkdown(message.bodyMarkdown || "").length > 0
Expand Down Expand Up @@ -544,23 +529,14 @@ export const CinematicStage: FunctionComponent<CinematicStageProps> = ({
}
};

/* Work tools — while the runtime is executing, the bot pulls a tool from
its toolbox and swaps to a fresh one every few seconds. */
const [activeTool, setActiveTool] = useState<AgentSceneTool | null>(readForcedTool);
useEffect(() => {
if (readForcedTool()) return; // pinned via ?stageTool= for design review
if (workingPhase !== "working") {
setActiveTool(null);
return;
}
let index = Math.floor(Math.random() * WORK_TOOLS.length);
setActiveTool(WORK_TOOLS[index]);
const timer = window.setInterval(() => {
index = (index + 1) % WORK_TOOLS.length;
setActiveTool(WORK_TOOLS[index]);
}, TOOL_SWAP_MS);
return () => window.clearInterval(timer);
}, [workingPhase]);
const activeTool = useCinematicWorkTool({
active: workingPhase === "working",
activityKey: activityState.foregroundCue?.id
?? selectedThread?.id
?? agentPreset?.id
?? "project-manager",
reducedMotion,
});

const mood: AgentMoodState = useAgentMood({
error,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
import { useEffect, useState } from "preact/hooks";
import {
AGENT_SCENE_TOOL_IDS,
isAgentSceneTool,
type AgentSceneTool,
} from "../../../lib/agent-scene-tools.js";

export interface UseCinematicWorkToolOptions {
active: boolean;
activityKey: string;
reducedMotion: boolean;
}

interface ActivityToolSelection {
activityKey: string;
tool: AgentSceneTool;
}

const TOOL_SWAP_MS = 7_000;

const readStageToolOverride = (): string | null => {
if (typeof window === "undefined") return null;
return new URLSearchParams(window.location.search).get("stageTool");
};

const getInitialToolIndex = (activityKey: string): number => {
let hash = 0;
for (let index = 0; index < activityKey.length; index += 1) {
hash = (Math.imul(hash, 31) + activityKey.charCodeAt(index)) >>> 0;
}
return hash % AGENT_SCENE_TOOL_IDS.length;
};

/**
* Selects the cinematic avatar's work tool for an activity already owned by
* the calling surface. Invocation ownership intentionally stays outside this
* hook so unrelated background work cannot activate the staged agent.
*/
export function useCinematicWorkTool({
active,
activityKey,
reducedMotion,
}: UseCinematicWorkToolOptions): AgentSceneTool | null {
const stageToolOverride = readStageToolOverride();
const forcedTool = isAgentSceneTool(stageToolOverride) ? stageToolOverride : null;
const initialToolIndex = getInitialToolIndex(activityKey);
const initialTool = AGENT_SCENE_TOOL_IDS[initialToolIndex];
const [selection, setSelection] = useState<ActivityToolSelection | null>(() => (
active && !forcedTool ? { activityKey, tool: initialTool } : null
));

useEffect(() => {
if (forcedTool || !active) {
setSelection(null);
return;
}

let toolIndex = initialToolIndex;
setSelection({ activityKey, tool: AGENT_SCENE_TOOL_IDS[toolIndex] });

if (reducedMotion) return;

const interval = window.setInterval(() => {
toolIndex = (toolIndex + 1) % AGENT_SCENE_TOOL_IDS.length;
setSelection({ activityKey, tool: AGENT_SCENE_TOOL_IDS[toolIndex] });
}, TOOL_SWAP_MS);

return () => window.clearInterval(interval);
}, [active, activityKey, forcedTool, initialToolIndex, reducedMotion, stageToolOverride]);

if (forcedTool) return forcedTool;
if (!active) return null;
if (selection?.activityKey !== activityKey) return initialTool;
return selection.tool;
}
2 changes: 1 addition & 1 deletion docs-web/content/docs/user-dashboard-chat.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ The 3D stage enters its Project Manager working state only while the selected th

The thought area turns known runtime fields into compact cues for container startup, provider work, planning, QA review, completion, and errors. Current stage cues come only from running records or the selected thread's awaited reply; old completed or failed invocations are not presented as live activity. A delegated-work cue stays visible while the Project Manager remains idle, and an active Project Manager cue takes precedence while retaining a count of other activity. The phase is shown directly without `Background` or provider-name prefixes. Its workplace-safe quote is keyed by stable agent, provider, phase, and runtime context and stays unchanged for at least twenty seconds. Delegated work uses 72 original agency and project-management jokes about coworker handoffs, meetings, scope creep, client feedback, and ticket rituals. The runtime shuffles the deck by context, uses every line before reshuffling, and avoids immediate repeats. Reduced-motion mode keeps the status text while stopping its decorative dots.

During the selected Project Manager's provider-working phase, the avatar rotates through five work tools every seven seconds: Power screwdriver (`screwdriver`), Jackhammer (`jackhammer`), Open-end wrench (`wrench`), Claw hammer (`hammer`), and Welding torch (`torch`). Container startup keeps the thinking state without a tool. Background work never equips a tool. Maintainers can pin a valid catalog identifier with `/chat?stageTool=<identifier>` for design review; missing or unsupported values leave normal runtime selection in place. Reduced-motion and WebGL fallback surfaces replace tool animation with the visible tool label and an accessible avatar description.
During the selected Project Manager's provider-working phase, the avatar rotates through five work tools every seven seconds: Power screwdriver (`screwdriver`), Jackhammer (`jackhammer`), Open-end wrench (`wrench`), Claw hammer (`hammer`), and Welding torch (`torch`). Each foreground activity key chooses a deterministic initial tool, then rotation follows catalog order without immediately repeating; a replacement activity starts its own deterministic sequence. Container startup keeps the thinking state without a tool, and background work never equips one because activity ownership is resolved before tool selection. Reduced motion keeps the initial tool static without a rotation timer. Maintainers can pin a valid catalog identifier with `/chat?stageTool=<identifier>` for design review, including while activity is inactive; missing or unsupported values leave normal runtime selection in place. Reduced-motion and WebGL fallback surfaces keep the visible tool label and accessible avatar description.

When a text-to-speech model or API is active under **Settings -> AI Models**, 3D Chat reads new Project Manager replies aloud. A compact control dock beneath the avatar identity holds the microphone and agent mute/unmute buttons, outside the composer. Voice defaults on, shows synthesis activity, and can be muted immediately. The preference is remembered per project in the current browser; opening an existing thread does not replay its latest historical message. Long replies start with the first ready sentence while a bounded two-chunk lookahead is synthesized, then continue through every chunk in transcript order. Muting, changing threads or Chat mode, leaving the page, or starting another replay cancels pending speech and releases the active audio. If synthesis or browser playback fails, playback stops and the accessible voice or transcript status reports the error without hiding the written reply.

Expand Down
2 changes: 1 addition & 1 deletion docs-web/user/dashboard/chat.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ The 3D stage enters its Project Manager working state only while the selected th

The thought area turns known runtime fields into compact cues for container startup, provider work, planning, QA review, completion, and errors. Current stage cues come only from running records or the selected thread's awaited reply; old completed or failed invocations are not presented as live activity. A delegated-work cue stays visible while the Project Manager remains idle, and an active Project Manager cue takes precedence while retaining a count of other activity. The phase is shown directly without `Background` or provider-name prefixes. Its workplace-safe quote is keyed by stable agent, provider, phase, and runtime context and stays unchanged for at least twenty seconds. Delegated work uses 72 original agency and project-management jokes about coworker handoffs, meetings, scope creep, client feedback, and ticket rituals. The runtime shuffles the deck by context, uses every line before reshuffling, and avoids immediate repeats. Reduced-motion mode keeps the status text while stopping its decorative dots.

During the selected Project Manager's provider-working phase, the avatar rotates through five work tools every seven seconds: Power screwdriver (`screwdriver`), Jackhammer (`jackhammer`), Open-end wrench (`wrench`), Claw hammer (`hammer`), and Welding torch (`torch`). Container startup keeps the thinking state without a tool. Background work never equips a tool. Maintainers can pin a valid catalog identifier with `/chat?stageTool=<identifier>` for design review; missing or unsupported values leave normal runtime selection in place. Reduced-motion and WebGL fallback surfaces replace tool animation with the visible tool label and an accessible avatar description.
During the selected Project Manager's provider-working phase, the avatar rotates through five work tools every seven seconds: Power screwdriver (`screwdriver`), Jackhammer (`jackhammer`), Open-end wrench (`wrench`), Claw hammer (`hammer`), and Welding torch (`torch`). Each foreground activity key chooses a deterministic initial tool, then rotation follows catalog order without immediately repeating; a replacement activity starts its own deterministic sequence. Container startup keeps the thinking state without a tool, and background work never equips one because activity ownership is resolved before tool selection. Reduced motion keeps the initial tool static without a rotation timer. Maintainers can pin a valid catalog identifier with `/chat?stageTool=<identifier>` for design review, including while activity is inactive; missing or unsupported values leave normal runtime selection in place. Reduced-motion and WebGL fallback surfaces keep the visible tool label and accessible avatar description.

When a text-to-speech model or API is active under **Settings -> AI Models**, 3D Chat reads new Project Manager replies aloud. A compact control dock beneath the avatar identity holds the microphone and agent mute/unmute buttons, outside the composer. Voice defaults on, shows synthesis activity, and can be muted immediately. The preference is remembered per project in the current browser; opening an existing thread does not replay its latest historical message. Long replies start with the first ready sentence while a bounded two-chunk lookahead is synthesized, then continue through every chunk in transcript order. Muting, changing threads or Chat mode, leaving the page, or starting another replay cancels pending speech and releases the active audio. If synthesis or browser playback fails, playback stops and the accessible voice or transcript status reports the error without hiding the written reply.

Expand Down
2 changes: 1 addition & 1 deletion docs/dashboard/design-system-chat.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ The chat page opens in a cinematic "3D Chat" stage (`components/chat/cinematic/C
- **Latest exchange spotlight**: the stage shows only the current beat — the newest agent reply as one glass speech bubble (markdown + widgets; long replies scroll *inside* the bubble) plus up to two user messages sent after it. A "Full conversation · N messages" link jumps to Threads for history. Quick actions use the open space beside the avatar while the bubble owns the right side. An empty thread shows a scripted greeting with suggestion chips that send directly without changing the composer.
- **Floating composer**: a bottom-center glass pill shared with the Threads data flow — Enter sends, first send auto-creates the thread, ArrowUp/Down recalls history.
- **Expressions catalog**: the avatar vocabulary (SVG + WebGL, kept in sync) now includes `curious`, `thinking`, `excited`, `laughing`, `surprised`, `wink`, `dance`, and `proud` in addition to the original eight.
- **Work tools**: while the selected Project Manager is in the provider-working phase the bot pulls an animated 3D work tool from its toolbox beside itself. The exact `AgentSceneTool` identifiers are `screwdriver` (spinning bit), `jackhammer` (piston/chisel), `wrench` (ratcheting swing), `hammer` (tapping swing), and `torch` (flickering welding tip). The stage starts at a random catalog position and advances in catalog order every 7,000 ms. `?stageTool=<identifier>` pins a valid tool for design review; an absent or unsupported value does not override runtime selection. Container startup shows the thinking state without a tool, and background project execution never equips one.
- **Work tools**: while the selected Project Manager is in the provider-working phase the bot pulls an animated 3D work tool from its toolbox beside itself. The exact `AgentSceneTool` identifiers are `screwdriver` (spinning bit), `jackhammer` (piston/chisel), `wrench` (ratcheting swing), `hammer` (tapping swing), and `torch` (flickering welding tip). The stage derives a deterministic initial catalog position from the foreground activity key and advances in catalog order every 7,000 ms without an immediate repeat; replacing the activity restarts selection predictably. Reduced motion keeps that activity's initial tool static and starts no rotation timer. `?stageTool=<identifier>` pins a valid tool for design review even while runtime activity is inactive; an absent or unsupported value does not override normal selection. Container startup shows the thinking state without a tool, and background project execution never equips one because the stage resolves ownership before calling the work-tool hook.
- **Idle quick actions**: the complete eligible 13-action stage set is **Create Web App**, **Create Desktop App**, **Create Onlineshop**, **Create Portfolio**, **Create Game**, **Status Report**, **Sprint Progress**, **What’s Failing?**, **Plan Next Steps**, **Add Nodes Workflow**, **Add Dashboard**, **Create Skill**, and **List Skills**. Desktop sorts them into subtle **Create**, **Project pulse**, and **Workflows** clusters contained entirely within the left stage viewbox. Each category uses a compact wrapping cluster, so Sprint Progress and Plan Next Steps remain beside the other Project pulse controls while Add Dashboard, Create Skill, and List Skills remain beside the other Workflows controls. Content-width neutral chips use small horizontal/vertical offsets, generous gaps, and staggered gentle drift to avoid both full-width controls and a mechanically aligned matrix while maintaining a whitespace buffer before the avatar. Mobile keeps the same category order in orderly content-width two-row horizontal groups. Each chip has a distinct colored icon tile for recognition while its card surface and interaction states remain consistent. Labels stay on one line, every action is a native keyboard-reachable button with a visible focus ring, and the floating animation stops under reduced motion. The five create-app actions dispatch typed `create_app` metadata and launch detached `Plan & Start` quicksprints; the eight informational and workflow actions send their catalog prompt through normal project chat without inserting into, replacing, or clearing the composer draft. All five create-app actions remain hidden until initial-project eligibility has loaded and is true; the other eight project actions remain available whenever the stage is idle. The full group hides without a selected project and while sending, working, or showing an error.
- **Reduced motion and fallback**: either the resolved reduced-motion preference or explicit `fallbackMode` selects the static SVG bot instead of creating a WebGL context. Aurora, thinking dots, quick-action float, drift, pointer gaze, tool motion, and response choreography stop. A selected work tool remains named in a visible static label and the fallback container's accessible image label. If WebGL renderer construction fails, the same SVG semantics apply. A response effect still exposes its validated semantic emotion and caption (or `Feeling <emotion>.`), so the reaction remains understandable without movement.

Expand Down
Loading
Loading