diff --git a/Unity/Assets/_SecondSpawn/Scripts/AI/PrototypeAgentBrain.cs b/Unity/Assets/_SecondSpawn/Scripts/AI/PrototypeAgentBrain.cs index 3f0a5fe..081c6e7 100644 --- a/Unity/Assets/_SecondSpawn/Scripts/AI/PrototypeAgentBrain.cs +++ b/Unity/Assets/_SecondSpawn/Scripts/AI/PrototypeAgentBrain.cs @@ -91,6 +91,7 @@ private enum BrainPhase private float _nextNpcSpeechResponseAt; private float _talkVisualUntil; private float _dialogHoldUntil; + private float _answeringStatusHoldUntil; private Vector3 _dialogLookPosition; private bool _hasDialogLookPosition; private int _heldDecisionSlots; @@ -384,6 +385,17 @@ private IEnumerator BrainLoop() continue; } + if (IsDialogFocusActive() && !HasFreshPendingPlayerChat()) + { + if (!IsAnsweringStatusActive()) + { + SetBrainStatus("AI listening", new Color(0.82f, 0.86f, 0.9f), "dialog focus"); + } + + yield return new WaitForSeconds(0.35f); + continue; + } + if (NpcDecisionRequestScheduler.IsPlayerChatFocusActive && !HasFreshPendingPlayerChat()) { SetBrainStatus("AI listening", new Color(0.82f, 0.86f, 0.9f), "player dialog nearby"); @@ -401,15 +413,28 @@ private IEnumerator BrainLoop() string decisionError = null; var isPlayerChatDecision = HasFreshPendingPlayerChat(); - if (!isPlayerChatDecision && NpcDecisionRequestScheduler.IsAutonomousQueueFull) + if (!isPlayerChatDecision && CanShowAutonomousDecisionStatus() && NpcDecisionRequestScheduler.IsAutonomousQueueFull) { SetBrainStatus("AI queued", new Color(0.72f, 0.82f, 0.95f)); } yield return NpcDecisionRequestScheduler.WaitForSlot( isPlayerChatDecision, - () => SetBrainStatus(isPlayerChatDecision ? "AI answering" : "AI queued", new Color(0.72f, 0.82f, 0.95f))); + () => + { + if (isPlayerChatDecision || CanShowAutonomousDecisionStatus()) + { + SetBrainStatus(isPlayerChatDecision ? "AI answering" : "AI queued", new Color(0.72f, 0.82f, 0.95f)); + } + }); _heldDecisionSlots++; + if (!isPlayerChatDecision && !CanShowAutonomousDecisionStatus()) + { + ReleaseHeldDecisionSlot(); + yield return new WaitForSeconds(0.1f); + continue; + } + SetBrainStatus("AI DOS.AI request", new Color(0.72f, 0.82f, 0.95f)); try { @@ -958,6 +983,21 @@ private bool HasFreshPendingPlayerChat() Time.time - _pendingPlayerChatAt <= PlayerChatContextLifetimeSeconds; } + private bool IsDialogFocusActive() + { + return Time.time < _dialogHoldUntil; + } + + private bool IsAnsweringStatusActive() + { + return _playerChatDecisionInFlight || Time.time < _answeringStatusHoldUntil; + } + + private bool CanShowAutonomousDecisionStatus() + { + return !HasFreshPendingPlayerChat() && !IsDialogFocusActive(); + } + private IEnumerator PlayerChatResponseLoop() { if (_playerChatDecisionInFlight) @@ -974,13 +1014,13 @@ private IEnumerator PlayerChatResponseLoop() AgentDecisionDto decision = null; string decisionError = null; var request = BuildDecisionRequest(); - SetBrainStatus("AI answering", new Color(0.72f, 0.82f, 0.95f), $"player chat attempt {_pendingPlayerChatAttemptCount}"); + SetAnsweringStatus($"player chat attempt {_pendingPlayerChatAttemptCount}"); PrototypeNearbyNpcChatBox.TryNotifyFocusedNpcResponseStarted(AgentId, DisplayName, _pendingPlayerChatAttemptCount); var stimulus = _pendingMessageFromNpc ? "npc_speech" : "player_chat"; Debug.Log($"[PrototypeAgentBrain] Player chat decision request agent={AgentId}, attempt={_pendingPlayerChatAttemptCount}, stimulus={stimulus}, allowed={string.Join(",", request.allowed ?? new string[0])}, message={request.world_snapshot?.last_player_message?.text}"); yield return NpcDecisionRequestScheduler.WaitForSlot( true, - () => SetBrainStatus("AI answering", new Color(0.72f, 0.82f, 0.95f))); + () => SetAnsweringStatus("priority slot wait")); _heldDecisionSlots++; try { @@ -1023,6 +1063,7 @@ private IEnumerator PlayerChatResponseLoop() } _playerChatDecisionInFlight = false; + _answeringStatusHoldUntil = 0f; _playerChatResponseLoop = null; } @@ -1242,6 +1283,10 @@ private IEnumerator ApplyDecision(AgentDecisionDto decision, AgentDecisionReques Debug.LogWarning($"[PrototypeAgentBrain] Suppressed non-model speech agent={AgentId}, source={decision.source}, source_reason={decision.source_reason}, action={decision.action}"); if (request.world_snapshot?.last_player_message != null) { + PrototypeNearbyNpcChatBox.TryNotifyFocusedNpcResponseFailed( + AgentId, + DisplayName, + FirstNonEmpty(decision.source_reason, "fallback")); ClearPendingPlayerChat(); } yield break; @@ -1254,6 +1299,7 @@ private IEnumerator ApplyDecision(AgentDecisionDto decision, AgentDecisionReques Debug.LogWarning($"[PrototypeAgentBrain] Suppressed empty model speech agent={AgentId}, source={decision.source}, source_reason={decision.source_reason}"); if (request.world_snapshot?.last_player_message != null) { + PrototypeNearbyNpcChatBox.TryNotifyFocusedNpcResponseFailed(AgentId, DisplayName, "empty_model_speech"); ClearPendingPlayerChat(); } yield break; @@ -1325,21 +1371,36 @@ private void BeginTalkVisuals() ApplyLocomotion(0f); } - public void BeginDialogFocus(Vector3 lookPosition, float seconds) + private void SetAnsweringStatus(string reason) { + _answeringStatusHoldUntil = Mathf.Max(_answeringStatusHoldUntil, Time.time + 45f); + SetBrainStatus("AI answering", new Color(0.72f, 0.82f, 0.95f), reason); + } + + public void BeginDialogFocus(Vector3 lookPosition, float seconds, bool updateStatus = true) + { + var holdSeconds = Mathf.Max(0f, seconds); _hasMoveTarget = false; - _dialogHoldUntil = Mathf.Max(_dialogHoldUntil, Time.time + Mathf.Max(0f, seconds)); + _dialogHoldUntil = Mathf.Max(_dialogHoldUntil, Time.time + holdSeconds); _dialogLookPosition = lookPosition; _hasDialogLookPosition = true; + _talkVisualUntil = Mathf.Max(_talkVisualUntil, Time.time + holdSeconds); FacePosition(lookPosition); ApplyLocomotion(0f); - SetBrainStatus("AI dialog", new Color(0.74f, 0.86f, 0.92f), "dialog focus"); + if (updateStatus) + { + SetBrainStatus("AI dialog", new Color(0.74f, 0.86f, 0.92f), "dialog focus"); + } } public void EndDialogFocus() { _dialogHoldUntil = 0f; + _answeringStatusHoldUntil = 0f; _hasDialogLookPosition = false; + _talkVisualUntil = 0f; + RestoreWeaponProps(); + ApplyLocomotion(0f); SetBrainStatus("AI ready", new Color(0.82f, 0.86f, 0.9f), "dialog ended"); } diff --git a/Unity/Assets/_SecondSpawn/Scripts/AI/PrototypeNearbyNpcChatBox.cs b/Unity/Assets/_SecondSpawn/Scripts/AI/PrototypeNearbyNpcChatBox.cs index 88c0ee0..e62a409 100644 --- a/Unity/Assets/_SecondSpawn/Scripts/AI/PrototypeNearbyNpcChatBox.cs +++ b/Unity/Assets/_SecondSpawn/Scripts/AI/PrototypeNearbyNpcChatBox.cs @@ -433,7 +433,7 @@ private static void EnterLocalDialogMode(NetworkPlayer player, PrototypeAgentBra PrototypeInputFocusGate.LockForDialog(DialogInputLockSeconds); player.FaceWorldPosition(brain.transform.position); player.BeginDialogVisual(DialogInputLockSeconds); - brain.BeginDialogFocus(player.transform.position, DialogInputLockSeconds); + brain.BeginDialogFocus(player.transform.position, DialogInputLockSeconds, true); Debug.Log($"[PrototypeNearbyNpcChatBox] Dialog mode entered player={player.name}, npc={brain.DisplayName}, hold_seconds={DialogInputLockSeconds:0.0}"); } @@ -470,7 +470,7 @@ private void MaintainDialogFacing() PrototypeInputFocusGate.LockForDialog(0.4f); player.FaceWorldPosition(brain.transform.position); player.BeginDialogVisual(0.75f); - brain.BeginDialogFocus(player.transform.position, 0.25f); + brain.BeginDialogFocus(player.transform.position, 0.25f, false); } private List ResolveFocusedNpcRecipient() diff --git a/Unity/Assets/_SecondSpawn/Scripts/UI/NearbyNpcChatPanel.cs b/Unity/Assets/_SecondSpawn/Scripts/UI/NearbyNpcChatPanel.cs index af7abb1..222e65d 100644 --- a/Unity/Assets/_SecondSpawn/Scripts/UI/NearbyNpcChatPanel.cs +++ b/Unity/Assets/_SecondSpawn/Scripts/UI/NearbyNpcChatPanel.cs @@ -326,7 +326,6 @@ private void RenderDialogueRows() return; } - var firstIsPlayer = _chat.DialogueLines[0].is_player; foreach (var line in _chat.DialogueLines) { if (line.is_system) @@ -335,7 +334,7 @@ private void RenderDialogueRows() continue; } - CreateMessageRow(line, line.is_player == firstIsPlayer); + CreateMessageRow(line, line.is_player); } } @@ -388,7 +387,8 @@ private void CreateMessageRow(PrototypeNearbyNpcChatBox.DialogueLine line, bool } var bubble = CreateRect("Bubble", row); - var bubbleWidth = _chat.IsChatModeActive ? 660f : 380f; + var lineText = $"{line.speaker}: {line.text}"; + var bubbleWidth = EstimateMessageWidth(lineText, _chat.IsChatModeActive); SetPreferredWidth(bubble, bubbleWidth); var image = bubble.gameObject.AddComponent(); image.color = line.is_player @@ -400,10 +400,10 @@ private void CreateMessageRow(PrototypeNearbyNpcChatBox.DialogueLine line, bool label.rectTransform.anchorMax = Vector2.one; label.rectTransform.offsetMin = new Vector2(12f, 7f); label.rectTransform.offsetMax = new Vector2(-12f, -7f); - label.alignment = line.is_player ? TextAnchor.MiddleRight : TextAnchor.MiddleLeft; + label.alignment = TextAnchor.MiddleLeft; label.horizontalOverflow = HorizontalWrapMode.Wrap; label.verticalOverflow = VerticalWrapMode.Truncate; - label.text = $"{line.speaker}: {line.text}"; + label.text = lineText; if (leftSide) { @@ -425,6 +425,32 @@ private static float EstimateMessageHeight(string text, bool dialogMode) return Mathf.Clamp(34f + lineCount * (dialogMode ? 27f : 22f), 54f, dialogMode ? 142f : 104f); } + private static float EstimateMessageWidth(string text, bool dialogMode) + { + var minWidth = dialogMode ? 180f : 150f; + var maxWidth = dialogMode ? 660f : 380f; + var averageGlyphWidth = dialogMode ? 10.5f : 8.5f; + var longestLine = LongestUnwrappedLineLength(text); + return Mathf.Clamp(28f + longestLine * averageGlyphWidth, minWidth, maxWidth); + } + + private static int LongestUnwrappedLineLength(string text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return 0; + } + + var longest = 0; + var lines = text.Split('\n'); + foreach (var line in lines) + { + longest = Mathf.Max(longest, line.Trim().Length); + } + + return longest; + } + private void QueueScrollToBottom() { _scrollToBottomPending = true; diff --git a/backend/nakama/local.example.yml b/backend/nakama/local.example.yml index 32c6cba..db8432e 100644 --- a/backend/nakama/local.example.yml +++ b/backend/nakama/local.example.yml @@ -12,7 +12,7 @@ runtime: - "SECOND_SPAWN_ENABLE_DEBUG_BODYTIME=false" - "DOS_AI_BASE_URL=https://api.dos.ai/v1" - "AGENT_DECISION_MODEL=dos-ai" - - "DOS_AI_DECISION_TIMEOUT_MS=120000" + - "DOS_AI_DECISION_TIMEOUT_MS=30000" - "DOS_AI_DECISION_BACKOFF_SECONDS=5" - "DOS_AI_DECISION_DAILY_REQUEST_LIMIT=1000" - "DOS_AI_DECISION_DAILY_TOKEN_BUDGET=250000" diff --git a/backend/nakama/modules/index.ts b/backend/nakama/modules/index.ts index 093d790..8b9c796 100644 --- a/backend/nakama/modules/index.ts +++ b/backend/nakama/modules/index.ts @@ -4344,7 +4344,7 @@ function tryDosAiAgentDecision( }; } - var decision = parseModelDecisionContent(content); + var decision = parseModelDecisionContent(content, allowed, world); sanitizeModelDecisionIntent(decision, world); var validationError = validateAgentDecisionIntent(decision, allowed, world); if (validationError) { @@ -5108,7 +5108,7 @@ function compactPlayerChatContext(message: any): any { }; } -function parseModelDecisionContent(content: string): any { +function parseModelDecisionContent(content: string, allowed?: string[], world?: any): any { var normalized = trimString(content); if (normalized.indexOf("```json") === 0) { normalized = trimString(normalized.substring(7)); @@ -5125,7 +5125,68 @@ function parseModelDecisionContent(content: string): any { normalized = trimString(normalized.substring(firstBrace, lastBrace + 1)); } } - return unwrapModelDecision(parseJson(normalized, "model decision")); + + try { + return unwrapModelDecision(parseJson(normalized, "model decision")); + } catch (err) { + var normalizedSpeech = normalizeModelSpeechDecision(normalized, allowed || [], world || {}); + if (normalizedSpeech) { + return normalizedSpeech; + } + + throw err; + } +} + +function normalizeModelSpeechDecision(content: string, allowed: string[], world: any): any { + if (!arrayContains(allowed, "say")) { + return null; + } + + var speech = extractModelSpeechText(content); + if (!speech) { + return null; + } + + var explicitSpeech = isExplicitSpeechText(content); + if (!explicitSpeech && !hasPlayerMessage(world)) { + return null; + } + + return { + action: "say", + target_id: selectFallbackSayTargetId(world), + say: truncateForLog(speech, hasPlayerMessage(world) ? 180 : 90), + reason: "model_speech_normalized", + confidence: 0.45 + }; +} + +function isExplicitSpeechText(content: string): boolean { + var value = lowercase(trimString(content)); + return value.indexOf("say:") === 0 || + value.indexOf("say ") === 0 || + value.indexOf("reply:") === 0 || + value.indexOf("speech:") === 0; +} + +function extractModelSpeechText(content: string): string { + var value = trimString(content); + if (!value) { + return ""; + } + + value = value.replace(/^(say|reply|speech)\s*:\s*/i, ""); + value = trimString(value); + if (value.length >= 2) { + var first = value.charAt(0); + var last = value.charAt(value.length - 1); + if ((first === "\"" && last === "\"") || (first === "'" && last === "'")) { + value = trimString(value.substring(1, value.length - 1)); + } + } + + return value; } function unwrapModelDecision(parsed: any): any { diff --git a/backend/nakama/tests/supabase_custom_auth.test.mjs b/backend/nakama/tests/supabase_custom_auth.test.mjs index 05c12d3..856158c 100644 --- a/backend/nakama/tests/supabase_custom_auth.test.mjs +++ b/backend/nakama/tests/supabase_custom_auth.test.mjs @@ -1457,6 +1457,66 @@ assert.equal(playerChatFallbackDecision.action, "stop"); assert.equal(playerChatFallbackDecision.reason, "model_unavailable_for_player_chat"); assert.equal(playerChatFallbackDecision.say, undefined); +const playerChatProseHarness = createRuntimeHarness(module); +playerChatProseHarness.nk.httpRequest = () => ({ + code: 200, + body: JSON.stringify({ choices: [{ message: { content: "I keep the south gate because the yard still leaks bad routes." } }] }) +}); +const playerChatProseDecision = JSON.parse(playerChatProseHarness.registeredRpcs.get("secondspawn_agent_decide")( + { + userId: "player-chat-prose-user", + env: { + DOS_AI_API_KEY: "dos-ai-test-key", + DOS_AI_BASE_URL: "https://api.dos.ai/v1", + AGENT_DECISION_MODEL: "dos-ai" + } + }, + playerChatProseHarness.logger, + playerChatProseHarness.nk, + JSON.stringify({ + world_snapshot: { + position: { x: 2, z: 3 }, + body_time_seconds: 3600, + last_player_message: { + player_actor_id: "player-body-local-1", + player_display_name: "JOY", + text: "Why are you here?" + }, + nearby_actors: [{ id: "player-body-local-1", distance: 2.5 }] + }, + allowed: ["say", "stop"] + }) +)); +assert.equal(playerChatProseDecision.source, "model"); +assert.equal(playerChatProseDecision.action, "say"); +assert.equal(playerChatProseDecision.say, "I keep the south gate because the yard still leaks bad routes."); +assert.equal(playerChatProseDecision.reason, "model_speech_normalized"); + +const explicitSayHarness = createRuntimeHarness(module); +explicitSayHarness.nk.httpRequest = () => ({ + code: 200, + body: JSON.stringify({ choices: [{ message: { content: "say: \"Your vitals are steady. Stay near Ward C.\"" } }] }) +}); +const explicitSayDecision = JSON.parse(explicitSayHarness.registeredRpcs.get("secondspawn_agent_decide")( + { + userId: "explicit-say-user", + env: { + DOS_AI_API_KEY: "dos-ai-test-key", + DOS_AI_BASE_URL: "https://api.dos.ai/v1", + AGENT_DECISION_MODEL: "dos-ai" + } + }, + explicitSayHarness.logger, + explicitSayHarness.nk, + JSON.stringify({ + world_snapshot: { position: { x: 2, z: 3 }, body_time_seconds: 3600 }, + allowed: ["say", "stop"] + }) +)); +assert.equal(explicitSayDecision.source, "model"); +assert.equal(explicitSayDecision.action, "say"); +assert.equal(explicitSayDecision.say, "Your vitals are steady. Stay near Ward C."); + const personaFallbackHarness = createRuntimeHarness(module); function npcPersonaFallbackSay(actorId) { return JSON.parse(personaFallbackHarness.registeredRpcs.get("secondspawn_agent_decide")(