Skip to content
122 changes: 111 additions & 11 deletions packages/runner/src/builtins/llm-dialog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -593,6 +593,30 @@ function extractToolCallParts(
);
}

/**
* Validates whether message content is non-empty and valid for the Anthropic API.
* Returns true if the content contains at least one non-empty text block or tool call.
*/
function hasValidContent(content: BuiltInLLMMessage["content"]): boolean {
if (typeof content === "string") {
return content.trim().length > 0;
}

if (Array.isArray(content)) {
return content.some((part) => {
if (part.type === "tool-call" || part.type === "tool-result") {
return true;
}
if (part.type === "text") {
return (part as BuiltInLLMTextPart).text?.trim().length > 0;
}
return false;
});
}

return false;
}

function buildAssistantMessage(
content: BuiltInLLMMessage["content"],
toolCallParts: BuiltInLLMToolCallPart[],
Expand Down Expand Up @@ -664,17 +688,29 @@ async function executeToolCalls(
function createToolResultMessages(
results: ToolCallExecutionResult[],
): BuiltInLLMMessage[] {
return results.map((toolResult) => ({
role: "tool",
content: [{
type: "tool-result",
toolCallId: toolResult.id,
toolName: toolResult.toolName || "unknown",
output: toolResult.error
? { type: "error-text", value: toolResult.error }
: toolResult.result,
}],
}));
return results.map((toolResult) => {
// Ensure output is never undefined/null - Anthropic API requires valid tool_result
// for every tool_use, even if the tool returns nothing
let output: any;
if (toolResult.error) {
output = { type: "error-text", value: toolResult.error };
} else if (toolResult.result === undefined || toolResult.result === null) {
// Tool returned nothing - use explicit null value
output = { type: "json", value: null };
} else {
output = toolResult.result;
}

return {
role: "tool",
content: [{
type: "tool-result",
toolCallId: toolResult.id,
toolName: toolResult.toolName || "unknown",
output,
}],
};
});
}

export const llmDialogTestHelpers = {
Expand All @@ -684,6 +720,7 @@ export const llmDialogTestHelpers = {
extractToolCallParts,
buildAssistantMessage,
createToolResultMessages,
hasValidContent,
};

/**
Expand Down Expand Up @@ -1080,6 +1117,34 @@ async function startRequest(

resultPromise
.then(async (llmResult) => {
// Validate that the response has valid content
if (!hasValidContent(llmResult.content)) {
// LLM returned empty or invalid content (e.g., stream aborted mid-flight,
// or AI SDK bug with empty text blocks). Insert a proper error message
// instead of storing invalid content.
logger.warn("LLM returned invalid/empty content, adding error message");
const errorMessage = {
[ID]: { llmDialog: { message: cause, id: crypto.randomUUID() } },
role: "assistant",
content:
"I encountered an error generating a response. Please try again.",
} satisfies BuiltInLLMMessage & { [ID]: unknown };

await safelyPerformUpdate(
runtime,
pending,
internal,
requestId,
(tx) => {
messagesCell.withTx(tx).push(
errorMessage as Schema<typeof LLMMessageSchema>,
);
pending.withTx(tx).set(false);
},
);
return;
}

// Extract tool calls from content if it's an array
const hasToolCalls = Array.isArray(llmResult.content) &&
llmResult.content.some((part) => part.type === "tool-call");
Expand All @@ -1098,6 +1163,41 @@ async function startRequest(
toolCatalog,
toolCallParts,
);

// Validate that we have a result for every tool call with matching IDs
const toolCallIds = new Set(toolCallParts.map((p) => p.toolCallId));
const resultIds = new Set(toolResults.map((r) => r.id));
const mismatch = toolResults.length !== toolCallParts.length ||
!toolCallParts.every((p) => resultIds.has(p.toolCallId));

if (mismatch) {
logger.error(
`Tool execution mismatch: ${toolCallParts.length} calls [${
Array.from(toolCallIds)
}] but ${toolResults.length} results [${Array.from(resultIds)}]`,
);
// Add error message instead of invalid partial results
const errorMessage = {
[ID]: { llmDialog: { message: cause, id: crypto.randomUUID() } },
role: "assistant",
content: "Some tool calls failed to execute. Please try again.",
} satisfies BuiltInLLMMessage & { [ID]: unknown };

await safelyPerformUpdate(
runtime,
pending,
internal,
requestId,
(tx) => {
messagesCell.withTx(tx).push(
errorMessage as Schema<typeof LLMMessageSchema>,
);
pending.withTx(tx).set(false);
},
);
return;
}

const newMessages: BuiltInLLMMessage[] = [
assistantMessage,
...createToolResultMessages(toolResults),
Expand Down
70 changes: 70 additions & 0 deletions packages/runner/test/llm-dialog-helpers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ const {
extractToolCallParts,
buildAssistantMessage,
createToolResultMessages,
hasValidContent,
} = llmDialogTestHelpers;

Deno.test("createCharmToolDefinitions slugifies charm names and returns tool metadata", () => {
Expand Down Expand Up @@ -112,3 +113,72 @@ Deno.test("createToolResultMessages converts execution results into tool message
assertEquals(failurePart.toolName, "failing");
assertEquals(failurePart.output, { type: "error-text", value: "boom" });
});

Deno.test("hasValidContent returns true for non-empty text", () => {
assert(hasValidContent("Hello"));
assert(hasValidContent([{ type: "text", text: "Hello" }]));
});

Deno.test("hasValidContent returns false for empty text", () => {
assert(!hasValidContent(""));
assert(!hasValidContent(" \n "));
assert(!hasValidContent([{ type: "text", text: "" }]));
assert(!hasValidContent([{ type: "text", text: " " }]));
});

Deno.test("hasValidContent returns true for tool calls", () => {
const content: BuiltInLLMMessage["content"] = [
{ type: "text", text: "" },
{ type: "tool-call", toolCallId: "1", toolName: "test", input: {} },
];
assert(hasValidContent(content));
});

Deno.test("hasValidContent returns true for tool results", () => {
const content: BuiltInLLMMessage["content"] = [
{
type: "tool-result",
toolCallId: "1",
toolName: "test",
output: { type: "json", value: null },
},
];
assert(hasValidContent(content));
});

Deno.test("hasValidContent returns false for only empty text parts", () => {
const content: BuiltInLLMMessage["content"] = [
{ type: "text", text: "" },
{ type: "text", text: " " },
];
assert(!hasValidContent(content));
});

Deno.test("createToolResultMessages handles undefined result with explicit null", () => {
const messages = createToolResultMessages([{
id: "call-1",
toolName: "empty",
result: undefined,
}]);

assertEquals(messages.length, 1);
assertEquals(messages[0].role, "tool");
assertEquals(messages[0].content?.[0], {
type: "tool-result",
toolCallId: "call-1",
toolName: "empty",
output: { type: "json", value: null },
});
});

Deno.test("createToolResultMessages handles null result with explicit null", () => {
const messages = createToolResultMessages([{
id: "call-1",
toolName: "empty",
result: null,
}]);

assertEquals(messages.length, 1);
const outputPart = messages[0].content?.[0] as any;
assertEquals(outputPart.output, { type: "json", value: null });
});