From 034a846b95035309185b0c032c25fb56ef0ee255 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Feb 2026 01:15:28 +0000 Subject: [PATCH 1/2] Filter out empty messages from LLM prompt construction Empty messages (with no content and no tool calls) could be sent to the LLM, which is wasteful and potentially causes issues. This change: - Skips assistant messages with empty body and no tool calls during history construction (matching the existing guard on user messages) - Adds a safety filter (hasMessageContent) on the final messages array to catch any empty messages from any source, while preserving assistant messages with tool calls and tool response messages CS-10116 https://claude.ai/code/session_01JMzG85yR6uQUXVfnUTegXz --- .../ai-bot/tests/prompt-construction-test.ts | 262 ++++++++++++++++++ packages/runtime-common/ai/prompt.ts | 38 ++- 2 files changed, 292 insertions(+), 8 deletions(-) diff --git a/packages/ai-bot/tests/prompt-construction-test.ts b/packages/ai-bot/tests/prompt-construction-test.ts index b050f1bfbec..8cd3dbe1fb2 100644 --- a/packages/ai-bot/tests/prompt-construction-test.ts +++ b/packages/ai-bot/tests/prompt-construction-test.ts @@ -5150,6 +5150,268 @@ new 'cache_control should be set to ephemeral', ); }); + + test('excludes assistant messages with empty body and no tool calls', async () => { + const history: DiscreteMatrixEvent[] = [ + { + type: 'm.room.message', + event_id: '1', + origin_server_ts: 1, + content: { + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + body: 'Hello', + isStreamingFinished: true, + data: { + context: { + submode: 'interact', + }, + }, + }, + sender: '@user:localhost', + room_id: 'room1', + unsigned: { + age: 1000, + transaction_id: '1', + }, + status: EventStatus.SENT, + }, + { + type: 'm.room.message', + event_id: '2', + origin_server_ts: 2, + content: { + body: '', + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + isStreamingFinished: true, + }, + sender: '@aibot:localhost', + room_id: 'room1', + unsigned: { + age: 1000, + transaction_id: '2', + }, + status: EventStatus.SENT, + }, + { + type: 'm.room.message', + event_id: '3', + origin_server_ts: 3, + content: { + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + body: 'Can you help me?', + isStreamingFinished: true, + data: { + context: { + submode: 'interact', + }, + }, + }, + sender: '@user:localhost', + room_id: 'room1', + unsigned: { + age: 1000, + transaction_id: '3', + }, + status: EventStatus.SENT, + }, + ]; + + const result = await buildPromptForModel( + history, + '@aibot:localhost', + undefined, + undefined, + [], + fakeMatrixClient, + ); + + const assistantMessages = result.filter( + (message) => message.role === 'assistant', + ); + assert.equal( + assistantMessages.length, + 0, + 'Empty assistant message should not be included', + ); + + const userMessages = result.filter((message) => message.role === 'user'); + assert.equal( + userMessages.length, + 2, + 'Both user messages should be included', + ); + }); + + test('keeps assistant messages with empty body when they have tool calls', async () => { + const history: DiscreteMatrixEvent[] = [ + { + type: 'm.room.message', + event_id: '1', + origin_server_ts: 1, + content: { + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + body: 'Update my card', + isStreamingFinished: true, + data: { + context: { + submode: 'interact', + }, + }, + }, + sender: '@user:localhost', + room_id: 'room1', + unsigned: { + age: 1000, + transaction_id: '1', + }, + status: EventStatus.SENT, + }, + { + type: 'm.room.message', + event_id: '2', + origin_server_ts: 2, + content: { + body: '', + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + isStreamingFinished: true, + [APP_BOXEL_COMMAND_REQUESTS_KEY]: [ + { + toolCallId: 'call_1', + name: 'patchCardInstance', + arguments: JSON.stringify({ + card_id: 'http://localhost/card/1', + attributes: { title: 'Updated' }, + }), + }, + ], + }, + sender: '@aibot:localhost', + room_id: 'room1', + unsigned: { + age: 1000, + transaction_id: '2', + }, + status: EventStatus.SENT, + }, + { + type: APP_BOXEL_COMMAND_RESULT_EVENT_TYPE, + event_id: '3', + origin_server_ts: 3, + content: { + msgtype: APP_BOXEL_COMMAND_RESULT_WITH_NO_OUTPUT_MSGTYPE, + body: 'Command result', + commandRequestId: 'call_1', + 'm.relates_to': { + rel_type: APP_BOXEL_COMMAND_RESULT_REL_TYPE, + event_id: '2', + key: 'applied', + }, + }, + sender: '@user:localhost', + room_id: 'room1', + unsigned: { + age: 1000, + transaction_id: '3', + }, + status: EventStatus.SENT, + }, + ]; + + const result = await buildPromptForModel( + history, + '@aibot:localhost', + undefined, + undefined, + [], + fakeMatrixClient, + ); + + const assistantMessages = result.filter( + (message) => message.role === 'assistant', + ); + assert.equal( + assistantMessages.length, + 1, + 'Assistant message with tool calls should be kept even with empty body', + ); + assert.ok( + assistantMessages[0].tool_calls?.length, + 'Assistant message should have tool calls', + ); + }); + + test('excludes user messages with empty body', async () => { + const history: DiscreteMatrixEvent[] = [ + { + type: 'm.room.message', + event_id: '1', + origin_server_ts: 1, + content: { + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + body: '', + isStreamingFinished: true, + data: { + context: { + submode: 'interact', + }, + }, + }, + sender: '@user:localhost', + room_id: 'room1', + unsigned: { + age: 1000, + transaction_id: '1', + }, + status: EventStatus.SENT, + }, + { + type: 'm.room.message', + event_id: '2', + origin_server_ts: 2, + content: { + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + body: 'Hello', + isStreamingFinished: true, + data: { + context: { + submode: 'interact', + }, + }, + }, + sender: '@user:localhost', + room_id: 'room1', + unsigned: { + age: 1000, + transaction_id: '2', + }, + status: EventStatus.SENT, + }, + ]; + + const result = await buildPromptForModel( + history, + '@aibot:localhost', + undefined, + undefined, + [], + fakeMatrixClient, + ); + + const userMessages = result.filter((message) => message.role === 'user'); + assert.equal( + userMessages.length, + 1, + 'Only the non-empty user message should be included', + ); + assert.equal(userMessages[0].content, 'Hello'); + }); }); module('set model in prompt', (hooks) => { diff --git a/packages/runtime-common/ai/prompt.ts b/packages/runtime-common/ai/prompt.ts index 57d8231479b..76d3f98131d 100644 --- a/packages/runtime-common/ai/prompt.ts +++ b/packages/runtime-common/ai/prompt.ts @@ -1208,15 +1208,18 @@ export async function buildPromptForModel( event as CardMessageEvent, history, ); - let historicalMessage: OpenAIPromptMessage = { - role: 'assistant', - content: elideCodeBlocks(body, codePatchResults), - }; + let content = elideCodeBlocks(body, codePatchResults); let toolCalls = toToolCalls(event as CardMessageEvent); - if (toolCalls.length) { - historicalMessage.tool_calls = toolCalls; + if (content || toolCalls.length) { + let historicalMessage: OpenAIPromptMessage = { + role: 'assistant', + content, + }; + if (toolCalls.length) { + historicalMessage.tool_calls = toolCalls; + } + historicalMessages.push(historicalMessage); } - historicalMessages.push(historicalMessage); let commandResults = getCommandResults( event as CardMessageEvent, history, @@ -1313,7 +1316,26 @@ export async function buildPromptForModel( } } - return messages; + return messages.filter(hasMessageContent); +} + +function hasMessageContent(message: OpenAIPromptMessage): boolean { + // Assistant messages with tool calls are valid even without text content + if (message.role === 'assistant' && message.tool_calls?.length) { + return true; + } + // Tool messages are required responses to tool calls + if (message.role === 'tool') { + return true; + } + let content = message.content; + if (typeof content === 'string') { + return content.length > 0; + } + if (Array.isArray(content)) { + return content.length > 0; + } + return false; } function collectPendingCodePatchCorrectnessCheck( From 01b3b74ac16496d0247cccfcc2e001a8c7b62f7e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Feb 2026 01:24:07 +0000 Subject: [PATCH 2/2] Remove over-engineered safety filter, fix test field name - Remove hasMessageContent filter: it was redundant with the inline guards already present for each message type and would silently mask construction bugs rather than surfacing them - Fix test: use correct field name `id` instead of `toolCallId` for encoded command requests (matching all other tests in the file) - Add tool result assertions to verify the assistant+tool message pairing is preserved when assistant body is empty https://claude.ai/code/session_01JMzG85yR6uQUXVfnUTegXz --- .../ai-bot/tests/prompt-construction-test.ts | 14 ++++++++++++- packages/runtime-common/ai/prompt.ts | 21 +------------------ 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/packages/ai-bot/tests/prompt-construction-test.ts b/packages/ai-bot/tests/prompt-construction-test.ts index 8cd3dbe1fb2..8e0e2037753 100644 --- a/packages/ai-bot/tests/prompt-construction-test.ts +++ b/packages/ai-bot/tests/prompt-construction-test.ts @@ -5281,7 +5281,7 @@ new isStreamingFinished: true, [APP_BOXEL_COMMAND_REQUESTS_KEY]: [ { - toolCallId: 'call_1', + id: 'call_1', name: 'patchCardInstance', arguments: JSON.stringify({ card_id: 'http://localhost/card/1', @@ -5343,6 +5343,18 @@ new assistantMessages[0].tool_calls?.length, 'Assistant message should have tool calls', ); + + const toolMessages = result.filter((message) => message.role === 'tool'); + assert.equal( + toolMessages.length, + 1, + 'Tool result message should be present alongside the assistant tool call', + ); + assert.equal( + toolMessages[0].tool_call_id, + 'call_1', + 'Tool result should reference the correct tool call id', + ); }); test('excludes user messages with empty body', async () => { diff --git a/packages/runtime-common/ai/prompt.ts b/packages/runtime-common/ai/prompt.ts index 76d3f98131d..4645abdbd1d 100644 --- a/packages/runtime-common/ai/prompt.ts +++ b/packages/runtime-common/ai/prompt.ts @@ -1316,26 +1316,7 @@ export async function buildPromptForModel( } } - return messages.filter(hasMessageContent); -} - -function hasMessageContent(message: OpenAIPromptMessage): boolean { - // Assistant messages with tool calls are valid even without text content - if (message.role === 'assistant' && message.tool_calls?.length) { - return true; - } - // Tool messages are required responses to tool calls - if (message.role === 'tool') { - return true; - } - let content = message.content; - if (typeof content === 'string') { - return content.length > 0; - } - if (Array.isArray(content)) { - return content.length > 0; - } - return false; + return messages; } function collectPendingCodePatchCorrectnessCheck(