Open History to see the last commit
1import { test, expect } from '../fixtures'
2import {
3 selectScenario,
4 runTest,
5 waitForTestComplete,
6 getMetadata,
7 getToolCalls,
8} from './helpers'
9
10/**
11 * Null Tool Input E2E Tests
12 *
13 * Regression test for GitHub issue #265 / PR #430.
14 *
15 * When a model (e.g. Anthropic Claude) produces a tool_use block with no input
16 * or literal null, JSON.parse('null') returns JavaScript null. Before the fix,
17 * this would fail Zod schema validation and silently kill the agent loop —
18 * the user would see "Let me check that..." then silence.
19 *
20 * The fix normalizes null/non-object parsed tool input to {} so the agent loop
21 * continues and the tool executes successfully.
22 *
23 * This test verifies the full end-to-end flow:
24 * 1. aimock returns a tool call with "null" as arguments
25 * 2. The adapter + server-side chat() normalizes null → {}
26 * 3. The tool executes successfully (schema has only optional fields)
27 * 4. The agent loop continues to the next iteration
28 * 5. The follow-up text response is received by the client
29 */
30
31test.describe('Null Tool Input E2E Tests (Regression #265)', () => {
32 test('server tool with null arguments executes and agent loop continues', async ({
33 page,
34 testId,
35 aimockPort,
36 }) => {
37 await selectScenario(page, 'null-tool-input', testId, aimockPort)
38 await runTest(page)
39 await waitForTestComplete(page, 30000)
40
41 // Verify the test completed (agent loop didn't die)
42 const metadata = await getMetadata(page)
43 expect(metadata.testComplete).toBe('true')
44 expect(metadata.isLoading).toBe('false')
45
46 // Verify the tool call was made and completed
47 expect(parseInt(metadata.toolCallCount)).toBeGreaterThanOrEqual(1)
48 expect(parseInt(metadata.completeToolCount)).toBeGreaterThanOrEqual(1)
49
50 // Verify check_status tool was called and completed
51 const toolCalls = await getToolCalls(page)
52 const statusTool = toolCalls.find((tc) => tc.name === 'check_status')
53 expect(statusTool).toBeTruthy()
54 expect(['complete', 'input-complete', 'output-available']).toContain(
55 statusTool?.state,
56 )
57
58 // Verify the tool result does NOT contain a validation error
59 // (proves null→{} normalization worked and the tool executed successfully)
60 const messages = await page.evaluate(() => {
61 const el = document.getElementById('messages-json-content')
62 if (!el) return []
63 try {
64 return JSON.parse(el.textContent || '[]')
65 } catch {
66 return []
67 }
68 })
69 const toolResults = messages.flatMap((m: any) =>
70 m.parts.filter((p: any) => p.type === 'tool-result'),
71 )
72 expect(toolResults.length).toBeGreaterThanOrEqual(1)
73 expect(toolResults[0].content).not.toContain('Input validation failed')
74 expect(toolResults[0].content).toContain('"status":"ok"')
75
76 // Verify the follow-up text was received (proves agent loop continued past
77 // the null-input tool call to produce the second iteration's response)
78 const assistantMessages = messages.filter(
79 (m: any) => m.role === 'assistant',
80 )
81 expect(assistantMessages.length).toBeGreaterThanOrEqual(2)
82
83 const lastAssistant = assistantMessages[assistantMessages.length - 1]
84 const textParts = lastAssistant.parts.filter((p: any) => p.type === 'text')
85 const allText = textParts.map((p: any) => p.content).join(' ')
86 expect(allText).toContain('status check is complete')
87 })
88
89 test.afterEach(async ({ page }, testInfo) => {
90 if (testInfo.status !== testInfo.expectedStatus) {
91 await page.screenshot({
92 path: `test-results/null-tool-input-failure-${testInfo.title.replace(/\s+/g, '-')}.png`,
93 fullPage: true,
94 })
95
96 const toolCalls = await getToolCalls(page)
97 const metadata = await getMetadata(page)
98
99 console.log('Test failed. Debug info:')
100 console.log('Metadata:', metadata)
101 console.log('Tool calls:', toolCalls)
102 }
103 })
104})