Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

client-tool.spec.ts

183 lines6.0 KB

1import { test, expect } from '../fixtures'
2import {
3 selectScenario,
4 runTest,
5 waitForTestComplete,
6 getMetadata,
7 getEventLog,
8 getToolCalls,
9} from './helpers'
10​
11/**
12 * Client Tool E2E Tests
13 *
14 * These tests verify that client-side tool execution works correctly,
15 * including proper event flow, continuation, and state management.
16 *
17 * These tests catch race conditions and event flow issues in the
18 * client-side code (useChat hook, ChatClient).
19 */
20​
21test.describe('Client Tool E2E Tests', () => {
22 test('single client tool executes and completes', async ({
23 page,
24 testId,
25 aimockPort,
26 }) => {
27 await selectScenario(page, 'client-tool-single', testId, aimockPort)
28 await runTest(page)
29​
30 // Wait for the test to complete
31 await waitForTestComplete(page)
32​
33 // Verify the results
34 const metadata = await getMetadata(page)
35 expect(metadata.testComplete).toBe('true')
36 expect(parseInt(metadata.toolCallCount)).toBeGreaterThanOrEqual(1)
37 expect(parseInt(metadata.completeToolCount)).toBeGreaterThanOrEqual(1)
38​
39 // Verify events were recorded
40 const events = await getEventLog(page)
41 const startEvents = events.filter((e) => e.type === 'execution-start')
42 const completeEvents = events.filter((e) => e.type === 'execution-complete')
43​
44 expect(startEvents.length).toBeGreaterThanOrEqual(1)
45 expect(completeEvents.length).toBeGreaterThanOrEqual(1)
46 expect(startEvents[0]?.toolName).toBe('show_notification')
47 })
48​
49 test('sequential client tools execute in order', async ({
50 page,
51 testId,
52 aimockPort,
53 }) => {
54 // This tests the specific issue: two client tool calls in a row
55 // The second call shouldn't be blocked by the first
56 await selectScenario(page, 'sequential-client-tools', testId, aimockPort)
57 await runTest(page)
58​
59 // Wait for the test to complete (expect 2 tools)
60 await waitForTestComplete(page, 15000, 2)
61​
62 // Wait for execution events to propagate
63 await page.waitForFunction(
64 () => {
65 const el = document.querySelector('#test-metadata')
66 return (
67 parseInt(el?.getAttribute('data-execution-complete-count') || '0') >=
68 2
69 )
70 },
71 { timeout: 10000 },
72 )
73​
74 // Verify the results
75 const metadata = await getMetadata(page)
76 expect(parseInt(metadata.toolCallCount)).toBeGreaterThanOrEqual(2)
77​
78 // Verify events show proper execution order
79 const events = await getEventLog(page)
80 const executionEvents = events.filter(
81 (e) => e.type === 'execution-start' || e.type === 'execution-complete',
82 )
83​
84 // Should have at least 4 events: start1, complete1, start2, complete2
85 expect(executionEvents.length).toBeGreaterThanOrEqual(4)
86 })
87​
88 test('parallel client tools execute concurrently', async ({
89 page,
90 testId,
91 aimockPort,
92 }) => {
93 await selectScenario(page, 'parallel-client-tools', testId, aimockPort)
94 await runTest(page)
95​
96 // Wait for the test to complete (expect 2 tools)
97 await waitForTestComplete(page, 15000, 2)
98​
99 // Verify the results
100 const metadata = await getMetadata(page)
101 expect(parseInt(metadata.toolCallCount)).toBeGreaterThanOrEqual(2)
102 expect(parseInt(metadata.completeToolCount)).toBeGreaterThanOrEqual(2)
103​
104 // Verify both tools were executed
105 const events = await getEventLog(page)
106 const toolNames = new Set(events.map((e) => e.toolName))
107 expect(toolNames.has('show_notification')).toBe(true)
108 expect(toolNames.has('display_chart')).toBe(true)
109 })
110​
111 test('triple client sequence completes all three', async ({
112 page,
113 testId,
114 aimockPort,
115 }) => {
116 // Stress test: three client tools in sequence
117 // NOTE: This tests a complex multi-step continuation flow that requires
118 // additional investigation. Currently verifying at least 2 tools complete.
119 await selectScenario(page, 'triple-client-sequence', testId, aimockPort)
120 await runTest(page)
121​
122 // Wait for the test to complete (expect at least 2 tools)
123 await waitForTestComplete(page, 20000, 2)
124​
125 // Wait for execution events to propagate
126 await page.waitForFunction(
127 () => {
128 const el = document.querySelector('#test-metadata')
129 return (
130 parseInt(el?.getAttribute('data-execution-complete-count') || '0') >=
131 2
132 )
133 },
134 { timeout: 10000 },
135 )
136​
137 // Verify at least 2 tools complete
138 const metadata = await getMetadata(page)
139 expect(parseInt(metadata.toolCallCount)).toBeGreaterThanOrEqual(2)
140 expect(parseInt(metadata.executionCompleteCount)).toBeGreaterThanOrEqual(2)
141 })
142​
143 test('server then two clients sequence completes', async ({
144 page,
145 testId,
146 aimockPort,
147 }) => {
148 // Tests complex flow: server tool -> client tool -> client tool
149 // NOTE: This tests a complex multi-step continuation flow involving
150 // both server and client tools. Currently verifying at least 1 client tool completes.
151 // Known issue: Server tool state tracking may need investigation.
152 await selectScenario(page, 'server-then-two-clients', testId, aimockPort)
153 await runTest(page)
154​
155 // Wait for at least 1 client tool to complete
156 await waitForTestComplete(page, 20000, 1)
157​
158 // Verify at least some tools appeared and 1 client tool completed
159 const metadata = await getMetadata(page)
160 expect(parseInt(metadata.toolCallCount)).toBeGreaterThanOrEqual(1)
161 expect(parseInt(metadata.completeToolCount)).toBeGreaterThanOrEqual(1)
162 })
163​
164 // Screenshot on failure
165 test.afterEach(async ({ page }, testInfo) => {
166 if (testInfo.status !== testInfo.expectedStatus) {
167 await page.screenshot({
168 path: `test-results/client-tool-failure-${testInfo.title.replace(/\s+/g, '-')}.png`,
169 fullPage: true,
170 })
171​
172 // Also log the event log and tool calls for debugging
173 const events = await getEventLog(page)
174 const toolCalls = await getToolCalls(page)
175 const metadata = await getMetadata(page)
176​
177 console.log('Test failed. Debug info:')
178 console.log('Metadata:', metadata)
179 console.log('Events:', events)
180 console.log('Tool calls:', toolCalls)
181 }
182 })
183})