Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

server-client-sequence.spec.ts

214 lines6.5 KB

1import { test, expect } from '../fixtures'
2import {
3 selectScenario,
4 runTest,
5 waitForTestComplete,
6 getMetadata,
7 getEventLog,
8 getToolCalls,
9} from './helpers'
10​
11/**
12 * Server-Client Sequence E2E Tests
13 *
14 * These tests verify that mixed server/client tool sequences work correctly,
15 * including:
16 * - Server tool followed by client tool
17 * - Client tool followed by server tool
18 * - Complex multi-step sequences
19 * - Proper state management across tool types
20 */
21​
22test.describe('Server-Client Sequence E2E Tests', () => {
23 test('server tool followed by client tool', async ({
24 page,
25 testId,
26 aimockPort,
27 }) => {
28 await selectScenario(page, 'sequence-server-client', testId, aimockPort)
29 await runTest(page)
30​
31 // Wait for the test to complete
32 await waitForTestComplete(page, 15000, 2)
33​
34 // Wait for client execution events to propagate
35 await page.waitForFunction(
36 () => {
37 const el = document.querySelector('#test-metadata')
38 return (
39 parseInt(el?.getAttribute('data-execution-complete-count') || '0') >=
40 1
41 )
42 },
43 { timeout: 10000 },
44 )
45​
46 // Verify results
47 const metadata = await getMetadata(page)
48 expect(metadata.testComplete).toBe('true')
49​
50 // Should have 2 tool calls (fetch_data server, display_chart client)
51 expect(parseInt(metadata.toolCallCount)).toBeGreaterThanOrEqual(2)
52​
53 // Verify client tool executed
54 const events = await getEventLog(page)
55 const chartExecution = events.find(
56 (e) => e.toolName === 'display_chart' && e.type === 'execution-complete',
57 )
58 expect(chartExecution).toBeTruthy()
59 })
60​
61 test('server then two client tools in sequence', async ({
62 page,
63 testId,
64 aimockPort,
65 }) => {
66 // Tests complex continuation: server -> client -> client
67 await selectScenario(page, 'server-then-two-clients', testId, aimockPort)
68 await runTest(page)
69​
70 // Wait for the test to complete — expect 3 tools (1 server + 2 clients)
71 await waitForTestComplete(page, 30000, 3)
72​
73 // Wait for client tool executions to propagate
74 await page.waitForFunction(
75 () => {
76 const el = document.querySelector('#test-metadata')
77 return (
78 parseInt(el?.getAttribute('data-execution-complete-count') || '0') >=
79 2
80 )
81 },
82 { timeout: 10000 },
83 )
84​
85 // Verify results
86 const metadata = await getMetadata(page)
87 expect(metadata.testComplete).toBe('true')
88​
89 // Should have 3 tool calls
90 expect(parseInt(metadata.toolCallCount)).toBeGreaterThanOrEqual(3)
91​
92 // Verify both client tools executed
93 const events = await getEventLog(page)
94 const notificationExecution = events.find(
95 (e) =>
96 e.toolName === 'show_notification' && e.type === 'execution-complete',
97 )
98 const chartExecution = events.find(
99 (e) => e.toolName === 'display_chart' && e.type === 'execution-complete',
100 )
101​
102 expect(notificationExecution).toBeTruthy()
103 expect(chartExecution).toBeTruthy()
104​
105 // Verify order: notification should complete before chart starts
106 // (they're in sequence, not parallel)
107 const notificationCompleteTime = events.find(
108 (e) =>
109 e.toolName === 'show_notification' && e.type === 'execution-complete',
110 )?.timestamp
111 const chartStartTime = events.find(
112 (e) => e.toolName === 'display_chart' && e.type === 'execution-start',
113 )?.timestamp
114​
115 if (notificationCompleteTime && chartStartTime) {
116 expect(notificationCompleteTime).toBeLessThanOrEqual(chartStartTime)
117 }
118 })
119​
120 test('parallel server tools complete correctly', async ({
121 page,
122 testId,
123 aimockPort,
124 }) => {
125 await selectScenario(page, 'parallel-tools', testId, aimockPort)
126 await runTest(page)
127​
128 // Wait for the test to complete
129 await waitForTestComplete(page)
130​
131 // Verify results
132 const metadata = await getMetadata(page)
133 expect(metadata.testComplete).toBe('true')
134​
135 // Should have 2 tool calls (get_weather, get_time)
136 expect(parseInt(metadata.toolCallCount)).toBeGreaterThanOrEqual(2)
137 expect(parseInt(metadata.completeToolCount)).toBeGreaterThanOrEqual(2)
138​
139 // Verify both tools are in the tool calls list
140 const toolCalls = await getToolCalls(page)
141 const toolNames = toolCalls.map((tc) => tc.name)
142 expect(toolNames).toContain('get_weather')
143 expect(toolNames).toContain('get_time')
144 })
145​
146 test('single server tool completes', async ({ page, testId, aimockPort }) => {
147 await selectScenario(page, 'server-tool-single', testId, aimockPort)
148 await runTest(page)
149​
150 // Wait for the test to complete
151 await waitForTestComplete(page)
152​
153 // Verify results
154 const metadata = await getMetadata(page)
155 expect(metadata.testComplete).toBe('true')
156​
157 // Should have 1 tool call (get_weather)
158 expect(parseInt(metadata.toolCallCount)).toBeGreaterThanOrEqual(1)
159 expect(parseInt(metadata.completeToolCount)).toBeGreaterThanOrEqual(1)
160​
161 // Verify tool is in the calls list
162 const toolCalls = await getToolCalls(page)
163 const weatherTool = toolCalls.find((tc) => tc.name === 'get_weather')
164 expect(weatherTool).toBeTruthy()
165 // Server tools stay at 'input-complete' state but are tracked as complete via tool-result parts
166 expect(['complete', 'input-complete', 'output-available']).toContain(
167 weatherTool?.state,
168 )
169 })
170​
171 test('text only scenario has no tool calls', async ({
172 page,
173 testId,
174 aimockPort,
175 }) => {
176 await selectScenario(page, 'text-only', testId, aimockPort)
177 await runTest(page)
178​
179 // Wait for loading to finish
180 await page.waitForFunction(
181 () => {
182 const metadata = document.getElementById('test-metadata')
183 return metadata?.getAttribute('data-is-loading') === 'false'
184 },
185 { timeout: 10000 },
186 )
187​
188 // Give it a moment to settle
189 await page.waitForTimeout(500)
190​
191 // Verify no tool calls
192 const metadata = await getMetadata(page)
193 expect(parseInt(metadata.toolCallCount)).toBe(0)
194 })
195​
196 // Screenshot on failure
197 test.afterEach(async ({ page }, testInfo) => {
198 if (testInfo.status !== testInfo.expectedStatus) {
199 await page.screenshot({
200 path: `test-results/sequence-failure-${testInfo.title.replace(/\s+/g, '-')}.png`,
201 fullPage: true,
202 })
203​
204 const events = await getEventLog(page)
205 const toolCalls = await getToolCalls(page)
206 const metadata = await getMetadata(page)
207​
208 console.log('Test failed. Debug info:')
209 console.log('Metadata:', metadata)
210 console.log('Events:', events)
211 console.log('Tool calls:', toolCalls)
212 }
213 })
214})