Open History to see the last commit
1import { test, expect } from '../fixtures'
2import {
3 selectScenario,
4 runTest,
5 waitForTestComplete,
6 getMetadata,
7 getEventLog,
8 getToolCalls,
9 getMessages,
10} from './helpers'
11
12/**
13 * Server-Client Sequence E2E Tests
14 *
15 * These tests verify that mixed server/client tool sequences work correctly,
16 * including:
17 * - Server tool followed by client tool
18 * - Client tool followed by server tool
19 * - Complex multi-step sequences
20 * - Proper state management across tool types
21 */
22
23test.describe('Server-Client Sequence E2E Tests', () => {
24 test('server tool followed by client tool', async ({
25 page,
26 testId,
27 aimockPort,
28 }) => {
29 await selectScenario(page, 'sequence-server-client', testId, aimockPort)
30 await runTest(page)
31
32 // Wait for the test to complete
33 await waitForTestComplete(page, 15000, 2)
34
35 // Wait for client execution events to propagate
36 await page.waitForFunction(
37 () => {
38 const el = document.querySelector('#test-metadata')
39 return (
40 parseInt(el?.getAttribute('data-execution-complete-count') || '0') >=
41 1
42 )
43 },
44 { timeout: 10000 },
45 )
46
47 // Verify results
48 const metadata = await getMetadata(page)
49 expect(metadata.testComplete).toBe('true')
50
51 // Should have 2 tool calls (fetch_data server, display_chart client)
52 expect(parseInt(metadata.toolCallCount)).toBeGreaterThanOrEqual(2)
53
54 // Verify client tool executed
55 const events = await getEventLog(page)
56 const chartExecution = events.find(
57 (e) => e.toolName === 'display_chart' && e.type === 'execution-complete',
58 )
59 expect(chartExecution).toBeTruthy()
60 })
61
62 test('server and client tool results populate tool-call output and complete state', async ({
63 page,
64 testId,
65 aimockPort,
66 }) => {
67 await selectScenario(page, 'sequence-server-client', testId, aimockPort)
68 await runTest(page)
69
70 await waitForTestComplete(page, 15000, 2)
71
72 await page.waitForFunction(
73 () => {
74 const messagesEl = document.getElementById('messages-json-content')
75 const messages = JSON.parse(messagesEl?.textContent || '[]')
76 const toolCalls = messages.flatMap((msg: any) =>
77 (msg.parts || []).filter((part: any) => part.type === 'tool-call'),
78 )
79
80 const fetchData = toolCalls.find(
81 (part: any) => part.name === 'fetch_data',
82 )
83 const displayChart = toolCalls.find(
84 (part: any) => part.name === 'display_chart',
85 )
86
87 return (
88 fetchData?.state === 'complete' &&
89 fetchData.output !== undefined &&
90 displayChart?.state === 'complete' &&
91 displayChart.output !== undefined
92 )
93 },
94 { timeout: 10000 },
95 )
96
97 const messages = await getMessages(page)
98 const toolCalls = messages.flatMap((msg) =>
99 (msg.parts || []).filter((part: any) => part.type === 'tool-call'),
100 )
101 const toolResults = messages.flatMap((msg) =>
102 (msg.parts || []).filter((part: any) => part.type === 'tool-result'),
103 )
104
105 const fetchData = toolCalls.find((part) => part.name === 'fetch_data')
106 expect(fetchData).toMatchObject({
107 state: 'complete',
108 output: {
109 source: 'api',
110 data: [1, 2, 3, 4, 5],
111 },
112 })
113
114 const displayChart = toolCalls.find((part) => part.name === 'display_chart')
115 expect(displayChart?.state).toBe('complete')
116 expect(displayChart?.output).toMatchObject({ rendered: true })
117 expect(typeof displayChart?.output?.chartId).toBe('string')
118
119 expect(
120 toolResults.some(
121 (part) =>
122 part.toolCallId === fetchData?.id &&
123 part.state === 'complete' &&
124 JSON.parse(part.content).source === 'api',
125 ),
126 ).toBe(true)
127 })
128
129 test('server then two client tools in sequence', async ({
130 page,
131 testId,
132 aimockPort,
133 }) => {
134 // Tests complex continuation: server -> client -> client
135 await selectScenario(page, 'server-then-two-clients', testId, aimockPort)
136 await runTest(page)
137
138 // Wait for the test to complete — expect 3 tools (1 server + 2 clients)
139 await waitForTestComplete(page, 30000, 3)
140
141 // Wait for client tool executions to propagate
142 await page.waitForFunction(
143 () => {
144 const el = document.querySelector('#test-metadata')
145 return (
146 parseInt(el?.getAttribute('data-execution-complete-count') || '0') >=
147 2
148 )
149 },
150 { timeout: 10000 },
151 )
152
153 // Verify results
154 const metadata = await getMetadata(page)
155 expect(metadata.testComplete).toBe('true')
156
157 // Should have 3 tool calls
158 expect(parseInt(metadata.toolCallCount)).toBeGreaterThanOrEqual(3)
159
160 // Verify both client tools executed
161 const events = await getEventLog(page)
162 const notificationExecution = events.find(
163 (e) =>
164 e.toolName === 'show_notification' && e.type === 'execution-complete',
165 )
166 const chartExecution = events.find(
167 (e) => e.toolName === 'display_chart' && e.type === 'execution-complete',
168 )
169
170 expect(notificationExecution).toBeTruthy()
171 expect(chartExecution).toBeTruthy()
172
173 // Verify order: notification should complete before chart starts
174 // (they're in sequence, not parallel)
175 const notificationCompleteTime = events.find(
176 (e) =>
177 e.toolName === 'show_notification' && e.type === 'execution-complete',
178 )?.timestamp
179 const chartStartTime = events.find(
180 (e) => e.toolName === 'display_chart' && e.type === 'execution-start',
181 )?.timestamp
182
183 if (notificationCompleteTime && chartStartTime) {
184 expect(notificationCompleteTime).toBeLessThanOrEqual(chartStartTime)
185 }
186 })
187
188 test('parallel server tools complete correctly', async ({
189 page,
190 testId,
191 aimockPort,
192 }) => {
193 await selectScenario(page, 'parallel-tools', testId, aimockPort)
194 await runTest(page)
195
196 // Wait for the test to complete
197 await waitForTestComplete(page)
198
199 // Verify results
200 const metadata = await getMetadata(page)
201 expect(metadata.testComplete).toBe('true')
202
203 // Should have 2 tool calls (get_weather, get_time)
204 expect(parseInt(metadata.toolCallCount)).toBeGreaterThanOrEqual(2)
205 expect(parseInt(metadata.completeToolCount)).toBeGreaterThanOrEqual(2)
206
207 // Verify both tools are in the tool calls list
208 const toolCalls = await getToolCalls(page)
209 const toolNames = toolCalls.map((tc) => tc.name)
210 expect(toolNames).toContain('get_weather')
211 expect(toolNames).toContain('get_time')
212 })
213
214 test('single server tool completes', async ({ page, testId, aimockPort }) => {
215 await selectScenario(page, 'server-tool-single', testId, aimockPort)
216 await runTest(page)
217
218 // Wait for the test to complete
219 await waitForTestComplete(page)
220
221 // Verify results
222 const metadata = await getMetadata(page)
223 expect(metadata.testComplete).toBe('true')
224
225 // Should have 1 tool call (get_weather)
226 expect(parseInt(metadata.toolCallCount)).toBeGreaterThanOrEqual(1)
227 expect(parseInt(metadata.completeToolCount)).toBeGreaterThanOrEqual(1)
228
229 // Verify tool is in the calls list
230 const toolCalls = await getToolCalls(page)
231 const weatherTool = toolCalls.find((tc) => tc.name === 'get_weather')
232 expect(weatherTool).toBeTruthy()
233 expect(weatherTool?.state).toBe('complete')
234 })
235
236 test('text only scenario has no tool calls', async ({
237 page,
238 testId,
239 aimockPort,
240 }) => {
241 await selectScenario(page, 'text-only', testId, aimockPort)
242 await runTest(page)
243
244 // Wait for loading to finish
245 await page.waitForFunction(
246 () => {
247 const metadata = document.getElementById('test-metadata')
248 return metadata?.getAttribute('data-is-loading') === 'false'
249 },
250 { timeout: 10000 },
251 )
252
253 // Give it a moment to settle
254 await page.waitForTimeout(500)
255
256 // Verify no tool calls
257 const metadata = await getMetadata(page)
258 expect(parseInt(metadata.toolCallCount)).toBe(0)
259 })
260
261 // Screenshot on failure
262 test.afterEach(async ({ page }, testInfo) => {
263 if (testInfo.status !== testInfo.expectedStatus) {
264 await page.screenshot({
265 path: `test-results/sequence-failure-${testInfo.title.replace(/\s+/g, '-')}.png`,
266 fullPage: true,
267 })
268
269 const events = await getEventLog(page)
270 const toolCalls = await getToolCalls(page)
271 const metadata = await getMetadata(page)
272
273 console.log('Test failed. Debug info:')
274 console.log('Metadata:', metadata)
275 console.log('Events:', events)
276 console.log('Tool calls:', toolCalls)
277 }
278 })
279})