Open History to see the last commit
1import type { Page } from '@playwright/test'
2
3/**
4 * Shared helpers for tools-test spec files.
5 *
6 * These helpers interact with the /tools-test page and the selectors it
7 * exposes:
8 *
9 * #scenario-select — dropdown for choosing a test scenario
10 * #run-test-button — button that starts the test
11 * #approval-section — container shown when approvals are pending
12 * .approve-button — class selector for approve buttons
13 * .deny-button — class selector for deny buttons
14 * #approve-{id} — specific approve button for a tool call
15 * #deny-{id} — specific deny button for a tool call
16 * #test-metadata — hidden div with data-* attributes used for assertions
17 * #event-log-json — <script type="application/json"> with event array
18 * #tool-calls-json — <script type="application/json"> with tool-call array
19 * #messages-json-content — <pre> with full messages JSON
20 */
21
22/**
23 * Navigate to /tools-test and select a scenario reliably (handles React
24 * hydration delays).
25 */
26export async function selectScenario(
27 page: Page,
28 scenario: string,
29 testId?: string,
30 aimockPort?: number,
31): Promise<void> {
32 const params = new URLSearchParams()
33 if (testId) params.set('testId', testId)
34 if (aimockPort) params.set('aimockPort', String(aimockPort))
35 params.set('scenario', scenario)
36 const qs = params.toString()
37 await page.goto(`/tools-test${qs ? '?' + qs : ''}`)
38 await page.waitForSelector('#run-test-button')
39
40 // Give React time to attach delegated event handlers before clicking.
41 await page.waitForTimeout(500)
42
43 const isScenarioSelected = async (timeout: number) =>
44 page
45 .waitForFunction(
46 (expected) => {
47 const metadata = document.getElementById('test-metadata')
48 return metadata?.getAttribute('data-scenario') === expected
49 },
50 scenario,
51 { timeout },
52 )
53 .then(() => true)
54 .catch(() => false)
55
56 if (await isScenarioSelected(5000)) {
57 return
58 }
59
60 // Try selecting multiple times
61 for (let attempt = 0; attempt < 3; attempt++) {
62 await page.focus('#scenario-select')
63 await page.selectOption('#scenario-select', scenario)
64 await page.waitForTimeout(200)
65
66 const currentScenario = await page.evaluate(() =>
67 document.getElementById('test-metadata')?.getAttribute('data-scenario'),
68 )
69 if (currentScenario === scenario) break
70 }
71
72 // Verify scenario is selected
73 await page.waitForFunction(
74 (expected) => {
75 const metadata = document.getElementById('test-metadata')
76 return metadata?.getAttribute('data-scenario') === expected
77 },
78 scenario,
79 { timeout: 10000 },
80 )
81
82 // Wait for new client to initialize
83 await page.waitForTimeout(300)
84}
85
86/**
87 * Click #run-test-button and wait for loading to start.
88 *
89 * Uses the pre-click message count as a baseline so that a fixture which is
90 * already populated (non-empty messages JSON) does not register as a started
91 * run on its own. A run is considered started when either:
92 * 1. data-is-loading === "true", OR
93 * 2. the parsed message count has grown beyond the pre-click baseline.
94 */
95export async function runTest(page: Page): Promise<void> {
96 const readMessageCount = () =>
97 page.evaluate(() => {
98 const text =
99 document.getElementById('messages-json-content')?.textContent || '[]'
100 try {
101 const parsed = JSON.parse(text)
102 return Array.isArray(parsed) ? parsed.length : 0
103 } catch {
104 return 0
105 }
106 })
107
108 for (let attempt = 0; attempt < 5; attempt++) {
109 const baselineMessageCount = await readMessageCount()
110 await page.click('#run-test-button')
111
112 // A run "starts" only when real stream activity appears — not when the
113 // optimistic user message lands. Clicking adds one user message
114 // synchronously (baseline + 1); that alone must NOT count as started, or a
115 // stalled run (the click registered but the stream produced nothing) would
116 // be reported as started and the test would later time out waiting for an
117 // approval / completion that never comes. Real activity is: loading turned
118 // on, a tool call appeared, the test completed, or a *second* message (the
119 // assistant response) was added beyond the optimistic user message. Poll
120 // briefly so a slow-but-real run under CI load isn't mistaken for a stall.
121 const started = await page
122 .waitForFunction(
123 (baseline) => {
124 const metadata = document.getElementById('test-metadata')
125 if (metadata?.getAttribute('data-is-loading') === 'true') return true
126 if (
127 parseInt(
128 metadata?.getAttribute('data-tool-call-count') || '0',
129 10,
130 ) > 0
131 )
132 return true
133 if (metadata?.getAttribute('data-test-complete') === 'true')
134 return true
135 const text =
136 document.getElementById('messages-json-content')?.textContent ||
137 '[]'
138 try {
139 const parsed = JSON.parse(text)
140 // > baseline + 1: the assistant message arrived (a real response),
141 // not just the optimistic user message.
142 return Array.isArray(parsed) && parsed.length > baseline + 1
143 } catch {
144 return false
145 }
146 },
147 baselineMessageCount,
148 { timeout: 2000 },
149 )
150 .then(() => true)
151 .catch(() => false)
152
153 if (started) {
154 return
155 }
156 }
157
158 throw new Error('Run test button did not start a chat run')
159}
160
161/**
162 * Wait for the test to complete.
163 *
164 * Considers the test complete when either:
165 * 1. data-test-complete === "true", OR
166 * 2. data-is-loading === "false" AND data-complete-tool-count >= expectedToolCount
167 */
168export async function waitForTestComplete(
169 page: Page,
170 timeout = 15000,
171 expectedToolCount = 1,
172): Promise<void> {
173 await page.waitForFunction(
174 ({ minTools }) => {
175 const metadata = document.getElementById('test-metadata')
176 const testComplete =
177 metadata?.getAttribute('data-test-complete') === 'true'
178 const isLoading = metadata?.getAttribute('data-is-loading') === 'true'
179 const completeToolCount = parseInt(
180 metadata?.getAttribute('data-complete-tool-count') || '0',
181 10,
182 )
183
184 // Consider complete if:
185 // 1. testComplete flag is true, OR
186 // 2. Not loading and we have at least the expected number of complete tools
187 return testComplete || (!isLoading && completeToolCount >= minTools)
188 },
189 { minTools: expectedToolCount },
190 { timeout },
191 )
192
193 // Give a little extra time for final state updates
194 await page.waitForTimeout(200)
195}
196
197/**
198 * Wait for client-side execution events to be reflected in the event log.
199 *
200 * Tool call completion can be visible in message state before React has flushed
201 * the separate event-log state update used by these assertions.
202 */
203export async function waitForExecutionComplete(
204 page: Page,
205 toolName: string,
206 timeout = 10000,
207): Promise<void> {
208 await page.waitForFunction(
209 (expectedToolName) => {
210 const el = document.getElementById('event-log-json')
211 if (!el) return false
212
213 try {
214 const events = JSON.parse(el.textContent || '[]') as Array<{
215 type?: string
216 toolName?: string
217 }>
218 return events.some(
219 (event) =>
220 event.type === 'execution-complete' &&
221 event.toolName === expectedToolName,
222 )
223 } catch {
224 return false
225 }
226 },
227 toolName,
228 { timeout },
229 )
230}
231
232/**
233 * Wait for the #approval-section element to become visible.
234 */
235export async function waitForApproval(
236 page: Page,
237 timeout = 10000,
238): Promise<void> {
239 await page.waitForSelector('#approval-section', { timeout })
240}
241
242/**
243 * Read all data-* attributes from #test-metadata.
244 */
245export async function getMetadata(page: Page): Promise<Record<string, string>> {
246 return page.evaluate(() => {
247 const el = document.getElementById('test-metadata')
248 if (!el) return {}
249 return {
250 scenario: el.getAttribute('data-scenario') || '',
251 isLoading: el.getAttribute('data-is-loading') || '',
252 testComplete: el.getAttribute('data-test-complete') || '',
253 toolCallCount: el.getAttribute('data-tool-call-count') || '',
254 pendingApprovalCount:
255 el.getAttribute('data-pending-approval-count') || '',
256 completeToolCount: el.getAttribute('data-complete-tool-count') || '',
257 eventCount: el.getAttribute('data-event-count') || '',
258 executionStartCount: el.getAttribute('data-execution-start-count') || '',
259 executionCompleteCount:
260 el.getAttribute('data-execution-complete-count') || '',
261 approvalGrantedCount:
262 el.getAttribute('data-approval-granted-count') || '',
263 approvalDeniedCount: el.getAttribute('data-approval-denied-count') || '',
264 hasError: el.getAttribute('data-has-error') || '',
265 errorMessage: el.getAttribute('data-error-message') || '',
266 errorRawEvent: el.getAttribute('data-error-raw-event') || '',
267 }
268 })
269}
270
271/**
272 * Parse the JSON event log from #event-log-json.
273 */
274export async function getEventLog(
275 page: Page,
276): Promise<Array<{ type: string; toolName: string; timestamp: number }>> {
277 return page.evaluate(() => {
278 const el = document.getElementById('event-log-json')
279 if (!el) return []
280 try {
281 return JSON.parse(el.textContent || '[]')
282 } catch {
283 return []
284 }
285 })
286}
287
288/**
289 * Parse the JSON tool-calls list from #tool-calls-json.
290 */
291export async function getToolCalls(
292 page: Page,
293): Promise<Array<{ id: string; name: string; state: string }>> {
294 return page.evaluate(() => {
295 const el = document.getElementById('tool-calls-json')
296 if (!el) return []
297 try {
298 return JSON.parse(el.textContent || '[]')
299 } catch {
300 return []
301 }
302 })
303}
304
305/**
306 * Parse the full UIMessage array from #messages-json-content.
307 */
308export async function getMessages(page: Page): Promise<Array<any>> {
309 return page.evaluate(() => {
310 const el = document.getElementById('messages-json-content')
311 if (!el) return []
312 try {
313 return JSON.parse(el.textContent || '[]')
314 } catch {
315 return []
316 }
317 })
318}
319
320/**
321 * Extract tool-call parts with parsed arguments from #messages-json-content.
322 */
323export async function getToolCallParts(
324 page: Page,
325): Promise<Array<{ name: string; arguments: Record<string, unknown> }>> {
326 return page.evaluate(() => {
327 const el = document.getElementById('messages-json-content')
328 if (!el) return []
329 try {
330 const messages = JSON.parse(el.textContent || '[]')
331 const parts: Array<{ name: string; arguments: Record<string, unknown> }> =
332 []
333 for (const msg of messages) {
334 for (const part of msg.parts || []) {
335 if (part.type === 'tool-call') {
336 parts.push({
337 name: part.name,
338 arguments:
339 typeof part.arguments === 'string'
340 ? JSON.parse(part.arguments)
341 : part.arguments,
342 })
343 }
344 }
345 }
346 return parts
347 } catch {
348 return []
349 }
350 })
351}