Open History to see the last commit
1import type { Page } from '@playwright/test'
2
3/**
4 * Shared helpers for tools-test spec files.
5 *
6 * These helpers interact with the /tools-test page and the selectors it
7 * exposes:
8 *
9 * #scenario-select — dropdown for choosing a test scenario
10 * #run-test-button — button that starts the test
11 * #approval-section — container shown when approvals are pending
12 * .approve-button — class selector for approve buttons
13 * .deny-button — class selector for deny buttons
14 * #approve-{id} — specific approve button for a tool call
15 * #deny-{id} — specific deny button for a tool call
16 * #test-metadata — hidden div with data-* attributes used for assertions
17 * #event-log-json — <script type="application/json"> with event array
18 * #tool-calls-json — <script type="application/json"> with tool-call array
19 * #messages-json-content — <pre> with full messages JSON
20 */
21
22/**
23 * Navigate to /tools-test and select a scenario reliably (handles React
24 * hydration delays).
25 */
26export async function selectScenario(
27 page: Page,
28 scenario: string,
29 testId?: string,
30 aimockPort?: number,
31): Promise<void> {
32 const params = new URLSearchParams()
33 if (testId) params.set('testId', testId)
34 if (aimockPort) params.set('aimockPort', String(aimockPort))
35 const qs = params.toString()
36 await page.goto(`/tools-test${qs ? '?' + qs : ''}`)
37 await page.waitForSelector('#run-test-button')
38
39 // Wait for hydration
40 await page.waitForTimeout(500)
41
42 // Try selecting multiple times
43 for (let attempt = 0; attempt < 3; attempt++) {
44 await page.focus('#scenario-select')
45 await page.selectOption('#scenario-select', scenario)
46 await page.waitForTimeout(200)
47
48 const currentScenario = await page.evaluate(() =>
49 document.getElementById('test-metadata')?.getAttribute('data-scenario'),
50 )
51 if (currentScenario === scenario) break
52 }
53
54 // Verify scenario is selected
55 await page.waitForFunction(
56 (expected) =>
57 document
58 .getElementById('test-metadata')
59 ?.getAttribute('data-scenario') === expected,
60 scenario,
61 { timeout: 10000 },
62 )
63
64 // Wait for new client to initialize
65 await page.waitForTimeout(300)
66}
67
68/**
69 * Click #run-test-button and wait for loading to start.
70 */
71export async function runTest(page: Page): Promise<void> {
72 await page.click('#run-test-button')
73}
74
75/**
76 * Wait for the test to complete.
77 *
78 * Considers the test complete when either:
79 * 1. data-test-complete === "true", OR
80 * 2. data-is-loading === "false" AND data-complete-tool-count >= expectedToolCount
81 */
82export async function waitForTestComplete(
83 page: Page,
84 timeout = 15000,
85 expectedToolCount = 1,
86): Promise<void> {
87 await page.waitForFunction(
88 ({ minTools }) => {
89 const metadata = document.getElementById('test-metadata')
90 const testComplete =
91 metadata?.getAttribute('data-test-complete') === 'true'
92 const isLoading = metadata?.getAttribute('data-is-loading') === 'true'
93 const completeToolCount = parseInt(
94 metadata?.getAttribute('data-complete-tool-count') || '0',
95 10,
96 )
97
98 // Consider complete if:
99 // 1. testComplete flag is true, OR
100 // 2. Not loading and we have at least the expected number of complete tools
101 return testComplete || (!isLoading && completeToolCount >= minTools)
102 },
103 { minTools: expectedToolCount },
104 { timeout },
105 )
106
107 // Give a little extra time for final state updates
108 await page.waitForTimeout(200)
109}
110
111/**
112 * Wait for the #approval-section element to become visible.
113 */
114export async function waitForApproval(
115 page: Page,
116 timeout = 10000,
117): Promise<void> {
118 await page.waitForSelector('#approval-section', { timeout })
119}
120
121/**
122 * Read all data-* attributes from #test-metadata.
123 */
124export async function getMetadata(page: Page): Promise<Record<string, string>> {
125 return page.evaluate(() => {
126 const el = document.getElementById('test-metadata')
127 if (!el) return {}
128 return {
129 scenario: el.getAttribute('data-scenario') || '',
130 isLoading: el.getAttribute('data-is-loading') || '',
131 testComplete: el.getAttribute('data-test-complete') || '',
132 toolCallCount: el.getAttribute('data-tool-call-count') || '',
133 pendingApprovalCount:
134 el.getAttribute('data-pending-approval-count') || '',
135 completeToolCount: el.getAttribute('data-complete-tool-count') || '',
136 eventCount: el.getAttribute('data-event-count') || '',
137 executionStartCount: el.getAttribute('data-execution-start-count') || '',
138 executionCompleteCount:
139 el.getAttribute('data-execution-complete-count') || '',
140 approvalGrantedCount:
141 el.getAttribute('data-approval-granted-count') || '',
142 approvalDeniedCount: el.getAttribute('data-approval-denied-count') || '',
143 hasError: el.getAttribute('data-has-error') || '',
144 errorMessage: el.getAttribute('data-error-message') || '',
145 }
146 })
147}
148
149/**
150 * Parse the JSON event log from #event-log-json.
151 */
152export async function getEventLog(
153 page: Page,
154): Promise<Array<{ type: string; toolName: string; timestamp: number }>> {
155 return page.evaluate(() => {
156 const el = document.getElementById('event-log-json')
157 if (!el) return []
158 try {
159 return JSON.parse(el.textContent || '[]')
160 } catch {
161 return []
162 }
163 })
164}
165
166/**
167 * Parse the JSON tool-calls list from #tool-calls-json.
168 */
169export async function getToolCalls(
170 page: Page,
171): Promise<Array<{ id: string; name: string; state: string }>> {
172 return page.evaluate(() => {
173 const el = document.getElementById('tool-calls-json')
174 if (!el) return []
175 try {
176 return JSON.parse(el.textContent || '[]')
177 } catch {
178 return []
179 }
180 })
181}
182
183/**
184 * Extract tool-call parts with parsed arguments from #messages-json-content.
185 */
186export async function getToolCallParts(
187 page: Page,
188): Promise<Array<{ name: string; arguments: Record<string, unknown> }>> {
189 return page.evaluate(() => {
190 const el = document.getElementById('messages-json-content')
191 if (!el) return []
192 try {
193 const messages = JSON.parse(el.textContent || '[]')
194 const parts: Array<{ name: string; arguments: Record<string, unknown> }> =
195 []
196 for (const msg of messages) {
197 for (const part of msg.parts || []) {
198 if (part.type === 'tool-call') {
199 parts.push({
200 name: part.name,
201 arguments:
202 typeof part.arguments === 'string'
203 ? JSON.parse(part.arguments)
204 : part.arguments,
205 })
206 }
207 }
208 }
209 return parts
210 } catch {
211 return []
212 }
213 })
214}