Open History to see the last commit
1/**
2 * Test helpers for E2E vendor tests
3 */
4
5import type { Page, APIRequestContext } from '@playwright/test'
6import type { ProviderId } from './vendor-config'
7
8/**
9 * Select a provider/model from the model dropdown on the chat page
10 */
11export async function selectProvider(
12 page: Page,
13 provider: ProviderId,
14 model: string,
15): Promise<void> {
16 // The model selector shows labels like "OpenAI - GPT-4o"
17 // We need to find the option that matches our provider and model
18 const select = page.locator('select').first()
19 await select.waitFor({ state: 'visible' })
20
21 // Get all options and find the one matching our provider/model
22 const options = await select.locator('option').all()
23 let targetIndex = -1
24
25 for (let i = 0; i < options.length; i++) {
26 const text = await options[i].textContent()
27 // Match by provider name in the label (e.g., "OpenAI - GPT-4o")
28 const providerName = getProviderDisplayName(provider)
29 if (text?.includes(providerName) && text?.includes(model)) {
30 targetIndex = i
31 break
32 }
33 // Also match if model contains the text (for models like "gpt-4o-mini")
34 if (text?.includes(providerName)) {
35 // Check if this is the model we want by looking at model substring
36 const modelPart = model.split('/').pop() || model // Handle openrouter models like "openai/gpt-4o"
37 if (text?.toLowerCase().includes(modelPart.toLowerCase())) {
38 targetIndex = i
39 break
40 }
41 }
42 }
43
44 // If we found a match, select it; otherwise try to find by provider only
45 if (targetIndex === -1) {
46 for (let i = 0; i < options.length; i++) {
47 const text = await options[i].textContent()
48 const providerName = getProviderDisplayName(provider)
49 if (text?.includes(providerName)) {
50 targetIndex = i
51 break
52 }
53 }
54 }
55
56 if (targetIndex >= 0) {
57 await select.selectOption({ index: targetIndex })
58 } else {
59 throw new Error(
60 `Could not find model option for provider: ${provider}, model: ${model}`,
61 )
62 }
63}
64
65/**
66 * Get the display name for a provider (as shown in the UI)
67 */
68function getProviderDisplayName(provider: ProviderId): string {
69 const names: Record<ProviderId, string> = {
70 openai: 'OpenAI',
71 anthropic: 'Anthropic',
72 gemini: 'Gemini',
73 ollama: 'Ollama',
74 grok: 'Grok',
75 openrouter: 'OpenRouter',
76 }
77 return names[provider]
78}
79
80/**
81 * Send a message in the chat UI
82 */
83export async function sendMessage(page: Page, message: string): Promise<void> {
84 // Find the textarea input
85 const textarea = page.locator('textarea').first()
86 await textarea.waitFor({ state: 'visible' })
87
88 // Click to focus and clear any existing content
89 await textarea.click()
90 await textarea.fill('')
91
92 // Type the message character by character to properly trigger React state updates
93 await textarea.pressSequentially(message, { delay: 10 })
94
95 // Verify the input has the message
96 await textarea.waitFor({ state: 'visible' })
97
98 // Use keyboard Enter to send (more reliable than finding the button)
99 // The chat UI handles Enter key to send messages
100 await textarea.press('Enter')
101}
102
103/**
104 * Wait for the assistant response to complete
105 */
106export async function waitForResponse(
107 page: Page,
108 timeout: number = 60_000,
109): Promise<void> {
110 // Wait for loading to start - the stop button should appear or we see loading indicator
111 const stopButton = page.locator('button:has-text("Stop")')
112
113 // First, wait a moment for loading to potentially start
114 await page.waitForTimeout(1000)
115
116 // Check if loading started by looking for the stop button
117 const loadingStarted = await stopButton.isVisible().catch(() => false)
118
119 if (loadingStarted) {
120 // Wait for loading to complete (stop button to disappear)
121 try {
122 await stopButton.waitFor({ state: 'hidden', timeout: timeout - 1000 })
123 } catch {
124 // Stop button might still be visible if test times out
125 }
126 } else {
127 // Loading might have been too fast or there's an error
128 // Wait for either an assistant message or an error to appear
129 try {
130 // Wait for the messages JSON to contain an assistant message
131 await page.waitForFunction(
132 () => {
133 const preElements = document.querySelectorAll('pre')
134 for (const pre of preElements) {
135 const text = pre.textContent || ''
136 if (text.includes('"assistant"')) {
137 return true
138 }
139 }
140 return false
141 },
142 { timeout: timeout - 1000 },
143 )
144 } catch {
145 // Timeout waiting for response
146 }
147 }
148
149 // Additional wait for message to fully render
150 await page.waitForTimeout(500)
151}
152
153/**
154 * Get the last assistant message text from the chat
155 */
156export async function getAssistantMessage(page: Page): Promise<string> {
157 // First try to get messages from the debug panel JSON
158 const messages = await getMessages(page)
159
160 // Find the last assistant message (searching from the end)
161 for (let i = messages.length - 1; i >= 0; i--) {
162 const msg = messages[i]
163 if (msg.role === 'assistant') {
164 // Extract text content from parts
165 const textParts = msg.parts?.filter(
166 (p: any) => p.type === 'text' && p.content,
167 )
168 if (textParts?.length > 0) {
169 return textParts.map((p: any) => p.content).join(' ')
170 }
171 // If no text parts, check if there's direct content
172 if (msg.content) {
173 return msg.content
174 }
175 }
176 }
177
178 // Fallback: try to get text from the rendered chat messages
179 // Look for the AI indicator badge and get the adjacent prose content
180 try {
181 // The chat shows messages with an "AI" badge for assistant messages
182 // Get all message containers and find ones with assistant role indicator
183 const aiMessages = page.locator('.rounded-lg.mb-2').filter({
184 has: page.locator('text="AI"'),
185 })
186 const count = await aiMessages.count()
187 if (count > 0) {
188 const lastAiMessage = aiMessages.last()
189 const proseContent = lastAiMessage.locator('.prose')
190 if ((await proseContent.count()) > 0) {
191 const textContent = await proseContent
192 .first()
193 .textContent({ timeout: 5000 })
194 return textContent || ''
195 }
196 }
197 } catch {
198 // Ignore errors in fallback
199 }
200
201 return ''
202}
203
204/**
205 * Get all messages as parsed JSON from the debug panel
206 */
207export async function getMessages(page: Page): Promise<any[]> {
208 const messagesJson = page.locator('pre').filter({ hasText: '"role"' }).first()
209
210 try {
211 const jsonText = await messagesJson.textContent({ timeout: 5000 })
212 if (jsonText) {
213 return JSON.parse(jsonText)
214 }
215 } catch {
216 // Return empty array if parsing fails
217 }
218
219 return []
220}
221
222/**
223 * Check if the last message has tool calls
224 */
225export async function hasToolCalls(page: Page): Promise<boolean> {
226 const messages = await getMessages(page)
227 for (let i = messages.length - 1; i >= 0; i--) {
228 if (messages[i].role === 'assistant') {
229 const toolCalls = messages[i].parts?.filter(
230 (p: any) => p.type === 'tool-call',
231 )
232 return toolCalls?.length > 0
233 }
234 }
235 return false
236}
237
238/**
239 * Get tool call names from the last assistant message
240 */
241export async function getToolCallNames(page: Page): Promise<string[]> {
242 const messages = await getMessages(page)
243 for (let i = messages.length - 1; i >= 0; i--) {
244 if (messages[i].role === 'assistant') {
245 const toolCalls = messages[i].parts?.filter(
246 (p: any) => p.type === 'tool-call',
247 )
248 return toolCalls?.map((tc: any) => tc.name) || []
249 }
250 }
251 return []
252}
253
254/**
255 * Options for summarization API call
256 */
257export interface SummarizeOptions {
258 text: string
259 provider: ProviderId
260 model?: string
261 maxLength?: number
262 style?: 'concise' | 'detailed' | 'bullet-points'
263 stream?: boolean
264}
265
266/**
267 * Call the summarize API directly (non-streaming)
268 */
269export async function callSummarizeAPI(
270 request: APIRequestContext,
271 baseURL: string,
272 options: SummarizeOptions,
273): Promise<{ summary: string; provider: string; model: string }> {
274 const response = await request.post(`${baseURL}/api/summarize`, {
275 data: {
276 text: options.text,
277 provider: options.provider,
278 model: options.model,
279 maxLength: options.maxLength || 100,
280 style: options.style || 'concise',
281 stream: false,
282 },
283 })
284
285 if (!response.ok()) {
286 const errorBody = await response.text()
287 throw new Error(`Summarize API failed: ${response.status()} - ${errorBody}`)
288 }
289
290 return response.json()
291}
292
293/**
294 * Call the summarize API with streaming
295 */
296export async function callSummarizeAPIStreaming(
297 request: APIRequestContext,
298 baseURL: string,
299 options: SummarizeOptions,
300): Promise<{
301 summary: string
302 provider: string
303 model: string
304 chunkCount: number
305}> {
306 const response = await request.post(`${baseURL}/api/summarize`, {
307 data: {
308 text: options.text,
309 provider: options.provider,
310 model: options.model,
311 maxLength: options.maxLength || 100,
312 style: options.style || 'concise',
313 stream: true,
314 },
315 })
316
317 if (!response.ok()) {
318 const errorBody = await response.text()
319 throw new Error(
320 `Summarize API streaming failed: ${response.status()} - ${errorBody}`,
321 )
322 }
323
324 // Parse SSE response
325 const text = await response.text()
326 const lines = text.split('\n')
327
328 let summary = ''
329 let provider = ''
330 let model = ''
331 let chunkCount = 0
332
333 for (const line of lines) {
334 if (line.startsWith('data: ')) {
335 const data = line.slice(6)
336 if (data === '[DONE]') continue
337
338 try {
339 const parsed = JSON.parse(data)
340 if (parsed.type === 'error') {
341 throw new Error(parsed.error)
342 }
343 if (parsed.type === 'TEXT_MESSAGE_CONTENT') {
344 chunkCount++
345 if (parsed.delta) {
346 summary += parsed.delta
347 } else if (parsed.content) {
348 summary = parsed.content
349 }
350 provider = parsed.provider || provider
351 model = parsed.model || model
352 }
353 } catch {
354 // Ignore parse errors
355 }
356 }
357 }
358
359 return { summary, provider, model, chunkCount }
360}
361
362/**
363 * Sample text for summarization tests
364 */
365export const SAMPLE_TEXT_FOR_SUMMARIZATION = `Artificial intelligence (AI) is intelligence demonstrated by machines, as opposed to natural intelligence displayed by animals including humans. AI research has been defined as the field of study of intelligent agents, which refers to any system that perceives its environment and takes actions that maximize its chance of achieving its goals.
366
367The term "artificial intelligence" had previously been used to describe machines that mimic and display "human" cognitive skills that are associated with the human mind, such as "learning" and "problem-solving". This definition has since been rejected by major AI researchers who now describe AI in terms of rationality and acting rationally, which does not limit how intelligence can be articulated.
368
369AI applications include advanced web search engines, recommendation systems, understanding human speech, self-driving cars, automated decision-making and competing at the highest level in strategic game systems. As machines become increasingly capable, tasks considered to require "intelligence" are often removed from the definition of AI, a phenomenon known as the AI effect.`
370
371/**
372 * Navigate to the chat page and wait for it to load
373 */
374export async function goToChatPage(page: Page): Promise<void> {
375 await page.goto('/')
376 // Wait for the model selector to be visible
377 await page.locator('select').first().waitFor({ state: 'visible' })
378 // Wait a bit for hydration
379 await page.waitForTimeout(500)
380}
381
382/**
383 * Navigate to the summarize page and wait for it to load
384 */
385export async function goToSummarizePage(page: Page): Promise<void> {
386 await page.goto('/summarize')
387 // Wait for the provider selector to be visible
388 await page.locator('select').first().waitFor({ state: 'visible' })
389 // Wait a bit for hydration
390 await page.waitForTimeout(500)
391}