Open History to see the last commit
1/**
2 * Tool Flow E2E Tests
3 *
4 * Tests that AI providers can correctly invoke tools when prompted.
5 * Uses the existing guitar recommendation tools in the testing panel.
6 *
7 * The guitar store system prompt instructs the AI to:
8 * 1. Call getGuitars() to fetch inventory
9 * 2. Call recommendGuitar(id) to display a recommendation
10 *
11 * NOTE: These tests are marked as flaky because they depend on real LLM behavior.
12 * LLMs may not always decide to call tools even when prompted, and API latency
13 * can cause timeouts. The retries help account for this non-determinism.
14 */
15
16import { test, expect } from '@playwright/test'
17import {
18 PROVIDERS,
19 isProviderAvailable,
20 getToolCapableProviders,
21} from './vendor-config'
22import {
23 goToChatPage,
24 selectProvider,
25 sendMessage,
26 waitForResponse,
27 getMessages,
28 hasToolCalls,
29 getToolCallNames,
30} from './helpers'
31
32// Only test providers that support tool calling
33const toolProviders = PROVIDERS.filter((p) => p.supportsTools)
34
35for (const provider of toolProviders) {
36 test.describe(`${provider.name} - Tool Flow`, () => {
37 // These tests are flaky due to LLM non-determinism - models may not always call tools
38 test.describe.configure({ retries: 2 })
39
40 // Skip if provider is not available
41 test.skip(
42 () => !isProviderAvailable(provider),
43 `${provider.name} API key not configured (requires ${provider.envKey || 'no key'})`,
44 )
45
46 test('should call getGuitars tool when asked for guitar recommendation', async ({
47 page,
48 }) => {
49 // Navigate to the chat page
50 await goToChatPage(page)
51
52 // Select the provider and model
53 await selectProvider(page, provider.id, provider.defaultModel)
54
55 // Send a prompt that should trigger tool calls
56 await sendMessage(
57 page,
58 'Use the getGuitars tool to show me what guitars you have in inventory.',
59 )
60
61 // Wait for the response (tool calls may take longer)
62 await waitForResponse(page, 90_000)
63
64 // Check that tool calls were made
65 const madeToolCalls = await hasToolCalls(page)
66
67 expect(madeToolCalls).toBe(true)
68
69 // Verify getGuitars was called
70 const toolNames = await getToolCallNames(page)
71 expect(toolNames).toContain('getGuitars')
72 })
73
74 test('should complete full recommendation flow with multiple tool calls', async ({
75 page,
76 }) => {
77 // Navigate to the chat page
78 await goToChatPage(page)
79
80 // Select the provider and model
81 await selectProvider(page, provider.id, provider.defaultModel)
82
83 // Send a prompt that should trigger the full flow
84 await sendMessage(
85 page,
86 'Recommend me an electric guitar from your inventory.',
87 )
88
89 // Wait for the response (multiple tool calls may take longer)
90 await waitForResponse(page, 120_000)
91
92 // Get all messages to inspect the tool calls
93 const messages = await getMessages(page)
94
95 // Find assistant messages with tool calls
96 const assistantMessages = messages.filter(
97 (m: any) => m.role === 'assistant',
98 )
99 expect(assistantMessages.length).toBeGreaterThan(0)
100
101 // Check that we have tool calls
102 const allToolCalls: string[] = []
103 for (const msg of assistantMessages) {
104 const toolCalls = msg.parts?.filter((p: any) => p.type === 'tool-call')
105 if (toolCalls) {
106 allToolCalls.push(...toolCalls.map((tc: any) => tc.name))
107 }
108 }
109
110 // Should have called getGuitars at minimum
111 expect(allToolCalls).toContain('getGuitars')
112 })
113
114 test('should handle tool with arguments correctly', async ({ page }) => {
115 // Navigate to the chat page
116 await goToChatPage(page)
117
118 // Select the provider and model
119 await selectProvider(page, provider.id, provider.defaultModel)
120
121 // Send a specific request that should use recommendGuitar with an ID
122 await sendMessage(
123 page,
124 'Show me the guitars you have and then recommend guitar #1 to me.',
125 )
126
127 // Wait for the response
128 await waitForResponse(page, 120_000)
129
130 // Get all messages
131 const messages = await getMessages(page)
132
133 // Find tool calls with arguments
134 const allToolCalls: Array<{ name: string; arguments?: string }> = []
135 for (const msg of messages) {
136 if (msg.role === 'assistant') {
137 const toolCalls = msg.parts?.filter(
138 (p: any) => p.type === 'tool-call',
139 )
140 if (toolCalls) {
141 for (const tc of toolCalls) {
142 allToolCalls.push({
143 name: tc.name,
144 arguments: tc.arguments,
145 })
146 }
147 }
148 }
149 }
150
151 // Should have some tool calls
152 expect(allToolCalls.length).toBeGreaterThan(0)
153 })
154 })
155}
156
157// ===========================
158// Multi-turn follow-up tests
159// ===========================
160// These test that providers can handle follow-up messages AFTER tool calls.
161// This specifically catches the Anthropic bug where consecutive user-role messages
162// (tool_result + new user message) violate the alternating role constraint.
163
164for (const provider of toolProviders) {
165 test.describe(`${provider.name} - Multi-turn Tool Follow-up`, () => {
166 // Extended timeout for multi-turn conversations (two full LLM round-trips)
167 test.describe.configure({ retries: 2, timeout: 180_000 })
168
169 // Skip if provider is not available
170 test.skip(
171 () => !isProviderAvailable(provider),
172 `${provider.name} API key not configured (requires ${provider.envKey || 'no key'})`,
173 )
174
175 test('should handle follow-up message after tool call completes', async ({
176 page,
177 }) => {
178 // Navigate to the chat page
179 await goToChatPage(page)
180
181 // Select the provider and model
182 await selectProvider(page, provider.id, provider.defaultModel)
183
184 // First message: trigger a tool call
185 await sendMessage(
186 page,
187 'Use the getGuitars tool to show me what acoustic guitars you have.',
188 )
189
190 // Wait for the first response to complete (tool call + model response)
191 await waitForResponse(page, 120_000)
192
193 // Verify the first turn worked - should have tool calls
194 const firstMessages = await getMessages(page)
195 const firstAssistant = firstMessages.filter(
196 (m: any) => m.role === 'assistant',
197 )
198 expect(firstAssistant.length).toBeGreaterThan(0)
199
200 // Send a follow-up message - this is where the bug manifested
201 // With the Anthropic bug, this would fail with consecutive user-role messages
202 await sendMessage(page, 'Now tell me about electric guitars instead.')
203
204 // Wait for the follow-up response
205 await waitForResponse(page, 120_000)
206
207 // Get all messages after the follow-up
208 const allMessages = await getMessages(page)
209
210 // Should have at least 2 user messages and 2 assistant messages
211 const userMessages = allMessages.filter((m: any) => m.role === 'user')
212 const assistantMessages = allMessages.filter(
213 (m: any) => m.role === 'assistant',
214 )
215
216 expect(userMessages.length).toBeGreaterThanOrEqual(2)
217 expect(assistantMessages.length).toBeGreaterThanOrEqual(2)
218
219 // The LAST assistant message should have non-empty text content
220 // (not just tool calls, and not an error)
221 const lastAssistant = assistantMessages[assistantMessages.length - 1]
222 const textParts = lastAssistant.parts?.filter(
223 (p: any) => p.type === 'text' && p.content && p.content.length > 0,
224 )
225
226 // The follow-up should have produced some text OR tool calls
227 // (both are valid responses - the key is it didn't error out)
228 const hasText = textParts?.length > 0
229 const hasTools =
230 lastAssistant.parts?.some((p: any) => p.type === 'tool-call') || false
231 expect(hasText || hasTools).toBe(true)
232 })
233 })
234}
235
236// Verify we have tool-capable providers to test
237test('at least one tool-capable provider should be available', async () => {
238 const available = getToolCapableProviders()
239 expect(available.length).toBeGreaterThanOrEqual(0)
240})