Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

basic-inference.spec.ts

103 lines3.2 KB

1/**
2 * Basic Inference E2E Tests
3 *
4 * Tests that each AI provider can respond to a simple "say hello" prompt.
5 * This validates that the adapter is correctly configured and can communicate
6 * with the vendor API.
7 */
8​
9import { test, expect } from '@playwright/test'
10import { PROVIDERS, isProviderAvailable } from './vendor-config'
11import {
12 goToChatPage,
13 selectProvider,
14 sendMessage,
15 waitForResponse,
16 getAssistantMessage,
17 getMessages,
18} from './helpers'
19​
20// Only test providers that support basic inference
21const inferenceProviders = PROVIDERS.filter((p) => p.supportsBasicInference)
22​
23for (const provider of inferenceProviders) {
24 test.describe(`${provider.name} - Basic Inference`, () => {
25 // Skip the entire describe block if provider is not available
26 test.skip(
27 () => !isProviderAvailable(provider),
28 `${provider.name} API key not configured (requires ${provider.envKey || 'no key'})`,
29 )
30​
31 test('should respond to a simple hello prompt', async ({ page }) => {
32 // Navigate to the chat page
33 await goToChatPage(page)
34​
35 // Select the provider and model
36 await selectProvider(page, provider.id, provider.defaultModel)
37​
38 // Send a simple prompt
39 await sendMessage(
40 page,
41 'Say hello in a friendly way. Just respond with a greeting.',
42 )
43​
44 // Wait for the response
45 await waitForResponse(page, 60_000)
46​
47 // Get the assistant's response
48 const response = await getAssistantMessage(page)
49​
50 // Verify we got a response
51 expect(response).toBeTruthy()
52 expect(response.length).toBeGreaterThan(0)
53​
54 // The response should contain some form of greeting
55 // We're flexible here since different models respond differently
56 const lowerResponse = response.toLowerCase()
57 const hasGreeting =
58 lowerResponse.includes('hello') ||
59 lowerResponse.includes('hi') ||
60 lowerResponse.includes('hey') ||
61 lowerResponse.includes('greetings') ||
62 lowerResponse.includes('welcome')
63​
64 expect(hasGreeting).toBe(true)
65 })
66​
67 test('should handle a follow-up question', async ({ page }) => {
68 // Navigate to the chat page
69 await goToChatPage(page)
70​
71 // Select the provider and model
72 await selectProvider(page, provider.id, provider.defaultModel)
73​
74 // Send first message
75 await sendMessage(page, 'What is 2 + 2? Just give me the number.')
76​
77 // Wait for response
78 await waitForResponse(page, 60_000)
79​
80 // Get all messages to verify structure
81 const messages = await getMessages(page)
82​
83 // Verify we have at least 2 messages (user + assistant)
84 expect(messages.length).toBeGreaterThanOrEqual(2)
85​
86 // Get assistant response
87 const response = await getAssistantMessage(page)
88​
89 // Verify we got a non-empty response that's not the user's message
90 expect(response).toBeTruthy()
91 expect(response).not.toContain('What is 2 + 2')
92​
93 // Response should contain "4" in some form
94 expect(response).toContain('4')
95 })
96 })
98​
99// Test that at least one provider is available
100test('at least one provider should be available', async () => {
101 const availableProviders = PROVIDERS.filter(isProviderAvailable)
102 expect(availableProviders.length).toBeGreaterThan(0)
103})