Open History to see the last commit
1/**
2 * Summarization E2E Tests
3 *
4 * Tests the summarization API endpoint with both streaming and non-streaming modes
5 * across all providers.
6 */
7
8import { test, expect } from '@playwright/test'
9import {
10 PROVIDERS,
11 isProviderAvailable,
12 getSummarizationCapableProviders,
13 getStreamingSummarizationCapableProviders,
14} from './vendor-config'
15import {
16 callSummarizeAPI,
17 callSummarizeAPIStreaming,
18 SAMPLE_TEXT_FOR_SUMMARIZATION,
19} from './helpers'
20
21const BASE_URL = 'http://localhost:3010'
22
23// Test non-streaming summarization for each provider
24for (const provider of PROVIDERS.filter((p) => p.supportsSummarization)) {
25 test.describe(`${provider.name} - Non-Streaming Summarization`, () => {
26 // Skip if provider is not available
27 test.skip(
28 () => !isProviderAvailable(provider),
29 `${provider.name} API key not configured (requires ${provider.envKey || 'no key'})`,
30 )
31
32 test('should summarize text successfully', async ({ request }) => {
33 const result = await callSummarizeAPI(request, BASE_URL, {
34 text: SAMPLE_TEXT_FOR_SUMMARIZATION,
35 provider: provider.id,
36 maxLength: 100,
37 style: 'concise',
38 })
39
40 // Verify we got a summary
41 expect(result.summary).toBeTruthy()
42 expect(result.summary.length).toBeGreaterThan(0)
43
44 // Summary should be shorter than the original text
45 expect(result.summary.length).toBeLessThan(
46 SAMPLE_TEXT_FOR_SUMMARIZATION.length,
47 )
48
49 // Verify provider info is returned
50 expect(result.provider).toBe(provider.id)
51 expect(result.model).toBeTruthy()
52 })
53
54 test('should handle different summary styles', async ({ request }) => {
55 // Test bullet-points style
56 const result = await callSummarizeAPI(request, BASE_URL, {
57 text: SAMPLE_TEXT_FOR_SUMMARIZATION,
58 provider: provider.id,
59 maxLength: 150,
60 style: 'bullet-points',
61 })
62
63 expect(result.summary).toBeTruthy()
64 })
65
66 test('should respect maxLength parameter', async ({ request }) => {
67 // Request a very short summary
68 const result = await callSummarizeAPI(request, BASE_URL, {
69 text: SAMPLE_TEXT_FOR_SUMMARIZATION,
70 provider: provider.id,
71 maxLength: 30,
72 style: 'concise',
73 })
74
75 expect(result.summary).toBeTruthy()
76
77 // The summary should be reasonably short (models don't always respect exact limits)
78 // We'll just verify it's significantly shorter than the input
79 expect(result.summary.length).toBeLessThan(
80 SAMPLE_TEXT_FOR_SUMMARIZATION.length / 2,
81 )
82 })
83 })
84}
85
86// Test streaming summarization for each provider that supports it
87for (const provider of PROVIDERS.filter(
88 (p) => p.supportsStreamingSummarization,
89)) {
90 test.describe(`${provider.name} - Streaming Summarization`, () => {
91 // Skip if provider is not available
92 test.skip(
93 () => !isProviderAvailable(provider),
94 `${provider.name} API key not configured (requires ${provider.envKey || 'no key'})`,
95 )
96
97 test('should stream summary chunks', async ({ request }) => {
98 const result = await callSummarizeAPIStreaming(request, BASE_URL, {
99 text: SAMPLE_TEXT_FOR_SUMMARIZATION,
100 provider: provider.id,
101 maxLength: 100,
102 style: 'concise',
103 })
104
105 // Verify we got a summary
106 expect(result.summary).toBeTruthy()
107 expect(result.summary.length).toBeGreaterThan(0)
108
109 // Verify we received multiple chunks (streaming)
110 expect(result.chunkCount).toBeGreaterThan(0)
111
112 // Verify provider info
113 expect(result.provider).toBe(provider.id)
114 expect(result.model).toBeTruthy()
115 })
116
117 test('should produce same quality summary as non-streaming', async ({
118 request,
119 }) => {
120 // Get non-streaming summary
121 const nonStreaming = await callSummarizeAPI(request, BASE_URL, {
122 text: SAMPLE_TEXT_FOR_SUMMARIZATION,
123 provider: provider.id,
124 maxLength: 100,
125 style: 'concise',
126 })
127
128 // Get streaming summary
129 const streaming = await callSummarizeAPIStreaming(request, BASE_URL, {
130 text: SAMPLE_TEXT_FOR_SUMMARIZATION,
131 provider: provider.id,
132 maxLength: 100,
133 style: 'concise',
134 })
135
136 // Both should produce valid summaries
137 expect(nonStreaming.summary).toBeTruthy()
138 expect(streaming.summary).toBeTruthy()
139
140 // Both should be reasonably sized (similar length, within 50% of each other)
141 const ratio =
142 streaming.summary.length / Math.max(nonStreaming.summary.length, 1)
143 expect(ratio).toBeGreaterThan(0.3)
144 expect(ratio).toBeLessThan(3)
145 })
146 })
147}
148
149// Test error handling
150test.describe('Summarization Error Handling', () => {
151 test('should handle empty text gracefully', async ({ request }) => {
152 // Get the first available provider
153 const availableProviders = getSummarizationCapableProviders()
154 test.skip(
155 availableProviders.length === 0,
156 'No summarization providers available',
157 )
158
159 const provider = availableProviders[0]
160
161 try {
162 await callSummarizeAPI(request, BASE_URL, {
163 text: '',
164 provider: provider.id,
165 maxLength: 100,
166 })
167 // If we get here, the API didn't reject empty text - that's also acceptable
168 } catch (error: any) {
169 // Error is expected for empty text
170 expect(error.message).toBeTruthy()
171 }
172 })
173})
174
175// Verify providers are available
176test('at least one summarization provider should be available', async () => {
177 const available = getSummarizationCapableProviders()
178 expect(available.length).toBeGreaterThanOrEqual(0)
179})
180
181test('at least one streaming summarization provider should be available', async () => {
182 const available = getStreamingSummarizationCapableProviders()
183 expect(available.length).toBeGreaterThanOrEqual(0)
184})