Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

middleware.spec.ts

226 lines8.1 KB

1import { SpanKind } from '@opentelemetry/api'
2import { test, expect } from './fixtures'
3​
4async function fetchOtelCapture(
5 page: import('@playwright/test').Page,
6 baseURL: string | undefined,
7 testId: string | undefined,
8) {
9 if (!testId) throw new Error('otel capture test requires a testId fixture')
10 const url = `${baseURL ?? ''}/api/middleware-test?testId=${encodeURIComponent(testId)}`
11 const response = await page.request.get(url)
12 if (!response.ok()) {
13 throw new Error(
14 `GET ${url} failed: ${response.status()} ${await response.text()}`,
15 )
16 }
17 return response.json()
19​
20test.describe('Middleware Lifecycle', () => {
21 test('onChunk transforms text content', async ({
22 page,
23 testId,
24 aimockPort,
25 }) => {
26 const params = new URLSearchParams()
27 if (testId) params.set('testId', testId)
28 if (aimockPort) params.set('aimockPort', String(aimockPort))
29 const qs = params.toString()
30 await page.goto(`/middleware-test${qs ? '?' + qs : ''}`)
31 await page.waitForTimeout(2000) // hydration
32 await page.locator('#mw-scenario-select').selectOption('basic-text')
33 await page.locator('#mw-mode-select').selectOption('chunk-transform')
34 await page.locator('#mw-run-button').click()
35​
36 await page.waitForFunction(
37 () =>
38 document
39 .querySelector('#mw-metadata')
40 ?.getAttribute('data-test-complete') === 'true',
41 { timeout: 10000 },
42 )
43​
44 const messagesJson = await page.locator('#mw-messages-json').textContent()
45 const messages = JSON.parse(messagesJson || '[]')
46 const assistantMsg = messages.find((m: any) => m.role === 'assistant')
47 const textPart = assistantMsg?.parts?.find((p: any) => p.type === 'text')
48 expect(textPart?.content).toContain('[MW]')
49 })
50​
51 test('onBeforeToolCall skips tool execution', async ({
52 page,
53 testId,
54 aimockPort,
55 }) => {
56 const params = new URLSearchParams()
57 if (testId) params.set('testId', testId)
58 if (aimockPort) params.set('aimockPort', String(aimockPort))
59 const qs = params.toString()
60 await page.goto(`/middleware-test${qs ? '?' + qs : ''}`)
61 await page.waitForTimeout(2000)
62 await page.locator('#mw-scenario-select').selectOption('with-tool')
63 await page.locator('#mw-mode-select').selectOption('tool-skip')
64 await page.locator('#mw-run-button').click()
65​
66 await page.waitForFunction(
67 () =>
68 document
69 .querySelector('#mw-metadata')
70 ?.getAttribute('data-test-complete') === 'true',
71 { timeout: 10000 },
72 )
73​
74 const messagesJson = await page.locator('#mw-messages-json').textContent()
75 const messages = JSON.parse(messagesJson || '[]')
76​
77 // Find tool result parts
78 const toolResults = messages.flatMap((m: any) =>
79 m.parts.filter((p: any) => p.type === 'tool-result'),
80 )
81 expect(toolResults.length).toBeGreaterThan(0)
82 expect(toolResults[0].content).toContain('skipped')
83 })
84​
85 test('otel middleware emits chat span + per-iteration token histograms', async ({
86 page,
87 testId,
88 aimockPort,
89 baseURL,
90 }) => {
91 const params = new URLSearchParams()
92 if (testId) params.set('testId', testId)
93 if (aimockPort) params.set('aimockPort', String(aimockPort))
94 const qs = params.toString()
95 await page.goto(`/middleware-test${qs ? '?' + qs : ''}`)
96 await page.waitForTimeout(2000)
97 await page.locator('#mw-scenario-select').selectOption('basic-text')
98 await page.locator('#mw-mode-select').selectOption('otel')
99 await page.locator('#mw-run-button').click()
100​
101 await page.waitForFunction(
102 () =>
103 document
104 .querySelector('#mw-metadata')
105 ?.getAttribute('data-test-complete') === 'true',
106 { timeout: 10000 },
107 )
108​
109 const capture = await fetchOtelCapture(page, baseURL, testId)
110​
111 // Root span is kind=INTERNAL; iteration spans are kind=CLIENT. This is a
112 // structural discriminator, immune to accidental attribute renames on
113 // either span.
114 const chatSpans = capture.spans.filter(
115 (s: any) => s.kind === SpanKind.INTERNAL,
116 )
117 expect(chatSpans).toHaveLength(1)
118 const chatSpan = chatSpans[0]
119 expect(chatSpan.ended).toBe(true)
120 // `gen_ai.operation.name` is intentionally NOT set on the root span —
121 // only iteration spans carry it (see otel.ts).
122 expect(chatSpan.attributes['gen_ai.operation.name']).toBeUndefined()
123​
124 const iterationSpans = capture.spans.filter(
125 (s: any) => s.kind === SpanKind.CLIENT,
126 )
127 expect(iterationSpans.length).toBeGreaterThanOrEqual(1)
128 for (const iter of iterationSpans) {
129 expect(iter.ended).toBe(true)
130 expect(iter.attributes['gen_ai.operation.name']).toBe('chat')
131 }
132​
133 // Token histogram records show up with correct unit and low-cardinality attrs.
134 const tokenRecords = capture.histograms.filter(
135 (h: any) => h.name === 'gen_ai.client.token.usage',
136 )
137 // Guard against the C1 regression: onUsage used to no-op in production order,
138 // losing every token histogram record. If we ever regress, this assertion fails.
139 expect(tokenRecords.length).toBeGreaterThanOrEqual(2)
140 for (const r of tokenRecords) {
141 expect(r.unit).toBe('{token}')
142 expect(r.attributes['gen_ai.response.id']).toBeUndefined()
143 expect(r.attributes['gen_ai.response.model']).toBeUndefined()
144 }
145​
146 // Duration histogram is per-run.
147 const durationRecords = capture.histograms.filter(
148 (h: any) => h.name === 'gen_ai.client.operation.duration',
149 )
150 expect(durationRecords.length).toBe(1)
151 expect(durationRecords[0].unit).toBe('s')
152 expect(
153 durationRecords[0].attributes['gen_ai.response.model'],
154 ).toBeUndefined()
155 })
156​
157 test('otel middleware nests tool spans under the iteration span that triggered them', async ({
158 page,
159 testId,
160 aimockPort,
161 baseURL,
162 }) => {
163 const params = new URLSearchParams()
164 if (testId) params.set('testId', testId)
165 if (aimockPort) params.set('aimockPort', String(aimockPort))
166 const qs = params.toString()
167 await page.goto(`/middleware-test${qs ? '?' + qs : ''}`)
168 await page.waitForTimeout(2000)
169 await page.locator('#mw-scenario-select').selectOption('with-tool')
170 await page.locator('#mw-mode-select').selectOption('otel')
171 await page.locator('#mw-run-button').click()
172​
173 await page.waitForFunction(
174 () =>
175 document
176 .querySelector('#mw-metadata')
177 ?.getAttribute('data-test-complete') === 'true',
178 { timeout: 15000 },
179 )
180​
181 const capture = await fetchOtelCapture(page, baseURL, testId)
182​
183 // Every tool span carries gen_ai.tool.name + ended outcome. This also
184 // guards against the "iteration span closed before onBeforeToolCall"
185 // regression — if it regressed, onBeforeToolCall would skip span creation.
186 const toolSpans = capture.spans.filter(
187 (s: any) => 'gen_ai.tool.name' in s.attributes,
188 )
189 expect(toolSpans.length).toBeGreaterThanOrEqual(1)
190 for (const tool of toolSpans) {
191 expect(tool.ended).toBe(true)
192 expect(tool.attributes['tanstack.ai.tool.outcome']).toBeDefined()
193 }
194 })
195​
196 test('no middleware passes content through unchanged', async ({
197 page,
198 testId,
199 aimockPort,
200 }) => {
201 const params = new URLSearchParams()
202 if (testId) params.set('testId', testId)
203 if (aimockPort) params.set('aimockPort', String(aimockPort))
204 const qs = params.toString()
205 await page.goto(`/middleware-test${qs ? '?' + qs : ''}`)
206 await page.waitForTimeout(2000)
207 await page.locator('#mw-scenario-select').selectOption('basic-text')
208 await page.locator('#mw-mode-select').selectOption('none')
209 await page.locator('#mw-run-button').click()
210​
211 await page.waitForFunction(
212 () =>
213 document
214 .querySelector('#mw-metadata')
215 ?.getAttribute('data-test-complete') === 'true',
216 { timeout: 10000 },
217 )
218​
219 const messagesJson = await page.locator('#mw-messages-json').textContent()
220 const messages = JSON.parse(messagesJson || '[]')
221 const assistantMsg = messages.find((m: any) => m.role === 'assistant')
222 const textPart = assistantMsg?.parts?.find((p: any) => p.type === 'text')
223 expect(textPart?.content).not.toContain('[MW]')
224 expect(textPart?.content).toContain('Hello')
225 })
226})