Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

middleware.spec.ts

511 lines18.7 KB

1import { SpanKind } from '@opentelemetry/api'
2import { test, expect } from './fixtures'
3​
4async function fetchOtelCapture(
5 page: import('@playwright/test').Page,
6 baseURL: string | undefined,
7 testId: string | undefined,
8) {
9 if (!testId) throw new Error('otel capture test requires a testId fixture')
10 const url = `${baseURL ?? ''}/api/middleware-test?testId=${encodeURIComponent(testId)}`
11 const response = await page.request.get(url)
12 if (!response.ok()) {
13 throw new Error(
14 `GET ${url} failed: ${response.status()} ${await response.text()}`,
15 )
16 }
17 return response.json()
19​
20test.describe('Middleware Lifecycle', () => {
21 test('onChunk transforms text content', async ({
22 page,
23 testId,
24 aimockPort,
25 }) => {
26 const params = new URLSearchParams()
27 if (testId) params.set('testId', testId)
28 if (aimockPort) params.set('aimockPort', String(aimockPort))
29 const qs = params.toString()
30 await page.goto(`/middleware-test${qs ? '?' + qs : ''}`)
31 await page.waitForTimeout(2000) // hydration
32 await page.locator('#mw-scenario-select').selectOption('basic-text')
33 await page.locator('#mw-mode-select').selectOption('chunk-transform')
34 await page.locator('#mw-run-button').click()
35​
36 await page.waitForFunction(
37 () =>
38 document
39 .querySelector('#mw-metadata')
40 ?.getAttribute('data-test-complete') === 'true',
41 { timeout: 10000 },
42 )
43​
44 const messagesJson = await page.locator('#mw-messages-json').textContent()
45 const messages = JSON.parse(messagesJson || '[]')
46 const assistantMsg = messages.find((m: any) => m.role === 'assistant')
47 const textPart = assistantMsg?.parts?.find((p: any) => p.type === 'text')
48 expect(textPart?.content).toContain('[MW]')
49 })
50​
51 test('onBeforeToolCall skips tool execution', async ({
52 page,
53 testId,
54 aimockPort,
55 }) => {
56 const params = new URLSearchParams()
57 if (testId) params.set('testId', testId)
58 if (aimockPort) params.set('aimockPort', String(aimockPort))
59 const qs = params.toString()
60 await page.goto(`/middleware-test${qs ? '?' + qs : ''}`)
61 await page.waitForTimeout(2000)
62 await page.locator('#mw-scenario-select').selectOption('with-tool')
63 await page.locator('#mw-mode-select').selectOption('tool-skip')
64 await page.locator('#mw-run-button').click()
65​
66 await page.waitForFunction(
67 () =>
68 document
69 .querySelector('#mw-metadata')
70 ?.getAttribute('data-test-complete') === 'true',
71 { timeout: 10000 },
72 )
73​
74 const messagesJson = await page.locator('#mw-messages-json').textContent()
75 const messages = JSON.parse(messagesJson || '[]')
76​
77 // Find tool result parts
78 const toolResults = messages.flatMap((m: any) =>
79 m.parts.filter((p: any) => p.type === 'tool-result'),
80 )
81 expect(toolResults.length).toBeGreaterThan(0)
82 expect(toolResults[0].content).toContain('skipped')
83 })
84​
85 test('capability provide/consume flow surfaces the consumed value in the stream', async ({
86 page,
87 testId,
88 aimockPort,
89 }) => {
90 const params = new URLSearchParams()
91 if (testId) params.set('testId', testId)
92 if (aimockPort) params.set('aimockPort', String(aimockPort))
93 const qs = params.toString()
94 await page.goto(`/middleware-test${qs ? '?' + qs : ''}`)
95 await page.waitForTimeout(2000) // hydration
96 await page.locator('#mw-scenario-select').selectOption('capability')
97 await page.locator('#mw-mode-select').selectOption('capability')
98 await page.locator('#mw-run-button').click()
99​
100 await page.waitForFunction(
101 () =>
102 document
103 .querySelector('#mw-metadata')
104 ?.getAttribute('data-test-complete') === 'true',
105 { timeout: 10000 },
106 )
107​
108 const messagesJson = await page.locator('#mw-messages-json').textContent()
109 const messages = JSON.parse(messagesJson || '[]')
110 const assistantMsg = messages.find((m: any) => m.role === 'assistant')
111 const textPart = assistantMsg?.parts?.find((p: any) => p.type === 'text')
112 // The provider middleware provides "[CAP]" in setup(); the consumer
113 // middleware reads it via the capability getter in onChunk and prefixes
114 // each text delta. Seeing "[CAP]" in the streamed text proves the provided
115 // capability value flowed across middleware to the consumer.
116 expect(textPart?.content).toContain('[CAP]')
117 expect(textPart?.content).toContain('Hello')
118 })
119​
120 test('otel middleware emits chat span + per-iteration token histograms', async ({
121 page,
122 testId,
123 aimockPort,
124 baseURL,
125 }) => {
126 const params = new URLSearchParams()
127 if (testId) params.set('testId', testId)
128 if (aimockPort) params.set('aimockPort', String(aimockPort))
129 const qs = params.toString()
130 await page.goto(`/middleware-test${qs ? '?' + qs : ''}`)
131 await page.waitForTimeout(2000)
132 await page.locator('#mw-scenario-select').selectOption('basic-text')
133 await page.locator('#mw-mode-select').selectOption('otel')
134 await page.locator('#mw-run-button').click()
135​
136 await page.waitForFunction(
137 () =>
138 document
139 .querySelector('#mw-metadata')
140 ?.getAttribute('data-test-complete') === 'true',
141 { timeout: 10000 },
142 )
143​
144 const capture = await fetchOtelCapture(page, baseURL, testId)
145​
146 // Root span is kind=INTERNAL; iteration spans are kind=CLIENT. This is a
147 // structural discriminator, immune to accidental attribute renames on
148 // either span.
149 const chatSpans = capture.spans.filter(
150 (s: any) => s.kind === SpanKind.INTERNAL,
151 )
152 expect(chatSpans).toHaveLength(1)
153 const chatSpan = chatSpans[0]
154 expect(chatSpan.ended).toBe(true)
155 // `gen_ai.operation.name` is intentionally NOT set on the root span —
156 // only iteration spans carry it (see otel.ts).
157 expect(chatSpan.attributes['gen_ai.operation.name']).toBeUndefined()
158​
159 const iterationSpans = capture.spans.filter(
160 (s: any) => s.kind === SpanKind.CLIENT,
161 )
162 expect(iterationSpans.length).toBeGreaterThanOrEqual(1)
163 for (const iter of iterationSpans) {
164 expect(iter.ended).toBe(true)
165 expect(iter.attributes['gen_ai.operation.name']).toBe('chat')
166 }
167​
168 // Token histogram records show up with correct unit and low-cardinality attrs.
169 const tokenRecords = capture.histograms.filter(
170 (h: any) => h.name === 'gen_ai.client.token.usage',
171 )
172 // Guard against the C1 regression: onUsage used to no-op in production order,
173 // losing every token histogram record. If we ever regress, this assertion fails.
174 expect(tokenRecords.length).toBeGreaterThanOrEqual(2)
175 for (const r of tokenRecords) {
176 expect(r.unit).toBe('{token}')
177 expect(r.attributes['gen_ai.response.id']).toBeUndefined()
178 expect(r.attributes['gen_ai.response.model']).toBeUndefined()
179 }
180​
181 // Duration histogram is per-run.
182 const durationRecords = capture.histograms.filter(
183 (h: any) => h.name === 'gen_ai.client.operation.duration',
184 )
185 expect(durationRecords.length).toBe(1)
186 expect(durationRecords[0].unit).toBe('s')
187 expect(
188 durationRecords[0].attributes['gen_ai.response.model'],
189 ).toBeUndefined()
190 })
191​
192 // #1054 — no-tools + outputSchema skips the agent loop. The only model
193 // call is structured-output finalization (`phase=structuredOutput`).
194 // otelMiddleware must open an iteration (CLIENT) span for that call so
195 // backends that key off generation spans (PostHog $ai_generation) and
196 // captureContent both work. Pin to claude-3-7-sonnet so we take the
197 // legacy finalization path, not native-combined (#605) which already
198 // emits a beforeModel iteration span.
199 test('otel middleware emits iteration span + captureContent for no-tools structured-output finalization', async ({
200 page,
201 testId,
202 aimockPort,
203 baseURL,
204 }) => {
205 const params = new URLSearchParams()
206 if (testId) params.set('testId', testId)
207 if (aimockPort) params.set('aimockPort', String(aimockPort))
208 params.set('provider', 'anthropic')
209 params.set('model', 'claude-3-7-sonnet')
210 const qs = params.toString()
211 await page.goto(`/middleware-test?${qs}`)
212 await page.waitForTimeout(2000)
213 await page.locator('#mw-scenario-select').selectOption('structured-output')
214 await page.locator('#mw-mode-select').selectOption('otel')
215 await page.locator('#mw-run-button').click()
216​
217 await page.waitForFunction(
218 () =>
219 document
220 .querySelector('#mw-metadata')
221 ?.getAttribute('data-test-complete') === 'true',
222 { timeout: 15000 },
223 )
224​
225 const capture = await fetchOtelCapture(page, baseURL, testId)
226​
227 const chatSpans = capture.spans.filter(
228 (s: any) => s.kind === SpanKind.INTERNAL,
229 )
230 expect(chatSpans).toHaveLength(1)
231​
232 const iterationSpans = capture.spans.filter(
233 (s: any) => s.kind === SpanKind.CLIENT,
234 )
235 // Exactly one provider call on the skip-agent-loop path → one generation.
236 expect(iterationSpans).toHaveLength(1)
237 const iter = iterationSpans[0]
238 expect(iter.ended).toBe(true)
239 expect(iter.attributes['gen_ai.operation.name']).toBe('chat')
240 expect(iter.attributes['tanstack.ai.iteration']).toBe(0)
241​
242 // captureContent is enabled on the harness otel middleware — prompt must
243 // land on the iteration span (the pre-fix silent no-op left this empty).
244 const inputMessages = iter.attributes['gen_ai.input.messages']
245 expect(typeof inputMessages).toBe('string')
246 expect(inputMessages.length).toBeGreaterThan(0)
247 })
248​
249 test('otel middleware nests tool spans under the iteration span that triggered them', async ({
250 page,
251 testId,
252 aimockPort,
253 baseURL,
254 }) => {
255 const params = new URLSearchParams()
256 if (testId) params.set('testId', testId)
257 if (aimockPort) params.set('aimockPort', String(aimockPort))
258 const qs = params.toString()
259 await page.goto(`/middleware-test${qs ? '?' + qs : ''}`)
260 await page.waitForTimeout(2000)
261 await page.locator('#mw-scenario-select').selectOption('with-tool')
262 await page.locator('#mw-mode-select').selectOption('otel')
263 await page.locator('#mw-run-button').click()
264​
265 await page.waitForFunction(
266 () =>
267 document
268 .querySelector('#mw-metadata')
269 ?.getAttribute('data-test-complete') === 'true',
270 { timeout: 15000 },
271 )
272​
273 const capture = await fetchOtelCapture(page, baseURL, testId)
274​
275 // Every tool span carries gen_ai.tool.name + ended outcome. This also
276 // guards against the "iteration span closed before onBeforeToolCall"
277 // regression — if it regressed, onBeforeToolCall would skip span creation.
278 const toolSpans = capture.spans.filter(
279 (s: any) => 'gen_ai.tool.name' in s.attributes,
280 )
281 expect(toolSpans.length).toBeGreaterThanOrEqual(1)
282 for (const tool of toolSpans) {
283 expect(tool.ended).toBe(true)
284 expect(tool.attributes['tanstack.ai.tool.outcome']).toBeDefined()
285 }
286 })
287​
288 test('otel middleware rolls up usage across a tool loop', async ({
289 request,
290 testId,
291 }) => {
292 const res = await request.post('/api/otel-usage', {
293 data: { provider: 'tool-loop', testId },
294 })
295 expect(res.ok()).toBe(true)
296 const { ok, error, spans } = await res.json()
297 expect(error ?? null).toBeNull()
298 expect(ok).toBe(true)
299​
300 const iterationSpans = spans.filter(
301 (span: any) => span.kind === SpanKind.CLIENT,
302 )
303 expect(iterationSpans).toHaveLength(2)
304 expect(iterationSpans.every((span: any) => span.ended)).toBe(true)
305​
306 const rootSpans = spans.filter(
307 (span: any) =>
308 span.kind === SpanKind.INTERNAL &&
309 span.attributes['tanstack.ai.iterations'] === 2,
310 )
311 expect(rootSpans).toHaveLength(1)
312 expect(rootSpans[0].ended).toBe(true)
313​
314 for (const key of [
315 'gen_ai.usage.input_tokens',
316 'gen_ai.usage.output_tokens',
317 'gen_ai.usage.total_tokens',
318 ]) {
319 expect(rootSpans[0].attributes[key]).toBe(
320 iterationSpans.reduce(
321 (total: number, span: any) => total + span.attributes[key],
322 0,
323 ),
324 )
325 }
326 })
327​
328 test('otel middleware emits total/cache/reasoning usage details on spans', async ({
329 request,
330 }) => {
331 // `/api/otel-usage` drives the OpenAI adapter against the
332 // `/openai-usage-details` aimock mount (total_tokens + cached_tokens +
333 // reasoning_tokens) with otelMiddleware attached, and returns the
334 // captured spans. End-to-end proof for #721: the full TokenUsage reaches
335 // span attributes, not just input/output tokens.
336 const res = await request.post('/api/otel-usage', {
337 data: { provider: 'openai' },
338 })
339 expect(res.ok()).toBe(true)
340 const { ok, error, spans } = await res.json()
341 expect(error ?? null).toBeNull()
342 expect(ok).toBe(true)
343​
344 const iterationSpans = spans.filter(
345 (s: any) => s.kind === SpanKind.CLIENT && s.ended,
346 )
347 expect(iterationSpans.length).toBeGreaterThanOrEqual(1)
348 expect(iterationSpans[0].attributes).toMatchObject({
349 'gen_ai.usage.input_tokens': 100,
350 'gen_ai.usage.output_tokens': 50,
351 'gen_ai.usage.total_tokens': 150,
352 'gen_ai.usage.cache_read.input_tokens': 80,
353 'gen_ai.usage.reasoning.output_tokens': 30,
354 })
355​
356 // The root span rolls up the final usage on onFinish.
357 const rootSpans = spans.filter((s: any) => s.kind === SpanKind.INTERNAL)
358 expect(rootSpans).toHaveLength(1)
359 expect(rootSpans[0].attributes).toMatchObject({
360 'gen_ai.usage.total_tokens': 150,
361 'gen_ai.usage.cache_read.input_tokens': 80,
362 'gen_ai.usage.reasoning.output_tokens': 30,
363 })
364 })
365​
366 test('otel middleware emits provider-reported cost on spans', async ({
367 request,
368 }) => {
369 // OpenRouter adapter against the `/openrouter-cost` mount, whose trailing
370 // usage chunk carries `cost` / `cost_details`. Backends like PostHog read
371 // `gen_ai.usage.cost` directly instead of re-deriving from price tables.
372 const res = await request.post('/api/otel-usage', {
373 data: { provider: 'openrouter' },
374 })
375 expect(res.ok()).toBe(true)
376 const { ok, error, spans } = await res.json()
377 expect(error ?? null).toBeNull()
378 expect(ok).toBe(true)
379​
380 const iterationSpans = spans.filter(
381 (s: any) => s.kind === SpanKind.CLIENT && s.ended,
382 )
383 expect(iterationSpans.length).toBeGreaterThanOrEqual(1)
384 expect(iterationSpans[0].attributes).toMatchObject({
385 'gen_ai.usage.input_tokens': 11,
386 'gen_ai.usage.output_tokens': 3,
387 'gen_ai.usage.total_tokens': 14,
388 'gen_ai.usage.cost': 0.0042,
389 'tanstack.ai.usage.upstream_cost': 0.0038,
390 'tanstack.ai.usage.upstream_input_cost': 0.0012,
391 'tanstack.ai.usage.upstream_output_cost': 0.0026,
392 })
393​
394 const rootSpans = spans.filter((s: any) => s.kind === SpanKind.INTERNAL)
395 expect(rootSpans).toHaveLength(1)
396 expect(rootSpans[0].attributes['gen_ai.usage.cost']).toBe(0.0042)
397 })
398​
399 test('otelMiddleware emits an image_generation span for generateImage', async ({
400 request,
401 testId,
402 aimockPort,
403 }) => {
404 // `/api/otel-media` drives generateImage with otelMiddleware against the
405 // image-gen aimock mount and returns the captured spans. End-to-end proof
406 // that the unified middleware tags a non-chat activity with the right
407 // `gen_ai.operation.name` — the same `otelMiddleware` value used for chat,
408 // passed through the media `middleware` slot.
409 const res = await request.post('/api/otel-media', {
410 data: {
411 prompt: 'a guitar in a music store',
412 provider: 'openai',
413 testId,
414 aimockPort,
415 },
416 })
417 expect(res.ok()).toBe(true)
418 const { ok, error, spans } = await res.json()
419 expect(error ?? null).toBeNull()
420 expect(ok).toBe(true)
421​
422 const mediaSpans = spans.filter(
423 (s: any) => s.attributes['gen_ai.operation.name'] === 'image_generation',
424 )
425 expect(mediaSpans).toHaveLength(1)
426 expect(mediaSpans[0].kind).toBe(SpanKind.CLIENT)
427 expect(mediaSpans[0].ended).toBe(true)
428 expect(mediaSpans[0].attributes).toMatchObject({
429 'gen_ai.system': 'openai',
430 'gen_ai.operation.name': 'image_generation',
431 'gen_ai.request.model': 'gpt-image-1',
432 })
433 })
434​
435 test('no middleware passes content through unchanged', async ({
436 page,
437 testId,
438 aimockPort,
439 }) => {
440 const params = new URLSearchParams()
441 if (testId) params.set('testId', testId)
442 if (aimockPort) params.set('aimockPort', String(aimockPort))
443 const qs = params.toString()
444 await page.goto(`/middleware-test${qs ? '?' + qs : ''}`)
445 await page.waitForTimeout(2000)
446 await page.locator('#mw-scenario-select').selectOption('basic-text')
447 await page.locator('#mw-mode-select').selectOption('none')
448 await page.locator('#mw-run-button').click()
449​
450 await page.waitForFunction(
451 () =>
452 document
453 .querySelector('#mw-metadata')
454 ?.getAttribute('data-test-complete') === 'true',
455 { timeout: 10000 },
456 )
457​
458 const messagesJson = await page.locator('#mw-messages-json').textContent()
459 const messages = JSON.parse(messagesJson || '[]')
460 const assistantMsg = messages.find((m: any) => m.role === 'assistant')
461 const textPart = assistantMsg?.parts?.find((p: any) => p.type === 'text')
462 expect(textPart?.content).not.toContain('[MW]')
463 expect(textPart?.content).toContain('Hello')
464 })
465​
466 test('memory middleware injects recalled prompt + tools and defers save', async ({
467 page,
468 testId,
469 aimockPort,
470 }) => {
471 const params = new URLSearchParams()
472 if (testId) params.set('testId', testId)
473 if (aimockPort) params.set('aimockPort', String(aimockPort))
474 const qs = params.toString()
475 await page.goto(`/middleware-test${qs ? '?' + qs : ''}`)
476 await page.waitForTimeout(2000) // hydration
477 await page.locator('#mw-scenario-select').selectOption('basic-text')
478 await page.locator('#mw-mode-select').selectOption('memory')
479 await page.locator('#mw-run-button').click()
480​
481 await page.waitForFunction(
482 () =>
483 document
484 .querySelector('#mw-metadata')
485 ?.getAttribute('data-test-complete') === 'true',
486 { timeout: 10000 },
487 )
488​
489 const memoryJson = await page.locator('#mw-memory-json').textContent()
490 const capture = JSON.parse(memoryJson || '{}') as {
491 configs: Array<{ systemPrompts: Array<string>; toolNames: Array<string> }>
492 saveCount: number
493 }
494​
495 // The recorder placed after memoryMiddleware saw the config the model
496 // receives: the recalled system prompt (+ tool guidance) and the injected
497 // tool must both be present.
498 expect(capture.configs.length).toBeGreaterThan(0)
499 const injected = capture.configs.find((c) =>
500 c.systemPrompts.some((p) => p.includes('love TanStack')),
501 )
502 expect(injected).toBeTruthy()
503 expect(injected?.systemPrompts.some((p) => p.includes('recall_more'))).toBe(
504 true,
505 )
506 expect(injected?.toolNames).toContain('recall_more')
507​
508 // The finished turn deferred a save through the adapter.
509 expect(capture.saveCount).toBeGreaterThanOrEqual(1)
510 })
511})