Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

structured-output-middleware.spec.ts

136 lines4.9 KB

1import { test, expect } from './fixtures'
2​
3/**
4 * E2E coverage for the structured-output × middleware interaction
5 * introduced in TanStack/ai#390. Two contracts under test:
6 *
7 * 1. Phase observation: the middleware chain sees chunks tagged with
8 * `ctx.phase === 'structuredOutput'` and `onFinish` is invoked exactly
9 * once. Guards against the pre-fix regression where the structured
10 * finalization adapter call bypassed the middleware chain entirely.
11 *
12 * 2. Streaming lifecycle: the consumer sees exactly one RUN_STARTED and
13 * one RUN_FINISHED chunk pair. Guards against the pre-fix regression
14 * where the streaming structured-output path emitted a synthetic
15 * terminal pair on top of the engine's, producing duplicate run
16 * lifecycle events to the consumer.
17 *
18 * Both tests drive the existing `/middleware-test` harness route with the
19 * new `phase-recorder` middleware mode, which records `ctx.phase` per chunk
20 * + `onFinish` count + post-middleware yielded chunks into a per-testId
21 * server-side store. The page fetches that store in `onFinish` and surfaces
22 * it via DOM elements (`#mw-phases-json`, `#mw-onfinish-count`,
23 * `#mw-yielded-chunks-json`) which the spec reads.
24 *
25 * NOTE on naming: both scenarios below run with `stream: true` against the
26 * harness route — the harness currently hard-codes streaming for both
27 * `structured-output` and `structured-output-stream` modes. The first test
28 * exercises the structured-output phase coverage via the SSE harness; the
29 * true non-streaming `Promise<T>` path is covered by unit tests in
30 * `packages/typescript/ai/tests/structured-output-middleware.test.ts`.
31 * Adding a real non-streaming E2E scenario would require page-side rewiring
32 * to await `Promise<T>` instead of iterating SSE and is out of scope here.
33 */
34​
35function buildHarnessUrl(testId?: string, aimockPort?: number): string {
36 const params = new URLSearchParams()
37 if (testId) params.set('testId', testId)
38 if (aimockPort) params.set('aimockPort', String(aimockPort))
39 const qs = params.toString()
40 return `/middleware-test${qs ? '?' + qs : ''}`
42​
43function parseStringArray(raw: string | null): Array<string> {
44 if (!raw) return []
45 let parsed: unknown
46 try {
47 parsed = JSON.parse(raw)
48 } catch {
49 return []
50 }
51 return Array.isArray(parsed) && parsed.every((p) => typeof p === 'string')
52 ? parsed
53 : []
55​
56function parseChunkSummaries(raw: string | null): Array<{ type: string }> {
57 if (!raw) return []
58 let parsed: unknown
59 try {
60 parsed = JSON.parse(raw)
61 } catch {
62 return []
63 }
64 if (!Array.isArray(parsed)) return []
65 const result: Array<{ type: string }> = []
66 for (const entry of parsed) {
67 if (!entry || typeof entry !== 'object') continue
68 const inner: Record<string, unknown> = { ...entry }
69 const t = inner.type
70 if (typeof t === 'string') result.push({ type: t })
71 }
72 return result
74​
75test.describe('Structured Output × Middleware Coverage', () => {
76 test('structured output with stream:true: middleware observes finalization phase chunks', async ({
77 page,
78 testId,
79 aimockPort,
80 }) => {
81 await page.goto(buildHarnessUrl(testId, aimockPort))
82 await page.waitForTimeout(2000)
83​
84 await page.locator('#mw-scenario-select').selectOption('structured-output')
85 await page.locator('#mw-mode-select').selectOption('phase-recorder')
86 await page.locator('#mw-run-button').click()
87​
88 await page.waitForFunction(
89 () =>
90 document
91 .querySelector('#mw-metadata')
92 ?.getAttribute('data-test-complete') === 'true',
93 { timeout: 15000 },
94 )
95​
96 const phasesJson = await page.locator('#mw-phases-json').textContent()
97 const phases = parseStringArray(phasesJson)
98 expect(phases).toContain('structuredOutput')
99​
100 const finishCountRaw = await page
101 .locator('#mw-onfinish-count')
102 .textContent()
103 const finishCount = Number(finishCountRaw ?? '0')
104 expect(finishCount).toBe(1)
105 })
106​
107 test('streaming structured output: consumer sees exactly one RUN_STARTED/RUN_FINISHED pair', async ({
108 page,
109 testId,
110 aimockPort,
111 }) => {
112 await page.goto(buildHarnessUrl(testId, aimockPort))
113 await page.waitForTimeout(2000)
114​
115 await page
116 .locator('#mw-scenario-select')
117 .selectOption('structured-output-stream')
118 await page.locator('#mw-mode-select').selectOption('phase-recorder')
119 await page.locator('#mw-run-button').click()
120​
121 await page.waitForFunction(
122 () =>
123 document
124 .querySelector('#mw-metadata')
125 ?.getAttribute('data-test-complete') === 'true',
126 { timeout: 15000 },
127 )
128​
129 const yieldedJson = await page
130 .locator('#mw-yielded-chunks-json')
131 .textContent()
132 const yielded = parseChunkSummaries(yieldedJson)
133 expect(yielded.filter((c) => c.type === 'RUN_STARTED')).toHaveLength(1)
134 expect(yielded.filter((c) => c.type === 'RUN_FINISHED')).toHaveLength(1)
135 })
136})