Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

agentic-structured-stream.spec.ts

98 lines4.3 KB

1import { test, expect } from './fixtures'
2import {
3 featureUrl,
4 getToolCalls,
5 sendMessage,
6 waitForAssistantText,
7 waitForResponse,
8} from './helpers'
9import { providersFor } from './test-matrix'
10​
11/**
12 * Per-provider coverage for #605 native combined-mode: `outputSchema` +
13 * `tools` + `stream: true` in a single chat call. The matrix is restricted
14 * to providers whose adapter declares `supportsCombinedToolsAndSchema`
15 * for the default (or feature-overridden) test model — see
16 * `feature-support.ts` and `features.ts`. The contracts below also hold for
17 * the legacy fallback path, but adding non-native-combined providers here
18 * would require an extra fixture sequence entry for the engine's
19 * `runStructuredFinalization` request, which is out of scope.
20 *
21 * Observable contracts pinned per provider:
22 * 1. A `getGuitars` tool call lands during the agent loop.
23 * 2. The schema-constrained final-turn content lands as a typed
24 * `structured-output` part on the assistant message (NOT a text part).
25 * Confirms the engine's synthetic `structured-output.start` reached
26 * the client and routed subsequent TEXT_MESSAGE_CONTENT deltas into a
27 * StructuredOutputPart.
28 * 3. The `structured-output.complete` custom event reaches the client
29 * with the parsed object matching the schema.
30 * 4. The content streamed (more than one TEXT_MESSAGE_CONTENT delta).
31 */
32for (const provider of providersFor('agentic-structured-stream')) {
33 test.describe(`${provider} — agentic-structured-stream`, () => {
34 test('streams tool calls and a schema-validated final message in one chat call', async ({
35 page,
36 testId,
37 aimockPort,
38 }) => {
39 // Raise the per-test cap (config default 30s) so the generous
40 // `structured-output-part` wait below isn't cut short by the test-level
41 // timeout under parallel-worker dev-server contention.
42 test.setTimeout(60_000)
43​
44 await page.goto(
45 featureUrl(provider, 'agentic-structured-stream', testId, aimockPort),
46 )
47 // Confirms useChat has hydrated before we send — under parallel workers
48 // the first POST can otherwise race React hydration and lose the
49 // synthetic `structured-output.start` event ordering at the processor.
50 await page.getByTestId('chat-input').waitFor({ state: 'visible' })
51​
52 await sendMessage(page, '[agentic-stream] check inventory and recommend')
53 await waitForResponse(page)
54​
55 const toolCalls = await getToolCalls(page)
56 expect(toolCalls.map((c) => c.name)).toContain('getGuitars')
57​
58 await waitForAssistantText(page, 'Fender Stratocaster')
59​
60 // Anchor on `structured-output-complete` first — this testid only
61 // renders after `useChat`'s `onCustomEvent('structured-output.complete')`
62 // fired. `waitForResponse` keys on `isLoading` flipping false
63 // (RUN_FINISHED), which can land before the React tree re-renders the
64 // streaming TextPart → completed StructuredOutputPart.
65 const completeEl = page.getByTestId('structured-output-complete')
66 await expect(completeEl).toBeAttached({ timeout: 10_000 })
67​
68 // Use a longer timeout — under parallel-worker dev-server contention
69 // the final React commit landing the structured-output part can lag
70 // well behind the `structured-output.complete` testid attachment (the
71 // `structured-output.complete` event has already arrived; only the React
72 // commit that swaps the streaming text part for the completed
73 // StructuredOutputPart is outstanding, so a generous wait is safe).
74 const assistantMessage = page.getByTestId('assistant-message').last()
75 await expect(
76 assistantMessage.getByTestId('structured-output-part'),
77 ).toHaveCount(1, { timeout: 30_000 })
78​
79 const structuredAttr = await completeEl.getAttribute(
80 'data-structured-output',
81 )
82 expect(structuredAttr).toBeTruthy()
83 const parsed = JSON.parse(structuredAttr!) as {
84 name: string
85 price: number
86 rating: number
87 }
88 expect(parsed.name).toContain('Fender Stratocaster')
89 expect(parsed.price).toBe(1299)
90 expect(parsed.rating).toBe(5)
91​
92 const countAttr = await page
93 .getByTestId('content-delta-count')
94 .getAttribute('data-count')
95 expect(Number(countAttr)).toBeGreaterThan(1)
96 })
97 })