Open History to see the last commit
1import { test, expect } from './fixtures'
2import { sendMessage, waitForResponse, featureUrl } from './helpers'
3import { providersFor } from './test-matrix'
4
5/**
6 * Multi-turn structured chat — the pattern documented at
7 * `docs/structured-outputs/multi-turn.md` and demonstrated by
8 * `examples/ts-react-chat/src/routes/generations.structured-chat.tsx`.
9 *
10 * Per-turn assertions:
11 * - Each user prompt produces a new assistant message with its own
12 * `structured-output` part attached (rendered as `structured-output-part`
13 * in the harness `ChatUI`).
14 * - Older turns' parts stay renderable — turn N+1 must not clobber turn N.
15 * - The hook-level `structured-output-complete` snapshot reflects only the
16 * latest turn's `data` (proving `final` derives from the most recent
17 * assistant message's part).
18 *
19 * Runs across every provider that supports both `multi-turn` and
20 * `structured-output` — non-native-streaming providers (anthropic, gemini,
21 * ollama) hit the `fallbackStructuredOutputStream` path, which still emits
22 * one `structured-output.complete` per turn and lands a typed part on the
23 * assistant message. The consumer code is identical across providers; the
24 * test pins that down.
25 */
26for (const provider of providersFor('multi-turn-structured')) {
27 test.describe(`${provider} — multi-turn-structured`, () => {
28 test('every assistant turn keeps its own typed structured-output part', async ({
29 page,
30 testId,
31 aimockPort,
32 }) => {
33 await page.goto(
34 featureUrl(provider, 'multi-turn-structured', testId, aimockPort),
35 )
36
37 // ---- Turn 1: pasta recipe lands as a structured-output part ----
38 await sendMessage(
39 page,
40 '[multiturn-structured-1] pasta dinner for two under $15',
41 )
42 await waitForResponse(page)
43
44 const partsAfterTurn1 = page.getByTestId('structured-output-part')
45 await expect(partsAfterTurn1).toHaveCount(1)
46 await expect(partsAfterTurn1.first()).toContainText('Pomodoro')
47 await expect(partsAfterTurn1.first()).not.toContainText('Vegan')
48
49 // Hook-level `final` reflects turn 1.
50 const completeEl = page.getByTestId('structured-output-complete')
51 await expect(completeEl).toBeAttached()
52 const turn1Data = JSON.parse(
53 (await completeEl.getAttribute('data-structured-output')) ?? '',
54 )
55 expect(turn1Data.title).toContain('Pomodoro')
56 expect(turn1Data.title).not.toContain('Vegan')
57
58 // ---- Turn 2: vegan variant — history must be preserved ----
59 await sendMessage(page, '[multiturn-structured-2] now make it vegan')
60 await waitForResponse(page)
61
62 const partsAfterTurn2 = page.getByTestId('structured-output-part')
63 await expect(partsAfterTurn2).toHaveCount(2)
64
65 // Turn 1's part still holds the original recipe — must NOT have been
66 // rewritten by the second run. This is the load-bearing assertion for
67 // the "per-message structured-output part" design: a single hook-level
68 // partial/final slot would have clobbered turn 1.
69 await expect(partsAfterTurn2.nth(0)).toContainText('Pomodoro')
70 await expect(partsAfterTurn2.nth(0)).not.toContainText('Vegan')
71
72 // Turn 2's part has the new vegan variant.
73 await expect(partsAfterTurn2.nth(1)).toContainText('Vegan')
74
75 // `final` snapshot now reflects turn 2 only — proves the hook derives
76 // from the latest assistant message's part, not from a sticky slot.
77 const turn2Data = JSON.parse(
78 (await completeEl.getAttribute('data-structured-output')) ?? '',
79 )
80 expect(turn2Data.title).toContain('Vegan')
81
82 // ---- Turn 3: gluten-free + salad — three independent recipes ----
83 await sendMessage(
84 page,
85 '[multiturn-structured-3] add a salad and make it gluten-free',
86 )
87 await waitForResponse(page)
88
89 const partsAfterTurn3 = page.getByTestId('structured-output-part')
90 await expect(partsAfterTurn3).toHaveCount(3)
91 await expect(partsAfterTurn3.nth(0)).toContainText('Pomodoro')
92 await expect(partsAfterTurn3.nth(0)).not.toContainText('Vegan')
93 await expect(partsAfterTurn3.nth(1)).toContainText('Vegan')
94 await expect(partsAfterTurn3.nth(1)).not.toContainText('Gluten-Free')
95 await expect(partsAfterTurn3.nth(2)).toContainText('Gluten-Free')
96 await expect(partsAfterTurn3.nth(2)).toContainText('Side Salad')
97
98 const turn3Data = JSON.parse(
99 (await completeEl.getAttribute('data-structured-output')) ?? '',
100 )
101 expect(turn3Data.title).toContain('Gluten-Free')
102 // The schema is honored on every turn — `ingredients` is an array of
103 // {item, amount}, even on the multi-ingredient salad turn.
104 expect(Array.isArray(turn3Data.ingredients)).toBe(true)
105 expect(turn3Data.ingredients.length).toBeGreaterThan(5)
106 })
107 })
108}