feat: streaming structured output across openai/openrouter/grok/groq + summarize fix #527
Mergedtombeckenhammerged 33 commits into
main fromrefs/pull/527/headmerged by AlemTuzlak View on GitHub- @@ -0,0 +1,5 @@1+---2+'@tanstack/ai': minor3+---4+5+feat: `chat({ outputSchema, stream: true })` returns `AsyncIterable<StreamChunk>` with raw JSON deltas plus a final `CUSTOM` `structured-output.complete` event carrying the validated parsed object. The existing `chat({ outputSchema })` (non-streaming) path is unchanged. Adapters expose this via a new optional `structuredOutputStream` method on `TextAdapter`; `BaseTextAdapter` provides a default that wraps the non-streaming `structuredOutput` so adapters without native streaming JSON support still satisfy the new combination.@@ -265,0 +420,21 @@420+ })421+ }422+423+ if (!accumulatedContent) {424+ // Mirrors the non-streaming `structuredOutput` empty-content error so425+ // refused/truncated responses surface as failures, not `null` data.426+ const message = 'Structured output response contained no content'427+ logger.errors(message, {428+ source: 'openrouter.structuredOutputStream',429+ })430+ yield asChunk({431+ type: 'RUN_ERROR',432+ runId: aguiState.runId,433+ model: resolvedModel,434+ timestamp,435+ message,436+ code: 'empty-response',437+ error: { message, code: 'empty-response' },438+ })439+ return440+ }@@ -1680,0 +1866,21 @@1866+ }1867+ } catch (engineError) {1868+ const message = (engineError as Error).message || 'Agent loop failed'1869+ logger.errors('runStreamingStructuredOutput agent loop failed', {1870+ error: engineError,1871+ source: 'runStreamingStructuredOutput',1872+ })1873+ yield {1874+ type: 'RUN_ERROR',1875+ runId,1876+ model,1877+ timestamp: Date.now(),1878+ message,1879+ code: 'agent-loop-failed',1880+ error: { message, code: 'agent-loop-failed' },1881+ } as unknown as StreamChunk1882+ return1883+ }1884+1885+ finalMessages = engine.getMessages()1886+ }@@ -122,4 +123,4 @@122123 }123124 ```124125125-Existing prefixes: `[chat]`, `[oneshot]`, `[reasoning]`, `[multiturn-1]`, `[multiturn-2]`, `[toolcall]`, `[parallel]`, `[approval]`, `[approval-deny]`, `[text-tool-text]`, `[structured]`, `[agentic]`, `[mmimage]`, `[mmstruct]`, `[summarize]`, `[imagegen]`, `[tts]`, `[transcription]`, `[abort-test]`, `[error-test]`.126+Existing prefixes: `[chat]`, `[oneshot]`, `[reasoning]`, `[multiturn-1]`, `[multiturn-2]`, `[toolcall]`, `[parallel]`, `[approval]`, `[approval-deny]`, `[text-tool-text]`, `[structured]`, `[structured-stream]`, `[agentic]`, `[mmimage]`, `[mmstruct]`, `[summarize]`, `[imagegen]`, `[tts]`, `[transcription]`, `[abort-test]`, `[error-test]`.
- @@ -21,3 +25,19 @@2125 nextSteps: z.array(z.string()).describe('Practical follow-up actions'),2226 })232728+type Provider = 'openai' | 'grok' | 'groq' | 'openrouter'29+30+function adapterFor(provider: Provider, model?: string): AnyTextAdapter {31+ switch (provider) {32+ case 'openai':33+ return openaiText((model || 'gpt-4o') as 'gpt-4o')34+ case 'grok':35+ return grokText((model || 'grok-3') as 'grok-3')36+ case 'groq':37+ return groqText(38+ (model || 'llama-3.3-70b-versatile') as 'llama-3.3-70b-versatile',39+ )40+ case 'openrouter':41+ return openRouterText((model || 'openai/gpt-5.2') as 'openai/gpt-5.2')42+ }43+}@@ -35,7 +63,18 @@63+ )64+ const streamIterable = chat({65+ adapter: adapterFor(resolvedProvider, model),66+ messages: [{ role: 'user', content: prompt }],67+ outputSchema: GuitarRecommendationSchema,68+ stream: true,69+ abortController,70+ }) as AsyncIterable<StreamChunk>71+ return toServerSentEventsResponse(streamIterable, {72+ abortController,73+ })74+ }75+3576 const result = await chat({36- adapter: openRouterText(37- (model || 'openai/gpt-5.2') as 'openai/gpt-5.2',38- ),77+ adapter: adapterFor(resolvedProvider, model),3978 messages: [{ role: 'user', content: prompt }],4079 outputSchema: GuitarRecommendationSchema,4180 })@@ -225,0 +430,21 @@430+ object: transformed,431+ raw: accumulatedContent,432+ },433+ model: resolvedModel,434+ timestamp,435+ })436+437+ yield asChunk({438+ type: 'RUN_FINISHED',439+ runId: aguiState.runId,440+ threadId: aguiState.threadId,441+ model: resolvedModel,442+ timestamp,443+ usage: aguiState.deferredUsage,444+ finishReason: aguiState.computedFinishReason ?? 'stop',445+ })446+ } catch (error) {447+ logger.errors('grok.structuredOutputStream fatal', {448+ error,449+ source: 'grok.structuredOutputStream',450+ })@@ -253,0 +404,21 @@404+ error: { message, code: 'refusal' },405+ })406+ return407+ }408+409+ if (chunk.type === 'response.completed') {410+ usage = {411+ promptTokens: chunk.response.usage?.input_tokens ?? 0,412+ completionTokens: chunk.response.usage?.output_tokens ?? 0,413+ totalTokens: chunk.response.usage?.total_tokens ?? 0,414+ }415+ finishReason = 'stop'416+ }417+418+ if (chunk.type === 'response.incomplete') {419+ finishReason =420+ chunk.response.incomplete_details?.reason === 'max_output_tokens'421+ ? 'length'422+ : 'stop'423+ }424+ }@@ -80,3 +80,6 @@8080 'grok',8181 'openrouter',8282 ]),83+ // Streaming structured output: only providers with native streaming JSON84+ // schema support are listed here. Other providers fall back to the default85+ // BaseTextAdapter implementation but aren't exercised by E2E yet.
- @@ -51,0 +154,21 @@154+ if (!stream) {155+ const payload = await response.json()156+ setResult(payload.data as PartialResult)157+ setHasFinalResult(true)158+ return159+ }160+161+ // Streaming path — parse SSE, accumulate raw JSON, render the partially162+ // parsed object live, snap to the validated terminal payload.163+ const reader = response.body!.getReader()164+ const decoder = new TextDecoder()165+ let buffer = ''166+ let accumulated = ''167+ let reasoning = ''168+ let deltas = 0169+170+ while (true) {171+ const { done, value } = await reader.read()172+ if (done) break173+ buffer += decoder.decode(value, { stream: true })174+@@ -52 +215,21 @@215+ } else if (216+ chunk.type === 'CUSTOM' &&217+ chunk.name === 'structured-output.complete' &&218+ chunk.value?.object219+ ) {220+ setResult(chunk.value.object as PartialResult)221+ setHasFinalResult(true)222+ if (223+ typeof (chunk.value as { reasoning?: string }).reasoning ===224+ 'string'225+ ) {226+ setReasoningFull(227+ (chunk.value as { reasoning: string }).reasoning,228+ )229+ }230+ } else if (chunk.type === 'RUN_ERROR') {231+ throw new Error(chunk.message || 'Stream failed')232+ }233+ }234+ }52235 }@@ -51,0 +209,21 @@209+ ) {210+ reasoning += chunk.delta211+ setReasoningFull(reasoning)212+ // One-liner: take the last non-empty line/sentence so consumers213+ // see "what it's thinking right now" without a wall of text.214+ setReasoningLine(latestThought(reasoning))215+ } else if (216+ chunk.type === 'CUSTOM' &&217+ chunk.name === 'structured-output.complete' &&218+ chunk.value?.object219+ ) {220+ setResult(chunk.value.object as PartialResult)221+ setHasFinalResult(true)222+ if (223+ typeof (chunk.value as { reasoning?: string }).reasoning ===224+ 'string'225+ ) {226+ setReasoningFull(227+ (chunk.value as { reasoning: string }).reasoning,228+ )229+ }@@ -90,0 +280,21 @@280+ <div className="space-y-2">281+ <label className="text-sm text-gray-400">Provider</label>282+ <select283+ value={provider}284+ onChange={(e) => onProviderChange(e.target.value as Provider)}285+ disabled={isLoading}286+ className="w-full rounded-lg border border-orange-500/20 bg-gray-800/50 px-3 py-2 text-sm text-white focus:outline-none focus:ring-2 focus:ring-orange-500/50 disabled:opacity-50"287+ >288+ <option value="openai">OpenAI</option>289+ <option value="grok">Grok (xAI)</option>290+ <option value="groq">Groq</option>291+ <option value="openrouter">OpenRouter</option>292+ </select>293+ </div>294+ <div className="space-y-2">295+ <label className="text-sm text-gray-400">Model</label>296+ <select297+ value={model}298+ onChange={(e) => setModel(e.target.value)}299+ disabled={isLoading}300+ className="w-full rounded-lg border border-orange-500/20 bg-gray-800/50 px-3 py-2 text-sm text-white focus:outline-none focus:ring-2 focus:ring-orange-500/50 disabled:opacity-50"@@ -225,0 +290,21 @@290+ },291+ { signal: chatOptions.request?.signal },292+ )293+294+ for await (const chunk of stream) {295+ logger.provider(`provider=grok`, { chunk })296+ if (chunk.model) currentModel = chunk.model297+298+ if (!aguiState.hasEmittedRunStarted) {299+ aguiState.hasEmittedRunStarted = true300+ yield asChunk({301+ type: 'RUN_STARTED',302+ runId: aguiState.runId,303+ threadId: aguiState.threadId,304+ model: currentModel || chatOptions.model,305+ timestamp,306+ })307+ }308+309+ const choice = chunk.choices[0]310+ if (!choice) continue
tombeckenham changed the title
AlemTuzlak reviewed
OutdatedResolvedby tombeckenham@@ -260,0 +346,21 @@346+ yield {347+ type: EventType.REASONING_START,348+ messageId: reasoningMessageId,349+ model,350+ timestamp,351+ } satisfies StreamChunk352+ yield {353+ type: EventType.REASONING_MESSAGE_START,354+ messageId: reasoningMessageId,355+ role: 'reasoning' as const,356+ model,357+ timestamp,358+ } satisfies StreamChunk359+ yield {360+ type: EventType.STEP_STARTED,361+ stepName: stepId,362+ stepId,363+ model,364+ timestamp,365+ stepType: 'thinking',366+ } satisfies StreamChunkAlemTuzlak reviewed
OutdatedResolvedby tombeckenham@@ -260,0 +355,21 @@355+ role: 'reasoning' as const,356+ model,357+ timestamp,358+ } satisfies StreamChunk359+ yield {360+ type: EventType.STEP_STARTED,361+ stepName: stepId,362+ stepId,363+ model,364+ timestamp,365+ stepType: 'thinking',366+ } satisfies StreamChunk367+ }.bind(this)368+369+ try {370+ chatOptions.logger.request(371+ `activity=structuredOutputStream provider=${this.name} model=${this.model} messages=${chatOptions.messages.length}`,372+ { provider: this.name, model: this.model },373+ )374+ const reqOptions = extractRequestOptions(chatOptions.request)375+ const rawStream = (await this.orClient.beta.responses.send(AlemTuzlak reviewed
OutdatedResolvedby tombeckenham@@ -260,0 +401,21 @@401+ yield {402+ type: EventType.RUN_STARTED,403+ runId: aguiState.runId,404+ threadId: aguiState.threadId,405+ model,406+ timestamp,407+ } satisfies StreamChunk408+ }409+410+ if (411+ chunk.type === 'response.created' ||412+ chunk.type === 'response.in_progress'413+ ) {414+ const r = chunk.response as { model?: string } | undefined415+ if (r?.model) model = r.model416+ continue417+ }418+419+ if (chunk.type === 'response.refusal.delta') {420+ const delta =421+ typeof (chunk as { delta?: unknown }).delta === 'string'AlemTuzlak reviewed
OutdatedResolvedby tombeckenham@@ -260,0 +438,21 @@438+ chunk.type === 'response.reasoning_summary_text.delta'439+ ) {440+ const raw = (chunk as { delta?: unknown }).delta441+ const reasoningDelta = Array.isArray(raw)442+ ? raw.join('')443+ : typeof raw === 'string'444+ ? raw445+ : ''446+ if (!reasoningDelta) continue447+ yield* openReasoning()448+ // openReasoning() guarantees reasoningMessageId is set on first449+ // call; TS can't see through the generator side-effect.450+ const messageId = reasoningMessageId!451+ accumulatedReasoning += reasoningDelta452+ yield {453+ type: EventType.REASONING_MESSAGE_CONTENT,454+ messageId,455+ delta: reasoningDelta,456+ model,457+ timestamp,458+ } satisfies StreamChunkAlemTuzlak reviewed
OutdatedResolvedby tombeckenham@@ -273,0 +364,21 @@364+ const stream = (await this.orClient.chat.send(365+ {366+ chatRequest: {367+ ...cleanParams,368+ stream: true,369+ streamOptions: { includeUsage: true },370+ responseFormat: {371+ type: 'json_schema',372+ jsonSchema: {373+ name: 'structured_output',374+ schema: jsonSchema,375+ strict: true,376+ },377+ },378+ },379+ },380+ {381+ signal: reqOptions.signal ?? undefined,382+ ...(reqOptions.headers && { headers: reqOptions.headers }),383+ },384+ )) as AsyncIterable<ChatStreamChunk>AlemTuzlak reviewed
Resolvedby tombeckenham@@ -1575,10 +1590,13 @@15751590 export function chat<15761591 TAdapter extends AnyTextAdapter,15771592 TSchema extends SchemaInput | undefined = undefined,1578- TStream extends boolean = true,1593+ TStream extends boolean = boolean,15791594 >(15801595 options: TextActivityOptions<TAdapter, TSchema, TStream>,15811596 ): TextActivityResult<TSchema, TStream> {15821597 const { outputSchema, stream } = options158315981584- // If outputSchema is provided, run agentic structured output1599+ // outputSchema + stream:true is the only branch that streams structured1600+ // output. Without an explicit `stream: true`, schema-bearing calls run the1601+ // agent loop and resolve to a typed Promise<InferSchemaType<TSchema>>.1602+ if (outputSchema && stream === true) {AlemTuzlak reviewed
OutdatedResolvedby tombeckenham@@ -1743,0 +1830,21 @@1830+ messageId,1831+ model,1832+ timestamp,1833+ } satisfies StreamChunk1834+1835+ yield {1836+ type: EventType.CUSTOM,1837+ name: 'structured-output.complete',1838+ value: { object: result.data, raw: result.rawText },1839+ model,1840+ timestamp,1841+ } satisfies StreamChunk1842+1843+ yield {1844+ type: EventType.RUN_FINISHED,1845+ runId,1846+ threadId,1847+ model,1848+ timestamp,1849+ finishReason: 'stop',1850+ } satisfies StreamChunkAlemTuzlak reviewed
OutdatedResolvedby tombeckenham@@ -1743,0 +1895,21 @@1895+ options1896+ const model = adapter.model1897+ const logger = resolveDebugOption(debug)1898+ const runId = textOptions.runId1899+1900+ // Inputs may be UIMessages (from useChat) or ModelMessages (from server-side1901+ // callers). The agent-loop branch converts via TextEngine; the no-tools1902+ // branch must convert here so the adapter sees a uniform ModelMessage shape.1903+ let finalMessages = convertMessagesToModelMessages(1904+ (textOptions.messages ?? []) as Array<any>,1905+ )1906+1907+ if (textOptions.tools?.length) {1908+ const engine = new TextEngine(1909+ {1910+ adapter,1911+ params: { ...textOptions, model, logger } as TextOptions<1912+ Record<string, unknown>,1913+ Record<string, unknown>1914+ >,1915+ middleware,AlemTuzlak requested changes
AlemTuzlak approved these changes
AlemTuzlak merged this98979f7
AlemTuzlak closed this
Sign in to join this conversation.
Note
Reviews paused#
It looks like this branch is under active development. To avoid overwhelming you with review comments due to an influx of new commits, CodeRabbit has automatically paused this review. You can configure this behavior by changing the
reviews.auto_review.auto_pause_after_reviewed_commitssetting.Use the following commands to manage reviews:
@coderabbitai resumeto resume automatic reviews.@coderabbitai reviewto trigger a single review.Use the checkboxes below for quick actions:
📝 Walkthrough
Walkthrough#
chat({ outputSchema, stream: true }) now produces an AsyncIterable that emits JSON delta chunks plus a terminal CUSTOM event
structured-output.completewith{ object, raw, reasoning? }. Adapters may implementstructuredOutputStream; BaseTextAdapter provides a fallback wrapping non-streaming structured output. Type and runtime plumbing added to route/validate streams.Changes#
Streaming Structured Output (single cohesive change DAG)
packages/typescript/ai/src/types.tsStructuredOutputCompleteEvent<T>,StructuredOutputStream<T>, andisStructuredOutputCompleteEvent()to represent/identify the terminal structured-output completion event.packages/typescript/ai/src/activities/chat/adapter.tsstructuredOutputStream(options)toTextAdapter, documenting expected AG-UI lifecycle emissions and the finalCUSTOM('structured-output.complete')payload.packages/typescript/ai/src/activities/chat/index.ts(type layer)TStreamdefaults andTextActivityResultso streaming structured output is selected only whenstream === truevia[TStream] extends [true].packages/typescript/ai/src/activities/chat/index.ts(runtime layer)outputSchema && stream === truebranch, implementsrunStreamingStructuredOutput/runStreamingStructuredOutputImpl, wiresabortControllerinto adapter requests, and providesfallbackStructuredOutputStreamthat synthesizes lifecycle events from non-streaming adapters.packages/typescript/ai-openai/...,packages/typescript/ai-grok/...,packages/typescript/ai-groq/...,packages/typescript/ai-openrouter/...structuredOutputStreamasync generator making a singlestream: trueJSON-schema request (strict), strips tools, emits AG-UI lifecycle events andTEXT_MESSAGE_CONTENTdeltas, accumulates raw JSON, parses/transforms (null→undefined), emitsCUSTOM('structured-output.complete')with{ object, raw, reasoning? }, and emitsRUN_FINISHEDorRUN_ERRORfor parse/empty/abort/fatal cases. OpenRouter also adjusts lifecycle finalization flags.packages/typescript/ai-*/tests/*CUSTOM, parse/empty/abort/failure handling, reasoning propagation, and null→undefined transform behavior..changeset/streaming-structured-output-*.mdtesting/e2e/src/lib/*,testing/e2e/src/lib/types.ts,testing/e2e/fixtures/structured-output-stream/*.json'structured-output-stream'feature, provider matrix entries, feature config, fixturesbasic.jsonandabort.json, and README updates for fixture prefixes.testing/e2e/src/components/ChatUI.tsx,testing/e2e/src/routes/$provider/$feature.tsx,testing/e2e/src/routes/api.chat.tsdata-structured-output,data-count), wiresonCustomEvent/onChunkto capturestructured-output.completeand delta counts, and conditions/api/chatto callchat()withoutputSchema+stream: truefor the feature.testing/e2e/tests/structured-output-stream.spec.tsstructured-output.completepayload delivery, multiple deltas (>1), and mid-stream abort preventing completion.examples/ts-react-chat/src/routes/*.ts*TEXT_MESSAGE_CONTENTdeltas and reasoning, progressive parse/update of partial JSON, delta counting and abort control, and UI updates for streaming state and partial/result rendering.Sequence Diagram#
Estimated code review effort#
🎯 4 (Complex) | ⏱️ ~50 minutes
🚥 Pre-merge checks | ✅ 4 | ❌ 1
❌ Failed checks (1 warning)#
✅ Passed checks (4 passed)
✨ Finishing Touches
🧪 Generate unit tests (beta)
Thanks for using CodeRabbit! It's free for OSS, and your support helps us grow. If you like it, consider giving us a shout-out.
❤️ Share