Transcription stream OpenAI #2179
Openrodriciru(GitHub profile, opens in a new tab)wants to merge 24 commits into
main fromrefs/pull/2179/headopened View on GitHubtanstack-source added labelsfeature
- docs/media/transcription-advanced.md lines 162 to 163Outdated@@ -159,3 +159,5 @@159159160160 ### Server function that streams161161162+With OpenAI, `gpt-transcribe`, `gpt-4o-transcribe`, and163+`gpt-4o-mini-transcribe` emit standard AG-UI text deltas when `stream: true`.docs/media/transcription-advanced.md line 181Outdated@@ -172,4 +178,4 @@172178 .handler(({ data }) => {173179 return toServerSentEventsResponse(174180 generateTranscription({175- adapter: openaiTranscription('gpt-4o-transcribe'),181+ adapter: openaiTranscription('gpt-transcribe'),docs/media/transcription.md line 104Outdated@@ -92,4 +101,4 @@92101 <input type="file" accept="audio/*" onChange={handleFile} />93102 {isLoading && <p>Transcribing...</p>}94103 {error && <p role="alert">Error: {error.message}</p>}95- {result && <p>{result.text}</p>}104+ {(transcript || result?.text) && <p>{transcript || result?.text}</p>}packages/ai/src/activities/generateTranscription/index.ts lines 301 to 304Outdated@@ -288,9 +293,12 @@288- const rawResult = await raceWithAbort(289- adapter.transcribe({290- ...rest,291- model,292- logger,293- ...(abortControls.signal ? { abortSignal: abortControls.signal } : {}),294- }),295- abortControls.signal,296- )293+ const transcriptionOptions = {294+ ...rest,295+ model,296+ logger,297+ ...(abortControls.signal ? { abortSignal: abortControls.signal } : {}),298+ }299+ const transcription =300+ emitTextDelta && adapter.transcribeStream301+ ? readTranscriptionStream(302+ adapter.transcribeStream(transcriptionOptions),303+ emitTextDelta,304+ )packages/ai/src/activities/stream-generation-result.ts line 107Outdated@@ -55,4 +93,15 @@5593 }56945795 try {58- const result = await generator({ runId, threadId })96+ const textQueue = streamOptions.text ? createAsyncQueue() : undefined97+ const generation = generator(98+ { runId, threadId },99+ textQueue ? (delta) => textQueue.enqueue(delta) : undefined,100+ ).then(101+ (result) => {102+ textQueue?.close()103+ return result104+ },105+ (error: unknown) => {106+ textQueue?.close()107+ throw errortesting/e2e/tests/transcription.spec.ts lines 17 to 19Outdated@@ -13,3 +13,7 @@1313 featureUrl(provider, 'transcription', testId, aimockPort, 'sse'),1414 )1515 await clickGenerate(page)16+ if (provider === 'openai') {17+ await expect(page.getByTestId('transcription-partial')).toContainText(18+ 'Fender Stratocaster',19+ )
github-actions[bot](GitHub profile, opens in a new tab) assignedtombeckenham
github-actions[bot](GitHub profile, opens in a new tab) added labelswaiting-on: maintainer
- docs/media/transcription-advanced.md line 237Outdated@@ -202,2 +218,20 @@218+ }219+ },202220 })203- // Render the same way as in the quick setup.221+222+ function handleFile(event: ChangeEvent<HTMLInputElement>) {223+ const file = event.target.files?.[0]224+ if (!file) return225+226+ const reader = new FileReader()227+ reader.onload = () => {228+ if (typeof reader.result === 'string') {229+ void generate({ audio: reader.result, language: 'en' })230+ }231+ }232+ reader.readAsDataURL(file)233+ }234+235+ return (236+ <div>237+ <input type="file" accept="audio/*" onChange={handleFile} />docs/media/transcription-advanced.md lines 240 to 241Outdated@@ -203,0 +221,21 @@221+222+ function handleFile(event: ChangeEvent<HTMLInputElement>) {223+ const file = event.target.files?.[0]224+ if (!file) return225+226+ const reader = new FileReader()227+ reader.onload = () => {228+ if (typeof reader.result === 'string') {229+ void generate({ audio: reader.result, language: 'en' })230+ }231+ }232+ reader.readAsDataURL(file)233+ }234+235+ return (236+ <div>237+ <input type="file" accept="audio/*" onChange={handleFile} />238+ {isLoading && <p>Transcribing...</p>}239+ {error && <p role="alert">{error.message}</p>}240+ {(result?.text || transcript) && (241+ <p>{result?.text || transcript}</p>docs/media/transcription.md line 104Outdated@@ -92,4 +101,4 @@92101 <input type="file" accept="audio/*" onChange={handleFile} />93102 {isLoading && <p>Transcribing...</p>}94103 {error && <p role="alert">Error: {error.message}</p>}95- {result && <p>{result.text}</p>}104+ {(result?.text || transcript) && <p>{result?.text || transcript}</p>}packages/ai/src/activities/stream-generation-result.ts lines 112 to 113Outdated@@ -55,4 +94,20 @@5594 }56955796 try {58- const result = await generator({ runId, threadId })97+ const textQueue = streamOptions.text ? createAsyncQueue() : undefined98+ const generation = generator(99+ { runId, threadId },100+ textQueue ? (delta) => textQueue.enqueue(delta) : undefined,101+ ).then(102+ (result) => {103+ textQueue?.close()104+ return { ok: true as const, result }105+ },106+ (error: unknown) => {107+ textQueue?.close()108+ return { ok: false as const, error }109+ },110+ )111+112+ const messageId = textQueue ? createId('message') : undefined113+ if (textQueue && messageId) {testing/e2e/global-setup.ts line 402Outdated@@ -353,0 +382,21 @@382+ const body = await readBody(req)383+ const boundary = req.headers['content-type']?.match(384+ /boundary=(?:"([^"]+)"|([^;]+))/i,385+ )386+ const boundaryValue = boundary?.[1] ?? boundary?.[2]?.trim()387+ const streamPart = boundaryValue388+ ? body389+ .split(`--${boundaryValue}`)390+ .find((part) =>391+ /Content-Disposition: form-data;\s*name="stream"/i.test(part),392+ )393+ : undefined394+ const streamValue = streamPart?.split('\r\n\r\n', 2)[1]?.trim()395+ const text = 'I would like to buy a Fender Stratocaster please'396+ const testId = req.headers['x-test-id']397+ let completion: Promise<void> | undefined398+ if (streamValue === 'true' && typeof testId === 'string') {399+ completion = new Promise<void>((resolve) => {400+ const release = () => {401+ clearTimeout(timeout)402+ pendingCompletions.delete(testId)testing/e2e/tests/transcription.spec.ts lines 19 to 21Outdated@@ -6,10 +6,16 @@66 test.describe(`${provider} -- transcription`, () => {77 test('sse -- transcribes audio via SSE connection', async ({88 page,9+ request,910 testId,1011 aimockPort,1112 }) => {1213 await page.goto(1314 featureUrl(provider, 'transcription', testId, aimockPort, 'sse'),1415 )1516 await clickGenerate(page)17+ if (provider === 'openai') {18+ try {19+ await expect(page.getByTestId('transcription-partial')).toContainText(20+ 'Fender Stratocaster',21+ )testing/e2e/tests/transcription.spec.ts lines 26 to 29Outdated@@ -9,7 +9,21 @@9+ request,910 testId,1011 aimockPort,1112 }) => {1213 await page.goto(1314 featureUrl(provider, 'transcription', testId, aimockPort, 'sse'),1415 )1516 await clickGenerate(page)17+ if (provider === 'openai') {18+ try {19+ await expect(page.getByTestId('transcription-partial')).toContainText(20+ 'Fender Stratocaster',21+ )22+ await expect(page.getByTestId('generation-status')).toHaveText(23+ 'loading',24+ )25+ } finally {26+ await request.post(27+ `http://127.0.0.1:${aimockPort}/transcription-stream/release`,28+ { headers: { 'x-test-id': testId } },29+ )
- docs/media/transcription-advanced.md line 223Outdated@@ -194,10 +205,19 @@194205 if (typeof input.audio !== 'string') {195206 throw new Error('Expected base64 or data URL audio')196207 }197208 return transcribeStreamFn({198- data: { ...input, audio: input.audio, threadId },209+ data: { ...input, threadId },199210 })200211 },201212 threadId,213+ onChunk(chunk) {214+ if (chunk.type === 'TEXT_MESSAGE_START') {215+ setTranscript('')216+ } else if (chunk.type === 'TEXT_MESSAGE_CONTENT') {217+ setTranscript((current) => current + chunk.delta)218+ }219+ },202220 })203- // Render the same way as in the quick setup.221+ const displayedTranscript = isLoading222+ ? transcript223+ : (result?.text ?? transcript)docs/media/transcription.md line 87Outdated@@ -67,9 +67,21 @@6767 ```tsx6868 import type { ChangeEvent } from 'react'69+import { useState } from 'react'6970 import { fetchServerSentEvents, useTranscription } from '@tanstack/ai-react'70717172 export function Transcriber() {73+ const [transcript, setTranscript] = useState('')7274 const { generate, result, isLoading, error } = useTranscription({7375 connection: fetchServerSentEvents('/api/transcribe'),7476 threadId: 'transcription',77+ onChunk(chunk) {78+ if (chunk.type === 'TEXT_MESSAGE_START') {79+ setTranscript('')80+ } else if (chunk.type === 'TEXT_MESSAGE_CONTENT') {81+ setTranscript((current) => current + chunk.delta)82+ }83+ },7584 })85+ const displayedTranscript = isLoading86+ ? transcript87+ : (result?.text ?? transcript)packages/ai/src/activities/generateTranscription/index.ts lines 211 to 214Outdated@@ -202,5 +204,11 @@202204 return streamGenerationResult(203205 // Only `runId` comes from the resolved wire identity. `threadId` is204206 // the caller's own id.205- (resolved) =>206- runGenerateTranscription({ ...options, runId: resolved.runId }),207+ (resolved, emitTextDelta, streamAbortSignal) =>208+ runGenerateTranscription(209+ {210+ ...options,211+ abortSignal: combineAbortSignals(212+ options.abortSignal,213+ streamAbortSignal,214+ ),packages/ai/src/activities/generateTranscription/index.ts line 398Outdated@@ -362,3 +382,17 @@362382 }363383 }364384385+async function readTranscriptionStream(386+ stream: AsyncGenerator<string, TranscriptionResult>,387+ emitTextDelta: (delta: string) => void,388+ signal?: AbortSignal,389+): Promise<TranscriptionResult> {390+ const iterator = stream[Symbol.asyncIterator]()391+ const closeIterator = () => {392+ void iterator.return?.(undefined as never).catch(() => undefined)393+ }394+395+ while (true) {396+ let next: IteratorResult<string, TranscriptionResult>397+ try {398+ next = await raceWithAbort(iterator.next(), signal)packages/ai/tests/stream-generation.test.ts lines 859 to 860Outdated@@ -614,0 +840,21 @@840+ })841+ const iterator = generateTranscription({842+ threadId: 'thread-1',843+ adapter,844+ audio: 'base64-audio-data',845+ stream: true,846+ })[Symbol.asyncIterator]()847+848+ await iterator.next()849+ await iterator.next()850+ expect((await iterator.next()).value?.type).toBe('TEXT_MESSAGE_CONTENT')851+ const pendingChunk = iterator.next()852+ await pendingReadStarted853+854+ const cancellation = iterator.return?.()855+ await transcriptionAborted856+ expect(await cancellation).toMatchObject({ done: true })857+ expect((await pendingChunk).value).toMatchObject({ type: 'RUN_ERROR' })858+859+ releasePendingRead()860+ await adapterClosedtesting/e2e/tests/transcription.spec.ts line 154Outdated@@ -53,0 +134,21 @@134+ const decoder = new TextDecoder()135+ let buffer = ''136+ while (true) {137+ const { done, value } = await reader.read()138+ if (done) return false139+ buffer += decoder.decode(value, { stream: true })140+ if (buffer.includes('"type":"TEXT_MESSAGE_CONTENT"')) {141+ const cancelled = await Promise.race([142+ reader.cancel().then(() => true),143+ new Promise<boolean>((resolve) =>144+ setTimeout(() => resolve(false), 5_000),145+ ),146+ ])147+ if (!cancelled) controller.abort()148+ return cancelled149+ }150+ }151+ },152+ { testId, aimockPort },153+ )154+ expect(cancellationCompleted).toBe(true)
github-actions[bot](GitHub profile, opens in a new tab) added labelswaiting-on: author
github-actions[bot](GitHub profile, opens in a new tab) removed labelswaiting-on: maintainer
- docs/media/transcription-advanced.md line 224Outdated@@ -193,11 +206,19 @@206+ const { generate, result, isLoading, error } = useTranscription({193207 fetcher: (input, { threadId }) => {194208 if (typeof input.audio !== 'string') {195209 throw new Error('Expected base64 or data URL audio')196210 }197211 return transcribeStreamFn({198- data: { ...input, audio: input.audio, threadId },212+ data: { ...input, threadId },199213 })200214 },201215 threadId,216+ onChunk(chunk) {217+ if (chunk.type === 'TEXT_MESSAGE_START') {218+ setTranscript('')219+ } else if (chunk.type === 'TEXT_MESSAGE_CONTENT') {220+ setTranscript((current) => current + chunk.delta)221+ }222+ },202223 })203- // Render the same way as in the quick setup.224+ const displayedTranscript = isLoading || errordocs/media/transcription.md line 89Outdated@@ -65,11 +69,21 @@6569 Add a component that reads a file and sends it:66706771 ```tsx6872 import type { ChangeEvent } from 'react'73+import { useState } from 'react'6974 import { fetchServerSentEvents, useTranscription } from '@tanstack/ai-react'70757176 export function Transcriber() {77+ const [transcript, setTranscript] = useState('')7278 const { generate, result, isLoading, error } = useTranscription({7379 connection: fetchServerSentEvents('/api/transcribe'),7480 threadId: 'transcription',81+ onChunk(chunk) {82+ if (chunk.type === 'TEXT_MESSAGE_START') {83+ setTranscript('')84+ } else if (chunk.type === 'TEXT_MESSAGE_CONTENT') {85+ setTranscript((current) => current + chunk.delta)86+ }87+ },7588 })89+ const displayedTranscript = isLoading || errorpackages/ai/src/activities/generateTranscription/index.ts line 285Outdated@@ -266,4 +282,4 @@266282 createId,267283 })268284269- await runGenerationStart(middleware, mwCtx)285+ await runGenerationStart(middleware, mwCtx).catch((error: unknown) => {packages/ai/src/activities/generateTranscription/index.ts line 317Outdated@@ -287,10 +306,12 @@287306 try {288- const rawResult = await raceWithAbort(289- adapter.transcribe({290- ...rest,291- model,292- logger,293- ...(abortControls.signal ? { abortSignal: abortControls.signal } : {}),294- }),295- abortControls.signal,296- )307+ if (abortControls.signal?.aborted) {308+ throw toAbortError(abortControls.signal.reason)309+ }310+311+ const transcriptionOptions = {312+ ...rest,313+ model,314+ logger,315+ ...(abortControls.signal ? { abortSignal: abortControls.signal } : {}),316+ }317+ const hasResultTransforms = mwCtx.resultTransforms.length > 0packages/ai/src/utilities/activity-abort.ts line 127Outdated@@ -97,12 +115,13 @@97115 }, ms)98116 }99117100- const signal = combineAbortSignals(options.abortSignal, timeoutSignal)118+ const combined = combineAbortSignals(options.abortSignal, timeoutSignal)101119102120 return {103- signal,121+ signal: combined.signal,104122 clear: () => {105123 if (timeoutId !== undefined) {106124 clearTimeout(timeoutId)107125 timeoutId = undefined108126 }127+ combined.dispose()testing/e2e/tests/transcription.spec.ts line 167Outdated@@ -53,0 +147,21 @@147+ if (!cancelled) controller.abort()148+ return cancelled149+ }150+ }151+ },152+ { testId, aimockPort },153+ )154+ expect(cancellationCompleted).toBe(true)155+ await expect156+ .poll(157+ async () => {158+ const response = await request.get(159+ `http://127.0.0.1:${aimockPort}/transcription-stream/status?testId=${encodeURIComponent(testId)}`,160+ )161+ if (!response.ok())162+ return { closedByClient: false, pending: false }163+ return response.json()164+ },165+ { timeout: 3_000 },166+ )167+ .toMatchObject({ closedByClient: true, pending: true })
github-actions[bot](GitHub profile, opens in a new tab) added labelswaiting-on: maintainer
github-actions[bot](GitHub profile, opens in a new tab) removed labelswaiting-on: author
Sign in to join this conversation.
🦋 Changeset detected#
Latest commit: a45a7581b35348c5bfe091dd0ae7d65368c100c1
The changes in this PR will be included in the next version bump.
This PR includes changesets to release 2 packages
Not sure what this means? Click here to learn what changesets are.
Click here if you're a maintainer who wants to add another changeset to this PR