Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

audio-providers.ts

388 lines11.1 KB

1/**
2 * Shared catalog of audio-related providers shown in the example pages.
3 *
4 * Each entry lists a display label plus the provider model we exercise so
5 * the UI can render consistent tabs/selectors across speech, transcription,
6 * and audio generation flows.
7 */
8​
9import type { TranscriptionGenerateInput } from '@tanstack/ai-client'
10​
11export type SpeechProviderId =
12 | 'openai'
13 | 'gemini'
14 | 'fal'
15 | 'grok'
16 | 'elevenlabs'
17 | 'byteplus'
18​
19export interface SpeechProviderConfig {
20 id: SpeechProviderId
21 label: string
22 model: string
23 /** Voices the UI will surface for this provider. */
24 voices: ReadonlyArray<{ id: string; label: string }>
25 /** Placeholder shown in the text area. */
26 placeholder: string
28​
29export const SPEECH_PROVIDERS: ReadonlyArray<SpeechProviderConfig> = [
30 {
31 id: 'openai',
32 label: 'OpenAI TTS',
33 model: 'tts-1',
34 voices: [
35 { id: 'alloy', label: 'Alloy' },
36 { id: 'echo', label: 'Echo' },
37 { id: 'fable', label: 'Fable' },
38 { id: 'onyx', label: 'Onyx' },
39 { id: 'nova', label: 'Nova' },
40 { id: 'shimmer', label: 'Shimmer' },
41 ],
42 placeholder: 'Enter text to read aloud with OpenAI TTS…',
43 },
44 {
45 id: 'gemini',
46 label: 'Gemini TTS',
47 model: 'gemini-2.5-flash-preview-tts',
48 voices: [
49 { id: 'Kore', label: 'Kore' },
50 { id: 'Puck', label: 'Puck' },
51 { id: 'Zephyr', label: 'Zephyr' },
52 ],
53 placeholder: 'Enter text for Gemini speech…',
54 },
55 {
56 id: 'fal',
57 label: 'Fal (Kokoro)',
58 model: 'fal-ai/kokoro/american-english',
59 voices: [
60 { id: 'af_heart', label: 'Heart' },
61 { id: 'af_sky', label: 'Sky' },
62 { id: 'am_adam', label: 'Adam' },
63 ],
64 placeholder: 'Enter text to synthesize with Fal Kokoro…',
65 },
66 {
67 id: 'grok',
68 label: 'Grok TTS',
69 model: 'grok-tts',
70 voices: [
71 { id: 'eve', label: 'Eve' },
72 { id: 'ara', label: 'Ara' },
73 { id: 'rex', label: 'Rex' },
74 { id: 'sal', label: 'Sal' },
75 { id: 'leo', label: 'Leo' },
76 ],
77 placeholder: 'Enter text for Grok speech…',
78 },
79 {
80 id: 'elevenlabs',
81 label: 'ElevenLabs',
82 model: 'eleven_multilingual_v2',
83 voices: [
84 { id: '21m00Tcm4TlvDq8ikWAM', label: 'Rachel' },
85 { id: 'AZnzlk1XvdvUeBnXmlld', label: 'Domi' },
86 { id: 'EXAVITQu4vr4xnSDxMaL', label: 'Bella' },
87 { id: 'pNInz6obpgDQGcFmaJgB', label: 'Adam' },
88 ],
89 placeholder: 'Enter text to synthesize with ElevenLabs…',
90 },
91 {
92 id: 'byteplus',
93 label: 'BytePlus Seed Speech',
94 model: 'seed-audio-1.0',
95 // Voice ids ("speakers") encode language, gender, character and TTS
96 // generation. The full roster lives at
97 // https://docs.byteplus.com/en/docs/byteplusvoice/voicelist; this is the
98 // adapter's default. Output formats: wav | mp3 | pcm | ogg_opus.
99 voices: [
100 { id: 'en_female_stokie_uranus_bigtts', label: 'Stokie (EN female)' },
101 ],
102 placeholder: 'Enter text for BytePlus Seed Speech…',
103 },
105​
106export type TranscriptionProviderId =
107 | 'openai'
108 | 'openai-diarize'
109 | 'fal'
110 | 'grok'
111 | 'elevenlabs'
112 | 'byteplus'
113​
114export interface TranscriptionProviderConfig {
115 id: TranscriptionProviderId
116 label: string
117 model: string
118 description: string
119 transcriptionOptions?: Pick<
120 TranscriptionGenerateInput,
121 'responseFormat' | 'modelOptions'
122 >
124​
125export const TRANSCRIPTION_PROVIDERS: ReadonlyArray<TranscriptionProviderConfig> =
126 [
127 {
128 id: 'openai',
129 label: 'OpenAI Whisper',
130 model: 'whisper-1',
131 description: 'OpenAI Whisper transcription with optional streaming.',
132 },
133 {
134 id: 'openai-diarize',
135 label: 'OpenAI Diarize',
136 model: 'gpt-4o-transcribe-diarize',
137 description:
138 'OpenAI diarized transcription with speaker-labeled segments.',
139 transcriptionOptions: {
140 modelOptions: {
141 response_format: 'diarized_json',
142 chunking_strategy: 'auto',
143 },
144 },
145 },
146 {
147 id: 'fal',
148 label: 'Fal Whisper',
149 model: 'fal-ai/whisper',
150 description: 'Fal-hosted Whisper with word-level timestamps.',
151 },
152 {
153 id: 'grok',
154 label: 'Grok STT',
155 model: 'grok-stt',
156 description: 'xAI speech-to-text with word-level timestamps.',
157 },
158 {
159 id: 'elevenlabs',
160 label: 'ElevenLabs Scribe',
161 model: 'scribe_v1',
162 description:
163 'ElevenLabs Scribe with diarization, keyterm biasing, and PII redaction.',
164 },
165 {
166 id: 'byteplus',
167 label: 'BytePlus Seed ASR',
168 model: 'seed-asr',
169 description:
170 'BytePlus Seed Speech ASR with utterance and word-level timestamps.',
171 },
172 ]
173​
174export type AudioProviderId =
175 | 'gemini-lyria'
176 | 'fal-audio'
177 | 'fal-sfx'
178 | 'elevenlabs-music'
179 | 'elevenlabs-sfx'
180​
181export interface AudioProviderConfig {
182 id: AudioProviderId
183 label: string
184 /** Default model when the provider does not expose a chooser. */
185 model: string
186 description: string
187 placeholder: string
188 /** Default generation length in seconds, when the provider accepts one. */
189 defaultDuration?: number
190 /** Ready-made prompts the UI can offer as one-click suggestions. */
191 samplePrompts: ReadonlyArray<{ label: string; prompt: string }>
192 /**
193 * Optional list of alternate models the UI can expose via a dropdown.
194 * By convention the entry matching {@link model} is listed first so it
195 * appears at the top of the selector, but nothing enforces that — the UI
196 * seeds the selection from {@link model} directly.
197 */
198 models?: ReadonlyArray<{ id: string; label: string }>
200​
201export const AUDIO_PROVIDERS: ReadonlyArray<AudioProviderConfig> = [
202 {
203 id: 'gemini-lyria',
204 label: 'Gemini Lyria',
205 model: 'lyria-3-clip-preview',
206 models: [
207 {
208 id: 'lyria-3-clip-preview',
209 label: 'Lyria 3 Clip (30s MP3)',
210 },
211 {
212 id: 'lyria-3-pro-preview',
213 label: 'Lyria 3 Pro (full-length MP3/WAV)',
214 },
215 ],
216 description: 'Google Lyria 3 music generation.',
217 placeholder: 'Ambient piano with warm pads and soft strings',
218 samplePrompts: [
219 {
220 label: 'Late-night jazz trio',
221 prompt:
222 'A contemplative jazz trio with brushed drums, walking upright bass, and smoky piano chords.',
223 },
224 {
225 label: 'Hero at dawn',
226 prompt:
227 'An epic film score for a hero cresting a mountain at dawn: horns, choir, and taiko drums.',
228 },
229 {
230 label: 'Haunted elevator',
231 prompt:
232 'Cheerful 1950s elevator music, but subtly out of tune, like the elevator is haunted.',
233 },
234 {
235 label: 'Polka funeral',
236 prompt:
237 'A funeral dirge unexpectedly remixed as an upbeat accordion polka.',
238 },
239 ],
240 },
241 {
242 id: 'fal-audio',
243 label: 'Fal Audio',
244 model: 'fal-ai/elevenlabs/music',
245 models: [
246 { id: 'fal-ai/elevenlabs/music', label: 'ElevenLabs Music' },
247 {
248 id: 'fal-ai/stable-audio-25/text-to-audio',
249 label: 'Stable Audio 2.5',
250 },
251 {
252 id: 'fal-ai/ace-step/prompt-to-audio',
253 label: 'ACE-Step (prompt-to-audio)',
254 },
255 ],
256 description: 'Fal-hosted open music generation models.',
257 placeholder: 'A lo-fi hip-hop beat with vinyl crackle',
258 defaultDuration: 10,
259 samplePrompts: [
260 {
261 label: 'Lo-fi study beat',
262 prompt:
263 'A mellow lo-fi hip-hop beat with warm vinyl crackle, dusty Rhodes chords, and soft swing.',
264 },
265 {
266 label: 'Ambient drone',
267 prompt:
268 'A slow ambient drone with shimmering reverb, distant field recordings, and evolving pads.',
269 },
270 {
271 label: 'Seagull Eurovision',
272 prompt:
273 'A Eurovision-style power ballad performed entirely by a choir of disgruntled seagulls.',
274 },
275 {
276 label: 'Hydrophobic pirate',
277 prompt:
278 'The rousing theme song for a swashbuckling pirate who is terrified of water.',
279 },
280 {
281 label: 'Death metal laundry',
282 prompt:
283 'A death metal ballad about losing your favorite socks in the dryer, complete with guttural vocals.',
284 },
285 ],
286 },
287 {
288 id: 'fal-sfx',
289 label: 'Fal SFX',
290 model: 'fal-ai/mmaudio-v2/text-to-audio',
291 models: [
292 {
293 id: 'fal-ai/mmaudio-v2/text-to-audio',
294 label: 'MMAudio v2 Text-to-Audio',
295 },
296 ],
297 description: 'Fal-hosted text-to-SFX models for short sound effects.',
298 placeholder: 'Glass shattering on a tile floor',
299 defaultDuration: 5,
300 samplePrompts: [
301 {
302 label: 'Rain on tin roof',
303 prompt: 'Steady rain pattering on a corrugated metal roof at night.',
304 },
305 {
306 label: 'Marble hallway steps',
307 prompt:
308 'Slow leather-soled footsteps echoing through an empty marble hallway.',
309 },
310 {
311 label: 'Interrogated duck',
312 prompt:
313 'A rubber duck being dramatically interrogated under a swinging lamp, squeaks only.',
314 },
315 {
316 label: 'Cartoon banana slip',
317 prompt:
318 'Classic cartoon banana slip: quick slide, comedic boing, and a distant crash.',
319 },
320 ],
321 },
322 {
323 id: 'elevenlabs-music',
324 label: 'ElevenLabs Music',
325 model: 'music_v1',
326 models: [{ id: 'music_v1', label: 'Music v1' }],
327 description:
328 'ElevenLabs Music — free-form prompts or structured composition plans.',
329 placeholder: 'An upbeat synthwave track with driving drums and arpeggios',
330 defaultDuration: 15,
331 samplePrompts: [
332 {
333 label: 'Synthwave drive',
334 prompt:
335 'An upbeat synthwave track with driving drums, warm analog pads, and glittering arpeggios.',
336 },
337 {
338 label: 'Cinematic reveal',
339 prompt:
340 'A cinematic reveal score with soaring strings, low brass stabs, and a sudden timpani hit.',
341 },
342 {
343 label: 'Acoustic campfire',
344 prompt:
345 'A gentle acoustic campfire tune: fingerpicked guitar, soft harmonica, and distant crickets.',
346 },
347 {
348 label: 'Angry kazoo orchestra',
349 prompt:
350 'A furious kazoo orchestra performing an operatic aria about overdue library books.',
351 },
352 ],
353 },
354 {
355 id: 'elevenlabs-sfx',
356 label: 'ElevenLabs SFX',
357 model: 'eleven_text_to_sound_v2',
358 models: [
359 { id: 'eleven_text_to_sound_v2', label: 'Text-to-Sound v2' },
360 { id: 'eleven_text_to_sound_v1', label: 'Text-to-Sound v1' },
361 ],
362 description:
363 'ElevenLabs text-to-sound for short effects, 0.5–30 seconds per clip.',
364 placeholder: 'A whoosh followed by a deep bass impact',
365 defaultDuration: 5,
366 samplePrompts: [
367 {
368 label: 'Trailer whoosh',
369 prompt: 'A cinematic whoosh followed by a deep sub-bass impact.',
370 },
371 {
372 label: 'Sword unsheathe',
373 prompt:
374 'The crisp metallic ring of a sword being drawn from a leather scabbard.',
375 },
376 {
377 label: 'UI confirmation',
378 prompt:
379 'A short, satisfying UI confirmation tone with a subtle sparkle tail.',
380 },
381 {
382 label: 'Anxious toaster',
383 prompt:
384 'A small kitchen toaster having an anxiety attack: frantic clicks, steam, and a plaintive ding.',
385 },
386 ],
387 },