Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

use-realtime.ts

115 lines3.3 KB

1import { createServerFn } from '@tanstack/react-start'
2import { realtimeToken } from '@tanstack/ai'
3import { useRealtimeChat } from '@tanstack/ai-react'
4import { openaiRealtime, openaiRealtimeToken } from '@tanstack/ai-openai'
5import {
6 elevenlabsRealtime,
7 elevenlabsRealtimeToken,
8} from '@tanstack/ai-elevenlabs'
9import { geminiRealtime, geminiRealtimeToken } from '@tanstack/ai-gemini'
10import { grokRealtime, grokRealtimeToken } from '@tanstack/ai-grok'
11import { realtimeClientTools } from '@/lib/realtime-tools'
12​
13type Provider = 'openai' | 'elevenlabs' | 'gemini' | 'grok'
14​
15const getRealtimeTokenFn = createServerFn({ method: 'POST' })
16 .inputValidator((data: { provider: Provider; language?: string }) => {
17 if (!data.provider) throw new Error('Provider is required')
18 return data
19 })
20 .handler(async ({ data }) => {
21 if (data.provider === 'openai') {
22 return realtimeToken({
23 adapter: openaiRealtimeToken({
24 model: 'gpt-realtime',
25 }),
26 })
27 }
28​
29 if (data.provider === 'gemini') {
30 return realtimeToken({
31 adapter: geminiRealtimeToken(),
32 })
33 }
34​
35 if (data.provider === 'elevenlabs') {
36 return realtimeToken({
37 adapter: elevenlabsRealtimeToken({
38 ...(data.language ? { overrides: { language: data.language } } : {}),
39 }),
40 })
41 }
42​
43 if (data.provider === 'grok') {
44 return realtimeToken({
45 adapter: grokRealtimeToken({ model: 'grok-voice-fast-1.0' }),
46 })
47 }
48​
49 throw new Error(`Unknown provider: ${data.provider}`)
50 })
51​
52function adapterForProvider(provider: Provider) {
53 switch (provider) {
54 case 'openai':
55 return openaiRealtime()
56 case 'elevenlabs':
57 return elevenlabsRealtime()
58 case 'gemini':
59 return geminiRealtime()
60 case 'grok':
61 return grokRealtime()
62 }
64​
65export function useRealtime({
66 provider,
67 language,
68 voice,
69 outputModalities,
70 temperature,
71 maxOutputTokens,
72 semanticEagerness,
73}: {
74 provider: Provider
75 language?: string
76 voice?: string
77 outputModalities?: Array<'audio' | 'text'>
78 temperature?: number
79 maxOutputTokens?: number | 'inf'
80 semanticEagerness?: 'low' | 'medium' | 'high'
81}) {
82 return useRealtimeChat({
83 getToken: () =>
84 getRealtimeTokenFn({
85 data: {
86 provider,
87 ...(provider === 'elevenlabs' && language ? { language } : {}),
88 },
89 }),
90 adapter: adapterForProvider(provider),
91 instructions: `You are a helpful, friendly voice assistant with access to several tools.
92​
93You can:
94- Tell the user the current time and date (getCurrentTime)
95- Get weather information for any location (getWeather)
96- Set reminders for the user (setReminder)
97- Search a knowledge base for information (searchKnowledge)
98​
99Keep your responses concise and conversational since this is a voice interface.
100When using tools, briefly explain what you're doing and then share the results naturally.
101If the user sends an image, describe what you see and answer any questions about it.
102Be friendly and engaging!`,
103 voice:
104 voice ??
105 (provider === 'grok' ? 'eve' : provider === 'gemini' ? 'Puck' : 'alloy'),
106 tools: realtimeClientTools,
107 outputModalities,
108 temperature,
109 maxOutputTokens,
110 semanticEagerness,
111 onError: (err) => {
112 console.error('Realtime error:', err)
113 },
114 })