Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

VideoGenerator.tsx

591 lines21.0 KB

1import { useCallback, useEffect, useRef, useState } from 'react'
2import { Film, Loader2, Shuffle, Upload, Wand2, X } from 'lucide-react'
3import { useGenerateVideo } from '@tanstack/ai-react'
4import type { VideoModel, VideoMode } from '@/lib/models'
5import type { AttachedMedia } from '@/lib/media'
6import type { MediaPrompt, MediaPromptPart } from '@tanstack/ai/client'
7import type { VideoBilling } from '@/lib/billing'
8​
9import { generateVideoFn } from '@/lib/server-functions'
10import { VIDEO_MODELS } from '@/lib/models'
11import { getRandomVideoPrompt } from '@/lib/prompts'
12import { imageUrlToPart, readMediaFile, toVideoPart } from '@/lib/media'
13import { readVideoBilling } from '@/lib/billing'
14​
15interface VideoGeneratorProps {
16 initialImageUrl?: string | null
18​
19/**
20 * One submission. Each card builds its own prompt from this, because which
21 * parts a model may receive depends on the model: only Omni takes a video
22 * part.
23 */
24interface VideoRequest {
25 prompt: string
26 mode: VideoMode
27 imageUrl: string | null
28 video: AttachedMedia | null
30​
31/** What each card hands the parent so it can drive that card's hook. */
32interface VideoCardHandle {
33 run: (request: VideoRequest) => void
34 clear: () => void
36​
37/**
38 * Image conditioning rides in the prompt: the start frame is an image part
39 * tagged `role: 'start_frame'`, which the fal adapter routes to the
40 * endpoint's start-image field (e.g. `image_url` on Kling i2v) and Omni takes
41 * as an interaction content block. Video parts (a reference clip or a clip to
42 * edit) are an Omni capability only, so they never reach the other providers.
43 */
44function buildVideoPrompt(
45 request: VideoRequest,
46 model: VideoModel,
47): MediaPrompt {
48 const parts: Array<MediaPromptPart> = [
49 { type: 'text', content: request.prompt },
50 ]
51 if (request.mode === 'image-to-video' && request.imageUrl) {
52 parts.push(imageUrlToPart(request.imageUrl, { role: 'start_frame' }))
53 }
54 if (request.video && model.provider === 'gemini') {
55 parts.push(toVideoPart(request.video))
56 }
57 return parts.length === 1 ? request.prompt : parts
59​
60export default function VideoGenerator({
61 initialImageUrl,
62}: VideoGeneratorProps) {
63 const [mode, setMode] = useState<VideoMode>('text-to-video')
64 const [prompt, setPrompt] = useState('')
65 const [selectedModel, setSelectedModel] = useState<string>('all')
66 const [imagePreview, setImagePreview] = useState<string | null>(
67 initialImageUrl ?? null,
68 )
69 const [attachedVideo, setAttachedVideo] = useState<AttachedMedia | null>(null)
70 /** Whether anything has been generated for the current model selection. */
71 const [submitted, setSubmitted] = useState(false)
72 const [running, setRunning] = useState<Record<string, boolean>>({})
73 const fileInputRef = useRef<HTMLInputElement>(null)
74 const videoInputRef = useRef<HTMLInputElement>(null)
75 // Generation is started from the click, not from an effect on the cards:
76 // a hook only exists inside a mounted component, so the cards mount with
77 // the picker and hand the parent a way to run and clear them.
78 const cardsRef = useRef(new Map<string, VideoCardHandle>())
79​
80 const filteredModels: ReadonlyArray<VideoModel> = VIDEO_MODELS.filter(
81 (m) => m.mode === mode,
82 )
83 const falModels = filteredModels.filter((m) => m.provider === 'fal')
84 const xaiModels = filteredModels.filter((m) => m.provider === 'xai')
85 const geminiModels = filteredModels.filter((m) => m.provider === 'gemini')
86 const byteplusModels = filteredModels.filter((m) => m.provider === 'byteplus')
87 const activeModels =
88 selectedModel === 'all'
89 ? filteredModels
90 : filteredModels.filter((model) => model.id === selectedModel)
91​
92 // Gemini Omni Flash additionally accepts video prompt parts (a reference
93 // clip or a video to edit). Offer the upload whenever an Omni model is in
94 // the running — other providers never receive the video part.
95 const omniInRun =
96 selectedModel === 'all'
97 ? geminiModels.length > 0
98 : selectedModel.startsWith('gemini-omni-flash-preview')
99​
100 // Each card runs its own generation, so the form is busy while any of them
101 // reports that it is.
102 const isGenerating = Object.values(running).some(Boolean)
103​
104 useEffect(() => {
105 if (initialImageUrl) {
106 setImagePreview(initialImageUrl)
107 }
108 }, [initialImageUrl])
109​
110 useEffect(() => {
111 // When mode changes, reset to "all" or first available model — and with
112 // it the results, which belong to the models that produced them.
113 setSelectedModel('all')
114 setSubmitted(false)
115 setRunning({})
116 }, [mode])
117​
118 const handleImageSelect = async (e: React.ChangeEvent<HTMLInputElement>) => {
119 const file = e.target.files?.[0]
120 if (fileInputRef.current) fileInputRef.current.value = ''
121 if (!file) return
122 const attached = await readMediaFile(file)
123 setImagePreview(attached.dataUrl)
124 }
125​
126 const clearImage = () => {
127 setImagePreview(null)
128 if (fileInputRef.current) fileInputRef.current.value = ''
129 }
130​
131 const handleVideoSelect = async (e: React.ChangeEvent<HTMLInputElement>) => {
132 const file = e.target.files?.[0]
133 if (videoInputRef.current) videoInputRef.current.value = ''
134 if (!file) return
135 setAttachedVideo(await readMediaFile(file))
136 }
137​
138 const clearVideo = () => {
139 setAttachedVideo(null)
140 if (videoInputRef.current) videoInputRef.current.value = ''
141 }
142​
143 const handleRunningChange = useCallback((modelId: string, value: boolean) => {
144 setRunning((prev) =>
145 prev[modelId] === value ? prev : { ...prev, [modelId]: value },
146 )
147 }, [])
148​
149 const registerCard = useCallback(
150 (modelId: string, card: VideoCardHandle | null) => {
151 if (card) cardsRef.current.set(modelId, card)
152 else cardsRef.current.delete(modelId)
153 },
154 [],
155 )
156​
157 const handleGenerate = () => {
158 if (!prompt.trim()) return
159 if (mode === 'image-to-video' && !imagePreview) return
160​
161 const request: VideoRequest = {
162 prompt,
163 mode,
164 imageUrl: imagePreview,
165 video: attachedVideo,
166 }
167 setSubmitted(true)
168 for (const model of activeModels) {
169 cardsRef.current.get(model.id)?.run(request)
170 }
171 }
172​
173 /** "Generate another": drop the results themselves, not just their panel. */
174 const clearResults = () => {
175 for (const card of cardsRef.current.values()) card.clear()
176 setSubmitted(false)
177 setRunning({})
178 }
179​
180 return (
181 <div className="space-y-6">
182 <div className="flex gap-2">
183 <button
184 onClick={() => setMode('text-to-video')}
185 className={`px-4 py-2 rounded-lg font-medium transition-colors ${
186 mode === 'text-to-video'
187 ? 'bg-blue-600 text-white'
188 : 'bg-gray-700 text-gray-300 hover:bg-gray-600'
189 }`}
190 >
191 Text-to-Video
192 </button>
193 <button
194 onClick={() => setMode('image-to-video')}
195 className={`px-4 py-2 rounded-lg font-medium transition-colors ${
196 mode === 'image-to-video'
197 ? 'bg-blue-600 text-white'
198 : 'bg-gray-700 text-gray-300 hover:bg-gray-600'
199 }`}
200 >
201 Image-to-Video
202 </button>
203 </div>
204​
205 <div className="space-y-4">
206 <div>
207 <label className="block text-sm font-medium text-gray-300 mb-2">
208 Model
209 </label>
210 <select
211 value={selectedModel}
212 onChange={(e) => {
213 setSelectedModel(e.target.value)
214 setSubmitted(false)
215 setRunning({})
216 }}
217 disabled={isGenerating}
218 className="w-full px-4 py-3 bg-gray-800 border border-gray-700 rounded-lg text-white focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent disabled:opacity-50"
219 >
220 <option value="all">All Models</option>
221 <optgroup label="fal.ai">
222 {falModels.map((model) => (
223 <option key={model.id} value={model.id}>
224 {model.name}
225 </option>
226 ))}
227 </optgroup>
228 <optgroup label="xAI (direct)">
229 {xaiModels.map((model) => (
230 <option key={model.id} value={model.id}>
231 {model.name}
232 </option>
233 ))}
234 </optgroup>
235 <optgroup label="Google (direct)">
236 {geminiModels.map((model) => (
237 <option key={model.id} value={model.id}>
238 {model.name}
239 </option>
240 ))}
241 </optgroup>
242 <optgroup label="BytePlus (direct)">
243 {byteplusModels.map((model) => (
244 <option key={model.id} value={model.id}>
245 {model.name}
246 </option>
247 ))}
248 </optgroup>
249 </select>
250 </div>
251​
252 {mode === 'image-to-video' && (
253 <div>
254 <label className="block text-sm font-medium text-gray-300 mb-2">
255 Source Image
256 </label>
257 {imagePreview ? (
258 <div className="relative">
259 <img
260 src={imagePreview}
261 alt="Source"
262 className="w-full max-h-64 object-contain rounded-lg border border-gray-700"
263 />
264 <button
265 onClick={clearImage}
266 disabled={isGenerating}
267 className="absolute top-2 right-2 p-1 bg-gray-900/80 hover:bg-gray-800 rounded-full text-white disabled:opacity-50"
268 >
269 <X className="w-4 h-4" />
270 </button>
271 </div>
272 ) : (
273 <button
274 onClick={() => fileInputRef.current?.click()}
275 className="w-full p-8 border-2 border-dashed border-gray-600 hover:border-gray-500 rounded-lg text-gray-400 hover:text-gray-300 transition-colors flex flex-col items-center gap-2"
276 >
277 <Upload className="w-8 h-8" />
278 <span>Click to upload an image</span>
279 </button>
280 )}
281 <input
282 ref={fileInputRef}
283 type="file"
284 accept="image/*"
285 onChange={handleImageSelect}
286 className="hidden"
287 />
288 </div>
289 )}
290​
291 {omniInRun && (
292 <div>
293 <label className="block text-sm font-medium text-gray-300 mb-2">
294 Reference video{' '}
295 <span className="text-gray-500 font-normal">
296 (optional — Gemini Omni Flash only, clips of 3s or less)
297 </span>
298 </label>
299 {attachedVideo ? (
300 <div className="relative">
301 <video
302 src={attachedVideo.dataUrl}
303 controls
304 muted
305 className="w-full max-h-64 rounded-lg border border-gray-700"
306 />
307 <button
308 onClick={clearVideo}
309 disabled={isGenerating}
310 className="absolute top-2 right-2 p-1 bg-gray-900/80 hover:bg-gray-800 rounded-full text-white disabled:opacity-50"
311 >
312 <X className="w-4 h-4" />
313 </button>
314 </div>
315 ) : (
316 <button
317 onClick={() => videoInputRef.current?.click()}
318 className="w-full p-6 border-2 border-dashed border-gray-600 hover:border-gray-500 rounded-lg text-gray-400 hover:text-gray-300 transition-colors flex flex-col items-center gap-2"
319 >
320 <Upload className="w-6 h-6" />
321 <span>Click to attach a video clip</span>
322 </button>
323 )}
324 <input
325 ref={videoInputRef}
326 type="file"
327 accept="video/*"
328 onChange={handleVideoSelect}
329 className="hidden"
330 />
331 </div>
332 )}
333​
334 <div>
335 <div className="flex items-center justify-between mb-2">
336 <label className="text-sm font-medium text-gray-300">Prompt</label>
337 <button
338 onClick={() => setPrompt(getRandomVideoPrompt(mode))}
339 disabled={isGenerating}
340 className="flex items-center gap-1.5 px-3 py-1 text-xs font-medium text-blue-400 hover:text-blue-300 bg-blue-500/10 hover:bg-blue-500/20 rounded-md transition-colors disabled:opacity-50 disabled:cursor-not-allowed"
341 >
342 <Shuffle className="w-3.5 h-3.5" />
343 Shuffle
344 </button>
345 </div>
346 <textarea
347 value={prompt}
348 onChange={(e) => setPrompt(e.target.value)}
349 placeholder={
350 mode === 'image-to-video'
351 ? 'Describe how you want the image to animate...'
352 : 'Describe the video you want to generate...'
353 }
354 className="w-full px-4 py-3 bg-gray-800 border border-gray-700 rounded-lg text-white placeholder-gray-500 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent resize-none"
355 rows={3}
356 disabled={isGenerating}
357 />
358 </div>
359​
360 <button
361 onClick={handleGenerate}
362 disabled={
363 isGenerating ||
364 !prompt.trim() ||
365 (mode === 'image-to-video' && !imagePreview)
366 }
367 className="w-full px-6 py-3 bg-blue-600 hover:bg-blue-700 disabled:bg-gray-700 disabled:cursor-not-allowed text-white font-medium rounded-lg transition-colors flex items-center justify-center gap-2"
368 >
369 {isGenerating ? (
370 <>
371 <Loader2 className="w-5 h-5 animate-spin" />
372 Generating...
373 </>
374 ) : (
375 <>
376 <Film className="w-5 h-5" />
377 Generate Video{selectedModel === 'all' ? 's' : ''}
378 </>
379 )}
380 </button>
381 </div>
382​
383 <div className="space-y-6">
384 {submitted && (
385 <div className="flex items-center justify-between">
386 <h3 className="text-lg font-medium text-white">
387 {activeModels.length > 1 ? 'Generated Videos' : 'Generated Video'}
388 </h3>
389 {!isGenerating && (
390 <button
391 onClick={clearResults}
392 className="text-sm text-gray-400 hover:text-white underline"
393 >
394 Generate another
395 </button>
396 )}
397 </div>
398 )}
399 {activeModels.map((model) => (
400 <VideoModelCard
401 key={model.id}
402 model={model}
403 visible={submitted}
404 showTitle={activeModels.length > 1}
405 onRegister={registerCard}
406 onRunningChange={handleRunningChange}
407 />
408 ))}
409 </div>
410 </div>
411 )
413​
414/**
415 * One model's generation. `useGenerateVideo` drives the whole job lifecycle —
416 * the server streams job id, status and the finished url back on one request,
417 * so there is no polling timer or job-state machine in the UI.
418 */
419function VideoModelCard({
420 model,
421 visible,
422 showTitle,
423 onRegister,
424 onRunningChange,
425}: {
426 model: VideoModel
427 visible: boolean
428 showTitle: boolean
429 onRegister: (modelId: string, card: VideoCardHandle | null) => void
430 onRunningChange: (modelId: string, running: boolean) => void
431}) {
432 const [editPrompt, setEditPrompt] = useState('')
433 const [billing, setBilling] = useState<VideoBilling | undefined>(undefined)
434 // The hook builds its client once, so the fetcher closure is created once
435 // too: anything that changes between runs (here, the Omni interaction being
436 // continued) has to be read from a ref at call time rather than captured.
437 const previousInteractionRef = useRef<string | undefined>(undefined)
438​
439 const { generate, reset, result, jobId, videoStatus, isLoading, error } =
440 useGenerateVideo({
441 threadId: `video:${model.id}`,
442 // `options.signal` is the hook's abort signal. Forwarding it matters
443 // more here than for images: cancelling the response is what ends the
444 // server's polling loop instead of leaving it running to the timeout.
445 fetcher: (input, options) =>
446 generateVideoFn({
447 data: {
448 prompt: input.prompt,
449 model: model.id,
450 ...(previousInteractionRef.current
451 ? { previousInteractionId: previousInteractionRef.current }
452 : {}),
453 },
454 signal: options?.signal,
455 }),
456 onChunk: (chunk) => {
457 const usage = readVideoBilling(chunk)
458 if (usage) setBilling(usage)
459 },
460 })
461​
462 const clear = useCallback(() => {
463 previousInteractionRef.current = undefined
464 setBilling(undefined)
465 reset()
466 }, [reset])
467​
468 const run = useCallback(
469 (request: VideoRequest) => {
470 // A new submission is a fresh video, never a continuation of the clip
471 // this card is showing. Deliberately not `reset()` first: reset stops
472 // the client, which clears the `isLoading` that makes a second
473 // `generate()` a no-op — that guard is what swallows a double-click.
474 // The stale result a failed re-run leaves behind is handled by the
475 // render gate below instead.
476 previousInteractionRef.current = undefined
477 setBilling(undefined)
478 void generate({ prompt: buildVideoPrompt(request, model) })
479 },
480 [generate, model],
481 )
482​
483 useEffect(() => {
484 onRegister(model.id, { run, clear })
485 return () => onRegister(model.id, null)
486 }, [model.id, run, clear, onRegister])
487​
488 useEffect(() => {
489 onRunningChange(model.id, isLoading)
490 }, [model.id, isLoading, onRunningChange])
491​
492 if (!visible) return null
493​
494 /**
495 * Gemini Omni Flash conversational editing: chain a new prompt onto the
496 * finished generation via its interaction id (the job id). The model
497 * applies the change while preserving everything else in the video.
498 */
499 const handleEditVideo = () => {
500 const edit = editPrompt.trim()
501 if (!edit || !result || isLoading) return
502 previousInteractionRef.current = result.jobId
503 setBilling(undefined)
504 setEditPrompt('')
505 void generate({ prompt: edit })
506 }
507​
508 const status = videoStatus?.status
509​
510 return (
511 <div className="space-y-2">
512 {showTitle && (
513 <h4 className="text-sm font-medium text-gray-300">{model.name}</h4>
514 )}
515 {isLoading && (
516 <div className="flex items-center gap-2 p-4 bg-gray-800 rounded-lg border border-gray-700">
517 <Loader2 className="w-5 h-5 animate-spin text-blue-400" />
518 <span className="text-gray-400">
519 {!jobId
520 ? 'Submitting...'
521 : status === 'processing'
522 ? `Processing${
523 videoStatus?.progress != null
524 ? ` (${videoStatus.progress}%)`
525 : '...'
526 }`
527 : 'Queued...'}
528 </span>
529 </div>
530 )}
531 {error && !isLoading && (
532 <div className="p-4 bg-red-500/10 border border-red-500/20 rounded-lg text-red-400">
533 {error.message}
534 </div>
535 )}
536 {/* `!error`: a failed re-run keeps the previous clip on the hook, and it
537 must not render underneath the new error. */}
538 {result && !isLoading && !error && (
539 <>
540 <div className="rounded-lg overflow-hidden border border-gray-700">
541 <video
542 src={result.url}
543 controls
544 autoPlay
545 loop
546 className="w-full h-auto"
547 />
548 </div>
549 {billing?.cost != null ? (
550 <p className="text-xs text-gray-500">
551 Billed ${billing.cost.toFixed(3)}
552 {billing.unitsBilled != null
553 ? ` for ${billing.unitsBilled} second${billing.unitsBilled === 1 ? '' : 's'} of video`
554 : ''}
555 </p>
556 ) : (
557 billing?.unitsBilled != null && (
558 <p className="text-xs text-gray-500">
559 Billed {billing.unitsBilled} fal unit
560 {billing.unitsBilled === 1 ? '' : 's'} — multiply by the
561 endpoint unit price for USD cost
562 </p>
563 )
564 )}
565 {model.provider === 'gemini' && (
566 <div className="flex gap-2">
567 <input
568 type="text"
569 value={editPrompt}
570 onChange={(e) => setEditPrompt(e.target.value)}
571 onKeyDown={(e) => {
572 if (e.key === 'Enter') handleEditVideo()
573 }}
574 placeholder="Describe an edit — e.g. 'make it nighttime'..."
575 className="flex-1 px-3 py-2 bg-gray-800 border border-gray-700 rounded-lg text-white text-sm placeholder-gray-500 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent disabled:opacity-50"
576 />
577 <button
578 onClick={handleEditVideo}
579 disabled={!editPrompt.trim()}
580 className="px-4 py-2 bg-purple-600 hover:bg-purple-700 disabled:bg-gray-700 disabled:cursor-not-allowed text-white text-sm font-medium rounded-lg transition-colors flex items-center gap-1.5"
581 >
582 <Wand2 className="w-4 h-4" />
583 Edit
584 </button>
585 </div>
586 )}
587 </>
588 )}
589 </div>
590 )