Skip to content
Source
Sign inSign up
TanStack/ai
Open History to see the last commit

media.ts

115 lines3.6 KB

1import type { MediaInputMetadata, MediaPromptPart } from '@tanstack/ai/client'
2​
3/**
4 * A media file (image or video) the user attached as conditioning input.
5 * `dataUrl` is the full `data:<mime>;base64,...` string used directly for
6 * the thumbnail preview; `base64` is the same payload with the prefix
7 * stripped for the prompt part.
8 */
9export interface AttachedMedia {
10 id: string
11 name: string
12 mimeType: string
13 /** Full data URL, used for the <img> / <video> preview. */
14 dataUrl: string
15 /** Base64 payload without the `data:` prefix, used for the prompt part. */
16 base64: string
18​
19/** Reads a File into an AttachedMedia (data URL preview + raw base64 payload). */
20export function readMediaFile(file: File): Promise<AttachedMedia> {
21 return new Promise((resolve, reject) => {
22 const reader = new FileReader()
23 reader.onerror = () =>
24 reject(reader.error ?? new Error('Failed to read file'))
25 reader.onload = () => {
26 const dataUrl = reader.result
27 if (typeof dataUrl !== 'string') {
28 reject(new Error('Unexpected file read result'))
29 return
30 }
31 const base64 = dataUrl.slice(dataUrl.indexOf(',') + 1)
32 resolve({
33 id: crypto.randomUUID(),
34 name: file.name,
35 mimeType: file.type,
36 dataUrl,
37 base64,
38 })
39 }
40 reader.readAsDataURL(file)
41 })
43​
44/** Builds an image prompt part from an attached image, with optional role hint. */
45export function toImagePart(
46 image: AttachedMedia,
47 metadata?: MediaInputMetadata,
48): MediaPromptPart {
49 return {
50 type: 'image',
51 source: { type: 'data', value: image.base64, mimeType: image.mimeType },
52 ...(metadata ? { metadata } : {}),
53 }
55​
56/**
57 * Builds a video prompt part from an attached video clip — e.g. a reference
58 * clip or a video to edit for Gemini Omni Flash, which accepts video inputs
59 * alongside text and images.
60 */
61export function toVideoPart(
62 video: AttachedMedia,
63 metadata?: MediaInputMetadata,
64): MediaPromptPart {
65 return {
66 type: 'video',
67 source: { type: 'data', value: video.base64, mimeType: video.mimeType },
68 ...(metadata ? { metadata } : {}),
69 }
71​
72/**
73 * Builds a prompt part from a remote media URL, for conditioning inputs the
74 * app never holds bytes for (the Seedance template presets). Adapters pass a
75 * `url` source straight through to the provider, so the file is fetched
76 * server-side by the provider rather than downloaded and re-encoded here.
77 *
78 * `metadata` is only meaningful for images — adapters route video and audio
79 * parts by part type alone.
80 */
81export function mediaUrlToPart(
82 kind: 'image' | 'video' | 'audio',
83 url: string,
84 metadata?: MediaInputMetadata,
85): MediaPromptPart {
86 const source = { type: 'url', value: url } as const
87 if (kind === 'video') return { type: 'video', source }
88 if (kind === 'audio') return { type: 'audio', source }
89 return { type: 'image', source, ...(metadata ? { metadata } : {}) }
91​
92/**
93 * Builds an image prompt part from a URL string — either a remote URL
94 * (passed through as a `url` source) or a `data:` URL (decomposed into a
95 * `data` source so adapters that upload files get the raw payload).
96 */
97export function imageUrlToPart(
98 url: string,
99 metadata?: MediaInputMetadata,
100): MediaPromptPart {
101 const meta = metadata ? { metadata } : {}
102 if (!url.startsWith('data:')) {
103 return { type: 'image', source: { type: 'url', value: url }, ...meta }
104 }
105 const comma = url.indexOf(',')
106 const mimeType = url.slice(5, comma).split(';')[0]
107 if (comma === -1 || !mimeType) {
108 throw new Error('data: URL is missing a mime type')
109 }
110 return {
111 type: 'image',
112 source: { type: 'data', value: url.slice(comma + 1), mimeType },
113 ...meta,
114 }