feat(ai): add experimental generateMotion() with fal and Uthana adapters #2194
MergedAlemTuzlakmerged 6 commits into
main fromt3/motion-generationmerged by AlemTuzlak View on GitHubtanstack-source requested a review fromjherr
tanstack-source requested a review fromTkDodo
- packages/ai-uthana/src/adapters/motion.ts line 205Outdated@@ -0,0 +185,21 @@185+ const url = `${this.config.baseUrl}/motion/file/motion_viewer/${encodeURIComponent(characterId)}/${encodeURIComponent(motionId)}/${format}/${fileName}${query}`186+ const response = await this.config.fetch(url, {187+ headers: { authorization: this.config.authorization },188+ ...(options?.abortSignal ? { signal: options.abortSignal } : {}),189+ })190+ if (!response.ok) {191+ throw new Error(192+ `Uthana download of ${format} failed (${response.status} ${response.statusText})`,193+ )194+ }195+ const bytes = await response.arrayBuffer()196+ const contentType = CONTENT_TYPES[format]197+ assets.push({198+ url: `data:${contentType};base64,${arrayBufferToBase64(bytes)}`,199+ format,200+ contentType,201+ fileName,202+ fileSize: bytes.byteLength,203+ })204+ }205+ this.downloadOptions.delete(jobId)
- packages/ai-fal/src/utils/client.ts line 55Outdated@@ -38,3 +38,18 @@3838 return createFalClient(toFalClientOptions(config))3939 }404041+/**42+ * fal can report a request COMPLETED and then answer the result fetch with a43+ * 422. Join its validation details as `field: message` pairs. Returns44+ * undefined when the error has no detail list.45+ */46+export function falValidationMessage(error: unknown): string | undefined {47+ if (typeof error !== 'object' || error === null || !('body' in error)) {48+ return undefined49+ }50+ const body = error.body51+ if (typeof body !== 'object' || body === null || !('detail' in body)) {52+ return undefined53+ }54+ const detail = body.detail55+ if (!Array.isArray(detail)) return undefined
- packages/ai/tests/generate-motion.test.ts line 71Outdated@@ -0,0 +51,21 @@51+ }52+}53+54+function customChunks(chunks: Array<StreamChunk>) {55+ return chunks.flatMap((c) =>56+ c.type === 'CUSTOM' ? [{ name: c.name, value: c.value }] : [],57+ )58+}59+60+function recordingMiddleware() {61+ const calls: Array<{ hook: string; runId: string | undefined }> = []62+ const middleware: GenerationMiddleware = {63+ onStart: (ctx) => void calls.push({ hook: 'start', runId: ctx.runId }),64+ onFinish: (ctx) => void calls.push({ hook: 'finish', runId: ctx.runId }),65+ onError: (ctx) => void calls.push({ hook: 'error', runId: ctx.runId }),66+ onAbort: (ctx) => void calls.push({ hook: 'abort', runId: ctx.runId }),67+ }68+ return { calls, middleware }69+}70+71+describe('generateMotion stream', () => {
AlemTuzlak reviewed
AlemTuzlak reviewed
AlemTuzlak reviewed
- packages/ai-uthana/package.json line 20Outdated@@ -0,0 +1,20 @@1+{2+ "name": "@tanstack/ai-uthana",3+ "version": "0.0.0",4+ "description": "Uthana adapter for TanStack AI motion generation: text-to-motion skeletal animation (FBX, GLB) retargeted onto your own characters.",5+ "author": "Tanner Linsley",6+ "license": "MIT",7+ "homepage": "https://tanstack.com/ai",8+ "repository": {9+ "type": "git",10+ "url": "git+https://github.com/TanStack/ai.git",11+ "directory": "packages/ai-uthana"12+ },13+ "bugs": {14+ "url": "https://github.com/TanStack/ai/issues"15+ },16+ "funding": {17+ "type": "github",18+ "url": "https://github.com/sponsors/tannerlinsley"19+ },20+ "type": "module",
AlemTuzlak reviewed
- docs/media/motion-generation.md line 491Outdated@@ -0,0 +471,21 @@471+ model: this.model,472+ assets: files.map((file) => ({ url: file.url, format: file.extension })),473+ }474+ }475+}476+```477+478+Use it like any other adapter: `generateMotion({ adapter: new StudioMotionAdapter('studio-motion-1'), ... })`.479+480+Rules for an adapter:481+482+Return only the metadata that the provider sends. Do not guess a clip length or a frame rate.483+Set `progress` only when the provider reports a real number.484+If the provider answers at once, do the work in `createMotionJob`. Then return `'completed'` on the first `getMotionStatus` call.485+Pass `abortSignal` to the HTTP call. It stops the request, not the provider job.486+487+## Advanced488+489+### Check a job from another request490+491+`getMotionJobStatus()` needs only the adapter and the job id. You can submit in one request and check in another request, or in another process. When the job is complete, `status.motion` holds the files.
AlemTuzlak reviewed
AlemTuzlak reviewed
tanstack-source reviewed
examples/react/generate-motion/scripts/generate-orc-clips.ts line 277Outdated@@ -0,0 +257,21 @@257+ // Keep earlier records when only some clips are regenerated.258+ const manifestPath = join(out, 'manifest.json')259+ const previous: Record<string, ClipRecord> = await readFile(260+ manifestPath,261+ 'utf8',262+ )263+ .then((text) => JSON.parse(text).clips as Record<string, ClipRecord>)264+ .catch(() => ({}))265+266+ const records = await Promise.all(clips.map((spec) => runClip(spec, out)))267+ const merged = { ...previous }268+ for (const record of records) merged[record.name] = record269+270+ await writeFile(271+ manifestPath,272+ JSON.stringify(273+ {274+ generatedWith:275+ '@tanstack/ai generateMotion + @tanstack/ai-fal falMotion',276+ provider: 'fal',277+ model: MODEL,examples/react/generate-motion/scripts/generate-orc-clips.ts line 287Outdated@@ -0,0 +267,21 @@267+ const merged = { ...previous }268+ for (const record of records) merged[record.name] = record269+270+ await writeFile(271+ manifestPath,272+ JSON.stringify(273+ {274+ generatedWith:275+ '@tanstack/ai generateMotion + @tanstack/ai-fal falMotion',276+ provider: 'fal',277+ model: MODEL,278+ updatedAt: new Date().toISOString(),279+ clips: merged,280+ },281+ null,282+ 2,283+ ),284+ )285+ console.log(`manifest: ${manifestPath}`)286+ if (records.some((r) => !r.result)) process.exitCode = 1287+}packages/ai-uthana/src/adapters/motion.ts line 176Outdated@@ -0,0 +156,21 @@156+ case 'FINISHED':157+ return error158+ ? { jobId, status: 'failed', error }159+ : { jobId, status: 'completed' }160+ case 'FAILED':161+ return { jobId, status: 'failed', error: error ?? 'Uthana job failed' }162+ default:163+ // RESERVED (a worker took the job) and any other running state.164+ return { jobId, status: 'processing' }165+ }166+ }167+168+ async getMotion(169+ jobId: string,170+ options?: { abortSignal?: AbortSignal },171+ ): Promise<MotionGenerationResult> {172+ const job = await this.readJob(jobId, options?.abortSignal)173+ const motionId = job.result?.result?.id174+ if (job.status !== 'FINISHED' || !motionId) {175+ throw new Error(176+ `Uthana job ${jobId} has no motion (status ${job.status}${jobError(job) ? `: ${jobError(job)}` : ''})`,- docs/tutorials/generate-motion.md line 374Outdated@@ -0,0 +354,21 @@354+Your engine reads this manifest. It retargets the clips, attaches the axe, and blends the clips. When the impact event fires, it applies damage, plays the reaction, and spawns the effects.355+356+## 6. Cancel a job on fal357+358+"Stop watching" does not stop the fal job. To stop it, the page asks your server to cancel it on fal. `byok.headers` sends the key in the `x-byok-fal` header:359+360+```tsx ignore361+async function cancelOnFal(jobId: string) {362+ const response = await fetch('/api/motion/cancel', {363+ method: 'POST',364+ headers: {365+ 'content-type': 'application/json',366+ ...byok.headers(falByok.id),367+ },368+ body: JSON.stringify({ jobId }),369+ })370+ if (!response.ok) throw new Error(await response.text())371+}372+```373+374+The clip panels and the page layout are in the [full `index.tsx`](https://github.com/TanStack/ai/blob/main/examples/react/generate-motion/src/routes/index.tsx). Each panel has a prompt box and three buttons: Generate, Stop watching, and Cancel on fal.examples/react/generate-motion/scripts/generate-orc-clips.ts line 266Outdated@@ -0,0 +246,21 @@246+ const { only, out, set } = parseArgs(process.argv.slice(2))247+ await mkdir(out, { recursive: true })248+ // --set ab runs the prompt/guidance/seed grid in ab-clips.ts.249+ const all =250+ set === 'ab'251+ ? (await import('./ab-clips.ts')).AB_CLIPS252+ : set === 'uthana'253+ ? (await import('./uthana-clips.ts')).UTHANA_CLIPS254+ : CLIPS255+ const clips = only ? all.filter((c) => only.includes(c.name)) : all256+257+ // Keep earlier records when only some clips are regenerated.258+ const manifestPath = join(out, 'manifest.json')259+ const previous: Record<string, ClipRecord> = await readFile(260+ manifestPath,261+ 'utf8',262+ )263+ .then((text) => JSON.parse(text).clips as Record<string, ClipRecord>)264+ .catch(() => ({}))265+266+ const records = await Promise.all(clips.map((spec) => runClip(spec, out)))examples/react/generate-motion/src/routes/index.tsx line 67Outdated@@ -0,0 +47,21 @@47+ onChunk: (chunk) => {48+ if (chunk.type !== 'CUSTOM') return49+ if (chunk.name === 'motion:job:created' && hasJobId(chunk.value)) {50+ setJobId(chunk.value.jobId)51+ }52+ if (chunk.name === 'motion:status' && isMotionStatus(chunk.value)) {53+ setJobStatus(chunk.value)54+ }55+ },56+ })57+58+ return {59+ ...generation,60+ jobId,61+ jobStatus,62+ stopped,63+ generate: (input: MotionInput) => {64+ setJobId(null)65+ setJobStatus(null)66+ setStopped(false)67+ return generation.generate(input)packages/ai/src/activities/generateMotion/index.ts lines 302 to 303Outdated@@ -0,0 +283,21 @@283+284+ aiEventClient.emit('motion:request:started', {285+ ...eventBase,286+ prompt,287+ timestamp: startTime,288+ ...(modelOptions !== undefined && {289+ modelOptions: modelOptions as Record<string, unknown>,290+ }),291+ })292+ logger.request(293+ `activity=generateMotion provider=${providerNameOf(adapter)}`,294+ {295+ provider: providerNameOf(adapter),296+ model,297+ },298+ )299+300+ let job: MotionJobResult301+ try {302+ job = await raceWithAbort(303+ adapter.createMotionJob({packages/ai/src/activities/generateMotion/index.ts line 313Outdated@@ -0,0 +293,21 @@293+ `activity=generateMotion provider=${providerNameOf(adapter)}`,294+ {295+ provider: providerNameOf(adapter),296+ model,297+ },298+ )299+300+ let job: MotionJobResult301+ try {302+ job = await raceWithAbort(303+ adapter.createMotionJob({304+ model,305+ prompt,306+ ...(duration !== undefined ? { duration } : {}),307+ ...(modelOptions !== undefined ? { modelOptions } : {}),308+ logger,309+ ...(abortControls.signal ? { abortSignal: abortControls.signal } : {}),310+ }),311+ abortControls.signal,312+ )313+ abortControls.clear()packages/ai/src/activities/generateMotion/index.ts lines 453 to 455Outdated@@ -0,0 +435,21 @@435+ ...(duration !== undefined ? { duration } : {}),436+ ...(modelOptions !== undefined ? { modelOptions } : {}),437+ logger,438+ ...(abortControls.signal ? { abortSignal: abortControls.signal } : {}),439+ }),440+ abortControls.signal,441+ )442+ jobId = job.jobId443+444+ yield {445+ type: 'CUSTOM',446+ name: MOTION_EVENTS.JOB_CREATED,447+ value: { jobId: job.jobId, model: job.model },448+ timestamp: Date.now(),449+ }450+451+ const pollStart = Date.now()452+ let first = true453+ while (Date.now() - pollStart < maxDuration) {454+ // Poll at once: a provider that finished synchronously needs no wait.455+ if (!first) await sleep(pollingInterval, abortControls.signal)packages/ai/src/activities/generateMotion/index.ts line 730Outdated@@ -0,0 +710,21 @@710+ signal,711+ )712+ } catch (error) {713+ // A cancelled call says nothing about the job.714+ if (signal?.aborted) throw error715+ // Completed but the result cannot be read: the job is terminal, so716+ // close the run as failed.717+ await runGenerationError(middleware, terminalContext(), {718+ error,719+ duration: Date.now() - startTime,720+ })721+ const message =722+ error instanceof Error ? error.message : 'Failed to fetch motion'723+ aiEventClient.emit('motion:request:completed', {724+ ...eventBase,725+ status: 'failed',726+ error: message,727+ duration: Date.now() - startTime,728+ timestamp: Date.now(),729+ })730+ return { ...base, status: 'failed', error: message }
AlemTuzlak pushed7746977
tanstack-source reviewed
packages/ai-uthana/src/adapters/motion.ts line 212Outdated@@ -0,0 +192,21 @@192+ case 'FAILED':193+ return { jobId, status: 'failed', error: error ?? 'Uthana job failed' }194+ default:195+ // RESERVED (a worker took the job) and any other running state.196+ return { jobId, status: 'processing' }197+ }198+ }199+200+ async getMotion(201+ jobId: string,202+ options?: { abortSignal?: AbortSignal },203+ ): Promise<MotionGenerationResult> {204+ const { id, options: jobOptions } = decodeJobId(jobId)205+ const job = await this.readJob(id, options?.abortSignal)206+ const motionId = job.result?.result?.id207+ if (job.status !== 'FINISHED' || !motionId) {208+ throw new Error(209+ `Uthana job ${jobId} has no motion (status ${job.status}${jobError(job) ? `: ${jobError(job)}` : ''})`,210+ )211+ }212+ const opts = { ...this.defaults, ...jobOptions }testing/e2e/tests/motion-generation.spec.ts line 262Outdated@@ -0,0 +242,21 @@242+ testId,243+ }) => {244+ const res = await request.post('/api/motion', {245+ data: {246+ threadId: `uthana-thread-${testId}`,247+ runId: `uthana-run-${testId}`,248+ state: {},249+ messages: [],250+ tools: [],251+ context: [],252+ forwardedProps: { prompt: PROMPT, mode: 'stream', provider: 'uthana' },253+ },254+ })255+ expect(res.ok()).toBe(true)256+ const events = parseSse(await res.text())257+ const custom = customEvents(events)258+259+ expect(custom.filter((e) => e.name === 'motion:status')).toEqual([260+ {261+ name: 'motion:status',262+ value: { jobId: 'uthana-job', status: 'pending' },examples/react/generate-motion/src/routes/index.tsx line 63Outdated@@ -0,0 +43,21 @@43+ const generation = useGeneration<MotionInput, MotionGenerationResult>({44+ connection: fetchServerSentEvents('/api/generate/motion'),45+ threadId,46+ byok,47+ onChunk: (chunk) => {48+ if (chunk.type !== 'CUSTOM') return49+ if (chunk.name === 'motion:job:created' && hasJobId(chunk.value)) {50+ setJobId(chunk.value.jobId)51+ }52+ if (chunk.name === 'motion:status' && isMotionStatus(chunk.value)) {53+ setJobStatus(chunk.value)54+ }55+ },56+ })57+58+ return {59+ ...generation,60+ jobId,61+ jobStatus,62+ stopped,63+ generate: (input: MotionInput) => {- examples/react/generate-motion/scripts/generate-orc-clips.ts line 261Outdated@@ -0,0 +241,21 @@241+ } else {242+ console.log(`[${spec.name}] FAILED: ${record.error}`)243+ }244+ return record245+}246+247+async function runClips(248+ clips: Array<ClipSpec>,249+ out: string,250+): Promise<Array<ClipRecord>> {251+ const records: Array<ClipRecord> = new Array(clips.length)252+ let next = 0253+ const workers = Math.min(CLIP_CONCURRENCY, clips.length)254+ await Promise.all(255+ Array.from({ length: workers }, async () => {256+ while (next < clips.length) {257+ const index = next258+ next += 1259+ const spec = clips[index]260+ if (!spec) continue261+ records[index] = await runClip(spec, out)packages/ai-uthana/src/adapters/motion.ts line 212Outdated@@ -0,0 +192,21 @@192+ case 'FAILED':193+ return { jobId, status: 'failed', error: error ?? 'Uthana job failed' }194+ default:195+ // RESERVED (a worker took the job) and any other running state.196+ return { jobId, status: 'processing' }197+ }198+ }199+200+ async getMotion(201+ jobId: string,202+ options?: { abortSignal?: AbortSignal },203+ ): Promise<MotionGenerationResult> {204+ const { id, options: jobOptions } = decodeJobId(jobId)205+ const job = await this.readJob(id, options?.abortSignal)206+ const motionId = job.result?.result?.id207+ if (job.status !== 'FINISHED' || !motionId) {208+ throw new Error(209+ `Uthana job ${jobId} has no motion (status ${job.status}${jobError(job) ? `: ${jobError(job)}` : ''})`,210+ )211+ }212+ const opts = { ...this.defaults, ...jobOptions }
Sign in to join this conversation.
🦋 Changeset detected#
Latest commit: bde858900f3df5aaba2174dffd04087ed108de43
The changes in this PR will be included in the next version bump.
This PR includes changesets to release 4 packages
Not sure what this means? Click here to learn what changesets are.
Click here if you're a maintainer who wants to add another changeset to this PR