1import {
2 CloudSpeechApiKey,
3 CloudSpeechError,
4 CloudSpeechModel,
5 CloudSpeechModelId,
6 SettingsSecretValue,
7 isSecretReference,
8 type SecretReference,
9 type ServerSettingsError,
10 type CloudSpeechKeys,
11 type CloudSpeechProvider,
12 EnvironmentSpeechTranscriptionResult,
13 type SpeechTranscriptionOptions,
14} from "@t3tools/contracts";
15import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver";
16import { cloudSpeechLanguages, effectiveSpeechLanguage } from "@t3tools/shared/speech";
17import * as Context from "effect/Context";
18import * as Effect from "effect/Effect";
19import * as Exit from "effect/Exit";
20import * as Layer from "effect/Layer";
21import * as Redacted from "effect/Redacted";
22import * as Schema from "effect/Schema";
23import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
24import * as ServerSettings from "../serverSettings.ts";
25import { requireSettingsSecret } from "../secrets/settingsSecretValue.ts";
26import { applySpeechAliases } from "./customWords.ts";
27import { removeSpeechFillerWords } from "./fillerWords.ts";
28import { decodeSpeechPcm, SpeechInvalidAudioError } from "./SpeechService.ts";
29
30const endpoints = {
31 openai: { url: "https://api.openai.com/v1/audio/transcriptions", model: "gpt-transcribe" },
32 xai: { url: "https://api.x.ai/v1/stt", model: "grok-voice-transcribe-2.0" },
33 openrouter: { url: "https://openrouter.ai/api/v1/audio/transcriptions" },
34};
35const OPENROUTER_CATALOG_URL =
36 "https://openrouter.ai/api/v1/models?output_modalities=transcription&zdr=true";
37const isApiKey = Schema.is(CloudSpeechApiKey);
38const isModelId = Schema.is(CloudSpeechModelId);
39const sameCredential = Schema.toEquivalence(SettingsSecretValue);
40const OpenRouterCatalog = Schema.Struct({
41 data: Schema.Array(Schema.Struct({ id: Schema.String, name: Schema.optionalKey(Schema.String) })),
42});
43
44function wave(samples: Float32Array) {
45 const buffer = new ArrayBuffer(44 + samples.length * 2);
46 const view = new DataView(buffer);
47 const tag = (at: number, value: string) =>
48 new Uint8Array(buffer, at, 4).set(new TextEncoder().encode(value));
49 tag(0, "RIFF");
50 view.setUint32(4, buffer.byteLength - 8, true);
51 tag(8, "WAVE");
52 tag(12, "fmt ");
53 view.setUint32(16, 16, true);
54 view.setUint16(20, 1, true);
55 view.setUint16(22, 1, true);
56 view.setUint32(24, 16_000, true);
57 view.setUint32(28, 32_000, true);
58 view.setUint16(32, 2, true);
59 view.setUint16(34, 16, true);
60 tag(36, "data");
61 view.setUint32(40, buffer.byteLength - 44, true);
62 for (let index = 0; index < samples.length; index++) {
63 const sample = samples[index]!;
64 view.setInt16(44 + index * 2, Math.round(sample * (sample < 0 ? 32768 : 32767)), true);
65 }
66 return buffer;
67}
68
69function audioBody(
70 provider: "openai" | "xai",
71 audio: ArrayBuffer,
72 language: string,
73 options: SpeechTranscriptionOptions,
74) {
75 const form = new FormData();
76 form.set("model", endpoints[provider].model);
77 if (language !== "auto") form.set(provider === "openai" ? "languages[]" : "language", language);
78 for (const { term } of options.speechCustomWords) {
79 if (provider === "openai" && /[<>\r\n]/.test(term)) continue;
80 form.append(provider === "openai" ? "keywords[]" : "keyterm", term);
81 }
82 if (provider === "xai") {
83 if (language !== "auto") form.set("format", "true");
84 form.set(
85 "filler_words",
86 String(!options.speechRemoveFillerWords || Boolean(options.speechCorrectionWord)),
87 );
88 if (!options.speechSilenceFiltering) form.set("vad_threshold", "0");
89 }
90 // xAI ignores options that follow the file part.
91 form.set("file", new Blob([audio], { type: "audio/wav" }), "dictation.wav");
92 return HttpClientRequest.bodyFormData(form);
93}
94
95function openRouterBody(model: string, audio: ArrayBuffer, language: string) {
96 return HttpClientRequest.bodyJsonUnsafe({
97 model,
98 input_audio: { data: Buffer.from(audio).toString("base64"), format: "wav" },
99 ...(language !== "auto" ? { language } : {}),
100 provider: { zdr: true },
101 });
102}
103
104export class CloudSpeech extends Context.Service<
105 CloudSpeech,
106 {
107 readonly configured: Effect.Effect<CloudSpeechKeys, ServerSettingsError>;
108 /** OpenRouter's public transcription catalog, cached briefly on this server. */
109 readonly models: Effect.Effect<ReadonlyArray<CloudSpeechModel>, CloudSpeechError>;
110 readonly setKey: (
111 provider: CloudSpeechProvider,
112 key: Redacted.Redacted<string> | SecretReference | null,
113 ) => Effect.Effect<void, ServerSettingsError>;
114 readonly transcribe: (
115 provider: CloudSpeechProvider,
116 pcm: Uint8Array,
117 options: SpeechTranscriptionOptions,
118 model?: string | undefined,
119 ) => Effect.Effect<string, CloudSpeechError | SpeechInvalidAudioError | ServerSettingsError>;
120 }
121>()("t3/speech/CloudSpeech") {}
122
123export const layer = Layer.effect(
124 CloudSpeech,
125 Effect.gen(function* () {
126 const settings = yield* ServerSettings.ServerSettingsService;
127 const resolver = yield* SecretResolver.SecretResolver;
128 const http = yield* HttpClient.HttpClient;
129 const credentials = settings.getSettings.pipe(
130 Effect.map((value) => value.cloudSpeechCredentials),
131 );
132 const configured = credentials.pipe(
133 Effect.map(({ openai, xai, openrouter }) => ({
134 openai: Boolean(openai),
135 xai: Boolean(xai),
136 openrouter: Boolean(openrouter),
137 })),
138 );
139 const execute = (request: HttpClientRequest.HttpClientRequest) =>
140 http
141 .execute(request)
142 .pipe(
143 Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }),
144 Effect.provideService(HttpClient.TracerPropagationEnabled, false),
145 );
146 const fetchCatalog = Effect.fn("CloudSpeech.catalog")(function* () {
147 const unavailable = () =>
148 new CloudSpeechError({ provider: "openrouter", reason: "unavailable" });
149 const response = yield* execute(HttpClientRequest.get(OPENROUTER_CATALOG_URL)).pipe(
150 Effect.mapError(unavailable),
151 );
152 if (response.status < 200 || response.status >= 300) return yield* unavailable();
153 const { data } = yield* HttpClientResponse.schemaBodyJson(OpenRouterCatalog)(response).pipe(
154 Effect.mapError(unavailable),
155 );
156 return data.flatMap(({ id, name }) =>
157 isModelId(id) ? [{ id, name: name?.trim() || id }] : [],
158 );
159 });
160 // The catalog is public, so the cache holds no credential or consent state.
161 const models = yield* Effect.cachedWithTTL(
162 fetchCatalog().pipe(
163 Effect.timeoutOrElse({
164 duration: "10 seconds",
165 orElse: () =>
166 Effect.fail(new CloudSpeechError({ provider: "openrouter", reason: "unavailable" })),
167 }),
168 ),
169 (exit) => (Exit.isSuccess(exit) ? "10 minutes" : 0),
170 );
171 const setKey = Effect.fn("CloudSpeech.setKey")(function* (
172 provider: CloudSpeechProvider,
173 key: Redacted.Redacted<string> | SecretReference | null,
174 ) {
175 yield* settings.updateSettings({
176 cloudSpeechCredentials: {
177 [provider]: Redacted.isRedacted(key) ? Redacted.value(key) : (key ?? ""),
178 },
179 });
180 });
181 const transcribe = Effect.fn("CloudSpeech.transcribe")(
182 function* (
183 provider: CloudSpeechProvider,
184 pcm: Uint8Array,
185 options: SpeechTranscriptionOptions,
186 model?: string,
187 ) {
188 const source = (yield* credentials)[provider];
189 if (!source) return yield* new CloudSpeechError({ provider, reason: "missing-key" });
190 if (provider === "openrouter" && (!model || !isModelId(model)))
191 return yield* new CloudSpeechError({ provider, reason: "model-unavailable" });
192 const samples = yield* Effect.try({
193 try: () => decodeSpeechPcm(pcm),
194 catch: () =>
195 new SpeechInvalidAudioError({
196 byteLength: pcm.byteLength,
197 message: "Invalid PCM audio.",
198 }),
199 });
200 const language = effectiveSpeechLanguage(cloudSpeechLanguages, options.speechLanguage);
201 if (provider === "openrouter" && !(yield* models).some((entry) => entry.id === model))
202 return yield* new CloudSpeechError({ provider, reason: "model-unavailable" });
203 // A vault that locked since the last recording must not authorize a new upload.
204 if (isSecretReference(source)) yield* resolver.invalidate([source]);
205 const unreadable = () => new CloudSpeechError({ provider, reason: "unreadable-key" });
206 const key = yield* requireSettingsSecret(resolver, source).pipe(
207 Effect.mapError(unreadable),
208 );
209 if (!isApiKey(key)) return yield* unreadable();
210 const audio = wave(samples);
211 const request = HttpClientRequest.post(endpoints[provider].url).pipe(
212 HttpClientRequest.bearerToken(Redacted.make(key)),
213 provider === "openrouter"
214 ? openRouterBody(model!, audio, language)
215 : audioBody(provider, audio, language, options),
216 );
217 const unavailable = () => new CloudSpeechError({ provider, reason: "unavailable" });
218 // A catalog fetch or vault lookup may have outlived the credential it started with.
219 const current = (yield* credentials)[provider];
220 if (!current || !sameCredential(source, current))
221 return yield* new CloudSpeechError({
222 provider,
223 reason: current ? "key-changed" : "missing-key",
224 });
225 const response = yield* execute(request).pipe(Effect.mapError(unavailable));
226 if (response.status < 200 || response.status >= 300)
227 return yield* new CloudSpeechError({
228 provider,
229 reason:
230 response.status === 429
231 ? "rate-limit"
232 : [401, 402, 403].includes(response.status)
233 ? "rejected-key"
234 : response.status === 404 && provider === "openrouter"
235 ? "model-unavailable"
236 : "unavailable",
237 });
238 const { text } = yield* HttpClientResponse.schemaBodyJson(
239 EnvironmentSpeechTranscriptionResult,
240 )(response).pipe(Effect.mapError(unavailable));
241 const corrected = applySpeechAliases(text, options.speechCustomWords);
242 return (
243 options.speechRemoveFillerWords
244 ? removeSpeechFillerWords(corrected, language, options.speechCorrectionWord)
245 : corrected
246 ).trim();
247 },
248 (effect, provider) =>
249 effect.pipe(
250 Effect.timeoutOrElse({
251 duration: "2 minutes",
252 orElse: () => Effect.fail(new CloudSpeechError({ provider, reason: "unavailable" })),
253 }),
254 ),
255 );
256 return CloudSpeech.of({ configured, models, setKey, transcribe });
257 }),
258);