1import type { TextGeneratorKind, TextGeneratorModel } from "@t3tools/contracts";
2import * as Effect from "effect/Effect";
3import * as Schema from "effect/Schema";
4import {
5 FetchHttpClient,
6 HttpClient,
7 HttpClientRequest,
8 type HttpClientResponse,
9} from "effect/http";
10
11/** Where a text generator is reached, with its key already read from wherever settings keep it. */
12export interface TextGeneratorConnection {
13 readonly baseUrl: string;
14 readonly apiKey: string;
15 /** Names the generator in errors, such as its label or host. */
16 readonly label: string;
17}
18
19/** Why a model API call failed, worded for the person who configured it. */
20export class TextGeneratorRequestError extends Schema.TaggedError<TextGeneratorRequestError>()(
21 "TextGeneratorRequestError",
22 { detail: Schema.String, cause: Schema.optional(Schema.Defect()) },
23) {}
24
25export interface TextGeneratorDriver {
26 readonly kind: TextGeneratorKind;
27 readonly generate: (
28 connection: TextGeneratorConnection,
29 model: string,
30 prompt: string,
31 ) => Effect.Effect<string, TextGeneratorRequestError, HttpClient.HttpClient>;
32 readonly listModels: (
33 connection: TextGeneratorConnection,
34 ) => Effect.Effect<
35 ReadonlyArray<TextGeneratorModel>,
36 TextGeneratorRequestError,
37 HttpClient.HttpClient
38 >;
39}
40
41const GENERATE_TIMEOUT = "2 minutes";
42const LIST_MODELS_TIMEOUT = "25 seconds";
43const MAX_MODEL_PAGES = 10;
44
45const ApiErrorBody = Schema.Struct({
46 error: Schema.Struct({ message: Schema.String }),
47});
48const decodeApiErrorBody = Schema.decodeUnknownOption(ApiErrorBody);
49
50const describeStatus = (label: string, status: number, message: string | undefined) => {
51 const reason =
52 status === 401 || status === 403
53 ? "rejected the API key"
54 : status === 404
55 ? "couldn't find that model or endpoint"
56 : status === 429
57 ? "is rate-limiting requests"
58 : `returned HTTP ${status}`;
59 const detail = message?.trim().slice(0, 300);
60 return `${label} ${reason}${detail ? `: ${detail}` : "."}`;
61};
62
63const endpoint = (connection: TextGeneratorConnection, path: string) =>
64 `${connection.baseUrl.replace(/\/+$/, "")}${path}`;
65
66/** Sends a request and decodes a 2xx JSON body; anything else becomes a readable error. */
67const sendJson = Effect.fn("TextGenerator.sendJson")(function* <S extends Schema.Top>(
68 label: string,
69 request: HttpClientRequest.HttpClientRequest,
70 schema: S,
71) {
72 const http = yield* HttpClient.HttpClient;
73 const response: HttpClientResponse.HttpClientResponse = yield* http.execute(request).pipe(
74 // A redirect could carry the key to another origin.
75 Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }),
76 Effect.provideService(HttpClient.TracerPropagationEnabled, false),
77 Effect.mapError(
78 (cause) =>
79 new TextGeneratorRequestError({
80 detail: `Couldn't reach ${label}. Check the base URL and your connection.`,
81 cause,
82 }),
83 ),
84 );
85 const body = yield* response.json.pipe(Effect.orElseSucceed((): unknown => undefined));
86 if (response.status < 200 || response.status >= 300) {
87 const message = decodeApiErrorBody(body).pipe((option) =>
88 option._tag === "Some" ? option.value.error.message : undefined,
89 );
90 return yield* new TextGeneratorRequestError({
91 detail: describeStatus(label, response.status, message),
92 });
93 }
94 return yield* Schema.decodeUnknownEffect(schema)(body).pipe(
95 Effect.mapError(
96 (cause) =>
97 new TextGeneratorRequestError({
98 detail: `${label} returned a reply T3 Code couldn't read.`,
99 cause,
100 }),
101 ),
102 );
103});
104
105const withTimeout =
106 (label: string, duration: typeof GENERATE_TIMEOUT | typeof LIST_MODELS_TIMEOUT) =>
107 <A, R>(effect: Effect.Effect<A, TextGeneratorRequestError, R>) =>
108 effect.pipe(
109 Effect.timeoutOrElse({
110 duration,
111 orElse: () =>
112 Effect.fail(
113 new TextGeneratorRequestError({ detail: `${label} took too long to respond.` }),
114 ),
115 }),
116 );
117
118const sortModels = (models: Iterable<TextGeneratorModel>) =>
119 [...models].sort((left, right) => left.id.localeCompare(right.id));
120
121const readModel = (entry: { readonly id: string; readonly name?: string | undefined }) => {
122 const id = entry.id.trim();
123 return id ? { id, name: entry.name?.trim() || id } : undefined;
124};
125
126const ChatCompletion = Schema.Struct({
127 choices: Schema.Array(
128 Schema.Struct({
129 finish_reason: Schema.NullOr(Schema.String),
130 message: Schema.Struct({ content: Schema.NullOr(Schema.String) }),
131 }),
132 ),
133});
134const OpenAIModelList = Schema.Struct({
135 data: Schema.Array(Schema.Struct({ id: Schema.String, name: Schema.optional(Schema.String) })),
136});
137
138const openAICompatible: TextGeneratorDriver = {
139 kind: "openaiCompatible",
140 generate: (connection, model, prompt) => {
141 const { label } = connection;
142 let request = HttpClientRequest.post(endpoint(connection, "/chat/completions")).pipe(
143 HttpClientRequest.bodyJsonUnsafe({ model, messages: [{ role: "user", content: prompt }] }),
144 );
145 if (connection.apiKey) request = HttpClientRequest.bearerToken(request, connection.apiKey);
146 return sendJson(label, request, ChatCompletion).pipe(
147 Effect.flatMap((completion) => {
148 const choice = completion.choices[0];
149 const text = choice?.message.content?.trim();
150 if (choice?.finish_reason === "length") {
151 return Effect.fail(
152 new TextGeneratorRequestError({ detail: `${label} reached its output limit.` }),
153 );
154 }
155 return text
156 ? Effect.succeed(text)
157 : Effect.fail(new TextGeneratorRequestError({ detail: `${label} returned no text.` }));
158 }),
159 withTimeout(label, GENERATE_TIMEOUT),
160 );
161 },
162 listModels: (connection) => {
163 const { label } = connection;
164 let request = HttpClientRequest.get(endpoint(connection, "/models"));
165 if (connection.apiKey) request = HttpClientRequest.bearerToken(request, connection.apiKey);
166 return sendJson(label, request, OpenAIModelList).pipe(
167 Effect.map((list) => sortModels(list.data.flatMap((entry) => readModel(entry) ?? []))),
168 withTimeout(label, LIST_MODELS_TIMEOUT),
169 );
170 },
171};
172
173const ANTHROPIC_VERSION = "2023-06-01";
174const AnthropicMessage = Schema.Struct({
175 stop_reason: Schema.NullOr(Schema.String),
176 content: Schema.Array(
177 Schema.Struct({ type: Schema.String, text: Schema.optional(Schema.String) }),
178 ),
179});
180const AnthropicModelPage = Schema.Struct({
181 data: Schema.Array(
182 Schema.Struct({ id: Schema.String, display_name: Schema.optional(Schema.String) }),
183 ),
184 has_more: Schema.optional(Schema.Boolean),
185 last_id: Schema.optional(Schema.NullOr(Schema.String)),
186});
187
188const anthropicRequest = (request: HttpClientRequest.HttpClientRequest, apiKey: string) =>
189 request.pipe(
190 HttpClientRequest.setHeader("x-api-key", apiKey),
191 HttpClientRequest.setHeader("anthropic-version", ANTHROPIC_VERSION),
192 );
193
194const anthropic: TextGeneratorDriver = {
195 kind: "anthropic",
196 generate: (connection, model, prompt) => {
197 const { label } = connection;
198 const request = anthropicRequest(
199 HttpClientRequest.post(endpoint(connection, "/messages")).pipe(
200 HttpClientRequest.bodyJsonUnsafe({
201 model,
202 max_tokens: 8192,
203 messages: [{ role: "user", content: prompt }],
204 }),
205 ),
206 connection.apiKey,
207 );
208 return sendJson(label, request, AnthropicMessage).pipe(
209 Effect.flatMap((message) => {
210 if (message.stop_reason === "max_tokens") {
211 return Effect.fail(
212 new TextGeneratorRequestError({ detail: `${label} reached its output limit.` }),
213 );
214 }
215 const text = message.content
216 .flatMap((block) => (block.type === "text" && block.text ? [block.text] : []))
217 .join("")
218 .trim();
219 return text
220 ? Effect.succeed(text)
221 : Effect.fail(new TextGeneratorRequestError({ detail: `${label} returned no text.` }));
222 }),
223 withTimeout(label, GENERATE_TIMEOUT),
224 );
225 },
226 listModels: Effect.fn("TextGenerator.anthropic.listModels")(
227 function* (connection) {
228 const { label } = connection;
229 const models = new Map<string, TextGeneratorModel>();
230 let after: string | undefined;
231 for (let page = 0; page < MAX_MODEL_PAGES; page++) {
232 let request = HttpClientRequest.get(endpoint(connection, "/models")).pipe(
233 HttpClientRequest.setUrlParam("limit", "1000"),
234 );
235 if (after) request = HttpClientRequest.setUrlParam(request, "after_id", after);
236 const result = yield* sendJson(
237 label,
238 anthropicRequest(request, connection.apiKey),
239 AnthropicModelPage,
240 );
241 for (const entry of result.data) {
242 const model = readModel({ id: entry.id, name: entry.display_name });
243 if (model) models.set(model.id, model);
244 }
245 if (!result.has_more || !result.last_id || result.last_id === after) break;
246 after = result.last_id;
247 }
248 return sortModels(models.values());
249 },
250 (effect, connection) => effect.pipe(withTimeout(connection.label, LIST_MODELS_TIMEOUT)),
251 ),
252};
253
254export const TEXT_GENERATOR_DRIVERS: Readonly<Record<TextGeneratorKind, TextGeneratorDriver>> = {
255 openaiCompatible: openAICompatible,
256 anthropic,
257};