1import {
2 type ListTextGeneratorModelsInput,
3 type ListTextGeneratorModelsResult,
4 type ProviderInstanceId,
5 TextGenerationError,
6 type TextGeneratorConfig,
7 TextGeneratorError,
8 type TextGeneratorModel,
9 textGeneratorForSelection,
10} from "@t3tools/contracts";
11import * as TextGenerationOperations from "@t3tools/provider-core/server/textGenerationOperations";
12import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver";
13import * as Context from "effect/Context";
14import * as Effect from "effect/Effect";
15import * as Equal from "effect/Equal";
16import * as Layer from "effect/Layer";
17import * as Ref from "effect/Ref";
18import { FetchHttpClient, HttpClient } from "effect/http";
19
20import * as SecretResolverLayer from "../secrets/SecretResolver.ts";
21import { requireSettingsSecret } from "../secrets/settingsSecretValue.ts";
22import * as ServerSettings from "../serverSettings.ts";
23import { SECRET_REDACTED } from "../settingsSecrets.ts";
24import {
25 TEXT_GENERATOR_DRIVERS,
26 type TextGeneratorConnection,
27 TextGeneratorRequestError,
28} from "./textGeneratorDrivers.ts";
29
30/** Text generators configured in settings: model APIs that generate text but cannot run agents. */
31export class TextGenerators extends Context.Service<
32 TextGenerators,
33 {
34 /** The runner for a text generator id, or undefined when the id names no text generator. */
35 readonly runner: (
36 id: ProviderInstanceId,
37 ) => Effect.Effect<TextGenerationOperations.Runner | undefined>;
38 readonly listModels: (
39 input: ListTextGeneratorModelsInput,
40 ) => Effect.Effect<ListTextGeneratorModelsResult, TextGeneratorError>;
41 }
42>()("t3/textGeneration/TextGenerators") {}
43
44interface CachedModels {
45 readonly config: TextGeneratorConfig;
46 readonly models: ReadonlyArray<TextGeneratorModel>;
47}
48
49/** The generator's name for messages: its label, else the host it reaches. */
50const generatorLabel = (config: TextGeneratorConfig) => {
51 if (config.label) return config.label;
52 try {
53 return new URL(config.baseUrl).host;
54 } catch {
55 return "The model API";
56 }
57};
58
59/** @public */
60export const make = Effect.gen(function* () {
61 const settingsService = yield* ServerSettings.ServerSettingsService;
62 const secrets = yield* SecretResolver.SecretResolver;
63 const http = yield* HttpClient.HttpClient;
64 const modelCache = yield* Ref.make(new Map<ProviderInstanceId, CachedModels>());
65 // Settings already drop saved generators under an allow-list; this stops unsaved ones too.
66 const allowListed =
67 settingsService.managedPolicy?.limits?.providers?.allowedDrivers !== undefined;
68
69 const readConfig = Effect.fn("TextGenerators.readConfig")(function* (id: ProviderInstanceId) {
70 const settings = yield* settingsService.getSettings.pipe(
71 Effect.mapError(
72 () =>
73 new TextGeneratorRequestError({ detail: "Couldn't read this environment's settings." }),
74 ),
75 );
76 return textGeneratorForSelection(settings, id);
77 });
78
79 const connect = Effect.fn("TextGenerators.connect")(function* (config: TextGeneratorConfig) {
80 const label = generatorLabel(config);
81 const apiKey = yield* requireSettingsSecret(secrets, config.apiKey).pipe(
82 Effect.mapError((cause) => new TextGeneratorRequestError({ detail: cause.message, cause })),
83 );
84 return { baseUrl: config.baseUrl, apiKey, label } satisfies TextGeneratorConnection;
85 });
86
87 const runnerFor =
88 (config: TextGeneratorConfig): TextGenerationOperations.Runner =>
89 (request) => {
90 const driver = TEXT_GENERATOR_DRIVERS[config.kind];
91 const label = generatorLabel(config);
92 return connect(config).pipe(
93 Effect.flatMap((connection) =>
94 driver.generate(connection, request.modelSelection.model, request.prompt),
95 ),
96 Effect.provideService(HttpClient.HttpClient, http),
97 Effect.mapError(
98 (error) =>
99 new TextGenerationError({
100 operation: request.operation,
101 detail: error.detail,
102 cause: error,
103 }),
104 ),
105 Effect.flatMap((text) => TextGenerationOperations.decodeJsonReply(request, label, text)),
106 );
107 };
108
109 const runner = Effect.fn("TextGenerators.runner")(function* (id: ProviderInstanceId) {
110 const config = yield* readConfig(id).pipe(Effect.orElseSucceed(() => undefined));
111 if (config === undefined) return undefined;
112 const label = generatorLabel(config);
113 if (!config.enabled) {
114 return ((request) =>
115 Effect.fail(
116 new TextGenerationError({
117 operation: request.operation,
118 detail: `${label} is off. Turn it on in Settings → Providers.`,
119 }),
120 )) satisfies TextGenerationOperations.Runner;
121 }
122 return TextGenerationOperations.withoutInvestigation(label, runnerFor(config));
123 });
124
125 const listModels = Effect.fn("TextGenerators.listModels")(
126 function* (input: ListTextGeneratorModelsInput) {
127 if (allowListed) {
128 return yield* new TextGeneratorRequestError({
129 detail: "Your organization's provider policy turns off text generators.",
130 });
131 }
132 const saved = yield* readConfig(input.id);
133 const config =
134 input.config === undefined
135 ? saved
136 : input.config.apiKey === SECRET_REDACTED
137 ? { ...input.config, apiKey: saved?.apiKey ?? "" }
138 : input.config;
139 if (config === undefined) {
140 return yield* new TextGeneratorRequestError({
141 detail: "That text generator no longer exists.",
142 });
143 }
144 // Checking unsaved settings, or testing, must reach the API rather than trust the cache.
145 const fresh = input.config !== undefined || input.test === true;
146 const cached = (yield* Ref.get(modelCache)).get(input.id);
147 // Keyed on the whole config so a changed URL or key never serves another API's list.
148 if (
149 !fresh &&
150 input.refresh !== true &&
151 cached !== undefined &&
152 Equal.equals(cached.config, config)
153 ) {
154 return { models: cached.models };
155 }
156 const connection = yield* connect(config);
157 const driver = TEXT_GENERATOR_DRIVERS[config.kind];
158 const models = yield* driver
159 .listModels(connection)
160 .pipe(Effect.provideService(HttpClient.HttpClient, http));
161 if (input.test === true) {
162 // An explicit test sends one tiny request: some APIs list models without checking the key.
163 const model =
164 input.model ??
165 (models.find((candidate) => /haiku|mini|flash|nano/i.test(candidate.id)) ?? models[0])
166 ?.id;
167 if (model === undefined) {
168 return yield* new TextGeneratorRequestError({
169 detail: `${connection.label} lists no models to test with.`,
170 });
171 }
172 yield* driver
173 .generate(connection, model, "Reply with OK.")
174 .pipe(Effect.provideService(HttpClient.HttpClient, http));
175 return { models, testedModel: model };
176 }
177 yield* Ref.update(modelCache, (current) =>
178 new Map(current).set(input.id, { config, models }),
179 );
180 return { models };
181 },
182 Effect.mapError((error) => new TextGeneratorError({ detail: error.detail })),
183 );
184
185 return TextGenerators.of({ runner, listModels });
186});
187
188export const layer = Layer.effect(TextGenerators, make).pipe(
189 Layer.provide([SecretResolverLayer.layer, FetchHttpClient.layer]),
190);