Open History to see the last commit
1/**
2 * Run the live AWS suites for services that have a Floci local provider
3 * (`flociDual` / `ProviderLayer.dual` in Providers.ts) under
4 * `ALCHEMY_TEST_DEV=1`.
5 *
6 * Dedicated `*.local.test.ts` files are excluded — those already set
7 * `Test.make({ dev: true })` and some use `Alchemy.remote()` for out-of-band
8 * live checks, which this env would redirect onto the emulator. Suites for
9 * live-only resources (`liveOnlySuites`) are excluded too.
10 *
11 * Extra alchemy-test args are forwarded (`-t`, `--retry`, paths, …).
12 * `--list` prints selected files without starting tests or touching Floci.
13 * `--external` uses an existing server without Docker fallback or state reset.
14 * `--dry-run` prints the resolved command without starting tests or touching Floci.
15 * Shared state is preserved unless `--reset-shared` is explicitly passed.
16 * Standing prerequisites of gated suites (see floci-fixtures.ts) are
17 * provisioned in Floci before the run and removed after it.
18 *
19 * Usage: node scripts/test-aws-floci.ts [--list] [--dry-run] [--external] [--reset-shared] [alchemy-test args...]
20 */
21import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
22import * as NodeServices from "@effect/platform-node/NodeServices";
23import { Argument, Command, Flag } from "effect/cli";
24import * as Config from "effect/Config";
25import * as Console from "effect/Console";
26import * as Data from "effect/Data";
27import * as Effect from "effect/Effect";
28import * as FileSystem from "effect/FileSystem";
29import { FetchHttpClient } from "effect/http";
30import * as HttpClient from "effect/http/HttpClient";
31import * as Layer from "effect/Layer";
32import * as Path from "effect/Path";
33import { ChildProcess } from "effect/process";
34import { ChildProcessSpawner } from "effect/process/ChildProcessSpawner";
35import * as Runtime from "effect/Runtime";
36import * as Stdio from "effect/Stdio";
37import { startFlociForTests } from "./floci-container.ts";
38import { provisionFlociStandingFixtures } from "./floci-fixtures.ts";
39import { preferLocalFlociImage } from "./floci-image.ts";
40import { resolveFlociDefaultNetwork } from "./floci-network.ts";
41
42const LABEL = "test:aws:floci";
43
44/** Ends the run with `code` after the reason has already been printed. */
45class ExitWith extends Data.TaggedError("ExitWith")<{ readonly code: number }> {
46 readonly [Runtime.errorReported] = false;
47 get [Runtime.errorExitCode]() {
48 return this.code;
49 }
50}
51
52const fail = (message: string) =>
53 Console.error(`${LABEL}: ${message}`).pipe(Effect.andThen(new ExitWith({ code: 1 })));
54
55const errorMessage = (error: unknown) =>
56 error instanceof Error
57 ? error.message
58 : typeof error === "object" && error !== null && "message" in error
59 ? String(error.message)
60 : String(error);
61
62const extraDirs: Record<string, ReadonlyArray<string>> = {
63 SecretsManager: ["Secret"],
64};
65
66// Opt-in gates whose tests are gated only for live-AWS cost, quota, slow
67// provisioning, or preview access. Floci emulates all of them, so enable them
68// here unless the caller set them explicitly. Gates that need standing
69// resources are provisioned by floci-fixtures.ts.
70const flociOptInGates = [
71 "ALCHEMY_TEST_IDENTITY_CENTER",
72 "AWS_LAMBDA_TEST_SHUTDOWN",
73 "AWS_ORG_MANAGEMENT_ACCOUNT",
74 "AWS_TEST_APIGWV2_VPCLINK",
75 "AWS_TEST_APPSYNC_CACHE",
76 "AWS_TEST_CLOUDHSM",
77 "AWS_TEST_CLOUDTRAIL_LAKE",
78 "AWS_TEST_CLOUDMAP_PUBLIC",
79 "AWS_TEST_CODEARTIFACT_EVENTS",
80 "AWS_TEST_CONFIG_RECORDER",
81 "AWS_TEST_EFS_MULTI_AZ",
82 "AWS_TEST_METRICSTREAM",
83 "AWS_TEST_NAT_GATEWAY",
84 "AWS_TEST_NETWORKFIREWALL",
85 "AWS_TEST_ORG_MANAGEMENT",
86 "AWS_TEST_POLICY_STORE_ALIAS",
87 "AWS_TEST_RDS_DBCLUSTER",
88 "AWS_TEST_RDS_DBCLUSTER_ENDPOINT",
89 "AWS_TEST_RDS_DBINSTANCE",
90 "AWS_TEST_RDS_DBSUBNETGROUP",
91 "AWS_TEST_RE2_AGGREGATOR",
92 "AWS_TEST_REDSHIFT",
93 "AWS_TEST_REGION_OPT_IN",
94 "AWS_TEST_SAGEMAKER_HYPERPOD",
95 "AWS_TEST_SERVICE_QUOTAS",
96 "AWS_TEST_SES_DEDICATED_IP",
97 "AWS_TEST_SLOW",
98 "AWS_TEST_TRANSFER_TRANSITIONS",
99 "AWS_TEST_WAF_ASSOCIATION",
100 "AWS_TEST_WAF_CLOUDFRONT",
101 "CLOUDFRONT_TEST_VPC_ORIGIN",
102 "INSPECTOR2_TEST_CIS",
103 "INSPECTOR2_TEST_ENABLER",
104 "LAMBDA_TEST_MICROVM",
105 "LAMBDA_TEST_NETWORK_CONNECTOR",
106 "RDS_TEST_LIFECYCLE",
107];
108
109// Values that gated suites need but Floci accepts without standing resources
110// (checked against each suite and Floci's validation). Applied with `??=`.
111const flociFixedValues: Record<string, string> = {
112 // Distinct from the delegated-admin principal: registering a delegated
113 // admin enables trusted access for its principal (Floci OrganizationsService).
114 AWS_ORG_TRUSTED_SERVICE_PRINCIPAL: "ssm.amazonaws.com",
115 AWS_ORG_DELEGATED_ADMIN_SERVICE_PRINCIPAL: "config.amazonaws.com",
116 AWS_TEST_SES_MAIL_FROM_IDENTITY: "ses-mailfrom.alchemy-test.example.com",
117 AWS_TEST_SES_MAIL_FROM_DOMAIN: "bounce.ses-mailfrom.alchemy-test.example.com",
118 AWS_TEST_SNS_PLATFORM: "1",
119 AWS_TEST_SNS_PLATFORM_NAME: "GCM",
120 // Floci accepts GCM credentials of 24+ chars without "invalid"/"fake"/"probe".
121 AWS_TEST_SNS_PLATFORM_CREDENTIAL: "alchemy-floci-gcm-server-key-000000000000",
122 AWS_TEST_SNS_PLATFORM_TOKEN: "alchemy-floci-device-token-0001",
123 // Floci's AgentCore control plane records the artifact without pulling it.
124 AWS_TEST_AGENTCORE: "1",
125 AWS_TEST_AGENTCORE_IMAGE:
126 "000000000000.dkr.ecr.us-east-1.amazonaws.com/alchemy-floci-agentcore:latest",
127 // Floci stores authorizer URIs without resolving the function.
128 ALCHEMY_TEST_AUTHORIZER_URI:
129 "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:000000000000:function:alchemy-floci-standing-authorizer/invocations",
130};
131
132/**
133 * Suites inside a dualized service folder whose resources stay live-only
134 * (registered without `flociDual` in Providers.ts). This runner points every
135 * AWS client at the emulator, so they cannot reach the AWS service they test.
136 */
137const liveOnlySuites = new Set([
138 // Floci does not emulate Lambda Web Functions.
139 "Lambda/WebFunction.test.ts",
140 "Lambda/WebFunctionEndpoint.test.ts",
141 "Lambda/WebFunctionPermission.test.ts",
142]);
143
144/** alchemy-test flags whose next argument is their value. */
145const flagsWithValue = new Set([
146 "-t",
147 "--test-name-pattern",
148 "--timeout",
149 "--retry",
150 "--concurrency",
151 "-c",
152 "--profile",
153]);
154
155/** Flags this runner owns; everything else is forwarded to alchemy-test. */
156const ownFlags = new Set([
157 "--list",
158 "--dry-run",
159 "--external",
160 "--reset-shared",
161 "--help",
162 "-h",
163 "--version",
164]);
165
166/**
167 * Moves forwarded alchemy-test args behind `--`, so the CLI parser takes them
168 * verbatim (`--concurrency=2`, `-t <pattern>`, paths) instead of rejecting
169 * flags it does not know.
170 */
171const splitArgs = (args: ReadonlyArray<string>) => {
172 const own: string[] = [];
173 const forwarded: string[] = [];
174 for (let i = 0; i < args.length; i++) {
175 const arg = args[i]!;
176 if (arg === "--") {
177 forwarded.push(...args.slice(i + 1));
178 break;
179 }
180 if (ownFlags.has(arg)) {
181 own.push(arg);
182 continue;
183 }
184 forwarded.push(arg);
185 if (flagsWithValue.has(arg) && args[i + 1] && !args[i + 1]!.startsWith("-")) {
186 forwarded.push(args[++i]!);
187 }
188 }
189 return [...own, "--", ...forwarded];
190};
191
192const dualizedServices = Effect.fn(function* (providersFile: string) {
193 const fs = yield* FileSystem.FileSystem;
194 const source = yield* fs.readFileString(providersFile);
195 const names = new Set<string>(["Local"]);
196 const pattern = /(?:flociDual|ProviderLayer\.dual)\(\s*([A-Za-z0-9]+)\./g;
197 for (const match of source.matchAll(pattern)) {
198 const service = match[1]!;
199 names.add(service);
200 for (const extra of extraDirs[service] ?? []) names.add(extra);
201 }
202 return [...names].sort();
203});
204
205/** Test files under `root`, matching `**\/*.test.ts` without dot directories. */
206const findTestFiles = Effect.fn(function* (root: string) {
207 const fs = yield* FileSystem.FileSystem;
208 const entries = yield* fs.readDirectory(root, { recursive: true });
209 return entries
210 .map((entry) => entry.replaceAll("\\", "/"))
211 .filter(
212 (entry) =>
213 entry.endsWith(".test.ts") && !entry.split("/").some((part) => part.startsWith(".")),
214 );
215});
216
217const optionalEnv = (name: string) => Config.String(name).pipe(Config.withDefault(""));
218
219const command = Command.make(
220 "test-aws-floci",
221 {
222 list: Flag.Boolean("list").pipe(
223 Flag.withDefault(false),
224 Flag.withDescription("Print the selected test files without starting tests or Floci"),
225 ),
226 dryRun: Flag.Boolean("dry-run").pipe(
227 Flag.withDefault(false),
228 Flag.withDescription("Print the resolved command without starting tests or Floci"),
229 ),
230 external: Flag.Boolean("external").pipe(
231 Flag.withDefault(false),
232 Flag.withDescription(
233 "Use an existing Floci server, without Docker fallback or state reset (or ALCHEMY_FLOCI_EXTERNAL=1)",
234 ),
235 ),
236 resetShared: Flag.Boolean("reset-shared").pipe(
237 Flag.withDefault(false),
238 Flag.withDescription("Reset the shared emulator state before the run"),
239 ),
240 args: Argument.String("alchemy-test-args").pipe(
241 Argument.withDescription("Test paths and alchemy-test flags (-t, --retry, -c, …), forwarded"),
242 Argument.atLeast(0),
243 ),
244 },
245 Effect.fn(function* (options) {
246 const fs = yield* FileSystem.FileSystem;
247 const path = yield* Path.Path;
248 const spawner = yield* ChildProcessSpawner;
249
250 const repoRoot = path.resolve(import.meta.dirname, "..");
251 const testPackageRoot = path.join(repoRoot, "packages/integration-tests");
252 const awsTestRoot = path.join(testPackageRoot, "test/AWS");
253 const providersFile = path.join(repoRoot, "packages/aws/src/Providers.ts");
254
255 const flags: string[] = [];
256 const paths: string[] = [];
257 const args = options.args;
258 for (let i = 0; i < args.length; i++) {
259 const arg = args[i]!;
260 if (arg.startsWith("-")) {
261 flags.push(arg);
262 if (flagsWithValue.has(arg) && args[i + 1] && !args[i + 1]!.startsWith("-")) {
263 flags.push(args[++i]!);
264 }
265 continue;
266 }
267 paths.push(arg);
268 }
269 const listOnly = options.list;
270 const dryRun = options.dryRun;
271 const resetShared = options.resetShared;
272 const external = options.external || (yield* optionalEnv("ALCHEMY_FLOCI_EXTERNAL")) === "1";
273
274 const allowedRoots = yield* Effect.filter(
275 (yield* dualizedServices(providersFile)).map((service) => path.join(awsTestRoot, service)),
276 (dir) => fs.exists(dir),
277 );
278 if (allowedRoots.length === 0) {
279 yield* Console.error(`${LABEL}: no dualized AWS service test dirs found`);
280 return yield* new ExitWith({ code: 1 });
281 }
282
283 const requestedRoots =
284 paths.length > 0 ? paths.map((p) => path.resolve(testPackageRoot, p)) : allowedRoots;
285
286 const files: string[] = [];
287 for (const root of requestedRoots) {
288 const info = yield* fs.stat(root).pipe(Effect.option);
289 if (info._tag === "None" || info.value.type !== "Directory") {
290 files.push(path.relative(testPackageRoot, root));
291 continue;
292 }
293 for (const file of yield* findTestFiles(root)) {
294 if (file.endsWith(".local.test.ts")) continue;
295 if (liveOnlySuites.has(path.relative(awsTestRoot, path.join(root, file)))) continue;
296 files.push(path.relative(testPackageRoot, path.join(root, file)));
297 }
298 }
299
300 if (listOnly) {
301 return yield* Console.log([...new Set(files)].sort().join("\n"));
302 }
303
304 if (resetShared && (external || (yield* optionalEnv("ALCHEMY_FLOCI_NO_RESET")) !== "")) {
305 return yield* fail("--reset-shared conflicts with --external or ALCHEMY_FLOCI_NO_RESET");
306 }
307
308 // Local providers currently use this gateway regardless of SDK overrides.
309 const endpointError =
310 "AWS_ENDPOINT_URL must be http://localhost:4566; isolated gateways are not supported yet";
311 const endpointValue = yield* Config.String("AWS_ENDPOINT_URL").pipe(
312 Config.withDefault("http://localhost:4566"),
313 );
314 if (!URL.canParse(endpointValue)) return yield* fail(endpointError);
315 const endpoint = new URL(endpointValue);
316 if (
317 endpoint.protocol !== "http:" ||
318 !["localhost", "127.0.0.1"].includes(endpoint.hostname) ||
319 endpoint.port !== "4566" ||
320 endpoint.pathname !== "/" ||
321 endpoint.search ||
322 endpoint.hash ||
323 endpoint.username ||
324 endpoint.password
325 ) {
326 return yield* fail(endpointError);
327 }
328
329 const hasFlag = (...names: string[]) =>
330 flags.some((flag) => names.some((name) => flag === name || flag.startsWith(`${name}=`)));
331 if (!hasFlag("--profile")) flags.unshift("--profile", "testing");
332 if (!hasFlag("--concurrency", "-c")) flags.unshift("--concurrency", "4");
333 const testCommand = ["bun", "alchemy-test", ...new Set(files.sort()), ...flags];
334 if (dryRun) {
335 return yield* Console.log(
336 JSON.stringify({
337 command: testCommand,
338 cwd: testPackageRoot,
339 external,
340 resetShared,
341 endpoint: "http://localhost:4566",
342 }),
343 );
344 }
345
346 // The env is written to `process.env` rather than only handed to the
347 // child: the in-process Floci helpers (`ensureFloci`, fixtures) read
348 // `ALCHEMY_FLOCI_*` from it too, and the child inherits it.
349 const setEnv = (values: Record<string, string>, overwrite: boolean) =>
350 Effect.sync(() => {
351 for (const [name, value] of Object.entries(values)) {
352 if (overwrite || process.env[name] === undefined) process.env[name] = value;
353 }
354 });
355 yield* setEnv(
356 {
357 ALCHEMY_TEST_DEV: "1",
358 AWS_ENDPOINT_URL: "http://localhost:4566",
359 ...(external ? { ALCHEMY_FLOCI_EXTERNAL: "1" } : {}),
360 },
361 true,
362 );
363 yield* setEnv(
364 {
365 ...Object.fromEntries(flociOptInGates.map((gate) => [gate, "1"])),
366 // One test process deploys many functions at once; each rolldown build
367 // holds its whole module graph in native memory.
368 ALCHEMY_BUNDLE_CONCURRENCY: "2",
369 ...flociFixedValues,
370 },
371 false,
372 );
373
374 if (external) {
375 yield* Console.log(`${LABEL}: using the existing Floci server (no Docker fallback)`);
376 } else {
377 yield* preferLocalFlociImage(LABEL);
378 yield* startFlociForTests(LABEL).pipe(
379 Effect.catch((error) =>
380 fail(
381 `could not start the Floci container: ${errorMessage(error)}; tests were not started`,
382 ),
383 ),
384 );
385 }
386
387 // Suites that take subnet/security-group IDs from the environment (e.g.
388 // Timestream DbInstance) get the emulator's default VPC network. Floci must
389 // be serving for this lookup, so start it here in non-external mode.
390 const subnetIds = yield* optionalEnv("AWS_TEST_SUBNET_IDS");
391 const securityGroupIds = yield* optionalEnv("AWS_TEST_SECURITY_GROUP_IDS");
392 if ((yield* optionalEnv("AWS_TEST_SLOW")) !== "" && (!subnetIds || !securityGroupIds)) {
393 const network = yield* resolveFlociDefaultNetwork(external).pipe(
394 Effect.catch((error) =>
395 fail(
396 `could not resolve the emulator's default VPC network: ${errorMessage(error)}; tests were not started`,
397 ),
398 ),
399 );
400 yield* setEnv(
401 {
402 AWS_TEST_SUBNET_IDS: network.subnetIds.join(","),
403 AWS_TEST_SECURITY_GROUP_IDS: network.securityGroupIds.join(","),
404 },
405 false,
406 );
407 }
408
409 if (resetShared) {
410 const client = yield* HttpClient.HttpClient;
411 const res = yield* client.post("http://localhost:4566/_floci/state/reset").pipe(
412 Effect.timeout("5 seconds"),
413 Effect.catch((error) =>
414 fail(`could not reset emulator state: ${errorMessage(error)}; tests were not started`),
415 ),
416 );
417 if (res.status < 200 || res.status >= 300) {
418 return yield* fail(`emulator state reset returned ${res.status}; tests were not started`);
419 }
420 yield* Console.log(`${LABEL}: reset shared emulator state`);
421 } else {
422 yield* Console.log(`${LABEL}: preserving shared emulator state`);
423 }
424
425 // Provisioned after any reset so the reset cannot wipe them. Interrupting
426 // provisioning (Ctrl-C) removes already-created fixtures first.
427 const fixtures = yield* provisionFlociStandingFixtures(external, { files }).pipe(
428 Effect.catch((error) =>
429 fail(
430 `could not provision standing fixtures: ${errorMessage(error)}; tests were not started`,
431 ),
432 ),
433 );
434 yield* setEnv(fixtures.env, false);
435
436 const teardown =
437 Object.keys(fixtures.env).length === 0
438 ? Effect.void
439 : Console.log(`${LABEL}: removing standing fixtures`).pipe(
440 Effect.andThen(fixtures.teardown),
441 Effect.catch((error) =>
442 Console.error(`${LABEL}: fixture teardown failed: ${errorMessage(error)}`),
443 ),
444 );
445
446 // Not detached: terminal Ctrl-C reaches the child through the shared
447 // process group. On interruption (Ctrl-C, or a SIGTERM aimed at this
448 // runner alone) the child is sent SIGTERM and awaited, then fixtures are
449 // torn down.
450 const exitCode = yield* Effect.scoped(
451 spawner
452 .spawn(
453 ChildProcess.make(testCommand[0]!, testCommand.slice(1), {
454 cwd: testPackageRoot,
455 detached: false,
456 stdin: "inherit",
457 stdout: "inherit",
458 stderr: "inherit",
459 }),
460 )
461 .pipe(Effect.flatMap((child) => child.exitCode)),
462 ).pipe(
463 Effect.catch((error) => Console.error(`${LABEL}: ${errorMessage(error)}`).pipe(Effect.as(1))),
464 Effect.ensuring(teardown),
465 );
466 if (exitCode !== 0) return yield* new ExitWith({ code: exitCode });
467 }),
468).pipe(Command.withDescription("Run the Floci-dualized AWS suites against the local emulator"));
469
470Stdio.Stdio.use(({ args }) =>
471 Effect.flatMap(args, (argv) => Command.runWith(command, { version: "0.0.0" })(splitArgs(argv))),
472).pipe(
473 Effect.provide(Layer.mergeAll(NodeServices.layer, FetchHttpClient.layer)),
474 NodeRuntime.runMain,
475);