Open History to see the last commit
1import * as Logs from "@distilled.cloud/aws/cloudwatch-logs";
2import { fromChain } from "@distilled.cloud/aws/Credentials";
3import * as DynamoDB from "@distilled.cloud/aws/dynamodb";
4import * as EC2 from "@distilled.cloud/aws/ec2";
5import * as ECS from "@distilled.cloud/aws/ecs";
6import * as EFS from "@distilled.cloud/aws/efs";
7import * as ELBv2 from "@distilled.cloud/aws/elastic-load-balancing-v2";
8import * as EventBridge from "@distilled.cloud/aws/eventbridge";
9import * as IAM from "@distilled.cloud/aws/iam";
10import * as Kinesis from "@distilled.cloud/aws/kinesis";
11import * as KMS from "@distilled.cloud/aws/kms";
12import * as Lambda from "@distilled.cloud/aws/lambda";
13import * as RDS from "@distilled.cloud/aws/rds";
14import { Region } from "@distilled.cloud/aws/Region";
15import {
16 getResources,
17 type GetResourcesOutput,
18} from "@distilled.cloud/aws/resource-groups-tagging-api";
19import * as S3 from "@distilled.cloud/aws/s3";
20import * as Scheduler from "@distilled.cloud/aws/scheduler";
21import * as SecretsManager from "@distilled.cloud/aws/secrets-manager";
22import * as SNS from "@distilled.cloud/aws/sns";
23import * as SQS from "@distilled.cloud/aws/sqs";
24import * as SSM from "@distilled.cloud/aws/ssm";
25import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
26import * as NodeServices from "@effect/platform-node/NodeServices";
27/**
28 * AWS leak sweep — finds (and optionally deletes) alchemy-tagged TEST resources
29 * left behind by crashed/interrupted test runs.
30 *
31 * Discovery is tag-driven: every alchemy resource is branded with
32 * `alchemy::stack` / `alchemy::stage` / `alchemy::id` (see src/Tags.ts
33 * createInternalTags). Test suites deploy with stage `test_$USER` (the default
34 * in packages/alchemy-test/src/Core.ts), so we sweep everything tagged
35 * `alchemy::stage = test_$USER`.
36 *
37 * Primary discovery path: resourcegroupstaggingapi GetResources (regional).
38 * Fallbacks for resources the tagging API can't see from a regional query:
39 * - IAM roles/policies (global service): listRoles/listPolicies + tag reads
40 * - EventBridge Scheduler schedules (not taggable): name-pattern heuristic
41 * (physical names embed `-{stage}-`, see src/PhysicalName.ts)
42 *
43 * Usage:
44 * node scripts/aws-leak-sweep.ts # DRY RUN (default)
45 * node scripts/aws-leak-sweep.ts --delete # actually delete
46 * node scripts/aws-leak-sweep.ts --older-than 12 # only >= 12h old (default 6)
47 * node scripts/aws-leak-sweep.ts --stage test_sam # tag stage filter (default test_$USER)
48 * node scripts/aws-leak-sweep.ts --region us-west-2 # region (default env or us-west-2)
49 *
50 * KMS keys are ALWAYS report-only — never deleted by this tool.
51 * Resources whose age can't be determined are treated as eligible (a leak with
52 * an unknown birthday is still a leak) and shown with age `?`.
53 */
54import { Command, Flag } from "effect/cli";
55import * as Config from "effect/Config";
56import * as Console from "effect/Console";
57import * as Effect from "effect/Effect";
58import { FetchHttpClient } from "effect/http";
59import type * as HttpClient from "effect/http/HttpClient";
60import * as Layer from "effect/Layer";
61import * as Result from "effect/Result";
62import * as Schedule from "effect/Schedule";
63
64// ---------------------------------------------------------------------------
65// CLI args
66// ---------------------------------------------------------------------------
67
68interface Args {
69 delete: boolean;
70 olderThanHours: number;
71 stage: string;
72 region: string;
73}
74
75// ---------------------------------------------------------------------------
76// Model
77// ---------------------------------------------------------------------------
78
79type Env = import("@distilled.cloud/aws/Credentials").Credentials | Region | HttpClient.HttpClient;
80
81interface Leak {
82 arn: string;
83 /** display kind, e.g. "sqs:queue" */
84 kind: string;
85 /** physical name / identifier used by delete calls */
86 name: string;
87 tags: Record<string, string>;
88 createdAt: Date | undefined;
89 hourly: boolean;
90 reportOnly: boolean;
91 reportOnlyReason?: string;
92 /** deletion ordering: lower phases first */
93 phase: number;
94}
95
96const toDate = (v: Date | string | number | undefined): Date | undefined => {
97 if (v === undefined || v === null) return undefined;
98 if (v instanceof Date) return v;
99 if (typeof v === "number") {
100 // seconds vs millis
101 return new Date(v > 10_000_000_000 ? v : v * 1000);
102 }
103 const n = Number(v);
104 if (!Number.isNaN(n) && v.trim() !== "") {
105 return new Date(n > 10_000_000_000 ? n : n * 1000);
106 }
107 const d = new Date(v);
108 return Number.isNaN(d.getTime()) ? undefined : d;
109};
110
111const ageHours = (d: Date | undefined): number | undefined =>
112 d === undefined ? undefined : (Date.now() - d.getTime()) / 3_600_000;
113
114const tagRecord = (
115 tags: readonly { Key?: string; Value?: string }[] | undefined,
116): Record<string, string> =>
117 Object.fromEntries(
118 (tags ?? []).filter((t) => t.Key !== undefined).map((t) => [t.Key as string, t.Value ?? ""]),
119 );
120
121/** best-effort: any failure (missing perms, races, region mismatch) -> undefined */
122const tryDate = <E, R>(
123 eff: Effect.Effect<Date | string | number | undefined, E, R>,
124): Effect.Effect<Date | undefined, never, R> =>
125 eff.pipe(
126 Effect.map(toDate),
127 Effect.orElseSucceed(() => undefined),
128 );
129
130// ---------------------------------------------------------------------------
131// ARN classification
132// ---------------------------------------------------------------------------
133
134interface Parsed {
135 service: string;
136 region: string;
137 account: string;
138 rest: string;
139}
140
141const parseArn = (arn: string): Parsed | undefined => {
142 const parts = arn.split(":");
143 if (parts.length < 6 || parts[0] !== "arn") return undefined;
144 return {
145 service: parts[2],
146 region: parts[3],
147 account: parts[4],
148 rest: parts.slice(5).join(":"),
149 };
150};
151
152interface Classified {
153 kind: string;
154 name: string;
155 hourly: boolean;
156 reportOnly: boolean;
157 reportOnlyReason?: string;
158 phase: number;
159}
160
161const classify = (arn: string): Classified => {
162 const p = parseArn(arn);
163 const unsupported = (reason: string): Classified => ({
164 kind: `${p?.service ?? "?"}:?`,
165 name: p?.rest ?? arn,
166 hourly: false,
167 reportOnly: true,
168 reportOnlyReason: reason,
169 phase: 99,
170 });
171 if (!p) return unsupported("unparseable arn");
172 const r = p.rest;
173 const mk = (
174 kind: string,
175 name: string,
176 phase: number,
177 opts?: { hourly?: boolean; reportOnly?: boolean; reason?: string },
178 ): Classified => ({
179 kind,
180 name,
181 phase,
182 hourly: opts?.hourly ?? false,
183 reportOnly: opts?.reportOnly ?? false,
184 reportOnlyReason: opts?.reason,
185 });
186
187 switch (p.service) {
188 case "sqs":
189 return mk("sqs:queue", r, 2);
190 case "sns":
191 return mk("sns:topic", r, 2);
192 case "dynamodb":
193 if (r.startsWith("table/") && !r.includes("/stream/")) {
194 return mk("dynamodb:table", r.slice("table/".length), 2);
195 }
196 return unsupported("only tables are swept");
197 case "lambda":
198 if (r.startsWith("function:")) {
199 return mk("lambda:function", r.split(":")[1], 2);
200 }
201 return unsupported("only functions are swept");
202 case "logs":
203 if (r.startsWith("log-group:")) {
204 return mk("logs:log-group", r.slice("log-group:".length).replace(/:\*$/, ""), 2);
205 }
206 return unsupported("only log groups are swept");
207 case "s3":
208 return mk("s3:bucket", r, 2);
209 case "kinesis":
210 if (r.startsWith("stream/")) {
211 return mk("kinesis:stream", r.slice("stream/".length), 2);
212 }
213 return unsupported("only streams are swept");
214 case "events": {
215 if (r.startsWith("rule/")) {
216 return mk("events:rule", r.slice("rule/".length), 1);
217 }
218 if (r.startsWith("event-bus/")) {
219 return mk("events:event-bus", r.slice("event-bus/".length), 3);
220 }
221 return unsupported("only rules and buses are swept");
222 }
223 case "ecs": {
224 if (r.startsWith("service/")) return mk("ecs:service", r, 1);
225 if (r.startsWith("cluster/")) {
226 return mk("ecs:cluster", r.slice("cluster/".length), 3);
227 }
228 if (r.startsWith("task-definition/")) {
229 return mk("ecs:task-definition", r.slice("task-definition/".length), 2);
230 }
231 if (r.startsWith("task/")) {
232 return mk("ecs:task", r, 99, {
233 reportOnly: true,
234 reason: "tasks die with their service/cluster",
235 });
236 }
237 return unsupported("unhandled ecs type");
238 }
239 case "elasticloadbalancing": {
240 if (r.startsWith("loadbalancer/")) {
241 return mk("elbv2:load-balancer", arn, 3, { hourly: true });
242 }
243 if (r.startsWith("targetgroup/")) {
244 return mk("elbv2:target-group", arn, 4);
245 }
246 if (r.startsWith("listener/")) {
247 return mk("elbv2:listener", arn, 99, {
248 reportOnly: true,
249 reason: "listeners die with their load balancer",
250 });
251 }
252 return unsupported("unhandled elb type");
253 }
254 case "ec2": {
255 if (r.startsWith("natgateway/")) {
256 return mk("ec2:nat-gateway", r.slice("natgateway/".length), 3, {
257 hourly: true,
258 });
259 }
260 if (r.startsWith("elastic-ip/")) {
261 return mk("ec2:elastic-ip", r.slice("elastic-ip/".length), 4, {
262 hourly: true,
263 });
264 }
265 if (r.startsWith("vpc/")) {
266 return mk("ec2:vpc", r.slice("vpc/".length), 5);
267 }
268 // subnets, route tables, IGWs, SGs etc. are swept as part of their
269 // VPC's teardown; report them so nothing is silently ignored.
270 return mk(`ec2:${r.split("/")[0]}`, r, 99, {
271 reportOnly: true,
272 reason: "deleted as part of the owning VPC's teardown",
273 });
274 }
275 case "rds": {
276 if (r.startsWith("db:")) {
277 return mk("rds:instance", r.slice("db:".length), 2, { hourly: true });
278 }
279 if (r.startsWith("cluster:")) {
280 return mk("rds:cluster", r.slice("cluster:".length), 3, {
281 hourly: true,
282 });
283 }
284 return mk(`rds:${r.split(":")[0]}`, r, 99, {
285 reportOnly: true,
286 reason: "only instances/clusters are swept",
287 });
288 }
289 case "elasticfilesystem": {
290 if (r.startsWith("file-system/")) {
291 return mk("efs:file-system", r.slice("file-system/".length), 2);
292 }
293 return unsupported("only file systems are swept");
294 }
295 case "kms":
296 if (r.startsWith("key/")) {
297 return mk("kms:key", r.slice("key/".length), 99, {
298 reportOnly: true,
299 reason: "KMS keys are NEVER deleted by this tool",
300 });
301 }
302 return unsupported("only keys are reported");
303 case "secretsmanager":
304 if (r.startsWith("secret:")) {
305 return mk("secretsmanager:secret", arn, 2);
306 }
307 return unsupported("only secrets are swept");
308 case "ssm":
309 if (r.startsWith("parameter/")) {
310 const raw = r.slice("parameter/".length);
311 return mk("ssm:parameter", raw.includes("/") ? `/${raw}` : raw, 2);
312 }
313 return unsupported("only parameters are swept");
314 case "scheduler": {
315 if (r.startsWith("schedule/")) {
316 return mk("scheduler:schedule", r.slice("schedule/".length), 1);
317 }
318 if (r.startsWith("schedule-group/")) {
319 return mk("scheduler:schedule-group", r.slice("schedule-group/".length), 3);
320 }
321 return unsupported("unhandled scheduler type");
322 }
323 case "iam": {
324 if (r.startsWith("role/")) {
325 return mk("iam:role", r.slice("role/".length).split("/").pop()!, 4);
326 }
327 if (r.startsWith("policy/")) {
328 return mk("iam:policy", arn, 4);
329 }
330 return unsupported("only roles/policies are swept");
331 }
332 default:
333 return unsupported("no sweep handler for this service");
334 }
335};
336
337// ---------------------------------------------------------------------------
338// Age enrichment (best-effort, one describe per resource)
339// ---------------------------------------------------------------------------
340
341const sqsQueueUrl = (leak: Leak, region: string): string => {
342 const p = parseArn(leak.arn)!;
343 return `https://sqs.${region}.amazonaws.com/${p.account}/${leak.name}`;
344};
345
346const lookupCreatedAt = (
347 leak: Leak,
348 region: string,
349 bucketDates: Map<string, Date>,
350): Effect.Effect<Date | undefined, never, Env> => {
351 switch (leak.kind) {
352 case "sqs:queue":
353 return tryDate(
354 SQS.getQueueAttributes({
355 QueueUrl: sqsQueueUrl(leak, region),
356 AttributeNames: ["CreatedTimestamp"],
357 }).pipe(Effect.map((r) => r.Attributes?.CreatedTimestamp)),
358 );
359 case "dynamodb:table":
360 return tryDate(
361 DynamoDB.describeTable({ TableName: leak.name }).pipe(
362 Effect.map((r) => r.Table?.CreationDateTime),
363 ),
364 );
365 case "lambda:function":
366 return tryDate(
367 Lambda.getFunctionConfiguration({ FunctionName: leak.name }).pipe(
368 Effect.map((r) => r.LastModified),
369 ),
370 );
371 case "logs:log-group":
372 return tryDate(
373 Logs.describeLogGroups({ logGroupNamePrefix: leak.name }).pipe(
374 Effect.map((r) => r.logGroups?.find((g) => g.logGroupName === leak.name)?.creationTime),
375 ),
376 );
377 case "s3:bucket":
378 return Effect.succeed(bucketDates.get(leak.name));
379 case "kinesis:stream":
380 return tryDate(
381 Kinesis.describeStreamSummary({ StreamName: leak.name }).pipe(
382 Effect.map((r) => r.StreamDescriptionSummary?.StreamCreationTimestamp),
383 ),
384 );
385 case "ecs:service": {
386 // name = "service/{cluster}/{service}"
387 const [, cluster, service] = leak.name.split("/");
388 return tryDate(
389 ECS.describeServices({ cluster, services: [service] }).pipe(
390 Effect.map((r) => r.services?.[0]?.createdAt),
391 ),
392 );
393 }
394 case "elbv2:load-balancer":
395 return tryDate(
396 ELBv2.describeLoadBalancers({ LoadBalancerArns: [leak.arn] }).pipe(
397 Effect.map((r) => r.LoadBalancers?.[0]?.CreatedTime),
398 ),
399 );
400 case "ec2:nat-gateway":
401 return tryDate(
402 EC2.describeNatGateways({ NatGatewayIds: [leak.name] }).pipe(
403 Effect.map((r) => r.NatGateways?.[0]?.CreateTime),
404 ),
405 );
406 case "rds:instance":
407 return tryDate(
408 RDS.describeDBInstances({ DBInstanceIdentifier: leak.name }).pipe(
409 Effect.map((r) => r.DBInstances?.[0]?.InstanceCreateTime),
410 ),
411 );
412 case "rds:cluster":
413 return tryDate(
414 RDS.describeDBClusters({ DBClusterIdentifier: leak.name }).pipe(
415 Effect.map((r) => r.DBClusters?.[0]?.ClusterCreateTime),
416 ),
417 );
418 case "efs:file-system":
419 return tryDate(
420 EFS.describeFileSystems({ FileSystemId: leak.name }).pipe(
421 Effect.map((r) => r.FileSystems?.[0]?.CreationTime),
422 ),
423 );
424 case "kms:key":
425 return tryDate(
426 KMS.describeKey({ KeyId: leak.name }).pipe(Effect.map((r) => r.KeyMetadata?.CreationDate)),
427 );
428 case "secretsmanager:secret":
429 return tryDate(
430 SecretsManager.describeSecret({ SecretId: leak.arn }).pipe(
431 Effect.map((r) => r.CreatedDate),
432 ),
433 );
434 case "ssm:parameter":
435 return tryDate(
436 SSM.describeParameters({
437 ParameterFilters: [{ Key: "Name", Option: "Equals", Values: [leak.name] }],
438 }).pipe(Effect.map((r) => r.Parameters?.[0]?.LastModifiedDate)),
439 );
440 case "scheduler:schedule": {
441 const [group, name] = leak.name.includes("/") ? leak.name.split("/") : ["default", leak.name];
442 return tryDate(
443 Scheduler.getSchedule({ Name: name, GroupName: group }).pipe(
444 Effect.map((r) => r.CreationDate),
445 ),
446 );
447 }
448 default:
449 return Effect.undefined;
450 }
451};
452
453// ---------------------------------------------------------------------------
454// Deletion (typed per service; not-found tags forgiven; bounded dep retries)
455// ---------------------------------------------------------------------------
456
457/** bounded retry on dependency-violation / in-use style typed tags */
458const retryDependencies = <A, E extends { _tag: string }, R>(
459 eff: Effect.Effect<A, E, R>,
460): Effect.Effect<A, E, R> =>
461 eff.pipe(
462 Effect.retry({
463 while: (e) =>
464 /InUse|DependencyViolation|Conflict|ContainsServices|ContainsTasks|BucketNotEmpty|InvalidDBClusterState|InvalidDBInstanceState|Throttl|TooManyRequests|LimitExceeded/i.test(
465 e._tag,
466 ),
467 schedule: Schedule.spaced("8 seconds"),
468 times: 6,
469 }),
470 );
471
472const GONE = /NotFound|NoSuchEntity|DoesNotExist|NoSuch|Missing|Gone|Deleted/i;
473
474const emptyBucket = (bucket: string) =>
475 Effect.gen(function* () {
476 // re-list from the start after each batch delete; bounded passes
477 for (let pass = 0; pass < 100; pass++) {
478 const listing = yield* S3.listObjectVersions({ Bucket: bucket });
479 const objects = [...(listing.Versions ?? []), ...(listing.DeleteMarkers ?? [])]
480 .filter((o) => o.Key !== undefined)
481 .map((o) => ({ Key: o.Key as string, VersionId: o.VersionId }));
482 if (objects.length === 0) return;
483 yield* S3.deleteObjects({
484 Bucket: bucket,
485 Delete: { Objects: objects, Quiet: true },
486 });
487 if (!listing.IsTruncated) return;
488 }
489 });
490
491const deleteIamRole = (roleName: string) =>
492 Effect.gen(function* () {
493 const profiles = yield* IAM.listInstanceProfilesForRole({
494 RoleName: roleName,
495 });
496 yield* Effect.forEach(
497 profiles.InstanceProfiles ?? [],
498 (p) =>
499 IAM.removeRoleFromInstanceProfile({
500 InstanceProfileName: p.InstanceProfileName,
501 RoleName: roleName,
502 }),
503 { discard: true },
504 );
505 const attached = yield* IAM.listAttachedRolePolicies({
506 RoleName: roleName,
507 });
508 yield* Effect.forEach(
509 attached.AttachedPolicies ?? [],
510 (p) =>
511 IAM.detachRolePolicy({
512 RoleName: roleName,
513 PolicyArn: p.PolicyArn!,
514 }),
515 { discard: true },
516 );
517 const inline = yield* IAM.listRolePolicies({ RoleName: roleName });
518 yield* Effect.forEach(
519 inline.PolicyNames ?? [],
520 (name) => IAM.deleteRolePolicy({ RoleName: roleName, PolicyName: name }),
521 { discard: true },
522 );
523 yield* retryDependencies(IAM.deleteRole({ RoleName: roleName }));
524 });
525
526const deleteIamPolicy = (policyArn: string) =>
527 Effect.gen(function* () {
528 const versions = yield* IAM.listPolicyVersions({ PolicyArn: policyArn });
529 yield* Effect.forEach(
530 (versions.Versions ?? []).filter((v) => !v.IsDefaultVersion),
531 (v) =>
532 IAM.deletePolicyVersion({
533 PolicyArn: policyArn,
534 VersionId: v.VersionId!,
535 }),
536 { discard: true },
537 );
538 yield* retryDependencies(IAM.deletePolicy({ PolicyArn: policyArn }));
539 });
540
541const deleteEcsService = (leak: Leak) =>
542 Effect.gen(function* () {
543 // name = "service/{cluster}/{service}"
544 const [, cluster, service] = leak.name.split("/");
545 yield* ECS.updateService({ cluster, service, desiredCount: 0 }).pipe(
546 Effect.orElseSucceed(() => undefined),
547 );
548 yield* retryDependencies(ECS.deleteService({ cluster, service, force: true }));
549 });
550
551const deleteEventsRule = (leak: Leak) =>
552 Effect.gen(function* () {
553 // name is "name" (default bus) or "{bus}/{name}"
554 const parts = leak.name.split("/");
555 const [bus, name] = parts.length === 2 ? [parts[0], parts[1]] : ["default", parts[0]];
556 const targets = yield* EventBridge.listTargetsByRule({
557 Rule: name,
558 EventBusName: bus,
559 }).pipe(Effect.orElseSucceed(() => ({ Targets: [] })));
560 const ids = (targets.Targets ?? [])
561 .map((t) => t.Id)
562 .filter((id): id is string => id !== undefined);
563 if (ids.length > 0) {
564 yield* EventBridge.removeTargets({
565 Rule: name,
566 EventBusName: bus,
567 Ids: ids,
568 Force: true,
569 }).pipe(Effect.orElseSucceed(() => undefined));
570 }
571 yield* EventBridge.deleteRule({
572 Name: name,
573 EventBusName: bus,
574 Force: true,
575 });
576 });
577
578/**
579 * A leaked VPC can't be deleted while its subnets / route tables / IGW /
580 * security groups exist, so tear those down first. NEVER touches the default
581 * VPC.
582 */
583const deleteVpcDeep = (vpcId: string) =>
584 Effect.gen(function* () {
585 const vpcs = yield* EC2.describeVpcs({ VpcIds: [vpcId] });
586 const vpc = vpcs.Vpcs?.[0];
587 if (!vpc) return; // already gone
588 if (vpc.IsDefault) {
589 return yield* Effect.fail({
590 _tag: "RefusingToDeleteDefaultVpc" as const,
591 });
592 }
593 const vpcFilter = [{ Name: "vpc-id", Values: [vpcId] }];
594
595 const subnets = yield* EC2.describeSubnets({ Filters: vpcFilter });
596 yield* Effect.forEach(
597 subnets.Subnets ?? [],
598 (s) =>
599 retryDependencies(EC2.deleteSubnet({ SubnetId: s.SubnetId! })).pipe(
600 Effect.orElseSucceed(() => undefined),
601 ),
602 { discard: true },
603 );
604
605 const igws = yield* EC2.describeInternetGateways({
606 Filters: [{ Name: "attachment.vpc-id", Values: [vpcId] }],
607 });
608 yield* Effect.forEach(
609 igws.InternetGateways ?? [],
610 (igw) =>
611 EC2.detachInternetGateway({
612 InternetGatewayId: igw.InternetGatewayId!,
613 VpcId: vpcId,
614 }).pipe(
615 Effect.andThen(
616 EC2.deleteInternetGateway({
617 InternetGatewayId: igw.InternetGatewayId!,
618 }),
619 ),
620 Effect.orElseSucceed(() => undefined),
621 ),
622 { discard: true },
623 );
624
625 const rts = yield* EC2.describeRouteTables({ Filters: vpcFilter });
626 yield* Effect.forEach(
627 (rts.RouteTables ?? []).filter((rt) => !(rt.Associations ?? []).some((a) => a.Main)),
628 (rt) =>
629 EC2.deleteRouteTable({ RouteTableId: rt.RouteTableId! }).pipe(
630 Effect.orElseSucceed(() => undefined),
631 ),
632 { discard: true },
633 );
634
635 const sgs = yield* EC2.describeSecurityGroups({ Filters: vpcFilter });
636 yield* Effect.forEach(
637 (sgs.SecurityGroups ?? []).filter((sg) => sg.GroupName !== "default"),
638 (sg) =>
639 retryDependencies(EC2.deleteSecurityGroup({ GroupId: sg.GroupId! })).pipe(
640 Effect.orElseSucceed(() => undefined),
641 ),
642 { discard: true },
643 );
644
645 yield* retryDependencies(EC2.deleteVpc({ VpcId: vpcId }));
646 });
647
648const deleteEfs = (fileSystemId: string) =>
649 Effect.gen(function* () {
650 const mts = yield* EFS.describeMountTargets({
651 FileSystemId: fileSystemId,
652 }).pipe(Effect.orElseSucceed(() => ({ MountTargets: [] })));
653 yield* Effect.forEach(
654 mts.MountTargets ?? [],
655 (mt) => EFS.deleteMountTarget({ MountTargetId: mt.MountTargetId }),
656 { discard: true },
657 );
658 yield* retryDependencies(EFS.deleteFileSystem({ FileSystemId: fileSystemId }));
659 });
660
661const destroyLeak = (leak: Leak, region: string): Effect.Effect<unknown, { _tag: string }, Env> => {
662 switch (leak.kind) {
663 case "sqs:queue":
664 return SQS.deleteQueue({ QueueUrl: sqsQueueUrl(leak, region) });
665 case "sns:topic":
666 return SNS.deleteTopic({ TopicArn: leak.arn });
667 case "dynamodb:table":
668 return retryDependencies(DynamoDB.deleteTable({ TableName: leak.name }));
669 case "lambda:function":
670 return Lambda.deleteFunction({ FunctionName: leak.name });
671 case "logs:log-group":
672 return Logs.deleteLogGroup({ logGroupName: leak.name });
673 case "s3:bucket":
674 return emptyBucket(leak.name).pipe(
675 Effect.andThen(retryDependencies(S3.deleteBucket({ Bucket: leak.name }))),
676 );
677 case "kinesis:stream":
678 return retryDependencies(
679 Kinesis.deleteStream({
680 StreamName: leak.name,
681 EnforceConsumerDeletion: true,
682 }),
683 );
684 case "events:rule":
685 return deleteEventsRule(leak);
686 case "events:event-bus":
687 return EventBridge.deleteEventBus({ Name: leak.name });
688 case "ecs:service":
689 return deleteEcsService(leak);
690 case "ecs:cluster":
691 return retryDependencies(ECS.deleteCluster({ cluster: leak.name }));
692 case "ecs:task-definition":
693 return ECS.deregisterTaskDefinition({ taskDefinition: leak.name });
694 case "elbv2:load-balancer":
695 return ELBv2.deleteLoadBalancer({ LoadBalancerArn: leak.arn });
696 case "elbv2:target-group":
697 return retryDependencies(ELBv2.deleteTargetGroup({ TargetGroupArn: leak.arn }));
698 case "ec2:nat-gateway":
699 return EC2.deleteNatGateway({ NatGatewayId: leak.name });
700 case "ec2:elastic-ip":
701 return retryDependencies(EC2.releaseAddress({ AllocationId: leak.name }));
702 case "ec2:vpc":
703 return deleteVpcDeep(leak.name);
704 case "rds:instance":
705 return retryDependencies(
706 RDS.deleteDBInstance({
707 DBInstanceIdentifier: leak.name,
708 SkipFinalSnapshot: true,
709 DeleteAutomatedBackups: true,
710 }),
711 );
712 case "rds:cluster":
713 return retryDependencies(
714 RDS.deleteDBCluster({
715 DBClusterIdentifier: leak.name,
716 SkipFinalSnapshot: true,
717 }),
718 );
719 case "efs:file-system":
720 return deleteEfs(leak.name);
721 case "secretsmanager:secret":
722 return SecretsManager.deleteSecret({
723 SecretId: leak.arn,
724 ForceDeleteWithoutRecovery: true,
725 });
726 case "ssm:parameter":
727 return SSM.deleteParameter({ Name: leak.name });
728 case "scheduler:schedule": {
729 const [group, name] = leak.name.includes("/") ? leak.name.split("/") : ["default", leak.name];
730 return Scheduler.deleteSchedule({ Name: name, GroupName: group });
731 }
732 case "scheduler:schedule-group":
733 return retryDependencies(Scheduler.deleteScheduleGroup({ Name: leak.name }));
734 case "iam:role":
735 return deleteIamRole(leak.name);
736 case "iam:policy":
737 return deleteIamPolicy(leak.arn);
738 default:
739 return Effect.void;
740 }
741};
742
743// ---------------------------------------------------------------------------
744// Discovery
745// ---------------------------------------------------------------------------
746
747/**
748 * NOTE: distilled's generated `.pages`/`.items` streams hang on APIs that
749 * signal the last page with an EMPTY-STRING token (the tagging API returns
750 * `PaginationToken: ""` when done, but src/client/api.ts only treats
751 * undefined/null as done). Paginate by hand, bounded, until that core bug is
752 * fixed.
753 */
754const MAX_PAGES = 50;
755
756const discoverTagged = (stage: string) =>
757 Effect.gen(function* () {
758 const all: import("@distilled.cloud/aws/resource-groups-tagging-api").ResourceTagMapping[] = [];
759 let token: string | undefined = undefined;
760 for (let page = 0; page < MAX_PAGES; page++) {
761 const res: GetResourcesOutput = yield* getResources({
762 TagFilters: [{ Key: "alchemy::stage", Values: [stage] }],
763 ResourcesPerPage: 100,
764 ...(token ? { PaginationToken: token } : {}),
765 });
766 all.push(...(res.ResourceTagMappingList ?? []));
767 token = res.PaginationToken;
768 if (!token) break;
769 }
770 return all;
771 });
772
773/**
774 * IAM is a global service — a regional GetResources call does not return IAM
775 * resources, so list roles/local policies directly and confirm ownership via
776 * their alchemy tags. To bound tag-read calls, only name-plausible candidates
777 * (physical names embed `-{stage}-`, see src/PhysicalName.ts) are tag-checked.
778 */
779const discoverIam = (stage: string, account: string) =>
780 Effect.gen(function* () {
781 const marker = `-${stage}-`;
782 const leaks: {
783 arn: string;
784 tags: Record<string, string>;
785 createdAt: Date | undefined;
786 }[] = [];
787
788 const roles: import("@distilled.cloud/aws/iam").Role[] = [];
789 {
790 let marker: string | undefined = undefined;
791 for (let page = 0; page < MAX_PAGES; page++) {
792 const res: IAM.ListRolesResponse = yield* IAM.listRoles({
793 MaxItems: 1000,
794 ...(marker ? { Marker: marker } : {}),
795 });
796 roles.push(...(res.Roles ?? []));
797 if (!res.IsTruncated || !res.Marker) break;
798 marker = res.Marker;
799 }
800 }
801 const roleCandidates = roles.filter((r) => (r.RoleName ?? "").includes(marker));
802 const roleTags = yield* Effect.forEach(
803 roleCandidates,
804 (r) =>
805 IAM.listRoleTags({ RoleName: r.RoleName }).pipe(
806 Effect.map((t) => ({ role: r, tags: tagRecord(t.Tags) })),
807 Effect.orElseSucceed(() => ({ role: r, tags: {} as Record<string, string> })),
808 ),
809 { concurrency: 5 },
810 );
811 for (const { role, tags } of roleTags) {
812 if (tags["alchemy::stage"] === stage) {
813 leaks.push({
814 arn: role.Arn ?? `arn:aws:iam::${account}:role/${role.RoleName}`,
815 tags,
816 createdAt: toDate(role.CreateDate),
817 });
818 }
819 }
820
821 const policies: import("@distilled.cloud/aws/iam").Policy[] = [];
822 {
823 let marker: string | undefined = undefined;
824 for (let page = 0; page < MAX_PAGES; page++) {
825 const res: IAM.ListPoliciesResponse = yield* IAM.listPolicies({
826 Scope: "Local",
827 MaxItems: 1000,
828 ...(marker ? { Marker: marker } : {}),
829 });
830 policies.push(...(res.Policies ?? []));
831 if (!res.IsTruncated || !res.Marker) break;
832 marker = res.Marker;
833 }
834 }
835 const policyCandidates = policies.filter((p) => (p.PolicyName ?? "").includes(marker));
836 const policyTags = yield* Effect.forEach(
837 policyCandidates,
838 (p) =>
839 IAM.listPolicyTags({ PolicyArn: p.Arn! }).pipe(
840 Effect.map((t) => ({ policy: p, tags: tagRecord(t.Tags) })),
841 Effect.orElseSucceed(() => ({ policy: p, tags: {} as Record<string, string> })),
842 ),
843 { concurrency: 5 },
844 );
845 for (const { policy, tags } of policyTags) {
846 if (tags["alchemy::stage"] === stage) {
847 leaks.push({
848 arn: policy.Arn!,
849 tags,
850 createdAt: toDate(policy.CreateDate),
851 });
852 }
853 }
854 return leaks;
855 });
856
857/**
858 * EventBridge Scheduler schedules are not taggable, so the tagging API can't
859 * find them. Fall back to listing all schedules and matching the alchemy
860 * physical-name shape (`{stack}-{id}-{stage}-{suffix}`).
861 */
862const discoverSchedules = (stage: string, region: string, account: string) =>
863 Effect.gen(function* () {
864 const marker = `-${stage}-`;
865 const schedules: import("@distilled.cloud/aws/scheduler").ScheduleSummary[] = [];
866 {
867 let token: string | undefined = undefined;
868 for (let page = 0; page < MAX_PAGES; page++) {
869 const res: Scheduler.ListSchedulesOutput = yield* Scheduler.listSchedules({
870 MaxResults: 100,
871 ...(token ? { NextToken: token } : {}),
872 });
873 schedules.push(...(res.Schedules ?? []));
874 token = res.NextToken;
875 if (!token) break;
876 }
877 }
878 return schedules
879 .filter((s) => (s.Name ?? "").includes(marker))
880 .map((s) => ({
881 arn:
882 s.Arn ??
883 `arn:aws:scheduler:${region}:${account}:schedule/${s.GroupName ?? "default"}/${s.Name}`,
884 group: s.GroupName ?? "default",
885 name: s.Name ?? "",
886 createdAt: toDate(s.CreationDate),
887 }));
888 }).pipe(Effect.orElseSucceed(() => []));
889
890// ---------------------------------------------------------------------------
891// Report
892// ---------------------------------------------------------------------------
893
894const fmtAge = (h: number | undefined): string =>
895 h === undefined ? "?" : h >= 48 ? `${(h / 24).toFixed(1)}d` : `${h.toFixed(1)}h`;
896
897const printTable = Effect.fn(function* (leaks: Leak[]) {
898 const rows = leaks.map((l) => ({
899 age: fmtAge(ageHours(l.createdAt)),
900 cost: l.hourly ? "HOURLY" : "",
901 kind: l.kind + (l.reportOnly ? " (report-only)" : ""),
902 stack: l.tags["alchemy::stack"] ?? "",
903 id: l.tags["alchemy::id"] ?? "",
904 arn: l.arn,
905 }));
906 const cols = ["age", "cost", "kind", "stack", "id", "arn"] as const;
907 const headers = {
908 age: "AGE",
909 cost: "COST",
910 kind: "KIND",
911 stack: "STACK",
912 id: "ID",
913 arn: "ARN",
914 };
915 const width = (c: (typeof cols)[number]) =>
916 Math.max(headers[c].length, ...rows.map((r) => r[c].length));
917 const line = (r: Record<(typeof cols)[number], string>) =>
918 cols.map((c) => r[c].padEnd(width(c))).join(" ");
919 yield* Console.log(line(headers));
920 yield* Console.log(cols.map((c) => "-".repeat(width(c))).join(" "));
921 for (const r of rows) yield* Console.log(line(r));
922});
923
924// ---------------------------------------------------------------------------
925// Main
926// ---------------------------------------------------------------------------
927
928const main = Effect.fn(function* (args: Args) {
929 yield* Console.log(
930 `aws-leak-sweep: region=${args.region} stage=${args.stage} older-than=${args.olderThanHours}h mode=${args.delete ? "DELETE" : "DRY-RUN"}`,
931 );
932
933 // -- discover ---------------------------------------------------------
934 const tagged = yield* discoverTagged(args.stage);
935 const account =
936 tagged.map((t) => parseArn(t.ResourceARN ?? "")?.account).find((a) => a && a.length > 0) ?? "";
937
938 const leaks: Leak[] = [];
939 const seen = new Set<string>();
940 const push = (leak: Leak) => {
941 if (!seen.has(leak.arn)) {
942 seen.add(leak.arn);
943 leaks.push(leak);
944 }
945 };
946
947 for (const mapping of tagged) {
948 const arn = mapping.ResourceARN;
949 if (!arn) continue;
950 const c = classify(arn);
951 push({
952 arn,
953 kind: c.kind,
954 name: c.name,
955 tags: tagRecord(mapping.Tags),
956 createdAt: undefined,
957 hourly: c.hourly,
958 reportOnly: c.reportOnly,
959 reportOnlyReason: c.reportOnlyReason,
960 phase: c.phase,
961 });
962 }
963
964 const iamLeaks = yield* discoverIam(args.stage, account);
965 for (const l of iamLeaks) {
966 const c = classify(l.arn);
967 push({
968 arn: l.arn,
969 kind: c.kind,
970 name: c.name,
971 tags: l.tags,
972 createdAt: l.createdAt,
973 hourly: c.hourly,
974 reportOnly: c.reportOnly,
975 reportOnlyReason: c.reportOnlyReason,
976 phase: c.phase,
977 });
978 }
979
980 const schedules = yield* discoverSchedules(args.stage, args.region, account);
981 for (const s of schedules) {
982 push({
983 arn: s.arn,
984 kind: "scheduler:schedule",
985 name: `${s.group}/${s.name}`,
986 tags: {},
987 createdAt: s.createdAt,
988 hourly: false,
989 reportOnly: false,
990 phase: 1,
991 });
992 }
993
994 if (leaks.length === 0) {
995 yield* Console.log("No alchemy-tagged test resources found. Clean.");
996 return;
997 }
998
999 // -- enrich ages ------------------------------------------------------
1000 const bucketDates = new Map<string, Date>();
1001 if (leaks.some((l) => l.kind === "s3:bucket")) {
1002 const buckets = yield* S3.listBuckets({}).pipe(Effect.orElseSucceed(() => ({ Buckets: [] })));
1003 for (const b of buckets.Buckets ?? []) {
1004 const d = toDate(b.CreationDate);
1005 if (b.Name && d) bucketDates.set(b.Name, d);
1006 }
1007 }
1008 yield* Effect.forEach(
1009 leaks,
1010 (leak) =>
1011 leak.createdAt !== undefined
1012 ? Effect.void
1013 : lookupCreatedAt(leak, args.region, bucketDates).pipe(
1014 Effect.map((d) => {
1015 leak.createdAt = d;
1016 }),
1017 ),
1018 { concurrency: 8, discard: true },
1019 );
1020
1021 // -- filter by age ----------------------------------------------------
1022 const eligible: Leak[] = [];
1023 let skippedYoung = 0;
1024 for (const leak of leaks) {
1025 const h = ageHours(leak.createdAt);
1026 // unknown age = still a leak; can't prove it's young
1027 if (h === undefined || h >= args.olderThanHours) eligible.push(leak);
1028 else skippedYoung++;
1029 }
1030
1031 eligible.sort((a, b) => (a.createdAt?.getTime() ?? 0) - (b.createdAt?.getTime() ?? 0));
1032
1033 yield* Console.log(
1034 `\nFound ${leaks.length} alchemy '${args.stage}'-stage resources; ${eligible.length} older than ${args.olderThanHours}h (or unknown age), ${skippedYoung} younger (skipped).\n`,
1035 );
1036 yield* printTable(eligible);
1037
1038 const hourlyCount = eligible.filter((l) => l.hourly).length;
1039 if (hourlyCount > 0) {
1040 yield* Console.log(
1041 `\n!! ${hourlyCount} resource(s) bill HOURLY (NAT/EIP/RDS/ELB) — sweep these first.`,
1042 );
1043 }
1044 const reportOnly = eligible.filter((l) => l.reportOnly);
1045 if (reportOnly.length > 0) {
1046 yield* Console.log(`\nReport-only (not deleted by this tool):`);
1047 for (const l of reportOnly) {
1048 yield* Console.log(` - ${l.arn} (${l.reportOnlyReason ?? ""})`);
1049 }
1050 }
1051
1052 if (!args.delete) {
1053 yield* Console.log(
1054 `\nDRY RUN — nothing deleted. Re-run with --delete to remove the ${
1055 eligible.filter((l) => !l.reportOnly).length
1056 } deletable resource(s).`,
1057 );
1058 return;
1059 }
1060
1061 // -- delete: phase order, oldest-first within phase --------------------
1062 const deletable = eligible
1063 .filter((l) => !l.reportOnly)
1064 .sort(
1065 (a, b) =>
1066 a.phase - b.phase ||
1067 (a.createdAt?.getTime() ?? Number.MAX_SAFE_INTEGER) -
1068 (b.createdAt?.getTime() ?? Number.MAX_SAFE_INTEGER),
1069 );
1070
1071 let ok = 0;
1072 let gone = 0;
1073 let failed = 0;
1074 for (const leak of deletable) {
1075 const result = yield* destroyLeak(leak, args.region).pipe(
1076 Effect.timeout("120 seconds"),
1077 Effect.result,
1078 );
1079 if (Result.isFailure(result)) {
1080 const tag =
1081 typeof result.failure === "object" && result.failure !== null && "_tag" in result.failure
1082 ? String(result.failure._tag)
1083 : String(result.failure);
1084 if (GONE.test(tag)) {
1085 gone++;
1086 yield* Console.log(` ~ already gone ${leak.kind} ${leak.arn}`);
1087 } else {
1088 failed++;
1089 yield* Console.log(` x FAILED ${leak.kind} ${leak.arn}: ${tag}`);
1090 }
1091 } else {
1092 ok++;
1093 yield* Console.log(` ✓ deleted ${leak.kind} ${leak.arn}`);
1094 }
1095 }
1096 yield* Console.log(`\nDelete complete: ${ok} deleted, ${gone} already gone, ${failed} failed.`);
1097 if (failed > 0) {
1098 yield* Console.log(
1099 "Failures are usually dependency ordering (NAT->EIP->VPC take minutes); re-run the sweep.",
1100 );
1101 yield* Effect.sync(() => {
1102 process.exitCode = 1;
1103 });
1104 }
1105});
1106
1107const command = Command.make(
1108 "aws-leak-sweep",
1109 {
1110 delete: Flag.Boolean("delete").pipe(
1111 Flag.withDescription("Actually delete the eligible resources (default is a dry run)"),
1112 Flag.withDefault(false),
1113 ),
1114 olderThanHours: Flag.Finite("older-than").pipe(
1115 Flag.withDescription("Only sweep resources at least this many hours old"),
1116 Flag.withDefault(6),
1117 ),
1118 stage: Flag.String("stage").pipe(
1119 Flag.withDescription("alchemy::stage tag to sweep (default test_$USER)"),
1120 Flag.withFallbackConfig(
1121 Config.NonEmptyString("USER").pipe(
1122 Config.orElse(() => Config.NonEmptyString("USERNAME")),
1123 Config.withDefault("unknown"),
1124 Config.map((user) => `test_${user}`),
1125 ),
1126 ),
1127 ),
1128 region: Flag.String("region").pipe(
1129 Flag.withDescription("AWS region (default $AWS_REGION, $AWS_DEFAULT_REGION or us-west-2)"),
1130 Flag.withFallbackConfig(
1131 Config.String("AWS_REGION").pipe(
1132 Config.orElse(() => Config.String("AWS_DEFAULT_REGION")),
1133 Config.withDefault("us-west-2"),
1134 ),
1135 ),
1136 ),
1137 },
1138 (args) =>
1139 main(args).pipe(
1140 Effect.provide(
1141 Layer.mergeAll(
1142 fromChain(),
1143 Layer.succeed(Region, Effect.succeed(args.region as any)),
1144 FetchHttpClient.layer,
1145 ),
1146 ),
1147 ),
1148).pipe(
1149 Command.withDescription("Find (and optionally delete) leaked alchemy-tagged AWS test resources"),
1150);
1151
1152Command.run(command, { version: "0.0.0" }).pipe(
1153 Effect.provide(NodeServices.layer),
1154 NodeRuntime.runMain,
1155);