1import { describe, expect, it } from "@effect/vitest";
2import * as DateTime from "effect/DateTime";
3import * as Effect from "effect/Effect";
4import * as Layer from "effect/Layer";
5import * as Redacted from "effect/Redacted";
6import * as TestClock from "effect/testing/TestClock";
7
8import * as RelayConfiguration from "../Config.ts";
9import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";
10import * as ManagedEndpointProvider from "./ManagedEndpointProvider.ts";
11import * as ManagedEndpointReaper from "./ManagedEndpointReaper.ts";
12
13const NOW = "2026-08-25T12:00:00.000Z";
14const NOW_MILLIS = DateTime.makeUnsafe(NOW).epochMilliseconds;
15const PREFIX = "t3coderelay-managedendpoint-prod-";
16
17function tunnel(input: {
18 readonly id: string;
19 readonly suffix: string;
20 readonly status: "down" | "inactive" | "healthy" | "degraded";
21 readonly timestamp?: string | null;
22 readonly prefix?: string;
23}): ManagedEndpointProvider.ManagedEndpointTunnel {
24 return {
25 id: input.id,
26 name: `${input.prefix ?? PREFIX}${input.suffix}`,
27 status: input.status,
28 ...(input.timestamp === undefined
29 ? {}
30 : input.status === "inactive"
31 ? { createdAt: input.timestamp }
32 : { connsInactiveAt: input.timestamp }),
33 };
34}
35
36function recoverableOwners(
37 tunnels: ReadonlyArray<ManagedEndpointProvider.ManagedEndpointTunnel>,
38): ReadonlyArray<ManagedEndpointAllocations.ManagedEndpointTunnelAllocation> {
39 return tunnels.map((entry) => allocation({ tunnelId: entry.id!, recoveryEnabled: true }));
40}
41
42function allocation(input: {
43 readonly tunnelId: string | null;
44 readonly recoveryEnabled: boolean;
45}): ManagedEndpointAllocations.ManagedEndpointTunnelAllocation {
46 return {
47 userId: "user-1",
48 environmentId: `environment-${input.tunnelId ?? "pending"}`,
49 hostname: `${input.tunnelId ?? "pending"}.example.test`,
50 tunnelId: input.tunnelId,
51 tunnelName: `${PREFIX}aaaaaaaaaaaaaaaa`,
52 dnsRecordId: "dns-1",
53 readyAt: "2026-08-25T11:00:00.000Z",
54 origin: { localHttpHost: "127.0.0.1", localHttpPort: 3773 },
55 updatedAt: "2026-08-25T11:00:00.000Z",
56 generation: 1,
57 tunnelReleasedAt: null,
58 recoveryEnabled: input.recoveryEnabled,
59 };
60}
61
62function harness(input?: {
63 readonly tunnels?: ReadonlyArray<ManagedEndpointProvider.ManagedEndpointTunnel>;
64 readonly allocations?: ReadonlyArray<ManagedEndpointAllocations.ManagedEndpointTunnelAllocation>;
65 readonly namespace?: string;
66 readonly failTunnelId?: string;
67 readonly rateLimitedTunnelId?: string;
68 readonly rateLimitCause?: unknown;
69 readonly failAllDeletes?: boolean;
70 readonly failDeleteWhen?: (tunnelId: string) => boolean;
71 readonly missingOnDeleteTunnelId?: string;
72 readonly missingOnGetTunnelId?: string;
73 readonly reserveOnGetTunnelId?: string;
74 readonly refreshedTunnels?: ReadonlyMap<string, ManagedEndpointProvider.ManagedEndpointTunnel>;
75 readonly skipTunnelId?: string;
76 readonly cleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode;
77 readonly legacyCleanupMode?: RelayConfiguration.ManagedEndpointCleanupMode;
78 readonly legacyTunnelGraceMinutes?: number;
79 readonly releaseDelayMs?: number;
80 readonly listDelayMs?: number;
81}) {
82 const listRequests: ManagedEndpointProvider.ManagedEndpointTunnelListRequest[] = [];
83 const deleted: string[] = [];
84 const releases: Array<
85 Parameters<ManagedEndpointProvider.ManagedEndpointProvider["Service"]["release"]>[0]
86 > = [];
87 const remaining = [...(input?.tunnels ?? [])];
88 const recorded = (input?.allocations ?? []).map((entry) => {
89 const matching = remaining.find((candidate) => candidate.id === entry.tunnelId);
90 return typeof matching?.name === "string" ? { ...entry, tunnelName: matching.name } : entry;
91 });
92 const tunnelClient = ManagedEndpointProvider.ManagedEndpointTunnelClient.of({
93 get: (tunnelId) =>
94 Effect.suspend(() => {
95 if (tunnelId === input?.missingOnGetTunnelId) {
96 return Effect.fail(
97 new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
98 operation: "get",
99 tunnelId,
100 cause: { _tag: "NotFound" },
101 }),
102 );
103 }
104 const found =
105 input?.refreshedTunnels?.get(tunnelId) ??
106 remaining.find((candidate) => candidate.id === tunnelId);
107 if (found === undefined) {
108 return Effect.fail(
109 new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
110 operation: "get",
111 tunnelId,
112 cause: { _tag: "NotFound" },
113 }),
114 );
115 }
116 if (tunnelId === input?.reserveOnGetTunnelId && typeof found.name === "string") {
117 recorded.push({
118 ...allocation({ tunnelId, recoveryEnabled: false }),
119 tunnelName: found.name,
120 });
121 }
122 return Effect.succeed(found);
123 }),
124 list: (request) =>
125 Effect.gen(function* () {
126 if (input?.listDelayMs !== undefined) {
127 yield* TestClock.adjust(input.listDelayMs);
128 }
129 listRequests.push(request);
130 const matching = remaining.filter((entry) => entry.status === request.status);
131 const start = ((request.page ?? 1) - 1) * (request.perPage ?? 100);
132 return {
133 result: matching.slice(start, start + (request.perPage ?? 100)),
134 resultInfo: {
135 page: request.page ?? 1,
136 perPage: request.perPage ?? 100,
137 totalCount: matching.length,
138 },
139 };
140 }),
141 create: () => Effect.die("unused"),
142 putConfiguration: () => Effect.die("unused"),
143 getConfiguration: () => Effect.die("unused"),
144 getToken: () => Effect.die("unused"),
145 delete: (tunnelId) =>
146 input?.failAllDeletes === true ||
147 input?.failDeleteWhen?.(tunnelId) === true ||
148 tunnelId === input?.failTunnelId ||
149 tunnelId === input?.rateLimitedTunnelId ||
150 tunnelId === input?.missingOnDeleteTunnelId
151 ? Effect.fail(
152 new ManagedEndpointProvider.ManagedEndpointTunnelClientError({
153 operation: "delete",
154 tunnelId,
155 cause:
156 tunnelId === input?.missingOnDeleteTunnelId
157 ? { _tag: "NotFound" }
158 : tunnelId === input?.rateLimitedTunnelId
159 ? (input.rateLimitCause ?? {
160 cause: {
161 _tag: "TooManyRequests",
162 message: "Cloudflare rate limit exceeded",
163 retryAfter: 60,
164 },
165 })
166 : "Cloudflare refused the deletion",
167 }),
168 )
169 : Effect.sync(() => {
170 deleted.push(tunnelId);
171 const index = remaining.findIndex((candidate) => candidate.id === tunnelId);
172 if (index !== -1) {
173 remaining.splice(index, 1);
174 }
175 }),
176 });
177 const allocationService = ManagedEndpointAllocations.ManagedEndpointAllocations.of({
178 getByTunnelName: () => Effect.die("unused"),
179 get: () => Effect.die("unused"),
180 reserve: () => Effect.die("unused"),
181 recordTunnel: () => Effect.die("unused"),
182 recordDns: () => Effect.die("unused"),
183 markReady: () => Effect.die("unused"),
184 enableRecovery: () => Effect.die("unused"),
185 listByTunnelNames: (tunnelNames) =>
186 Effect.succeed(recorded.filter((entry) => tunnelNames.includes(entry.tunnelName))),
187 claimRelease: () => Effect.die("unused"),
188 withClaimedTunnel: () => Effect.die("unused"),
189 claimDeprovision: () => Effect.die("unused"),
190 removeClaimed: () => Effect.die("unused"),
191 });
192 const provider = ManagedEndpointProvider.ManagedEndpointProvider.of({
193 provision: () => Effect.die("unused"),
194 reconcileOrigin: () => Effect.die("unused"),
195 prepareDeprovision: () => Effect.die("unused"),
196 deprovision: () => Effect.die("unused"),
197 release: (request) =>
198 Effect.gen(function* () {
199 releases.push(request);
200 if (input?.releaseDelayMs !== undefined) {
201 yield* TestClock.adjust(input.releaseDelayMs);
202 }
203 if (request.expectedTunnelId === input?.skipTunnelId) {
204 return false;
205 }
206 if (request.expectedTunnelId !== undefined) {
207 yield* tunnelClient.delete(request.expectedTunnelId).pipe(
208 Effect.mapError(
209 (cause) =>
210 new ManagedEndpointProvider.ManagedEndpointDeprovisioningFailed({
211 stage: "delete-tunnel",
212 userId: request.userId,
213 environmentId: request.environmentId,
214 tunnelId: request.expectedTunnelId!,
215 cause,
216 }),
217 ),
218 );
219 }
220 return true;
221 }),
222 });
223 const config = RelayConfiguration.RelayConfiguration.of({
224 relayIssuer: "https://relay.example.test",
225 apns: {
226 environment: "sandbox",
227 teamId: "team-id",
228 keyId: "key-id",
229 privateKey: Redacted.make("private-key"),
230 bundleId: "com.t3tools.t3code.dev",
231 },
232 apnsDeliveryJobSigningSecret: Redacted.make("job-secret"),
233 clerkSecretKey: Redacted.make("clerk-secret"),
234 clerkPublishableKey: "pk_test_test",
235 clerkJwtAudience: "t3-code-relay",
236 cloudMintPrivateKey: Redacted.make("cloud-private-key"),
237 cloudMintPublicKey: "cloud-public-key",
238 managedEndpointBaseDomain: "example.test",
239 managedEndpointNamespace: input?.namespace ?? "prod",
240 managedEndpointCleanupMode: input?.cleanupMode ?? "enabled",
241 legacyManagedEndpointCleanupMode: input?.legacyCleanupMode ?? "off",
242 ...(input?.legacyTunnelGraceMinutes === undefined
243 ? {}
244 : { legacyTunnelGraceMinutes: input.legacyTunnelGraceMinutes }),
245 });
246
247 return {
248 listRequests,
249 deleted,
250 releases,
251 layer: ManagedEndpointReaper.layer.pipe(
252 Layer.provide(
253 Layer.mergeAll(
254 RelayConfiguration.layer(config),
255 ManagedEndpointProvider.layerTunnelClient(tunnelClient),
256 Layer.succeed(ManagedEndpointProvider.ManagedEndpointProvider, provider),
257 Layer.succeed(ManagedEndpointAllocations.ManagedEndpointAllocations, allocationService),
258 ),
259 ),
260 ),
261 };
262}
263
264describe("ManagedEndpointReaper", () => {
265 it.effect("removes expired down and inactive tunnels from recoverable environments", () => {
266 const state = harness({
267 tunnels: [
268 tunnel({
269 id: "down-1",
270 suffix: "aaaaaaaaaaaaaaaa",
271 status: "down",
272 timestamp: "2026-08-25T11:55:00.000Z",
273 }),
274 tunnel({
275 id: "inactive-1",
276 suffix: "bbbbbbbbbbbbbbbb",
277 status: "inactive",
278 timestamp: "2026-08-25T10:59:00.000Z",
279 }),
280 ],
281 allocations: [
282 allocation({ tunnelId: "down-1", recoveryEnabled: true }),
283 allocation({ tunnelId: "inactive-1", recoveryEnabled: true }),
284 ],
285 });
286
287 return Effect.gen(function* () {
288 yield* TestClock.setTime(NOW_MILLIS);
289 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
290 expect(yield* reaper.sweep).toMatchObject({
291 scanned: 2,
292 deleted: 2,
293 skippedLegacy: 0,
294 failed: 0,
295 });
296 expect(state.deleted).toEqual(["down-1", "inactive-1"]);
297 expect(state.releases.some((release) => release.markReleased === true)).toBe(false);
298 expect(state.releases.map((request) => request.expectedTunnelId)).toEqual([
299 "down-1",
300 "inactive-1",
301 ]);
302 expect(state.listRequests).toEqual([
303 {
304 isDeleted: false,
305 includePrefix: PREFIX,
306 status: "down",
307 existedAt: "2026-08-25T11:55:00.000Z",
308 wasInactiveAt: "2026-08-25T11:55:00.000Z",
309 page: 1,
310 perPage: 100,
311 },
312 {
313 isDeleted: false,
314 includePrefix: PREFIX,
315 status: "inactive",
316 existedAt: "2026-08-25T11:00:00.000Z",
317 page: 1,
318 perPage: 100,
319 },
320 ]);
321 }).pipe(Effect.provide(state.layer));
322 });
323
324 it.effect("keeps recent tunnels, other stages, and tunnels without valid timestamps", () => {
325 const state = harness({
326 tunnels: [
327 tunnel({
328 id: "recent",
329 suffix: "aaaaaaaaaaaaaaaa",
330 status: "down",
331 timestamp: "2026-08-25T11:55:01.000Z",
332 }),
333 tunnel({
334 id: "other-stage",
335 prefix: `${PREFIX}julius-`,
336 suffix: "bbbbbbbbbbbbbbbb",
337 status: "down",
338 timestamp: "2026-08-25T11:00:00.000Z",
339 }),
340 tunnel({
341 id: "missing-time",
342 suffix: "cccccccccccccccc",
343 status: "inactive",
344 timestamp: null,
345 }),
346 ],
347 });
348
349 return Effect.gen(function* () {
350 yield* TestClock.setTime(NOW_MILLIS);
351 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
352 expect(yield* reaper.sweep).toMatchObject({
353 scanned: 0,
354 deleted: 0,
355 skippedLegacy: 0,
356 failed: 0,
357 });
358 expect(state.deleted).toEqual([]);
359 }).pipe(Effect.provide(state.layer));
360 });
361
362 it.effect("keeps tunnels owned by environments that cannot recover yet", () => {
363 const state = harness({
364 tunnels: [
365 tunnel({
366 id: "legacy",
367 suffix: "aaaaaaaaaaaaaaaa",
368 status: "down",
369 timestamp: "2026-08-25T11:00:00.000Z",
370 }),
371 ],
372 allocations: [allocation({ tunnelId: "legacy", recoveryEnabled: false })],
373 });
374
375 return Effect.gen(function* () {
376 yield* TestClock.setTime(NOW_MILLIS);
377 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
378 expect(yield* reaper.sweep).toMatchObject({
379 scanned: 1,
380 deleted: 0,
381 skippedLegacy: 1,
382 failed: 0,
383 });
384 expect(state.deleted).toEqual([]);
385 }).pipe(Effect.provide(state.layer));
386 });
387
388 it.effect("counts an expired tunnel with no allocation instead of deleting it", () => {
389 const state = harness({
390 tunnels: [
391 tunnel({
392 id: "orphan",
393 suffix: "cccccccccccccccc",
394 status: "down",
395 timestamp: "2026-08-25T11:00:00.000Z",
396 }),
397 ],
398 });
399
400 return Effect.gen(function* () {
401 yield* TestClock.setTime(NOW_MILLIS);
402 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
403 expect(yield* reaper.sweep).toMatchObject({
404 scanned: 1,
405 wouldDelete: 0,
406 deleted: 0,
407 skippedOrphan: 1,
408 });
409 expect(state.deleted).toEqual([]);
410 }).pipe(Effect.provide(state.layer));
411 });
412 it.effect("keeps an expired tunnel while its allocation is incomplete", () => {
413 const state = harness({
414 tunnels: [
415 tunnel({
416 id: "unrecorded",
417 suffix: "aaaaaaaaaaaaaaaa",
418 status: "inactive",
419 timestamp: "2026-08-25T11:00:00.000Z",
420 }),
421 ],
422 allocations: [allocation({ tunnelId: null, recoveryEnabled: false })],
423 });
424
425 return Effect.gen(function* () {
426 yield* TestClock.setTime(NOW_MILLIS);
427 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
428 expect((yield* reaper.sweep).deleted).toBe(0);
429 expect(state.deleted).toEqual([]);
430 }).pipe(Effect.provide(state.layer));
431 });
432
433 describe("legacy cleanup", () => {
434 const legacyTunnels = [
435 tunnel({
436 id: "legacy-recent",
437 suffix: "1111111111111111",
438 status: "down",
439 timestamp: "2026-08-20T12:00:00.000Z",
440 }),
441 tunnel({
442 id: "legacy-old",
443 suffix: "2222222222222222",
444 status: "down",
445 timestamp: "2026-07-16T12:00:00.000Z",
446 }),
447 tunnel({
448 id: "legacy-never",
449 suffix: "3333333333333333",
450 status: "inactive",
451 timestamp: "2026-07-16T12:00:00.000Z",
452 }),
453 ];
454 const legacyOwners = legacyTunnels.map((entry) =>
455 allocation({ tunnelId: entry.id!, recoveryEnabled: false }),
456 );
457
458 it.effect("leaves legacy tunnels alone while legacy cleanup is off", () => {
459 const state = harness({ tunnels: legacyTunnels, allocations: legacyOwners });
460 return Effect.gen(function* () {
461 yield* TestClock.setTime(NOW_MILLIS);
462 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
463 expect(yield* reaper.sweep).toMatchObject({
464 skippedLegacy: 3,
465 wouldDeleteLegacy: 0,
466 deletedLegacy: 0,
467 });
468 expect(state.deleted).toEqual([]);
469 }).pipe(Effect.provide(state.layer));
470 });
471
472 it.effect("deletes only legacy tunnels past the grace period", () => {
473 const state = harness({
474 tunnels: legacyTunnels,
475 allocations: legacyOwners,
476 legacyCleanupMode: "enabled",
477 });
478 return Effect.gen(function* () {
479 yield* TestClock.setTime(NOW_MILLIS);
480 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
481 expect(yield* reaper.sweep).toMatchObject({
482 wouldDeleteLegacy: 2,
483 deletedLegacy: 2,
484 deleted: 2,
485 });
486 expect([...state.deleted].sort()).toEqual(["legacy-never", "legacy-old"]);
487 expect(state.releases.every((release) => release.markReleased === true)).toBe(true);
488 for (const release of state.releases) {
489 expect(release.expectedInactiveBefore).toBe("2026-08-18T12:00:00.000Z");
490 }
491 }).pipe(Effect.provide(state.layer));
492 });
493
494 it.effect.each([
495 { stage: "canary", expected: 3 },
496 { stage: "prod", expected: 2 },
497 ] as const)(
498 "applies the grace-period override only off prod ($stage)",
499 ({ stage, expected }) => {
500 const stageLegacyTunnels = legacyTunnels.map((entry) => ({
501 ...entry,
502 name: entry.name!.replace(PREFIX, `t3coderelay-managedendpoint-${stage}-`),
503 }));
504 const state = harness({
505 namespace: stage,
506 tunnels: stageLegacyTunnels,
507 allocations: legacyOwners,
508 legacyCleanupMode: "dry-run",
509 legacyTunnelGraceMinutes: 10,
510 });
511 return Effect.gen(function* () {
512 yield* TestClock.setTime(NOW_MILLIS);
513 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
514 expect((yield* reaper.sweep).wouldDeleteLegacy).toBe(expected);
515 }).pipe(Effect.provide(state.layer));
516 },
517 );
518
519 it.effect("runs legacy cleanup while recoverable cleanup is off", () => {
520 const recoverable = tunnel({
521 id: "recoverable",
522 suffix: "4444444444444444",
523 status: "down",
524 timestamp: "2026-08-25T11:00:00.000Z",
525 });
526 const state = harness({
527 tunnels: [...legacyTunnels, recoverable],
528 allocations: [
529 ...legacyOwners,
530 allocation({ tunnelId: "recoverable", recoveryEnabled: true }),
531 ],
532 cleanupMode: "off",
533 legacyCleanupMode: "enabled",
534 });
535 return Effect.gen(function* () {
536 yield* TestClock.setTime(NOW_MILLIS);
537 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
538 expect(yield* reaper.sweep).toMatchObject({ wouldDelete: 0, deletedLegacy: 2 });
539 expect(state.deleted).not.toContain("recoverable");
540 }).pipe(Effect.provide(state.layer));
541 });
542 });
543
544 it.effect.each([
545 ["a structured Cloudflare rate limit error", undefined],
546 [
547 "a Cloudflare API pause recorded by another isolate",
548 new ManagedEndpointProvider.CloudflareApiPaused({ retryAfterMillis: 30_000 }),
549 ],
550 ] as const)("stops the sweep after %s", ([, rateLimitCause]) => {
551 const state = harness({
552 tunnels: [
553 tunnel({
554 id: "limited",
555 suffix: "aaaaaaaaaaaaaaaa",
556 status: "down",
557 timestamp: "2026-08-25T11:00:00.000Z",
558 }),
559 tunnel({
560 id: "next",
561 suffix: "bbbbbbbbbbbbbbbb",
562 status: "down",
563 timestamp: "2026-08-25T11:00:00.000Z",
564 }),
565 ],
566 allocations: [
567 allocation({ tunnelId: "limited", recoveryEnabled: true }),
568 allocation({ tunnelId: "next", recoveryEnabled: true }),
569 ],
570 rateLimitedTunnelId: "limited",
571 rateLimitCause,
572 });
573
574 return Effect.gen(function* () {
575 yield* TestClock.setTime(NOW_MILLIS);
576 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
577 expect(yield* reaper.sweep).toMatchObject({
578 attempted: 1,
579 deleted: 0,
580 failed: 1,
581 truncated: true,
582 });
583 expect(state.deleted).toEqual([]);
584 }).pipe(Effect.provide(state.layer));
585 });
586
587 it.effect("continues past a page of older hosts to find recoverable tunnels", () => {
588 const entries = Array.from({ length: 101 }, (_, index) =>
589 tunnel({
590 id: `tunnel-${index}`,
591 suffix: index.toString(16).padStart(16, "0"),
592 status: "down",
593 timestamp: "2026-08-25T11:00:00.000Z",
594 }),
595 );
596 const state = harness({
597 tunnels: entries,
598 allocations: entries.map((entry, index) =>
599 allocation({ tunnelId: entry.id!, recoveryEnabled: index === 100 }),
600 ),
601 });
602
603 return Effect.gen(function* () {
604 yield* TestClock.setTime(NOW_MILLIS);
605 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
606 expect(yield* reaper.sweep).toMatchObject({
607 scanned: 101,
608 deleted: 1,
609 skippedLegacy: 100,
610 failed: 0,
611 });
612 expect(state.deleted).toEqual(["tunnel-100"]);
613 expect(
614 state.listRequests
615 .filter((request) => request.status === "down")
616 .map((request) => request.page),
617 ).toEqual([1, 2]);
618 }).pipe(Effect.provide(state.layer));
619 });
620
621 it.effect("collects every page before deletions shift Cloudflare pagination", () => {
622 const entries = Array.from({ length: 120 }, (_, index) =>
623 tunnel({
624 id: `tunnel-${index}`,
625 suffix: index.toString(16).padStart(16, "0"),
626 status: "down",
627 timestamp: "2026-08-25T11:00:00.000Z",
628 }),
629 );
630 const state = harness({
631 tunnels: entries,
632 allocations: entries.map((entry, index) =>
633 allocation({ tunnelId: entry.id!, recoveryEnabled: index < 50 || index >= 100 }),
634 ),
635 });
636
637 return Effect.gen(function* () {
638 yield* TestClock.setTime(NOW_MILLIS);
639 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
640 expect(yield* reaper.sweep).toMatchObject({
641 scanned: 120,
642 deleted: 70,
643 skippedLegacy: 50,
644 failed: 0,
645 });
646 expect(state.deleted).toContain("tunnel-119");
647 }).pipe(Effect.provide(state.layer));
648 });
649
650 it.effect("reports candidates without mutating them in dry-run mode", () => {
651 const state = harness({
652 cleanupMode: "dry-run",
653 tunnels: [
654 tunnel({
655 id: "expired",
656 suffix: "aaaaaaaaaaaaaaaa",
657 status: "down",
658 timestamp: "2026-08-25T11:00:00.000Z",
659 }),
660 ],
661 allocations: [allocation({ tunnelId: "expired", recoveryEnabled: true })],
662 });
663
664 return Effect.gen(function* () {
665 yield* TestClock.setTime(NOW_MILLIS);
666 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
667 expect(yield* reaper.sweep).toMatchObject({
668 mode: "dry-run",
669 scanned: 1,
670 attempted: 0,
671 deleted: 0,
672 wouldDelete: 1,
673 });
674 expect(state.deleted).toEqual([]);
675 expect(state.releases).toEqual([]);
676 }).pipe(Effect.provide(state.layer));
677 });
678
679 it.effect("rotates the deletion order so persistent failures do not starve later tunnels", () => {
680 const limit = ManagedEndpointReaper.MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT;
681 const entries = Array.from({ length: limit * 2 }, (_, index) =>
682 tunnel({
683 id: `tunnel-${index}`,
684 suffix: index.toString(16).padStart(16, "0"),
685 status: "down",
686 timestamp: "2026-08-25T11:00:00.000Z",
687 }),
688 );
689 const state = harness({
690 tunnels: entries,
691 allocations: recoverableOwners(entries),
692 failDeleteWhen: (id) => Number(id.split("-")[1]) < limit,
693 });
694
695 return Effect.gen(function* () {
696 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
697 yield* TestClock.setTime(NOW_MILLIS);
698 yield* reaper.sweep;
699 yield* TestClock.setTime(NOW_MILLIS + 5 * 60 * 1_000);
700 yield* reaper.sweep;
701 const later = state.deleted.filter((id) => Number(id.split("-")[1]) >= limit);
702 expect(later.length).toBeGreaterThan(0);
703 }).pipe(Effect.provide(state.layer));
704 });
705
706 it.effect("rotates the attempt budget across both statuses over consecutive sweeps", () => {
707 const limit = ManagedEndpointReaper.MANAGED_ENDPOINT_SWEEP_ATTEMPT_LIMIT;
708 const entries = (["down", "inactive"] as const).flatMap((status) =>
709 Array.from({ length: limit }, (_, index) =>
710 tunnel({
711 id: `${status}-${index}`,
712 suffix: `${status === "down" ? "a" : "b"}${index.toString(16).padStart(15, "0")}`,
713 status,
714 timestamp: "2026-08-25T10:00:00.000Z",
715 }),
716 ),
717 );
718 const state = harness({ tunnels: entries, allocations: recoverableOwners(entries) });
719
720 return Effect.gen(function* () {
721 const reaper = yield* ManagedEndpointReaper.ManagedEndpointReaper;
722 yield* TestClock.setTime(NOW_MILLIS);
723 yield* reaper.sweep;
724 const firstStatus = state.deleted[0]!.split("-")[0];
725 expect(state.deleted).toHaveLength(limit);
726 expect(state.deleted.every((id) => id.startsWith(`${firstStatus}-`))).toBe(true);
727
728 state.deleted.length = 0;
729 yield* TestClock.setTime(NOW_MILLIS + 5 * 60 * 1_000);
730 yield* reaper.sweep;
731 expect(state.deleted).toHaveLength(limit);
732 expect(state.deleted.every((id) => !id.startsWith(`${firstStatus}-`))).toBe(true);
733 }).pipe(Effect.provide(state.layer));
734 });
735});