Open History to see the last commit
1/**
2 * The pack hasher on AWS Lambda, end to end (DESIGN §22.11): the Git host
3 * Worker binds `InvokeFunction` cross-cloud, pushes stream through Lambda
4 * chunks, and the result clones back byte-identical under `fsck --strict`.
5 * Needs BOTH provider sets (`--profile testing`; AWS via SSO).
6 */
7import * as AWS from "@alchemy.run/aws";
8import * as Cloudflare from "@alchemy.run/cloudflare";
9import { GitApi } from "@alchemy.run/git/Api";
10import { expect } from "@alchemy.run/test";
11import * as Test from "@alchemy.run/test/Alchemy";
12import * as Data from "effect/Data";
13import * as Effect from "effect/Effect";
14import * as FileSystem from "effect/FileSystem";
15import * as HttpApiClient from "effect/http-api/HttpApiClient";
16import * as HttpClient from "effect/http/HttpClient";
17import * as HttpClientRequest from "effect/http/HttpClientRequest";
18import * as Layer from "effect/Layer";
19import * as ChildProcess from "effect/process/ChildProcess";
20import { MinimumLogLevel } from "effect/References";
21import * as Schedule from "effect/Schedule";
22import * as Stream from "effect/Stream";
23import { makeLambdaTestStack, TEST_SECRET } from "./fixtures/lambda-stack.ts";
24
25const { test, beforeAll, afterAll, deploy, destroy } = Test.make({
26 providers: Layer.mergeAll(AWS.providers(), Cloudflare.providers()),
27});
28
29const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info");
30
31const Stack = makeLambdaTestStack("GitHasherLambdaStack");
32
33const makeClient = (url: string, token: string) =>
34 HttpApiClient.make(GitApi, {
35 baseUrl: url,
36 transformClient: HttpClient.mapRequest((request) =>
37 request.pipe(HttpClientRequest.bearerToken(token)),
38 ),
39 });
40
41const edgeRetry = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
42 effect.pipe(
43 Effect.timeout("10 seconds"),
44 Effect.retry({
45 while: (error: unknown) =>
46 (error as { _tag?: string })._tag === "TimeoutError" ||
47 (error as { _tag?: string })._tag === "HttpClientError",
48 schedule: Schedule.spaced("1500 millis"),
49 times: 20,
50 }),
51 );
52
53class ShellError extends Data.TaggedError("ShellError")<{
54 readonly script: string;
55 readonly exitCode: number;
56 readonly stdout: string;
57 readonly stderr: string;
58}> {
59 override get message(): string {
60 return `shell script exited ${this.exitCode}\n--- script ---\n${this.script}\n--- stdout ---\n${this.stdout}\n--- stderr ---\n${this.stderr}`;
61 }
62}
63
64const sh = Effect.fn(function* (cwd: string, script: string) {
65 const handle = yield* ChildProcess.make("sh", ["-ec", script], {
66 cwd,
67 env: {
68 GIT_TERMINAL_PROMPT: "0",
69 GIT_CONFIG_NOSYSTEM: "1",
70 GIT_CONFIG_GLOBAL: "/dev/null",
71 GIT_AUTHOR_NAME: "Hasher Bot",
72 GIT_AUTHOR_EMAIL: "hasher@example.com",
73 GIT_COMMITTER_NAME: "Hasher Bot",
74 GIT_COMMITTER_EMAIL: "hasher@example.com",
75 GIT_AUTHOR_DATE: "2026-01-01T00:00:00Z",
76 GIT_COMMITTER_DATE: "2026-01-01T00:00:00Z",
77 },
78 extendEnv: true,
79 });
80 const [exitCode, stdout, stderr] = yield* Effect.all(
81 [
82 handle.exitCode,
83 Stream.mkString(Stream.decodeText(handle.stdout)),
84 Stream.mkString(Stream.decodeText(handle.stderr)),
85 ],
86 { concurrency: 3 },
87 );
88 if (exitCode !== 0) {
89 return yield* new ShellError({ script, exitCode, stdout, stderr });
90 }
91 return { stdout: stdout.trim(), stderr };
92}, Effect.timeout("180 seconds"));
93
94const stack = beforeAll(
95 deploy(Stack).pipe(
96 Effect.tap(({ url }) =>
97 Effect.gen(function* () {
98 yield* Effect.logInfo(`git-service (lambda hasher) deployed at ${url}`);
99 const admin = yield* makeClient(url, TEST_SECRET);
100 yield* admin.repos.list({ query: {} }).pipe(
101 edgeRetry,
102 Effect.retry({
103 schedule: Schedule.spaced("1500 millis"),
104 times: 40,
105 }),
106 );
107 }),
108 ),
109 logLevel,
110 ),
111);
112afterAll.skipIf(!!process.env.NO_DESTROY)(destroy(Stack).pipe(logLevel));
113
114test(
115 "a multi-part push is hashed on Lambda and clones back under fsck --strict",
116 Effect.gen(function* () {
117 const { url } = yield* stack;
118 const admin = yield* makeClient(url, TEST_SECRET);
119 const owner = "lambda";
120 const name = "hashed";
121 yield* admin.repos.delete({ params: { owner, repo: name } }).pipe(
122 Effect.catchTag("RepoNotFound", () => Effect.void),
123 edgeRetry,
124 );
125 const created = yield* admin.repos.create({ payload: { owner, name } }).pipe(edgeRetry);
126 const parsed = new URL(url);
127 const remote = `${parsed.protocol}//x:${TEST_SECRET}@${parsed.host}/${owner}/${name}.git`;
128 const fs = yield* FileSystem.FileSystem;
129 const dir = yield* fs.makeTempDirectory({ prefix: "git-lambda-" });
130 // ~14 MiB of incompressible blobs plus text files that delta well: a
131 // body that spills (four Lambda chunks, two R2 parts) and carries
132 // both non-delta and delta entries across chunk edges.
133 yield* sh(
134 dir,
135 `git init -q -b main src && cd src
136 for i in $(seq 1 14); do head -c 1000000 /dev/urandom > bin-$i.dat; done
137 for i in $(seq 1 200); do seq 1 400 | sed "s/^/line $i /" > text-$i.txt; done
138 git add -A && git commit -qm base
139 for i in $(seq 1 60); do echo "edit $i" >> text-$i.txt; done
140 git add -A && git commit -qm edits
141 git push -q ${remote} main`,
142 );
143 const back = yield* sh(
144 dir,
145 `git clone -q ${remote} back && cd back && git fsck --strict && git rev-parse HEAD`,
146 );
147 const head = yield* sh(dir, "cd src && git rev-parse HEAD");
148 expect(back.stdout.trim().split("\n").pop()).toBe(head.stdout.trim());
149 const repo = yield* admin.repos.get({ params: { owner, repo: name } }).pipe(edgeRetry);
150 const push = repo.lastPush;
151 expect(push).not.toBeNull();
152 // Four 4 MiB chunks were dispatched to Lambda; nothing fell back.
153 expect(push?.phases?.chunks ?? 0).toBeGreaterThanOrEqual(4);
154 yield* Effect.logInfo(`lastPush: ${JSON.stringify(push)}`);
155 }),
156 {
157 tags: [
158 "provider:aws",
159 "provider:aws:lambda",
160 "provider:cloudflare",
161 "provider:cloudflare:r2",
162 "provider:cloudflare:worker",
163 "live",
164 ],
165 timeout: 600_000,
166 },
167);