Open History to see the last commit
1#!/usr/bin/env node
2// Runs one scenario with `e2e run` on a base and a head commit and says whether it fails on
3// base and passes on head. See docs/operations/scenarios.md.
4import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
5import * as NodeServices from "@effect/platform-node/NodeServices";
6import * as HostProcess from "@t3tools/shared/HostProcess";
7import { fromJsonStringPretty } from "@t3tools/shared/schemaJson";
8import * as Console from "effect/Console";
9import * as Effect from "effect/Effect";
10import * as FileSystem from "effect/FileSystem";
11import * as Option from "effect/Option";
12import * as Path from "effect/Path";
13import * as Schema from "effect/Schema";
14import { Argument, Command, Flag } from "effect/cli";
15
16import { run, ScenarioError } from "./scenarios/shell.ts";
17
18/** The parts of e2e's report.json (schema/report-v1.schema.json) a proof reads. */
19const E2EReport = Schema.Struct({
20 run: Schema.Struct({
21 results: Schema.Array(
22 Schema.Struct({
23 titlePath: Schema.Array(Schema.String),
24 attempts: Schema.Array(
25 Schema.Struct({
26 status: Schema.optional(Schema.String),
27 steps: Schema.Array(
28 Schema.Struct({
29 api: Schema.String,
30 label: Schema.optional(Schema.String),
31 status: Schema.String,
32 }),
33 ),
34 error: Schema.optional(Schema.Struct({ code: Schema.String, message: Schema.String })),
35 artifacts: Schema.optional(
36 Schema.Array(
37 Schema.Struct({ kind: Schema.String, path: Schema.optional(Schema.String) }),
38 ),
39 ),
40 }),
41 ),
42 }),
43 ),
44 }),
45});
46type E2EReport = typeof E2EReport.Type;
47const decodeReport = Schema.decodeUnknownEffect(Schema.fromJsonString(E2EReport));
48const encodeProof = Schema.encodeEffect(fromJsonStringPretty(Schema.Unknown));
49
50export interface SideResult {
51 readonly sha: string;
52 /** `error`: the scenario could not run, which says nothing about the behavior. */
53 readonly status: "passed" | "failed" | "error";
54 readonly title: string;
55 readonly message?: string;
56 readonly steps: ReadonlyArray<{ readonly step: string; readonly status: string }>;
57 /** Capture paths under the run's results directory, in the order the scenario took them. */
58 readonly captures: ReadonlyArray<string>;
59 readonly videos: ReadonlyArray<string>;
60}
61
62/** Only a failed expectation shows the bug. A missing control or a dead app shows nothing. */
63export function readSide(
64 sha: string,
65 resultsDir: string,
66 report: E2EReport | undefined,
67): SideResult {
68 // The report also lists the tests the run did not select, with no attempts.
69 const executed = report?.run.results.filter((candidate) => candidate.attempts.length > 0) ?? [];
70 // One scenario is one test: with several, a pass in the first could hide a failure in another.
71 const result = executed.length === 1 ? executed[0] : undefined;
72 const attempt = result?.attempts.at(-1);
73 if (result === undefined || attempt === undefined) {
74 const message =
75 executed.length > 1
76 ? `the scenario file ran ${executed.length} tests; a proof needs exactly one`
77 : "e2e wrote no report: the app did not start, or no test was selected";
78 return { sha, status: "error", title: "", message, steps: [], captures: [], videos: [] };
79 }
80 const files = (kind: string) =>
81 (attempt.artifacts ?? []).flatMap((artifact) =>
82 artifact.kind === kind && artifact.path !== undefined
83 ? [`${resultsDir}/${artifact.path}`]
84 : [],
85 );
86 const error = attempt.error;
87 return {
88 sha,
89 // e2e exits 0 for a skipped test too, and a test that did not run proves nothing.
90 status:
91 attempt.status === "passed"
92 ? "passed"
93 : error?.code === "ASSERTION_FAILED"
94 ? "failed"
95 : "error",
96 title: result.titlePath.join(" > "),
97 ...(error === undefined
98 ? attempt.status === "passed"
99 ? {}
100 : { message: `the test ended ${attempt.status ?? "without a status"}` }
101 : { message: `${error.code}: ${error.message}` }),
102 steps: attempt.steps.map((step) => ({
103 // The sign-in step's label holds the run's pairing token.
104 step: `${step.api} ${step.label?.includes("token=") ? "<pairing link>" : (step.label ?? "")}`,
105 status: step.status,
106 })),
107 captures: files("screenshot"),
108 videos: files("video"),
109 };
110}
111
112const short = (sha: string) => sha.slice(0, 10);
113const captureLabel = (path: string) =>
114 (path.split("/").at(-1) ?? "")
115 .replace(/\.png$/, "")
116 .replace(/^\d+-/, "")
117 .replaceAll("-", " ");
118
119/** `link` maps a capture path to what the Markdown should point at, such as an uploaded URL. */
120export function buildProof(base: SideResult, head: SideResult, link: (path: string) => string) {
121 const baseSha = `\`${short(base.sha)}\``;
122 const headSha = `\`${short(head.sha)}\``;
123 const broken = base.status === "error" ? base : head.status === "error" ? head : undefined;
124 const verdict =
125 broken !== undefined
126 ? `Not proven: the scenario could not run on ${broken === base ? `base ${baseSha}` : `head ${headSha}`} (${broken.message ?? "unknown error"}).`
127 : base.status === "passed"
128 ? head.status === "passed"
129 ? `Not proven: the scenario passes on base ${baseSha} too, so it does not reproduce the bug.`
130 : `Not proven: the scenario passes on base ${baseSha} and fails on head ${headSha}, so head breaks it.`
131 : head.status === "passed"
132 ? `Proven: the scenario fails on base ${baseSha} and passes on head ${headSha}.`
133 : `Not proven: the scenario fails on base ${baseSha} and still fails on head ${headSha}.`;
134
135 const labels = [...new Set([...base.captures, ...head.captures].map(captureLabel))];
136 const cell = (side: string, result: SideResult, label: string) => {
137 const capture = result.captures.find((path) => captureLabel(path) === label);
138 return capture === undefined
139 ? ""
140 : `})`;
141 };
142 const outcome = (result: SideResult) =>
143 result.status === "passed"
144 ? "✅ passed"
145 : `${result.status === "failed" ? "❌ failed" : "⚠️ could not run"}<br>${(result.message ?? "").replaceAll("\n", "<br>").replaceAll("|", "\\|").replaceAll("<br><br>", "<br>")}`;
146 const markdown = [
147 `### Scenario: ${head.title || base.title}`,
148 "",
149 `**${verdict}**`,
150 "",
151 `| | Base ${baseSha} | Head ${headSha} |`,
152 "| --- | --- | --- |",
153 `| Result | ${outcome(base)} | ${outcome(head)} |`,
154 ...labels.map(
155 (label) => `| ${label} | ${cell("Base", base, label)} | ${cell("Head", head, label)} |`,
156 ),
157 "",
158 ].join("\n");
159 return {
160 scenario: head.title || base.title,
161 failsOnBase: base.status === "failed",
162 passesOnHead: head.status === "passed",
163 baseSha: base.sha,
164 headSha: head.sha,
165 verdict,
166 steps: { base: base.steps, head: head.steps },
167 captures: [...base.captures, ...head.captures],
168 videos: [...base.videos, ...head.videos],
169 markdown,
170 base,
171 head,
172 };
173}
174
175const roots = Effect.gen(function* () {
176 const path = yield* Path.Path;
177 const scripts = yield* path.fromFileUrl(new URL(".", import.meta.url));
178 return { scripts, repo: path.dirname(path.resolve(scripts)) };
179});
180
181const git = Effect.fn("prove.git")(function* (args: ReadonlyArray<string>) {
182 return (yield* run("git", args, { cwd: (yield* roots).repo })).stdout.trim();
183});
184
185/** Runs `use` with a checkout of `sha`: this one when it already is that commit, else a worktree. */
186const withCheckout = Effect.fn("prove.withCheckout")(function* <A, E, R>(
187 sha: string,
188 use: (appDir: string) => Effect.Effect<A, E, R>,
189) {
190 const { repo } = yield* roots;
191 const clean = (yield* git(["status", "--porcelain", "--untracked-files=no"])).length === 0;
192 if (clean && (yield* git(["rev-parse", "HEAD"])) === sha) return yield* use(repo);
193 // Beside this checkout, not in the temp directory, so tooling resolves as it does here.
194 const worktree = `${repo}-scenario-${short(sha)}`;
195 return yield* Effect.acquireUseRelease(
196 git(["worktree", "add", "--quiet", "--detach", "--force", worktree, sha]),
197 () => run("vp", ["i"], { cwd: worktree, echo: true }).pipe(Effect.andThen(use(worktree))),
198 () => git(["worktree", "remove", "--force", worktree]).pipe(Effect.ignore),
199 );
200});
201
202/** Runs the scenario with `e2e run` against the app at `sha` and reads the report it wrote. */
203const runSide = Effect.fn("prove.runSide")(function* (
204 side: "base" | "head",
205 rev: string,
206 scenarioPath: string,
207) {
208 const fs = yield* FileSystem.FileSystem;
209 const path = yield* Path.Path;
210 const { scripts } = yield* roots;
211 const sha = yield* git(["rev-parse", "--verify", `${rev}^{commit}`]);
212 const outDir = path.join(scripts, ".e2e/prove", side);
213 yield* fs.remove(outDir, { recursive: true, force: true });
214 const environment = yield* HostProcess.Environment;
215 yield* withCheckout(sha, (appDir) =>
216 // The scenario and e2e come from this checkout; only the app comes from `appDir`.
217 run(
218 path.join(scripts, "node_modules/.bin/e2e"),
219 ["run", scenarioPath, "--output", path.relative(scripts, outDir)],
220 {
221 cwd: scripts,
222 env: { ...environment, T3_SCENARIO_APP_DIR: appDir, E2E_TELEMETRY_DISABLED: "1" },
223 // The report says how the run went; e2e's exit code only repeats it.
224 allowExit: [0, 1, 2, 3, 4],
225 echo: true,
226 },
227 ),
228 );
229 const report = yield* fs.readFileString(path.join(outDir, "report.json")).pipe(
230 Effect.flatMap(decodeReport),
231 Effect.orElseSucceed(() => undefined),
232 );
233 return readSide(sha, path.join(outDir, "results"), report);
234});
235
236/** Uploads the captures and maps each to its URL; a failed upload leaves the map empty. */
237const upload = Effect.fn("prove.upload")(function* (
238 uploadCommand: string,
239 captures: ReadonlyArray<string>,
240) {
241 const [command = "", ...args] = uploadCommand.split(/\s+/).filter(Boolean);
242 if (captures.length === 0) return new Map<string, string>();
243 return yield* run(command, [...args, ...captures]).pipe(
244 Effect.map((result) => result.stdout.trim().split("\n")),
245 // Lines pair with captures by position, so a different count would mislabel every image.
246 Effect.filterOrFail(
247 (urls) => urls.length === captures.length,
248 () => new ScenarioError({ message: "The upload command printed an unexpected line count." }),
249 ),
250 Effect.map((urls) => new Map(captures.map((capture, index) => [capture, urls[index] ?? ""]))),
251 // Two finished runs are worth more than their links: the proof then names local files.
252 Effect.catch((error) =>
253 Console.error(`Upload failed, linking local captures: ${error.message}`).pipe(
254 Effect.as(new Map<string, string>()),
255 ),
256 ),
257 );
258});
259
260const prove = Effect.fn("prove")(function* (input: {
261 readonly scenario: string;
262 readonly base: string;
263 readonly head: string;
264 readonly uploadCommand: Option.Option<string>;
265}) {
266 const fs = yield* FileSystem.FileSystem;
267 const path = yield* Path.Path;
268 const { scripts } = yield* roots;
269 const proveDir = path.join(scripts, ".e2e/prove");
270 const scenarioPath = path.resolve(yield* HostProcess.WorkingDirectory, input.scenario);
271 // A proof left by an earlier run must not be read as this one's.
272 yield* fs.remove(proveDir, { recursive: true, force: true });
273 yield* fs.makeDirectory(proveDir, { recursive: true });
274 const base = yield* runSide("base", input.base, scenarioPath);
275 const head = yield* runSide("head", input.head, scenarioPath);
276 const links = Option.isSome(input.uploadCommand)
277 ? yield* upload(input.uploadCommand.value, [...base.captures, ...head.captures])
278 : new Map<string, string>();
279 const proof = buildProof(
280 base,
281 head,
282 (capture) => links.get(capture) ?? path.relative(proveDir, capture),
283 );
284 const json = yield* encodeProof(proof);
285 yield* fs.writeFileString(path.join(proveDir, "proof.md"), proof.markdown);
286 yield* fs.writeFileString(path.join(proveDir, "proof.json"), `${json}\n`);
287 yield* Console.log(json);
288 const couldNotRun = base.status === "error" || head.status === "error";
289 return proof.failsOnBase && proof.passesOnHead ? 0 : couldNotRun ? 2 : 1;
290});
291
292const command = Command.make(
293 "prove-scenario",
294 {
295 scenario: Argument.String("scenario").pipe(
296 Argument.withDescription("Scenario file, such as scripts/scenarios/name.e2e.ts."),
297 ),
298 base: Flag.String("base").pipe(Flag.withDescription("Commit the bug shows on.")),
299 head: Flag.String("head").pipe(
300 Flag.withDescription("Commit with the fix."),
301 Flag.withDefault("HEAD"),
302 ),
303 uploadCommand: Flag.String("upload-command").pipe(
304 Flag.withDescription("Command that takes capture paths and prints one URL per line."),
305 Flag.optional,
306 ),
307 },
308 (input) =>
309 prove(input).pipe(
310 // A bad revision or a failed install is "could not run" (2), not "did not prove" (1).
311 Effect.catch((error) => Console.error(String(error)).pipe(Effect.as(2))),
312 Effect.flatMap((exitCode) =>
313 Effect.sync(() => {
314 process.exitCode = exitCode;
315 }),
316 ),
317 ),
318);
319
320if (import.meta.main) {
321 Command.run(command, { version: "0.0.0" }).pipe(
322 Effect.provide(NodeServices.layer),
323 NodeRuntime.runMain,
324 );
325}