1import * as NodeServices from "@effect/platform-node/NodeServices";
2import { assert, it } from "@effect/vitest";
3import * as Deferred from "effect/Deferred";
4import * as Effect from "effect/Effect";
5import * as FileSystem from "effect/FileSystem";
6import * as Layer from "effect/Layer";
7import * as Path from "effect/Path";
8import * as Option from "effect/Option";
9import * as Stream from "effect/Stream";
10import * as Tracer from "effect/Tracer";
11import * as TestClock from "effect/testing/TestClock";
12import * as SqlClient from "effect/sql/SqlClient";
13
14import * as ServerConfig from "../config.ts";
15import * as ProjectionMaintenance from "../orchestration-v2/ProjectionMaintenance.ts";
16import * as DatabaseMaintenance from "./DatabaseMaintenance.ts";
17import * as DatabaseStorage from "./DatabaseStorage.ts";
18import * as Sqlite from "./Sqlite.ts";
19
20const freePages = Effect.map(
21 Effect.flatMap(
22 SqlClient.SqlClient,
23 (sql) => sql<{ readonly freelist_count: number }>`PRAGMA freelist_count`,
24 ),
25 (rows) => rows[0]?.freelist_count ?? 0,
26);
27
28const storageLayer = <E>(
29 config: Layer.Layer<ServerConfig.ServerConfig, E, FileSystem.FileSystem | Path.Path>,
30) =>
31 DatabaseStorage.layer.pipe(
32 Layer.provide(ProjectionMaintenance.layer),
33 Layer.provideMerge(Sqlite.layerMemory),
34 Layer.provideMerge(config),
35 Layer.provide(NodeServices.layer),
36 );
37
38const testConfig = ServerConfig.layerTest(process.cwd(), { prefix: "database-storage-" });
39
40it.effect("a new database returns freed pages online without losing data", () =>
41 Effect.gen(function* () {
42 const sql = yield* SqlClient.SqlClient;
43 const storage = yield* DatabaseStorage.DatabaseStorage;
44 assert.deepEqual(yield* sql`PRAGMA auto_vacuum`, [{ auto_vacuum: 2 }]);
45 yield* sql`CREATE TABLE retained (value TEXT)`;
46 yield* sql`INSERT INTO retained VALUES ('keep')`;
47 yield* sql`CREATE TABLE discarded (value BLOB)`;
48 yield* sql`INSERT INTO discarded VALUES (zeroblob(6291456))`;
49 yield* sql`DROP TABLE discarded`;
50 const before = yield* freePages;
51 assert.isAbove(before, 1024);
52 const spans: Array<Tracer.NativeSpan> = [];
53 const tracer = Tracer.make({
54 span(options) {
55 const span = new Tracer.NativeSpan(options);
56 spans.push(span);
57 return span;
58 },
59 });
60 yield* storage.reclaim.pipe(Effect.withTracer(tracer), Effect.withTracerEnabled(true));
61 assert.equal(before - (yield* freePages), 1024);
62 assert.lengthOf(spans, 1);
63 assert.deepEqual(
64 spans.map((span) => span.name),
65 ["DatabaseStorage.reclaim"],
66 );
67 assert.equal(spans[0]?.attributes.get("db.reclaimed_pages"), 1024);
68 yield* storage.reclaim;
69 assert.equal(yield* freePages, 0);
70 assert.deepEqual(yield* sql`SELECT value FROM retained`, [{ value: "keep" }]);
71 }).pipe(Effect.provide(storageLayer(testConfig))),
72);
73
74const compactionLayer = (
75 pruneEventStore: ProjectionMaintenance.ProjectionMaintenanceV2["Service"]["pruneEventStore"],
76) =>
77 DatabaseStorage.layer.pipe(
78 Layer.provide(
79 Layer.succeed(ProjectionMaintenance.ProjectionMaintenanceV2, {
80 pruneEventStore,
81 }),
82 ),
83 Layer.provide(Sqlite.layerMemory),
84 );
85
86const minute = 60_000;
87const startupTime = Date.UTC(2026, 9, 19);
88const emptyCompaction = { deletedEventCount: 0, deletedReceiptCount: 0, reclaimableBytes: 0 };
89
90it.effect("remembers completion only after the paced pass finishes", () =>
91 Effect.gen(function* () {
92 yield* TestClock.setTime(startupTime);
93 const fs = yield* FileSystem.FileSystem;
94 const path = yield* Path.Path;
95 const config = yield* ServerConfig.ServerConfig;
96 const timestampPath = path.join(path.dirname(config.dbPath), "compacted-at");
97 const betweenPagesReached = yield* Deferred.make<void>();
98 const completed = yield* Deferred.make<void>();
99 const timestampWritten = yield* Deferred.make<void>();
100 const compact = (betweenPages: Effect.Effect<void> = Effect.void) =>
101 Effect.gen(function* () {
102 yield* Effect.sleep(30);
103 yield* Deferred.succeed(betweenPagesReached, undefined);
104 yield* betweenPages;
105 yield* Deferred.succeed(completed, undefined);
106 return emptyCompaction;
107 });
108 yield* Effect.gen(function* () {
109 yield* TestClock.adjust(minute);
110 yield* TestClock.adjust(30);
111 yield* Deferred.await(betweenPagesReached);
112 assert.isFalse(yield* Deferred.isDone(completed));
113 assert.isFalse(yield* fs.exists(timestampPath));
114 yield* TestClock.adjust(269);
115 assert.isFalse(yield* Deferred.isDone(completed));
116 assert.isFalse(yield* fs.exists(timestampPath));
117 yield* TestClock.adjust(1);
118 yield* Deferred.await(timestampWritten);
119 assert.isTrue(yield* Deferred.isDone(completed));
120 assert.equal(Number(yield* fs.readFileString(timestampPath)), startupTime + minute + 300);
121 }).pipe(
122 Effect.provide(compactionLayer(compact)),
123 Effect.provideService(FileSystem.FileSystem, {
124 ...fs,
125 writeFileString: (file, content, options) =>
126 fs
127 .writeFileString(file, content, options)
128 .pipe(Effect.ensuring(Deferred.succeed(timestampWritten, undefined))),
129 }),
130 );
131 }).pipe(Effect.provide(testConfig.pipe(Layer.provideMerge(NodeServices.layer)))),
132);
133
134// NONE is a database from before automatic shrinking, which the repair converts.
135const reclaimLayer = (
136 restart: Effect.Effect<void> | null,
137 autoVacuum: "NONE" | "FULL" | "INCREMENTAL" = "NONE",
138) =>
139 DatabaseStorage.layer.pipe(
140 Layer.provide(ProjectionMaintenance.layer),
141 Layer.provide(Layer.succeed(DatabaseStorage.ServerRestart, restart)),
142 Layer.provide(
143 Layer.effectDiscard(
144 Effect.gen(function* () {
145 const sql = yield* SqlClient.SqlClient;
146 yield* sql.unsafe(`PRAGMA auto_vacuum = ${autoVacuum}`);
147 yield* sql`VACUUM`;
148 }),
149 ).pipe(Layer.provideMerge(Sqlite.layerMemory)),
150 ),
151 );
152
153it.effect(
154 "a reclaim from Settings saves its request, then restarts the server after replying",
155 () =>
156 Effect.gen(function* () {
157 const fs = yield* FileSystem.FileSystem;
158 const path = yield* Path.Path;
159 const config = yield* ServerConfig.ServerConfig;
160 const files = DatabaseMaintenance.reclaimFiles(path, config.dbPath);
161 const request = Effect.flatMap(
162 DatabaseStorage.DatabaseStorage,
163 (storage) => storage.requestReclaim,
164 );
165 const restarted = yield* Deferred.make<void>();
166 const restart = Deferred.succeed(restarted, undefined).pipe(Effect.asVoid);
167
168 // Nothing starts a server run by hand again, so it keeps pointing at the command.
169 const unsupervised = yield* request.pipe(Effect.flip, Effect.provide(reclaimLayer(null)));
170 assert.equal(unsupervised._tag, "DatabaseReclaimError");
171 // A database that already shrinks on its own (or a stale dialog after another client's
172 // repair) has nothing to reclaim, so it must not restart.
173 for (const autoVacuum of ["FULL", "INCREMENTAL"] as const) {
174 const refused = yield* request.pipe(
175 Effect.flip,
176 Effect.provide(reclaimLayer(restart, autoVacuum)),
177 );
178 assert.equal(refused._tag, "DatabaseReclaimError", autoVacuum);
179 }
180 assert.isFalse(yield* fs.exists(files.request));
181
182 yield* fs.writeFileString(files.failure, "an earlier failure");
183 yield* Effect.gen(function* () {
184 const storage = yield* DatabaseStorage.DatabaseStorage;
185 const shownFailure = storage.status.pipe(
186 Stream.filter((status) => status !== null),
187 Stream.runHead,
188 Effect.map((status) => Option.getOrThrow(status)?.reclaimFailure),
189 );
190 assert.equal(yield* shownFailure, "an earlier failure");
191 yield* storage.requestReclaim;
192 // The retry clears it at once, so Settings shows the restart, not the old failure.
193 assert.isUndefined(yield* shownFailure);
194 }).pipe(Effect.provide(reclaimLayer(restart)));
195 assert.isTrue(yield* fs.exists(files.request));
196 assert.isFalse(yield* fs.exists(files.failure));
197 assert.isFalse(yield* Deferred.isDone(restarted), "the reply must go out before the restart");
198 yield* TestClock.adjust(500);
199 yield* Deferred.await(restarted);
200 }).pipe(Effect.provide(testConfig.pipe(Layer.provideMerge(NodeServices.layer)))),
201);