1import * as AssetSigner from "./AssetSigner.ts";
2import * as DocumentText from "../provider/DocumentText.ts";
3// @effect-diagnostics nodeBuiltinImport:off
4import * as NodeFS from "node:fs";
5import * as NodePath from "node:path";
6
7import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
8import * as NodeServices from "@effect/platform-node/NodeServices";
9import { describe, expect, it } from "@effect/vitest";
10import { ChatAttachmentId } from "@t3tools/contracts";
11import * as Deferred from "effect/Deferred";
12import * as Effect from "effect/Effect";
13import * as Fiber from "effect/Fiber";
14import * as FileSystem from "effect/FileSystem";
15import * as Layer from "effect/Layer";
16import * as PlatformError from "effect/PlatformError";
17import * as Schema from "effect/Schema";
18import * as Stream from "effect/Stream";
19
20import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
21import { base64UrlEncode } from "../auth/utils.ts";
22import * as ServerConfig from "../config.ts";
23import {
24 claimPendingAttachments,
25 releaseClaimedAttachments,
26} from "../orchestration-v2/AttachmentClaims.ts";
27import {
28 ATTACHMENT_UPLOAD_ROUTE_PREFIX,
29 deletePendingAttachment,
30 issueAttachmentUploadUrl,
31 storeAttachmentUpload,
32 validateAttachmentUploadToken,
33} from "./AttachmentUpload.ts";
34
35const layerTest = AssetSigner.layer.pipe(
36 Layer.provideMerge(ServerSecretStore.layer),
37 Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "t3-attachment-upload-" })),
38 Layer.provideMerge(
39 DocumentText.layer.pipe(
40 Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)),
41 ),
42 ),
43);
44
45const uploadInput = {
46 name: "screenshot.png",
47 mimeType: "image/png",
48 sizeBytes: 6,
49} as const;
50
51const LegacyAttachmentUploadClaims = Schema.Struct({
52 version: Schema.Literal(1),
53 kind: Schema.Literal("attachment-upload"),
54 attachmentId: Schema.String,
55 name: Schema.String,
56 mimeType: Schema.String,
57 sizeBytes: Schema.Number,
58 expiresAt: Schema.Number,
59});
60const encodeLegacyAttachmentUploadClaims = Schema.encodeEffect(
61 Schema.fromJsonString(LegacyAttachmentUploadClaims),
62);
63
64describe("AttachmentUpload", () => {
65 it.effect.each([false, true])(
66 "persists and replays immutable uploads (hard links unsupported: %s)",
67 (unsupported) =>
68 Effect.gen(function* () {
69 const config = yield* ServerConfig.ServerConfig;
70 const fileSystem = yield* FileSystem.FileSystem;
71 const store = (claims: Parameters<typeof storeAttachmentUpload>[0], bytes: Uint8Array) =>
72 storeAttachmentUpload(claims, bytes).pipe(
73 Effect.provideService(
74 FileSystem.FileSystem,
75 unsupported
76 ? {
77 ...fileSystem,
78 link: () =>
79 Effect.fail(
80 PlatformError.systemError({
81 _tag: "Unknown",
82 module: "FileSystem",
83 method: "link",
84 }),
85 ),
86 }
87 : fileSystem,
88 ),
89 );
90 const bytes = new TextEncoder().encode(
91 JSON.stringify({
92 cells: [{ cell_type: "markdown", source: ["# Cargo\n", "Violet crates: 19"] }],
93 }),
94 );
95 const issued = yield* issueAttachmentUploadUrl({
96 type: "file",
97 name: "cargo.ipynb",
98 mimeType: "application/x-ipynb+json",
99 sizeBytes: bytes.length,
100 });
101 const claims = yield* validateAttachmentUploadToken(
102 issued.relativeUrl.slice(`${ATTACHMENT_UPLOAD_ROUTE_PREFIX}/`.length),
103 );
104 if (!claims) throw new Error("Expected upload claims");
105 expect(yield* store(claims, bytes)).toEqual({ ok: true });
106 expect(yield* store(claims, bytes)).toEqual({ ok: true });
107 const path = NodePath.join(config.attachmentsDir, `${issued.attachmentId}.ipynb`);
108 expect(NodeFS.readFileSync(`${path}.txt`, "utf8")).toContain("Violet crates: 19");
109 const replacement = new TextEncoder().encode(
110 new TextDecoder().decode(bytes).replace("Violet crates: 19", "Violet crates: 28"),
111 );
112 expect(yield* store(claims, replacement)).toMatchObject({
113 ok: false,
114 status: 409,
115 });
116 expect(NodeFS.readFileSync(path)).toEqual(Buffer.from(bytes));
117 expect(NodeFS.readFileSync(`${path}.txt`, "utf8")).toContain("Violet crates: 19");
118 const claimed = yield* claimPendingAttachments({
119 threadId: "document-thread",
120 attachments: [
121 {
122 type: "file",
123 id: ChatAttachmentId.make(issued.attachmentId),
124 name: claims.name,
125 mimeType: claims.mimeType,
126 sizeBytes: bytes.length,
127 },
128 ],
129 });
130 expect(claimed.claimedPaths).toHaveLength(2);
131 expect(NodeFS.readFileSync(claimed.claimedPaths[1]!, "utf8")).toContain(
132 "Violet crates: 19",
133 );
134 yield* deletePendingAttachment(issued.attachmentId);
135 expect(NodeFS.existsSync(path)).toBe(false);
136 expect(NodeFS.existsSync(`${path}.txt`)).toBe(false);
137 expect(NodeFS.existsSync(claimed.claimedPaths[1]!)).toBe(true);
138 yield* releaseClaimedAttachments(claimed.claimedPaths);
139 expect(NodeFS.existsSync(claimed.claimedPaths[1]!)).toBe(false);
140 }).pipe(Effect.provide(layerTest)),
141 );
142 it.effect("rejects tampered and malformed upload tokens", () =>
143 Effect.gen(function* () {
144 const issued = yield* issueAttachmentUploadUrl(uploadInput);
145 const token = issued.relativeUrl.slice(`${ATTACHMENT_UPLOAD_ROUTE_PREFIX}/`.length);
146 const [payload, signature] = token.split(".");
147
148 expect(yield* validateAttachmentUploadToken(`${payload}x.${signature}`)).toBeNull();
149 expect(yield* validateAttachmentUploadToken(`${token}.extra`)).toBeNull();
150 expect(yield* validateAttachmentUploadToken("garbage")).toBeNull();
151 }).pipe(Effect.provide(layerTest)),
152 );
153
154 it.effect("accepts unexpired image upload tokens issued before file support", () =>
155 Effect.gen(function* () {
156 const issued = yield* issueAttachmentUploadUrl(uploadInput);
157 const signer = yield* AssetSigner.AssetSigner;
158 const encodedPayload = base64UrlEncode(
159 yield* encodeLegacyAttachmentUploadClaims({
160 version: 1,
161 kind: "attachment-upload",
162 attachmentId: issued.attachmentId,
163 name: uploadInput.name,
164 mimeType: uploadInput.mimeType,
165 sizeBytes: uploadInput.sizeBytes,
166 expiresAt: issued.expiresAt,
167 }),
168 );
169 const legacyToken = `${encodedPayload}.${yield* signer.sign(encodedPayload)}`;
170
171 expect(yield* validateAttachmentUploadToken(legacyToken)).toMatchObject({
172 type: "image",
173 attachmentId: issued.attachmentId,
174 });
175 }).pipe(Effect.provide(layerTest)),
176 );
177
178 it.effect("stores the expected bytes without leaving temporary files", () =>
179 Effect.gen(function* () {
180 const config = yield* ServerConfig.ServerConfig;
181 const issued = yield* issueAttachmentUploadUrl(uploadInput);
182 const token = issued.relativeUrl.slice(`${ATTACHMENT_UPLOAD_ROUTE_PREFIX}/`.length);
183 const claims = yield* validateAttachmentUploadToken(token);
184 if (!claims) {
185 throw new Error("Expected valid upload claims.");
186 }
187
188 expect(yield* storeAttachmentUpload(claims, new Uint8Array([1, 2, 3]))).toMatchObject({
189 ok: false,
190 status: 400,
191 });
192 expect(yield* storeAttachmentUpload(claims, new Uint8Array(6))).toEqual({ ok: true });
193 expect(
194 NodeFS.existsSync(NodePath.join(config.attachmentsDir, `${issued.attachmentId}.png`)),
195 ).toBe(true);
196 expect(
197 NodeFS.readdirSync(config.attachmentsDir).filter((entry) => entry.endsWith(".part")),
198 ).toEqual([]);
199 }).pipe(Effect.provide(layerTest)),
200 );
201
202 it.effect("streams generic files to a path with their original extension", () =>
203 Effect.gen(function* () {
204 const config = yield* ServerConfig.ServerConfig;
205 const issued = yield* issueAttachmentUploadUrl({
206 type: "file",
207 name: "report.PDF",
208 mimeType: "application/pdf",
209 sizeBytes: 6,
210 });
211 const token = issued.relativeUrl.slice(`${ATTACHMENT_UPLOAD_ROUTE_PREFIX}/`.length);
212 const claims = yield* validateAttachmentUploadToken(token);
213 if (!claims) {
214 throw new Error("Expected valid upload claims.");
215 }
216
217 expect(
218 yield* storeAttachmentUpload(
219 claims,
220 Stream.make(new Uint8Array([1, 2, 3]), new Uint8Array([4, 5, 6])),
221 ),
222 ).toEqual({ ok: true });
223 expect(issued.attachmentId).toMatch(/-pdf$/);
224 expect(
225 NodeFS.readFileSync(NodePath.join(config.attachmentsDir, `${issued.attachmentId}.pdf`)),
226 ).toEqual(Buffer.from([1, 2, 3, 4, 5, 6]));
227
228 yield* deletePendingAttachment(issued.attachmentId);
229 expect(NodeFS.readdirSync(config.attachmentsDir)).toEqual([]);
230 }).pipe(Effect.provide(layerTest)),
231 );
232
233 it.effect("stores MIME-identified SVG files with a previewable signed extension", () =>
234 Effect.gen(function* () {
235 const config = yield* ServerConfig.ServerConfig;
236 const bytes = new TextEncoder().encode('<svg xmlns="http://www.w3.org/2000/svg"/>');
237 for (const name of ["diagram", "diagram.bin"]) {
238 const issued = yield* issueAttachmentUploadUrl({
239 type: "file",
240 name,
241 mimeType: "image/svg+xml",
242 sizeBytes: bytes.length,
243 });
244 const token = issued.relativeUrl.slice(`${ATTACHMENT_UPLOAD_ROUTE_PREFIX}/`.length);
245 const claims = yield* validateAttachmentUploadToken(token);
246 if (!claims) throw new Error("Expected valid SVG upload claims.");
247 expect(claims.name).toBe(`${name}.svg`);
248 expect(issued.attachmentId).toMatch(/-svg$/);
249 expect(yield* storeAttachmentUpload(claims, bytes)).toEqual({ ok: true });
250 expect(
251 NodeFS.readFileSync(NodePath.join(config.attachmentsDir, `${issued.attachmentId}.svg`)),
252 ).toEqual(Buffer.from(bytes));
253 const claimed = yield* claimPendingAttachments({
254 threadId: "svg-upload-thread",
255 attachments: [
256 {
257 type: "file",
258 id: ChatAttachmentId.make(issued.attachmentId),
259 name,
260 mimeType: "image/svg+xml",
261 sizeBytes: bytes.length,
262 },
263 ],
264 });
265 expect(claimed.attachments[0]?.name).toBe(`${name}.svg`);
266 expect(NodeFS.readFileSync(claimed.claimedPaths[0]!)).toEqual(Buffer.from(bytes));
267 yield* releaseClaimedAttachments(claimed.claimedPaths);
268 yield* deletePendingAttachment(issued.attachmentId);
269 }
270 expect(NodeFS.readdirSync(config.attachmentsDir)).toEqual([]);
271 }).pipe(Effect.provide(layerTest)),
272 );
273
274 it.effect("removes partial streamed uploads that exceed their signed size", () =>
275 Effect.gen(function* () {
276 const config = yield* ServerConfig.ServerConfig;
277 const issued = yield* issueAttachmentUploadUrl(uploadInput);
278 const token = issued.relativeUrl.slice(`${ATTACHMENT_UPLOAD_ROUTE_PREFIX}/`.length);
279 const claims = yield* validateAttachmentUploadToken(token);
280 if (!claims) {
281 throw new Error("Expected valid upload claims.");
282 }
283
284 expect(yield* storeAttachmentUpload(claims, Stream.make(new Uint8Array(7)))).toMatchObject({
285 ok: false,
286 status: 400,
287 });
288 expect(NodeFS.readdirSync(config.attachmentsDir)).toEqual([]);
289 }).pipe(Effect.provide(layerTest)),
290 );
291
292 it.effect("removes partial streamed uploads when the upload is interrupted", () =>
293 Effect.gen(function* () {
294 const config = yield* ServerConfig.ServerConfig;
295 const issued = yield* issueAttachmentUploadUrl(uploadInput);
296 const token = issued.relativeUrl.slice(`${ATTACHMENT_UPLOAD_ROUTE_PREFIX}/`.length);
297 const claims = yield* validateAttachmentUploadToken(token);
298 if (!claims) {
299 throw new Error("Expected valid upload claims.");
300 }
301
302 const nextChunkRequested = yield* Deferred.make<void>();
303 const body = Stream.make(new Uint8Array([1, 2, 3])).pipe(
304 Stream.concat(
305 Stream.fromEffect(
306 Deferred.succeed(nextChunkRequested, undefined).pipe(Effect.andThen(Effect.never)),
307 ),
308 ),
309 );
310 const upload = yield* storeAttachmentUpload(claims, body).pipe(Effect.forkScoped);
311
312 yield* Deferred.await(nextChunkRequested);
313 expect(
314 NodeFS.readdirSync(config.attachmentsDir).filter((entry) => entry.endsWith(".part")),
315 ).toHaveLength(1);
316
317 yield* Fiber.interrupt(upload);
318 expect(NodeFS.readdirSync(config.attachmentsDir)).toEqual([]);
319 }).pipe(Effect.provide(layerTest)),
320 );
321
322 it.effect("deletes pending uploads without deleting thread-owned copies", () =>
323 Effect.gen(function* () {
324 const config = yield* ServerConfig.ServerConfig;
325 const uuid = "00000000-0000-4000-8000-0000000000dd";
326 const pendingPath = NodePath.join(config.attachmentsDir, `pending-${uuid}.png`);
327 const claimedPath = NodePath.join(config.attachmentsDir, `thread-1-${uuid}.png`);
328 NodeFS.writeFileSync(pendingPath, Buffer.from("pixels"));
329 NodeFS.writeFileSync(claimedPath, Buffer.from("pixels"));
330
331 yield* deletePendingAttachment(`pending-${uuid}`);
332 yield* deletePendingAttachment(`pending-${uuid}`);
333 yield* deletePendingAttachment(`thread-1-${uuid}`);
334
335 expect(NodeFS.existsSync(pendingPath)).toBe(false);
336 expect(NodeFS.existsSync(claimedPath)).toBe(true);
337 }).pipe(Effect.provide(layerTest)),
338 );
339});