Open History to see the last commit
1// @effect-diagnostics nodeBuiltinImport:off
2import * as NodeChildProcess from "node:child_process";
3import * as NodeFS from "node:fs";
4import * as NodePath from "node:path";
5import * as NodeServices from "@effect/platform-node/NodeServices";
6import { describe, expect, it } from "@effect/vitest";
7import * as Effect from "effect/Effect";
8import * as Deferred from "effect/Deferred";
9import * as Fiber from "effect/Fiber";
10import * as Queue from "effect/Queue";
11import * as FileSystem from "effect/FileSystem";
12import * as Layer from "effect/Layer";
13import * as TestClock from "effect/testing/TestClock";
14import { vi } from "vite-plus/test";
15import * as HostProcess from "@t3tools/shared/HostProcess";
16import * as ServerConfig from "../config.ts";
17import * as DocumentText from "./DocumentText.ts";
18import { extractDocumentText } from "./documentTextWorker.ts";
19import * as Cleanup from "../orchestration-v2/ResourceCleanupService.ts";
20import * as DeviceService from "../device/DeviceService.ts";
21import * as TerminalManager from "../terminal/Manager.ts";
22import * as PreviewManager from "../preview/Manager.ts";
23import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts";
24import * as NodeZlib from "node:zlib";
25import { appendUserInputAttachmentPaths } from "./userInputAttachments.ts";
26
27vi.mock("node:child_process", { spy: true });
28
29const layer = Layer.merge(
30 ServerConfig.layerTest(process.cwd(), { prefix: "t3-answer-paths-" }),
31 DocumentText.layer,
32).pipe(Layer.provideMerge(NodeServices.layer));
33const attachment = {
34 type: "file" as const,
35 id: "thread-1-00000000-0000-4000-8000-0000000000aa-txt",
36 name: 'spec "final".txt',
37 mimeType: "text/plain",
38 sizeBytes: 4,
39};
40describe("question answer paths", () => {
41 it.effect.each([false, true])(
42 "keeps unrelated files available during extraction (failure: %s)",
43 (failure) =>
44 Effect.gen(function* () {
45 const config = yield* ServerConfig.ServerConfig;
46 const fs = yield* FileSystem.FileSystem;
47 const path = NodePath.join(config.attachmentsDir, "blocked.rtf");
48 const cached = NodePath.join(config.attachmentsDir, "cached.rtf");
49 NodeFS.writeFileSync(path, failure ? "invalid document" : String.raw`{\rtf1 Available}`);
50 NodeFS.writeFileSync(cached + ".txt", "Cached text");
51 const started = yield* Deferred.make<void>();
52 const release = yield* Deferred.make<void>();
53 let reads = 0;
54 yield* Effect.gen(function* () {
55 const documents = yield* DocumentText.DocumentText;
56 const extraction = yield* documents.ensure(path).pipe(Effect.forkChild);
57 yield* Deferred.await(started);
58 const duplicate = yield* documents.ensure(path).pipe(Effect.forkChild);
59 expect(yield* documents.ensure(path + ".png")).toBeUndefined();
60 expect(yield* documents.ensure(path + ".pdf")).toBeUndefined();
61 expect(yield* documents.ensure(cached)).toBe(cached + ".txt");
62 expect(NodeFS.readFileSync(cached + ".txt", "utf8")).toBe("Cached text");
63 yield* Deferred.succeed(release, undefined);
64 const expected = failure ? undefined : path + ".txt";
65 expect(yield* Fiber.join(extraction)).toBe(expected);
66 expect(yield* Fiber.join(duplicate)).toBe(expected);
67 expect(yield* documents.ensure(path)).toBe(expected);
68 expect(reads).toBe(1);
69 if (failure) {
70 NodeFS.writeFileSync(path + ".txt", "Recovered text");
71 expect(yield* documents.ensure(path)).toBe(path + ".txt");
72 expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe("Recovered text");
73 } else expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe("Available");
74 }).pipe(
75 Effect.provide(Layer.fresh(DocumentText.layer)),
76 Effect.provideService(FileSystem.FileSystem, {
77 ...fs,
78 stat: (file) =>
79 file === path
80 ? Effect.gen(function* () {
81 reads++;
82 yield* Deferred.succeed(started, undefined);
83 yield* Deferred.await(release);
84 return yield* fs.stat(file);
85 })
86 : fs.stat(file),
87 }),
88 );
89 }).pipe(Effect.provide(layer)),
90 );
91
92 it.effect.each(["interrupted", "failed"])(
93 "extracts two documents at once and frees a slot when one is %s",
94 (outcome) =>
95 Effect.gen(function* () {
96 const config = yield* ServerConfig.ServerConfig;
97 const fs = yield* FileSystem.FileSystem;
98 const [first, second, third] = ["first", "second", "third"].map((name) =>
99 NodePath.join(config.attachmentsDir, `${name}.rtf`),
100 );
101 NodeFS.writeFileSync(first!, String.raw`{\rtf1 First}`);
102 NodeFS.writeFileSync(second!, "invalid document");
103 NodeFS.writeFileSync(third!, String.raw`{\rtf1 Third}`);
104 const started = yield* Queue.unbounded<string>();
105 const releases = new Map<string, Deferred.Deferred<void>>();
106 for (const path of [first!, second!, third!])
107 releases.set(path, yield* Deferred.make<void>());
108 yield* Effect.gen(function* () {
109 const documents = yield* DocumentText.DocumentText;
110 const leaving = yield* documents.ensure(first!).pipe(Effect.forkChild);
111 const staying = yield* documents.ensure(first!).pipe(Effect.forkChild);
112 const freed = yield* documents.ensure(second!).pipe(Effect.forkChild);
113 expect(new Set([yield* Queue.take(started), yield* Queue.take(started)])).toEqual(
114 new Set([first, second]),
115 );
116 const waiting = yield* documents.ensure(third!).pipe(Effect.forkChild);
117 yield* Fiber.interrupt(leaving);
118 if (outcome === "interrupted") yield* Fiber.interrupt(freed);
119 else {
120 yield* Deferred.succeed(releases.get(second!)!, undefined);
121 expect(yield* Fiber.join(freed)).toBeUndefined();
122 }
123 expect(yield* Queue.take(started)).toBe(third);
124 yield* Deferred.succeed(releases.get(first!)!, undefined);
125 yield* Deferred.succeed(releases.get(third!)!, undefined);
126 expect(yield* Fiber.join(staying)).toBe(first + ".txt");
127 expect(yield* Fiber.join(waiting)).toBe(third + ".txt");
128 expect(yield* Queue.size(started)).toBe(0);
129 expect(NodeFS.existsSync(second + ".txt")).toBe(false);
130 expect(
131 NodeFS.readdirSync(config.attachmentsDir).filter((file) => file.endsWith(".part")),
132 ).toEqual([]);
133 }).pipe(
134 Effect.provide(Layer.fresh(DocumentText.layer)),
135 Effect.provideService(FileSystem.FileSystem, {
136 ...fs,
137 stat: (file) =>
138 Effect.gen(function* () {
139 yield* Queue.offer(started, file);
140 yield* Deferred.await(releases.get(file)!);
141 return yield* fs.stat(file);
142 }),
143 }),
144 );
145 }).pipe(Effect.provide(layer)),
146 );
147
148 it.effect("retries a document while its cancelled extraction is still publishing", () =>
149 Effect.gen(function* () {
150 const config = yield* ServerConfig.ServerConfig;
151 const fs = yield* FileSystem.FileSystem;
152 const path = NodePath.join(config.attachmentsDir, "retried.rtf");
153 NodeFS.writeFileSync(path, String.raw`{\rtf1 Retried}`);
154 const writing = yield* Deferred.make<void>();
155 const writeReleased = yield* Deferred.make<void>();
156 const renaming = yield* Deferred.make<void>();
157 const renameReleased = yield* Deferred.make<void>();
158 let writes = 0;
159 let renames = 0;
160 yield* Effect.gen(function* () {
161 const documents = yield* DocumentText.DocumentText;
162 const cancelled = yield* documents.ensure(path).pipe(Effect.forkChild);
163 yield* Deferred.await(writing);
164 const interrupting = yield* Fiber.interrupt(cancelled).pipe(Effect.forkChild);
165 const retry = yield* documents.ensure(path).pipe(Effect.forkChild);
166 yield* Deferred.await(renaming);
167 yield* Deferred.succeed(writeReleased, undefined);
168 yield* Fiber.join(interrupting);
169 yield* Deferred.succeed(renameReleased, undefined);
170 expect(yield* Fiber.join(retry)).toBe(path + ".txt");
171 expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe("Retried");
172 expect(
173 NodeFS.readdirSync(config.attachmentsDir).filter((file) => file.endsWith(".part")),
174 ).toEqual([]);
175 }).pipe(
176 Effect.provide(Layer.fresh(DocumentText.layer)),
177 Effect.provideService(FileSystem.FileSystem, {
178 ...fs,
179 writeFileString: (file, data) =>
180 Effect.gen(function* () {
181 if (++writes === 1) {
182 yield* Deferred.succeed(writing, undefined);
183 yield* Deferred.await(writeReleased);
184 }
185 return yield* fs.writeFileString(file, data);
186 }),
187 rename: (from, to) =>
188 Effect.gen(function* () {
189 if (++renames === 1) {
190 yield* Deferred.succeed(renaming, undefined);
191 yield* Deferred.await(renameReleased);
192 }
193 return yield* fs.rename(from, to);
194 }),
195 }),
196 );
197 }).pipe(Effect.provide(layer)),
198 );
199
200 it.effect.each(["spawn", "read"])("retries after a transient %s failure expires", (failure) =>
201 Effect.acquireUseRelease(
202 Effect.promise(async () => {
203 const { execFile } = await vi.importActual<typeof NodeChildProcess>("node:child_process");
204 const worker = vi.spyOn(NodeChildProcess, "execFile");
205 worker.mockImplementationOnce(((
206 file: string,
207 args: readonly string[],
208 options: NodeChildProcess.ExecFileOptionsWithStringEncoding,
209 callback: (
210 error: NodeChildProcess.ExecFileException | null,
211 stdout: string,
212 stderr: string,
213 ) => void,
214 ) => {
215 if (failure === "spawn") {
216 throw Object.assign(new Error("spawn EAGAIN"), { code: "EAGAIN" });
217 }
218 const preload = `import fs from 'node:fs/promises';
219 import { syncBuiltinESMExports } from 'node:module';
220 fs.readFile = async () => { throw Object.assign(new Error('read EAGAIN'), { code: 'EAGAIN' }); };
221 syncBuiltinESMExports();`;
222 return execFile(
223 file,
224 ["--import", `data:text/javascript,${encodeURIComponent(preload)}`, ...args],
225 options,
226 callback,
227 );
228 }) as typeof NodeChildProcess.execFile);
229 return worker;
230 }),
231 (worker) =>
232 Effect.gen(function* () {
233 const config = yield* ServerConfig.ServerConfig;
234 const path = NodePath.join(config.attachmentsDir, "retry.rtf");
235 NodeFS.writeFileSync(path, String.raw`{\rtf1 Available}`);
236 const documents = yield* DocumentText.DocumentText;
237 expect(yield* documents.ensure(path)).toBeUndefined();
238 expect(yield* documents.ensure(path)).toBeUndefined();
239 expect(worker).toHaveBeenCalledTimes(1);
240 expect(NodeFS.existsSync(path + ".txt")).toBe(false);
241 yield* TestClock.adjust("1 second");
242 expect(yield* documents.ensure(path)).toBe(path + ".txt");
243 expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe("Available");
244 expect(worker).toHaveBeenCalledTimes(2);
245 expect(yield* documents.ensure(path)).toBe(path + ".txt");
246 expect(worker).toHaveBeenCalledTimes(2);
247 }).pipe(Effect.provide(layer)),
248 (worker) => Effect.sync(() => worker.mockRestore()),
249 ),
250 );
251
252 it.effect.each([
253 ["ipynb", "{"],
254 ["rtf", "invalid document"],
255 ["rtf", String.raw`{\rtf1}`],
256 ["rtf", String.raw`{\rtf1\ansi\ansicpg99999 Unsupported}`],
257 ] as const)("caches unchanged document rejections: %s %s", ([extension, source]) =>
258 Effect.acquireUseRelease(
259 Effect.sync(() => vi.spyOn(NodeChildProcess, "execFile")),
260 (worker) =>
261 Effect.gen(function* () {
262 const config = yield* ServerConfig.ServerConfig;
263 const path = NodePath.join(config.attachmentsDir, `invalid.${extension}`);
264 NodeFS.writeFileSync(path, source);
265 const documents = yield* DocumentText.DocumentText;
266 for (const seconds of [0, 1, 60]) {
267 yield* TestClock.adjust(`${seconds} seconds`);
268 expect(yield* documents.ensure(path)).toBeUndefined();
269 expect(worker).toHaveBeenCalledTimes(1);
270 expect(NodeFS.existsSync(path + ".txt")).toBe(false);
271 }
272 }).pipe(Effect.provide(layer)),
273 (worker) => Effect.sync(() => worker.mockRestore()),
274 ),
275 );
276
277 it.effect.each([
278 [
279 "text symbols",
280 String.raw`{\rtf1\ansi Pages 1\endash 5\emdash\lquote left\rquote\ldblquote right\rdblquote\bullet\b End\b0\unknown{\*\secret\bullet hidden}}`,
281 "Pages 1–5—‘left’“right”•End",
282 ],
283 [
284 "space and hyphen symbols",
285 String.raw`{\rtf1\ansi A\enspace B\emspace C\qmspace D\~E\_F\-G}`,
286 "A\u2002B\u2003C\u2005D\u00a0E\u2011FG",
287 ],
288 [
289 "text symbol fallback",
290 String.raw`{\rtf1\ansi\uc10\u945\endash\emdash\enspace\emspace\qmspace\lquote\rquote\ldblquote\rdblquote\bullet Z}`,
291 "αZ",
292 ],
293 ["hyphen symbol fallback", String.raw`{\rtf1\ansi\uc3\u945\~\_\-Z}`, "αZ"],
294 ["control word", String.raw`{\rtf1\ansi A\u9\tab X}`, "A\tX"],
295 ["line break paragraphs", "{\\rtf1\\ansi First\\\nSecond\\\r\nThird}", "First\nSecond\nThird"],
296 ["group close", String.raw`{\rtf1\ansi{\u945}Z}`, "αZ"],
297 ["group open", String.raw`{\rtf1\ansi\u945{Z}}`, "αZ"],
298 ["hex byte", String.raw`{\rtf1\ansi\u945\'3fZ}`, "αZ"],
299 ["escaped symbol", String.raw`{\rtf1\ansi\u945\{Z}`, "αZ"],
300 ["destination symbol", String.raw`{\rtf1\ansi\u945\*Z}`, "αZ"],
301 ["multiple units", String.raw`{\rtf1\ansi\uc4\u945?\tab\'3f\~Z}`, "αZ"],
302 ["binary unit", String.raw`{\rtf1\ansi\u945\bin3 abcZ}`, "αZ"],
303 ["large fallback", String.raw`{\rtf1\ansi\uc20\u945????????????????????Z}`, "αZ"],
304 ["no fallback", String.raw`{\rtf1\ansi\uc0\u945Z}`, "αZ"],
305 ["scoped count", String.raw`{\rtf1\ansi{\uc2\u945??}\u946?Z}`, "αβZ"],
306 ] as const)("extracts RTF symbols and Unicode fallbacks: %s", ([name, source, expected]) =>
307 Effect.gen(function* () {
308 const config = yield* ServerConfig.ServerConfig;
309 const path = NodePath.join(config.attachmentsDir, `${name}.rtf`);
310 NodeFS.writeFileSync(path, source);
311 expect(yield* Effect.promise(() => extractDocumentText(path, ".rtf"))).toBe(expected);
312 expect(yield* (yield* DocumentText.DocumentText).ensure(path)).toBe(path + ".txt");
313 expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe(expected);
314 }).pipe(Effect.provide(layer)),
315 );
316
317 it.effect.each([false, true])("limits the child heap for standalone=%s", (standalone) =>
318 Effect.acquireUseRelease(
319 Effect.promise(async () => {
320 const { execFile } = await vi.importActual<typeof NodeChildProcess>("node:child_process");
321 const probe = vi.spyOn(NodeChildProcess, "execFile");
322 probe.mockImplementation(((
323 file: string,
324 _args: readonly string[],
325 options: NodeChildProcess.ExecFileOptionsWithStringEncoding,
326 callback: (
327 error: NodeChildProcess.ExecFileException | null,
328 stdout: string,
329 stderr: string,
330 ) => void,
331 ) =>
332 execFile(
333 file,
334 [
335 "-e",
336 'process.stdout.write(String(require("node:v8").getHeapStatistics().heap_size_limit))',
337 ],
338 options,
339 callback,
340 )) as typeof NodeChildProcess.execFile);
341 return probe;
342 }),
343 () =>
344 Effect.gen(function* () {
345 const config = yield* ServerConfig.ServerConfig;
346 const path = NodePath.join(config.attachmentsDir, "heap.rtf");
347 NodeFS.writeFileSync(path, String.raw`{\rtf1 hello}`);
348 const extracted = yield* Effect.gen(function* () {
349 return yield* (yield* DocumentText.DocumentText).ensure(path);
350 }).pipe(
351 Effect.provide(Layer.fresh(DocumentText.layer)),
352 Effect.provideService(HostProcess.IsExecutable, standalone),
353 Effect.provideService(HostProcess.Environment, {
354 ...process.env,
355 NODE_OPTIONS: "--max-old-space-size=1024",
356 }),
357 );
358 expect(extracted).toBe(path + ".txt");
359 const heapLimit = Number(NodeFS.readFileSync(path + ".txt", "utf8"));
360 expect(heapLimit).toBeGreaterThan(192 * 1024 * 1024);
361 // Newer V8 versions reserve up to 192 MiB for the young generation as well.
362 expect(heapLimit).toBeLessThanOrEqual(384 * 1024 * 1024);
363 }).pipe(Effect.provide(layer)),
364 (probe) => Effect.sync(() => probe.mockRestore()),
365 ),
366 );
367
368 it.effect("extracts notebook cells and outputs once and keeps both readable paths", () =>
369 Effect.gen(function* () {
370 const config = yield* ServerConfig.ServerConfig;
371 const file = {
372 ...attachment,
373 id: attachment.id.replace(/txt$/, "ipynb"),
374 name: "survey.ipynb",
375 mimeType: "application/x-ipynb+json",
376 };
377 const path = NodePath.join(config.attachmentsDir, `${file.id}.ipynb`);
378 const notebook = JSON.stringify({
379 cells: [
380 { cell_type: "markdown", source: ["# Survey\n", "Vessel: Larkspur"] },
381 {
382 cell_type: "code",
383 source: ["print(43 * 7)"],
384 outputs: [
385 { text: ["301\n"] },
386 { data: { "text/plain": ["42"] } },
387 {
388 output_type: "error",
389 ename: "ZeroDivisionError",
390 evalue: "division by zero",
391 traceback: [
392 "\u001b[32mCell In[1], line 1\u001b[0m",
393 "\u001b[31m----> 1 1 / 0\u001b[0m",
394 "ZeroDivisionError: division by zero",
395 ],
396 },
397 ],
398 },
399 ],
400 });
401 NodeFS.writeFileSync(path, notebook);
402 const answers = yield* appendUserInputAttachmentPaths({
403 answers: { q: "Read this" },
404 attachmentsDir: config.attachmentsDir,
405 attachmentsByQuestionId: { q: [file] },
406 });
407 expect(answers.q).toContain(`text extracted to: ${JSON.stringify(path + ".txt")}`);
408 expect(NodeFS.readFileSync(`${path}.txt`, "utf8")).toContain("Vessel: Larkspur");
409 expect(NodeFS.readFileSync(`${path}.txt`, "utf8")).toContain("301");
410 expect(NodeFS.readFileSync(`${path}.txt`, "utf8")).toContain("42");
411 const extracted = NodeFS.readFileSync(`${path}.txt`, "utf8");
412 expect(extracted).toContain(
413 "ZeroDivisionError: division by zero\nCell In[1], line 1\n----> 1 1 / 0\nZeroDivisionError: division by zero",
414 );
415 expect(extracted).not.toContain("\u001b");
416 expect(NodeFS.readFileSync(path, "utf8")).toBe(notebook);
417 const before = NodeFS.statSync(path + ".txt").mtimeMs;
418 // A changed binary would fail parsing if the cached extraction were not reused.
419 NodeFS.writeFileSync(path, "invalid notebook");
420 const documents = yield* DocumentText.DocumentText;
421 const results = yield* Effect.all([documents.ensure(path), documents.ensure(path)], {
422 concurrency: "unbounded",
423 });
424 expect(results).toEqual([path + ".txt", path + ".txt"]);
425 expect(NodeFS.statSync(path + ".txt").mtimeMs).toBe(before);
426 yield* Effect.gen(function* () {
427 const cleanup = yield* Cleanup.ResourceCleanupService;
428 yield* cleanup.cleanupAttachments([file.id]);
429 expect(NodeFS.existsSync(path)).toBe(false);
430 expect(NodeFS.existsSync(path + ".txt")).toBe(false);
431 NodeFS.writeFileSync(path + ".txt", "orphaned extraction");
432 yield* cleanup.cleanupAttachments([file.id]);
433 expect(NodeFS.existsSync(path + ".txt")).toBe(false);
434 }).pipe(
435 Effect.provide(
436 Cleanup.layer.pipe(
437 Layer.provide(
438 Layer.mergeAll(
439 Layer.mock(TerminalManager.TerminalManager)({}),
440 Layer.mock(PreviewManager.PreviewManager)({}),
441 Layer.mock(DeviceService.DeviceService)({}),
442 Layer.mock(ScheduledTaskService.ScheduledTaskService)({}),
443 ),
444 ),
445 ),
446 ),
447 );
448 }).pipe(Effect.provide(layer)),
449 );
450
451 it.effect(
452 "preserves corrupt documents, PDFs and unknown binaries without an extracted path",
453 () =>
454 Effect.gen(function* () {
455 const config = yield* ServerConfig.ServerConfig;
456 for (const extension of ["docx", "ipynb", "zip", "pdf"]) {
457 const file = {
458 ...attachment,
459 id: attachment.id.replace(/txt$/, extension),
460 name: `broken.${extension}`,
461 };
462 const path = NodePath.join(config.attachmentsDir, `${file.id}.${extension}`);
463 NodeFS.writeFileSync(path, "invalid document");
464 const answers = yield* appendUserInputAttachmentPaths({
465 answers: {},
466 attachmentsDir: config.attachmentsDir,
467 attachmentsByQuestionId: { q: [file] },
468 });
469 expect(answers.q).toContain(JSON.stringify(path));
470 expect(answers.q).not.toContain("text extracted");
471 expect(NodeFS.existsSync(`${path}.txt`)).toBe(false);
472 expect(NodeFS.readFileSync(path, "utf8")).toBe("invalid document");
473 }
474 }).pipe(Effect.provide(layer)),
475 );
476
477 it.effect("preserves office structure and EPUB reading order with bounded archives", () =>
478 Effect.gen(function* () {
479 const config = yield* ServerConfig.ServerConfig;
480 const cases = [
481 {
482 ext: "docx",
483 bounded: true,
484 parts: {
485 "word/document.xml": "<document><p>Readable despite embedded media</p></document>",
486 "word/media/black.bmp": "\0".repeat(64 * 1024),
487 "word/embeddings/unused.bin": "\0".repeat(33 * 1024 * 1024),
488 },
489 expected: ["Readable despite embedded media"],
490 absent: ["[Extracted text truncated"],
491 },
492 ...[false, true].map((deep) => ({
493 ext: "docx",
494 bounded: true,
495 parts: {
496 "word/document.xml": `<document>${deep ? "<div>".repeat(129) + "Lost" + "</div>".repeat(129) : "<p/>".repeat(200_001)}</document>`,
497 },
498 expected: ["[Extracted text truncated at document extraction limits.]"],
499 absent: ["Lost"],
500 })),
501 ...["ods", "odt", "odp", "docx"].flatMap((ext) =>
502 [false, true].map((rows) => ({
503 ext,
504 bounded: true,
505 parts: {
506 [ext === "docx" ? "word/document.xml" : "content.xml"]:
507 `<document><page><table><table-row${rows ? ' number-rows-repeated="10001"' : ""}><table-cell${rows ? "" : ' number-columns-repeated="513"'}><p>Repeated value</p></table-cell></table-row></table></page></document>`,
508 },
509 expected: [
510 "Repeated value",
511 "[Extracted text truncated at document extraction limits.]",
512 ],
513 occurrences: { text: "Repeated value", count: rows ? 10_000 : 512 },
514 })),
515 ),
516 ...["ods", "odt", "odp", "docx"].map((ext) => ({
517 ext,
518 bounded: true,
519 parts: {
520 [ext === "docx" ? "word/document.xml" : "content.xml"]:
521 `<document><page><table><table-row><table-cell><p>Kept</p></table-cell><table-cell number-columns-repeated="16383"> \n <p> \t </p> <s c="1048575"/> </table-cell></table-row><table-row number-rows-repeated="1048575"><table-cell> \n <p> \t </p> </table-cell></table-row></table></page></document>`,
522 },
523 expected: ["Kept"],
524 absent: ["[Extracted text truncated", "\t"],
525 })),
526 ...["ods", "odt", "odp"].map((ext) => ({
527 ext,
528 bounded: true,
529 parts: {
530 "content.xml": `<document><page><table><table-row>
531 <table-cell office:value-type="string" office:string-value="Harbor"/>
532 <table-cell office:value-type="boolean" office:boolean-value="false"/>
533 <table-cell office:value-type="date" office:date-value="2026-10-20"/>
534 <table-cell office:value-type="time" office:time-value="PT2H30M"/>
535 <table-cell office:value-type="currency" office:value="125.50" office:currency="USD"/>
536 <table-cell office:value-type="percentage" office:value="0.25"/>
537 <table-cell office:value-type="float" office:value="0"/>
538 <table-cell office:value-type="string" office:string-value="Fallback"><p> \n </p></table-cell>
539 <table-cell office:value-type="string" office:string-value="Unshown"><p>Displayed</p></table-cell>
540 <table-cell office:value-type="void" office:value="Unshown"/>
541 <table-cell office:value-type="currency" office:value=" " office:currency="Unshown"/>
542 <table-cell office:value-type="percentage" office:value=" "/>
543 </table-row></table></page></document>`,
544 },
545 expected: [
546 "Harbor\tfalse\t2026-10-20\tPT2H30M\t125.50 USD\t0.25 [ODF value-type=percentage]\t0\tFallback\tDisplayed",
547 ],
548 absent: ["Unshown", "[Extracted text truncated"],
549 })),
550 ...[false, true].map((rows) => ({
551 ext: "xlsx",
552 bounded: true,
553 parts: {
554 "xl/workbook.xml": '<workbook><sheet name="Limited" r:id="a"/></workbook>',
555 "xl/_rels/workbook.xml.rels":
556 '<Relationships><Relationship Id="a" Target="worksheets/a.xml"/></Relationships>',
557 "xl/worksheets/a.xml": `<worksheet>${rows ? "<row><c><v>7</v></c></row>".repeat(10_001) : '<row><c r="A1"><v>7</v></c><c r="SS1"><v>Beyond</v></c></row>'}</worksheet>`,
558 },
559 expected: ["7", "[Extracted text truncated at document extraction limits.]"],
560 absent: ["Beyond"],
561 })),
562 {
563 ext: "xlsx",
564 bounded: true,
565 parts: {
566 "xl/workbook.xml": '<workbook><sheet name="Padding" r:id="a"/></workbook>',
567 "xl/_rels/workbook.xml.rels":
568 '<Relationships><Relationship Id="a" Target="worksheets/a.xml"/></Relationships>',
569 "xl/sharedStrings.xml": "<sst><si><t> \n </t></si><si><t>Present</t></si></sst>",
570 "xl/worksheets/a.xml": `<worksheet>${'<row><c t="s"><v>0</v></c><c r="XFD1" t="inlineStr"><is><t> \t </t></is></c></row>'.repeat(10_001)}<row><c t="s"><v>0</v></c><c t="s"><v>1</v></c><c r="XFD2"><v> \n </v></c></row></worksheet>`,
571 },
572 expected: ["Present"],
573 absent: ["[Extracted text truncated"],
574 },
575 {
576 ext: "epub",
577 parts: {
578 "META-INF/container.xml": '<container><rootfile full-path="book.opf"/></container>',
579 "book.opf":
580 '<package><item id="bad" href="ch%ZZ.xhtml"/><item id="good" href="good.svg"/><itemref idref="bad"/><itemref idref="good"/></package>',
581 "good.svg":
582 '<svg xmlns="http://www.w3.org/2000/svg"><text>Readable SVG chapter</text></svg>',
583 },
584 expected: [
585 "Readable SVG chapter",
586 "[Skipped document part: chapter 1 (missing or invalid XML).]",
587 ],
588 absent: ["[Extracted text truncated"],
589 },
590 ...[false, true].flatMap((bom) =>
591 [false, true].map((bigEndian) => {
592 const encode = (source: string) => {
593 const data = Buffer.from(`${bom ? "\ufeff" : ""}${source}`, "utf16le");
594 return bigEndian ? data.swap16() : data;
595 };
596 return {
597 ext: "epub",
598 parts: {
599 "META-INF/container.xml": encode(
600 '<?xml version="1.0" encoding="UTF-16"?><container><rootfile full-path="book.opf"/></container>',
601 ),
602 "book.opf": encode(
603 '<?xml version="1.0" encoding="UTF-16"?><package><item id="a" href="a.xhtml"/><itemref idref="a"/></package>',
604 ),
605 "a.xhtml": encode(
606 '<?xml version="1.0" encoding="UTF-16"?><html><p>UTF-16 chapter: café 日本語</p></html>',
607 ),
608 },
609 expected: ["UTF-16 chapter: café 日本語"],
610 absent: ["[Skipped document part", "[Extracted text truncated"],
611 };
612 }),
613 ),
614 {
615 ext: "epub",
616 bounded: true,
617 parts: {
618 "META-INF/container.xml": '<container><rootfile full-path="book.opf"/></container>',
619 "book.opf":
620 '<package><item id="bad" href="bad.xhtml"/><item id="good" href="good.xhtml"/><itemref idref="bad"/><itemref idref="good"/></package>',
621 "bad.xhtml": "<html><p>Discard this broken chapter</html>",
622 "good.xhtml":
623 '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"><html><p>North Harbor ≂̸ ∳ & <tag> "quoted"</p><p><![CDATA[Literal ]]></p><table><tr><td> </td></tr><tr><td>Kept cell</td></tr></table></html>',
624 },
625 expected: [
626 "## Chapter 2",
627 'North\u00a0Harbor ≂̸ ∳ & <tag> "quoted"',
628 "Literal ",
629 "Kept cell",
630 "[Skipped document part: bad.xhtml (missing or invalid XML).]",
631 ],
632 absent: ["Discard this broken chapter", "[Extracted text truncated"],
633 },
634 ...["pptx", "xlsx"].map((ext) => ({
635 ext,
636 bounded: true,
637 parts:
638 ext === "pptx"
639 ? {
640 "ppt/presentation.xml":
641 '<presentation><sldId r:id="bad"/><sldId r:id="good"/></presentation>',
642 "ppt/_rels/presentation.xml.rels":
643 '<Relationships><Relationship Id="bad" Target="bad.xml"/><Relationship Id="good" Target="good.xml"/></Relationships>',
644 "ppt/bad.xml": "<slide><p>Discard</slide>",
645 "ppt/good.xml": "<slide><p>Healthy part</p></slide>",
646 }
647 : {
648 "xl/workbook.xml":
649 '<workbook><sheet name="Bad" r:id="bad"/><sheet name="Good" r:id="good"/></workbook>',
650 "xl/_rels/workbook.xml.rels":
651 '<Relationships><Relationship Id="bad" Target="bad.xml"/><Relationship Id="good" Target="good.xml"/></Relationships>',
652 "xl/bad.xml": "<worksheet><row>Discard</worksheet>",
653 "xl/good.xml":
654 '<worksheet><row><c t="inlineStr"><is><t>Healthy part</t></is></c></row></worksheet>',
655 },
656 expected: [
657 "Healthy part",
658 `[Skipped document part: ${ext === "pptx" ? "ppt" : "xl"}/bad.xml (missing or invalid XML).]`,
659 ],
660 absent: ["Discard", "[Extracted text truncated"],
661 })),
662 {
663 ext: "odt",
664 bounded: true,
665 parts: { "content.xml": '<document><p>Start<s c="1000001"/>Unneeded</p></document>' },
666 expected: ["Start", "[Extracted text truncated at document extraction limits.]"],
667 absent: ["Unneeded"],
668 },
669 {
670 ext: "docx",
671 bounded: true,
672 parts: {
673 "word/document.xml":
674 '<w:document xmlns:w="urn:w" xmlns:mc="urn:mc"><w:body><w:p><w:r><w:t>Body</w:t></w:r></w:p><w:p><w:r><mc:AlternateContent><mc:Choice Requires="wps"><w:drawing><wps:wsp><wps:txbx><w:txbxContent><w:p><w:r><w:t>Callout box</w:t></w:r></w:p></w:txbxContent></wps:txbx></wps:wsp></w:drawing></mc:Choice><mc:Fallback><w:pict><v:shape><v:textbox><w:txbxContent><w:p><w:r><w:t>Callout box</w:t></w:r></w:p></w:txbxContent></v:textbox></v:shape></w:pict></mc:Fallback></mc:AlternateContent></w:r></w:p></w:body></w:document>',
675 },
676 expected: ["Body\nCallout box"],
677 occurrences: { text: "Callout box", count: 1 },
678 },
679 {
680 ext: "docx",
681 bounded: true,
682 parts: {
683 "word/document.xml":
684 '<w:document xmlns:w="urn:w" xmlns:mc="urn:mc"><w:body><w:p><w:r><mc:AlternateContent><mc:Choice Requires="cx"><w:drawing><cx:chart/></w:drawing></mc:Choice><mc:Fallback><w:pict><v:shape><v:textbox><w:txbxContent><w:p><w:r><w:t>Chart caption</w:t></w:r></w:p></w:txbxContent></v:textbox></v:shape></w:pict></mc:Fallback></mc:AlternateContent></w:r></w:p></w:body></w:document>',
685 },
686 expected: ["Chart caption"],
687 },
688 {
689 ext: "docx",
690 bounded: true,
691 parts: {
692 "word/document.xml":
693 '<w:document xmlns:w="urn:w"><w:body><w:p><w:pPr><w:pStyle w:val="Heading2"/></w:pPr><w:r><w:t>Report & totals</w:t></w:r></w:p><w:tbl><w:tr><w:tc><w:p><w:r><w:t>North</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>12</w:t></w:r></w:p><w:p><w:r><w:t>34</w:t></w:r></w:p></w:tc></w:tr><w:tr><w:tc><w:p><w:r><w:t>Before</w:t></w:r></w:p><w:tbl><w:tr><w:tc><w:p><w:r><w:t>Inner</w:t></w:r></w:p></w:tc></w:tr></w:tbl><w:p><w:r><w:t>After</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>Neighbor</w:t></w:r></w:p></w:tc></w:tr></w:tbl></w:body></w:document>',
694 },
695 expected: ["## Report & totals", "North\t12 34", "Before Inner After\tNeighbor"],
696 },
697 {
698 ext: "docx",
699 bounded: true,
700 parts: {
701 "word/document.xml":
702 '<w:document xmlns:w="urn:w"><w:body><w:p><w:r><w:t>First</w:t><w:cr/><w:t>Second</w:t><w:br/><w:t>Third</w:t></w:r></w:p></w:body></w:document>',
703 },
704 expected: ["First\nSecond\nThird"],
705 },
706 {
707 ext: "docx",
708 bounded: true,
709 parts: {
710 "word/document.xml":
711 '<w:document xmlns:w="urn:w"><w:body><w:p><w:r><w:t>Intro</w:t></w:r><w:r><w:fldChar w:fldCharType="begin"/></w:r><w:r><w:instrText> PAGEREF _Toc1 \\h </w:instrText></w:r><w:r><w:fldChar w:fldCharType="separate"/></w:r><w:r><w:t> 3</w:t></w:r><w:r><w:fldChar w:fldCharType="end"/></w:r></w:p><w:p><w:pPr><w:tabs><w:tab w:val="right" w:pos="9350"/></w:tabs></w:pPr><w:r><w:t xml:space="preserve">Fee </w:t></w:r><w:del><w:r><w:delText>$500</w:delText></w:r></w:del><w:ins><w:r><w:t>$700</w:t></w:r></w:ins><w:moveFrom><w:r><w:t>Moved</w:t></w:r></w:moveFrom><w:r><w:tab/><w:t>due</w:t></w:r></w:p></w:body></w:document>',
712 },
713 expected: ["Intro 3\nFee $700\tdue"],
714 absent: ["PAGEREF", "$500", "Moved"],
715 },
716 {
717 ext: "odt",
718 bounded: true,
719 parts: {
720 "content.xml":
721 "<document><text><tracked-changes><changed-region><deletion><change-info><creator>Reviewer</creator></change-info><p>$500</p></deletion></changed-region></tracked-changes><p>Fee <change/>$700</p></text></document>",
722 },
723 expected: ["Fee $700"],
724 absent: ["Reviewer", "$500"],
725 },
726 {
727 ext: "docx",
728 bounded: true,
729 parts: {
730 "word/document.xml": `<w:document><w:body><w:tbl><w:tr><w:tc><w:p><w:r><w:t>${"A".repeat(1_000_000)}</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>Later cell</w:t></w:r></w:p></w:tc></w:tr></w:tbl></w:body></w:document>`,
731 },
732 expected: ["A".repeat(32), "[Extracted text truncated at document extraction limits.]"],
733 },
734 {
735 ext: "xlsx",
736 bounded: true,
737 parts: {
738 "xl/workbook.xml": '<workbook><sheet name="Formulas" r:id="a"/></workbook>',
739 "xl/_rels/workbook.xml.rels":
740 '<Relationships><Relationship Id="a" Target="worksheets/a.xml"/></Relationships>',
741 "xl/styles.xml": '<styleSheet><cellXfs><xf numFmtId="9"/></cellXfs></styleSheet>',
742 "xl/worksheets/a.xml":
743 "<worksheet><row><c><f>SUM(11,22)</f><v/></c></row><row><c><f>SUM(33,44)</f><v> \n </v></c></row><row><c><f>SUM(55,66)</f></c></row><row><c><f>SUM(11,22)</f><v>33</v></c><c><f>SUM(0,0)</f><v>0</v></c></row></worksheet>",
744 },
745 expected: [
746 "SUM(11,22)\nSUM(33,44)\nSUM(55,66)",
747 "33 [Excel numFmtId=9; date system 1900]\t0 [Excel numFmtId=9; date system 1900]",
748 ],
749 absent: ["SUM(0,0)", "[Extracted text truncated"],
750 },
751 {
752 ext: "xlsx",
753 parts: {
754 "xl/workbook.xml":
755 '<workbook><sheets><sheet name="Inventory" r:id="a"/><sheet name="Budget" r:id="b"/></sheets></workbook>',
756 "xl/_rels/workbook.xml.rels":
757 '<Relationships><Relationship Id="a" Target="worksheets/second.xml"/><Relationship Id="b" Target="worksheets/first.xml"/></Relationships>',
758 "xl/sharedStrings.xml": "<sst><si><t>Lantern</t></si></sst>",
759 "xl/styles.xml":
760 '<styleSheet><numFmts><numFmt numFmtId="164" formatCode="yyyy-mm-dd"/></numFmts><cellXfs><xf numFmtId="0"/><xf numFmtId="14"/><xf numFmtId="9"/><xf numFmtId="164"/></cellXfs></styleSheet>',
761 "xl/worksheets/second.xml":
762 '<worksheet><sheetData><row><c r="A1" t="s"><v>0</v></c><c r="C1"><v>37</v></c></row></sheetData></worksheet>',
763 "xl/worksheets/first.xml":
764 '<worksheet><sheetData><row><c t="inlineStr"><is><t>Ferry</t></is></c><c><v>628</v></c><c s="1"><v>45292</v></c><c s="2"><v>0.25</v></c><c s="3"><v>45293</v></c></row></sheetData></worksheet>',
765 },
766 expected: [
767 "## Sheet: Inventory",
768 "Lantern\t\t37",
769 "## Sheet: Budget",
770 "Ferry\t628",
771 "45292 [Excel numFmtId=14; date system 1900]",
772 "0.25 [Excel numFmtId=9; date system 1900]",
773 "45293 [Excel numFmtId=164; format=yyyy-mm-dd; date system 1900]",
774 ],
775 },
776 {
777 ext: "xlsx",
778 bounded: true,
779 parts: {
780 "xl/workbook.xml": '<workbook><sheet name="Phonetics" r:id="a"/></workbook>',
781 "xl/_rels/workbook.xml.rels":
782 '<Relationships><Relationship Id="a" Target="worksheets/a.xml"/></Relationships>',
783 "xl/sharedStrings.xml":
784 '<sst><si><t>東京</t><rPh sb="0" eb="2"><t>とうきょう</t></rPh></si></sst>',
785 "xl/worksheets/a.xml":
786 '<worksheet><row><c t="s"><v>0</v></c><c t="inlineStr"><is><t>東京</t><rPh sb="0" eb="2"><t>とうきょう</t></rPh></is></c></row></worksheet>',
787 },
788 expected: ["## Sheet: Phonetics", "東京\t東京"],
789 absent: ["とうきょう"],
790 },
791 {
792 ext: "xlsx",
793 bounded: true,
794 parts: {
795 "xl/workbook.xml":
796 '<workbook><workbookPr date1904="true"/><sheets><sheet name="Dates" r:id="a"/></sheets></workbook>',
797 "xl/_rels/workbook.xml.rels":
798 '<Relationships><Relationship Id="a" Target="worksheets/dates.xml"/></Relationships>',
799 "xl/styles.xml": '<styleSheet><cellXfs><xf numFmtId="14"/></cellXfs></styleSheet>',
800 "xl/worksheets/dates.xml":
801 "<worksheet><sheetData><row><c><v>1</v></c></row></sheetData></worksheet>",
802 },
803 expected: ["## Sheet: Dates", "1 [Excel numFmtId=14; date system 1904]"],
804 },
805 {
806 ext: "xlsx",
807 bounded: true,
808 parts: {
809 "xl/workbook.xml":
810 '<workbook><sheets><sheet name="Repeated" r:id="a"/><sheet name="Unneeded" r:id="missing"/></sheets></workbook>',
811 "xl/_rels/workbook.xml.rels":
812 '<Relationships><Relationship Id="a" Target="worksheets/repeated.xml"/></Relationships>',
813 "xl/sharedStrings.xml": `<sst><si><t>${"A".repeat(1_000_000 - "## Sheet: Repeated\n".length)}</t></si></sst>`,
814 "xl/worksheets/repeated.xml": `<worksheet><sheetData>${'<row><c t="s"><v>0</v></c><c t="s"><v>0</v></c></row>'.repeat(1_500)}</sheetData></worksheet>`,
815 },
816 expected: [
817 "## Sheet: Repeated",
818 "A".repeat(32),
819 "[Extracted text truncated at document extraction limits.]",
820 ],
821 },
822 {
823 ext: "pptx",
824 bounded: true,
825 parts: {
826 "ppt/presentation.xml":
827 '<p:presentation><p:sldIdLst><p:sldId id="1" r:id="b"/><p:sldId id="2" r:id="a"/></p:sldIdLst></p:presentation>',
828 "ppt/_rels/presentation.xml.rels":
829 '<Relationships><Relationship Id="a" Target="slides/slide1.xml"/><Relationship Id="b" Target="slides/slide2.xml"/></Relationships>',
830 "ppt/slides/slide1.xml": "<p:sld><a:p><a:r><a:t>Departure</a:t></a:r></a:p></p:sld>",
831 "ppt/slides/slide2.xml": "<p:sld><a:p><a:r><a:t>Juniper Pier</a:t></a:r></a:p></p:sld>",
832 "ppt/slides/_rels/slide2.xml.rels":
833 '<Relationships><Relationship Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/slideLayout" Target="../notesSlides/ignored.xml"/><Relationship Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/notesSlide" TargetMode="External" Target="https://example.com/notes.xml"/><Relationship Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/notesSlide" Target="../notesSlides/notes7.xml"/></Relationships>',
834 "ppt/slides/_rels/slide1.xml.rels":
835 '<Relationships><Relationship Type="http://purl.oclc.org/ooxml/officeDocument/relationships/notesSlide" Target="/ppt/notesSlides/notes3.xml"/></Relationships>',
836 "ppt/notesSlides/notes7.xml":
837 "<p:notes><a:p><a:r><a:t>evacuation code ORCHID-81</a:t></a:r></a:p></p:notes>",
838 "ppt/notesSlides/notes3.xml":
839 "<p:notes><a:p><a:r><a:t>Board at gate 4</a:t></a:r></a:p></p:notes>",
840 "ppt/notesSlides/ignored.xml": "<p:notes><a:p>Unrelated notes</a:p></p:notes>",
841 },
842 expected: [
843 "## Slide 1",
844 "Juniper Pier\nNotes:\nevacuation code ORCHID-81",
845 "## Slide 2",
846 "Departure\nNotes:\nBoard at gate 4",
847 ],
848 absent: ["Unrelated notes", "[Skipped document part", "[Extracted text truncated"],
849 },
850 ...[false, true].map((missing) => ({
851 ext: "pptx",
852 bounded: true,
853 parts: {
854 "ppt/presentation.xml": '<presentation><sldId r:id="a"/></presentation>',
855 "ppt/_rels/presentation.xml.rels":
856 '<Relationships><Relationship Id="a" Target="slides/a.xml"/></Relationships>',
857 "ppt/slides/a.xml": "<slide><p>Slide text</p></slide>",
858 "ppt/slides/_rels/a.xml.rels": `<Relationships><Relationship Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/notesSlide" Target="../notesSlides/${missing ? "missing" : "bad"}.xml"/></Relationships>`,
859 "ppt/notesSlides/bad.xml": "<notes><p>Discard broken notes</notes>",
860 },
861 expected: [
862 "Slide text",
863 `[Skipped document part: ppt/notesSlides/${missing ? "missing" : "bad"}.xml (missing or invalid XML).]`,
864 ],
865 absent: ["Discard broken notes", "[Extracted text truncated"],
866 })),
867 {
868 ext: "pptx",
869 bounded: true,
870 parts: {
871 "ppt/presentation.xml":
872 '<presentation><sldId r:id="a"/><sldId r:id="b"/></presentation>',
873 "ppt/_rels/presentation.xml.rels":
874 '<Relationships><Relationship Id="a" Target="slides/a.xml"/><Relationship Id="b" Target="slides/b.xml"/></Relationships>',
875 "ppt/slides/a.xml": `<slide><p>${"A".repeat(600_000)}</p></slide>`,
876 "ppt/slides/b.xml": "<slide><p>Unneeded slide</p></slide>",
877 "ppt/slides/_rels/a.xml.rels":
878 '<Relationships><Relationship Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/notesSlide" Target="../notesSlides/a.xml"/></Relationships>',
879 "ppt/notesSlides/a.xml": `<notes><p>evacuation code ORCHID-81 ${"B".repeat(600_000)} Unneeded notes</p></notes>`,
880 },
881 expected: [
882 "## Slide 1",
883 "A".repeat(32),
884 "Notes:\nevacuation code ORCHID-81",
885 "B".repeat(32),
886 "[Extracted text truncated at document extraction limits.]",
887 ],
888 absent: ["Unneeded notes", "Unneeded slide", "## Slide 2"],
889 },
890 {
891 ext: "odt",
892 parts: {
893 "content.xml":
894 '<office:document><text:h text:outline-level="2">Orchard</text:h><text:p>19<text:s text:c="3"/>violet<text:s/>crates</text:p></office:document>',
895 },
896 expected: ["## Orchard", "19 violet crates"],
897 },
898 {
899 ext: "ods",
900 parts: {
901 "content.xml":
902 '<office:document><table:table table:name="Cargo"><table:table-row><table:table-cell><text:p>Violet</text:p></table:table-cell><table:table-cell office:value="19"/><table:table-cell office:value="27"><text:p/></table:table-cell></table:table-row></table:table></office:document>',
903 },
904 expected: ["## Sheet: Cargo", "Violet\t19\t27"],
905 },
906 {
907 ext: "ods",
908 bounded: true,
909 parts: {
910 "content.xml":
911 '<office:document><table:table table:name="Padded rows"><table:table-row><table:table-cell><text:p>Left</text:p></table:table-cell><table:table-cell table:number-columns-repeated="2"/><table:table-cell><text:p>Right</text:p></table:table-cell></table:table-row><table:table-row table:number-rows-repeated="1048575"><table:table-cell/></table:table-row></table:table></office:document>',
912 },
913 expected: ["## Sheet: Padded rows", "Left\t\t\tRight"],
914 absent: ["[Extracted text truncated"],
915 },
916 {
917 ext: "ods",
918 bounded: true,
919 parts: {
920 "content.xml":
921 '<office:document><table:table table:name="Padded"><table:table-row><table:table-cell><text:p>Populated</text:p></table:table-cell><table:table-cell table:number-columns-repeated="16383"/></table:table-row><table:table-row table:number-rows-repeated="1048575"><table:table-cell table:number-columns-repeated="16384"><text:p/></table:table-cell></table:table-row></table:table></office:document>',
922 },
923 expected: ["## Sheet: Padded", "Populated"],
924 absent: ["[Extracted text truncated", "\t"],
925 },
926 {
927 ext: "ods",
928 bounded: true,
929 parts: {
930 "content.xml": `<office:document><table:table table:name="Repeated"><table:table-row><table:table-cell table:number-columns-repeated="512"><text:p>Lead ${"a".repeat(600_000)} Tail</text:p></table:table-cell></table:table-row></table:table></office:document>`,
931 },
932 expected: [
933 "## Sheet: Repeated",
934 "Lead ",
935 " Tail\tLead ",
936 "[Extracted text truncated at document extraction limits.]",
937 ],
938 },
939 {
940 ext: "odp",
941 parts: {
942 "content.xml":
943 "<office:document><draw:page><text:p>First</text:p></draw:page><draw:page><text:p>Second</text:p></draw:page></office:document>",
944 },
945 expected: ["## Slide 1", "First", "## Slide 2", "Second"],
946 },
947 {
948 ext: "ods",
949 bounded: true,
950 parts: {
951 "content.xml": `<office:document>${Array.from(
952 { length: 256 },
953 (_, index) =>
954 `<table:table table:name="${index === 0 ? "Shared" : "Unneeded"}"><table:table-row><table:table-cell table:number-columns-repeated="512"><text:p>${"B".repeat(4096)}</text:p></table:table-cell></table:table-row></table:table>`,
955 ).join("")}</office:document>`,
956 },
957 expected: [
958 "## Sheet: Shared",
959 "B".repeat(32),
960 "[Extracted text truncated at document extraction limits.]",
961 ],
962 absent: ["## Sheet: Unneeded"],
963 },
964 {
965 ext: "odt",
966 bounded: true,
967 parts: {
968 "content.xml": `<office:document><text:p>Before whitespace</text:p><table:table><table:table-row table:number-rows-repeated="10000"><table:table-cell table:number-columns-repeated="512"><text:p>${" ".repeat(10_000)}</text:p></table:table-cell></table:table-row></table:table><text:p>Unneeded</text:p></office:document>`,
969 },
970 expected: ["Before whitespace", "Unneeded"],
971 absent: ["[Extracted text truncated"],
972 },
973 ...["odt", "odp", "ods"].flatMap((ext) =>
974 [false, true].map((rows) => ({
975 ext,
976 bounded: true,
977 parts: {
978 "content.xml": `<office:document><draw:page>${Array.from(
979 { length: 256 },
980 () =>
981 `<table:table><table:table-row${rows ? ' table:number-rows-repeated="10000"' : ""}><table:table-cell table:number-columns-repeated="512"><text:p>${"Budget ".repeat(rows ? 160 : 585)}</text:p></table:table-cell></table:table-row></table:table>`,
982 ).join(
983 "",
984 )}</draw:page><draw:page><text:p>Unneeded</text:p></draw:page></office:document>`,
985 },
986 expected: ["Budget ", "[Extracted text truncated at document extraction limits.]"],
987 absent: ["Unneeded"],
988 })),
989 ),
990 {
991 ext: "docx",
992 bounded: true,
993 parts: {
994 "word/document.xml": `<w:document><w:p><w:t>${"A".repeat(1_000_100)}</w:t></w:p><w:p><w:t>Unneeded</w:t></w:p></w:document>`,
995 },
996 expected: ["A".repeat(32), "[Extracted text truncated at document extraction limits.]"],
997 absent: ["Unneeded"],
998 },
999 {
1000 ext: "pptx",
1001 bounded: true,
1002 parts: {
1003 "ppt/presentation.xml":
1004 '<p:presentation><p:sldId r:id="a"/><p:sldId r:id="missing"/></p:presentation>',
1005 "ppt/_rels/presentation.xml.rels":
1006 '<Relationships><Relationship Id="a" Target="slides/slide.xml"/></Relationships>',
1007 "ppt/slides/slide.xml": `<p:sld><a:p><a:t>${"A".repeat(1_000_100)}</a:t></a:p></p:sld>`,
1008 },
1009 expected: [
1010 "## Slide 1",
1011 "A".repeat(32),
1012 "[Extracted text truncated at document extraction limits.]",
1013 ],
1014 absent: ["## Slide 2"],
1015 },
1016 {
1017 ext: "epub",
1018 bounded: true,
1019 parts: {
1020 "META-INF/container.xml": '<container><rootfile full-path="book.opf"/></container>',
1021 "book.opf":
1022 '<package><item id="a" href="a.xhtml"/><itemref idref="a"/><itemref idref="missing"/></package>',
1023 "a.xhtml": `<html><body><p>${"A".repeat(1_000_100)}</p></body></html>`,
1024 },
1025 expected: [
1026 "## Chapter 1",
1027 "A".repeat(32),
1028 "[Extracted text truncated at document extraction limits.]",
1029 ],
1030 absent: ["## Chapter 2"],
1031 },
1032 {
1033 ext: "epub",
1034 parts: {
1035 "META-INF/container.xml":
1036 '<container><rootfile full-path="OEBPS/book.opf"/></container>',
1037 "OEBPS/book.opf":
1038 '<package><manifest><item id="a" href="a.xhtml"/><item id="b" href="b.xhtml"/></manifest><spine><itemref idref="b"/><itemref idref="a"/></spine></package>',
1039 "OEBPS/a.xhtml":
1040 '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"><html><body><p>Last chapter</p><p>Price <s>10</s> 8</p></body></html>',
1041 "OEBPS/b.xhtml":
1042 "<!DOCTYPE html><html><body><h1>First chapter</h1><p>Larkspur</p><div>Direct div<section>Direct section<li>Direct item <em>emphasis</em> trailing</li></section></div><div>Account: A<br/>Balance: 731<line-break/>Currency: USD</div><table><thead><tr><th>Heading</th><th>Total</th></tr></thead><tbody><tr><td>Important value</td><td>731</td></tr><tr><td>Before<table><tr><td>Inner</td></tr></table>After</td><td>Neighbor</td></tr></tbody></table></body></html>",
1043 },
1044 expected: [
1045 "## Chapter 1",
1046 "# First chapter",
1047 "Larkspur",
1048 "Direct div",
1049 "Direct section",
1050 "Direct item emphasis trailing",
1051 "Account: A\nBalance: 731\nCurrency: USD",
1052 "Heading\tTotal",
1053 "Important value\t731",
1054 "Before Inner After\tNeighbor",
1055 "## Chapter 2",
1056 "Last chapter",
1057 "Price 10 8",
1058 ],
1059 },
1060 ];
1061 for (const [caseIndex, item] of cases.entries()) {
1062 const local: Buffer[] = [];
1063 const central: Buffer[] = [];
1064 let offset = 0;
1065 for (const [name, content] of Object.entries(item.parts)) {
1066 const data = Buffer.from(content!);
1067 const compressed = /\.(bmp|bin)$/.test(name) ? NodeZlib.deflateRawSync(data) : data;
1068 const filename = Buffer.from(name);
1069 const header = Buffer.alloc(30);
1070 header.writeUInt32LE(0x04034b50);
1071 header.writeUInt16LE(20, 4);
1072 header.writeUInt16LE(compressed === data ? 0 : 8, 8);
1073 header.writeUInt32LE(NodeZlib.crc32(data), 14);
1074 header.writeUInt32LE(compressed.length, 18);
1075 header.writeUInt32LE(data.length, 22);
1076 header.writeUInt16LE(filename.length, 26);
1077 const directory = Buffer.alloc(46);
1078 directory.writeUInt32LE(0x02014b50);
1079 directory.writeUInt16LE(20, 4);
1080 directory.writeUInt16LE(20, 6);
1081 directory.writeUInt16LE(compressed === data ? 0 : 8, 10);
1082 directory.writeUInt32LE(NodeZlib.crc32(data), 16);
1083 directory.writeUInt32LE(compressed.length, 20);
1084 directory.writeUInt32LE(data.length, 24);
1085 directory.writeUInt16LE(filename.length, 28);
1086 directory.writeUInt32LE(offset, 42);
1087 local.push(header, filename, compressed);
1088 central.push(directory, filename);
1089 offset += header.length + filename.length + compressed.length;
1090 }
1091 const end = Buffer.alloc(22);
1092 end.writeUInt32LE(0x06054b50);
1093 end.writeUInt16LE(central.length / 2, 8);
1094 end.writeUInt16LE(central.length / 2, 10);
1095 end.writeUInt32LE(Buffer.concat(central).length, 12);
1096 end.writeUInt32LE(offset, 16);
1097 const path = NodePath.join(config.attachmentsDir, `structure-${caseIndex}.${item.ext}`);
1098 NodeFS.writeFileSync(path, Buffer.concat([...local, ...central, end]));
1099 const extracted = yield* Effect.promise(() => extractDocumentText(path, `.${item.ext}`));
1100 let previous = -1;
1101 for (const expected of item.expected) {
1102 const index = extracted.indexOf(expected);
1103 expect(index, `${item.ext}: ${expected} in ${extracted.slice(0, 500)}`).toBeGreaterThan(
1104 previous,
1105 );
1106 previous = index;
1107 }
1108 if ("occurrences" in item)
1109 expect(extracted.split(item.occurrences.text).length - 1).toBe(item.occurrences.count);
1110 if ("absent" in item)
1111 for (const absent of item.absent) expect(extracted).not.toContain(absent);
1112 if (("bounded" in item && item.bounded) || item.ext === "epub") {
1113 expect(extracted.length).toBeLessThan(1_000_100);
1114 expect(extracted.match(/\[Extracted text truncated/g)?.length ?? 0).toBeLessThanOrEqual(
1115 1,
1116 );
1117 const documents = yield* DocumentText.DocumentText;
1118 expect(yield* documents.ensure(path)).toBe(path + ".txt");
1119 expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe(extracted);
1120 }
1121 if (item.ext === "docx") {
1122 end.writeUInt16LE(2049, 8);
1123 end.writeUInt16LE(2049, 10);
1124 NodeFS.writeFileSync(path, Buffer.concat([...local, ...central, end]));
1125 yield* Effect.promise(async () => {
1126 expect(await extractDocumentText(path, ".docx")).toContain("[Extracted text truncated");
1127 });
1128 end.writeUInt16LE(central.length / 2, 8);
1129 end.writeUInt16LE(central.length / 2, 10);
1130 central[0]!.writeUInt16LE(8, 10);
1131 central[0]!.writeUInt32LE(33 * 1024 * 1024, 24);
1132 NodeFS.writeFileSync(path, Buffer.concat([...local, ...central, end]));
1133 yield* Effect.promise(async () => {
1134 expect(await extractDocumentText(path, ".docx")).toContain("[Extracted text truncated");
1135 });
1136 }
1137 }
1138 const path = NodePath.join(config.attachmentsDir, "unicode.rtf");
1139 NodeFS.writeFileSync(
1140 path,
1141 String.raw`{\rtf1\ansi{\fonttbl hidden}Hello\tab world\par Unicode \u945?{\*\secret hidden}}`,
1142 );
1143 const extracted = yield* Effect.promise(() => extractDocumentText(path, ".rtf"));
1144 expect(extracted).toContain("Hello\tworld\nUnicode α");
1145 expect(extracted).not.toContain("hidden");
1146
1147 for (const [name, content, expected] of [
1148 [
1149 "cyrillic-escaped",
1150 String.raw`{\rtf1\ansi\ansicpg1251 \'cf\'f0\'e8\'e2\'e5\'f2 {\ansicpg1252 \'e9} \'cc\'ee\'f1\'ea\'e2\'e0}`,
1151 "Привет é Москва",
1152 ],
1153 [
1154 "cyrillic-raw",
1155 Buffer.concat([
1156 Buffer.from(String.raw`{\rtf1\ansi\ansicpg1251 `),
1157 Buffer.from([0xcf, 0xf0, 0xe8, 0xe2, 0xe5, 0xf2]),
1158 Buffer.from("}"),
1159 ]),
1160 "Привет",
1161 ],
1162 ] as const) {
1163 const encodedPath = NodePath.join(config.attachmentsDir, `${name}.rtf`);
1164 NodeFS.writeFileSync(encodedPath, content);
1165 expect(yield* Effect.promise(() => extractDocumentText(encodedPath, ".rtf"))).toBe(
1166 expected,
1167 );
1168 const documents = yield* DocumentText.DocumentText;
1169 expect(yield* documents.ensure(encodedPath)).toBe(encodedPath + ".txt");
1170 expect(NodeFS.readFileSync(encodedPath + ".txt", "utf8")).toBe(expected);
1171 }
1172 const notebookPath = NodePath.join(config.attachmentsDir, "limited.ipynb");
1173 NodeFS.writeFileSync(
1174 notebookPath,
1175 JSON.stringify({ cells: Array.from({ length: 10_001 }, () => ({ source: "Kept" })) }),
1176 );
1177 const notebookLimited = yield* Effect.promise(() =>
1178 extractDocumentText(notebookPath, ".ipynb"),
1179 );
1180 expect(notebookLimited).toContain("## Cell 10000");
1181 expect(notebookLimited).not.toContain("## Cell 10001");
1182 expect(notebookLimited.match(/\[Extracted text truncated/g)).toHaveLength(1);
1183 expect(yield* (yield* DocumentText.DocumentText).ensure(notebookPath)).toBe(
1184 notebookPath + ".txt",
1185 );
1186 expect(NodeFS.readFileSync(notebookPath + ".txt", "utf8")).toBe(notebookLimited);
1187 const errorPath = NodePath.join(config.attachmentsDir, "limited-error.ipynb");
1188 NodeFS.writeFileSync(
1189 errorPath,
1190 JSON.stringify({
1191 cells: [
1192 {
1193 cell_type: "code",
1194 source: "1 / 0",
1195 outputs: [
1196 {
1197 output_type: "error",
1198 ename: "ZeroDivisionError",
1199 evalue: "division by zero",
1200 traceback: ["\u001b[31m" + "trace".repeat(250_000) + "\u001b[0m", "After limit"],
1201 },
1202 ],
1203 },
1204 ],
1205 }),
1206 );
1207 const errorLimited = yield* Effect.promise(() => extractDocumentText(errorPath, ".ipynb"));
1208 expect(errorLimited).toContain("ZeroDivisionError: division by zero\ntrace");
1209 expect(errorLimited.length).toBeLessThan(1_000_100);
1210 expect(errorLimited.match(/\[Extracted text truncated/g)).toHaveLength(1);
1211 expect(errorLimited).not.toContain("After limit");
1212 expect(errorLimited).not.toContain("\u001b");
1213 const deepPath = NodePath.join(config.attachmentsDir, "deep.rtf");
1214 NodeFS.writeFileSync(
1215 deepPath,
1216 String.raw`{\rtf1 Kept` + "{".repeat(128) + "Lost" + "}".repeat(129),
1217 );
1218 const deepLimited = yield* Effect.promise(() => extractDocumentText(deepPath, ".rtf"));
1219 expect(deepLimited).toBe("Kept\n\n[Extracted text truncated at document extraction limits.]");
1220 const unsupportedPath = NodePath.join(config.attachmentsDir, "unsupported-codepage.rtf");
1221 NodeFS.writeFileSync(unsupportedPath, String.raw`{\rtf1\ansi\ansicpg932 \'82\'a0}`);
1222 yield* Effect.promise(async () => {
1223 await expect(extractDocumentText(unsupportedPath, ".rtf")).rejects.toThrow(
1224 "Unsupported RTF code page",
1225 );
1226 });
1227 expect(yield* (yield* DocumentText.DocumentText).ensure(unsupportedPath)).toBeUndefined();
1228 expect(NodeFS.existsSync(unsupportedPath + ".txt")).toBe(false);
1229 NodeFS.writeFileSync(path, String.raw`{\rtf1 ` + "x".repeat(1_000_100) + "}");
1230 expect(yield* Effect.promise(() => extractDocumentText(path, ".rtf"))).toContain(
1231 "[Extracted text truncated",
1232 );
1233 NodeFS.writeFileSync(path, String.raw`{\rtf1 First` + " ".repeat(1_000_100) + "Last}");
1234 const whitespaceTruncated = yield* Effect.promise(() => extractDocumentText(path, ".rtf"));
1235 expect(whitespaceTruncated).toBe(
1236 "First\n\n[Extracted text truncated at document extraction limits.]",
1237 );
1238 const documents = yield* DocumentText.DocumentText;
1239 expect(yield* documents.ensure(path)).toBe(path + ".txt");
1240 expect(NodeFS.readFileSync(path + ".txt", "utf8")).toBe(whitespaceTruncated);
1241 NodeFS.truncateSync(path, 17 * 1024 * 1024);
1242 yield* Effect.promise(async () => {
1243 expect(await extractDocumentText(path, ".rtf")).toContain("[Extracted text truncated");
1244 });
1245 }).pipe(Effect.provide(layer)),
1246 );
1247 it.effect("keeps selected values intact and appends an actual readable server path", () =>
1248 Effect.gen(function* () {
1249 const config = yield* ServerConfig.ServerConfig;
1250 const path = NodePath.join(config.attachmentsDir, `${attachment.id}.txt`);
1251 NodeFS.writeFileSync(path, "spec");
1252 const original = { q: ["First", "Second"], other: "No file" };
1253 const answers = yield* appendUserInputAttachmentPaths({
1254 answers: original,
1255 attachmentsDir: config.attachmentsDir,
1256 attachmentsByQuestionId: { q: [attachment] },
1257 });
1258 expect(answers.q).toEqual([
1259 "First",
1260 "Second",
1261 `Attached file "spec \\"final\\".txt": ${JSON.stringify(path)}`,
1262 ]);
1263 expect(answers.other).toBe("No file");
1264 expect(original.q).toEqual(["First", "Second"]);
1265 expect(NodeFS.readFileSync(path, "utf8")).toBe("spec");
1266 const specialKey = yield* appendUserInputAttachmentPaths({
1267 answers: {},
1268 attachmentsDir: config.attachmentsDir,
1269 attachmentsByQuestionId: { ["__proto__"]: [attachment] },
1270 });
1271 expect(Object.keys(specialKey)).toEqual(["__proto__"]);
1272 expect(specialKey["__proto__"]).toContain(JSON.stringify(path));
1273 }).pipe(Effect.provide(layer)),
1274 );
1275 it.effect("does not send an unavailable attachment path", () =>
1276 Effect.gen(function* () {
1277 const config = yield* ServerConfig.ServerConfig;
1278 const result = yield* appendUserInputAttachmentPaths({
1279 answers: { q: "" },
1280 attachmentsDir: config.attachmentsDir,
1281 attachmentsByQuestionId: { q: [attachment] },
1282 }).pipe(Effect.result);
1283 expect(result._tag).toBe("Failure");
1284 }).pipe(Effect.provide(layer)),
1285 );
1286});