Skip to content
Source
Sign inSign up
pingdotgg/t3code
Open History to see the last commit

largeText.worker.ts

189 lines6.2 KB

1import { parseDelimitedPreview } from "@t3tools/shared/delimitedPreview";
2import {
3 LARGE_TEXT_MAX_BYTES,
4 LARGE_TEXT_MAX_ROWS,
5 LARGE_TEXT_MAX_COLUMNS,
6 PREVIEW_PAGE_ROWS,
7 type TextPreviewRequest,
8 type TextPreviewResult,
9} from "./largeTextProtocol";
10​
11const chunks: { start: number; blob: Blob }[] = [];
12const offsets: Uint32Array[] = [new Uint32Array(65536)];
13let entries = 1;
14let size = 0;
15let delimiter: "," | "\t" | null = null;
16let complete = false;
17let columns = 1;
18function reply(value: TextPreviewResult) {
19 self.postMessage(value, { transfer: [] });
21function at(index: number) {
22 return offsets[index >>> 16]![index & 65535]!;
24function set(index: number, value: number) {
25 offsets[index >>> 16] ??= new Uint32Array(65536);
26 offsets[index >>> 16]![index & 65535] = value;
28async function read(start: number, end: number) {
29 let low = 0;
30 let high = chunks.length;
31 while (low + 1 < high) {
32 const mid = (low + high) >>> 1;
33 if (chunks[mid]!.start <= start) low = mid;
34 else high = mid;
35 }
36 const parts: Blob[] = [];
37 for (let i = low; i < chunks.length; i++) {
38 const part = chunks[i]!;
39 if (part.start >= end) break;
40 parts.push(part.blob.slice(Math.max(0, start - part.start), end - part.start));
41 }
42 return new TextDecoder("utf-8", { fatal: true }).decode(await new Blob(parts).arrayBuffer());
44async function page(index: number) {
45 const start = index * PREVIEW_PAGE_ROWS;
46 const end = Math.min(entries - 1, start + PREVIEW_PAGE_ROWS);
47 if (start >= end) return;
48 const text = await read(at(start), at(end));
49 const rows = delimiter
50 ? parseDelimitedPreview(text, delimiter, {
51 maxRows: PREVIEW_PAGE_ROWS,
52 maxColumns: LARGE_TEXT_MAX_COLUMNS,
53 maxCellLength: 1024 * 1024,
54 }).rows
55 : text
56 .replace(/\r?\n$|\r$/, "")
57 .split(/\r\n|\n|\r/)
58 .map((line) => [line]);
59 reply({ type: "page", index, rows });
61async function open(input: Extract<TextPreviewRequest, { type: "open" }>) {
62 delimiter = input.delimiter;
63 let response =
64 input.sizeBytes === 0
65 ? new Response(new Blob([]))
66 : input.text !== undefined
67 ? new Response(new Blob([input.text]))
68 : await fetch(input.src!, {
69 headers: { Range: `bytes=0-${LARGE_TEXT_MAX_BYTES}` },
70 cache: "no-store",
71 });
72 if (response.status === 416 && response.headers.get("content-range") === "bytes */0") {
73 await response.body?.cancel();
74 response = new Response(new Blob([]));
75 }
76 if (!response.ok || !response.body)
77 throw new Error("The file could not be loaded. Close it and try again.");
78 const total = Number(
79 response.headers.get("content-range")?.split("/")[1] ?? response.headers.get("content-length"),
80 );
81 if (total > LARGE_TEXT_MAX_BYTES) {
82 await response.body.cancel();
83 throw new Error("Text previews support files up to 256 MB.");
84 }
85 const reader = response.body.getReader();
86 let state = 0;
87 let fieldStart = true;
88 let cr = false;
89 let rowColumns = 1;
90 let first = false;
91 try {
92 while (true) {
93 const { value, done } = await reader.read();
94 if (done) break;
95 if (size + value.length > LARGE_TEXT_MAX_BYTES)
96 throw new Error("Text previews support files up to 256 MB.");
97 chunks.push({ start: size, blob: new Blob([value]) });
98 for (let i = 0; i < value.length; i++) {
99 const byte = value[i]!;
100 const pos = size + i;
101 if (
102 (pos === 0 && byte === 239) ||
103 (pos === 1 && byte === 187) ||
104 (pos === 2 && byte === 191)
105 )
106 continue;
107 if (byte === 0) throw new Error("This file contains binary data.");
108 if (pos - at(entries - 1) > 1024 * 1024)
109 throw new Error("A line or record exceeds the 1 MB preview limit.");
110 if (cr) {
111 cr = false;
112 if (byte === 10) {
113 set(entries - 1, pos + 1);
114 continue;
115 }
116 }
117 if (delimiter) {
118 if (state === 1) {
119 if (byte === 34) state = 2;
120 continue;
121 }
122 if (state === 2) {
123 if (byte === 34) {
124 state = 1;
125 continue;
126 }
127 state = 0;
128 }
129 if (byte === 34 && fieldStart) {
130 state = 1;
131 fieldStart = false;
132 continue;
133 }
134 if (byte === delimiter.charCodeAt(0)) {
135 rowColumns++;
136 fieldStart = true;
137 continue;
138 }
139 }
140 if (byte === 10 || byte === 13) {
141 if (entries > LARGE_TEXT_MAX_ROWS)
142 throw new Error("Text previews support up to 5 million rows.");
143 set(entries++, pos + 1);
144 columns = Math.min(LARGE_TEXT_MAX_COLUMNS, Math.max(columns, rowColumns));
145 rowColumns = 1;
146 fieldStart = true;
147 cr = byte === 13;
148 } else fieldStart = false;
149 }
150 size += value.length;
151 if (!first && entries > 1) {
152 reply({ type: "meta", count: entries - 1, columns, complete: false, sizeBytes: size });
153 await page(0);
154 first = true;
155 }
156 }
157 if (total > 0 && size !== total)
158 throw new Error("The file download was incomplete. Close it and try again.");
159 if (size > at(entries - 1)) {
160 if (entries > LARGE_TEXT_MAX_ROWS)
161 throw new Error("Text previews support up to 5 million rows.");
162 set(entries++, size);
163 columns = Math.min(LARGE_TEXT_MAX_COLUMNS, Math.max(columns, rowColumns));
164 }
165 complete = true;
166 reply({ type: "meta", count: entries - 1, columns, complete, sizeBytes: size });
167 await page(0);
168 } finally {
169 await reader.cancel();
170 }
172self.addEventListener("message", (event: MessageEvent<TextPreviewRequest>) => {
173 const input = event.data;
174 void (
175 input.type === "open"
176 ? open(input)
177 : input.type === "page"
178 ? page(input.index)
179 : complete
180 ? read(0, size).then((text) => reply({ type: "copy", text }))
181 : Promise.reject(new Error("Wait for the file to finish loading before copying."))
182 ).catch((cause: unknown) =>
183 reply({
184 type: "error",
185 operation: input.type,
186 message: cause instanceof Error ? cause.message : "Could not read this file.",
187 }),
188 );
189});