Open History to see the last commit
1import { getFiletypeFromFileName } from "@pierre/diffs/utils/getFiletypeFromFileName";
2import {
3 parse,
4 type Command,
5 type DoubleQuotedChild,
6 type HereDoc,
7 type Word,
8 type WordPart,
9} from "unbash";
10
11/** starts[i]..ends[i] spans the original spelling of text[i], including escapes. */
12export interface EmbeddedScript {
13 readonly language: string;
14 readonly text: string;
15 readonly starts: ReadonlyArray<number>;
16 readonly ends: ReadonlyArray<number>;
17}
18
19export interface SyntaxToken {
20 readonly content: string;
21 readonly offset: number;
22 readonly color?: string;
23 readonly fontStyle?: number;
24}
25
26type TokenLines = ReadonlyArray<ReadonlyArray<SyntaxToken>>;
27
28interface MappedText {
29 readonly text: string;
30 readonly starts: ReadonlyArray<number>;
31 readonly ends: ReadonlyArray<number>;
32}
33
34interface Interpreter {
35 readonly language: string;
36 readonly inline: (flag: string) => boolean;
37 readonly valueOptions?: ReadonlySet<string>;
38 readonly scriptFileOptions?: ReadonlySet<string>;
39 readonly stdin: "always" | "without-script-file" | "never";
40 readonly positionalScript?: number;
41 readonly repeatable?: boolean;
42 readonly subcommand?: string;
43 /** PowerShell matches parameters case-insensitively; the lists above are lowercase. */
44 readonly ignoreCase?: boolean;
45}
46
47const MAX_DEPTH = 4;
48
49const flagIn =
50 (...flags: string[]) =>
51 (flag: string) =>
52 flags.includes(flag);
53
54const SHELL: Interpreter = {
55 language: "shellscript",
56 inline: (flag) => /^-[a-z]*c[a-z]*$/u.test(flag),
57 valueOptions: new Set(["-o", "-O", "--rcfile", "--init-file"]),
58 stdin: "without-script-file",
59};
60
61const PYTHON: Interpreter = {
62 language: "python",
63 inline: flagIn("-c"),
64 valueOptions: new Set(["-W", "-X"]),
65 scriptFileOptions: new Set(["-m"]),
66 stdin: "without-script-file",
67};
68
69const SQL_SHELL = {
70 language: "sql",
71 valueOptions: new Set([
72 "-d",
73 "-h",
74 "-u",
75 "-U",
76 "--dbname",
77 "--host",
78 "--user",
79 "--username",
80 "-separator",
81 "-nullvalue",
82 "-init",
83 ]),
84 stdin: "always",
85 repeatable: true,
86} satisfies Omit<Interpreter, "inline">;
87
88const INTERPRETERS: Record<string, Interpreter> = {
89 sh: SHELL,
90 bash: SHELL,
91 zsh: SHELL,
92 dash: SHELL,
93 ash: SHELL,
94 ksh: SHELL,
95 fish: { language: "fish", inline: flagIn("-c", "--command"), stdin: "without-script-file" },
96 node: {
97 language: "javascript",
98 inline: flagIn("-e", "--eval", "-p", "--print"),
99 valueOptions: new Set(["-r", "--require", "--import", "--loader", "-C", "--conditions"]),
100 stdin: "without-script-file",
101 },
102 bun: { language: "typescript", inline: flagIn("-e", "--eval", "-p", "--print"), stdin: "never" },
103 deno: {
104 language: "typescript",
105 inline: () => false,
106 // `--env-file` takes its optional path only as `--env-file=path`.
107 valueOptions: new Set(["-c", "--config", "--import-map", "--ext"]),
108 stdin: "never",
109 subcommand: "eval",
110 },
111 ruby: {
112 language: "ruby",
113 inline: flagIn("-e"),
114 valueOptions: new Set(["-r", "-I", "-C"]),
115 stdin: "without-script-file",
116 },
117 perl: { language: "perl", inline: flagIn("-e", "-E"), stdin: "without-script-file" },
118 osascript: {
119 language: "applescript",
120 inline: flagIn("-e"),
121 valueOptions: new Set(["-l", "-s"]),
122 stdin: "without-script-file",
123 },
124 pwsh: {
125 language: "powershell",
126 inline: flagIn("-c", "-command"),
127 valueOptions: new Set(["-executionpolicy", "-workingdirectory", "-windowstyle"]),
128 scriptFileOptions: new Set(["-f", "-file", "-encodedcommand"]),
129 stdin: "never",
130 ignoreCase: true,
131 },
132 psql: {
133 ...SQL_SHELL,
134 inline: flagIn("-c", "--command"),
135 scriptFileOptions: new Set(["-f", "--file"]),
136 },
137 mysql: { ...SQL_SHELL, inline: flagIn("-e", "--execute") },
138 sqlite3: { ...SQL_SHELL, inline: flagIn("-cmd"), positionalScript: 1 },
139 duckdb: {
140 ...SQL_SHELL,
141 inline: flagIn("-c", "-cmd"),
142 scriptFileOptions: new Set(["-f"]),
143 positionalScript: 1,
144 },
145};
146INTERPRETERS.powershell = INTERPRETERS.pwsh!;
147INTERPRETERS.mariadb = INTERPRETERS.mysql!;
148
149const PROGRAM_WRAPPERS: Record<string, ReadonlySet<string>> = {
150 env: new Set(["-u", "--unset", "-C", "--chdir"]),
151 sudo: new Set(["-u", "--user", "-g", "--group", "-C", "-D", "--chdir"]),
152 nice: new Set(["-n", "--adjustment"]),
153 nohup: new Set(),
154 exec: new Set(["-a"]),
155 command: new Set(),
156 builtin: new Set(),
157};
158
159const DELIMITER_LANGUAGES: Record<string, string> = {
160 py: "python",
161 python: "python",
162 js: "javascript",
163 javascript: "javascript",
164 ts: "typescript",
165 typescript: "typescript",
166 sql: "sql",
167 json: "json",
168 yaml: "yaml",
169 yml: "yaml",
170 toml: "toml",
171 sh: "shellscript",
172 bash: "shellscript",
173 shell: "shellscript",
174 rb: "ruby",
175 ruby: "ruby",
176 html: "html",
177 css: "css",
178 md: "markdown",
179 markdown: "markdown",
180 xml: "xml",
181 diff: "diff",
182 patch: "diff",
183};
184
185/** Returns nothing for commands that do not parse cleanly. */
186export function embeddedScripts(command: string): EmbeddedScript[] {
187 const identity = Array.from({ length: command.length }, (_, index) => index);
188 return scriptsIn(
189 { text: command, starts: identity, ends: identity.map((index) => index + 1) },
190 0,
191 );
192}
193
194function scriptsIn(source: MappedText, depth: number): EmbeddedScript[] {
195 const script = parse(source.text);
196 if (script.errors?.length) return [];
197 const found: EmbeddedScript[] = [];
198 forEachCommand(script.commands, (command) => {
199 for (const { language, mapped } of commandScripts(command, source.text)) {
200 if (mapped.text.trim() === "") continue;
201 const embedded: EmbeddedScript = {
202 language,
203 text: mapped.text,
204 starts: mapped.starts.map((start) => source.starts[start]!),
205 ends: mapped.ends.map((end) => source.ends[end - 1]!),
206 };
207 found.push(embedded);
208 if (language === "shellscript" && depth + 1 < MAX_DEPTH) {
209 found.push(...scriptsIn(embedded, depth + 1));
210 }
211 }
212 });
213 return found;
214}
215
216function isNode(value: unknown): value is { readonly type: string } {
217 return typeof value === "object" && value !== null && "type" in value;
218}
219
220function forEachCommand(
221 node: unknown,
222 visit: (command: Command) => void,
223 seen = new WeakSet<object>(),
224): void {
225 if (Array.isArray(node)) {
226 for (const child of node) forEachCommand(child, visit, seen);
227 return;
228 }
229 if (!isNode(node) || seen.has(node)) return;
230 seen.add(node);
231 switch (node.type) {
232 case "Command":
233 visit(node as Command);
234 break;
235 case "CommandExpansion":
236 case "ProcessSubstitution": {
237 const { script } = node as Extract<WordPart, { type: "CommandExpansion" }>;
238 if (script && !script.errors?.length && script.source === undefined) {
239 forEachCommand(script.commands, visit, seen);
240 }
241 return;
242 }
243 }
244 // Word parts are lazy getters, which Object.values skips.
245 if ("parts" in node) forEachCommand(node.parts, visit, seen);
246 for (const value of Object.values(node)) {
247 if (typeof value === "object") forEachCommand(value, visit, seen);
248 }
249}
250
251function programName(word: Word): string {
252 // Bash reads `C:\Python312\python.exe` unquoted as escapes; Windows meant a path.
253 const path = /\\/u.test(word.text) ? word.text.replace(/^["']|["']$/gu, "") : word.value;
254 return (path.split(/[\\/]/u).at(-1) ?? "").toLowerCase().replace(/\.exe$/u, "");
255}
256
257function interpreterFor(name: string): Interpreter | undefined {
258 if (/^python(?:\d+(?:\.\d+)?)?$|^pypy3?$|^py$/u.test(name)) return PYTHON;
259 return Object.hasOwn(INTERPRETERS, name) ? INTERPRETERS[name] : undefined;
260}
261
262function programIndex(words: ReadonlyArray<Word>): number {
263 let index = 0;
264 while (index < words.length) {
265 const name = programName(words[index]!);
266 if (!Object.hasOwn(PROGRAM_WRAPPERS, name)) return index;
267 const valueOptions = PROGRAM_WRAPPERS[name]!;
268 index += 1;
269 let optionsEnded = false;
270 while (index < words.length) {
271 const value = words[index]!.value;
272 const isOption = !optionsEnded && value.startsWith("-");
273 if (isOption && value === "--") optionsEnded = true;
274 // `env -- FOO=1 cmd` still takes assignments after the options end.
275 if (isOption || /^[A-Za-z_]\w*=/u.test(value)) {
276 index += isOption && valueOptions.has(value) ? 2 : 1;
277 } else {
278 break;
279 }
280 }
281 }
282 return index;
283}
284
285function commandScripts(
286 command: Command,
287 source: string,
288): Array<{ language: string; mapped: MappedText }> {
289 if (!command.name) return [];
290 const words = [command.name, ...command.args.filter((arg): arg is Word => arg.type === "Word")];
291 const index = programIndex(words);
292 const program = words[index];
293 if (!program) return [];
294 const name = programName(program);
295 const interpreter = interpreterFor(name);
296 const args = words.slice(index + 1);
297 const scripts: Array<{ language: string; mapped: MappedText }> = [];
298
299 let readsStdinScript = false;
300 if (interpreter) {
301 const found = interpreterArgs(interpreter, args);
302 for (const script of found.scripts) {
303 const mapped = wordText(script.word, source);
304 if (!mapped) continue;
305 const start = script.inValue ? mapped.text.indexOf("=") + 1 : 0;
306 scripts.push({ language: interpreter.language, mapped: sliceMapped(mapped, start) });
307 }
308 readsStdinScript =
309 interpreter.stdin !== "never" && found.scripts.length === 0 && !found.scriptFromFile;
310 }
311
312 const stdinLanguage = readsStdinScript ? interpreter?.language : undefined;
313 const writtenLanguage = writtenFileLanguage(name, command, args);
314 for (const redirect of command.redirects) {
315 const toStdin = isDescriptor(redirect.descriptor, 0);
316 if (redirect.type === "HereString" && toStdin && stdinLanguage && redirect.target) {
317 const mapped = wordText(redirect.target, source);
318 if (mapped) scripts.push({ language: stdinLanguage, mapped });
319 }
320 if (redirect.type === "HereDoc") {
321 const language =
322 (toStdin ? (stdinLanguage ?? writtenLanguage) : undefined) ?? delimiterLanguage(redirect);
323 if (language) scripts.push({ language, mapped: hereDocText(redirect, source) });
324 }
325 }
326 return scripts;
327}
328
329function isDescriptor(descriptor: HereDoc["descriptor"], fd: number): boolean {
330 return (
331 descriptor === undefined || (descriptor.type === "FileDescriptor" && descriptor.value === fd)
332 );
333}
334
335interface ScriptArgument {
336 readonly word: Word;
337 readonly inValue: boolean;
338}
339
340function interpreterArgs(
341 interpreter: Interpreter,
342 args: ReadonlyArray<Word>,
343): { scripts: ScriptArgument[]; scriptFromFile: boolean } {
344 const positionals: Word[] = [];
345 const scripts: ScriptArgument[] = [];
346 const whole = (word: Word): ScriptArgument => ({ word, inValue: false });
347 let optionsEnded = false;
348 let scriptFromFile = false;
349 for (let index = 0; index < args.length; index += 1) {
350 const arg = args[index]!;
351 const flag = interpreter.ignoreCase ? arg.value.toLowerCase() : arg.value;
352 if (!optionsEnded && flag === "--") {
353 optionsEnded = true;
354 continue;
355 }
356 if (!optionsEnded && flag.startsWith("-") && flag !== "-") {
357 const [name = flag, value] = flag.startsWith("--") ? flag.split(/=(.*)/su) : [flag];
358 if (interpreter.scriptFileOptions?.has(name)) {
359 if (!interpreter.repeatable) return { scripts, scriptFromFile: true };
360 // SQL shells run `-f` files and `-c` statements alike, in order.
361 scriptFromFile = true;
362 if (value === undefined) index += 1;
363 continue;
364 }
365 if (interpreter.inline(name)) {
366 const next = args[index + 1];
367 if (value !== undefined) scripts.push({ word: arg, inValue: true });
368 else if (next) scripts.push(whole(next));
369 if (!interpreter.repeatable) return { scripts, scriptFromFile: false };
370 if (value === undefined) index += 1;
371 continue;
372 }
373 if (value === undefined && interpreter.valueOptions?.has(flag)) index += 1;
374 continue;
375 }
376 positionals.push(arg);
377 if (interpreter.stdin === "without-script-file" && !interpreter.subcommand) break;
378 }
379
380 if (interpreter.subcommand) {
381 if (positionals[0]?.value !== interpreter.subcommand) return { scripts, scriptFromFile: true };
382 return { scripts: positionals.slice(1, 2).map(whole), scriptFromFile: false };
383 }
384 if (interpreter.positionalScript !== undefined) {
385 scripts.push(...positionals.slice(interpreter.positionalScript).map(whole));
386 }
387 // SQL shells take databases as positional arguments, not script files.
388 if (interpreter.stdin === "always") return { scripts, scriptFromFile };
389 const first = positionals[0];
390 return { scripts, scriptFromFile: first !== undefined && first.value !== "-" };
391}
392
393function sliceMapped(mapped: MappedText, start: number): MappedText {
394 if (start === 0) return mapped;
395 return {
396 text: mapped.text.slice(start),
397 starts: mapped.starts.slice(start),
398 ends: mapped.ends.slice(start),
399 };
400}
401
402function writtenFileLanguage(
403 name: string,
404 command: Command,
405 args: ReadonlyArray<Word>,
406): string | undefined {
407 let path: string | undefined;
408 if (name === "cat") {
409 for (const redirect of command.redirects) {
410 if (
411 redirect.type === "Redirect" &&
412 [">", ">>", ">|", "&>", "&>>"].includes(redirect.operator) &&
413 isDescriptor(redirect.descriptor, 1)
414 ) {
415 path = redirect.target?.value;
416 }
417 }
418 } else if (name === "tee") {
419 path = args.find((arg) => !arg.value.startsWith("-"))?.value;
420 }
421 if (!path) return undefined;
422 const language = getFiletypeFromFileName(path);
423 return language === "text" ? undefined : language;
424}
425
426function delimiterLanguage(hereDoc: HereDoc): string | undefined {
427 const name = hereDoc.delimiter?.value.toLowerCase();
428 return name && Object.hasOwn(DELIMITER_LANGUAGES, name) ? DELIMITER_LANGUAGES[name] : undefined;
429}
430
431function hereDocText(hereDoc: HereDoc, source: string): MappedText {
432 const { pos, end } = hereDoc.body;
433 const starts = Array.from({ length: end - pos }, (_, index) => pos + index);
434 return {
435 text: source.slice(pos, end),
436 starts,
437 ends: starts.map((start) => start + 1),
438 };
439}
440
441// Preserve literal expansions; return undefined for unmappable spellings.
442function wordText(word: Word, source: string): MappedText | undefined {
443 const builder = new MappedTextBuilder();
444 const parts: ReadonlyArray<WordPart> = word.parts ?? [
445 { type: "Literal", pos: word.pos, end: word.end, text: word.text, value: word.value },
446 ];
447 for (const part of parts) {
448 switch (part.type) {
449 case "Literal":
450 if (!builder.addEscaped(part, false)) return undefined;
451 break;
452 case "SingleQuoted":
453 builder.addRaw(part.value, part.pos + 1);
454 break;
455 case "DoubleQuoted":
456 case "LocaleString":
457 for (const child of part.parts) {
458 if (!builder.addDoubleQuotedChild(child)) return undefined;
459 }
460 break;
461 case "AnsiCQuoted":
462 // $'…' escapes such as \x41 do not map character for character.
463 if (part.value !== part.text.slice(2, -1)) return undefined;
464 builder.addRaw(part.value, part.pos + 2);
465 break;
466 default:
467 builder.addRaw(source.slice(part.pos, part.end), part.pos);
468 }
469 }
470 return builder.build();
471}
472
473class MappedTextBuilder {
474 private text = "";
475 private readonly starts: number[] = [];
476 private readonly ends: number[] = [];
477
478 addRaw(text: string, pos: number) {
479 for (let index = 0; index < text.length; index += 1) this.push(text[index]!, pos + index, 1);
480 }
481
482 addDoubleQuotedChild(child: DoubleQuotedChild): boolean {
483 if (child.type === "Literal") return this.addEscaped(child, true);
484 this.addRaw(child.text, child.pos);
485 return true;
486 }
487
488 addEscaped(literal: { pos: number; text: string; value: string }, doubleQuoted: boolean) {
489 const before = this.text.length;
490 const { text, pos } = literal;
491 for (let index = 0; index < text.length; index += 1) {
492 const next = text[index + 1];
493 if (
494 text[index] === "\\" &&
495 next !== undefined &&
496 (!doubleQuoted || '$`"\\\n'.includes(next))
497 ) {
498 // A backslash-newline joins lines and spells nothing.
499 if (next !== "\n") this.push(next, pos + index, 2);
500 index += 1;
501 } else {
502 this.push(text[index]!, pos + index, 1);
503 }
504 }
505 return this.text.slice(before) === literal.value;
506 }
507
508 build(): MappedText {
509 return { text: this.text, starts: this.starts, ends: this.ends };
510 }
511
512 private push(character: string, start: number, length: number) {
513 this.text += character;
514 this.starts.push(start);
515 this.ends.push(start + length);
516 }
517}
518
519export function withEmbeddedScripts(
520 code: string,
521 lines: TokenLines,
522 embedded: ReadonlyArray<EmbeddedScript>,
523 tokenize: (text: string, language: string) => TokenLines,
524): TokenLines {
525 if (embedded.length === 0) return lines;
526 const styles = Array.from<SyntaxToken | undefined>({ length: code.length });
527 for (const token of lines.flat()) {
528 styles.fill(token, token.offset, token.offset + token.content.length);
529 }
530 for (const script of embedded) {
531 let scriptLines: TokenLines;
532 try {
533 scriptLines = tokenize(script.text, script.language);
534 } catch {
535 continue;
536 }
537 for (const token of scriptLines.flat()) {
538 if (token.content === "") continue;
539 const last = token.offset + token.content.length - 1;
540 styles.fill(token, script.starts[token.offset], script.ends[last]);
541 }
542 }
543 return restyledLines(code, styles);
544}
545
546function restyledLines(
547 code: string,
548 styles: ReadonlyArray<SyntaxToken | undefined>,
549): SyntaxToken[][] {
550 const lines: SyntaxToken[][] = [];
551 let start = 0;
552 for (const lineBreak of [...code.matchAll(/\r?\n/gu), undefined]) {
553 const end = lineBreak?.index ?? code.length;
554 const tokens: SyntaxToken[] = [];
555 for (let index = start; index < end;) {
556 const style = styles[index];
557 let next = index + 1;
558 while (
559 next < end &&
560 styles[next]?.color === style?.color &&
561 styles[next]?.fontStyle === style?.fontStyle
562 ) {
563 next += 1;
564 }
565 tokens.push({
566 content: code.slice(index, next),
567 offset: index,
568 ...(style?.color ? { color: style.color } : {}),
569 ...(style?.fontStyle ? { fontStyle: style.fontStyle } : {}),
570 });
571 index = next;
572 }
573 lines.push(tokens);
574 if (lineBreak) start = lineBreak.index + lineBreak[0].length;
575 }
576 return lines;
577}