1import type { Root, RootContent } from "mdast";
2import { unified, type Processor } from "unified";
3import remarkParse from "remark-parse";
4import remarkGfm from "remark-gfm";
5import { remarkKeepWindowsProsePaths } from "@t3tools/shared/markdownWindowsPaths";
6import { absoluteFilePathMatches, parseMarkdownFileLink } from "@t3tools/shared/markdownLinks";
7import { isWindowsAbsolutePath } from "@t3tools/shared/path";
8
9const parser = unified().use(remarkParse).use(remarkGfm).use(remarkKeepWindowsProsePaths);
10export type MarkdownNode = Root | RootContent;
11
12interface MarkdownEdit {
13 readonly start: number;
14 readonly end: number;
15 readonly content: string;
16}
17
18function applyMarkdownEdits(markdown: string, edits: MarkdownEdit[]): string {
19 if (edits.length === 0) return markdown;
20 const parts: string[] = [];
21 let offset = 0;
22 // AST traversal emits disjoint edits in source order.
23 for (const edit of edits) {
24 parts.push(markdown.slice(offset, edit.start), edit.content);
25 offset = edit.end;
26 }
27 parts.push(markdown.slice(offset));
28 return parts.join("");
29}
30
31function decodeCodePoint(codePoint: number, entity: string): string {
32 if (!Number.isInteger(codePoint) || codePoint < 0 || codePoint > 0x10ffff) {
33 return entity;
34 }
35 return String.fromCodePoint(codePoint);
36}
37
38export function decodeHtmlEntities(value: string): string {
39 return value.replace(
40 /&(?:#(\d+)|#x([0-9a-f]+)|amp|apos|gt|lt|nbsp|quot);/gi,
41 (entity, decimal: string | undefined, hexadecimal: string | undefined) => {
42 if (decimal) {
43 return decodeCodePoint(Number.parseInt(decimal, 10), entity);
44 }
45 if (hexadecimal) {
46 return decodeCodePoint(Number.parseInt(hexadecimal, 16), entity);
47 }
48 switch (entity.toLowerCase()) {
49 case "&":
50 return "&";
51 case "'":
52 return "'";
53 case ">":
54 return ">";
55 case "<":
56 return "<";
57 case " ":
58 return "\u00a0";
59 case """:
60 return '"';
61 default:
62 return entity;
63 }
64 },
65 );
66}
67
68const HTML_TAG = /<!--[\s\S]*?-->|<\/?[a-z](?:[^>"']|"[^"]*"|'[^']*')*>/gi;
69
70function attribute(tag: string, name: string): string | null {
71 const attributes = /([\w:-]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))/g;
72 for (const match of tag.matchAll(attributes)) {
73 if (match[1]?.toLowerCase() === name) {
74 return decodeHtmlEntities(match[2] ?? match[3] ?? match[4] ?? "");
75 }
76 }
77 return null;
78}
79
80function markdownDestination(url: string): string {
81 return `<${url.replace(/&/g, "\\&").replace(/</g, "%3C").replace(/>/g, "%3E").replace(/\|/g, "%7C").replace(/\n/g, "")}>`;
82}
83
84const ANCHOR_TAG = /^<\/?a[\s/>]/i;
85
86/** Replacements for the anchor tags still ahead, and how many emitted links are open. */
87type AnchorState = { readonly pending: string[]; open: number };
88
89// Code spans keep their text; an odd trailing backslash would escape the closing "]".
90function escapeLinkLabel(text: string): string {
91 return text
92 .replace(/(`+)[\s\S]*?\1|\\*[[\]]/g, (match) =>
93 match.startsWith("`") || match.length % 2 === 0
94 ? match
95 : `${match.slice(0, -1)}\\${match.slice(-1)}`,
96 )
97 .replace(/\\+$/, (slashes) => (slashes.length % 2 ? `${slashes}\\` : slashes));
98}
99
100// An unescaped "!" before the inserted "[" would turn the link into an image.
101function bangBefore(text: string, index: number): boolean {
102 if (text[index - 1] !== "!") return false;
103 let backslashes = 0;
104 while (text[index - 2 - backslashes] === "\\") backslashes += 1;
105 return backslashes % 2 === 0;
106}
107
108/**
109 * Pairs `<a>` tags in order. Only a closed anchor with an http(s) or mailto href becomes a
110 * Markdown link; any other anchor keeps its text unlinked.
111 */
112function anchorMarkdown(tags: readonly string[]): AnchorState {
113 const replacements = tags.map(() => "");
114 const open: Array<{ index: number; href: string | undefined }> = [];
115 tags.forEach((tag, index) => {
116 if (tag[1] !== "/") {
117 const href = attribute(tag, "href")?.trim();
118 open.push({ index, href: href && /^(?:https?|mailto):/i.test(href) ? href : undefined });
119 return;
120 }
121 const opening = open.pop();
122 if (!opening?.href) return;
123 replacements[opening.index] = "[";
124 replacements[index] = `](${markdownDestination(opening.href)})`;
125 });
126 return { pending: replacements, open: 0 };
127}
128
129export function markdownHtmlTags(content: string): RegExpExecArray[] {
130 // An HTML block such as <details> can also contain Markdown fences and code spans.
131 const codeRanges: Array<{ start: number; end: number }> = [];
132 const tags = [...content.matchAll(HTML_TAG)];
133 let tagIndex = 0;
134 const delimiters = [...content.matchAll(/`+|~{3,}/g)].filter((delimiter) => {
135 while (tags[tagIndex] && tags[tagIndex]!.index + tags[tagIndex]![0].length <= delimiter.index)
136 tagIndex += 1;
137 const tag = tags[tagIndex];
138 return !tag || delimiter.index < tag.index;
139 });
140 for (let index = 0; index < delimiters.length; index += 1) {
141 const opening = delimiters[index]!;
142 const lineStart = content.lastIndexOf("\n", opening.index - 1) + 1;
143 const fenced =
144 opening[0].length >= 3 && /^ {0,3}$/.test(content.slice(lineStart, opening.index));
145 if (opening[0][0] === "~" && !fenced) continue;
146 let closingIndex = -1;
147 for (let candidate = index + 1; candidate < delimiters.length; candidate += 1) {
148 const closing = delimiters[candidate]!;
149 if (!fenced && closing[0] === opening[0]) {
150 closingIndex = candidate;
151 break;
152 }
153 if (!fenced || closing[0][0] !== opening[0][0] || closing[0].length < opening[0].length)
154 continue;
155 const start = content.lastIndexOf("\n", closing.index - 1) + 1;
156 const nextLine = content.indexOf("\n", closing.index);
157 const end = nextLine === -1 ? content.length : nextLine;
158 if (
159 /^ {0,3}$/.test(content.slice(start, closing.index)) &&
160 /^\s*$/.test(content.slice(closing.index + closing[0].length, end))
161 ) {
162 closingIndex = candidate;
163 break;
164 }
165 }
166 if (closingIndex === -1) {
167 if (!fenced) continue;
168 // An unclosed fence runs to the end of the content, so no later opener matters.
169 codeRanges.push({ start: opening.index, end: content.length });
170 break;
171 }
172 const closing = delimiters[closingIndex]!;
173 codeRanges.push({ start: opening.index, end: closing.index + closing[0].length });
174 index = closingIndex;
175 }
176 return tags.filter(
177 (tag) => !codeRanges.some((range) => tag.index >= range.start && tag.index < range.end),
178 );
179}
180
181/**
182 * Called only for parsed HTML nodes, so Markdown code examples keep their literal tags.
183 * `anchors` is shared across sibling inline HTML nodes because each of those carries a single tag.
184 */
185export function markdownHtmlContent(
186 content: string,
187 options: { readonly anchors?: AnchorState; readonly inTableCell?: boolean } = {},
188): string {
189 const tags = markdownHtmlTags(content);
190 const anchors =
191 options.anchors ??
192 anchorMarkdown(tags.map((tag) => tag[0]).filter((tag) => ANCHOR_TAG.test(tag)));
193 let result = "";
194 let end = 0;
195 for (const match of tags) {
196 const text = content.slice(end, match.index);
197 result += anchors.open > 0 ? escapeLinkLabel(text) : text;
198 const replacement = htmlTagMarkdown(match[0], anchors, options);
199 if (replacement === "[" && bangBefore(result, result.length))
200 result = `${result.slice(0, -1)}\\!`;
201 result +=
202 anchors.open > 0 && /[\s[]/.test(result.at(-1) ?? "")
203 ? replacement.replace(/^ /, "")
204 : replacement;
205 end = match.index + match[0].length;
206 }
207 const text = content.slice(end);
208 return result + (anchors.open > 0 ? escapeLinkLabel(text) : text);
209}
210
211function htmlTagMarkdown(
212 tag: string,
213 anchors: AnchorState,
214 options: { readonly inTableCell?: boolean },
215): string {
216 if (tag.startsWith("<!--")) return "";
217 if (ANCHOR_TAG.test(tag)) {
218 const replacement = anchors.pending.shift() ?? "";
219 if (replacement === "[") anchors.open += 1;
220 else if (replacement) anchors.open -= 1;
221 return replacement;
222 }
223 if (/^<img\b/i.test(tag)) {
224 const src = attribute(tag, "src");
225 if (!src) return "";
226 const alt = (attribute(tag, "alt") ?? "").replace(/[\\[\]&]/g, "\\$&");
227 return `})`;
228 }
229 // A raw newline would end the table row; both parsers decode this reference to one inside the cell.
230 const cellBreak = " ";
231 // A blank line inside an open link label would end the paragraph before its "]".
232 const blockBreak = anchors.open > 0 ? " " : options.inTableCell ? cellBreak : "\n\n";
233 if (/^<summary\b/i.test(tag)) return `${blockBreak}**`;
234 if (/^<\/summary\s*>/i.test(tag)) return `**${blockBreak}`;
235 if (/^<br\b/i.test(tag)) return options.inTableCell ? cellBreak : " \n";
236 if (/^<\/?(?:p|div|details)\b/i.test(tag)) return blockBreak;
237 return "";
238}
239
240export function markdownWithHtml(markdown: string): string {
241 if (!/<(?:!--|\/?[a-z])/i.test(markdown)) return markdown;
242 const edits: Array<{ start: number; end: number; content: string }> = [];
243 const visit = (node: MarkdownNode, inTableCell: boolean, inLink: boolean) => {
244 if (!("children" in node)) return;
245 inTableCell ||= node.type === "tableCell";
246 const children: readonly MarkdownNode[] = node.children;
247 // Inline HTML arrives one tag per node, so anchors pair across phrasing siblings.
248 const anchors =
249 node.type === "root" || node.type === "blockquote" || node.type === "listItem"
250 ? undefined
251 : anchorMarkdown(
252 children.flatMap((child) =>
253 child.type === "html"
254 ? markdownHtmlTags(child.value)
255 .map((tag) => tag[0])
256 .filter((tag) => ANCHOR_TAG.test(tag))
257 : [],
258 ),
259 );
260 for (const child of children) {
261 const start = child.position?.start.offset;
262 const end = child.position?.end.offset;
263 const linkLabel = inLink || (anchors?.open ?? 0) > 0;
264 if (child.type === "text") {
265 if (linkLabel && start !== undefined && end !== undefined) {
266 const content = escapeLinkLabel(markdown.slice(start, end));
267 if (content !== markdown.slice(start, end)) edits.push({ start, end, content });
268 }
269 continue;
270 }
271 if (child.type !== "html") {
272 visit(child, inTableCell, linkLabel);
273 continue;
274 }
275 const content = markdownHtmlContent(child.value, { anchors, inTableCell });
276 if (start === undefined || end === undefined) continue;
277 // Text before an outermost link is never edited, so this edit cannot overlap another.
278 if (content.startsWith("[") && !linkLabel && bangBefore(markdown, start))
279 edits.push({ start: start - 1, end: start, content: "\\!" });
280 edits.push({ start, end, content });
281 }
282 };
283 visit(parser.parse(markdown), false, false);
284 return applyMarkdownEdits(markdown, edits);
285}
286
287// Start only at the beginning of a backtick run to avoid retrying every suffix.
288const INLINE_CODE_SPAN = /(?<!`)`+([^`\r\n]+)`+/g;
289
290export function markdownWithInlineFileLinks(markdown: string, fileLink: RegExp): string {
291 return transformNativeMarkdown(markdown, fileLink, false);
292}
293
294function transformNativeMarkdown(
295 markdown: string,
296 fileLink: RegExp | undefined,
297 tableBreaks: boolean,
298): string {
299 const breakTables = tableBreaks && mayHaveTableUnderText(markdown);
300 let linkFiles = false;
301 if (fileLink) {
302 for (const [, code] of markdown.matchAll(INLINE_CODE_SPAN)) {
303 if (fileLink.test(code!.trim())) {
304 linkFiles = true;
305 break;
306 }
307 }
308 }
309 const linkProseFiles = fileLink !== undefined && absoluteFilePathMatches(markdown).length > 0;
310 if (!linkFiles && !linkProseFiles && !breakTables) return markdown;
311 const edits: MarkdownEdit[] = [];
312 const fileEdits: MarkdownEdit[] = [];
313 let rewritesPipe = false;
314 const visit = (node: MarkdownNode) => {
315 if (linkFiles && node.type === "inlineCode") {
316 const start = node.position?.start.offset;
317 const end = node.position?.end.offset;
318 const escaped = node.value.replace(/[\\[\]&]/g, "\\$&");
319 if (
320 start !== undefined &&
321 end !== undefined &&
322 fileLink?.test(node.value) &&
323 parseMarkdownFileLink(node.value) !== null &&
324 !/[<>]/.test(node.value)
325 ) {
326 const edit = {
327 start,
328 end,
329 content: `[${escaped}](<${escaped}>)`,
330 };
331 edits.push(edit);
332 fileEdits.push(edit);
333 if (node.value.includes("|")) rewritesPipe = true;
334 }
335 } else if (
336 linkProseFiles &&
337 node.type === "text" &&
338 node.position?.start.offset !== undefined
339 ) {
340 const start = node.position.start.offset;
341 const raw = markdown.slice(start, node.position.end.offset);
342 for (const { index, path } of absoluteFilePathMatches(raw)) {
343 // Source offsets stay valid for paths whose own text needs no Markdown decoding.
344 if (raw !== node.value && /[\\&]/.test(path)) continue;
345 const escaped = path.replace(/[\\[\]&]/g, "\\$&");
346 const edit = {
347 start: start + index,
348 end: start + index + path.length,
349 content: `[${escaped}](<${escaped}>)`,
350 };
351 edits.push(edit);
352 fileEdits.push(edit);
353 }
354 } else if (node.type !== "link" && node.type !== "linkReference" && "children" in node)
355 node.children.forEach((child, index) => {
356 const previous = node.children[index - 1];
357 const start = child.position?.start;
358 if (
359 breakTables &&
360 child.type === "table" &&
361 previous?.type === "paragraph" &&
362 start?.offset !== undefined &&
363 previous.position?.end.line === start.line - 1
364 ) {
365 let lineStart = start.offset;
366 while (lineStart > 0 && !/[\r\n]/.test(markdown[lineStart - 1]!)) lineStart -= 1;
367 // Replace only the container prefix, leaving cell edits disjoint from the table break.
368 const [, header = "", , ending = "\n", markers = ""] =
369 /^([ \t>]*)([^\r\n]*)(\r\n|\r|\n)([ \t>]*)/.exec(markdown.slice(lineStart)) ?? [];
370 edits.push({
371 start: lineStart,
372 end: lineStart + header.length,
373 content: `${markers.trimEnd()}${ending}${markers}`,
374 });
375 }
376 visit(child);
377 });
378 };
379 visit(parser.parse(markdown));
380 // Rewritten pipes can change table structure, so preserve the original pass ordering.
381 if (breakTables && rewritesPipe)
382 return markdownWithTableBreaks(applyMarkdownEdits(markdown, fileEdits));
383 return applyMarkdownEdits(markdown, edits);
384}
385
386// A delimiter row such as `|---|:--:|`, possibly inside a blockquote or list item.
387function isTableDelimiterRow(line: string): boolean {
388 if (!line.includes("-")) return false;
389 const row = line.replace(/^[ \t>]+/, "").trim();
390 return row
391 .replace(/^\||\|$/g, "")
392 .split("|")
393 .every((cell) => /^:?-+:?$/.test(cell.trim()));
394}
395
396function mayHaveTableUnderText(markdown: string): boolean {
397 if (!markdown.includes("|")) return false;
398 const lines = markdown.split(/\r\n|\r|\n/);
399 for (let index = 2; index < lines.length; index += 1) {
400 if (isTableDelimiterRow(lines[index]!) && /[^\s>]/.test(lines[index - 2]!)) return true;
401 }
402 return false;
403}
404
405/** md4c starts tables only at paragraph starts; insert breaks to match GFM. */
406export function markdownWithTableBreaks(markdown: string): string {
407 return transformNativeMarkdown(markdown, undefined, true);
408}
409
410// `>` covers a destination continued on a blockquote line.
411const WINDOWS_DESTINATION = /\](?:\(|:)[\s>]*<?(?:[A-Za-z]:[\\/]|\\\\)/;
412
413interface DestinationToken {
414 readonly start: { readonly offset: number };
415 readonly end: { readonly offset: number };
416}
417
418/**
419 * Markdown reads `C:\work\.t3` as `C:\work.t3`. Web keeps authored Windows destinations
420 * literally (see `remarkKeepWindowsPaths` in shared), so escape their backslashes to give
421 * both parsers here the same path. `\&` stays an escape, as `markdownWithHtml` emits it.
422 * Expects authored Markdown: already escaped output would be escaped twice.
423 */
424function markdownWithWindowsPathDestinations(markdown: string): string {
425 // Skips the extra parse for the common message that has no Windows destination.
426 if (!WINDOWS_DESTINATION.test(markdown)) return markdown;
427 const edits: MarkdownEdit[] = [];
428 function record(
429 this: { resume(): string; sliceSerialize(token: DestinationToken): string },
430 token: DestinationToken,
431 ) {
432 this.resume();
433 const authored = this.sliceSerialize(token);
434 if (!isWindowsAbsolutePath(authored) || !authored.includes("\\") || /[<>]/.test(authored))
435 return;
436 const start = token.start.offset;
437 const end = token.end.offset;
438 const escaped = authored.replace(/\\(?!&)/g, "\\\\");
439 // A bare destination ends at an unescaped `)`, so wrap it to keep `\)` inside.
440 const angled = markdown[start - 1] === "<";
441 edits.push({ start, end, content: angled ? escaped : `<${escaped}>` });
442 }
443 unified()
444 .use(remarkParse)
445 .use(remarkGfm)
446 .use(function (this: Processor) {
447 (this.data().fromMarkdownExtensions ??= []).push({
448 exit: { resourceDestinationString: record, definitionDestinationString: record },
449 });
450 })
451 .parse(markdown);
452 return applyMarkdownEdits(markdown, edits);
453}
454
455// HTML wrappers can expose Markdown that wasn't in the first AST, so normalize them first.
456export function markdownForNativeParsing(markdown: string): string {
457 return markdownWithWindowsPathDestinations(markdownWithHtml(markdown));
458}
459
460export function parseMarkdownDocument(markdown: string): Root {
461 return parser.parse(markdownForNativeParsing(markdown));
462}
463
464export function markdownForNativeRendering(markdown: string, fileLink: RegExp): string {
465 return transformNativeMarkdown(markdownForNativeParsing(markdown), fileLink, true);
466}
467
468/** Native media slots only cover standalone images; nested file embeds must become links. */
469export function markdownWithFileEmbeds(
470 markdown: string,
471 label: (href: string, standalone: boolean) => string | null,
472): { markdown: string; fileHrefs: ReadonlySet<string> } {
473 const fileHrefs = new Set<string>();
474 if (!markdown.includes("![")) return { markdown, fileHrefs };
475 const root = parser.parse(markdown);
476 const topLevelBlocks = new Set<MarkdownNode>(root.children);
477 const definitions = new Map<string, string>();
478 const collect = (node: MarkdownNode) => {
479 if (node.type === "definition") {
480 const identifier = node.identifier.toUpperCase();
481 if (!definitions.has(identifier)) definitions.set(identifier, node.url);
482 }
483 if ("children" in node) node.children.forEach(collect);
484 };
485 collect(root);
486 const edits: Array<{ start: number; end: number; content: string }> = [];
487 const visit = (node: MarkdownNode, standalone = false, insideLink = false) => {
488 if (node.type === "image" || node.type === "imageReference") {
489 const href =
490 node.type === "image" ? node.url : definitions.get(node.identifier.toUpperCase());
491 const name = href === undefined ? null : label(href, standalone);
492 const start = node.position?.start.offset;
493 const end = node.position?.end.offset;
494 if (href !== undefined && name !== null && start !== undefined && end !== undefined) {
495 const text = name.replace(/[\\[\]`*_~|&<>]/g, "\\$&");
496 const url = href.replace(/[\\&]/g, "\\$&").replace(/</g, "%3C").replace(/>/g, "%3E");
497 edits.push({ start, end, content: insideLink ? text : `[${text}](<${url}>)` });
498 fileHrefs.add(href);
499 }
500 }
501 if ("children" in node)
502 node.children.forEach((child) =>
503 visit(
504 child,
505 node.type === "paragraph" && node.children.length === 1 && topLevelBlocks.has(node),
506 insideLink || node.type === "link" || node.type === "linkReference",
507 ),
508 );
509 };
510 visit(root);
511 const parts: string[] = [];
512 let offset = 0;
513 for (const edit of edits) {
514 parts.push(markdown.slice(offset, edit.start), edit.content);
515 offset = edit.end;
516 }
517 parts.push(markdown.slice(offset));
518 return { markdown: parts.join(""), fileHrefs };
519}