Open History to see the last commit
1import { isLocalEditorFileUrl, LOCAL_FILE_LINK_SCHEMES } from "@t3tools/contracts";
2import type {
3 Heading,
4 Link,
5 Paragraph,
6 PhrasingContent,
7 Root,
8 RootContent,
9 TableCell,
10 Text,
11} from "mdast";
12import { remarkKeepWindowsPaths } from "./markdownWindowsPaths.ts";
13import { type PluggableList, type Processor, type Transformer, unified } from "unified";
14import remarkParse from "remark-parse";
15import remarkRehype from "remark-rehype";
16import { find, html } from "property-information";
17import rehypeRaw from "rehype-raw";
18import rehypeSanitize, { defaultSchema } from "rehype-sanitize";
19import remarkBreaks from "remark-breaks";
20import remarkGfm from "remark-gfm";
21import { remarkGithubAlerts } from "./markdownGithubAlerts.ts";
22import { remarkNormalizeListItemIndentation } from "./markdownListIndentation.ts";
23import { remarkKeepOrderedListNumbers } from "./markdownOrderedListNumbers.ts";
24import {
25 CODEX_ARTIFACT_TEMPLATE_HAST_PROPERTIES,
26 remarkCodexDirectives,
27} from "./codexMarkdownDirectives.ts";
28import {
29 absoluteFilePathMatches,
30 isWindowsDrivePathHref,
31 normalizeMarkdownLinkDestination,
32} from "./markdownLinks.ts";
33import { THREAD_LINK_PROTOCOL } from "./threadLinks.ts";
34
35export { remarkEnableOrderedListNumbers } from "./markdownOrderedListNumbers.ts";
36
37type MarkdownImageHastNode = {
38 type?: string;
39 value?: string;
40 tagName?: string;
41 properties?: Record<string, unknown>;
42 children?: MarkdownImageHastNode[];
43};
44
45function meaningfulHastChildren(node: MarkdownImageHastNode): MarkdownImageHastNode[] {
46 return (node.children ?? []).filter(
47 (child) => !(child.type === "text" && (child as { value?: string }).value?.trim() === ""),
48 );
49}
50
51const STANDALONE_IMAGE_BLOCKS = new Set([
52 "p",
53 "div",
54 "li",
55 "td",
56 "th",
57 "figure",
58 "center",
59 "blockquote",
60]);
61
62function soleImageDescendant(node: MarkdownImageHastNode): MarkdownImageHastNode | undefined {
63 const children = meaningfulHastChildren(node);
64 if (children.length !== 1) return undefined;
65 const only = children[0];
66 if (only?.type !== "element") return undefined;
67 if (only.tagName === "img") return only;
68 return only.tagName === "a" || only.tagName === "strong" || only.tagName === "em"
69 ? soleImageDescendant(only)
70 : undefined;
71}
72
73function markStandaloneImages(node: MarkdownImageHastNode) {
74 if (node.type === "root" || (node.tagName && STANDALONE_IMAGE_BLOCKS.has(node.tagName))) {
75 const image = soleImageDescendant(node);
76 if (image) image.properties = { ...image.properties, dataStandalone: true };
77 }
78 node.children?.forEach((child) => {
79 if (child.type === "element") markStandaloneImages(child);
80 });
81}
82
83// Whole tags, attributes, and comments, so closing-anchor text inside them never counts as a tag.
84const RAW_HTML_TAG_PATTERN = /<!--[\s\S]*?(?:-->|$)|<\/?[A-Za-z](?:[^"'<>]|"[^"]*"|'[^']*')*>/g;
85
86function rehypePreserveBareAnchorPlaceholders() {
87 return (tree: MarkdownImageHastNode) => {
88 const anchors: Array<MarkdownImageHastNode | null> = [];
89 let rawTextTag: string | undefined;
90 const visit = (node: MarkdownImageHastNode) => {
91 if (node.type === "raw" && typeof node.value === "string") {
92 const tags = new RegExp(RAW_HTML_TAG_PATTERN);
93 let offset = 0;
94 while (rawTextTag !== "plaintext") {
95 // Raw text ends at its closing tag even inside comment-looking text.
96 const matcher = rawTextTag ? new RegExp(`</${rawTextTag}\\s*>`, "gi") : tags;
97 matcher.lastIndex = offset;
98 const match = matcher.exec(node.value);
99 if (!match) break;
100 const [tag] = match;
101 offset = matcher.lastIndex;
102 if (rawTextTag) {
103 rawTextTag = undefined;
104 continue;
105 }
106 if (tag.startsWith("<!--")) continue;
107 const closing = /^<\/([a-z]+)\s*>$/i.exec(tag)?.[1]?.toLowerCase();
108 const opening = /^<([a-z]+)(?:\s|\/?>)/i.exec(tag)?.[1]?.toLowerCase();
109 if (
110 opening &&
111 /^(?:script|style|textarea|title|xmp|iframe|noembed|noframes|plaintext)$/.test(opening)
112 ) {
113 rawTextTag = opening;
114 } else if (opening === "a") {
115 anchors.push(node.value === tag && /^<a\s*\/?>$/i.test(tag) ? node : null);
116 } else if (closing === "a") {
117 anchors.pop();
118 }
119 }
120 }
121 node.children?.forEach(visit);
122 };
123
124 visit(tree);
125 for (const anchor of anchors) {
126 if (anchor) anchor.type = "text";
127 }
128 };
129}
130
131const COLOR_SCHEME_MEDIA = /^\(\s*prefers-color-scheme\s*:\s*(dark|light)\s*\)$/i;
132
133/**
134 * The renderer resolves an `<img>`'s `src` (workspace files, host media) but never a `<source>`'s
135 * `srcset`, which would load unresolved whenever its media matched. A color-scheme source moves
136 * onto the `<img>` for the renderer to pick by theme; every source is then dropped.
137 */
138function moveColorSchemeSourcesToImage(picture: MarkdownImageHastNode) {
139 if (!picture.children) return;
140 const image = picture.children.find((child) => child.tagName === "img");
141 for (const source of picture.children) {
142 if (source.tagName !== "source" || !image) continue;
143 const media = source.properties?.media;
144 const scheme = typeof media === "string" ? COLOR_SCHEME_MEDIA.exec(media)?.[1] : undefined;
145 const srcSet = source.properties?.srcSet;
146 const candidate = Array.isArray(srcSet) ? srcSet[0] : srcSet;
147 // A candidate's URL runs to the first space (a descriptor such as `2x` may follow).
148 const url =
149 typeof candidate === "string" ? candidate.trim().split(/\s/, 1)[0]?.replace(/,+$/, "") : "";
150 const key = scheme?.toLowerCase() === "dark" ? "dataDarkSrc" : "dataLightSrc";
151 // As in a browser, the first source that matches a scheme is the one shown.
152 if (scheme && url && image.properties?.[key] === undefined) {
153 // The sanitizer reads a drive letter as a protocol; as a file URL it resolves to the same path.
154 const value = isWindowsDrivePathHref(url) ? `file:///${url.replaceAll("\\", "/")}` : url;
155 image.properties = { ...image.properties, [key]: value };
156 }
157 }
158 picture.children = picture.children.filter((child) => child.tagName !== "source");
159}
160
161function rehypePreserveImageSourceMeta() {
162 return (tree: MarkdownImageHastNode) => {
163 const visit = (node: MarkdownImageHastNode) => {
164 const src = node.properties?.src;
165 const title = node.properties?.title;
166 if (node.type === "element" && node.tagName === "picture") {
167 moveColorSchemeSourcesToImage(node);
168 }
169 if (node.type === "element" && node.tagName === "img") {
170 node.properties = {
171 ...node.properties,
172 ...(typeof src === "string" && isWindowsDrivePathHref(src) ? { dataLocalSrc: src } : {}),
173 ...(typeof title === "string" ? { dataMarkdownTitle: title } : {}),
174 };
175 }
176 node.children?.forEach(visit);
177 };
178
179 visit(tree);
180 markStandaloneImages(tree);
181 };
182}
183
184const IMAGE_SRC_PROTOCOLS = [...(defaultSchema.protocols?.src ?? []), "file", "t3-context"];
185
186// The sanitizer compares schemes case-sensitively, so lowercase accepted editor links first.
187function rehypeNormalizeLocalEditorLinks() {
188 return (tree: MarkdownImageHastNode) => {
189 const visit = (node: MarkdownImageHastNode) => {
190 const href = node.properties?.href;
191 if (node.type === "element" && node.tagName === "a" && typeof href === "string") {
192 try {
193 const url = new URL(href);
194 if (isLocalEditorFileUrl(url)) node.properties = { ...node.properties, href: url.href };
195 } catch {
196 // Not an absolute URL; the sanitizer decides.
197 }
198 }
199 node.children?.forEach(visit);
200 };
201
202 visit(tree);
203 };
204}
205
206const CHAT_MARKDOWN_SANITIZE_SCHEMA = {
207 ...defaultSchema,
208 attributes: {
209 ...defaultSchema.attributes,
210 "*": (defaultSchema.attributes?.["*"] ?? []).filter((attribute) => attribute !== "title"),
211 // Fence languages plus the two math classes the web chat's math plugin emits.
212 code: [
213 ["className", /^language-./, "math-inline", "math-display"],
214 "dataCodeMeta",
215 "dataInlineCode",
216 ],
217 blockquote: [...(defaultSchema.attributes?.blockquote ?? []), "dataAlert"],
218 div: [...(defaultSchema.attributes?.div ?? []), ...CODEX_ARTIFACT_TEMPLATE_HAST_PROPERTIES],
219 a: [...(defaultSchema.attributes?.a ?? []), "dataPullRequestAutolink"],
220 img: [
221 ...(defaultSchema.attributes?.img ?? []),
222 "dataDarkSrc",
223 "dataLightSrc",
224 "dataLocalSrc",
225 "dataMarkdownTitle",
226 "dataStandalone",
227 ],
228 },
229 protocols: {
230 ...defaultSchema.protocols,
231 href: [
232 ...(defaultSchema.protocols?.href ?? []),
233 "file",
234 "t3-citation",
235 "t3-context",
236 THREAD_LINK_PROTOCOL,
237 // Scheme only: the renderer's URL transform checks the rest of the link shape.
238 ...LOCAL_FILE_LINK_SCHEMES,
239 ],
240 src: IMAGE_SRC_PROTOCOLS,
241 dataDarkSrc: IMAGE_SRC_PROTOCOLS,
242 dataLightSrc: IMAGE_SRC_PROTOCOLS,
243 },
244} satisfies Parameters<typeof rehypeSanitize>[0];
245
246export const CHAT_MARKDOWN_REMARK_PLUGINS: PluggableList = [
247 remarkGfm,
248 remarkEndAutolinksAtCjkPunctuation,
249 remarkKeepAutolinkSuffixes,
250 remarkLinkAbsoluteFilePaths,
251 remarkKeepWindowsPaths,
252 remarkGithubAlerts,
253 remarkKeepOrderedListNumbers,
254 remarkNormalizeListItemIndentation,
255 remarkCodexDirectives,
256 remarkBreakAroundImageLines,
257 remarkPreserveCodeMeta,
258 remarkNormalizeLinksAndTagInlineCode,
259];
260
261export const CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS: PluggableList = [
262 remarkGfm,
263 remarkEndAutolinksAtCjkPunctuation,
264 remarkKeepAutolinkSuffixes,
265 remarkLinkAbsoluteFilePaths,
266 remarkKeepWindowsPaths,
267 remarkGithubAlerts,
268 remarkKeepOrderedListNumbers,
269 remarkNormalizeListItemIndentation,
270 remarkCodexDirectives,
271 remarkBreakAroundImageLines,
272 remarkBreaks,
273 remarkPreserveCodeMeta,
274 remarkNormalizeLinksAndTagInlineCode,
275];
276
277// Users type paths like ~/a and (~/b, which GFM's single-tilde strikethrough hides as <del>.
278// Appended after the base list, this reconfigures its remark-gfm entry; ~~double~~ still strikes.
279export const CHAT_USER_MARKDOWN_REMARK_PLUGINS: PluggableList = [
280 [remarkGfm, { singleTilde: false }],
281];
282
283// Start tags an HTML parser closes an open `<p>` for.
284const CLOSES_PARAGRAPH = new Set(
285 "address article aside blockquote center details dialog dir div dl fieldset figcaption figure footer form h1 h2 h3 h4 h5 h6 header hgroup hr li listing main menu nav ol p plaintext pre search section summary table ul xmp dd dt".split(
286 " ",
287 ),
288);
289const HEADING = /^h[1-6]$/;
290
291interface HtmlScope {
292 readonly anchor: boolean;
293 readonly paragraph: boolean;
294 readonly heading: boolean;
295}
296
297/**
298 * Whether the tree needs the HTML parser: raw HTML, or nesting it repairs (a footnote call in a
299 * link label nests `<a>`). Otherwise it only rewrites properties as hast-util-to-parse5 does and
300 * drops `data`, both repeated here, and moves blank text that renders the same.
301 */
302function needsHtmlParser(node: MarkdownImageHastNode, scope: HtmlScope): boolean {
303 if (node.type === "raw") return true;
304 let inner = scope;
305 if (node.type === "element" && node.tagName) {
306 const { tagName } = node;
307 const heading = HEADING.test(tagName);
308 if (
309 (tagName === "a" && scope.anchor) ||
310 (scope.paragraph && CLOSES_PARAGRAPH.has(tagName)) ||
311 (heading && scope.heading)
312 ) {
313 return true;
314 }
315 inner = {
316 anchor: scope.anchor || tagName === "a",
317 paragraph: scope.paragraph || tagName === "p",
318 heading: scope.heading || heading,
319 };
320 const properties = node.properties ?? {};
321 for (const key of Object.keys(properties)) {
322 const value = properties[key];
323 const info = find(html, key);
324 if (
325 value === false ||
326 value === null ||
327 value === undefined ||
328 Number.isNaN(value) ||
329 (!value && info.boolean)
330 ) {
331 delete properties[key];
332 } else if (value === true && !info.boolean && !info.overloadedBoolean) {
333 properties[key] = "";
334 }
335 }
336 delete (node as { data?: unknown }).data;
337 }
338 return (node.children ?? []).some((child) => needsHtmlParser(child, inner));
339}
340
341const TOP_SCOPE: HtmlScope = { anchor: false, paragraph: false, heading: false };
342
343/** rehype-raw reparses the whole tree through an HTML parser, which most trees can skip. */
344function rehypeRawWhenPresent(this: Processor) {
345 const transform = rehypeRaw.call(this) as unknown as (
346 tree: MarkdownImageHastNode,
347 file: unknown,
348 ) => unknown;
349 return (tree: MarkdownImageHastNode, file: unknown) =>
350 needsHtmlParser(tree, TOP_SCOPE) ? transform(tree, file) : undefined;
351}
352
353export const CHAT_MARKDOWN_REHYPE_PLUGINS: PluggableList = [
354 rehypePreserveBareAnchorPlaceholders,
355 rehypeRawWhenPresent,
356 rehypePreserveImageSourceMeta,
357 rehypeNormalizeLocalEditorLinks,
358 [rehypeSanitize, CHAT_MARKDOWN_SANITIZE_SCHEMA],
359];
360
361const isImageNode = (node: PhrasingContent | undefined): boolean =>
362 node?.type === "image" ||
363 node?.type === "imageReference" ||
364 ((node?.type === "link" || node?.type === "linkReference") &&
365 node.children.length === 1 &&
366 isImageNode(node.children[0]));
367const endsLine = (node: PhrasingContent | undefined) =>
368 node === undefined ||
369 node.type === "break" ||
370 (node.type === "text" && /\n[ \t]*$/.test(node.value));
371const startsLine = (node: PhrasingContent | undefined) =>
372 node === undefined ||
373 node.type === "break" ||
374 (node.type === "text" && /^[ \t]*\n/.test(node.value));
375
376function breakLinesAroundImages(paragraph: Paragraph) {
377 const children = paragraph.children;
378 if (!children.some(isImageNode)) return;
379 const ownsLine = (index: number) =>
380 isImageNode(children[index]) &&
381 endsLine(children[index - 1]) &&
382 startsLine(children[index + 1]);
383 paragraph.children = children.flatMap((child, index): PhrasingContent[] => {
384 if (child.type !== "text") return [child];
385 const afterImage = ownsLine(index - 1);
386 const beforeImage = ownsLine(index + 1);
387 if (!afterImage && !beforeImage) return [child];
388 const parts: PhrasingContent[] = [];
389 let value = child.value;
390 if (afterImage) {
391 value = value.replace(/^[ \t]*\n/, "");
392 parts.push({ type: "break" });
393 }
394 const breakAfter = beforeImage && /\n[ \t]*$/.test(value);
395 if (breakAfter) value = value.replace(/[ \t]*\n[ \t]*$/, "");
396 if (value) parts.push({ type: "text", value });
397 if (breakAfter) parts.push({ type: "break" });
398 return parts;
399 });
400}
401
402/**
403 * Agents label screenshots with a line like `**Palette**` right above the image. CommonMark
404 * joins those lines into one paragraph, which puts the image inline beside its label. An
405 * image alone on its source line keeps that line; prose and inline badges still wrap.
406 */
407function remarkBreakAroundImageLines() {
408 return (tree: Root) => {
409 const visit = (node: Root | RootContent) => {
410 if (node.type === "paragraph") breakLinesAroundImages(node);
411 else if ("children" in node) node.children.forEach(visit);
412 };
413 visit(tree);
414 };
415}
416
417type MarkdownAstNode = {
418 type?: string;
419 meta?: unknown;
420 url?: string;
421 data?: {
422 hProperties?: Record<string, unknown>;
423 };
424 children?: MarkdownAstNode[];
425};
426
427function remarkKeepAutolinkSuffixes(): Transformer<Root> {
428 return (tree, file) => {
429 const source = String(file);
430 const visit = (node: Root | RootContent) => {
431 if (!("children" in node) || node.type === "link" || node.type === "linkReference") return;
432 for (let index = 0; index < node.children.length; index++) {
433 const child = node.children[index];
434 const next = node.children[index + 1];
435 if (child?.type === "link" && /^https?:\/\//i.test(child.url) && next?.type === "text") {
436 const suffix = /^[_~]+/.exec(next.value)?.[0];
437 if (!suffix) continue;
438 const label = child.children.length === 1 ? child.children[0] : undefined;
439 if (label?.type !== "text") continue;
440 if (label.value !== child.url && `http://${label.value}` !== child.url) continue;
441 const start = child.position?.start.offset;
442 const end = child.position?.end.offset;
443 const bare =
444 !child.position ||
445 (start !== undefined &&
446 end !== undefined &&
447 source.slice(start, end) === label.value &&
448 source.slice(end, end + suffix.length) === suffix);
449 if (bare) {
450 child.url += suffix;
451 label.value += suffix;
452 next.value = next.value.slice(suffix.length);
453 if (child.position && end !== undefined) {
454 const boundary = {
455 ...child.position.end,
456 column: child.position.end.column + suffix.length,
457 offset: end + suffix.length,
458 };
459 child.position.end = boundary;
460 if (label.position) label.position.end = boundary;
461 if (next.position) next.position.start = boundary;
462 }
463 if (!next.value) node.children.splice(index + 1, 1);
464 }
465 }
466 if (child) visit(child);
467 }
468 };
469 visit(tree);
470 };
471}
472
473// Punctuation and symbols from CJK Symbols and Punctuation and from Halfwidth and Fullwidth Forms.
474const CJK_PUNCTUATION = /(?=[ -〿-])[\p{P}\p{S}]/u;
475
476/**
477 * GFM runs a bare URL on until whitespace, so Chinese and Japanese punctuation joins it, along with
478 * the rest of the sentence and any `**` that should have closed around it. A URL that is ASCII up to
479 * such a mark ends there. Unicode earlier in the URL means the mark may belong to it, so it stays.
480 */
481function cjkPunctuationIndex(link: Link): number {
482 const label = link.children.length === 1 ? link.children[0] : undefined;
483 if (label?.type !== "text" || !/^https?:\/\//i.test(link.url)) return -1;
484 if (link.url !== label.value && link.url !== `http://${label.value}`) return -1;
485 const index = label.value.search(/[^!-~]/u);
486 return index > 0 && CJK_PUNCTUATION.test(label.value.charAt(index)) ? index : -1;
487}
488
489/** GFM's text pass links bare URLs without positions; its tokenizer's links span their URL. */
490function isBareLink(link: Link, source: string): boolean {
491 const start = link.position?.start.offset;
492 const end = link.position?.end.offset;
493 const label = link.children[0];
494 return (
495 !link.position ||
496 (start !== undefined &&
497 end !== undefined &&
498 label?.type === "text" &&
499 source.slice(start, end) === label.value)
500 );
501}
502
503/** Source offsets of the marks that tokenized bare URLs run past. */
504function cjkPunctuationOffsets(node: RootContent, source: string): number[] {
505 if (node.type === "link") {
506 const index = cjkPunctuationIndex(node);
507 const start = node.position?.start.offset;
508 return index > 0 && start !== undefined && isBareLink(node, source) ? [start + index] : [];
509 }
510 return "children" in node && node.type !== "linkReference"
511 ? (node.children as RootContent[]).flatMap((child) => cjkPunctuationOffsets(child, source))
512 : [];
513}
514
515const hasReference = (node: RootContent): boolean =>
516 node.type === "linkReference" ||
517 node.type === "imageReference" ||
518 node.type === "footnoteReference" ||
519 ("children" in node && (node.children as RootContent[]).some(hasReference));
520
521/** How many of the ascending numbers are below `value`. */
522function countBelow(sorted: readonly number[], value: number): number {
523 let low = 0;
524 let high = sorted.length;
525 while (low < high) {
526 const middle = (low + high) >> 1;
527 if (sorted[middle]! < value) low = middle + 1;
528 else high = middle;
529 }
530 return low;
531}
532
533/**
534 * Parses the block's text again with a tab before each mark. The tab ends the URL as a space would,
535 * so emphasis around the URL closes and later URLs and code get their own nodes. The tabs are then
536 * dropped and positions mapped back to the source. Returns false and leaves the block alone when its
537 * text would parse differently on its own, as a quote's continuation lines do.
538 */
539function reparseAtCjkPunctuation(
540 processor: Processor,
541 block: Paragraph | Heading | TableCell,
542 source: string,
543 offsets: number[],
544): boolean {
545 // A task item's paragraph starts at its checkbox; its first child starts after it.
546 const from = block.children[0]?.position?.start;
547 const to = block.children.at(-1)?.position?.end;
548 if (from?.offset === undefined || to?.offset === undefined || hasReference(block)) return false;
549 const base = from.offset;
550 const text = source.slice(base, to.offset);
551 // A tab is the marker, a quote's `>` on a later line would read as a new quote, a list's
552 // indentation would join a code span, and only a table unescapes `\|` in code.
553 if (
554 text.includes("\t") ||
555 /[\r\n][ ]*>/.test(text) ||
556 (/[\r\n] /.test(text) && text.includes("`")) ||
557 (block.type === "tableCell" && text.includes("\\|"))
558 ) {
559 return false;
560 }
561 const marks = offsets.map((offset) => offset - base);
562 // Each pass finds the next URL in a run with no spaces, like `http://a，http://b，http://c`.
563 for (let attempt = 0; attempt < 16; attempt++) {
564 marks.sort((a, b) => a - b);
565 const tabs = marks.map((mark, index) => mark + index);
566 const unmarked = (offset: number) => offset - countBelow(tabs, offset);
567 const marked = [0, ...marks]
568 .map((start, index) => text.slice(start, marks[index] ?? text.length))
569 .join("\t");
570 const [paragraph, ...rest] = (processor.parse(marked) as Root).children;
571 if (paragraph?.type !== "paragraph" || rest.length > 0) return false;
572 const more = cjkPunctuationOffsets(paragraph, marked);
573 if (more.length > 0) {
574 marks.push(...more.map(unmarked));
575 continue;
576 }
577
578 let tabCount = 0;
579 const moved = new Set<object>();
580 const restore = (node: RootContent): boolean => {
581 for (const point of node.position ? [node.position.start, node.position.end] : []) {
582 const offset = point.offset;
583 if (offset === undefined || moved.has(point)) continue;
584 moved.add(point);
585 const tabsOnLine = countBelow(tabs, offset) - countBelow(tabs, offset - point.column + 1);
586 point.column = (point.line === 1 ? from.column - 1 : 0) + point.column - tabsOnLine;
587 point.line += from.line - 1;
588 point.offset = base + unmarked(offset);
589 }
590 if (node.type === "text") {
591 tabCount += node.value.split("\t").length - 1;
592 node.value = node.value.replaceAll("\t", "");
593 } else if (
594 ("value" in node && node.value.includes("\t")) ||
595 ("url" in node && node.url.includes("\t"))
596 ) {
597 return false;
598 }
599 if (!("children" in node)) return true;
600 const children = node.children as RootContent[];
601 if (!children.every(restore)) return false;
602 node.children = children.filter(
603 (child) => child.type !== "text" || child.value !== "",
604 ) as typeof node.children;
605 return true;
606 };
607 if (!restore(paragraph) || tabCount !== tabs.length) return false;
608 block.children = paragraph.children;
609 return true;
610 }
611 return false;
612}
613
614/** Ends the bare URLs a reparse could not, and those GFM's text pass linked after emphasis. */
615function splitAtCjkPunctuation(node: RootContent, source: string) {
616 if (!("children" in node) || node.type === "link" || node.type === "linkReference") return;
617 const children = node.children as RootContent[];
618 for (let index = 0; index < children.length; index++) {
619 const child = children[index]!;
620 if (child.type !== "link") {
621 splitAtCjkPunctuation(child, source);
622 continue;
623 }
624 const at = isBareLink(child, source) ? cjkPunctuationIndex(child) : -1;
625 const label = child.children[0];
626 if (at < 0 || label?.type !== "text") continue;
627 // GFM drops this punctuation from the end of a URL; remarkKeepAutolinkSuffixes restores `_` and `~`.
628 const length = label.value.slice(0, at).replace(/[!"'*,.:;?_~]+$/, "").length;
629 const tail: Text = { type: "text", value: label.value.slice(length) };
630 const start = child.position?.start;
631 if (child.position && start?.offset !== undefined) {
632 const end = {
633 line: start.line,
634 column: start.column + length,
635 offset: start.offset + length,
636 };
637 tail.position = { start: end, end: child.position.end };
638 child.position = { start, end };
639 label.position = { start, end };
640 }
641 child.url = child.url.slice(0, child.url.length - label.value.length + length);
642 label.value = label.value.slice(0, length);
643 children.splice(index + 1, 0, tail);
644 index++;
645 }
646}
647
648function remarkEndAutolinksAtCjkPunctuation(this: Processor): Transformer<Root> {
649 return (tree, file) => {
650 const source = String(file);
651 if (!CJK_PUNCTUATION.test(source)) return;
652 const visit = (node: RootContent) => {
653 if (node.type === "paragraph" || node.type === "heading" || node.type === "tableCell") {
654 const offsets = cjkPunctuationOffsets(node, source);
655 if (offsets.length > 0) reparseAtCjkPunctuation(this, node, source, offsets);
656 splitAtCjkPunctuation(node, source);
657 } else if ("children" in node) {
658 (node.children as RootContent[]).forEach(visit);
659 }
660 };
661 tree.children.forEach(visit);
662 };
663}
664
665function remarkLinkAbsoluteFilePaths(): Transformer<Root> {
666 return (tree) => {
667 // Raw `<a>` tags are sibling html nodes, so text inside one is not under a link node.
668 let rawAnchorDepth = 0;
669 const visit = (node: Root | RootContent) => {
670 if (!("children" in node) || node.type === "link" || node.type === "linkReference") return;
671 node.children = node.children.flatMap((child): typeof node.children => {
672 if (child.type === "html") {
673 for (const [tag] of child.value.matchAll(RAW_HTML_TAG_PATTERN)) {
674 if (/^<a[\s/>]/i.test(tag)) rawAnchorDepth += 1;
675 else if (/^<\/a[\s>]/i.test(tag)) rawAnchorDepth = Math.max(0, rawAnchorDepth - 1);
676 }
677 }
678 if (child.type !== "text" || rawAnchorDepth > 0) {
679 visit(child);
680 return [child];
681 }
682 const parts: PhrasingContent[] = [];
683 let offset = 0;
684 for (const { index, path } of absoluteFilePathMatches(child.value)) {
685 if (index > offset) parts.push({ type: "text", value: child.value.slice(offset, index) });
686 parts.push({ type: "link", url: path, children: [{ type: "text", value: path }] });
687 offset = index + path.length;
688 }
689 if (offset === 0) return [child];
690 if (offset < child.value.length)
691 parts.push({ type: "text", value: child.value.slice(offset) });
692 return parts;
693 });
694 };
695 visit(tree);
696 };
697}
698
699function remarkPreserveCodeMeta() {
700 return (tree: MarkdownAstNode) => {
701 const visit = (node: MarkdownAstNode) => {
702 if (node.type === "code" && typeof node.meta === "string" && node.meta.trim().length > 0) {
703 node.data = {
704 ...node.data,
705 hProperties: {
706 ...node.data?.hProperties,
707 dataCodeMeta: node.meta.trim(),
708 },
709 };
710 }
711 node.children?.forEach(visit);
712 };
713
714 visit(tree);
715 };
716}
717
718function remarkNormalizeLinksAndTagInlineCode() {
719 return (tree: MarkdownAstNode) => {
720 const visit = (node: MarkdownAstNode, insideLink: boolean) => {
721 if (
722 (node.type === "link" || node.type === "definition") &&
723 typeof node.url === "string" &&
724 /^[A-Za-z]:[\\/]/.test(node.url)
725 ) {
726 node.url = `file:///${node.url.replaceAll("\\", "/")}`;
727 }
728 if (node.type === "inlineCode" && !insideLink) {
729 node.data = {
730 ...node.data,
731 hProperties: {
732 ...node.data?.hProperties,
733 dataInlineCode: "",
734 },
735 };
736 }
737 const childInsideLink = insideLink || node.type === "link" || node.type === "linkReference";
738 node.children?.forEach((child) => visit(child, childInsideLink));
739 };
740
741 visit(tree, false);
742 };
743}
744
745export function shouldPreserveAssistantLineBreaks(text: string): boolean {
746 return /^★ Insight(?:\s|─)/mu.test(text);
747}
748
749export function remarkAssistantInsights(this: Processor) {
750 return (tree: Root, file: { value: unknown }) => {
751 const source = String(file.value);
752 if (!source.includes("★ Insight")) return;
753 const protectedRanges: { start: number; end: number; code: boolean }[] = [];
754 const inspect = (node: RootContent | PhrasingContent) => {
755 const start = node.position?.start.offset;
756 const end = node.position?.end.offset;
757 if (start !== undefined && end !== undefined) {
758 const code =
759 node.type === "code" ||
760 node.type === "html" ||
761 (node.type === "inlineCode" && node.position?.start.line !== node.position?.end.line);
762 if (code || node.type === "list" || node.type === "blockquote")
763 protectedRanges.push({ start, end, code });
764 }
765 if ("children" in node) node.children.forEach(inspect);
766 };
767 tree.children.forEach(inspect);
768 const starts = new Set<number>();
769 const offsets: { original: number; rewritten: number; prefix: number }[] = [];
770 let indent: string | undefined;
771 let original = 0;
772 let rewritten = 0;
773 const lines = source.split(/(?<=\n)|(?<=\r)(?!\n)/u).map((line, index) => {
774 const value = line.replace(/[\r\n]+$/u, "");
775 const blocked = protectedRanges.some(
776 (range) =>
777 original >= range.start && original < range.end && (indent === undefined || range.code),
778 );
779 const opening =
780 indent === undefined && !blocked
781 ? /^( {0,3})(`?)★ Insight[ \t]+─{3,}\2[ \t]*$/u.exec(value)
782 : null;
783 const closing =
784 indent !== undefined && !blocked ? /^( {0,3})(`?)─{3,}\2[ \t]*$/u.exec(value) : null;
785 let prefix = 0;
786 let output = line;
787 if (opening) {
788 indent = opening[1] ?? "";
789 starts.add(index + 1);
790 output = ">" + line.slice(value.length);
791 } else if (closing && closing[1] === indent) {
792 indent = undefined;
793 output = line.slice(value.length);
794 } else if (indent !== undefined) {
795 prefix = 2;
796 output = "> " + line;
797 }
798 offsets.push({ original, rewritten, prefix });
799 original += line.length;
800 rewritten += output.length;
801 return output;
802 });
803 if (starts.size === 0) return;
804 const parsed = this.parse(lines.join("")) as Root;
805 const restorePosition = (node: RootContent | PhrasingContent) => {
806 if (node.position) {
807 for (const point of [node.position.start, node.position.end]) {
808 const offset = offsets[point.line - 1];
809 if (!offset) continue;
810 point.column = Math.max(1, point.column - offset.prefix);
811 if (point.offset !== undefined)
812 point.offset += offset.original - offset.rewritten - offset.prefix;
813 }
814 }
815 if ("children" in node) node.children.forEach(restorePosition);
816 };
817 for (const node of parsed.children) {
818 if (node.type === "blockquote" && starts.has(node.position?.start.line ?? 0)) {
819 node.data = { ...node.data, hProperties: { dataAlert: "insight" } };
820 }
821 restorePosition(node);
822 }
823 tree.children = parsed.children;
824 };
825}
826
827const imageSourceProcessor = unified()
828 .use(remarkParse)
829 .use(CHAT_MARKDOWN_REMARK_PLUGINS)
830 .use(remarkRehype, { allowDangerousHtml: true })
831 .use(CHAT_MARKDOWN_REHYPE_PLUGINS);
832
833interface ImageSourceNode {
834 readonly type: string;
835 readonly tagName?: string;
836 readonly properties?: {
837 readonly src?: unknown;
838 readonly dataLocalSrc?: unknown;
839 readonly dataDarkSrc?: unknown;
840 readonly dataLightSrc?: unknown;
841 };
842 readonly children?: ReadonlyArray<ImageSourceNode>;
843}
844
845/**
846 * The sources of the images chat markdown shows, read as ChatMarkdown's `img` renderer reads
847 * them. Code, comments, and plain links show no image, so they add none.
848 */
849export function chatMarkdownImageSources(markdown: string): Array<string> {
850 const sources: Array<string> = [];
851 const visit = (node: ImageSourceNode) => {
852 if (node.type === "element" && node.tagName === "img") {
853 const { src, dataLocalSrc, dataDarkSrc, dataLightSrc } = node.properties ?? {};
854 // The sanitizer drops a Windows drive path from `src`; the renderer reads it from here.
855 if (typeof dataLocalSrc === "string") {
856 sources.push(normalizeMarkdownLinkDestination(dataLocalSrc).replaceAll("\\", "/"));
857 } else if (typeof src === "string") {
858 sources.push(normalizeMarkdownLinkDestination(src));
859 }
860 // A `<picture>` shows one of these instead, by the viewer's theme.
861 for (const variant of [dataDarkSrc, dataLightSrc]) {
862 if (typeof variant === "string") sources.push(normalizeMarkdownLinkDestination(variant));
863 }
864 }
865 node.children?.forEach(visit);
866 };
867 visit(imageSourceProcessor.runSync(imageSourceProcessor.parse(markdown)) as ImageSourceNode);
868 return sources;
869}