Open History to see the last commit
1import { isLocalEditorFileUrl, LOCAL_FILE_LINK_SCHEMES } from "@t3tools/contracts";
2import type {
3 Heading,
4 Link,
5 Paragraph,
6 PhrasingContent,
7 Root,
8 RootContent,
9 TableCell,
10 Text,
11} from "mdast";
12import { remarkKeepWindowsPaths } from "./markdownWindowsPaths.ts";
13import { type PluggableList, type Processor, type Transformer, unified } from "unified";
14import remarkParse from "remark-parse";
15import remarkRehype from "remark-rehype";
16import { find, html } from "property-information";
17import rehypeRaw from "rehype-raw";
18import rehypeSanitize, { defaultSchema } from "rehype-sanitize";
19import remarkBreaks from "remark-breaks";
20import remarkGfm from "remark-gfm";
21import { remarkGithubAlerts } from "./markdownGithubAlerts.ts";
22import { remarkNormalizeListItemIndentation } from "./markdownListIndentation.ts";
23import { remarkKeepOrderedListNumbers } from "./markdownOrderedListNumbers.ts";
24import {
25 CODEX_ARTIFACT_TEMPLATE_HAST_PROPERTIES,
26 remarkCodexDirectives,
27} from "./codexMarkdownDirectives.ts";
28import {
29 absoluteFilePathMatches,
30 isWindowsDrivePathHref,
31 normalizeMarkdownLinkDestination,
32} from "./markdownLinks.ts";
33import { THREAD_LINK_PROTOCOL } from "./threadLinks.ts";
34
35export { remarkEnableOrderedListNumbers } from "./markdownOrderedListNumbers.ts";
36
37type MarkdownImageHastNode = {
38 type?: string;
39 value?: string;
40 tagName?: string;
41 properties?: Record<string, unknown>;
42 children?: MarkdownImageHastNode[];
43};
44
45function meaningfulHastChildren(node: MarkdownImageHastNode): MarkdownImageHastNode[] {
46 return (node.children ?? []).filter(
47 (child) => !(child.type === "text" && (child as { value?: string }).value?.trim() === ""),
48 );
49}
50
51const STANDALONE_IMAGE_BLOCKS = new Set([
52 "p",
53 "div",
54 "li",
55 "td",
56 "th",
57 "figure",
58 "center",
59 "blockquote",
60]);
61
62function soleImageDescendant(node: MarkdownImageHastNode): MarkdownImageHastNode | undefined {
63 const children = meaningfulHastChildren(node);
64 if (children.length !== 1) return undefined;
65 const only = children[0];
66 if (only?.type !== "element") return undefined;
67 if (only.tagName === "img") return only;
68 return only.tagName === "a" || only.tagName === "strong" || only.tagName === "em"
69 ? soleImageDescendant(only)
70 : undefined;
71}
72
73function markStandaloneImages(node: MarkdownImageHastNode) {
74 if (node.type === "root" || (node.tagName && STANDALONE_IMAGE_BLOCKS.has(node.tagName))) {
75 const image = soleImageDescendant(node);
76 if (image) image.properties = { ...image.properties, dataStandalone: true };
77 }
78 node.children?.forEach((child) => {
79 if (child.type === "element") markStandaloneImages(child);
80 });
81}
82
83// Whole tags, attributes, and comments, so closing-anchor text inside them never counts as a tag.
84const RAW_HTML_TAG_PATTERN = /<!--[\s\S]*?(?:-->|$)|<\/?[A-Za-z](?:[^"'<>]|"[^"]*"|'[^']*')*>/g;
85
86function rehypePreserveBareAnchorPlaceholders() {
87 return (tree: MarkdownImageHastNode) => {
88 const anchors: Array<MarkdownImageHastNode | null> = [];
89 let rawTextTag: string | undefined;
90 const visit = (node: MarkdownImageHastNode) => {
91 if (node.type === "raw" && typeof node.value === "string") {
92 const tags = new RegExp(RAW_HTML_TAG_PATTERN);
93 let offset = 0;
94 while (rawTextTag !== "plaintext") {
95 // Raw text ends at its closing tag even inside comment-looking text.
96 const matcher = rawTextTag ? new RegExp(`</${rawTextTag}\\s*>`, "gi") : tags;
97 matcher.lastIndex = offset;
98 const match = matcher.exec(node.value);
99 if (!match) break;
100 const [tag] = match;
101 offset = matcher.lastIndex;
102 if (rawTextTag) {
103 rawTextTag = undefined;
104 continue;
105 }
106 if (tag.startsWith("<!--")) continue;
107 const closing = /^<\/([a-z]+)\s*>$/i.exec(tag)?.[1]?.toLowerCase();
108 const opening = /^<([a-z]+)(?:\s|\/?>)/i.exec(tag)?.[1]?.toLowerCase();
109 if (
110 opening &&
111 /^(?:script|style|textarea|title|xmp|iframe|noembed|noframes|plaintext)$/.test(opening)
112 ) {
113 rawTextTag = opening;
114 } else if (opening === "a") {
115 anchors.push(node.value === tag && /^<a\s*\/?>$/i.test(tag) ? node : null);
116 } else if (closing === "a") {
117 anchors.pop();
118 }
119 }
120 }
121 node.children?.forEach(visit);
122 };
123
124 visit(tree);
125 for (const anchor of anchors) {
126 if (anchor) anchor.type = "text";
127 }
128 };
129}
130
131const COLOR_SCHEME_MEDIA = /^\(\s*prefers-color-scheme\s*:\s*(dark|light)\s*\)$/i;
132
133/**
134 * The renderer resolves an `<img>`'s `src` (workspace files, host media) but never a `<source>`'s
135 * `srcset`, which would load unresolved whenever its media matched. A color-scheme source moves
136 * onto the `<img>` for the renderer to pick by theme; every source is then dropped.
137 */
138function moveColorSchemeSourcesToImage(picture: MarkdownImageHastNode) {
139 if (!picture.children) return;
140 const image = picture.children.find((child) => child.tagName === "img");
141 for (const source of picture.children) {
142 if (source.tagName !== "source" || !image) continue;
143 const media = source.properties?.media;
144 const scheme = typeof media === "string" ? COLOR_SCHEME_MEDIA.exec(media)?.[1] : undefined;
145 const srcSet = source.properties?.srcSet;
146 const candidate = Array.isArray(srcSet) ? srcSet[0] : srcSet;
147 // A candidate's URL runs to the first space (a descriptor such as `2x` may follow).
148 const url =
149 typeof candidate === "string" ? candidate.trim().split(/\s/, 1)[0]?.replace(/,+$/, "") : "";
150 const key = scheme?.toLowerCase() === "dark" ? "dataDarkSrc" : "dataLightSrc";
151 // As in a browser, the first source that matches a scheme is the one shown.
152 if (scheme && url && image.properties?.[key] === undefined) {
153 // The sanitizer reads a drive letter as a protocol; as a file URL it resolves to the same path.
154 const value = isWindowsDrivePathHref(url) ? `file:///${url.replaceAll("\\", "/")}` : url;
155 image.properties = { ...image.properties, [key]: value };
156 }
157 }
158 picture.children = picture.children.filter((child) => child.tagName !== "source");
159}
160
161function rehypePreserveImageSourceMeta() {
162 return (tree: MarkdownImageHastNode) => {
163 const visit = (node: MarkdownImageHastNode) => {
164 const src = node.properties?.src;
165 const title = node.properties?.title;
166 if (node.type === "element" && node.tagName === "picture") {
167 moveColorSchemeSourcesToImage(node);
168 }
169 if (node.type === "element" && node.tagName === "img") {
170 node.properties = {
171 ...node.properties,
172 ...(typeof src === "string" && isWindowsDrivePathHref(src) ? { dataLocalSrc: src } : {}),
173 ...(typeof title === "string" ? { dataMarkdownTitle: title } : {}),
174 };
175 }
176 node.children?.forEach(visit);
177 };
178
179 visit(tree);
180 markStandaloneImages(tree);
181 };
182}
183
184const IMAGE_SRC_PROTOCOLS = [...(defaultSchema.protocols?.src ?? []), "file", "t3-context"];
185
186// The sanitizer compares schemes case-sensitively, so lowercase accepted editor links first.
187function rehypeNormalizeLocalEditorLinks() {
188 return (tree: MarkdownImageHastNode) => {
189 const visit = (node: MarkdownImageHastNode) => {
190 const href = node.properties?.href;
191 if (node.type === "element" && node.tagName === "a" && typeof href === "string") {
192 try {
193 const url = new URL(href);
194 if (isLocalEditorFileUrl(url)) node.properties = { ...node.properties, href: url.href };
195 } catch {
196 // Not an absolute URL; the sanitizer decides.
197 }
198 }
199 node.children?.forEach(visit);
200 };
201
202 visit(tree);
203 };
204}
205
206const CHAT_MARKDOWN_SANITIZE_SCHEMA = {
207 ...defaultSchema,
208 attributes: {
209 ...defaultSchema.attributes,
210 "*": (defaultSchema.attributes?.["*"] ?? []).filter((attribute) => attribute !== "title"),
211 // Fence languages plus the two math classes the web chat's math plugin emits.
212 code: [
213 ["className", /^language-./, "math-inline", "math-display"],
214 "dataCodeMeta",
215 "dataInlineCode",
216 ],
217 blockquote: [...(defaultSchema.attributes?.blockquote ?? []), "dataAlert"],
218 div: [...(defaultSchema.attributes?.div ?? []), ...CODEX_ARTIFACT_TEMPLATE_HAST_PROPERTIES],
219 a: [...(defaultSchema.attributes?.a ?? []), "dataPullRequestAutolink"],
220 img: [
221 ...(defaultSchema.attributes?.img ?? []),
222 "dataDarkSrc",
223 "dataLightSrc",
224 "dataLocalSrc",
225 "dataMarkdownTitle",
226 "dataStandalone",
227 ],
228 },
229 protocols: {
230 ...defaultSchema.protocols,
231 href: [
232 ...(defaultSchema.protocols?.href ?? []),
233 "file",
234 "t3-citation",
235 "t3-context",
236 THREAD_LINK_PROTOCOL,
237 // Scheme only: the renderer's URL transform checks the rest of the link shape.
238 ...LOCAL_FILE_LINK_SCHEMES,
239 ],
240 src: IMAGE_SRC_PROTOCOLS,
241 dataDarkSrc: IMAGE_SRC_PROTOCOLS,
242 dataLightSrc: IMAGE_SRC_PROTOCOLS,
243 },
244} satisfies Parameters<typeof rehypeSanitize>[0];
245
246export const CHAT_MARKDOWN_REMARK_PLUGINS: PluggableList = [
247 remarkGfm,
248 remarkEndAutolinksAtCjkPunctuation,
249 remarkKeepAutolinkSuffixes,
250 remarkLinkAbsoluteFilePaths,
251 remarkKeepWindowsPaths,
252 remarkGithubAlerts,
253 remarkKeepOrderedListNumbers,
254 remarkNormalizeListItemIndentation,
255 remarkCodexDirectives,
256 remarkBreakAroundImageLines,
257 remarkPreserveCodeMeta,
258 remarkNormalizeLinksAndTagInlineCode,
259];
260
261export const CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS: PluggableList = [
262 remarkGfm,
263 remarkEndAutolinksAtCjkPunctuation,
264 remarkKeepAutolinkSuffixes,
265 remarkLinkAbsoluteFilePaths,
266 remarkKeepWindowsPaths,
267 remarkGithubAlerts,
268 remarkKeepOrderedListNumbers,
269 remarkNormalizeListItemIndentation,
270 remarkCodexDirectives,
271 remarkBreakAroundImageLines,
272 remarkBreaks,
273 remarkPreserveCodeMeta,
274 remarkNormalizeLinksAndTagInlineCode,
275];
276
277// Start tags an HTML parser closes an open `<p>` for.
278const CLOSES_PARAGRAPH = new Set(
279 "address article aside blockquote center details dialog dir div dl fieldset figcaption figure footer form h1 h2 h3 h4 h5 h6 header hgroup hr li listing main menu nav ol p plaintext pre search section summary table ul xmp dd dt".split(
280 " ",
281 ),
282);
283const HEADING = /^h[1-6]$/;
284
285interface HtmlScope {
286 readonly anchor: boolean;
287 readonly paragraph: boolean;
288 readonly heading: boolean;
289}
290
291/**
292 * Whether the tree needs the HTML parser: raw HTML, or nesting it repairs (a footnote call in a
293 * link label nests `<a>`). Otherwise it only rewrites properties as hast-util-to-parse5 does and
294 * drops `data`, both repeated here, and moves blank text that renders the same.
295 */
296function needsHtmlParser(node: MarkdownImageHastNode, scope: HtmlScope): boolean {
297 if (node.type === "raw") return true;
298 let inner = scope;
299 if (node.type === "element" && node.tagName) {
300 const { tagName } = node;
301 const heading = HEADING.test(tagName);
302 if (
303 (tagName === "a" && scope.anchor) ||
304 (scope.paragraph && CLOSES_PARAGRAPH.has(tagName)) ||
305 (heading && scope.heading)
306 ) {
307 return true;
308 }
309 inner = {
310 anchor: scope.anchor || tagName === "a",
311 paragraph: scope.paragraph || tagName === "p",
312 heading: scope.heading || heading,
313 };
314 const properties = node.properties ?? {};
315 for (const key of Object.keys(properties)) {
316 const value = properties[key];
317 const info = find(html, key);
318 if (
319 value === false ||
320 value === null ||
321 value === undefined ||
322 Number.isNaN(value) ||
323 (!value && info.boolean)
324 ) {
325 delete properties[key];
326 } else if (value === true && !info.boolean && !info.overloadedBoolean) {
327 properties[key] = "";
328 }
329 }
330 delete (node as { data?: unknown }).data;
331 }
332 return (node.children ?? []).some((child) => needsHtmlParser(child, inner));
333}
334
335const TOP_SCOPE: HtmlScope = { anchor: false, paragraph: false, heading: false };
336
337/** rehype-raw reparses the whole tree through an HTML parser, which most trees can skip. */
338function rehypeRawWhenPresent(this: Processor) {
339 const transform = rehypeRaw.call(this) as unknown as (
340 tree: MarkdownImageHastNode,
341 file: unknown,
342 ) => unknown;
343 return (tree: MarkdownImageHastNode, file: unknown) =>
344 needsHtmlParser(tree, TOP_SCOPE) ? transform(tree, file) : undefined;
345}
346
347export const CHAT_MARKDOWN_REHYPE_PLUGINS: PluggableList = [
348 rehypePreserveBareAnchorPlaceholders,
349 rehypeRawWhenPresent,
350 rehypePreserveImageSourceMeta,
351 rehypeNormalizeLocalEditorLinks,
352 [rehypeSanitize, CHAT_MARKDOWN_SANITIZE_SCHEMA],
353];
354
355const isImageNode = (node: PhrasingContent | undefined): boolean =>
356 node?.type === "image" ||
357 node?.type === "imageReference" ||
358 ((node?.type === "link" || node?.type === "linkReference") &&
359 node.children.length === 1 &&
360 isImageNode(node.children[0]));
361const endsLine = (node: PhrasingContent | undefined) =>
362 node === undefined ||
363 node.type === "break" ||
364 (node.type === "text" && /\n[ \t]*$/.test(node.value));
365const startsLine = (node: PhrasingContent | undefined) =>
366 node === undefined ||
367 node.type === "break" ||
368 (node.type === "text" && /^[ \t]*\n/.test(node.value));
369
370function breakLinesAroundImages(paragraph: Paragraph) {
371 const children = paragraph.children;
372 if (!children.some(isImageNode)) return;
373 const ownsLine = (index: number) =>
374 isImageNode(children[index]) &&
375 endsLine(children[index - 1]) &&
376 startsLine(children[index + 1]);
377 paragraph.children = children.flatMap((child, index): PhrasingContent[] => {
378 if (child.type !== "text") return [child];
379 const afterImage = ownsLine(index - 1);
380 const beforeImage = ownsLine(index + 1);
381 if (!afterImage && !beforeImage) return [child];
382 const parts: PhrasingContent[] = [];
383 let value = child.value;
384 if (afterImage) {
385 value = value.replace(/^[ \t]*\n/, "");
386 parts.push({ type: "break" });
387 }
388 const breakAfter = beforeImage && /\n[ \t]*$/.test(value);
389 if (breakAfter) value = value.replace(/[ \t]*\n[ \t]*$/, "");
390 if (value) parts.push({ type: "text", value });
391 if (breakAfter) parts.push({ type: "break" });
392 return parts;
393 });
394}
395
396/**
397 * Agents label screenshots with a line like `**Palette**` right above the image. CommonMark
398 * joins those lines into one paragraph, which puts the image inline beside its label. An
399 * image alone on its source line keeps that line; prose and inline badges still wrap.
400 */
401function remarkBreakAroundImageLines() {
402 return (tree: Root) => {
403 const visit = (node: Root | RootContent) => {
404 if (node.type === "paragraph") breakLinesAroundImages(node);
405 else if ("children" in node) node.children.forEach(visit);
406 };
407 visit(tree);
408 };
409}
410
411type MarkdownAstNode = {
412 type?: string;
413 meta?: unknown;
414 url?: string;
415 data?: {
416 hProperties?: Record<string, unknown>;
417 };
418 children?: MarkdownAstNode[];
419};
420
421function remarkKeepAutolinkSuffixes(): Transformer<Root> {
422 return (tree, file) => {
423 const source = String(file);
424 const visit = (node: Root | RootContent) => {
425 if (!("children" in node) || node.type === "link" || node.type === "linkReference") return;
426 for (let index = 0; index < node.children.length; index++) {
427 const child = node.children[index];
428 const next = node.children[index + 1];
429 if (child?.type === "link" && /^https?:\/\//i.test(child.url) && next?.type === "text") {
430 const suffix = /^[_~]+/.exec(next.value)?.[0];
431 if (!suffix) continue;
432 const label = child.children.length === 1 ? child.children[0] : undefined;
433 if (label?.type !== "text") continue;
434 if (label.value !== child.url && `http://${label.value}` !== child.url) continue;
435 const start = child.position?.start.offset;
436 const end = child.position?.end.offset;
437 const bare =
438 !child.position ||
439 (start !== undefined &&
440 end !== undefined &&
441 source.slice(start, end) === label.value &&
442 source.slice(end, end + suffix.length) === suffix);
443 if (bare) {
444 child.url += suffix;
445 label.value += suffix;
446 next.value = next.value.slice(suffix.length);
447 if (child.position && end !== undefined) {
448 const boundary = {
449 ...child.position.end,
450 column: child.position.end.column + suffix.length,
451 offset: end + suffix.length,
452 };
453 child.position.end = boundary;
454 if (label.position) label.position.end = boundary;
455 if (next.position) next.position.start = boundary;
456 }
457 if (!next.value) node.children.splice(index + 1, 1);
458 }
459 }
460 if (child) visit(child);
461 }
462 };
463 visit(tree);
464 };
465}
466
467// Punctuation and symbols from CJK Symbols and Punctuation and from Halfwidth and Fullwidth Forms.
468const CJK_PUNCTUATION = /(?=[ -〿-])[\p{P}\p{S}]/u;
469
470/**
471 * GFM runs a bare URL on until whitespace, so Chinese and Japanese punctuation joins it, along with
472 * the rest of the sentence and any `**` that should have closed around it. A URL that is ASCII up to
473 * such a mark ends there. Unicode earlier in the URL means the mark may belong to it, so it stays.
474 */
475function cjkPunctuationIndex(link: Link): number {
476 const label = link.children.length === 1 ? link.children[0] : undefined;
477 if (label?.type !== "text" || !/^https?:\/\//i.test(link.url)) return -1;
478 if (link.url !== label.value && link.url !== `http://${label.value}`) return -1;
479 const index = label.value.search(/[^!-~]/u);
480 return index > 0 && CJK_PUNCTUATION.test(label.value.charAt(index)) ? index : -1;
481}
482
483/** GFM's text pass links bare URLs without positions; its tokenizer's links span their URL. */
484function isBareLink(link: Link, source: string): boolean {
485 const start = link.position?.start.offset;
486 const end = link.position?.end.offset;
487 const label = link.children[0];
488 return (
489 !link.position ||
490 (start !== undefined &&
491 end !== undefined &&
492 label?.type === "text" &&
493 source.slice(start, end) === label.value)
494 );
495}
496
497/** Source offsets of the marks that tokenized bare URLs run past. */
498function cjkPunctuationOffsets(node: RootContent, source: string): number[] {
499 if (node.type === "link") {
500 const index = cjkPunctuationIndex(node);
501 const start = node.position?.start.offset;
502 return index > 0 && start !== undefined && isBareLink(node, source) ? [start + index] : [];
503 }
504 return "children" in node && node.type !== "linkReference"
505 ? (node.children as RootContent[]).flatMap((child) => cjkPunctuationOffsets(child, source))
506 : [];
507}
508
509const hasReference = (node: RootContent): boolean =>
510 node.type === "linkReference" ||
511 node.type === "imageReference" ||
512 node.type === "footnoteReference" ||
513 ("children" in node && (node.children as RootContent[]).some(hasReference));
514
515/** How many of the ascending numbers are below `value`. */
516function countBelow(sorted: readonly number[], value: number): number {
517 let low = 0;
518 let high = sorted.length;
519 while (low < high) {
520 const middle = (low + high) >> 1;
521 if (sorted[middle]! < value) low = middle + 1;
522 else high = middle;
523 }
524 return low;
525}
526
527/**
528 * Parses the block's text again with a tab before each mark. The tab ends the URL as a space would,
529 * so emphasis around the URL closes and later URLs and code get their own nodes. The tabs are then
530 * dropped and positions mapped back to the source. Returns false and leaves the block alone when its
531 * text would parse differently on its own, as a quote's continuation lines do.
532 */
533function reparseAtCjkPunctuation(
534 processor: Processor,
535 block: Paragraph | Heading | TableCell,
536 source: string,
537 offsets: number[],
538): boolean {
539 // A task item's paragraph starts at its checkbox; its first child starts after it.
540 const from = block.children[0]?.position?.start;
541 const to = block.children.at(-1)?.position?.end;
542 if (from?.offset === undefined || to?.offset === undefined || hasReference(block)) return false;
543 const base = from.offset;
544 const text = source.slice(base, to.offset);
545 // A tab is the marker, a quote's `>` on a later line would read as a new quote, a list's
546 // indentation would join a code span, and only a table unescapes `\|` in code.
547 if (
548 text.includes("\t") ||
549 /[\r\n][ ]*>/.test(text) ||
550 (/[\r\n] /.test(text) && text.includes("`")) ||
551 (block.type === "tableCell" && text.includes("\\|"))
552 ) {
553 return false;
554 }
555 const marks = offsets.map((offset) => offset - base);
556 // Each pass finds the next URL in a run with no spaces, like `http://a，http://b，http://c`.
557 for (let attempt = 0; attempt < 16; attempt++) {
558 marks.sort((a, b) => a - b);
559 const tabs = marks.map((mark, index) => mark + index);
560 const unmarked = (offset: number) => offset - countBelow(tabs, offset);
561 const marked = [0, ...marks]
562 .map((start, index) => text.slice(start, marks[index] ?? text.length))
563 .join("\t");
564 const [paragraph, ...rest] = (processor.parse(marked) as Root).children;
565 if (paragraph?.type !== "paragraph" || rest.length > 0) return false;
566 const more = cjkPunctuationOffsets(paragraph, marked);
567 if (more.length > 0) {
568 marks.push(...more.map(unmarked));
569 continue;
570 }
571
572 let tabCount = 0;
573 const moved = new Set<object>();
574 const restore = (node: RootContent): boolean => {
575 for (const point of node.position ? [node.position.start, node.position.end] : []) {
576 const offset = point.offset;
577 if (offset === undefined || moved.has(point)) continue;
578 moved.add(point);
579 const tabsOnLine = countBelow(tabs, offset) - countBelow(tabs, offset - point.column + 1);
580 point.column = (point.line === 1 ? from.column - 1 : 0) + point.column - tabsOnLine;
581 point.line += from.line - 1;
582 point.offset = base + unmarked(offset);
583 }
584 if (node.type === "text") {
585 tabCount += node.value.split("\t").length - 1;
586 node.value = node.value.replaceAll("\t", "");
587 } else if (
588 ("value" in node && node.value.includes("\t")) ||
589 ("url" in node && node.url.includes("\t"))
590 ) {
591 return false;
592 }
593 if (!("children" in node)) return true;
594 const children = node.children as RootContent[];
595 if (!children.every(restore)) return false;
596 node.children = children.filter(
597 (child) => child.type !== "text" || child.value !== "",
598 ) as typeof node.children;
599 return true;
600 };
601 if (!restore(paragraph) || tabCount !== tabs.length) return false;
602 block.children = paragraph.children;
603 return true;
604 }
605 return false;
606}
607
608/** Ends the bare URLs a reparse could not, and those GFM's text pass linked after emphasis. */
609function splitAtCjkPunctuation(node: RootContent, source: string) {
610 if (!("children" in node) || node.type === "link" || node.type === "linkReference") return;
611 const children = node.children as RootContent[];
612 for (let index = 0; index < children.length; index++) {
613 const child = children[index]!;
614 if (child.type !== "link") {
615 splitAtCjkPunctuation(child, source);
616 continue;
617 }
618 const at = isBareLink(child, source) ? cjkPunctuationIndex(child) : -1;
619 const label = child.children[0];
620 if (at < 0 || label?.type !== "text") continue;
621 // GFM drops this punctuation from the end of a URL; remarkKeepAutolinkSuffixes restores `_` and `~`.
622 const length = label.value.slice(0, at).replace(/[!"'*,.:;?_~]+$/, "").length;
623 const tail: Text = { type: "text", value: label.value.slice(length) };
624 const start = child.position?.start;
625 if (child.position && start?.offset !== undefined) {
626 const end = {
627 line: start.line,
628 column: start.column + length,
629 offset: start.offset + length,
630 };
631 tail.position = { start: end, end: child.position.end };
632 child.position = { start, end };
633 label.position = { start, end };
634 }
635 child.url = child.url.slice(0, child.url.length - label.value.length + length);
636 label.value = label.value.slice(0, length);
637 children.splice(index + 1, 0, tail);
638 index++;
639 }
640}
641
642function remarkEndAutolinksAtCjkPunctuation(this: Processor): Transformer<Root> {
643 return (tree, file) => {
644 const source = String(file);
645 if (!CJK_PUNCTUATION.test(source)) return;
646 const visit = (node: RootContent) => {
647 if (node.type === "paragraph" || node.type === "heading" || node.type === "tableCell") {
648 const offsets = cjkPunctuationOffsets(node, source);
649 if (offsets.length > 0) reparseAtCjkPunctuation(this, node, source, offsets);
650 splitAtCjkPunctuation(node, source);
651 } else if ("children" in node) {
652 (node.children as RootContent[]).forEach(visit);
653 }
654 };
655 tree.children.forEach(visit);
656 };
657}
658
659function remarkLinkAbsoluteFilePaths(): Transformer<Root> {
660 return (tree) => {
661 // Raw `<a>` tags are sibling html nodes, so text inside one is not under a link node.
662 let rawAnchorDepth = 0;
663 const visit = (node: Root | RootContent) => {
664 if (!("children" in node) || node.type === "link" || node.type === "linkReference") return;
665 node.children = node.children.flatMap((child): typeof node.children => {
666 if (child.type === "html") {
667 for (const [tag] of child.value.matchAll(RAW_HTML_TAG_PATTERN)) {
668 if (/^<a[\s/>]/i.test(tag)) rawAnchorDepth += 1;
669 else if (/^<\/a[\s>]/i.test(tag)) rawAnchorDepth = Math.max(0, rawAnchorDepth - 1);
670 }
671 }
672 if (child.type !== "text" || rawAnchorDepth > 0) {
673 visit(child);
674 return [child];
675 }
676 const parts: PhrasingContent[] = [];
677 let offset = 0;
678 for (const { index, path } of absoluteFilePathMatches(child.value)) {
679 if (index > offset) parts.push({ type: "text", value: child.value.slice(offset, index) });
680 parts.push({ type: "link", url: path, children: [{ type: "text", value: path }] });
681 offset = index + path.length;
682 }
683 if (offset === 0) return [child];
684 if (offset < child.value.length)
685 parts.push({ type: "text", value: child.value.slice(offset) });
686 return parts;
687 });
688 };
689 visit(tree);
690 };
691}
692
693function remarkPreserveCodeMeta() {
694 return (tree: MarkdownAstNode) => {
695 const visit = (node: MarkdownAstNode) => {
696 if (node.type === "code" && typeof node.meta === "string" && node.meta.trim().length > 0) {
697 node.data = {
698 ...node.data,
699 hProperties: {
700 ...node.data?.hProperties,
701 dataCodeMeta: node.meta.trim(),
702 },
703 };
704 }
705 node.children?.forEach(visit);
706 };
707
708 visit(tree);
709 };
710}
711
712function remarkNormalizeLinksAndTagInlineCode() {
713 return (tree: MarkdownAstNode) => {
714 const visit = (node: MarkdownAstNode, insideLink: boolean) => {
715 if (
716 (node.type === "link" || node.type === "definition") &&
717 typeof node.url === "string" &&
718 /^[A-Za-z]:[\\/]/.test(node.url)
719 ) {
720 node.url = `file:///${node.url.replaceAll("\\", "/")}`;
721 }
722 if (node.type === "inlineCode" && !insideLink) {
723 node.data = {
724 ...node.data,
725 hProperties: {
726 ...node.data?.hProperties,
727 dataInlineCode: "",
728 },
729 };
730 }
731 const childInsideLink = insideLink || node.type === "link" || node.type === "linkReference";
732 node.children?.forEach((child) => visit(child, childInsideLink));
733 };
734
735 visit(tree, false);
736 };
737}
738
739export function shouldPreserveAssistantLineBreaks(text: string): boolean {
740 return /^★ Insight(?:\s|─)/mu.test(text);
741}
742
743export function remarkAssistantInsights(this: Processor) {
744 return (tree: Root, file: { value: unknown }) => {
745 const source = String(file.value);
746 if (!source.includes("★ Insight")) return;
747 const protectedRanges: { start: number; end: number; code: boolean }[] = [];
748 const inspect = (node: RootContent | PhrasingContent) => {
749 const start = node.position?.start.offset;
750 const end = node.position?.end.offset;
751 if (start !== undefined && end !== undefined) {
752 const code =
753 node.type === "code" ||
754 node.type === "html" ||
755 (node.type === "inlineCode" && node.position?.start.line !== node.position?.end.line);
756 if (code || node.type === "list" || node.type === "blockquote")
757 protectedRanges.push({ start, end, code });
758 }
759 if ("children" in node) node.children.forEach(inspect);
760 };
761 tree.children.forEach(inspect);
762 const starts = new Set<number>();
763 const offsets: { original: number; rewritten: number; prefix: number }[] = [];
764 let indent: string | undefined;
765 let original = 0;
766 let rewritten = 0;
767 const lines = source.split(/(?<=\n)|(?<=\r)(?!\n)/u).map((line, index) => {
768 const value = line.replace(/[\r\n]+$/u, "");
769 const blocked = protectedRanges.some(
770 (range) =>
771 original >= range.start && original < range.end && (indent === undefined || range.code),
772 );
773 const opening =
774 indent === undefined && !blocked
775 ? /^( {0,3})(`?)★ Insight[ \t]+─{3,}\2[ \t]*$/u.exec(value)
776 : null;
777 const closing =
778 indent !== undefined && !blocked ? /^( {0,3})(`?)─{3,}\2[ \t]*$/u.exec(value) : null;
779 let prefix = 0;
780 let output = line;
781 if (opening) {
782 indent = opening[1] ?? "";
783 starts.add(index + 1);
784 output = ">" + line.slice(value.length);
785 } else if (closing && closing[1] === indent) {
786 indent = undefined;
787 output = line.slice(value.length);
788 } else if (indent !== undefined) {
789 prefix = 2;
790 output = "> " + line;
791 }
792 offsets.push({ original, rewritten, prefix });
793 original += line.length;
794 rewritten += output.length;
795 return output;
796 });
797 if (starts.size === 0) return;
798 const parsed = this.parse(lines.join("")) as Root;
799 const restorePosition = (node: RootContent | PhrasingContent) => {
800 if (node.position) {
801 for (const point of [node.position.start, node.position.end]) {
802 const offset = offsets[point.line - 1];
803 if (!offset) continue;
804 point.column = Math.max(1, point.column - offset.prefix);
805 if (point.offset !== undefined)
806 point.offset += offset.original - offset.rewritten - offset.prefix;
807 }
808 }
809 if ("children" in node) node.children.forEach(restorePosition);
810 };
811 for (const node of parsed.children) {
812 if (node.type === "blockquote" && starts.has(node.position?.start.line ?? 0)) {
813 node.data = { ...node.data, hProperties: { dataAlert: "insight" } };
814 }
815 restorePosition(node);
816 }
817 tree.children = parsed.children;
818 };
819}
820
821const imageSourceProcessor = unified()
822 .use(remarkParse)
823 .use(CHAT_MARKDOWN_REMARK_PLUGINS)
824 .use(remarkRehype, { allowDangerousHtml: true })
825 .use(CHAT_MARKDOWN_REHYPE_PLUGINS);
826
827interface ImageSourceNode {
828 readonly type: string;
829 readonly tagName?: string;
830 readonly properties?: {
831 readonly src?: unknown;
832 readonly dataLocalSrc?: unknown;
833 readonly dataDarkSrc?: unknown;
834 readonly dataLightSrc?: unknown;
835 };
836 readonly children?: ReadonlyArray<ImageSourceNode>;
837}
838
839/**
840 * The sources of the images chat markdown shows, read as ChatMarkdown's `img` renderer reads
841 * them. Code, comments, and plain links show no image, so they add none.
842 */
843export function chatMarkdownImageSources(markdown: string): Array<string> {
844 const sources: Array<string> = [];
845 const visit = (node: ImageSourceNode) => {
846 if (node.type === "element" && node.tagName === "img") {
847 const { src, dataLocalSrc, dataDarkSrc, dataLightSrc } = node.properties ?? {};
848 // The sanitizer drops a Windows drive path from `src`; the renderer reads it from here.
849 if (typeof dataLocalSrc === "string") {
850 sources.push(normalizeMarkdownLinkDestination(dataLocalSrc).replaceAll("\\", "/"));
851 } else if (typeof src === "string") {
852 sources.push(normalizeMarkdownLinkDestination(src));
853 }
854 // A `<picture>` shows one of these instead, by the viewer's theme.
855 for (const variant of [dataDarkSrc, dataLightSrc]) {
856 if (typeof variant === "string") sources.push(normalizeMarkdownLinkDestination(variant));
857 }
858 }
859 node.children?.forEach(visit);
860 };
861 visit(imageSourceProcessor.runSync(imageSourceProcessor.parse(markdown)) as ImageSourceNode);
862 return sources;
863}