Open History to see the last commit
1import {
2 matchInlineSkills,
3 formatProviderSkillDisplayName,
4 type InlineSkill,
5} from "./inlineSkills.ts";
6import {
7 artifactTemplateFromHastProperties,
8 renderCodexInlineDirectivesAsMarkdown,
9} from "./codexMarkdownDirectives.ts";
10import { codexArtifactTemplatePresentationLabel } from "./codexArtifactTemplates.ts";
11import {
12 buildFileLinkParentSuffixByPath,
13 splitFilePathPosition,
14 fileLinkLabel,
15} from "./fileLinks.ts";
16import {
17 isMarkdownFileLinkLabel,
18 extractMarkdownLinkHrefs,
19 extractInlineCodeSpans,
20 inlineCodeFilePathCandidate,
21 resolveMarkdownFileLinkTarget,
22} from "./markdownLinks.ts";
23import { upgradeLegacyContextMessage } from "./composerContextLegacy.ts";
24import {
25 isAttachmentOnlyBootstrapPrompt,
26 parseComposerContextHref,
27} from "./composerContextReferences.ts";
28import { parseAssistantCitationHref } from "./assistantCitations.ts";
29import type { OrchestrationV2ConversationMessage } from "@t3tools/contracts";
30import { proposedPlanTitle, stripDisplayedPlanMarkdown } from "./proposedPlanText.ts";
31import { unified } from "unified";
32import remarkParse from "remark-parse";
33import remarkRehype from "remark-rehype";
34import {
35 shouldPreserveAssistantLineBreaks,
36 remarkAssistantInsights,
37 CHAT_MARKDOWN_REHYPE_PLUGINS,
38 CHAT_MARKDOWN_REMARK_PLUGINS,
39 CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS,
40} from "./markdownPipeline.ts";
41
42// Inline wrappers (including Shiki token spans) must not split a search phrase.
43export const THREAD_FIND_BLOCK_TAGS = new Set([
44 "address",
45 "article",
46 "aside",
47 "blockquote",
48 "br",
49 "dd",
50 "details",
51 "div",
52 "dl",
53 "dt",
54 "figcaption",
55 "figure",
56 "footer",
57 "h1",
58 "h2",
59 "h3",
60 "h4",
61 "h5",
62 "h6",
63 "header",
64 "hr",
65 "li",
66 "main",
67 "nav",
68 "ol",
69 "p",
70 "pre",
71 "section",
72 "summary",
73 "table",
74 "tbody",
75 "td",
76 "th",
77 "thead",
78 "tr",
79 "ul",
80]);
81
82function assistantTextProcessor(lineBreaks = false, renderInsights = false) {
83 return unified()
84 .use(remarkParse)
85 .use(renderInsights ? [remarkAssistantInsights] : [])
86 .use(lineBreaks ? CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS : CHAT_MARKDOWN_REMARK_PLUGINS)
87 .use(remarkRehype, { allowDangerousHtml: true })
88 .use(CHAT_MARKDOWN_REHYPE_PLUGINS);
89}
90const assistantProcessor = assistantTextProcessor();
91const assistantBreaksProcessor = assistantTextProcessor(true);
92const insightProcessor = assistantTextProcessor(false, true);
93const insightBreaksProcessor = assistantTextProcessor(true, true);
94const userProcessor = unified()
95 .use(remarkParse)
96 .use(CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS)
97 .use(remarkRehype, { allowDangerousHtml: true });
98
99interface TextTree {
100 readonly type: string;
101 readonly tagName?: string;
102 readonly value?: string;
103 readonly properties?: {
104 readonly href?: unknown;
105 readonly src?: unknown;
106 readonly dataInlineCode?: unknown;
107 };
108 readonly children?: ReadonlyArray<TextTree>;
109}
110
111function markdownThreadFindText(
112 markdown: string,
113 userMessage = false,
114 cwd?: string,
115 skills: readonly InlineSkill[] = [],
116 lineBreaks = false,
117 renderInsights = false,
118): string[] {
119 const processor = userMessage
120 ? userProcessor
121 : lineBreaks
122 ? assistantBreaksProcessor
123 : assistantProcessor;
124 const renderer = renderInsights
125 ? lineBreaks
126 ? insightBreaksProcessor
127 : insightProcessor
128 : processor;
129 const tree = renderer.runSync(renderer.parse(markdown), markdown);
130 const paths = [
131 ...extractMarkdownLinkHrefs(renderCodexInlineDirectivesAsMarkdown(markdown)),
132 ...extractInlineCodeSpans(markdown).flatMap((span) => inlineCodeFilePathCandidate(span) ?? []),
133 ].flatMap((href) => {
134 const target = resolveMarkdownFileLinkTarget(href, cwd);
135 return target ? [splitFilePathPosition(target).path] : [];
136 });
137 const parentSuffixes = buildFileLinkParentSuffixByPath(paths);
138 const segments: string[] = [];
139 let text = "";
140 const flush = () => {
141 if (text.trim()) segments.push(text);
142 text = "";
143 };
144 const visit = (node: TextTree, inPre = false, inlineSkills = false) => {
145 const href = node.properties?.href ?? node.properties?.src;
146 if (userMessage && typeof href === "string" && parseComposerContextHref(href)) {
147 flush();
148 return;
149 }
150 const citation =
151 node.tagName === "a" && typeof href === "string" ? parseAssistantCitationHref(href) : null;
152 if (citation) {
153 // The quote card is not searched, and it splits the text before it from its comment.
154 flush();
155 text += (citation.comment ?? "").replace(/\r?\n/g, " ");
156 return;
157 }
158 {
159 let candidate: string | null = null;
160 if (node.tagName === "a" && typeof href === "string") {
161 candidate = href;
162 } else if (
163 node.tagName === "code" &&
164 !inPre &&
165 node.properties?.dataInlineCode !== undefined
166 ) {
167 candidate = inlineCodeFilePathCandidate(
168 (node.children ?? []).map((child) => child.value ?? "").join(""),
169 );
170 }
171 const target = candidate ? resolveMarkdownFileLinkTarget(candidate, cwd) : null;
172 if (target) {
173 const labelText = (node: TextTree): string =>
174 node.value ?? (node.children ?? []).map(labelText).join("");
175 const label = labelText(node);
176 if (node.tagName === "a" && candidate && !isMarkdownFileLinkLabel(label, candidate))
177 text += `${label} `;
178 text += fileLinkLabel(splitFilePathPosition(target), parentSuffixes);
179 return;
180 }
181 }
182 const template = artifactTemplateFromHastProperties(node.properties);
183 if (template) {
184 flush();
185 segments.push(
186 template.displayName,
187 codexArtifactTemplatePresentationLabel(template.artifactKind),
188 );
189 return;
190 }
191 const block = THREAD_FIND_BLOCK_TAGS.has(node.tagName ?? "");
192 if (block) flush();
193 if (node.type === "text" || (userMessage && node.type === "raw")) {
194 let value = node.value ?? "";
195 if (inlineSkills) {
196 let cursor = 0;
197 let rendered = "";
198 for (const { start, end, skill } of matchInlineSkills(value, skills)) {
199 rendered += value.slice(cursor, start) + formatProviderSkillDisplayName(skill);
200 cursor = end;
201 }
202 value = rendered + value.slice(cursor);
203 }
204 text += inPre ? value : value.replace(/\r?\n/g, " ");
205 }
206 const renderSkills =
207 node.tagName === "code" || node.tagName === "a"
208 ? false
209 : inlineSkills || node.tagName === "p" || node.tagName === "li";
210 for (const child of node.children ?? [])
211 visit(child, inPre || node.tagName === "pre", renderSkills);
212 if (block) flush();
213 };
214 visit(tree);
215 flush();
216 return segments;
217}
218
219/** Plans render `$skill` tokens literally (no skill chips), so they are indexed as written. */
220export function searchablePlanSegments(markdown: string, cwd?: string): readonly string[] {
221 return [
222 proposedPlanTitle(markdown) ?? "Proposed plan",
223 ...markdownThreadFindText(stripDisplayedPlanMarkdown(markdown), false, cwd, []),
224 ];
225}
226
227export function searchableMessageSegments(
228 message: Pick<OrchestrationV2ConversationMessage, "role" | "text" | "streaming" | "context">,
229 cwd?: string,
230 skills: readonly InlineSkill[] = [],
231): readonly string[] | null {
232 if (message.role === "user") {
233 // Clients hide this prompt, so find must not match text nobody can see.
234 if (isAttachmentOnlyBootstrapPrompt(message.text)) return [];
235 const text = message.context ? message.text : upgradeLegacyContextMessage(message.text).text;
236 return markdownThreadFindText(text, true, cwd, skills);
237 }
238 if (message.role !== "assistant") return null;
239 return markdownThreadFindText(
240 message.text || (message.streaming ? "" : "(empty response)"),
241 false,
242 cwd,
243 skills,
244 shouldPreserveAssistantLineBreaks(message.text),
245 true,
246 );
247}