1import {
2 matchInlineSkills,
3 formatProviderSkillDisplayName,
4 type InlineSkill,
5} from "./inlineSkills.ts";
6import {
7 artifactTemplateFromHastProperties,
8 renderCodexInlineDirectivesAsMarkdown,
9} from "./codexMarkdownDirectives.ts";
10import { codexArtifactTemplatePresentationLabel } from "./codexArtifactTemplates.ts";
11import {
12 buildFileLinkParentSuffixByPath,
13 splitFilePathPosition,
14 fileLinkLabel,
15} from "./fileLinks.ts";
16import {
17 isMarkdownFileLinkLabel,
18 extractMarkdownLinkHrefs,
19 extractInlineCodeSpans,
20 inlineCodeFilePathCandidate,
21 resolveMarkdownFileLinkTarget,
22} from "./markdownLinks.ts";
23import { upgradeLegacyContextMessage } from "./composerContextLegacy.ts";
24import {
25 isAttachmentOnlyBootstrapPrompt,
26 parseComposerContextHref,
27} from "./composerContextReferences.ts";
28import { parseAssistantCitationHref } from "./assistantCitations.ts";
29import type { OrchestrationV2ConversationMessage } from "@t3tools/contracts";
30import { proposedPlanTitle, stripDisplayedPlanMarkdown } from "./proposedPlanText.ts";
31import { unified } from "unified";
32import remarkParse from "remark-parse";
33import remarkRehype from "remark-rehype";
34import {
35 shouldPreserveAssistantLineBreaks,
36 remarkAssistantInsights,
37 CHAT_MARKDOWN_REHYPE_PLUGINS,
38 CHAT_MARKDOWN_REMARK_PLUGINS,
39 CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS,
40 CHAT_USER_MARKDOWN_REMARK_PLUGINS,
41} from "./markdownPipeline.ts";
42
43// Inline wrappers (including Shiki token spans) must not split a search phrase.
44export const THREAD_FIND_BLOCK_TAGS = new Set([
45 "address",
46 "article",
47 "aside",
48 "blockquote",
49 "br",
50 "dd",
51 "details",
52 "div",
53 "dl",
54 "dt",
55 "figcaption",
56 "figure",
57 "footer",
58 "h1",
59 "h2",
60 "h3",
61 "h4",
62 "h5",
63 "h6",
64 "header",
65 "hr",
66 "li",
67 "main",
68 "nav",
69 "ol",
70 "p",
71 "pre",
72 "section",
73 "summary",
74 "table",
75 "tbody",
76 "td",
77 "th",
78 "thead",
79 "tr",
80 "ul",
81]);
82
83function assistantTextProcessor(lineBreaks = false, renderInsights = false) {
84 return unified()
85 .use(remarkParse)
86 .use(renderInsights ? [remarkAssistantInsights] : [])
87 .use(lineBreaks ? CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS : CHAT_MARKDOWN_REMARK_PLUGINS)
88 .use(remarkRehype, { allowDangerousHtml: true })
89 .use(CHAT_MARKDOWN_REHYPE_PLUGINS);
90}
91const assistantProcessor = assistantTextProcessor();
92const assistantBreaksProcessor = assistantTextProcessor(true);
93const insightProcessor = assistantTextProcessor(false, true);
94const insightBreaksProcessor = assistantTextProcessor(true, true);
95const userProcessor = unified()
96 .use(remarkParse)
97 .use(CHAT_MARKDOWN_REMARK_PLUGINS_WITH_BREAKS)
98 .use(CHAT_USER_MARKDOWN_REMARK_PLUGINS)
99 .use(remarkRehype, { allowDangerousHtml: true });
100
101interface TextTree {
102 readonly type: string;
103 readonly tagName?: string;
104 readonly value?: string;
105 readonly properties?: {
106 readonly href?: unknown;
107 readonly src?: unknown;
108 readonly dataInlineCode?: unknown;
109 };
110 readonly children?: ReadonlyArray<TextTree>;
111}
112
113function markdownThreadFindText(
114 markdown: string,
115 userMessage = false,
116 cwd?: string,
117 skills: readonly InlineSkill[] = [],
118 lineBreaks = false,
119 renderInsights = false,
120): string[] {
121 const processor = userMessage
122 ? userProcessor
123 : lineBreaks
124 ? assistantBreaksProcessor
125 : assistantProcessor;
126 const renderer = renderInsights
127 ? lineBreaks
128 ? insightBreaksProcessor
129 : insightProcessor
130 : processor;
131 const tree = renderer.runSync(renderer.parse(markdown), markdown);
132 const paths = [
133 ...extractMarkdownLinkHrefs(renderCodexInlineDirectivesAsMarkdown(markdown)),
134 ...extractInlineCodeSpans(markdown).flatMap((span) => inlineCodeFilePathCandidate(span) ?? []),
135 ].flatMap((href) => {
136 const target = resolveMarkdownFileLinkTarget(href, cwd);
137 return target ? [splitFilePathPosition(target).path] : [];
138 });
139 const parentSuffixes = buildFileLinkParentSuffixByPath(paths);
140 const segments: string[] = [];
141 let text = "";
142 const flush = () => {
143 if (text.trim()) segments.push(text);
144 text = "";
145 };
146 const visit = (node: TextTree, inPre = false, inlineSkills = false) => {
147 const href = node.properties?.href ?? node.properties?.src;
148 if (userMessage && typeof href === "string" && parseComposerContextHref(href)) {
149 flush();
150 return;
151 }
152 const citation =
153 node.tagName === "a" && typeof href === "string" ? parseAssistantCitationHref(href) : null;
154 if (citation) {
155 // The quote card is not searched, and it splits the text before it from its comment.
156 flush();
157 text += (citation.comment ?? "").replace(/\r?\n/g, " ");
158 return;
159 }
160 {
161 let candidate: string | null = null;
162 if (node.tagName === "a" && typeof href === "string") {
163 candidate = href;
164 } else if (
165 node.tagName === "code" &&
166 !inPre &&
167 node.properties?.dataInlineCode !== undefined
168 ) {
169 candidate = inlineCodeFilePathCandidate(
170 (node.children ?? []).map((child) => child.value ?? "").join(""),
171 );
172 }
173 const target = candidate ? resolveMarkdownFileLinkTarget(candidate, cwd) : null;
174 if (target) {
175 const labelText = (node: TextTree): string =>
176 node.value ?? (node.children ?? []).map(labelText).join("");
177 const label = labelText(node);
178 if (node.tagName === "a" && candidate && !isMarkdownFileLinkLabel(label, candidate))
179 text += `${label} `;
180 text += fileLinkLabel(splitFilePathPosition(target), parentSuffixes);
181 return;
182 }
183 }
184 const template = artifactTemplateFromHastProperties(node.properties);
185 if (template) {
186 flush();
187 segments.push(
188 template.displayName,
189 codexArtifactTemplatePresentationLabel(template.artifactKind),
190 );
191 return;
192 }
193 const block = THREAD_FIND_BLOCK_TAGS.has(node.tagName ?? "");
194 if (block) flush();
195 if (node.type === "text" || (userMessage && node.type === "raw")) {
196 let value = node.value ?? "";
197 if (inlineSkills) {
198 let cursor = 0;
199 let rendered = "";
200 for (const { start, end, skill } of matchInlineSkills(value, skills)) {
201 rendered += value.slice(cursor, start) + formatProviderSkillDisplayName(skill);
202 cursor = end;
203 }
204 value = rendered + value.slice(cursor);
205 }
206 text += inPre ? value : value.replace(/\r?\n/g, " ");
207 }
208 const renderSkills =
209 node.tagName === "code" || node.tagName === "a"
210 ? false
211 : inlineSkills || node.tagName === "p" || node.tagName === "li";
212 for (const child of node.children ?? [])
213 visit(child, inPre || node.tagName === "pre", renderSkills);
214 if (block) flush();
215 };
216 visit(tree);
217 flush();
218 return segments;
219}
220
221/** Plans render `$skill` tokens literally (no skill chips), so they are indexed as written. */
222export function searchablePlanSegments(markdown: string, cwd?: string): readonly string[] {
223 return [
224 proposedPlanTitle(markdown) ?? "Proposed plan",
225 ...markdownThreadFindText(stripDisplayedPlanMarkdown(markdown), false, cwd, []),
226 ];
227}
228
229export function searchableMessageSegments(
230 message: Pick<OrchestrationV2ConversationMessage, "role" | "text" | "streaming" | "context">,
231 cwd?: string,
232 skills: readonly InlineSkill[] = [],
233): readonly string[] | null {
234 if (message.role === "user") {
235 // Clients hide this prompt, so find must not match text nobody can see.
236 if (isAttachmentOnlyBootstrapPrompt(message.text)) return [];
237 const text = message.context ? message.text : upgradeLegacyContextMessage(message.text).text;
238 return markdownThreadFindText(text, true, cwd, skills);
239 }
240 if (message.role !== "assistant") return null;
241 return markdownThreadFindText(
242 message.text || (message.streaming ? "" : "(empty response)"),
243 false,
244 cwd,
245 skills,
246 shouldPreserveAssistantLineBreaks(message.text),
247 true,
248 );
249}