1import { directiveFromMarkdown } from "mdast-util-directive";
2import { directive } from "micromark-extension-directive";
3import {
4 markdownLineEnding,
5 unicodePunctuation,
6 unicodeWhitespace,
7} from "micromark-util-character";
8import type { Construct, Extension, Tokenizer } from "micromark-util-types";
9import remarkParse from "remark-parse";
10import { unified, type Processor } from "unified";
11
12import {
13 codexArtifactTemplatePresentationLabel,
14 resolveCodexArtifactTemplate,
15 type CodexArtifactTemplate,
16} from "./codexArtifactTemplates.ts";
17import { codexFileCitationMarkdown, resolveCodexFileCitationLink } from "./codexFileCitations.ts";
18
19const COLON = 58;
20const DASH = 45;
21const UNDERSCORE = 95;
22const CODEX_FILE_CITATION_NAME = "codex-file-citation";
23const CODEX_FOLLOW_UP_NAME = "codex-followup";
24const CODEX_ARTIFACT_TEMPLATE_NAME = "artifact-template";
25
26export const CODEX_ARTIFACT_TEMPLATE_HAST_PROPERTIES = [
27 "dataCodexArtifactTemplate",
28 "dataArtifactKind",
29 "dataDisplayName",
30 "dataGalleryKind",
31 "dataSkillDirectory",
32 "dataSkillName",
33] as const;
34
35interface MarkdownPosition {
36 readonly start: { readonly offset?: number };
37 readonly end: { readonly offset?: number };
38}
39
40interface MarkdownAstNode {
41 type?: string;
42 name?: string;
43 value?: string;
44 url?: string;
45 attributes?: Readonly<Record<string, string | null>>;
46 position?: MarkdownPosition;
47 data?: {
48 codexArtifactTemplate?: CodexArtifactTemplate;
49 codexFileCitationMarkdown?: string;
50 codexFollowUp?: true;
51 hName?: string;
52 hProperties?: Record<string, unknown>;
53 };
54 children?: MarkdownAstNode[];
55}
56
57interface MarkdownFile {
58 readonly value: unknown;
59}
60
61export type CodexArtifactTemplateMarkdownSegment =
62 | { readonly kind: "markdown"; readonly markdown: string; readonly sourceOffset: number }
63 | {
64 readonly kind: "artifact-template";
65 readonly sourceOffset: number;
66 readonly template: CodexArtifactTemplate;
67 };
68
69function asConstruct(value: Construct | Construct[] | undefined, label: string): Construct {
70 const construct = Array.isArray(value) ? value[0] : value;
71 if (!construct) throw new Error(`Missing ${label} directive construct`);
72 return construct;
73}
74
75function directiveNameEnds(code: number | null): boolean {
76 return (
77 code === null ||
78 markdownLineEnding(code) ||
79 unicodeWhitespace(code) ||
80 (unicodePunctuation(code) && code !== DASH && code !== UNDERSCORE)
81 );
82}
83
84function directiveNameGate(markerCount: number, name: string): Construct {
85 const tokenize: Tokenizer = (effects, ok, nok) => {
86 let markerIndex = 0;
87 let nameIndex = 0;
88
89 return marker;
90
91 function marker(code: number | null) {
92 if (code !== COLON) return nok(code);
93 if (markerIndex === 0) effects.enter("data");
94 effects.consume(code);
95 markerIndex += 1;
96 return markerIndex === markerCount ? nameCharacter : marker;
97 }
98
99 function nameCharacter(code: number | null) {
100 if (code !== name.charCodeAt(nameIndex)) return nok(code);
101 effects.consume(code);
102 nameIndex += 1;
103 return nameIndex === name.length ? afterName : nameCharacter;
104 }
105
106 function afterName(code: number | null) {
107 effects.exit("data");
108 return directiveNameEnds(code) ? ok(code) : nok(code);
109 }
110 };
111
112 return { partial: true, tokenize };
113}
114
115function restrictedDirective(construct: Construct, markerCount: number, name: string): Construct {
116 const gate = directiveNameGate(markerCount, name);
117 return {
118 ...construct,
119 tokenize(effects, ok, nok) {
120 return effects.check(gate, construct.tokenize.call(this, effects, ok, nok), nok);
121 },
122 };
123}
124
125function codexDirectiveSyntax(): Extension {
126 const genericSyntax = directive();
127 const textDirective = asConstruct(genericSyntax.text?.[COLON], CODEX_FILE_CITATION_NAME);
128 const flowDirectives = genericSyntax.flow?.[COLON];
129 const leafDirective = Array.isArray(flowDirectives)
130 ? flowDirectives.find((construct) => construct.concrete !== true)
131 : flowDirectives;
132 if (!leafDirective)
133 throw new Error(`Missing ${CODEX_ARTIFACT_TEMPLATE_NAME} directive construct`);
134
135 return {
136 text: {
137 [COLON]: [
138 restrictedDirective(textDirective, 1, CODEX_FILE_CITATION_NAME),
139 restrictedDirective(textDirective, 1, CODEX_FOLLOW_UP_NAME),
140 ],
141 },
142 flow: {
143 [COLON]: restrictedDirective(leafDirective, 2, CODEX_ARTIFACT_TEMPLATE_NAME),
144 },
145 };
146}
147
148const CODEX_DIRECTIVE_SYNTAX = codexDirectiveSyntax();
149const CODEX_DIRECTIVE_FROM_MARKDOWN = directiveFromMarkdown();
150
151function sourceForNode(node: MarkdownAstNode, source: string): string {
152 const start = node.position?.start.offset;
153 const end = node.position?.end.offset;
154 return start === undefined || end === undefined ? "" : source.slice(start, end);
155}
156
157function textContent(node: MarkdownAstNode): string {
158 return node.value ?? (node.children ?? []).map(textContent).join("");
159}
160
161function sourceForDirective(node: MarkdownAstNode, source: string, marker: ":" | "::"): string {
162 const prefix = `${marker}${node.name ?? ""}`;
163 const slicedSource = sourceForNode(node, source);
164 if (slicedSource.startsWith(prefix)) return slicedSource;
165
166 const label = node.children?.length ? `[${textContent(node)}]` : "";
167 const attributes = Object.entries(node.attributes ?? {}).map(([name, value]) =>
168 value === null ? name : `${name}=${JSON.stringify(value)}`,
169 );
170 return `${prefix}${label}${attributes.length === 0 ? "" : `{${attributes.join(" ")}}`}`;
171}
172
173function restoreTextDirective(node: MarkdownAstNode, source: string): void {
174 node.type = "text";
175 node.value = sourceForDirective(node, source, ":");
176 delete node.name;
177 delete node.attributes;
178 delete node.url;
179 delete node.data;
180 delete node.children;
181}
182
183function restoreLeafDirective(node: MarkdownAstNode, source: string): void {
184 const value = sourceForDirective(node, source, "::");
185 node.type = "paragraph";
186 node.children = [
187 { type: "text", value, ...(node.position === undefined ? {} : { position: node.position }) },
188 ];
189 delete node.name;
190 delete node.attributes;
191 delete node.value;
192 delete node.url;
193 delete node.data;
194}
195
196function renderFileCitation(node: MarkdownAstNode, source: string, insideLink: boolean): void {
197 const citation = resolveCodexFileCitationLink(node.attributes);
198 if (!citation || insideLink) {
199 restoreTextDirective(node, source);
200 return;
201 }
202
203 node.type = "link";
204 node.url = citation.href;
205 node.children = [{ type: "text", value: citation.label }];
206 node.data = { codexFileCitationMarkdown: codexFileCitationMarkdown(citation) };
207 delete node.name;
208 delete node.attributes;
209 delete node.value;
210}
211
212function renderFollowUp(node: MarkdownAstNode, source: string): void {
213 if (!node.attributes?.prompt?.trim() || !textContent(node).trim()) {
214 restoreTextDirective(node, source);
215 return;
216 }
217
218 node.data = { codexFollowUp: true, hName: "span" };
219 delete node.name;
220 delete node.attributes;
221}
222
223function renderArtifactTemplate(node: MarkdownAstNode, source: string): void {
224 const template = resolveCodexArtifactTemplate(node.attributes);
225 if (!template) {
226 restoreLeafDirective(node, source);
227 return;
228 }
229
230 node.type = "paragraph";
231 node.children = [];
232 node.data = {
233 codexArtifactTemplate: template,
234 hName: "div",
235 hProperties: {
236 dataCodexArtifactTemplate: "true",
237 dataArtifactKind: template.artifactKind,
238 dataDisplayName: template.displayName,
239 ...(template.galleryKind === undefined ? {} : { dataGalleryKind: template.galleryKind }),
240 dataSkillDirectory: template.skillDirectory,
241 dataSkillName: template.skillName,
242 },
243 };
244 delete node.name;
245 delete node.attributes;
246 delete node.value;
247 delete node.url;
248}
249
250function transformCodexDirectives(node: MarkdownAstNode, source: string, insideLink = false): void {
251 if (node.type === "textDirective" && node.name === CODEX_FILE_CITATION_NAME) {
252 renderFileCitation(node, source, insideLink);
253 return;
254 }
255 if (node.type === "textDirective" && node.name === CODEX_FOLLOW_UP_NAME) {
256 renderFollowUp(node, source);
257 }
258 if (node.type === "leafDirective" && node.name === CODEX_ARTIFACT_TEMPLATE_NAME) {
259 renderArtifactTemplate(node, source);
260 return;
261 }
262
263 const childrenInsideLink = insideLink || node.type === "link" || node.type === "linkReference";
264 for (const child of node.children ?? []) {
265 transformCodexDirectives(child, source, childrenInsideLink);
266 }
267}
268
269function attachCodexDirectives(this: Processor) {
270 const data = this.data();
271 const micromarkExtensions = data.micromarkExtensions ?? (data.micromarkExtensions = []);
272 const fromMarkdownExtensions = data.fromMarkdownExtensions ?? (data.fromMarkdownExtensions = []);
273 micromarkExtensions.push(CODEX_DIRECTIVE_SYNTAX);
274 fromMarkdownExtensions.push(CODEX_DIRECTIVE_FROM_MARKDOWN);
275
276 return (tree: unknown, file: MarkdownFile) => {
277 transformCodexDirectives(tree as MarkdownAstNode, String(file.value));
278 };
279}
280
281export const remarkCodexDirectives = attachCodexDirectives;
282
283const directiveParser = unified().use(remarkParse).use(remarkCodexDirectives).freeze();
284
285function parseCodexMarkdown(markdown: string): MarkdownAstNode {
286 return directiveParser.runSync(directiveParser.parse(markdown), {
287 value: markdown,
288 }) as MarkdownAstNode;
289}
290
291interface DirectiveMatch {
292 readonly start: number;
293 readonly end: number;
294 readonly markdown?: string;
295 readonly template?: CodexArtifactTemplate;
296 readonly labelRange?: readonly [start: number, end: number];
297}
298
299function collectDirectiveMatches(node: MarkdownAstNode, matches: DirectiveMatch[]): void {
300 const start = node.position?.start.offset;
301 const end = node.position?.end.offset;
302 if (start !== undefined && end !== undefined) {
303 if (node.data?.codexFileCitationMarkdown !== undefined) {
304 matches.push({ start, end, markdown: node.data.codexFileCitationMarkdown });
305 return;
306 }
307 if (node.data?.codexArtifactTemplate !== undefined) {
308 matches.push({ start, end, template: node.data.codexArtifactTemplate });
309 return;
310 }
311 const labelStart = node.children?.[0]?.position?.start.offset;
312 const labelEnd = node.children?.at(-1)?.position?.end.offset;
313 if (node.data?.codexFollowUp && labelStart !== undefined && labelEnd !== undefined) {
314 matches.push({ start, end, labelRange: [labelStart, labelEnd] });
315 return;
316 }
317 }
318 for (const child of node.children ?? []) collectDirectiveMatches(child, matches);
319}
320
321function renderDirectiveMatches(
322 markdown: string,
323 replacementFor: (match: DirectiveMatch) => string | undefined,
324): string {
325 const matches: DirectiveMatch[] = [];
326 collectDirectiveMatches(parseCodexMarkdown(markdown), matches);
327 let rendered = markdown;
328 for (const match of matches.sort((left, right) => right.start - left.start)) {
329 const replacement = match.labelRange
330 ? markdown.slice(...match.labelRange)
331 : replacementFor(match);
332 if (replacement !== undefined) {
333 rendered = rendered.slice(0, match.start) + replacement + rendered.slice(match.end);
334 }
335 }
336 return rendered;
337}
338
339export function renderCodexInlineDirectivesAsMarkdown(markdown: string): string {
340 if (
341 !markdown.includes(`:${CODEX_FILE_CITATION_NAME}`) &&
342 !markdown.includes(`:${CODEX_FOLLOW_UP_NAME}`)
343 ) {
344 return markdown;
345 }
346
347 return renderDirectiveMatches(markdown, (match) => match.markdown);
348}
349
350export function renderCodexDirectivesForCopy(markdown: string): string {
351 if (
352 !markdown.includes(`:${CODEX_FILE_CITATION_NAME}`) &&
353 !markdown.includes(`:${CODEX_FOLLOW_UP_NAME}`) &&
354 !markdown.includes(`::${CODEX_ARTIFACT_TEMPLATE_NAME}`)
355 ) {
356 return markdown;
357 }
358
359 return renderDirectiveMatches(markdown, (match) => {
360 if (match.markdown !== undefined) return match.markdown;
361 if (match.template === undefined) return undefined;
362 return `${match.template.displayName} (${codexArtifactTemplatePresentationLabel(match.template.artifactKind)})`;
363 });
364}
365
366export function splitCodexArtifactTemplateMarkdown(
367 markdown: string,
368): ReadonlyArray<CodexArtifactTemplateMarkdownSegment> {
369 if (!markdown.includes(`::${CODEX_ARTIFACT_TEMPLATE_NAME}`)) {
370 return [{ kind: "markdown", markdown, sourceOffset: 0 }];
371 }
372
373 const matches: DirectiveMatch[] = [];
374 collectDirectiveMatches(parseCodexMarkdown(markdown), matches);
375 const templates = matches
376 .filter(
377 (match): match is DirectiveMatch & { readonly template: CodexArtifactTemplate } =>
378 match.template !== undefined,
379 )
380 .sort((left, right) => left.start - right.start);
381 if (templates.length === 0) {
382 return [{ kind: "markdown", markdown, sourceOffset: 0 }];
383 }
384
385 const segments: CodexArtifactTemplateMarkdownSegment[] = [];
386 let cursor = 0;
387 for (const match of templates) {
388 if (match.start > cursor) {
389 segments.push({
390 kind: "markdown",
391 markdown: markdown.slice(cursor, match.start),
392 sourceOffset: cursor,
393 });
394 }
395 segments.push({
396 kind: "artifact-template",
397 sourceOffset: match.start,
398 template: match.template,
399 });
400 cursor = match.end;
401 }
402 if (cursor < markdown.length) {
403 segments.push({ kind: "markdown", markdown: markdown.slice(cursor), sourceOffset: cursor });
404 }
405 return segments;
406}
407
408export function artifactTemplateFromHastProperties(
409 properties: Readonly<Record<string, unknown>> | null | undefined,
410): CodexArtifactTemplate | null {
411 if (properties?.dataCodexArtifactTemplate !== "true") return null;
412 const stringProperty = (name: string) => {
413 const value = properties[name];
414 return typeof value === "string" ? value : undefined;
415 };
416 return resolveCodexArtifactTemplate({
417 artifact_kind: stringProperty("dataArtifactKind"),
418 display_name: stringProperty("dataDisplayName"),
419 gallery_kind: stringProperty("dataGalleryKind"),
420 skill_directory: stringProperty("dataSkillDirectory"),
421 skill_name: stringProperty("dataSkillName"),
422 });
423}