1/** Shared with turn playback so paragraph boundaries match server streaming. */
2// List fences can open at any indentation; closers may be at most three spaces deeper.
3const MARKDOWN_FENCE_PATTERN = /^( *)(`{3,}|~{3,})/;
4// CommonMark blank lines contain only spaces and tabs.
5const BLANK_LINE_PATTERN = /^[ \t]*$/;
6// Require trailing space so partial list markers cannot match during streaming.
7const LIST_ITEM_START_PATTERN = /^[ \t]*(?:[-*+]|\d{1,9}[.)])[ \t]/;
8const SECTION_TITLE_PATTERN = /^ {0,3}(?:#{1,6}(?:[ \t]|$)|\*\*(?:[^*]|\*(?!\*))+\*\*:?$)/;
9// Unindented ATX headings end preceding paragraphs and lists without blank lines; bold lines do not.
10const TOP_LEVEL_HEADING_PATTERN = /^#{1,6}(?:[ \t]|$)/;
11
12// Matches LIST_ITEM_START_PATTERN on the trimmed partial line without copying it.
13const PARTIAL_LIST_ITEM_START_PATTERN = /[ \t]*(?:[-*+]|\d{1,9}[.)])[ \t][ \t\r]*[^ \t\r]/y;
14
15export interface AssistantTextScan {
16 readonly text: string;
17 // Start of the partial line; every line before it has been scanned.
18 readonly lineStart: number;
19 readonly openFence: { readonly marker: string; readonly indent: number } | null;
20 readonly boundary: number;
21 readonly titleAwaitingContent: boolean;
22}
23
24export const EMPTY_SCAN: AssistantTextScan = {
25 text: "",
26 lineStart: 0,
27 openFence: null,
28 boundary: -1,
29 titleAwaitingContent: false,
30};
31
32function trimLineEnd(text: string, start: number, end: number): string {
33 while (end > start) {
34 const char = text.charCodeAt(end - 1);
35 if (char !== 32 && char !== 9 && char !== 13) break;
36 end--;
37 }
38 return text.slice(start, end);
39}
40
41/** Scans the terminated lines of `text`, resuming after `previous` when `text` extends it. */
42export function advanceScan(previous: AssistantTextScan, text: string): AssistantTextScan {
43 const resumed = text.startsWith(previous.text);
44 let { openFence, boundary, lineStart, titleAwaitingContent } = resumed ? previous : EMPTY_SCAN;
45 let searchFrom = resumed ? previous.text.length : 0;
46 for (;;) {
47 const newline = text.indexOf("\n", searchFrom);
48 if (newline === -1) break;
49 const line = trimLineEnd(text, lineStart, newline);
50 if (
51 openFence === null &&
52 lineStart > 0 &&
53 !titleAwaitingContent &&
54 LIST_ITEM_START_PATTERN.test(line)
55 ) {
56 boundary = lineStart;
57 }
58 const fenceMatch = MARKDOWN_FENCE_PATTERN.exec(line);
59 if (fenceMatch) {
60 const indent = fenceMatch[1]!.length;
61 const marker = fenceMatch[2]!;
62 if (openFence === null) {
63 openFence = { marker, indent };
64 titleAwaitingContent = false;
65 } else if (
66 marker[0] === openFence.marker[0] &&
67 marker.length >= openFence.marker.length &&
68 indent <= openFence.indent + 3 &&
69 line.length === indent + marker.length
70 ) {
71 // CommonMark: a closing fence carries no info string.
72 openFence = null;
73 boundary = newline + 1;
74 }
75 } else if (openFence === null && BLANK_LINE_PATTERN.test(line) && lineStart > 0) {
76 if (!titleAwaitingContent) {
77 boundary = newline + 1;
78 }
79 } else if (openFence === null) {
80 if (lineStart > 0 && !titleAwaitingContent && TOP_LEVEL_HEADING_PATTERN.test(line)) {
81 boundary = lineStart;
82 }
83 titleAwaitingContent = SECTION_TITLE_PATTERN.test(line);
84 }
85 lineStart = newline + 1;
86 searchFrom = lineStart;
87 }
88 return { text, lineStart, openFence, boundary, titleAwaitingContent };
89}
90
91// The partial line may still change, so its list item start is checked here, not kept in the scan.
92export function readyBoundary(scan: AssistantTextScan): number {
93 if (scan.openFence === null && scan.lineStart > 0 && !scan.titleAwaitingContent) {
94 PARTIAL_LIST_ITEM_START_PATTERN.lastIndex = scan.lineStart;
95 if (PARTIAL_LIST_ITEM_START_PATTERN.test(scan.text)) return scan.lineStart;
96 }
97 return scan.boundary;
98}
99
100/** Emit terminated lines only, except partial list-start lookahead for tight lists. Hold section titles until content follows. */
101export function splitBufferedAssistantText(text: string): { ready: string; rest: string } {
102 const boundary = readyBoundary(advanceScan(EMPTY_SCAN, text));
103 if (boundary === -1) {
104 return { ready: "", rest: text };
105 }
106 return { ready: text.slice(0, boundary), rest: text.slice(boundary) };
107}