Open History to see the last commit
1import {
2 embeddedScripts,
3 withEmbeddedScripts,
4 type SyntaxToken,
5} from "@t3tools/client-runtime/work-log/embedded-scripts";
6import type { T3Highlights } from "@t3tools/mobile-highlights-native";
7import { NitroModules } from "react-native-nitro-modules";
8
9import { highlightLanguageId } from "./highlightLanguage";
10import { HIGHLIGHTS_THEME_STYLES } from "./highlights.generated";
11
12export type HighlightScheme = "light" | "dark";
13
14export interface HighlightedToken {
15 readonly content: string;
16 readonly color: string | null;
17 readonly fontStyle: number | null;
18}
19
20export type HighlightedLines = ReadonlyArray<ReadonlyArray<HighlightedToken>>;
21
22const LINE_END = 0xffffffff;
23// Longer lines stay one plain token so a minified line does not become
24// thousands of native text runs.
25const MAX_HIGHLIGHTED_LINE_LENGTH = 1_000;
26
27// @pierre/highlights compiled to native code (modules/t3-highlights). Missing
28// from binaries built before it shipped, where code renders as plain text.
29const nativeHighlights = NitroModules.hasHybridObject("T3Highlights")
30 ? NitroModules.createHybridObject<T3Highlights>("T3Highlights")
31 : undefined;
32
33export interface TokenizedCode {
34 /** Retained UTF-16 source, compact native records, and line indexes. */
35 readonly byteLength: number;
36 readonly lineCount: number;
37 readonly highlightLine: (
38 index: number,
39 scheme: HighlightScheme,
40 ) => ReadonlyArray<HighlightedToken> | undefined;
41}
42
43/** Keep native records compact; expand only lines a client will display. */
44export function tokenizeCode(code: string, language: string): TokenizedCode {
45 const id = highlightLanguageId(language);
46 const buffer = id === 0 ? null : (nativeHighlights?.tokenize(code, id) ?? null);
47 const records = buffer ? new Uint32Array(buffer) : null;
48 const offsets = [0];
49 if (records) {
50 for (let record = 0; record < records.length; record += 2) {
51 if (records[record + 1] === LINE_END) offsets.push(record + 2);
52 }
53 } else {
54 for (let at = code.indexOf("\n"); at !== -1; at = code.indexOf("\n", at + 1))
55 offsets.push(at + 1);
56 }
57 const lineStarts = Uint32Array.from(offsets);
58 return {
59 byteLength: code.length * 2 + (records?.byteLength ?? 0) + lineStarts.byteLength,
60 lineCount: lineStarts.length,
61 highlightLine(index, scheme) {
62 if (index < 0 || index >= lineStarts.length) return undefined;
63 const first = lineStarts[index]!;
64 const next = lineStarts[index + 1];
65 if (!records) {
66 const end = next === undefined ? code.length : next - (code[next - 2] === "\r" ? 2 : 1);
67 const content = code.slice(first, end);
68 return content ? [{ content, color: null, fontStyle: null }] : [];
69 }
70 let start = first === 0 ? 0 : records[first - 2]!;
71 const last = next === undefined ? records.length : next - 2;
72 const end = last === first ? start : records[last - 2]!;
73 if (end - start > MAX_HIGHLIGHTED_LINE_LENGTH) {
74 return [{ content: code.slice(start, end), color: null, fontStyle: null }];
75 }
76 const styles = HIGHLIGHTS_THEME_STYLES[scheme];
77 const line: HighlightedToken[] = [];
78 for (let at = first; at < last; at += 2) {
79 const tokenEnd = records[at]!;
80 if (tokenEnd <= start) continue;
81 const [color, fontStyle] = styles[records[at + 1]!] ?? styles[0]!;
82 line.push({ content: code.slice(start, tokenEnd), color, fontStyle });
83 start = tokenEnd;
84 }
85 return line;
86 },
87 };
88}
89
90/** Shell commands also color the scripts they embed, such as heredocs and `python -c`. */
91export function highlightCommand(
92 code: string,
93 language: "shellscript" | "powershell",
94 scheme: HighlightScheme,
95): HighlightedLines {
96 const tokenize = (text: string, lang: string) => {
97 // A missing grammar keeps the shell colors, as on web.
98 if (highlightLanguageId(lang) === 0) throw new Error(`No grammar for ${lang}`);
99 return withOffsets(text, highlightCode(text, lang, scheme));
100 };
101 const embedded = language === "shellscript" ? embeddedScripts(code) : [];
102 return withEmbeddedScripts(code, tokenize(code, language), embedded, tokenize).map((line) =>
103 line.map((token) => ({
104 content: token.content,
105 color: token.color ?? null,
106 fontStyle: token.fontStyle ?? null,
107 })),
108 );
109}
110
111function withOffsets(code: string, lines: HighlightedLines): SyntaxToken[][] {
112 const endings = code.match(/\r?\n/gu) ?? [];
113 let offset = 0;
114 return lines.map((line, index) => {
115 const tokens = line.map((token) => {
116 const start = offset;
117 offset += token.content.length;
118 return {
119 content: token.content,
120 offset: start,
121 ...(token.color ? { color: token.color } : {}),
122 ...(token.fontStyle ? { fontStyle: token.fontStyle } : {}),
123 };
124 });
125 offset += endings[index]?.length ?? 0;
126 return tokens;
127 });
128}
129
130/** LF and CRLF split lines; a trailing line break yields a final empty line. */
131export function highlightCode(
132 code: string,
133 language: string,
134 scheme: HighlightScheme,
135): HighlightedLines {
136 const source = tokenizeCode(code, language);
137 return Array.from({ length: source.lineCount }, (_, index) =>
138 source.highlightLine(index, scheme)!,
139 );
140}