Open History to see the last commit
1// @effect-diagnostics nodeBuiltinImport:off - Knip and the TypeScript compiler host use synchronous Node paths.
2import * as NodeOS from "node:os";
3import * as NodePath from "node:path";
4import * as NodeWorkerThreads from "node:worker_threads";
5import type { Preprocessor } from "knip";
6// Knip needs the legacy compiler API, which TypeScript 7 no longer exports.
7import ts from "typescript-legacy";
8
9const schemaTypeId = "~effect/Schema/Schema";
10
11// One tsconfig's files, each with the export names its issues mention. Only
12// those names are checked: inferring the type of every export cost seconds.
13interface Project {
14 readonly cwd: string;
15 readonly configPath: string | undefined;
16 readonly files: ReadonlyArray<readonly [string, ReadonlyArray<string>]>;
17}
18
19// Each tsconfig needs its own program. The programs share parsed and bound
20// files (Effect's and Node's declarations) through one registry, as the
21// language service does. Without it, building 17 programs took about 13 s.
22const registry = ts.createDocumentRegistry();
23
24// Returns each file with those of its names that are Effect schemas.
25function findSchemas({ cwd, configPath, files }: Project) {
26 const config = configPath
27 ? ts.getParsedCommandLineOfConfigFile(
28 configPath,
29 {},
30 {
31 ...ts.sys,
32 onUnRecoverableConfigFileDiagnostic: (diagnostic) => {
33 throw new Error(ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n"));
34 },
35 },
36 )
37 : undefined;
38 if (config?.errors.length) {
39 throw new Error(
40 config.errors
41 .map((error) => ts.flattenDiagnosticMessageText(error.messageText, "\n"))
42 .join("\n"),
43 );
44 }
45 const options: ts.CompilerOptions = {
46 module: ts.ModuleKind.NodeNext,
47 allowJs: true,
48 ...config?.options,
49 noEmit: true,
50 };
51 const host = ts.createCompilerHost(options);
52 const key = registry.getKeyForCompilationSettings(options);
53 host.getSourceFile = (fileName, languageVersionOrOptions) => {
54 const text = host.readFile(fileName);
55 if (text === undefined) return undefined;
56 return registry.acquireDocumentWithKey(
57 fileName,
58 ts.sys.resolvePath(fileName) as ts.Path,
59 options,
60 key,
61 ts.ScriptSnapshot.fromString(text),
62 // Files do not change during one run.
63 "1",
64 undefined,
65 languageVersionOrOptions,
66 );
67 };
68 const program = ts.createProgram(
69 files.map(([file]) => NodePath.resolve(cwd, file)),
70 options,
71 host,
72 );
73 const checker = program.getTypeChecker();
74 return files.flatMap(([filePath, names]) => {
75 const source = program.getSourceFile(NodePath.resolve(cwd, filePath));
76 const module = source && checker.getSymbolAtLocation(source);
77 if (!source || !module) return [];
78 const schemas = checker
79 .getExportsOfModule(module)
80 .filter((symbol) => names.includes(symbol.name))
81 .flatMap((symbol) => {
82 const exported =
83 symbol.flags & ts.SymbolFlags.Alias ? checker.getAliasedSymbol(symbol) : symbol;
84 const target =
85 exported.flags & ts.SymbolFlags.Value
86 ? exported
87 : checker.resolveName(symbol.name, source, ts.SymbolFlags.Value, false);
88 if (!target) return [];
89 const type = checker.getTypeOfSymbolAtLocation(target, target.valueDeclaration ?? source);
90 const marker = type.getProperty(schemaTypeId);
91 if (!marker) return [];
92 const markerType = checker.getTypeOfSymbolAtLocation(marker, source);
93 return markerType.isStringLiteral() && markerType.value === schemaTypeId
94 ? [symbol.name]
95 : [];
96 });
97 return [[filePath, schemas] as const];
98 });
99}
100
101// The preprocessor loads this file again as each worker.
102if (!NodeWorkerThreads.isMainThread) {
103 const projects: ReadonlyArray<Project> = NodeWorkerThreads.workerData;
104 NodeWorkerThreads.parentPort?.postMessage(projects.flatMap(findSchemas));
105}
106
107// Three workers measured fastest. Each one parses Effect again, so more
108// workers cost more than they save.
109const WORKERS = Math.min(3, NodeOS.availableParallelism());
110
111// Drops unused-export issues whose every symbol is an Effect schema.
112const preprocess = async (options: Parameters<Preprocessor>[0]) => {
113 const categories = ["exports", "nsExports", "duplicates"] as const;
114 const projects = new Map<string | undefined, Map<string, Set<string>>>();
115 for (const category of categories) {
116 for (const [filePath, issues] of Object.entries(options.issues[category])) {
117 if (Object.keys(issues).length === 0) continue;
118 const configPath = ts.findConfigFile(
119 NodePath.dirname(NodePath.resolve(options.cwd, filePath)),
120 ts.sys.fileExists,
121 );
122 const files = projects.get(configPath) ?? new Map<string, Set<string>>();
123 const names = files.get(filePath) ?? new Set<string>();
124 for (const issue of Object.values(issues)) {
125 for (const { symbol } of issue.symbols ?? [{ symbol: issue.symbol }]) names.add(symbol);
126 }
127 files.set(filePath, names);
128 projects.set(configPath, files);
129 }
130 }
131
132 // Deal the projects out largest first so the workers get similar shares.
133 const shares = Array.from({ length: WORKERS }, (): Array<Project> => []);
134 [...projects]
135 .sort(([, a], [, b]) => b.size - a.size)
136 .forEach(([configPath, files], index) => {
137 shares[index % WORKERS]!.push({
138 cwd: options.cwd,
139 configPath,
140 files: [...files].map(([file, names]) => [file, [...names]]),
141 });
142 });
143 const workers = shares
144 .filter((share) => share.length > 0)
145 .map((share) => new NodeWorkerThreads.Worker(new URL(import.meta.url), { workerData: share }));
146 const results = await Promise.all(
147 workers.map(
148 (worker) =>
149 new Promise<ReturnType<typeof findSchemas>>((resolve, reject) => {
150 worker.once("message", resolve);
151 worker.once("error", reject);
152 worker.once("exit", (code) => {
153 if (code !== 0) reject(new Error(`knip-schemas worker exited with code ${code}`));
154 });
155 }),
156 ),
157 ).catch(async (error: unknown) => {
158 // knip only sets an exit code on a failed preprocessor, so stop the
159 // siblings or the failed run waits for their work.
160 await Promise.allSettled(workers.map((worker) => worker.terminate()));
161 throw error;
162 });
163
164 for (const [filePath, names] of results.flat()) {
165 for (const category of categories) {
166 const issues = options.issues[category][filePath];
167 if (!issues) continue;
168 for (const [key, issue] of Object.entries(issues)) {
169 const symbols = issue.symbols ?? [{ symbol: issue.symbol }];
170 if (symbols.length > 0 && symbols.every(({ symbol }) => names.includes(symbol))) {
171 delete issues[key];
172 options.counters[category]--;
173 }
174 }
175 if (Object.keys(issues).length === 0) delete options.issues[category][filePath];
176 }
177 }
178 return options;
179};
180
181export default preprocess;