Open History to see the last commit
1// @effect-diagnostics nodeBuiltinImport:off - readdir with file types avoids an lstat per child
2import * as NodeFSP from "node:fs/promises";
3import * as NodeModule from "node:module";
4
5import type {
6 DirItem,
7 FileItem,
8 FileFinder as FileFinderType,
9 GrepCursor,
10 MixedItem,
11 Result,
12 WatchEvent,
13 WatchUnsubscribe,
14} from "@ff-labs/fff-node";
15import * as Clock from "effect/Clock";
16import * as Context from "effect/Context";
17import * as Deferred from "effect/Deferred";
18import * as Effect from "effect/Effect";
19import * as FileSystem from "effect/FileSystem";
20import * as Layer from "effect/Layer";
21import * as LayerMap from "effect/LayerMap";
22import * as Option from "effect/Option";
23import * as Path from "effect/Path";
24import * as Schema from "effect/Schema";
25
26import type {
27 ProjectEntry,
28 ProjectEntryKind,
29 ProjectListEntriesResult,
30 ProjectSearchContentsInput,
31 ProjectSearchContentsResult,
32 ProjectSearchEntriesResult,
33} from "@t3tools/contracts";
34import * as HostProcess from "@t3tools/shared/HostProcess";
35import { isProjectFaviconPath } from "@t3tools/shared/projectFavicon";
36import { scoreSubsequenceMatch } from "@t3tools/shared/searchRanking";
37
38import * as VcsProcess from "../vcs/VcsProcess.ts";
39import { type IgnoreDecision, type IgnoreFile, parseIgnoreFile } from "./IgnorePatterns.ts";
40
41// fff-node's native library stays external; single-executables require disk packages instead of importing them.
42const requireForFff = NodeModule.createRequire(import.meta.url);
43const { FileFinder } = requireForFff("@ff-labs/fff-node") as typeof import("@ff-labs/fff-node");
44
45const WORKSPACE_INDEX_MAX_ENTRIES = 25_000;
46const WORKSPACE_INDEX_PAGE_SIZE = WORKSPACE_INDEX_MAX_ENTRIES + 2;
47const WORKSPACE_INDEX_SCAN_TIMEOUT = "15 seconds";
48const WORKSPACE_INDEX_SCAN_TIMEOUT_MS = 15_000;
49const WORKSPACE_INDEX_IDLE_TTL = "15 minutes";
50const CONTENT_SEARCH_TIME_BUDGET_MS = 250;
51const CONTENT_SEARCH_MAX_MATCHES_PER_FILE = 100;
52const SYMLINK_SCAN_CONCURRENCY = 16;
53// A scan no watcher can invalidate is reused this long, so typing does not rescan every folder.
54const UNWATCHED_SYMLINK_SCAN_TTL_MS = 2_000;
55// A rescan that outlasts this leaves the read on the previous scan.
56const SYMLINK_RESCAN_WAIT = "5 millis";
57// Each page is a synchronous native call; small pages let searches run between them.
58const SYMLINK_DIRECTORY_PAGE_SIZE = 1_000;
59
60export class WorkspaceSearchIndexCreateFailed extends Schema.TaggedError<WorkspaceSearchIndexCreateFailed>()(
61 "WorkspaceSearchIndexCreateFailed",
62 {
63 cwd: Schema.String,
64 reason: Schema.String,
65 cause: Schema.optional(Schema.Defect()),
66 },
67) {
68 override get message(): string {
69 return `Failed to create the workspace search index for '${this.cwd}'.`;
70 }
71}
72
73export class WorkspaceSearchIndexScanTimedOut extends Schema.TaggedError<WorkspaceSearchIndexScanTimedOut>()(
74 "WorkspaceSearchIndexScanTimedOut",
75 {
76 cwd: Schema.String,
77 timeout: Schema.String,
78 },
79) {
80 override get message(): string {
81 return `Workspace search index for '${this.cwd}' did not finish scanning within ${this.timeout}`;
82 }
83}
84
85export class WorkspaceSearchIndexSearchFailed extends Schema.TaggedError<WorkspaceSearchIndexSearchFailed>()(
86 "WorkspaceSearchIndexSearchFailed",
87 {
88 cwd: Schema.String,
89 queryLength: Schema.Number,
90 pageSize: Schema.Number,
91 reason: Schema.String,
92 cause: Schema.optional(Schema.Defect()),
93 },
94) {
95 override get message(): string {
96 return `Workspace search failed for '${this.cwd}'.`;
97 }
98}
99
100export class WorkspaceSearchIndexRefreshFailed extends Schema.TaggedError<WorkspaceSearchIndexRefreshFailed>()(
101 "WorkspaceSearchIndexRefreshFailed",
102 {
103 cwd: Schema.String,
104 reason: Schema.String,
105 cause: Schema.optional(Schema.Defect()),
106 },
107) {
108 override get message(): string {
109 return `Failed to refresh the workspace search index for '${this.cwd}'.`;
110 }
111}
112
113export class WorkspaceSearchIndexDestroyFailed extends Schema.TaggedError<WorkspaceSearchIndexDestroyFailed>()(
114 "WorkspaceSearchIndexDestroyFailed",
115 {
116 cwd: Schema.String,
117 cause: Schema.Defect(),
118 },
119) {
120 override get message(): string {
121 return `Failed to destroy the workspace search index for '${this.cwd}'.`;
122 }
123}
124
125export class WorkspaceSearchIndex extends Context.Service<
126 WorkspaceSearchIndex,
127 {
128 readonly list: () => Effect.Effect<ProjectListEntriesResult, WorkspaceSearchIndexSearchFailed>;
129 readonly search: (
130 query: string,
131 limit: number,
132 kind?: ProjectEntryKind,
133 imageOnly?: boolean,
134 ) => Effect.Effect<ProjectSearchEntriesResult, WorkspaceSearchIndexSearchFailed>;
135 readonly searchContents: (
136 input: Omit<ProjectSearchContentsInput, "cwd">,
137 ) => Effect.Effect<ProjectSearchContentsResult, WorkspaceSearchIndexSearchFailed>;
138 readonly refresh: () => Effect.Effect<void, WorkspaceSearchIndexRefreshFailed>;
139 }
140>()("t3/workspace/WorkspaceSearchIndex") {}
141
142// Only the host separator becomes `/`: on POSIX `\` is a file name character.
143function toPosixPath(input: string, hostIsWindows: boolean): string {
144 return hostIsWindows ? input.replaceAll("\\", "/") : input;
145}
146
147function trimDirectorySeparator(input: string): string {
148 return input.endsWith("/") ? input.slice(0, -1) : input;
149}
150
151function parentPathOf(input: string): string | undefined {
152 const separatorIndex = input.lastIndexOf("/");
153 return separatorIndex === -1 ? undefined : input.slice(0, separatorIndex);
154}
155
156function toProjectEntry(item: MixedItem, hostIsWindows: boolean): ProjectEntry | null {
157 const normalizedPath = trimDirectorySeparator(toPosixPath(item.item.relativePath, hostIsWindows));
158 if (!normalizedPath) {
159 return null;
160 }
161
162 return {
163 path: normalizedPath,
164 kind: item.type,
165 };
166}
167
168function toFileEntry(item: FileItem, hostIsWindows: boolean): ProjectEntry | null {
169 const normalizedPath = trimDirectorySeparator(toPosixPath(item.relativePath, hostIsWindows));
170 return normalizedPath ? { path: normalizedPath, kind: "file" } : null;
171}
172
173function toDirectoryEntry(item: DirItem, hostIsWindows: boolean): ProjectEntry | null {
174 const normalizedPath = trimDirectorySeparator(toPosixPath(item.relativePath, hostIsWindows));
175 return normalizedPath ? { path: normalizedPath, kind: "directory" } : null;
176}
177
178type SearchPage<I> = { readonly items: ReadonlyArray<I>; readonly totalMatched: number };
179
180function mapFileSearchResult(
181 result: SearchPage<FileItem>,
182 limit: number,
183 hostIsWindows: boolean,
184 imageOnly = false,
185): ProjectSearchEntriesResult {
186 const entries = result.items.flatMap((item) => {
187 const entry = toFileEntry(item, hostIsWindows);
188 return entry && (!imageOnly || isProjectFaviconPath(entry.path)) ? [entry] : [];
189 });
190 return {
191 entries: entries.slice(0, limit),
192 truncated: entries.length > limit || result.totalMatched > result.items.length,
193 };
194}
195
196function mapDirectorySearchResult(
197 result: SearchPage<DirItem>,
198 limit: number,
199 hostIsWindows: boolean,
200): ProjectSearchEntriesResult {
201 const entries = result.items.flatMap((item) => {
202 const entry = toDirectoryEntry(item, hostIsWindows);
203 return entry ? [entry] : [];
204 });
205 const rootDirectoryCount = result.items.some((item) => item.relativePath.length === 0) ? 1 : 0;
206 return {
207 entries: entries.slice(0, limit),
208 truncated: result.totalMatched - rootDirectoryCount > limit,
209 };
210}
211
212function mapMixedSearchResult(
213 result: SearchPage<MixedItem>,
214 limit: number,
215 hostIsWindows: boolean,
216): { readonly entries: ProjectEntry[]; readonly truncated: boolean } {
217 const entries: ProjectEntry[] = [];
218 for (const item of result.items) {
219 const entry = toProjectEntry(item, hostIsWindows);
220 if (entry) {
221 entries.push(entry);
222 }
223 if (entries.length >= limit) {
224 break;
225 }
226 }
227
228 const rootDirectoryCount = result.items.some(
229 (item) => item.type === "directory" && item.item.relativePath.length === 0,
230 )
231 ? 1
232 : 0;
233 return {
234 entries,
235 truncated: result.totalMatched - rootDirectoryCount > limit,
236 };
237}
238
239const WORD_CHARACTER = /[\p{Letter}\p{Mark}\p{Number}_]/u;
240
241function codePointAt(line: string, index: number): string | undefined {
242 const codePoint = line.codePointAt(index);
243 return codePoint === undefined ? undefined : String.fromCodePoint(codePoint);
244}
245
246function codePointBefore(line: string, index: number): string | undefined {
247 if (index <= 0) return undefined;
248 const previousCodeUnit = line.charCodeAt(index - 1);
249 const previousIndex =
250 previousCodeUnit >= 0xdc00 && previousCodeUnit <= 0xdfff ? index - 2 : index - 1;
251 return codePointAt(line, previousIndex);
252}
253
254function buildContentSearchQuery(input: Omit<ProjectSearchContentsInput, "cwd">): {
255 readonly searchQuery: string;
256 readonly regexMode: boolean;
257} {
258 if (input.caseSensitive) {
259 return { searchQuery: input.query, regexMode: input.useRegex };
260 }
261 return input.useRegex
262 ? { searchQuery: `(?i)${input.query}`, regexMode: true }
263 : { searchQuery: input.query.toLowerCase(), regexMode: false };
264}
265
266function mapContentMatchRanges(
267 line: string,
268 byteRanges: ReadonlyArray<readonly [number, number]>,
269): Array<{ readonly start: number; readonly end: number }> {
270 const lineBytes = Buffer.from(line);
271 const toStringIndex = (byteOffset: number) => lineBytes.subarray(0, byteOffset).toString().length;
272 return byteRanges.map(([startByte, endByte]) => ({
273 start: toStringIndex(startByte),
274 end: toStringIndex(endByte),
275 }));
276}
277
278// Post-filter whole words: consuming boundary regexes swallow adjacent separators and widen ranges; \b misses punctuation edges.
279function isWholeWordRange(
280 line: string,
281 range: { readonly start: number; readonly end: number },
282): boolean {
283 if (range.end <= range.start) return false;
284 const isWord = (character: string | undefined) =>
285 character !== undefined && WORD_CHARACTER.test(character);
286 const leftIsBoundary =
287 range.start === 0 ||
288 !isWord(codePointBefore(line, range.start)) ||
289 !isWord(codePointAt(line, range.start));
290 const rightIsBoundary =
291 range.end >= line.length ||
292 !isWord(codePointAt(line, range.end)) ||
293 !isWord(codePointBefore(line, range.end));
294 return leftIsBoundary && rightIsBoundary;
295}
296
297function withDirectoryAncestors(entries: ReadonlyArray<ProjectEntry>): ProjectEntry[] {
298 const entryByPath = new Map(entries.map((entry) => [entry.path, entry]));
299 for (const entry of entries) {
300 let parentPath = parentPathOf(entry.path);
301 while (parentPath) {
302 if (!entryByPath.has(parentPath)) {
303 entryByPath.set(parentPath, { path: parentPath, kind: "directory" });
304 }
305 parentPath = parentPathOf(parentPath);
306 }
307 }
308 return [...entryByPath.values()];
309}
310
311interface ScannedDirectory {
312 readonly directory: string;
313 readonly links: ReadonlyArray<ProjectEntry>;
314 /** Every symlink in the folder, including ones dropped for leaving the workspace. */
315 readonly linkPaths: ReadonlyArray<string>;
316 /** A nested repository (or the workspace's own), whose ignore rules own what is below it. */
317 readonly isRepository: boolean;
318 readonly ignoreFile: string | undefined;
319}
320
321const joinEntryPath = (directory: string, name: string) =>
322 directory ? `${directory}/${name}` : name;
323
324/**
325 * fff skips symlinks, and `followSymlinks` drops file links, dedupes folder links, and walks out
326 * of the workspace. So links are read from the folders fff already indexed (ignore rules applied),
327 * without descending into them, and kept only when they resolve inside the workspace.
328 */
329const scanDirectory = Effect.fn("WorkspaceSearchIndex.scanDirectory")(function* (
330 fileSystem: FileSystem.FileSystem,
331 path: Path.Path,
332 cwd: string,
333 root: string,
334 directory: string,
335) {
336 // Effect's readDirectory returns bare names; one lstat per child would cost far more.
337 const children = yield* Effect.promise(() =>
338 NodeFSP.readdir(path.join(cwd, directory), { withFileTypes: true }).catch(() => []),
339 );
340 const links: ProjectEntry[] = [];
341 const linkPaths: string[] = [];
342 let isRepository = false;
343 let ignoreFile: string | undefined;
344 for (const child of children) {
345 if (child.name === ".git") isRepository = true;
346 if (child.name === ".ignore" && child.isFile()) {
347 ignoreFile = yield* fileSystem
348 .readFileString(path.join(cwd, directory, child.name))
349 .pipe(Effect.orElseSucceed(() => undefined));
350 }
351 if (!child.isSymbolicLink()) continue;
352 const entryPath = joinEntryPath(directory, child.name);
353 linkPaths.push(entryPath);
354 const absolutePath = path.join(cwd, entryPath);
355 const target = yield* Effect.all([
356 fileSystem.realPath(absolutePath),
357 fileSystem.stat(absolutePath),
358 ]).pipe(Effect.orElseSucceed(() => undefined));
359 if (!target) continue;
360 const relative = path.relative(root, target[0]);
361 if (
362 relative === ".." ||
363 relative.startsWith(`..${path.sep}`) ||
364 path.isAbsolute(relative) ||
365 relative.split(path.sep).includes(".git") ||
366 (target[1].type !== "Directory" && target[1].type !== "File")
367 ) {
368 continue;
369 }
370 links.push({ path: entryPath, kind: target[1].type === "Directory" ? "directory" : "file" });
371 }
372 return { directory, links, linkPaths, isRepository, ignoreFile } satisfies ScannedDirectory;
373});
374
375/** Whether `entryPath` is `directory` itself or below it. */
376const isWithin = (entryPath: string, directory: string) =>
377 directory === "" || entryPath.startsWith(`${directory}/`);
378
379/**
380 * fff reads `.ignore` files alongside git's rules, and like ripgrep the nearest one that matches
381 * decides, ahead of any `.gitignore`. `ignoreFiles` is ordered deepest folder first.
382 */
383function decideByIgnoreFiles(
384 entry: ProjectEntry,
385 ignoreFiles: ReadonlyArray<{ readonly directory: string; readonly file: IgnoreFile }>,
386): IgnoreDecision | undefined {
387 for (const { directory, file } of ignoreFiles) {
388 if (!isWithin(entry.path, directory) || entry.path === directory) continue;
389 const relative = directory ? entry.path.slice(directory.length + 1) : entry.path;
390 // Like git and ripgrep, a link is never a folder to a folder-only rule, whatever it targets.
391 const decision = file.decide(relative, false);
392 if (decision) return decision;
393 }
394 return undefined;
395}
396
397/** Places each symlink before the first index result that matches the query less closely. */
398function mergeSymlinkMatches(
399 indexed: ProjectSearchEntriesResult,
400 linkPaths: ReadonlySet<string>,
401 symlinks: ReadonlyArray<ProjectEntry>,
402 query: string,
403 limit: number,
404): ProjectSearchEntriesResult {
405 if (linkPaths.size === 0) return indexed;
406 // fff's watcher indexes links created after its scan, under a kind of its own choosing, so a
407 // path it already returned takes the link's kind (or drops out of a kind-filtered search).
408 const symlinkByPath = new Map(symlinks.map((entry) => [entry.path, entry]));
409 const score = (entry: ProjectEntry) =>
410 scoreSubsequenceMatch(entry.path.toLowerCase(), query) ?? Number.POSITIVE_INFINITY;
411 const indexedPaths = new Set(indexed.entries.map((entry) => entry.path));
412 const pending = symlinks
413 .flatMap((entry) => {
414 const entryScore = score(entry);
415 return Number.isFinite(entryScore) && !indexedPaths.has(entry.path)
416 ? [{ entry, score: entryScore }]
417 : [];
418 })
419 .toSorted((left, right) => left.score - right.score);
420 const entries: ProjectEntry[] = [];
421 let nextPending = 0;
422 for (const indexedEntry of indexed.entries) {
423 const entry = linkPaths.has(indexedEntry.path)
424 ? symlinkByPath.get(indexedEntry.path)
425 : indexedEntry;
426 if (!entry) continue;
427 const entryScore = score(entry);
428 while (nextPending < pending.length && pending[nextPending]!.score < entryScore) {
429 entries.push(pending[nextPending++]!.entry);
430 }
431 entries.push(entry);
432 }
433 for (; nextPending < pending.length; nextPending += 1) entries.push(pending[nextPending]!.entry);
434 return {
435 entries: entries.slice(0, limit),
436 truncated: indexed.truncated || entries.length > limit,
437 };
438}
439
440// fff applies .gitignore files only; Git also ignores paths from .git/info/exclude and the global excludes file.
441const listGitIgnoredPaths = Effect.fn("WorkspaceSearchIndex.listGitIgnoredPaths")(function* (
442 vcsProcess: VcsProcess.VcsProcess["Service"],
443 cwd: string,
444 hostIsWindows: boolean,
445) {
446 const result = yield* vcsProcess
447 .run({
448 operation: "WorkspaceSearchIndex.listGitIgnoredPaths",
449 command: "git",
450 args: [
451 "-c",
452 "core.fsmonitor=false",
453 "ls-files",
454 "--others",
455 "--ignored",
456 "--exclude-standard",
457 "--directory",
458 "-z",
459 ],
460 cwd,
461 allowNonZeroExit: true,
462 timeoutMs: 10_000,
463 maxOutputBytes: 16 * 1024 * 1024,
464 })
465 .pipe(Effect.orElseSucceed(() => undefined));
466 if (!result || result.exitCode !== 0 || result.stdoutTruncated) return new Set<string>();
467 return new Set(
468 result.stdout
469 .split("\0")
470 .filter((entry) => entry.length > 0)
471 .map((entry) => trimDirectorySeparator(toPosixPath(entry, hostIsWindows))),
472 );
473});
474
475function mixedItemPath(item: MixedItem): string {
476 return item.item.relativePath;
477}
478
479function isIgnoredPath(
480 ignoredPaths: ReadonlySet<string>,
481 relativePath: string,
482 hostIsWindows: boolean,
483): boolean {
484 if (ignoredPaths.size === 0) return false;
485 let candidate: string | undefined = trimDirectorySeparator(
486 toPosixPath(relativePath, hostIsWindows),
487 );
488 while (candidate) {
489 if (ignoredPaths.has(candidate)) return true;
490 candidate = parentPathOf(candidate);
491 }
492 return false;
493}
494
495const createFinder = Effect.fn("WorkspaceSearchIndex.createFinder")(function* (
496 cwd: string,
497 variant: WorkspaceSearchIndexVariant,
498) {
499 const result = yield* Effect.try({
500 try: () =>
501 FileFinder.create({
502 basePath: cwd,
503 disableMmapCache: true,
504 // Content indexing costs CPU/memory; path-only consumers use the lightweight index.
505 disableContentIndexing: variant !== "content",
506 aiMode: false,
507 }),
508 catch: (cause) =>
509 new WorkspaceSearchIndexCreateFailed({
510 cwd,
511 reason: "FileFinder.create threw unexpectedly.",
512 cause,
513 }),
514 });
515 if (result.ok) return result.value;
516 return yield* new WorkspaceSearchIndexCreateFailed({
517 cwd,
518 reason: result.error,
519 });
520});
521
522const waitForIndexReady = Effect.fn("WorkspaceSearchIndex.waitForIndexReady")(function* <E>(
523 cwd: string,
524 finder: FileFinderType,
525 onFailure: (input: { readonly reason: string; readonly cause?: unknown }) => E,
526): Effect.fn.Return<void, E | WorkspaceSearchIndexScanTimedOut> {
527 const result = yield* Effect.tryPromise({
528 try: () => finder.waitForIndexReady(WORKSPACE_INDEX_SCAN_TIMEOUT_MS),
529 catch: (cause) =>
530 onFailure({
531 reason: "FileFinder.waitForIndexReady rejected unexpectedly.",
532 cause,
533 }),
534 });
535 if (!result.ok) {
536 return yield* Effect.fail(onFailure({ reason: result.error }));
537 }
538 if (!result.value) {
539 return yield* new WorkspaceSearchIndexScanTimedOut({
540 cwd,
541 timeout: WORKSPACE_INDEX_SCAN_TIMEOUT,
542 });
543 }
544});
545
546export const make = Effect.fn("WorkspaceSearchIndex.make")(function* (
547 cwd: string,
548 variant: WorkspaceSearchIndexVariant = "paths",
549) {
550 const hostIsWindows = yield* HostProcess.isWindows;
551 const fileSystem = yield* FileSystem.FileSystem;
552 const path = yield* Path.Path;
553 const finder = yield* Effect.acquireRelease(createFinder(cwd, variant), (finder) =>
554 Effect.try({
555 try: () => finder.destroy(),
556 catch: (cause) => new WorkspaceSearchIndexDestroyFailed({ cwd, cause }),
557 }).pipe(Effect.orDie),
558 );
559 let initialScanTimedOut = false;
560 const vcsProcess = yield* VcsProcess.VcsProcess;
561 let ignoredPaths = yield* listGitIgnoredPaths(vcsProcess, cwd, hostIsWindows);
562 yield* waitForIndexReady(
563 cwd,
564 finder,
565 ({ reason, cause }) =>
566 new WorkspaceSearchIndexCreateFailed({
567 cwd,
568 reason,
569 cause,
570 }),
571 ).pipe(
572 Effect.catchTags({
573 WorkspaceSearchIndexScanTimedOut: (error) =>
574 variant === "paths"
575 ? Effect.sync(() => {
576 initialScanTimedOut = true;
577 })
578 : Effect.fail(error),
579 }),
580 );
581
582 // fff's isScanning takes the picker lock too, so progress is read off the JS thread.
583 const isScanning = Effect.promise(async () => {
584 try {
585 const progress = await finder.getScanProgressAsync();
586 return !progress.ok || progress.value.isScanning;
587 } catch {
588 return true;
589 }
590 });
591 const hasIncompleteInitialScan = () => (initialScanTimedOut ? isScanning : Effect.succeed(false));
592
593 const runSearch = Effect.fn("WorkspaceSearchIndex.runSearch")(function* <A>(
594 query: string,
595 pageSize: number,
596 operation: "directorySearch" | "fileSearch" | "grep" | "mixedSearch",
597 execute: () => Promise<Result<A>>,
598 ): Effect.fn.Return<A, WorkspaceSearchIndexSearchFailed> {
599 // fff's searches wait on its picker lock, which its watcher can hold for minutes.
600 const result = yield* Effect.tryPromise({
601 try: execute,
602 catch: (cause) =>
603 new WorkspaceSearchIndexSearchFailed({
604 cwd,
605 queryLength: query.length,
606 pageSize,
607 reason: `FileFinder.${operation} rejected unexpectedly.`,
608 cause,
609 }),
610 });
611 if (!result.ok) {
612 return yield* new WorkspaceSearchIndexSearchFailed({
613 cwd,
614 queryLength: query.length,
615 pageSize,
616 reason: result.error,
617 });
618 }
619 return result.value;
620 });
621
622 const root = yield* fileSystem.realPath(cwd).pipe(Effect.orElseSucceed(() => cwd));
623
624 const listIndexedDirectories = Effect.fn("WorkspaceSearchIndex.listIndexedDirectories")(
625 function* () {
626 const directories: string[] = [];
627 // fff's pageIndex is an offset into the results, not a page number.
628 for (let pageIndex = 0; ; pageIndex += SYMLINK_DIRECTORY_PAGE_SIZE) {
629 const result = yield* Effect.promise(async () => {
630 try {
631 return await finder.directorySearchAsync("", {
632 pageIndex,
633 pageSize: SYMLINK_DIRECTORY_PAGE_SIZE,
634 });
635 } catch {
636 return undefined;
637 }
638 });
639 if (!result?.ok) return directories;
640 for (const item of result.value.items) {
641 const directory = trimDirectorySeparator(toPosixPath(item.relativePath, hostIsWindows));
642 if (directory) directories.push(directory);
643 }
644 if (result.value.items.length < SYMLINK_DIRECTORY_PAGE_SIZE) return directories;
645 }
646 },
647 );
648
649 /** Ignore rules belong to the repository that owns a path, so each one answers for its own. */
650 const listGitIgnored = Effect.fn("WorkspaceSearchIndex.listGitIgnored")(function* (
651 repository: string,
652 entries: ReadonlyArray<ProjectEntry>,
653 ) {
654 const relativePaths = entries.map((entry) =>
655 repository ? entry.path.slice(repository.length + 1) : entry.path,
656 );
657 const repositoryCwd = path.join(cwd, repository);
658 const result = yield* vcsProcess
659 .run({
660 operation: "WorkspaceSearchIndex.listGitIgnored",
661 command: "git",
662 // fff applies ignore rules to tracked paths too, so the index is not consulted.
663 args: ["-c", "core.fsmonitor=false", "check-ignore", "--no-index", "-z", "--stdin"],
664 cwd: repositoryCwd,
665 stdin: `${relativePaths.join("\0")}\0`,
666 allowNonZeroExit: true,
667 timeoutMs: 10_000,
668 maxOutputBytes: 16 * 1024 * 1024,
669 })
670 .pipe(Effect.orElseSucceed(() => undefined));
671 // Exit 1 means nothing matched.
672 if (result?.exitCode === 1) return [];
673 if (result?.exitCode !== 0 || result.stdoutTruncated) {
674 // Outside git no rules apply (git exits before reading stdin, so the cause is lost).
675 // Any other failure hides the repository's links rather than leaking ones it ignores.
676 const insideRepository = yield* vcsProcess
677 .run({
678 operation: "WorkspaceSearchIndex.listGitIgnored",
679 command: "git",
680 args: ["rev-parse", "--git-dir"],
681 cwd: repositoryCwd,
682 timeoutMs: 10_000,
683 })
684 .pipe(
685 Effect.as(true),
686 Effect.orElseSucceed(() => false),
687 );
688 return insideRepository ? entries.map((entry) => entry.path) : [];
689 }
690 return result.stdout
691 .split("\0")
692 .filter(Boolean)
693 .map((relativePath) => (repository ? `${repository}/${relativePath}` : relativePath));
694 });
695
696 const scanSymlinkEntries = Effect.fn("WorkspaceSearchIndex.scanSymlinkEntries")(function* () {
697 const indexComplete = !(yield* isScanning);
698 const directories = yield* listIndexedDirectories();
699 const scanned = yield* Effect.forEach(
700 ["", ...directories],
701 (directory) => scanDirectory(fileSystem, path, cwd, root, directory),
702 { concurrency: SYMLINK_SCAN_CONCURRENCY },
703 );
704 const candidates = scanned.flatMap((directory) => directory.links);
705 const linkPaths = new Set(scanned.flatMap((directory) => directory.linkPaths));
706 if (candidates.length === 0) return { entries: candidates, linkPaths, indexComplete };
707
708 const ignoreFiles = scanned
709 .flatMap(({ directory, ignoreFile }) =>
710 ignoreFile === undefined ? [] : [{ directory, file: parseIgnoreFile(ignoreFile) }],
711 )
712 .toSorted((left, right) => right.directory.length - left.directory.length);
713 const decisions = new Map(
714 candidates.map((entry) => [entry.path, decideByIgnoreFiles(entry, ignoreFiles)]),
715 );
716 const repositories = [
717 "",
718 ...scanned.flatMap(({ directory, isRepository }) =>
719 isRepository && directory ? [directory] : [],
720 ),
721 ].toSorted((left, right) => right.length - left.length);
722 const entriesByRepository = Map.groupBy(
723 candidates.filter((entry) => decisions.get(entry.path) === undefined),
724 (entry) =>
725 repositories.find(
726 (repository) => isWithin(entry.path, repository) && entry.path !== repository,
727 ) ?? "",
728 );
729 const ignored = new Set(
730 (yield* Effect.forEach(
731 entriesByRepository,
732 ([repository, entries]) => listGitIgnored(repository, entries),
733 { concurrency: 4 },
734 )).flat(),
735 );
736 return {
737 entries: candidates.filter((entry) =>
738 decisions.get(entry.path) === undefined
739 ? !ignored.has(entry.path)
740 : decisions.get(entry.path) === "included",
741 ),
742 linkPaths,
743 indexComplete,
744 };
745 });
746
747 // One scan runs at a time and outlives the reads that started it. Only the first read waits for
748 // it in full: after that a change or expiry marks the result stale, and reads wait briefly for
749 // the rescan before answering from the last finished one, so file churn cannot stall search.
750 type SymlinkScan = Effect.Success<ReturnType<typeof scanSymlinkEntries>>;
751 let lastSymlinkScan: SymlinkScan | undefined;
752 let lastSymlinkScanExpiresAt: number | undefined;
753 let pendingSymlinkScan: Deferred.Deferred<SymlinkScan> | undefined;
754 let symlinkScanStale = false;
755 let knownLinkPaths: ReadonlySet<string> = new Set();
756 const invalidateSymlinkScan = () => {
757 symlinkScanStale = true;
758 };
759 const isKnownLink = (absolutePath: string) =>
760 [cwd, root].some((base) =>
761 knownLinkPaths.has(toPosixPath(path.relative(base, absolutePath), hostIsWindows)),
762 );
763 const onWatchBatch = (events: ReadonlyArray<WatchEvent>) => {
764 // fff reports a link replaced in place as `modified`, so edits to known links count too.
765 const editsOnly = events.every(
766 (event) =>
767 event.kind === "modified" &&
768 !/(^|[\\/])\.(git)?ignore$/.test(event.path) &&
769 !isKnownLink(event.path),
770 );
771 if (!editsOnly) invalidateSymlinkScan();
772 };
773 // fff refuses subscriptions until its watcher is up, which can trail the initial scan, so
774 // each read retries until one is in place.
775 let unwatch: WatchUnsubscribe | undefined;
776 yield* Effect.addFinalizer(() => Effect.sync(() => unwatch?.()));
777 const ensureWatching = () => {
778 if (unwatch) return true;
779 try {
780 const result = finder.watch(onWatchBatch);
781 if (result.ok) unwatch = result.value;
782 } catch {
783 // Not ready yet; the next read tries again.
784 }
785 return unwatch !== undefined;
786 };
787 const indexScope = yield* Effect.scope;
788 const startSymlinkScan = Effect.fn("WorkspaceSearchIndex.startSymlinkScan")(function* () {
789 symlinkScanStale = false;
790 // Subscribing before the scan starts means no change during it goes unseen.
791 const watched = ensureWatching();
792 const done = yield* Deferred.make<SymlinkScan>();
793 pendingSymlinkScan = done;
794 yield* scanSymlinkEntries().pipe(
795 Effect.tap((result) =>
796 Effect.gen(function* () {
797 lastSymlinkScan = result;
798 knownLinkPaths = result.linkPaths;
799 // Without a watcher, or before fff has seen every folder, the result is kept only briefly.
800 lastSymlinkScanExpiresAt =
801 watched && result.indexComplete
802 ? undefined
803 : (yield* Clock.currentTimeMillis) + UNWATCHED_SYMLINK_SCAN_TTL_MS;
804 }),
805 ),
806 Effect.onExit((exit) =>
807 Effect.sync(() => {
808 pendingSymlinkScan = undefined;
809 }).pipe(Effect.andThen(Deferred.done(done, exit))),
810 ),
811 Effect.forkIn(indexScope),
812 );
813 return done;
814 });
815 const listSymlinkEntries = Effect.fn("WorkspaceSearchIndex.listSymlinkEntries")(function* () {
816 if (
817 lastSymlinkScanExpiresAt !== undefined &&
818 (yield* Clock.currentTimeMillis) >= lastSymlinkScanExpiresAt
819 ) {
820 symlinkScanStale = true;
821 }
822 const pending =
823 pendingSymlinkScan ??
824 (lastSymlinkScan === undefined || symlinkScanStale ? yield* startSymlinkScan() : undefined);
825 let result = lastSymlinkScan;
826 if (pending && result === undefined) {
827 result = yield* Deferred.await(pending);
828 } else if (pending) {
829 const fresh = yield* Deferred.await(pending).pipe(Effect.timeoutOption(SYMLINK_RESCAN_WAIT));
830 if (Option.isSome(fresh)) result = fresh.value;
831 }
832 if (result === undefined)
833 return yield* Effect.die("The symlink scan finished without a result.");
834 return {
835 // Git's ignore rules hide a link the same way searchVisible hides the files it indexes.
836 entries: result.entries.filter(
837 (entry) => !isIgnoredPath(ignoredPaths, entry.path, hostIsWindows),
838 ),
839 linkPaths: result.linkPaths,
840 };
841 });
842
843 // Grows the first page until hidden entries can no longer crowd visible ones out of it.
844 const searchVisible = Effect.fn("WorkspaceSearchIndex.searchVisible")(function* <I>(
845 query: string,
846 pageSize: number,
847 operation: "directorySearch" | "fileSearch" | "mixedSearch",
848 execute: (pageSize: number) => Promise<Result<SearchPage<I>>>,
849 pathOf: (item: I) => string,
850 ): Effect.fn.Return<SearchPage<I>, WorkspaceSearchIndexSearchFailed> {
851 let size = pageSize;
852 while (true) {
853 const fetchSize = size;
854 const result = yield* runSearch(query, fetchSize, operation, () => execute(fetchSize));
855 if (ignoredPaths.size === 0) return result;
856 const items = result.items.filter(
857 (item) => !isIgnoredPath(ignoredPaths, pathOf(item), hostIsWindows),
858 );
859 if (
860 items.length >= pageSize ||
861 result.items.length >= result.totalMatched ||
862 fetchSize >= WORKSPACE_INDEX_PAGE_SIZE
863 ) {
864 return {
865 items,
866 totalMatched: result.totalMatched - (result.items.length - items.length),
867 };
868 }
869 size = Math.min(WORKSPACE_INDEX_PAGE_SIZE, fetchSize * 4);
870 }
871 });
872
873 const refresh: WorkspaceSearchIndex["Service"]["refresh"] = Effect.fn(
874 "WorkspaceSearchIndex.refresh",
875 )(function* () {
876 // scanFiles waits on fff's picker lock, which its watcher can hold for seconds.
877 const result = yield* Effect.tryPromise({
878 try: () => finder.scanFilesAsync(),
879 catch: (cause) =>
880 new WorkspaceSearchIndexRefreshFailed({
881 cwd,
882 reason: "FileFinder.scanFilesAsync rejected unexpectedly.",
883 cause,
884 }),
885 });
886 if (!result.ok) {
887 return yield* new WorkspaceSearchIndexRefreshFailed({
888 cwd,
889 reason: result.error,
890 });
891 }
892 // A rescan that outlasts the wait keeps running, and the finder answers
893 // from its previous index until it finishes, so the index stays usable.
894 yield* waitForIndexReady(
895 cwd,
896 finder,
897 ({ reason, cause }) =>
898 new WorkspaceSearchIndexRefreshFailed({
899 cwd,
900 reason,
901 cause,
902 }),
903 ).pipe(Effect.catchTags({ WorkspaceSearchIndexScanTimedOut: () => Effect.void }));
904 ignoredPaths = yield* listGitIgnoredPaths(vcsProcess, cwd, hostIsWindows);
905 invalidateSymlinkScan();
906 });
907
908 const list: WorkspaceSearchIndex["Service"]["list"] = Effect.fn("WorkspaceSearchIndex.list")(
909 function* () {
910 const incompleteBeforeQuery = yield* hasIncompleteInitialScan();
911 const result = yield* searchVisible(
912 "",
913 WORKSPACE_INDEX_PAGE_SIZE,
914 "mixedSearch",
915 (pageSize) => finder.mixedSearchAsync("", { pageSize }),
916 mixedItemPath,
917 );
918 const mapped = mapMixedSearchResult(result, WORKSPACE_INDEX_MAX_ENTRIES, hostIsWindows);
919 const symlinks = yield* listSymlinkEntries();
920 // fff's watcher can index a link the scan rejected; only accepted links are listed.
921 const indexed = mapped.entries.filter((entry) => !symlinks.linkPaths.has(entry.path));
922 const sortedEntries = withDirectoryAncestors([...indexed, ...symlinks.entries]).toSorted(
923 (left, right) => left.path.localeCompare(right.path),
924 );
925 const entries = sortedEntries.slice(0, WORKSPACE_INDEX_MAX_ENTRIES);
926 return {
927 entries,
928 truncated:
929 incompleteBeforeQuery || mapped.truncated || entries.length < sortedEntries.length,
930 };
931 },
932 );
933
934 const searchNames = Effect.fn("WorkspaceSearchIndex.searchNames")(function* (
935 query: string,
936 limit: number,
937 kind: ProjectEntryKind | undefined,
938 imageOnly: boolean | undefined,
939 ) {
940 const { entries: allSymlinks, linkPaths } = yield* listSymlinkEntries();
941 const symlinks = allSymlinks.filter((entry) =>
942 imageOnly
943 ? entry.kind === "file" && isProjectFaviconPath(entry.path)
944 : kind === undefined || entry.kind === kind,
945 );
946 // Index results that are filtered-out links get dropped in the merge, so fetch past them.
947 const indexLimit = limit + linkPaths.size - symlinks.length;
948 const pageSize = imageOnly ? WORKSPACE_INDEX_PAGE_SIZE : Math.max(1, indexLimit + 1);
949 const merge = (indexed: ProjectSearchEntriesResult) =>
950 mergeSymlinkMatches(indexed, linkPaths, symlinks, query, limit);
951 if (kind === "file" || imageOnly) {
952 const result = yield* searchVisible(
953 query,
954 pageSize,
955 "fileSearch",
956 (size) => finder.fileSearchAsync(query, { pageSize: size }),
957 (item) => item.relativePath,
958 );
959 return merge(mapFileSearchResult(result, indexLimit, hostIsWindows, imageOnly));
960 }
961 if (kind === "directory") {
962 const result = yield* searchVisible(
963 query,
964 pageSize,
965 "directorySearch",
966 (size) => finder.directorySearchAsync(query, { pageSize: size }),
967 (item) => item.relativePath,
968 );
969 return merge(mapDirectorySearchResult(result, indexLimit, hostIsWindows));
970 }
971 const result = yield* searchVisible(
972 query,
973 pageSize,
974 "mixedSearch",
975 (size) => finder.mixedSearchAsync(query, { pageSize: size }),
976 mixedItemPath,
977 );
978 return merge(mapMixedSearchResult(result, indexLimit, hostIsWindows));
979 });
980
981 const search: WorkspaceSearchIndex["Service"]["search"] = Effect.fn(
982 "WorkspaceSearchIndex.search",
983 )(function* (rawQuery, limit, kind, imageOnly) {
984 // fff parses any word containing "[" or "{" as a glob and has no escape for them; fuzzy
985 // matching is subsequence-based, so dropping them still matches "[id]" and "a{b}" paths.
986 // "{" is dropped only when the query finds nothing as a glob, so "*.{ts,tsx}" keeps working.
987 const query = rawQuery.replaceAll("[", "");
988 const incompleteBeforeQuery = yield* hasIncompleteInitialScan();
989 let mapped = yield* searchNames(query, limit, kind, imageOnly);
990 if (mapped.entries.length === 0 && query.includes("{")) {
991 mapped = yield* searchNames(query.replaceAll("{", ""), limit, kind, imageOnly);
992 }
993 return { ...mapped, truncated: incompleteBeforeQuery || mapped.truncated };
994 });
995
996 const searchContents: WorkspaceSearchIndex["Service"]["searchContents"] = Effect.fn(
997 "WorkspaceSearchIndex.searchContents",
998 )(function* (input) {
999 let { searchQuery, regexMode } = buildContentSearchQuery(input);
1000 const deadline = performance.now() + CONTENT_SEARCH_TIME_BUDGET_MS;
1001 // Grep cursors advance per file; whole-word filtering needs enough raw candidates before advancing.
1002 const rawPageSize = input.wholeWord
1003 ? Math.max(input.limit, CONTENT_SEARCH_MAX_MATCHES_PER_FILE)
1004 : input.limit;
1005 const matches: Array<ProjectSearchContentsResult["matches"][number]> = [];
1006 let nextCursor: GrepCursor | null = null;
1007 let regexFallbackError: string | undefined;
1008 let retryLiterally = false;
1009
1010 do {
1011 const remainingTimeBudgetMs = Math.max(1, Math.ceil(deadline - performance.now()));
1012 const result = yield* runSearch(input.query, input.limit, "grep", () =>
1013 finder.grepAsync(searchQuery, {
1014 mode: regexMode ? "regex" : "plain",
1015 smartCase: !input.caseSensitive && !regexMode,
1016 maxMatchesPerFile: Math.min(CONTENT_SEARCH_MAX_MATCHES_PER_FILE, rawPageSize),
1017 pageSize: rawPageSize,
1018 cursor: nextCursor,
1019 timeBudgetMs: remainingTimeBudgetMs,
1020 }),
1021 );
1022 // fff searches an invalid pattern as a literal, which would keep the `(?i)` prefix.
1023 retryLiterally = regexMode && !input.caseSensitive && result.regexFallbackError !== undefined;
1024 if (retryLiterally) {
1025 regexFallbackError = result.regexFallbackError;
1026 ({ searchQuery, regexMode } = buildContentSearchQuery({ ...input, useRegex: false }));
1027 continue;
1028 }
1029
1030 for (const match of result.items) {
1031 if (isIgnoredPath(ignoredPaths, match.relativePath, hostIsWindows)) continue;
1032 const matchRanges = mapContentMatchRanges(match.lineContent, match.matchRanges).filter(
1033 (range) => !input.wholeWord || isWholeWordRange(match.lineContent, range),
1034 );
1035 // fff reports hits past its 512-char lineContent cap with no ranges, so whole words can't be verified there.
1036 if (input.wholeWord && matchRanges.length === 0) continue;
1037 matches.push({
1038 path: toPosixPath(match.relativePath, hostIsWindows),
1039 lineNumber: match.lineNumber,
1040 lineContent: match.lineContent,
1041 matchRanges,
1042 });
1043 }
1044 nextCursor = result.nextCursor;
1045 regexFallbackError ??= result.regexFallbackError;
1046 } while (
1047 retryLiterally ||
1048 (matches.length < input.limit && nextCursor !== null && performance.now() < deadline)
1049 );
1050
1051 return {
1052 matches: matches.slice(0, input.limit),
1053 truncated: matches.length > input.limit || nextCursor !== null,
1054 ...(regexFallbackError !== undefined ? { regexFallbackError } : {}),
1055 };
1056 });
1057
1058 return WorkspaceSearchIndex.of({ list, refresh, search, searchContents });
1059});
1060
1061export const WORKSPACE_SEARCH_INDEX_VARIANTS = ["paths", "content"] as const;
1062export type WorkspaceSearchIndexVariant = (typeof WORKSPACE_SEARCH_INDEX_VARIANTS)[number];
1063
1064export const workspaceSearchIndexKey = (cwd: string, variant: WorkspaceSearchIndexVariant) =>
1065 `${variant}\n${cwd}`;
1066
1067function parseWorkspaceSearchIndexKey(key: string): {
1068 readonly cwd: string;
1069 readonly variant: WorkspaceSearchIndexVariant;
1070} {
1071 const separatorIndex = key.indexOf("\n");
1072 return {
1073 variant: key.slice(0, separatorIndex) as WorkspaceSearchIndexVariant,
1074 cwd: key.slice(separatorIndex + 1),
1075 };
1076}
1077
1078/** @public */
1079export const layer = (key: string) => {
1080 const { cwd, variant } = parseWorkspaceSearchIndexKey(key);
1081 return Layer.effect(WorkspaceSearchIndex, make(cwd, variant));
1082};
1083
1084export class WorkspaceSearchIndexMap extends LayerMap.Service<WorkspaceSearchIndexMap>()(
1085 "t3/workspace/WorkspaceSearchIndexMap",
1086 {
1087 lookup: layer,
1088 idleTimeToLive: WORKSPACE_INDEX_IDLE_TTL,
1089 },
1090) {}