1import { assert, it } from "@effect/vitest";
2import {
3 EnvironmentId,
4 EventId,
5 MessageId,
6 ProjectId,
7 ProviderInstanceId,
8 ThreadId,
9 type OrchestrationV2DomainEvent,
10 type ThreadLinkedPullRequest,
11} from "@t3tools/contracts";
12import * as NodeServices from "@effect/platform-node/NodeServices";
13import * as DateTime from "effect/DateTime";
14import * as Deferred from "effect/Deferred";
15import * as Effect from "effect/Effect";
16import * as TestClock from "effect/testing/TestClock";
17import * as FileSystem from "effect/FileSystem";
18import * as Fiber from "effect/Fiber";
19import * as Layer from "effect/Layer";
20import * as Path from "effect/Path";
21import * as SqlClient from "effect/sql/SqlClient";
22import { serializeAssistantCitation } from "@t3tools/shared/assistantCitations";
23import { ATTACHMENT_ONLY_BOOTSTRAP_PROMPT } from "@t3tools/shared/composerContextReferences";
24
25import * as SqlitePersistence from "../persistence/Sqlite.ts";
26import { rebuildMessageSearch } from "./messageSearchIndex.ts";
27import * as ProjectionStore from "./ProjectionStore.ts";
28import * as ProjectStore from "./ProjectStore.ts";
29import * as ThreadSearch from "./ThreadSearch.ts";
30
31const layerTest = Layer.mergeAll(
32 ThreadSearch.layer,
33 ProjectionStore.layer,
34 ProjectStore.layer,
35).pipe(Layer.provideMerge(SqlitePersistence.layerMemory));
36
37const providerInstanceId = ProviderInstanceId.make("codex");
38const at = (minute: number) => DateTime.makeUnsafe(Date.UTC(2026, 8, 27, 0, minute));
39
40const createProject = (projectId: ProjectId) =>
41 Effect.flatMap(ProjectStore.ProjectStoreV2, (projects) =>
42 projects.apply({
43 sequence: 0,
44 eventId: EventId.make(`created:${projectId}`),
45 aggregateKind: "project",
46 aggregateId: projectId,
47 occurredAt: DateTime.formatIso(at(0)),
48 commandId: null,
49 causationEventId: null,
50 correlationId: null,
51 metadata: {},
52 type: "project.created",
53 payload: {
54 projectId,
55 title: projectId,
56 workspaceRoot: `/work/${projectId}`,
57 defaultModelSelection: null,
58 scripts: [],
59 createdAt: DateTime.formatIso(at(0)),
60 updatedAt: DateTime.formatIso(at(0)),
61 },
62 }),
63 );
64
65const thread = (
66 threadId: ThreadId,
67 projectId: ProjectId,
68 overrides: {
69 readonly archivedAt?: DateTime.Utc;
70 readonly deletedAt?: DateTime.Utc;
71 readonly subagentOf?: ThreadId;
72 readonly sideChatOf?: ThreadId;
73 readonly title?: string;
74 readonly settledAt?: DateTime.Utc;
75 readonly linkedPullRequest?: ThreadLinkedPullRequest;
76 readonly branchPullRequest?: ThreadLinkedPullRequest;
77 } = {},
78): OrchestrationV2DomainEvent => ({
79 id: EventId.make(`created:${threadId}`),
80 type: "thread.created",
81 threadId,
82 providerInstanceId,
83 occurredAt: at(0),
84 payload: {
85 createdBy: "user",
86 creationSource: "web",
87 id: threadId,
88 projectId,
89 title: overrides.title ?? threadId,
90 providerInstanceId,
91 modelSelection: { instanceId: providerInstanceId, model: "gpt-5" },
92 runtimeMode: "full-access",
93 interactionMode: "default",
94 branch: null,
95 worktreePath: null,
96 linkedPullRequest: overrides.linkedPullRequest ?? null,
97 branchPullRequest: overrides.branchPullRequest ?? null,
98 activeProviderThreadId: null,
99 lineage: overrides.subagentOf
100 ? {
101 parentThreadId: overrides.subagentOf,
102 relationshipToParent: "subagent",
103 rootThreadId: overrides.subagentOf,
104 }
105 : overrides.sideChatOf
106 ? {
107 parentThreadId: overrides.sideChatOf,
108 relationshipToParent: "fork",
109 rootThreadId: overrides.sideChatOf,
110 sideChat: true,
111 }
112 : { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId },
113 forkedFrom: null,
114 createdAt: at(0),
115 updatedAt: at(0),
116 archivedAt: overrides.archivedAt ?? null,
117 settledOverride: overrides.settledAt === undefined ? null : "settled",
118 settledAt: overrides.settledAt ?? null,
119 lastVisitedAt: null,
120 deletedAt: overrides.deletedAt ?? null,
121 },
122});
123
124const message = (
125 threadId: ThreadId,
126 id: string,
127 role: "user" | "assistant" | "system",
128 text: string,
129 options: {
130 readonly minute?: number;
131 readonly streaming?: boolean;
132 readonly createdBy?: "user" | "agent";
133 } = {},
134): OrchestrationV2DomainEvent => ({
135 id: EventId.make(`message:${id}`),
136 type: "message.updated",
137 threadId,
138 providerInstanceId,
139 occurredAt: at(options.minute ?? 1),
140 payload: {
141 createdBy: options.createdBy ?? (role === "user" ? "user" : "agent"),
142 creationSource: role === "user" ? "web" : "provider",
143 id: MessageId.make(id),
144 threadId,
145 runId: null,
146 nodeId: null,
147 role,
148 text,
149 attachments: [],
150 streaming: options.streaming ?? false,
151 createdAt: at(options.minute ?? 1),
152 updatedAt: at(options.minute ?? 1),
153 },
154});
155
156// The prefix and suffix are hidden selection context, not text the reader sees.
157const quote = serializeAssistantCitation({
158 version: 1,
159 environmentId: EnvironmentId.make("environment:local"),
160 threadId: ThreadId.make("thread:quoted"),
161 messageId: MessageId.make("quoted-answer"),
162 text: "A lighthouse guides ships safely through the darkness.",
163 comment: "what are you talking about",
164 start: 0,
165 end: 54,
166 prefix: "beacon01",
167 suffix: "harbor",
168});
169
170it.effect.each([true, false])(
171 "ThreadSearch excludes bootstrap text before the hit and candidate limits (indexed: %s)",
172 (indexed) =>
173 Effect.gen(function* () {
174 const projections = yield* ProjectionStore.ProjectionStoreV2;
175 const search = yield* ThreadSearch.ThreadSearch;
176 const sql = yield* SqlClient.SqlClient;
177 const project = ProjectId.make("project:attachments");
178 const threadId = ThreadId.make("thread:attachments");
179 yield* createProject(project);
180 yield* Effect.forEach(
181 [
182 thread(threadId, project),
183 message(threadId, "ordinary", "user", "Review the attached files in context", {
184 minute: 1,
185 }),
186 message(threadId, "attachment", "user", ATTACHMENT_ONLY_BOOTSTRAP_PROMPT, { minute: 2 }),
187 message(
188 threadId,
189 "ultrathink",
190 "user",
191 `Ultrathink:\n${ATTACHMENT_ONLY_BOOTSTRAP_PROMPT}`,
192 { minute: 3 },
193 ),
194 ],
195 projections.apply,
196 { discard: true },
197 );
198 if (!indexed) {
199 yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`;
200 yield* sql`DROP TRIGGER orchestration_v2_message_search_update`;
201 yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`;
202 yield* sql`DROP TABLE orchestration_v2_message_search`;
203 }
204 assert.deepEqual((yield* search.search({ query: "conversation context" })).matches, []);
205 assert.deepEqual((yield* search.search({ query: "ultrathink" })).matches, []);
206 assert.deepEqual(
207 (yield* search.search({ query: "attached files" })).matches.map((match) => [
208 match.messageId,
209 match.snippet,
210 ]),
211 [[MessageId.make("ordinary"), "Review the attached files in context"]],
212 );
213 for (const text of [
214 ATTACHMENT_ONLY_BOOTSTRAP_PROMPT,
215 `Ultrathink:\n${ATTACHMENT_ONLY_BOOTSTRAP_PROMPT}`,
216 `uLtRaThInK:\r\n\t\u00a0${ATTACHMENT_ONLY_BOOTSTRAP_PROMPT}`,
217 ]) {
218 yield* sql`
219 WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < ${indexed ? 10_000 : 2000})
220 INSERT INTO orchestration_v2_projection_messages
221 (message_id, thread_id, role, streaming, created_at, updated_at, payload_json)
222 SELECT 'attachment-limit:' || i, ${threadId}, 'user', 0,
223 ${DateTime.formatIso(at(5))}, ${DateTime.formatIso(at(5))}, json_object('text', ${text})
224 FROM n
225 `;
226 assert.deepEqual(
227 (yield* search.search({ query: "context" })).matches.map((match) => match.messageId),
228 [MessageId.make("ordinary")],
229 );
230 yield* sql`
231 DELETE FROM orchestration_v2_projection_messages WHERE message_id LIKE 'attachment-limit:%'
232 `;
233 }
234 }).pipe(Effect.provide(layerTest)),
235);
236
237it.effect("ThreadSearch excludes legacy placeholders still in the index", () =>
238 Effect.gen(function* () {
239 const projections = yield* ProjectionStore.ProjectionStoreV2;
240 const search = yield* ThreadSearch.ThreadSearch;
241 const sql = yield* SqlClient.SqlClient;
242 const project = ProjectId.make("project:legacy-attachments");
243 const threadId = ThreadId.make("thread:legacy-attachments");
244 yield* createProject(project);
245 // Simulate the pre-upgrade trigger, which indexed attachment placeholders.
246 yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`;
247 yield* sql`
248 CREATE TRIGGER orchestration_v2_message_search_insert
249 AFTER INSERT ON orchestration_v2_projection_messages
250 BEGIN
251 INSERT INTO orchestration_v2_message_search (rowid, text)
252 VALUES (new.rowid, json_extract(new.payload_json, '$.text'));
253 END
254 `;
255 yield* Effect.forEach(
256 [
257 thread(threadId, project),
258 message(threadId, "ordinary", "user", "Review the attached files in context"),
259 message(threadId, "attachment", "user", ATTACHMENT_ONLY_BOOTSTRAP_PROMPT),
260 message(
261 threadId,
262 "ultrathink",
263 "user",
264 `uLtRaThInK:\r\n\t\u00a0${ATTACHMENT_ONLY_BOOTSTRAP_PROMPT}`,
265 ),
266 ],
267 projections.apply,
268 { discard: true },
269 );
270 assert.lengthOf(
271 yield* sql`SELECT rowid FROM orchestration_v2_message_search WHERE orchestration_v2_message_search MATCH '"context"'`,
272 3,
273 );
274 assert.deepEqual((yield* search.search({ query: "conversation context" })).matches, []);
275 assert.deepEqual((yield* search.search({ query: "ultrathink" })).matches, []);
276 assert.deepEqual(
277 (yield* search.search({ query: "context" })).matches.map((match) => match.messageId),
278 [MessageId.make("ordinary")],
279 );
280 }).pipe(Effect.provide(layerTest)),
281);
282
283it.layer(layerTest)("ThreadSearch", (it) => {
284 it.effect("reads the newest hits without sorting all of them", () =>
285 Effect.gen(function* () {
286 const sql = yield* SqlClient.SqlClient;
287 for (const match of ['"needle"', null]) {
288 const plan = yield* sql<{
289 readonly detail: string;
290 }>`EXPLAIN QUERY PLAN ${ThreadSearch.messageHitsQuery(sql, { match, substrings: "[]", shortSpellings: "[]", projectId: null, excludedThreadIds: "[]" })}`;
291 assert.isFalse(
292 plan.some((row) => row.detail.includes("TEMP B-TREE")),
293 plan.map((row) => row.detail).join("\n"),
294 );
295 if (match !== null) {
296 assert.lengthOf(
297 plan.filter((row) =>
298 row.detail.startsWith("SEARCH messages USING INTEGER PRIMARY KEY"),
299 ),
300 1,
301 plan.map((row) => row.detail).join("\n"),
302 );
303 }
304 }
305 }),
306 );
307
308 it.effect("returns one finished user or assistant match per active thread", () =>
309 Effect.gen(function* () {
310 const projections = yield* ProjectionStore.ProjectionStoreV2;
311 const search = yield* ThreadSearch.ThreadSearch;
312 const project = ProjectId.make("project:search");
313 const deletedProject = ProjectId.make("project:search-deleted");
314 yield* createProject(project);
315 yield* createProject(deletedProject);
316 yield* Effect.flatMap(ProjectStore.ProjectStoreV2, (projects) =>
317 projects.apply({
318 sequence: 0,
319 eventId: EventId.make("deleted:project"),
320 aggregateKind: "project",
321 aggregateId: deletedProject,
322 occurredAt: DateTime.formatIso(at(2)),
323 commandId: null,
324 causationEventId: null,
325 correlationId: null,
326 metadata: {},
327 type: "project.deleted",
328 payload: { projectId: deletedProject, deletedAt: DateTime.formatIso(at(2)) },
329 }),
330 );
331
332 const both = ThreadId.make("thread:both");
333 const assistantOnly = ThreadId.make("thread:assistant");
334 const events: ReadonlyArray<OrchestrationV2DomainEvent> = [
335 thread(both, project),
336 message(both, "both-assistant", "assistant", "needle from the answer", { minute: 3 }),
337 message(both, "both-user-old", "user", "older needle question", { minute: 1 }),
338 message(both, "both-user-new", "user", "newer needle question", { minute: 2 }),
339 thread(assistantOnly, project),
340 message(assistantOnly, "assistant-only", "assistant", "needle in an answer"),
341 message(assistantOnly, "assistant-streaming", "user", "needle still typing", {
342 streaming: true,
343 }),
344 message(assistantOnly, "assistant-system", "system", "needle system prompt"),
345 thread(ThreadId.make("thread:archived"), project, { archivedAt: at(1) }),
346 message(ThreadId.make("thread:archived"), "archived", "user", "needle archived"),
347 thread(ThreadId.make("thread:deleted"), project, { deletedAt: at(1) }),
348 message(ThreadId.make("thread:deleted"), "deleted", "user", "needle deleted"),
349 thread(ThreadId.make("thread:orphaned"), deletedProject),
350 message(ThreadId.make("thread:orphaned"), "orphaned", "user", "needle orphaned"),
351 ];
352 yield* Effect.forEach(events, projections.apply, { discard: true });
353
354 const result = yield* search.search({ query: "NEEDLE", limit: 20 });
355 assert.deepEqual(
356 result.matches
357 .map((match) => [match.threadId, match.source, match.snippet, match.messageId])
358 .toSorted(([left], [right]) => String(left).localeCompare(String(right))),
359 [
360 [assistantOnly, "assistant", "needle in an answer", "assistant-only"],
361 [both, "user", "newer needle question", "both-user-new"],
362 ],
363 );
364 assert.lengthOf((yield* search.search({ query: "needle", limit: 1 })).matches, 1);
365 assert.deepEqual((yield* search.search({ query: "ne%le" })).matches, []);
366 }),
367 );
368
369 it.effect("matches the readable text of quotes and their comments", () =>
370 Effect.gen(function* () {
371 const projections = yield* ProjectionStore.ProjectionStoreV2;
372 const search = yield* ThreadSearch.ThreadSearch;
373 const project = ProjectId.make("project:search-citation");
374 const threadId = ThreadId.make("thread:citation");
375 yield* createProject(project);
376 yield* Effect.forEach(
377 [
378 thread(threadId, project),
379 message(threadId, "citation", "user", `${quote} quiet silver lantern`),
380 ],
381 projections.apply,
382 { discard: true },
383 );
384 const snippets = (query: string) =>
385 Effect.map(search.search({ query }), (result) =>
386 result.matches
387 .filter((match) => match.threadId === threadId)
388 .map((match) => match.snippet),
389 );
390
391 const readable =
392 "A lighthouse guides ships safely through the darkness. Comment: what are you talking about quiet silver lantern";
393 assert.deepEqual(yield* snippets("what are you talking about"), [
394 `…${readable.slice(readable.indexOf("the darkness"))}`,
395 ]);
396 assert.deepEqual(yield* snippets("ships safely through"), [readable]);
397 assert.deepEqual(yield* snippets("beacon01"), []);
398 assert.deepEqual(yield* snippets("harbor"), []);
399
400 yield* projections.apply(message(threadId, "citation", "user", "quiet silver lantern"));
401 assert.deepEqual(yield* snippets("what are you talking about"), []);
402 assert.deepEqual(yield* snippets("lantern"), ["quiet silver lantern"]);
403 }),
404 );
405
406 it.effect("finds accented, CJK, and apostrophe words of a quote through the index", () =>
407 Effect.gen(function* () {
408 const sql = yield* SqlClient.SqlClient;
409 const projections = yield* ProjectionStore.ProjectionStoreV2;
410 const search = yield* ThreadSearch.ThreadSearch;
411 const project = ProjectId.make("project:search-citation-index");
412 const threadId = ThreadId.make("thread:citation-index");
413 // The stored link percent-encodes these words; the index must hold them as they read.
414 const encodedQuote = serializeAssistantCitation({
415 version: 1,
416 environmentId: EnvironmentId.make("environment:local"),
417 threadId: ThreadId.make("thread:quoted"),
418 messageId: MessageId.make("quoted-answer"),
419 text: "The kittiwake's résumé mentions 灯台の光 twice.",
420 comment: "naïve question в логах",
421 start: 0,
422 end: 44,
423 prefix: "",
424 suffix: "",
425 });
426 yield* createProject(project);
427 yield* Effect.forEach(
428 [thread(threadId, project), message(threadId, "citation-index", "user", encodedQuote)],
429 projections.apply,
430 { discard: true },
431 );
432 const index = yield* sql`
433 SELECT 1 FROM sqlite_master WHERE name = 'orchestration_v2_message_search'
434 `;
435 assert.lengthOf(index, 1);
436 const threadIds = (query: string) =>
437 search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
438
439 for (const query of [
440 "kittiwake's",
441 "kittiwake’s",
442 "résumé",
443 "灯台の光",
444 "naïve",
445 "логах в",
446 ]) {
447 assert.deepEqual(yield* threadIds(query), [threadId], query);
448 }
449 // `t3 doctor` rebuilds the index after VACUUM; quotes must still read.
450 yield* rebuildMessageSearch(sql);
451 assert.deepEqual(yield* threadIds("résumé"), [threadId]);
452
453 yield* projections.apply(message(threadId, "citation-index", "user", "plain words"));
454 assert.deepEqual(yield* threadIds("résumé"), []);
455 assert.deepEqual(yield* threadIds("plain"), [threadId]);
456 }),
457 );
458
459 it.effect("indexes a message when it finishes streaming and follows later edits", () =>
460 Effect.gen(function* () {
461 const projections = yield* ProjectionStore.ProjectionStoreV2;
462 const search = yield* ThreadSearch.ThreadSearch;
463 const project = ProjectId.make("project:search-sync");
464 const threadId = ThreadId.make("thread:sync");
465 yield* createProject(project);
466 yield* projections.apply(thread(threadId, project));
467 const threadIds = (query: string) =>
468 search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
469
470 yield* projections.apply(
471 message(threadId, "sync", "assistant", "streaming draft", { streaming: true }),
472 );
473 assert.deepEqual(yield* threadIds("draft"), []);
474
475 yield* projections.apply(message(threadId, "sync", "assistant", "finished answer"));
476 assert.deepEqual(yield* threadIds("finished"), [threadId]);
477
478 yield* projections.apply(message(threadId, "sync", "assistant", "edited reply"));
479 assert.deepEqual(yield* threadIds("finished"), []);
480 assert.deepEqual(yield* threadIds("edited"), [threadId]);
481 }),
482 );
483
484 it.effect("matches inside identifiers and skips hidden threads and short tokens", () =>
485 Effect.gen(function* () {
486 const projections = yield* ProjectionStore.ProjectionStoreV2;
487 const search = yield* ThreadSearch.ThreadSearch;
488 const project = ProjectId.make("project:search-tokens");
489 const parent = ThreadId.make("thread:tokens-parent");
490 const subagent = ThreadId.make("thread:tokens-subagent");
491 const sideChat = ThreadId.make("thread:tokens-side-chat");
492 yield* createProject(project);
493 yield* Effect.forEach(
494 [
495 thread(parent, project),
496 message(parent, "parent", "user", "Rename ThreadSearchService before shipping"),
497 thread(subagent, project, { subagentOf: parent }),
498 message(subagent, "subagent", "assistant", "ThreadSearchService renamed"),
499 thread(sideChat, project, { sideChatOf: parent, title: "ThreadSearchService question" }),
500 message(sideChat, "side-chat", "user", "Why rename ThreadSearchService?"),
501 ],
502 projections.apply,
503 { discard: true },
504 );
505
506 const result = yield* search.search({ query: "search servi" });
507 assert.deepEqual(
508 result.matches.map((match) => [match.threadId, match.snippet]),
509 [[parent, "Rename ThreadSearchService before shipping"]],
510 );
511 // "ab" alone cannot use the index; with a longer token it narrows the hits.
512 assert.deepEqual((yield* search.search({ query: "ab" })).matches, []);
513 assert.lengthOf((yield* search.search({ query: "renam be" })).matches, 1);
514 assert.lengthOf((yield* search.search({ query: "renam ab" })).matches, 0);
515 }),
516 );
517
518 it.effect("indexes neither subagent messages nor rewrites that keep the text", () =>
519 Effect.gen(function* () {
520 const sql = yield* SqlClient.SqlClient;
521 const projections = yield* ProjectionStore.ProjectionStoreV2;
522 const project = ProjectId.make("project:search-index-writes");
523 const parent = ThreadId.make("thread:index-parent");
524 const subagent = ThreadId.make("thread:index-subagent");
525 yield* createProject(project);
526 yield* Effect.forEach(
527 [
528 thread(parent, project),
529 message(parent, "index-parent", "assistant", "parent answer"),
530 thread(subagent, project, { subagentOf: parent }),
531 message(subagent, "index-subagent", "assistant", "subagent answer"),
532 ],
533 projections.apply,
534 { discard: true },
535 );
536 const indexedThreads = sql<{ readonly threadId: string }>`
537 SELECT messages.thread_id AS "threadId"
538 FROM orchestration_v2_message_search AS search
539 INNER JOIN orchestration_v2_projection_messages AS messages
540 ON messages.rowid = search.rowid
541 WHERE messages.thread_id IN (${parent}, ${subagent})
542 `.pipe(Effect.map((rows) => rows.map((row) => row.threadId)));
543 assert.deepEqual(yield* indexedThreads, [parent]);
544
545 // total_changes counts trigger writes, so 1 means the index was untouched.
546 const changes = sql<{ readonly n: number }>`SELECT total_changes() AS n`.pipe(
547 Effect.map(([row]) => row?.n ?? 0),
548 );
549 const before = yield* changes;
550 yield* sql`
551 UPDATE orchestration_v2_projection_messages
552 SET payload_json = json_set(payload_json, '$.updatedAt', '2026-09-27T01:00:00.000Z')
553 WHERE message_id = 'index-parent'
554 `;
555 assert.strictEqual((yield* changes) - before, 1);
556 assert.deepEqual(yield* indexedThreads, [parent]);
557 }),
558 );
559});
560
561it.layer(layerTest)("ThreadSearch filters before it ranks", (it) => {
562 it.effect("orders message matches by when they were written, not by row order", () =>
563 Effect.gen(function* () {
564 const projections = yield* ProjectionStore.ProjectionStoreV2;
565 const search = yield* ThreadSearch.ThreadSearch;
566 const project = ProjectId.make("project:written");
567 yield* createProject(project);
568 const recent = ThreadId.make("thread:written-recent");
569 const imported = ThreadId.make("thread:written-imported");
570 // An imported history lands after newer messages in row order.
571 yield* Effect.forEach(
572 [
573 thread(recent, project),
574 message(recent, "written-recent", "user", "kestrel deploy notes", { minute: 30 }),
575 thread(imported, project),
576 message(imported, "written-imported", "user", "kestrel deploy notes", { minute: 1 }),
577 ],
578 projections.apply,
579 { discard: true },
580 );
581 const { matches } = yield* search.search({ query: "kestrel" });
582 assert.deepEqual(
583 matches.map((match) => match.threadId),
584 [recent, imported],
585 );
586 }),
587 );
588
589 it.effect("ranks a thread about the word above a passing mention", () =>
590 Effect.gen(function* () {
591 const projections = yield* ProjectionStore.ProjectionStoreV2;
592 const search = yield* ThreadSearch.ThreadSearch;
593 const project = ProjectId.make("project:about");
594 yield* createProject(project);
595 const about = ThreadId.make("thread:about-osprey");
596 const mention = ThreadId.make("thread:mention-osprey");
597 yield* Effect.forEach(
598 [
599 thread(about, project),
600 ...Array.from({ length: 12 }, (_, index) =>
601 message(about, `about-osprey-${index}`, "assistant", `osprey step ${index}`, {
602 minute: 2,
603 }),
604 ),
605 thread(mention, project),
606 message(mention, "mention-osprey", "user", "also check osprey", { minute: 5 }),
607 ],
608 projections.apply,
609 { discard: true },
610 );
611 const { matches } = yield* search.search({ query: "osprey" });
612 assert.deepEqual(
613 matches.map((match) => [match.threadId, match.hits]),
614 [
615 [about, 12],
616 [mention, 1],
617 ],
618 );
619 }),
620 );
621
622 it.effect("counts a thread's hits at its best match only", () =>
623 Effect.gen(function* () {
624 const projections = yield* ProjectionStore.ProjectionStoreV2;
625 const search = yield* ThreadSearch.ThreadSearch;
626 const project = ProjectId.make("project:best-hits");
627 yield* createProject(project);
628 const clarks = ThreadId.make("thread:clarks");
629 const larks = ThreadId.make("thread:lark-many");
630 yield* Effect.forEach(
631 [
632 thread(clarks, project),
633 message(clarks, "clarks-lark", "user", "Lark once", { minute: 3 }),
634 ...Array.from({ length: 31 }, (_, index) =>
635 message(clarks, `clarks-${index}`, "assistant", "Clark the plan", { minute: 3 }),
636 ),
637 thread(larks, project),
638 ...Array.from({ length: 8 }, (_, index) =>
639 message(larks, `lark-many-${index}`, "assistant", "Lark again", { minute: 3 }),
640 ),
641 ],
642 projections.apply,
643 { discard: true },
644 );
645 const { matches } = yield* search.search({ query: "lark" });
646 assert.deepEqual(
647 matches.map((match) => [match.threadId, match.hits]),
648 [
649 [larks, 8],
650 [clarks, 1],
651 ],
652 );
653 }),
654 );
655
656 it.effect("finds a visible match behind hundreds of hidden or narrower hits", () =>
657 Effect.gen(function* () {
658 const projections = yield* ProjectionStore.ProjectionStoreV2;
659 const search = yield* ThreadSearch.ThreadSearch;
660 const project = ProjectId.make("project:crowded");
661 const parent = ThreadId.make("thread:crowded-parent");
662 const subagent = ThreadId.make("thread:crowded-subagent");
663 const older = ThreadId.make("thread:crowded-older");
664 yield* createProject(project);
665 yield* Effect.forEach(
666 [
667 thread(parent, project),
668 thread(subagent, project, { subagentOf: parent }),
669 thread(older, project),
670 ...Array.from({ length: 450 }, (_, index) =>
671 message(subagent, `subagent-${index}`, "assistant", "needle needle"),
672 ),
673 ...Array.from({ length: 450 }, (_, index) =>
674 message(older, `older-${index}`, "assistant", "pnpm 11 pnpm"),
675 ),
676 message(parent, "parent-needle", "user", "a needle in a long haystack of other words"),
677 // Agent-written notices in user messages are not indexed, so they never read as "You".
678 message(older, "older-notice", "user", "needle update from a watched pull request", {
679 createdBy: "agent",
680 }),
681 message(parent, "parent-pnpm", "user", "moved the repo to pnpm 12 today"),
682 ],
683 projections.apply,
684 { discard: true },
685 );
686
687 const threadIds = (query: string) =>
688 search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
689 assert.deepEqual(yield* threadIds("needle"), [parent]);
690 assert.deepEqual(yield* threadIds("pnpm 12"), [parent]);
691 }),
692 );
693
694 it.effect("keeps a thread picked for the query when the limit cuts", () =>
695 Effect.gen(function* () {
696 const projections = yield* ProjectionStore.ProjectionStoreV2;
697 const search = yield* ThreadSearch.ThreadSearch;
698 const sql = yield* SqlClient.SqlClient;
699 const project = ProjectId.make("project:plover");
700 const busy = ThreadId.make("thread:plover-busy");
701 const picked = ThreadId.make("thread:plover-picked");
702 yield* createProject(project);
703 yield* Effect.forEach(
704 [
705 thread(busy, project),
706 message(busy, "plover-1", "user", "plover notes", { minute: 3 }),
707 message(busy, "plover-2", "user", "more plover notes", { minute: 4 }),
708 thread(picked, project),
709 message(picked, "plover-old", "user", "one plover mention", { minute: 1 }),
710 ],
711 projections.apply,
712 { discard: true },
713 );
714 const threadIds = () =>
715 search
716 .search({ query: "plover", limit: 1 })
717 .pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
718 assert.deepEqual(yield* threadIds(), [busy]);
719 // A pick at time 0 would read as no pick.
720 yield* TestClock.adjust("1 day");
721 const at = DateTime.toEpochMillis(yield* DateTime.now);
722 yield* sql`
723 UPDATE orchestration_v2_projection_threads
724 SET payload_json = json_set(payload_json, '$.opens', json(${JSON.stringify({
725 openedAt: at,
726 score: 1,
727 picks: [{ query: "plover", at }],
728 })}))
729 WHERE thread_id = ${picked}
730 `;
731 assert.deepEqual(yield* threadIds(), [picked]);
732 }),
733 );
734
735 it.effect("finds a curly apostrophe for a straight one and back", () =>
736 Effect.gen(function* () {
737 const projections = yield* ProjectionStore.ProjectionStoreV2;
738 const search = yield* ThreadSearch.ThreadSearch;
739 const project = ProjectId.make("project:apostrophes");
740 const curly = ThreadId.make("thread:apostrophe-curly");
741 const straight = ThreadId.make("thread:apostrophe-straight");
742 yield* createProject(project);
743 yield* Effect.forEach(
744 [
745 thread(curly, project),
746 message(curly, "apostrophe-curly", "user", "the gannet\u2019s nest"),
747 thread(straight, project),
748 message(straight, "apostrophe-straight", "user", "a kittiwake's nest"),
749 ],
750 projections.apply,
751 { discard: true },
752 );
753 const threadIds = (query: string) =>
754 search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
755 assert.deepEqual(yield* threadIds("gannet's"), [curly]);
756 assert.deepEqual(yield* threadIds("kittiwake\u2019s"), [straight]);
757 }),
758 );
759
760 it.effect("leaves out answers to an agent's question, which show in its card", () =>
761 Effect.gen(function* () {
762 const projections = yield* ProjectionStore.ProjectionStoreV2;
763 const search = yield* ThreadSearch.ThreadSearch;
764 const project = ProjectId.make("project:answers");
765 const asked = ThreadId.make("thread:answers");
766 yield* createProject(project);
767 yield* Effect.forEach(
768 [
769 thread(asked, project),
770 message(asked, "async-answer:request-1", "user", "use the albatross drive"),
771 ],
772 projections.apply,
773 { discard: true },
774 );
775 assert.deepEqual((yield* search.search({ query: "albatross" })).matches, []);
776 }),
777 );
778
779 it.effect("matches messages as they read, with links as their text", () =>
780 Effect.gen(function* () {
781 const projections = yield* ProjectionStore.ProjectionStoreV2;
782 const search = yield* ThreadSearch.ThreadSearch;
783 const project = ProjectId.make("project:heron");
784 const labeled = ThreadId.make("thread:heron-label");
785 const addressOnly = ThreadId.make("thread:heron-address");
786 yield* createProject(project);
787 yield* Effect.forEach(
788 [
789 thread(labeled, project),
790 message(labeled, "heron-label", "assistant", "See [the heron plan](https://x.dev/a) now"),
791 thread(addressOnly, project),
792 message(addressOnly, "heron-address", "user", "Read [this](https://heron.dev/b)"),
793 ],
794 projections.apply,
795 { discard: true },
796 );
797 const { matches } = yield* search.search({ query: "heron" });
798 assert.deepEqual(
799 matches.map((match) => [match.threadId, match.snippet]),
800 [[labeled, "See the heron plan now"]],
801 );
802 }),
803 );
804
805 it.effect("reads at most the newest 10,000 index hits, however many the filters drop", () =>
806 Effect.gen(function* () {
807 const projections = yield* ProjectionStore.ProjectionStoreV2;
808 const search = yield* ThreadSearch.ThreadSearch;
809 const sql = yield* SqlClient.SqlClient;
810 const project = ProjectId.make("project:kestrel");
811 const old = ThreadId.make("thread:kestrel-old");
812 const busy = ThreadId.make("thread:kestrel-busy");
813 yield* createProject(project);
814 yield* Effect.forEach(
815 [
816 thread(old, project),
817 message(old, "kestrel-old", "user", "kestrel ci is red"),
818 thread(busy, project),
819 ],
820 projections.apply,
821 { discard: true },
822 );
823 const insertBusy = (count: number) => sql`
824 WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < ${count})
825 INSERT INTO orchestration_v2_projection_messages
826 (message_id, thread_id, role, streaming, created_at, updated_at, payload_json)
827 SELECT 'kestrel-busy-' || i, ${busy}, 'assistant', 0, ${DateTime.formatIso(at(5))},
828 ${DateTime.formatIso(at(5))}, json_object('text', 'kestrel packages')
829 FROM n
830 `;
831 const threadIds = (query: string) =>
832 search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
833 yield* insertBusy(9_999);
834 assert.deepEqual(yield* threadIds("kestrel ci"), [old]);
835 yield* sql`DELETE FROM orchestration_v2_projection_messages WHERE thread_id = ${busy}`;
836 yield* insertBusy(10_000);
837 assert.deepEqual(yield* threadIds("kestrel ci"), []);
838 }),
839 );
840});
841
842it.effect.each([true, false])(
843 "ThreadSearch narrows short words outside ASCII before the hit limit (indexed: %s)",
844 (indexed) =>
845 Effect.gen(function* () {
846 const projections = yield* ProjectionStore.ProjectionStoreV2;
847 const search = yield* ThreadSearch.ThreadSearch;
848 const sql = yield* SqlClient.SqlClient;
849 const project = ProjectId.make("project:short-limit");
850 const older = ThreadId.make("thread:short-older");
851 const newer = ThreadId.make("thread:short-newer");
852 yield* createProject(project);
853 yield* Effect.forEach(
854 [
855 thread(older, project),
856 message(older, "older", "user", "В логах ошибка"),
857 thread(newer, project),
858 ],
859 projections.apply,
860 { discard: true },
861 );
862 if (!indexed) {
863 yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`;
864 yield* sql`DROP TRIGGER orchestration_v2_message_search_update`;
865 yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`;
866 yield* sql`DROP TABLE orchestration_v2_message_search`;
867 }
868 // More newer hits for the long word than one read holds, none with the short one.
869 yield* sql`
870 WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 2000)
871 INSERT INTO orchestration_v2_projection_messages
872 (message_id, thread_id, role, streaming, created_at, updated_at, payload_json)
873 SELECT 'short-newer-' || i, ${newer}, 'assistant', 0, ${DateTime.formatIso(at(5))},
874 ${DateTime.formatIso(at(5))}, json_object('text', 'ошибка здесь')
875 FROM n
876 `;
877 const { matches } = yield* search.search({ query: "ошибка в" });
878 assert.deepEqual(
879 matches.map((match) => match.threadId),
880 [older],
881 );
882 }).pipe(Effect.provide(layerTest)),
883);
884
885it.layer(layerTest)("ThreadSearch short words outside ASCII", (it) => {
886 it.effect("finds them in any case or accent, with or without the index", () =>
887 Effect.gen(function* () {
888 const projections = yield* ProjectionStore.ProjectionStoreV2;
889 const search = yield* ThreadSearch.ThreadSearch;
890 const sql = yield* SqlClient.SqlClient;
891 const project = ProjectId.make("project:short-non-ascii");
892 const russian = ThreadId.make("thread:short-russian");
893 const french = ThreadId.make("thread:short-french");
894 yield* createProject(project);
895 yield* Effect.forEach(
896 [
897 thread(russian, project),
898 message(russian, "russian", "user", "В логах ошибка"),
899 thread(french, project),
900 message(french, "french", "user", "École fermée à Lyon"),
901 ],
902 projections.apply,
903 { discard: true },
904 );
905 const threadIds = (query: string) =>
906 search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
907
908 const assertMatches = Effect.gen(function* () {
909 assert.deepEqual(yield* threadIds("ошибка в"), [russian]);
910 assert.deepEqual(yield* threadIds("fermée éc"), [french]);
911 // They still narrow: a word the message lacks rules it out.
912 assert.deepEqual(yield* threadIds("ошибка я"), []);
913 });
914 yield* assertMatches;
915 yield* sql`DROP TABLE orchestration_v2_message_search`;
916 yield* assertMatches;
917 }),
918 );
919});
920
921it.layer(layerTest)("ThreadSearch across scripts and without the index", (it) => {
922 it.effect(
923 "matches Korean, Japanese, and two-character Chinese words, and whole short words",
924 () =>
925 Effect.gen(function* () {
926 const projections = yield* ProjectionStore.ProjectionStoreV2;
927 const search = yield* ThreadSearch.ThreadSearch;
928 const sql = yield* SqlClient.SqlClient;
929 const project = ProjectId.make("project:scripts");
930 const threadId = ThreadId.make("thread:scripts");
931 const drafts = ThreadId.make("thread:drafts");
932 const addresses = ThreadId.make("thread:addresses");
933 yield* createProject(project);
934 yield* Effect.forEach(
935 [
936 thread(threadId, project),
937 message(threadId, "korean", "user", "검색 기능을 고쳐 주세요"),
938 message(threadId, "japanese", "user", "がんばって検索します"),
939 message(threadId, "chinese", "user", "明天部署到生产环境"),
940 message(threadId, "fts", "assistant", "the FTS index is ready"),
941 thread(drafts, project),
942 message(drafts, "drafts", "assistant", "saved two drafts and shifts"),
943 message(threadId, "camel", "assistant", "Configure enableFTS today"),
944 message(threadId, "accented", "user", "Review Résumé indexing"),
945 thread(addresses, project),
946 message(addresses, "addresses", "user", "This addresses Résumé indexing"),
947 ],
948 projections.apply,
949 { discard: true },
950 );
951 const threadIds = (query: string) =>
952 search
953 .search({ query })
954 .pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
955
956 // Short words must start a word, which camelCase and accented words do.
957 for (const query of ["검색 기능", "がんばって", "部署", "fts", "enable"]) {
958 assert.deepEqual(yield* threadIds(query), [threadId], query);
959 }
960 // "res" inside "addresses" does not hide the accented word after it.
961 assert.sameMembers(yield* threadIds("res"), [threadId, addresses]);
962 // A word beats the same letters inside a longer word, even in an older message.
963 const trust = ThreadId.make("thread:trust");
964 const rust = ThreadId.make("thread:rust");
965 yield* Effect.forEach(
966 [
967 thread(rust, project),
968 message(rust, "rust", "assistant", "Rust rewrite notes"),
969 thread(trust, project),
970 message(trust, "trust", "user", "We trust the build"),
971 ],
972 projections.apply,
973 { discard: true },
974 );
975 const { matches } = yield* search.search({ query: "rust" });
976 assert.deepEqual(
977 matches.map((match) => [match.threadId, (match.quality ?? 1) <= 0.3]),
978 [
979 [rust, false],
980 [trust, true],
981 ],
982 );
983 // Without the index (an SQLite build without FTS5 trigrams) search scans instead.
984 yield* sql`DROP TABLE orchestration_v2_message_search`;
985 assert.deepEqual(yield* threadIds("検索"), [threadId]);
986 assert.deepEqual(yield* threadIds("fts"), [threadId]);
987 }),
988 );
989});
990
991it.layer(layerTest)("ThreadSearch.searchOlderSettled", (it) => {
992 it.effect("returns older settled title matches in scope, then older message hits", () =>
993 Effect.gen(function* () {
994 const projections = yield* ProjectionStore.ProjectionStoreV2;
995 const search = yield* ThreadSearch.ThreadSearch;
996 const project = ProjectId.make("project:kestrel");
997 const other = ProjectId.make("project:kestrel-other");
998 yield* createProject(project);
999 yield* createProject(other);
1000 const id = (name: string) => ThreadId.make(`thread:kestrel-${name}`);
1001 const old = at(10);
1002 yield* Effect.forEach(
1003 [
1004 thread(id("old"), project, { title: "Kestrel nest", settledAt: old }),
1005 thread(id("recent"), project, { title: "Kestrel nest", settledAt: at(50) }),
1006 thread(id("open"), project, { title: "Kestrel nest" }),
1007 thread(id("child"), project, { title: "Kestrel nest", subagentOf: id("old") }),
1008 thread(id("elsewhere"), other, { title: "Kestrel nest", settledAt: old }),
1009 thread(id("note"), project, { title: "Unrelated", settledAt: old }),
1010 thread(id("new-note"), project, { title: "Unrelated", settledAt: at(50) }),
1011 ],
1012 projections.apply,
1013 { discard: true },
1014 );
1015 const before = DateTime.formatIso(at(30));
1016 const messageThreadIds = [id("new-note"), id("note"), id("old")];
1017 assert.sameMembers(
1018 [...(yield* search.searchOlderSettled({ query: "kestrel", before, messageThreadIds }))],
1019 [id("old"), id("elsewhere"), id("note")],
1020 );
1021 // The scope applies to titles and message hits; message hits follow the titles in order.
1022 assert.deepEqual(
1023 yield* search.searchOlderSettled({
1024 query: "kestrel",
1025 before,
1026 projectIds: [project],
1027 messageThreadIds: [id("elsewhere"), ...messageThreadIds],
1028 }),
1029 [id("old"), id("note")],
1030 );
1031 // A palette also matches the environment's name.
1032 assert.include(
1033 yield* search.searchOlderSettled({
1034 query: "laptop unrelated",
1035 before,
1036 environmentLabel: "Laptop",
1037 messageThreadIds: [],
1038 }),
1039 id("note"),
1040 );
1041 }),
1042 );
1043
1044 it.effect("matches the branch and linked pull requests, like the palette", () =>
1045 Effect.gen(function* () {
1046 const projections = yield* ProjectionStore.ProjectionStoreV2;
1047 const search = yield* ThreadSearch.ThreadSearch;
1048 const project = ProjectId.make("project:heron");
1049 yield* createProject(project);
1050 const pullRequest = (number: number): ThreadLinkedPullRequest => ({
1051 projectId: project,
1052 repository: "acme/heron",
1053 number,
1054 url: `https://github.com/acme/heron/pull/${number}`,
1055 });
1056 const branch = ThreadId.make("thread:heron-branch");
1057 const linked = ThreadId.make("thread:heron-linked");
1058 yield* Effect.forEach(
1059 [
1060 thread(branch, project, {
1061 title: "Heron",
1062 settledAt: at(10),
1063 branchPullRequest: pullRequest(4101),
1064 }),
1065 thread(linked, project, {
1066 title: "Heron",
1067 settledAt: at(10),
1068 linkedPullRequest: pullRequest(4202),
1069 }),
1070 ],
1071 projections.apply,
1072 { discard: true },
1073 );
1074 const find = (query: string) =>
1075 search.searchOlderSettled({
1076 query,
1077 before: DateTime.formatIso(at(30)),
1078 messageThreadIds: [],
1079 });
1080 assert.deepEqual(yield* find("#4101"), [branch]);
1081 assert.deepEqual(yield* find("acme/heron#4202"), [linked]);
1082 }),
1083 );
1084});
1085
1086it.layer(layerTest)("ThreadSearch.searchThreadsWithOlderSettledRows", (it) => {
1087 it.effect("adds older settled rows to message hits, or alone for titles only", () =>
1088 Effect.gen(function* () {
1089 const projections = yield* ProjectionStore.ProjectionStoreV2;
1090 const search = yield* ThreadSearch.ThreadSearch;
1091 const sql = yield* SqlClient.SqlClient;
1092 const project = ProjectId.make("project:plover");
1093 yield* createProject(project);
1094 const old = ThreadId.make("thread:plover-old");
1095 const recent = ThreadId.make("thread:plover-recent");
1096 yield* Effect.forEach(
1097 [
1098 thread(old, project, { title: "Plover nest", settledAt: at(10) }),
1099 thread(recent, project, { title: "Plover nest", settledAt: at(50) }),
1100 message(recent, "plover-note", "user", "a plover note", { minute: 2 }),
1101 ],
1102 projections.apply,
1103 { discard: true },
1104 );
1105 const run = (request: { readonly titlesOnly?: boolean }) =>
1106 ThreadSearch.searchThreadsWithOlderSettledRows({
1107 request: { query: "plover", olderSettledBefore: DateTime.formatIso(at(30)), ...request },
1108 search: search.search,
1109 searchOlderSettled: search.searchOlderSettled,
1110 sql,
1111 readThreads: (threadIds) =>
1112 projections.readShellSnapshot({ location: "active", threadIds }),
1113 latestSequence: Effect.succeed(7),
1114 });
1115 const withMessages = yield* run({});
1116 assert.deepEqual(
1117 withMessages.matches.map((match) => match.threadId),
1118 [recent],
1119 );
1120 assert.equal(withMessages.snapshotSequence, 7);
1121 assert.deepEqual(
1122 withMessages.threads?.map((row) => row.id),
1123 [old],
1124 );
1125 const titles = yield* run({ titlesOnly: true });
1126 assert.deepEqual(titles.matches, []);
1127 assert.deepEqual(
1128 titles.threads?.map((row) => row.id),
1129 [old],
1130 );
1131 // Without the cut it is the plain message search.
1132 assert.isUndefined(
1133 (yield* ThreadSearch.searchThreadsWithOlderSettledRows({
1134 request: { query: "plover" },
1135 search: search.search,
1136 searchOlderSettled: search.searchOlderSettled,
1137 sql,
1138 readThreads: (threadIds) =>
1139 projections.readShellSnapshot({ location: "active", threadIds }),
1140 latestSequence: Effect.succeed(7),
1141 })).threads,
1142 );
1143 }),
1144 );
1145});
1146
1147it.layer(layerTest)("ThreadSearch.searchThreads", (it) => {
1148 it.effect("keeps room for a message-only thread behind many titled ones", () =>
1149 Effect.gen(function* () {
1150 const projections = yield* ProjectionStore.ProjectionStoreV2;
1151 const search = yield* ThreadSearch.ThreadSearch;
1152 const project = ProjectId.make("project:petrel");
1153 yield* createProject(project);
1154 const messageOnly = ThreadId.make("thread:petrel-message");
1155 yield* Effect.forEach(
1156 [
1157 thread(messageOnly, project),
1158 // The oldest message, so a cut taken before titles drop would lose it.
1159 message(messageOnly, "petrel-old", "user", "a petrel note", { minute: 1 }),
1160 ...Array.from({ length: 50 }, (_, index) => {
1161 const id = ThreadId.make(`thread:petrel-${index}`);
1162 return [
1163 thread(id, project, { title: `Petrel ${index}` }),
1164 message(id, `petrel-${index}`, "user", "petrel again", { minute: 2 }),
1165 ];
1166 }).flat(),
1167 ],
1168 projections.apply,
1169 { discard: true },
1170 );
1171 const { matches } = yield* search.searchThreads({ query: "petrel", limit: 5 });
1172 assert.include(
1173 matches.map((match) => match.threadId),
1174 messageOnly,
1175 );
1176 }),
1177 );
1178
1179 it.effect("leaves titled threads out before the hit limit", () =>
1180 Effect.gen(function* () {
1181 const projections = yield* ProjectionStore.ProjectionStoreV2;
1182 const search = yield* ThreadSearch.ThreadSearch;
1183 const sql = yield* SqlClient.SqlClient;
1184 const project = ProjectId.make("project:hit-limit");
1185 const messageOnly = ThreadId.make("thread:grebe-message");
1186 const titled = ThreadId.make("thread:grebe-titled");
1187 yield* createProject(project);
1188 yield* Effect.forEach(
1189 [
1190 thread(messageOnly, project, { title: "Unrelated" }),
1191 message(messageOnly, "grebe-old", "user", "a grebe note", { minute: 1 }),
1192 thread(titled, project, { title: "Grebe" }),
1193 ],
1194 projections.apply,
1195 { discard: true },
1196 );
1197 // Newer hits than one read holds, all in the titled thread.
1198 yield* sql`
1199 WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 2000)
1200 INSERT INTO orchestration_v2_projection_messages
1201 (message_id, thread_id, role, streaming, created_at, updated_at, payload_json)
1202 SELECT 'grebe-titled-' || i, ${titled}, 'assistant', 0, ${DateTime.formatIso(at(5))},
1203 ${DateTime.formatIso(at(5))}, json_object('text', 'grebe again')
1204 FROM n
1205 `;
1206 const { matches } = yield* search.searchThreads({ query: "grebe", limit: 5 });
1207 assert.include(
1208 matches.map((match) => match.threadId),
1209 messageOnly,
1210 );
1211 }),
1212 );
1213
1214 it.effect("ranks title matches like the palette, then message matches, within one project", () =>
1215 Effect.gen(function* () {
1216 const projections = yield* ProjectionStore.ProjectionStoreV2;
1217 const search = yield* ThreadSearch.ThreadSearch;
1218 const project = ProjectId.make("project:titles");
1219 const otherProject = ProjectId.make("project:titles-other");
1220 yield* createProject(project);
1221 yield* createProject(otherProject);
1222 const wordStart = ThreadId.make("thread:word-start");
1223 const midWord = ThreadId.make("thread:mid-word");
1224 const messageOnly = ThreadId.make("thread:message-only");
1225 const elsewhere = ThreadId.make("thread:elsewhere");
1226 yield* Effect.forEach(
1227 [
1228 thread(midWord, project, { title: "Fix oauth redirect" }),
1229 thread(wordStart, project, { title: "Clean up auth tokens" }),
1230 thread(messageOnly, project, { title: "Unrelated" }),
1231 message(messageOnly, "message-only", "user", "the auth flow broke again"),
1232 thread(elsewhere, otherProject, { title: "Auth everywhere" }),
1233 ],
1234 projections.apply,
1235 { discard: true },
1236 );
1237
1238 // Stronger message hits in another project must not use up this project's limit.
1239 yield* Effect.forEach(
1240 Array.from({ length: 55 }, (_, index) => {
1241 const busy = ThreadId.make(`thread:elsewhere-${index}`);
1242 return [
1243 thread(busy, otherProject),
1244 message(busy, `elsewhere-${index}`, "user", "auth auth auth"),
1245 ];
1246 }).flat(),
1247 projections.apply,
1248 { discard: true },
1249 );
1250 const result = yield* search.searchThreads({ query: "auth", projectId: project, limit: 3 });
1251 assert.deepEqual(
1252 result.matches.map((match) => [match.threadId, match.source]),
1253 [
1254 [wordStart, "title"],
1255 [midWord, "title"],
1256 [messageOnly, "user"],
1257 ],
1258 );
1259 // With one slot, an exact title keeps it.
1260 assert.deepEqual(
1261 (yield* search.searchThreads({ query: "auth", projectId: project, limit: 1 })).matches.map(
1262 (match) => match.threadId,
1263 ),
1264 [wordStart],
1265 );
1266 // Exact matching finds nothing, so one typo is forgiven.
1267 assert.deepEqual(
1268 (yield* search.searchThreads({ query: "tokesn", projectId: project })).matches.map(
1269 (match) => match.title,
1270 ),
1271 ["Clean up auth tokens"],
1272 );
1273 }),
1274 );
1275});
1276
1277it.effect("starts snippets a few words before the match so one line shows it", () =>
1278 Effect.sync(() => {
1279 const snippet = (text: string, word: string) =>
1280 ThreadSearch.buildSearchSnippet(text, text.indexOf(word));
1281 const long = `${"lead ".repeat(60)}the NEEDLE sits here ${"tail ".repeat(60)}`;
1282 const cut = snippet(long, "NEEDLE");
1283 assert.isAtMost(cut.length, 240);
1284 assert.match(cut, /^…(lead ){2,4}the NEEDLE sits here/);
1285 assert.isTrue(cut.endsWith("…"));
1286 // Whitespace collapses only in the shown part, so the offset still lands.
1287 assert.match(snippet(`${"word\n\n ".repeat(60)}un café`, "café"), /un café$/);
1288 // A cut never splits an emoji's surrogate pair.
1289 assert.notMatch(
1290 snippet(`x${"🚀".repeat(200)}needle`, "needle"),
1291 /[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/,
1292 );
1293 // Emphasis and code marks go, as the thread shows the text.
1294 assert.equal(
1295 ThreadSearch.buildSearchSnippet("Use **bold** and `code` here", 0),
1296 "Use bold and code here",
1297 );
1298 assert.equal(ThreadSearch.buildSearchSnippet("Call __init__ first", 0), "Call __init__ first");
1299 // Short text still moves a late match into view.
1300 assert.equal(snippet(`${"word ".repeat(20)}needle`, "needle"), "…word word word word needle");
1301 }),
1302);
1303
1304it.effect("searches on the reader worker while the writer is busy", () =>
1305 Effect.gen(function* () {
1306 const fileSystem = yield* FileSystem.FileSystem;
1307 const path = yield* Path.Path;
1308 const directory = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-thread-search-" });
1309 const layerFile = Layer.mergeAll(
1310 ThreadSearch.layer,
1311 ProjectionStore.layer,
1312 ProjectStore.layer,
1313 ).pipe(
1314 Layer.provideMerge(SqlitePersistence.layerFromPath(path.join(directory, "state.sqlite"))),
1315 Layer.provide(NodeServices.layer),
1316 );
1317
1318 yield* Effect.gen(function* () {
1319 const projections = yield* ProjectionStore.ProjectionStoreV2;
1320 const search = yield* ThreadSearch.ThreadSearch;
1321 const sql = yield* SqlClient.SqlClient;
1322 const project = ProjectId.make("project:search-reader");
1323 const threadId = ThreadId.make("thread:reader");
1324 yield* createProject(project);
1325 yield* Effect.forEach(
1326 [thread(threadId, project), message(threadId, "reader", "user", "needle on the reader")],
1327 projections.apply,
1328 { discard: true },
1329 );
1330
1331 const writerHeld = yield* Deferred.make<void>();
1332 const releaseWriter = yield* Deferred.make<void>();
1333 const writer = yield* Effect.forkChild(
1334 sql.withTransaction(
1335 Deferred.succeed(writerHeld, undefined).pipe(
1336 Effect.andThen(Deferred.await(releaseWriter)),
1337 ),
1338 ),
1339 );
1340 yield* Deferred.await(writerHeld);
1341 const result = yield* search.search({ query: "needle" });
1342 yield* Deferred.succeed(releaseWriter, undefined);
1343 yield* Fiber.join(writer);
1344
1345 assert.deepEqual(
1346 result.matches.map((match) => match.threadId),
1347 [threadId],
1348 );
1349 }).pipe(Effect.provide(layerFile));
1350 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
1351);