1import type { ThreadTitleMessage } from "@t3tools/provider-core/server/threadTitleContext";
2
3export const threadTitleEvaluationCases = [
4 {
5 id: "referenced-thread-startup",
6 source: "Maintainer report of chart thread references overriding a startup question",
7 request: "Explain why one example thread failed to start and another needed a retry.",
8 previousTitle: "Fix Chart Formatting",
9 messages: [
10 {
11 role: "user",
12 text: "[xlsx: chart data labels use the cells' number format](t3-context://v1/thread/thread_labels) why could this thread not start up? [Run chart format tests on external target](t3-context://v1/thread/thread_tests) why did i have to say it like this over here to get it working?",
13 },
14 ],
15 rubric:
16 "Name thread startup or prompt/retry behavior. Chart formatting and tests are titles of example threads, not the requested work. Replace the inaccurate previous title.",
17 },
18 {
19 id: "referenced-thread-subject",
20 source: "Control for maintainer report of referenced thread titles overriding the request",
21 request: "Continue the actual chart formatting work in a referenced thread.",
22 previousTitle: "Chart Label Number Formats",
23 messages: [
24 {
25 role: "user",
26 text: "Continue [xlsx: chart data labels use the cells' number format](t3-context://v1/thread/thread_labels). Make the chart labels inherit the cells' number formats.",
27 },
28 ],
29 rubric:
30 "Name chart label number formatting. Referenced titles can supply the subject when the user explicitly continues that work.",
31 },
32 {
33 id: "linked-reset-credits",
34 source: "https://github.com/pingdotgg/t3code/pull/10462",
35 request: "Review the reset credit routing change.",
36 previousTitle: "Review PR 10462",
37 messages: [{ role: "user", text: "Review https://github.com/pingdotgg/t3code/pull/10462" }],
38 rubric: "Name reset credit routing. Distinguish it from displaying credit balances.",
39 },
40 {
41 id: "onboarding-merge",
42 source: "https://github.com/pingdotgg/t3code/pull/10465",
43 request: "Make onboarding one shared wizard across computers, then merge when green.",
44 previousTitle: "Finish onboarding PR",
45 messages: [
46 { role: "user", text: "Make onboarding one shared wizard across computers." },
47 {
48 role: "assistant",
49 text: "The wizard now handles pairing, agent selection, and project import.",
50 },
51 { role: "user", text: "File a PR and merge it when green." },
52 ],
53 rubric: "Keep the multi-computer onboarding subject. Do not title it after merging.",
54 },
55 {
56 id: "vague-opening",
57 source: "Existing lazy thread feed title scenario",
58 request: "A failing test is later identified as a lazy thread feed mismatch.",
59 previousTitle: "Fix failing test",
60 messages: [
61 { role: "user", text: "Fix this failing test." },
62 {
63 role: "assistant",
64 text: "The lazy thread feed test expects a full message body before the client requests it.",
65 },
66 ],
67 rubric: "Name the lazy thread feed test. Do not invent a wider mobile regression.",
68 },
69 {
70 id: "scope-change",
71 source: "Title context budget scenario",
72 request: "Change the goal from QR layout to pairing expiry, despite long assistant replies.",
73 previousTitle: "Improve QR layout",
74 messages: [
75 { role: "user", text: "Improve QR sharing layout." },
76 {
77 role: "user",
78 text: "Change of plan. Fix pairing token expiry. Keep remote access working.",
79 },
80 {
81 role: "assistant",
82 text: "The token expires before redemption. " + "Implementation detail. ".repeat(800),
83 },
84 { role: "user", text: "Ship it." },
85 ],
86 rubric: "Name pairing expiry and honor the explicit scope change.",
87 },
88 {
89 id: "review-umbrella",
90 source: "Existing subagent monitoring title scenario",
91 request: "Review subagent monitoring risks. A Codex roster issue is one finding.",
92 previousTitle: "Review subagent monitoring risks",
93 messages: [
94 { role: "user", text: "Review subagent monitoring risks." },
95 {
96 role: "assistant",
97 text: "One finding is a stale Codex roster. " + "Roster detail. ".repeat(800),
98 },
99 { role: "user", text: "Fix the findings and babysit CI." },
100 ],
101 rubric: "Preserve the monitoring review scope. The previous title can stay unchanged.",
102 },
103 {
104 id: "long-opening",
105 source: "Title message truncation scenario",
106 request: "Investigate Android pairing while preserving the iOS flow.",
107 previousTitle: "Inspect logs",
108 messages: [
109 {
110 role: "user",
111 text:
112 "Investigate Android pairing. " +
113 "Connection logs. ".repeat(800) +
114 " Preserve the iOS pairing flow.",
115 },
116 ],
117 rubric: "Name Android pairing. Logs are supporting evidence.",
118 },
119 {
120 id: "research",
121 source: "Maintainer title generation request",
122 request: "How can we improve title generation in T3 Code?",
123 previousTitle: "Research title gen improvements",
124 messages: [
125 { role: "user", text: "How can we improve title gen further in T3 Code?" },
126 {
127 role: "assistant",
128 text: "Prioritize user messages, refine vague titles once, and resolve PR subjects.",
129 },
130 {
131 role: "user",
132 text: "Make these changes and file a PR. Babysit until everything is green.",
133 },
134 ],
135 rubric: "Keep title generation as the subject. Do not focus on filing the PR.",
136 },
137 {
138 id: "project-name",
139 source: "https://github.com/pingdotgg/t3code/pull/33418",
140 request: "Make hyperdrive init create a default config instead of crashing.",
141 projectTitle: "Hyperdrive",
142 previousTitle: "Hyperdrive Init Missing Config Crash",
143 messages: [
144 {
145 role: "user",
146 text: "hyperdrive init crashes when ~/.hyperdrive/config.toml is missing. Make hyperdrive create a default config instead of throwing.",
147 },
148 {
149 role: "assistant",
150 text: "Hyperdrive's loadConfig now writes a default config.toml when the file is absent.",
151 },
152 ],
153 rubric: "Name the default config on init. Leave out Hyperdrive, the project name.",
154 },
155] satisfies ReadonlyArray<{
156 id: string;
157 source: string;
158 request: string;
159 projectTitle?: string;
160 previousTitle: string;
161 messages: ReadonlyArray<ThreadTitleMessage>;
162 rubric: string;
163}>;