1import * as Alchemy from "alchemy";
2import * as Axiom from "@alchemy.run/axiom";
3import * as Output from "alchemy/Output";
4import * as Config from "effect/Config";
5import * as Effect from "effect/Effect";
6import * as Option from "effect/Option";
7
8import { relayResourceNameForStage } from "./deploymentConfig.ts";
9
10const RUNBOOK_URL =
11 "https://github.com/pingdotgg/t3code/blob/main/docs/operations/relay-observability.md#alerts";
12
13const LINK_ROUTE = `"/v1/client/environment-links"`;
14const TUNNEL_ROUTES = `"/v1/environments/:environmentId/tunnel", "/v1/environments/:environmentId/tunnel/recovery"`;
15const CLOUDFLARE_API = `['attributes.url.full'] startswith "https://api.cloudflare.com"`;
16const STATUS = `toint(['attributes.http.response.status_code'])`;
17// The tunnel reaper records its counters on this span every five minutes,
18// including while cleanup is off.
19const SWEEP = `name == "relay.managed_endpoint_reaper.sweep"`;
20const sweepCounter = (field: string) =>
21 `toint(['attributes.custom']['relay.managed_endpoint_reaper.${field}'])`;
22
23// Status codes, not span errors: a request cut off by the relay deadline ends
24// as an interrupted span with no error flag, but still answers 504.
25// Below `minimum` requests the rate reads 0, so a quiet window cannot page.
26const serverFailurePercent = (dataset: string, routes: string, minimum: number) =>
27 [
28 `['${dataset}']`,
29 `| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] in (${routes})`,
30 `| summarize total = count(), failed = countif(${STATUS} >= 500)`,
31 `| project failure_pct = iff(total >= ${minimum}, 100.0 * failed / total, 0.0)`,
32 ].join("\n");
33
34type Severity = "page" | "warn";
35
36interface RelayMonitor {
37 readonly id: string;
38 readonly severity: Severity;
39 readonly name: string;
40 /** Why it fires and the healthy baseline. The runbook link is appended. */
41 readonly description: string;
42 readonly query: (dataset: string) => string;
43 readonly operator: "Above" | "AboveOrEqual" | "Below";
44 readonly threshold: number;
45 readonly rangeMinutes: number;
46 readonly intervalMinutes?: number;
47 readonly triggerFromNRuns?: number;
48 readonly alertOnNoData?: boolean;
49}
50
51// Thresholds come from 2026-10-16..19 prod data: the worst healthy window,
52// and the outage (Cloudflare API 429s stalling every provision).
53// Paging monitors need two bad runs in a row; warnings catch the slow
54// degradations an outage threshold has to ignore.
55const relayMonitors: ReadonlyArray<RelayMonitor> = [
56 {
57 id: "RelayLinkFailureRateMonitor",
58 severity: "page",
59 name: "T3 Connect relay: links failing",
60 description:
61 "Share of POST /v1/client/environment-links (turning on T3 Connect) answering 5xx over 15 min. Healthy: median 11%, worst 75%. #16810: 87-96%.",
62 query: (dataset) => serverFailurePercent(dataset, LINK_ROUTE, 20),
63 operator: "Above",
64 threshold: 80,
65 rangeMinutes: 15,
66 triggerFromNRuns: 2,
67 },
68 {
69 id: "RelayTunnelFailureRateMonitor",
70 severity: "page",
71 name: "T3 Connect relay: tunnel recovery failing",
72 description:
73 "Share of host tunnel registration and recovery answering 5xx over 15 min. Hosts whose tunnel drops cannot come back. Healthy: median 4%, worst 52%. #16810: 83-94%.",
74 query: (dataset) => serverFailurePercent(dataset, TUNNEL_ROUTES, 200),
75 operator: "Above",
76 threshold: 60,
77 rangeMinutes: 15,
78 triggerFromNRuns: 2,
79 },
80 {
81 id: "RelayConnectReachabilityMonitor",
82 severity: "page",
83 name: "T3 Connect relay: environments unreachable",
84 description:
85 "Share of environments clients tried to reach that connected at least once in 30 min. Offline hosts keep this below 100%, and retry storms against them do not move it. Healthy: 65-72%.",
86 query: (dataset) =>
87 [
88 `['${dataset}']`,
89 `| where name == "relay.environment_connector.connect"`,
90 `| extend environment = tostring(['attributes.custom']['relay.environment_id'])`,
91 `| summarize environments = dcount(environment), connected = dcountif(environment, error != true)`,
92 `| project reachable_pct = iff(environments >= 1000, 100.0 * connected / environments, 100.0)`,
93 ].join("\n"),
94 operator: "Below",
95 threshold: 55,
96 rangeMinutes: 30,
97 triggerFromNRuns: 2,
98 },
99 {
100 id: "RelayPushFailureRateMonitor",
101 severity: "page",
102 name: "T3 Connect relay: push delivery failing",
103 description:
104 "Share of APNs and FCM delivery jobs failing over 15 min, e.g. an expired APNs key or FCM credential. Healthy: APNs under 0.3%, FCM under 4%.",
105 query: (dataset) =>
106 [
107 `['${dataset}']`,
108 `| where name in ("relay.apns_deliveries.process_signed_job", "relay.fcm.process")`,
109 `| summarize total = count(), failed = countif(error == true)`,
110 `| project failure_pct = iff(total >= 100, 100.0 * failed / total, 0.0)`,
111 ].join("\n"),
112 operator: "Above",
113 threshold: 10,
114 rangeMinutes: 15,
115 triggerFromNRuns: 2,
116 },
117 {
118 id: "RelayCloudflareRateLimitedMonitor",
119 severity: "page",
120 name: "T3 Connect relay: Cloudflare API rate limited",
121 description:
122 "Cloudflare API 429s, plus calls the relay refused while paused after one, per 10 min. Sustained at this level, links and recoveries stall. Peak 429s before #16810: 1,286; during it: ~11k.",
123 query: (dataset) =>
124 `['${dataset}']\n| where (${CLOUDFLARE_API} and ${STATUS} == 429) or name == "relay.cloudflare_api_pause.rejected"\n| summarize rate_limited = count()`,
125 operator: "AboveOrEqual",
126 threshold: 500,
127 rangeMinutes: 10,
128 triggerFromNRuns: 2,
129 },
130 {
131 id: "RelayTrafficDroppedMonitor",
132 severity: "page",
133 name: "T3 Connect relay: traffic dropped",
134 description:
135 "Relay requests per 10 min collapsed: Worker down, broken deploy, or trace export stopped. Healthy minimum: ~69k.",
136 query: (dataset) => `['${dataset}']\n| where kind == "server"\n| summarize requests = count()`,
137 operator: "Below",
138 threshold: 10_000,
139 rangeMinutes: 10,
140 triggerFromNRuns: 2,
141 alertOnNoData: true,
142 },
143 {
144 id: "RelayCloudflareRateLimitWarningMonitor",
145 severity: "warn",
146 name: "T3 Connect relay: Cloudflare API near its limit",
147 description:
148 "Any sustained Cloudflare API 429s or paused calls. 429 bursts ran for days before #16810; raise the account limit or cut call volume before they grow.",
149 query: (dataset) =>
150 `['${dataset}']\n| where (${CLOUDFLARE_API} and ${STATUS} == 429) or name == "relay.cloudflare_api_pause.rejected"\n| summarize rate_limited = count()`,
151 operator: "AboveOrEqual",
152 threshold: 20,
153 rangeMinutes: 10,
154 triggerFromNRuns: 2,
155 },
156 {
157 id: "RelayLinkChronicFailureMonitor",
158 severity: "warn",
159 name: "T3 Connect relay: links degraded",
160 description:
161 "Share of links answering 5xx over 6 h, mostly 9 s deadline 504s from slow provisioning. Healthy days: 10-18%. Degraded days before #16810: 28-33%.",
162 query: (dataset) => serverFailurePercent(dataset, LINK_ROUTE, 100),
163 operator: "Above",
164 threshold: 20,
165 rangeMinutes: 360,
166 intervalMinutes: 30,
167 },
168 {
169 id: "RelayDatabaseLatencyMonitor",
170 severity: "warn",
171 name: "T3 Connect relay: database slow",
172 description:
173 "p90 Postgres query time over 15 min, in ms. Every relay request runs several queries in sequence, so this sets how close provisioning gets to the 9 s deadline. Healthy: ~200 ms.",
174 query: (dataset) =>
175 `['${dataset}']\n| where name == "postgres"\n| summarize p90 = percentile(duration, 90)\n| project p90_ms = p90 / 1000000`,
176 operator: "Above",
177 threshold: 500,
178 rangeMinutes: 15,
179 triggerFromNRuns: 2,
180 },
181 {
182 id: "RelayTunnelCleanupStoppedMonitor",
183 severity: "warn",
184 name: "T3 Connect relay: tunnel cleanup stopped",
185 description:
186 "Tunnel cleanup sweeps that finished in 30 min. Healthy: 6. The cron is not running, or every sweep fails, as during the 2026-10-19 Cloudflare 429s when listing tunnels timed out. Idle tunnels pile up and are billed meanwhile.",
187 query: (dataset) =>
188 `['${dataset}']\n| where ${SWEEP} and error != true\n| summarize sweeps = count()`,
189 operator: "Below",
190 threshold: 1,
191 rangeMinutes: 30,
192 triggerFromNRuns: 2,
193 alertOnNoData: true,
194 },
195 {
196 id: "RelayTunnelCleanupFailingMonitor",
197 severity: "warn",
198 name: "T3 Connect relay: tunnel deletions failing",
199 description:
200 "Cloudflare tunnel deletions that failed in 30 min. Healthy: 0; the worst 30 min so far had 3. The relay logs a warning with the Cloudflare error for each.",
201 query: (dataset) =>
202 `['${dataset}']\n| where ${SWEEP}\n| summarize failed = sum(${sweepCounter("failed")})`,
203 operator: "Above",
204 threshold: 10,
205 rangeMinutes: 30,
206 triggerFromNRuns: 2,
207 },
208];
209
210interface QueryChart {
211 readonly id: string;
212 readonly name: string;
213 readonly type: "TimeSeries" | "Table";
214 readonly query: { readonly apl: string };
215}
216
217const timeSeries = (id: string, name: string, apl: string): QueryChart => ({
218 id,
219 name,
220 type: "TimeSeries",
221 query: { apl },
222});
223
224const relayDashboardCharts = (dataset: string): ReadonlyArray<QueryChart> => {
225 const source = `['${dataset}']`;
226 const linkRequests = `${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] == ${LINK_ROUTE}`;
227 const tunnelRequests = `${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and ['attributes.http.route'] in (${TUNNEL_ROUTES})`;
228 const clientConnections = `${source}\n| where name == "relay.connection.attempt" and error != true\n| extend surface = tostring(['resource.custom']['t3.client.surface'])\n| where isnotempty(surface)`;
229 return [
230 timeSeries(
231 "links",
232 "Links by outcome",
233 `${linkRequests}\n| summarize succeeded = countif(${STATUS} < 400), rejected = countif(${STATUS} >= 400 and ${STATUS} < 500), failed = countif(${STATUS} >= 500) by bin_auto(_time)`,
234 ),
235 timeSeries(
236 "link-latency",
237 "Link latency (9 s deadline)",
238 `${linkRequests}\n| summarize p50 = percentile(duration, 50), p90 = percentile(duration, 90) by bin_auto(_time)`,
239 ),
240 timeSeries(
241 "reachability",
242 "Environments reachable per 30 min (%)",
243 `${source}\n| where name == "relay.environment_connector.connect"\n| extend environment = tostring(['attributes.custom']['relay.environment_id'])\n| summarize environments = dcount(environment), connected = dcountif(environment, error != true) by bin(_time, 30m)\n| project _time, reachable_pct = 100.0 * connected / environments`,
244 ),
245 timeSeries(
246 "tunnel",
247 "Tunnel registration and recovery by outcome",
248 `${tunnelRequests}\n| summarize succeeded = countif(${STATUS} < 400), rejected = countif(${STATUS} >= 400 and ${STATUS} < 500), failed = countif(${STATUS} >= 500) by bin_auto(_time)`,
249 ),
250 timeSeries(
251 "cloudflare-api",
252 "Cloudflare API calls by status",
253 // Calls refused during a rate-limit pause never reach Cloudflare, so
254 // they are their own series or a storm would look like a quiet period.
255 `${source}\n| where ${CLOUDFLARE_API} or name == "relay.cloudflare_api_pause.rejected"\n| summarize count() by bin_auto(_time), status = case(name == "relay.cloudflare_api_pause.rejected", "paused", isnull(${STATUS}), "no response", tostring(${STATUS}))`,
256 ),
257 timeSeries(
258 "provision-paths",
259 "Tunnel provisions by path",
260 `${source}\n| where name == "relay.managed_endpoint_provider.provision"\n| summarize reused_ready_tunnel = countif(error != true and tobool(['attributes.custom']['relay.managed_endpoint.reused_ready_tunnel']) == true), full_provision = countif(error != true and isnull(['attributes.custom']['relay.managed_endpoint.reused_ready_tunnel'])), failed = countif(error == true) by bin_auto(_time)`,
261 ),
262 timeSeries(
263 "postgres",
264 "Postgres query time",
265 `${source}\n| where name == "postgres"\n| summarize p50 = percentile(duration, 50), p90 = percentile(duration, 90) by bin_auto(_time)`,
266 ),
267 timeSeries(
268 "push",
269 "Push delivery jobs",
270 `${source}\n| where name in ("relay.apns_deliveries.process_signed_job", "relay.fcm.process")\n| summarize delivered = countif(error != true), failed = countif(error == true) by bin_auto(_time), service = iff(name == "relay.fcm.process", "fcm", "apns")`,
271 ),
272 timeSeries(
273 "tunnel-cleanup",
274 "Tunnel cleanup per sweep",
275 `${source}\n| where ${SWEEP}\n| summarize deleted = sum(${sweepCounter("deleted")}), deleted_legacy = sum(${sweepCounter("deletedLegacy")}), would_delete = sum(${sweepCounter("wouldDelete")}), would_delete_legacy = sum(${sweepCounter("wouldDeleteLegacy")}), failed = sum(${sweepCounter("failed")}), sweeps_failed = countif(error == true) by bin_auto(_time)`,
276 ),
277 // Cloudflare's own counts for this stage, including tunnels the reaper
278 // skips, so this is the idle tunnels still billed.
279 timeSeries(
280 "tunnel-backlog",
281 "Idle tunnels in Cloudflare",
282 `${source}\n| where ${SWEEP}\n| summarize down = max(${sweepCounter("totalDown")}), never_connected = max(${sweepCounter("totalInactive")}) by bin_auto(_time)`,
283 ),
284 timeSeries(
285 "server-errors",
286 "5xx by route",
287 `${source}\n| where kind == "server" and ${STATUS} >= 500\n| summarize count() by bin_auto(_time), route = tostring(['attributes.http.route']), status = ${STATUS}`,
288 ),
289 {
290 id: "link-failures",
291 name: "Link failures by reason",
292 type: "Table",
293 query: {
294 apl: `${source}\n| where name == "relay.environment_linker.link" and error == true\n| extend message = tostring(['status.message'])\n| summarize failures = count() by reason = case(message contains "failed during '", strcat("provisioning: ", extract("failed during '([^']+)'", 1, message)), message startswith "Managed tunnel limit reached", "managed tunnel limit reached", substring(message, 0, 80))\n| top 10 by failures`,
295 },
296 },
297 // Connecting, as users see it: web, desktop and mobile export their
298 // connection spans into this dataset, in the same traces as the relay's
299 // and the host's spans. An attempt lasts until initial sync finishes.
300 timeSeries(
301 "connect-time",
302 "Connecting: time to connected by surface",
303 `${clientConnections}\n| summarize p50 = percentile(duration, 50), p90 = percentile(duration, 90) by bin_auto(_time), surface`,
304 ),
305 timeSeries(
306 "connect-outcomes",
307 "Connecting: attempts by outcome",
308 `${source}\n| where name == "relay.connection.attempt" and isnotempty(['resource.custom']['t3.client.surface'])\n| summarize connected = countif(error != true), failed = countif(error == true) by bin_auto(_time)`,
309 ),
310 {
311 id: "connect-phases",
312 name: "Connecting: where the time goes",
313 type: "Table",
314 query: {
315 // Failed direct route checks stay in: waiting out an unreachable LAN
316 // address before falling back to the relay is the slowest step.
317 apl: [
318 source,
319 `| where isnotempty(['resource.custom']['t3.client.surface'])`,
320 `| extend url = tostring(['attributes.url.full'])`,
321 `| extend phase = case(`,
322 ` name == "ConnectionDriver.checkRoute", "0. Direct route check, before any relay fallback",`,
323 ` name == "relay.connection.cloudSessionToken.resolve", "1. Clerk session token",`,
324 ` name == "clientRuntime.managedRelay.exchangeAccessToken", "2. Relay token exchange",`,
325 ` kind == "client" and url startswith "https://relay.t3.codes/v1/environments/" and url endswith "/connect", "3. Relay connect, minting a host credential",`,
326 ` name == "environment.authorization.descriptor", "4. Host descriptor",`,
327 ` name == "environment.authorization.accessToken.exchange", "5. Host token exchange",`,
328 ` name == "clientRuntime.connection.remote.createDpopSocketUrl", "6. Socket ticket",`,
329 ` name == "environment.initialSync", "7. Initial sync",`,
330 ` name == "relay.connection.attempt", "Total: time to connected",`,
331 ` "")`,
332 `| where phase != "" and (error != true or phase startswith "0.")`,
333 `| summarize calls = count(), p50 = percentile(duration, 50), p90 = percentile(duration, 90) by phase`,
334 `| order by phase asc`,
335 ].join("\n"),
336 },
337 },
338 timeSeries(
339 "relay-latency-by-continent",
340 "Relay token and connect latency by user continent (p50)",
341 `${source}\n| where kind == "server" and ['attributes.http.request.method'] == "POST" and (['attributes.http.route'] == "/v1/client/dpop-token" or ['attributes.http.route'] == "/v1/environments/:environmentId/connect")\n| extend continent = tostring(['attributes.custom']['relay.edge.continent'])\n| where isnotempty(continent)\n| summarize p50 = percentile(duration, 50) by bin_auto(_time), continent`,
342 ),
343 ];
344};
345
346export const notifierConfig = (name: string) =>
347 Config.String(name).pipe(
348 Config.option,
349 Config.map(Option.flatMap((value) => Option.liftPredicate(value.trim(), Boolean))),
350 );
351
352/**
353 * A deployment opts in by naming the notifier that pages (incident.io for
354 * prod). Personal stages and forks leave it unset and get no monitors or
355 * dashboard. Warnings go to their own notifier when one is set, and are only
356 * visible in Axiom otherwise.
357 */
358export const RelayMonitoring = Effect.fn("RelayMonitoring")(function* (traces: Axiom.Dataset) {
359 const { stage } = yield* Alchemy.Stack;
360 const page = yield* notifierConfig("RELAY_ALERT_NOTIFIER_ID");
361 if (Option.isNone(page)) return;
362 const notifiers: Record<Severity, Option.Option<string>> = {
363 page,
364 warn: yield* notifierConfig("RELAY_WARNING_NOTIFIER_ID"),
365 };
366
367 for (const monitor of relayMonitors) {
368 yield* Axiom.Monitor(monitor.id, {
369 name: stage === "prod" ? monitor.name : `[${stage}] ${monitor.name}`,
370 description: `${monitor.description} Runbook: ${RUNBOOK_URL}`,
371 type: "Threshold",
372 aplQuery: Output.map(traces.name, monitor.query),
373 operator: monitor.operator,
374 threshold: monitor.threshold,
375 intervalMinutes: monitor.intervalMinutes ?? 5,
376 rangeMinutes: monitor.rangeMinutes,
377 triggerFromNRuns: monitor.triggerFromNRuns ?? 1,
378 alertOnNoData: monitor.alertOnNoData ?? false,
379 notifierIds: Option.toArray(notifiers[monitor.severity]),
380 });
381 }
382
383 yield* Axiom.Dashboard("RelayHealthDashboard", {
384 dashboard: Output.map(traces.name, (dataset) => {
385 const charts = relayDashboardCharts(dataset);
386 return {
387 name: relayResourceNameForStage("t3-code-relay-health", stage),
388 // Shared with the whole org. An empty owner only means that for API
389 // tokens; deploys with a personal token would make it private.
390 owner: "X-AXIOM-EVERYONE",
391 description: `T3 Connect relay health. Alerts and runbook: ${RUNBOOK_URL}`,
392 refreshTime: 60,
393 schemaVersion: 2,
394 timeWindowStart: "qr-now-24h",
395 timeWindowEnd: "qr-now",
396 charts: [...charts],
397 // Two charts per row on Axiom's 12-column grid.
398 layout: charts.map((chart, index) => ({
399 i: chart.id,
400 x: (index % 2) * 6,
401 y: Math.floor(index / 2) * 6,
402 w: 6,
403 h: 6,
404 })),
405 };
406 }),
407 });
408});