1import * as Alchemy from "alchemy";
2import * as Axiom from "@alchemy.run/axiom";
3import * as Output from "alchemy/Output";
4import * as Effect from "effect/Effect";
5import * as Option from "effect/Option";
6
7import { relayResourceNameForStage } from "../deploymentConfig.ts";
8import { notifierConfig } from "../monitoring.ts";
9import { isPreviewStage } from "../previewStages.ts";
10
11const RUNBOOK_URL =
12 "https://github.com/pingdotgg/t3code/blob/main/docs/operations/relay-observability.md#mobile-crashes-and-errors";
13
14// A new dataset has none of these fields until a report carries them, and an
15// unknown field fails the whole query, so every one has a default.
16const field = (name: string, fallback: string) => `column_ifexists('${name}', ${fallback})`;
17const reports = (dataset: string) =>
18 [
19 `['${dataset}']`,
20 `| extend source = tostring(${field("attributes.expo.error.source", "''")}),`,
21 ` fatal = tobool(${field("attributes.expo.error.is_fatal", "false")}),`,
22 ` js_fatal = tobool(${field("attributes.relay.crash.js_fatal", "false")}),`,
23 ` platform = tostring(${field("resource.os.name", "''")}),`,
24 ` install = tostring(${field("resource.expo.eas_client.id", "''")}),`,
25 ` error_type = tostring(${field("attributes.exception.type", "''")}),`,
26 ` message = substring(tostring(${field("attributes.exception.message", "''")}), 0, 140),`,
27 ` version = strcat(tostring(${field("service.version", "''")}), ' (', tostring(${field("resource.expo.app.build_number", "''")}), ')')`,
28 `| where isnotempty(source) and isnotempty(platform)`,
29 // A fatal JS error also ends in a native abort; counting both would double every JS crash.
30 `| extend kind = case(source == 'nativeCrash' and js_fatal, 'abort after JS crash', source == 'nativeCrash', 'native crash', fatal, 'JS crash', 'handled error')`,
31 ].join("\n");
32const crashes = (dataset: string) =>
33 `${reports(dataset)}\n| where kind in ('JS crash', 'native crash')`;
34const handledErrors = (dataset: string) => `${reports(dataset)}\n| where kind == 'handled error'`;
35const ingest = (traces: string) =>
36 `['${traces}']\n| where name == 'relay.mobile_telemetry.ingest'\n| extend signal = tostring(['attributes.custom']['relay.mobile_telemetry.signal']), outcome = tostring(['attributes.custom']['relay.mobile_telemetry.outcome'])`;
37
38type Severity = "page" | "warn";
39
40interface Datasets {
41 readonly logs: string;
42 readonly traces: string;
43}
44
45interface MobileMonitor {
46 readonly id: string;
47 readonly severity: Severity;
48 readonly name: string;
49 /** Why it fires. The runbook link is appended. */
50 readonly description: string;
51 readonly query: (datasets: Datasets) => string;
52 readonly props: Pick<
53 Axiom.MonitorProps,
54 | "type"
55 | "operator"
56 | "threshold"
57 | "rangeMinutes"
58 | "intervalMinutes"
59 | "compareDays"
60 | "tolerance"
61 | "alertOnNoData"
62 | "triggerFromNRuns"
63 >;
64}
65
66// No production baseline exists yet: the spike monitor learns its own, and the
67// thresholds below are starting points to tune after two weeks of data.
68const mobileMonitors: ReadonlyArray<MobileMonitor> = [
69 {
70 id: "MobileCrashSpikeMonitor",
71 severity: "page",
72 name: "T3 Code mobile: crash spike",
73 description:
74 "Crashes (JS and native, each counted once) per 15 min well above the same time over the last 7 days. Usually a bad OTA update or store build: compare app versions on the dashboard.",
75 query: ({ logs }) => `${crashes(logs)}\n| summarize crashes = count() by bin_auto(_time)`,
76 props: {
77 type: "AnomalyDetection",
78 operator: "Above",
79 compareDays: 7,
80 tolerance: 50,
81 rangeMinutes: 15,
82 intervalMinutes: 15,
83 },
84 },
85 {
86 id: "MobileCrashSpreadMonitor",
87 severity: "page",
88 name: "T3 Code mobile: one crash hitting many installs",
89 description:
90 "The most widespread crash reached this many installs in an hour. Its message and versions are at the top of the dashboard's Top crashes table.",
91 query: ({ logs }) =>
92 `${crashes(logs)}\n| summarize installs = dcount(install) by platform, error_type, message\n| summarize worst = max(installs)`,
93 props: {
94 type: "Threshold",
95 operator: "AboveOrEqual",
96 threshold: 25,
97 rangeMinutes: 60,
98 intervalMinutes: 15,
99 },
100 },
101 {
102 id: "MobileTelemetryStoppedMonitor",
103 severity: "warn",
104 name: "T3 Code mobile: reports stopped",
105 description:
106 "No startup metrics forwarded in an hour. Every launch sends some, so the relay route, the Axiom token, or the app's endpoint is broken, and crashes go unseen too.",
107 query: ({ traces }) =>
108 `${ingest(traces)}\n| where signal == 'metrics' and outcome == 'forwarded'\n| summarize forwarded = count()`,
109 props: {
110 type: "Threshold",
111 operator: "Below",
112 threshold: 1,
113 rangeMinutes: 60,
114 intervalMinutes: 15,
115 alertOnNoData: true,
116 triggerFromNRuns: 2,
117 },
118 },
119 {
120 id: "MobileSymbolicationFailingMonitor",
121 severity: "warn",
122 name: "T3 Code mobile: crash stacks not symbolicated",
123 description:
124 "Share of release JS exceptions over 6 h whose stack stayed minified. Map uploads are failing or the relay cannot read them; relay.symbolication.status says which.",
125 query: ({ logs }) =>
126 [
127 `['${logs}']`,
128 `| extend status = tostring(${field("attributes.relay.symbolication.status", "''")})`,
129 `| where isnotempty(status) and status !in ('no_bundle_id', 'no_bundle_frames')`,
130 `| summarize total = count(), minified = countif(status != 'symbolicated')`,
131 `| project minified_pct = iff(total >= 20, 100.0 * minified / total, 0.0)`,
132 ].join("\n"),
133 props: {
134 type: "Threshold",
135 operator: "Above",
136 threshold: 50,
137 rangeMinutes: 360,
138 intervalMinutes: 60,
139 },
140 },
141];
142
143interface QueryChart {
144 readonly id: string;
145 readonly name: string;
146 readonly type: "TimeSeries" | "Table" | "Statistic";
147 readonly query: { readonly apl: string };
148 /** Grid columns out of 12. */
149 readonly width: 4 | 6 | 12;
150}
151
152const chart = (
153 type: QueryChart["type"],
154 width: QueryChart["width"],
155 id: string,
156 name: string,
157 apl: string,
158): QueryChart => ({ id, name, type, width, query: { apl } });
159
160const mobileDashboardCharts = ({ logs, traces }: Datasets): ReadonlyArray<QueryChart> => [
161 chart("Statistic", 4, "crashes", "Crashes", `${crashes(logs)}\n| summarize count()`),
162 chart(
163 "Statistic",
164 4,
165 "crashed-installs",
166 "Installs that crashed",
167 `${crashes(logs)}\n| summarize dcount(install)`,
168 ),
169 chart(
170 "Statistic",
171 4,
172 "handled-errors",
173 "Handled errors",
174 `${handledErrors(logs)}\n| summarize count()`,
175 ),
176 chart(
177 "TimeSeries",
178 6,
179 "crashes-over-time",
180 "Crashes by platform",
181 `${crashes(logs)}\n| summarize count() by bin_auto(_time), series = strcat(platform, ' ', kind)`,
182 ),
183 chart(
184 "TimeSeries",
185 6,
186 "handled-over-time",
187 "Handled errors by platform",
188 `${handledErrors(logs)}\n| summarize count() by bin_auto(_time), platform`,
189 ),
190 chart(
191 "Table",
192 12,
193 "top-crashes",
194 "Top crashes",
195 `${crashes(logs)}\n| summarize installs = dcount(install), crashes = count(), first_seen = min(_time), last_seen = max(_time), versions = make_set(version, 3) by platform, kind, error_type, message\n| order by installs desc, crashes desc\n| take 15`,
196 ),
197 chart(
198 "Table",
199 12,
200 "top-handled",
201 "Top handled errors",
202 `${handledErrors(logs)}\n| summarize installs = dcount(install), errors = count(), last_seen = max(_time), versions = make_set(version, 3) by platform, error_type, message\n| order by installs desc, errors desc\n| take 15`,
203 ),
204 chart(
205 "Table",
206 6,
207 "by-version",
208 "Crashes by app version",
209 `${crashes(logs)}\n| summarize crashes = count(), installs = dcount(install) by platform, version\n| order by crashes desc\n| take 10`,
210 ),
211 chart(
212 "TimeSeries",
213 6,
214 "symbolication",
215 "JS stacks by symbolication outcome",
216 `['${logs}']\n| extend status = tostring(${field("attributes.relay.symbolication.status", "''")})\n| where isnotempty(status)\n| summarize count() by bin_auto(_time), status`,
217 ),
218 chart(
219 "TimeSeries",
220 12,
221 "ingest",
222 "Reports reaching the relay",
223 `${ingest(traces)}\n| summarize count() by bin_auto(_time), series = strcat(signal, ' ', outcome)`,
224 ),
225];
226
227/** Lays charts left to right, wrapping at Axiom's 12-column grid. */
228function layout(charts: ReadonlyArray<QueryChart>) {
229 let x = 0;
230 let y = 0;
231 let rowHeight = 0;
232 return charts.map((chart) => {
233 const h = chart.type === "Statistic" ? 3 : 6;
234 if (x + chart.width > 12) {
235 x = 0;
236 y += rowHeight;
237 rowHeight = 0;
238 }
239 const cell = { i: chart.id, x, y, w: chart.width, h };
240 x += chart.width;
241 rowHeight = Math.max(rowHeight, h);
242 return cell;
243 });
244}
245
246/**
247 * Monitors follow the relay's: a deployment opts in by naming its paging notifier. The
248 * dashboard also comes up on PR previews, so a change to mobile telemetry can be checked
249 * there; the preview's teardown removes it.
250 */
251export const MobileMonitoring = Effect.fn("MobileMonitoring")(function* (
252 logs: Axiom.Dataset,
253 traces: Axiom.Dataset,
254) {
255 const { stage } = yield* Alchemy.Stack;
256 const page = yield* notifierConfig("RELAY_ALERT_NOTIFIER_ID");
257 if (Option.isNone(page) && !isPreviewStage(stage)) return;
258 const datasets = Output.map(Output.all(logs.name, traces.name), ([logs, traces]) => ({
259 logs,
260 traces,
261 }));
262
263 if (Option.isSome(page)) {
264 const notifiers: Record<Severity, Option.Option<string>> = {
265 page,
266 warn: yield* notifierConfig("RELAY_WARNING_NOTIFIER_ID"),
267 };
268 for (const monitor of mobileMonitors) {
269 yield* Axiom.Monitor(monitor.id, {
270 ...monitor.props,
271 name: stage === "prod" ? monitor.name : `[${stage}] ${monitor.name}`,
272 description: `${monitor.description} Runbook: ${RUNBOOK_URL}`,
273 aplQuery: Output.map(datasets, monitor.query),
274 triggerFromNRuns: monitor.props.triggerFromNRuns ?? 1,
275 alertOnNoData: monitor.props.alertOnNoData ?? false,
276 notifierIds: Option.toArray(notifiers[monitor.severity]),
277 });
278 }
279 }
280
281 yield* Axiom.Dashboard("MobileHealthDashboard", {
282 dashboard: Output.map(datasets, (names) => {
283 const charts = mobileDashboardCharts(names);
284 return {
285 name: relayResourceNameForStage("t3-code-mobile-health", stage),
286 owner: "X-AXIOM-EVERYONE",
287 description: `T3 Code mobile app crashes and errors. Crashes count JS and native once each. Runbook: ${RUNBOOK_URL}`,
288 refreshTime: 300,
289 schemaVersion: 2,
290 timeWindowStart: "qr-now-7d",
291 timeWindowEnd: "qr-now",
292 charts: charts.map(({ width: _width, ...chart }) => chart),
293 layout: layout(charts),
294 };
295 }),
296 });
297});