Open History to see the last commit
1import { DuplicateTransactionIdError, createTransaction } from '@tanstack/db'
2import { OutboxTransactionNotFoundError } from '../outbox/OutboxManager'
3import { DefaultRetryPolicy } from '../retry/RetryPolicy'
4import { NonRetriableError } from '../types'
5import { withNestedSpan } from '../telemetry/tracer'
6import type { KeyScheduler } from './KeyScheduler'
7import type { OutboxManager } from '../outbox/OutboxManager'
8import type {
9 OfflineConfig,
10 OfflineTransaction,
11 TransactionSignaler,
12} from '../types'
13
14const HANDLED_EXECUTION_ERROR = Symbol(`HandledExecutionError`)
15
16function toError(value: unknown): Error {
17 return value instanceof Error ||
18 Object.prototype.toString.call(value) === `[object Error]`
19 ? (value as Error)
20 : new Error(String(value))
21}
22
23export class TransactionExecutor {
24 private scheduler: KeyScheduler
25 private outbox: OutboxManager
26 private config: OfflineConfig
27 private retryPolicy: DefaultRetryPolicy
28 private isExecuting = false
29 private executionPromise: Promise<void> | null = null
30 private offlineExecutor: TransactionSignaler
31 private retryTimer: ReturnType<typeof setTimeout> | null = null
32 private fatalError: Error | null = null
33
34 constructor(
35 scheduler: KeyScheduler,
36 outbox: OutboxManager,
37 config: OfflineConfig,
38 offlineExecutor: TransactionSignaler,
39 ) {
40 this.scheduler = scheduler
41 this.outbox = outbox
42 this.config = config
43 this.retryPolicy = new DefaultRetryPolicy(
44 Number.POSITIVE_INFINITY,
45 config.jitter ?? true,
46 )
47 this.offlineExecutor = offlineExecutor
48 }
49
50 async execute(transaction: OfflineTransaction): Promise<void> {
51 this.assertHealthy()
52 this.scheduler.schedule(transaction)
53 await this.executeAll()
54 }
55
56 async executeAll(): Promise<void> {
57 this.assertHealthy()
58 if (this.isExecuting) {
59 return this.executionPromise!
60 }
61
62 this.isExecuting = true
63 this.executionPromise = this.runExecution()
64
65 try {
66 await this.executionPromise
67 } finally {
68 this.isExecuting = false
69 this.executionPromise = null
70 this.scheduleNextRetry()
71 }
72 }
73
74 private async runExecution(): Promise<void> {
75 while (this.scheduler.getPendingCount() > 0) {
76 if (!this.canExecute()) {
77 break
78 }
79
80 const transaction = this.scheduler.getNext()
81
82 if (!transaction) {
83 break
84 }
85
86 await this.executeTransaction(transaction)
87 }
88 }
89
90 private async executeTransaction(
91 transaction: OfflineTransaction,
92 ): Promise<void> {
93 try {
94 await withNestedSpan(
95 `transaction.execute`,
96 {
97 'transaction.id': transaction.id,
98 'transaction.mutationFnName': transaction.mutationFnName,
99 'transaction.retryCount': transaction.retryCount,
100 'transaction.keyCount': transaction.keys.length,
101 },
102 async (span) => {
103 const terminalError =
104 transaction.outboxPhase === `rejection-pending`
105 ? this.getTerminalError(transaction)
106 : undefined
107 this.scheduler.markStarted(transaction)
108
109 if (transaction.retryCount > 0) {
110 span.setAttribute(`retry.attempt`, transaction.retryCount)
111 }
112
113 if (!transaction.outboxPhase) {
114 try {
115 await this.runMutationFn(transaction)
116 } catch (error) {
117 const err = toError(error)
118
119 span.setAttribute(`result`, `error`)
120
121 await this.handleError(transaction, err)
122 ;(err as any)[HANDLED_EXECUTION_ERROR] = true
123 throw err
124 }
125 }
126
127 // A successful provider call can outlive this executor. Persist that
128 // fact before removal so a restarted executor only retries deletion.
129 const deletionPending: OfflineTransaction = transaction.outboxPhase
130 ? transaction
131 : { ...transaction, outboxPhase: `deletion-pending` }
132 try {
133 await this.removeSettledTransaction(
134 deletionPending,
135 !transaction.outboxPhase,
136 )
137 } catch (error) {
138 const storageError = error as Error
139 span.recordException(storageError)
140 span.setAttribute(`result`, `outbox_failure`)
141 this.offlineExecutor.rejectTransaction(
142 transaction.id,
143 terminalError ?? storageError,
144 )
145 throw storageError
146 }
147
148 if (terminalError) {
149 span.setAttribute(`result`, `permanent_failure`)
150 this.offlineExecutor.rejectTransaction(
151 transaction.id,
152 terminalError,
153 )
154 } else {
155 span.setAttribute(`result`, `success`)
156 this.offlineExecutor.resolveTransaction(transaction.id, undefined)
157 }
158 },
159 )
160 } catch (error) {
161 if (
162 error !== null &&
163 typeof error === `object` &&
164 (error as Record<symbol, unknown>)[HANDLED_EXECUTION_ERROR] === true
165 ) {
166 return
167 }
168
169 throw error
170 }
171 }
172
173 private async runMutationFn(transaction: OfflineTransaction): Promise<void> {
174 const mutationFn = this.config.mutationFns[transaction.mutationFnName]
175
176 if (!mutationFn) {
177 const errorMessage = `Unknown mutation function: ${transaction.mutationFnName}`
178
179 if (this.config.onUnknownMutationFn) {
180 this.config.onUnknownMutationFn(transaction.mutationFnName, transaction)
181 }
182
183 throw new NonRetriableError(errorMessage)
184 }
185
186 // Mutations are already PendingMutation objects with collections attached
187 // from the deserializer, so we can use them directly
188 const transactionWithMutations = {
189 id: transaction.id,
190 mutations: transaction.mutations,
191 metadata: transaction.metadata ?? {},
192 }
193
194 await mutationFn({
195 transaction: transactionWithMutations as any,
196 idempotencyKey: transaction.idempotencyKey,
197 })
198 }
199
200 private async handleError(
201 transaction: OfflineTransaction,
202 error: Error,
203 ): Promise<void> {
204 return withNestedSpan(
205 `transaction.handleError`,
206 {
207 'transaction.id': transaction.id,
208 'error.name': error.name,
209 'error.message': error.message,
210 },
211 async (span) => {
212 let shouldRetry: boolean
213 let hookFailure: Error | undefined
214 if (error instanceof NonRetriableError) {
215 shouldRetry = false
216 } else {
217 let decision: boolean | undefined
218 try {
219 decision = this.config.shouldRetry?.(error, transaction.retryCount)
220 if (decision !== undefined && typeof decision !== `boolean`) {
221 // An async hook is invalid, but its rejection must stay row-local.
222 void Promise.resolve(decision).catch(() => {})
223 throw new TypeError(
224 `OfflineConfig.shouldRetry must return true, false, or undefined`,
225 )
226 }
227 } catch (hookError) {
228 hookFailure = toError(hookError)
229 }
230 if (hookFailure) {
231 shouldRetry = false
232 } else {
233 shouldRetry =
234 decision ??
235 this.retryPolicy.shouldRetry(error, transaction.retryCount)
236 }
237 }
238
239 span.setAttribute(`shouldRetry`, shouldRetry)
240
241 if (!shouldRetry) {
242 const terminalError = hookFailure ?? error
243 const rejectionPending: OfflineTransaction = {
244 ...transaction,
245 outboxPhase: `rejection-pending`,
246 lastError: {
247 name: terminalError.name,
248 message: terminalError.message,
249 stack: terminalError.stack,
250 },
251 }
252 if (hookFailure) {
253 span.recordException(hookFailure)
254 console.warn(
255 `Retry decision failed for transaction ${transaction.id}:`,
256 hookFailure,
257 `Named mutation function error:`,
258 error,
259 )
260 } else {
261 console.warn(
262 `Transaction ${transaction.id} failed permanently:`,
263 error,
264 )
265 }
266 try {
267 await this.removeSettledTransaction(rejectionPending, true)
268 } catch (storageError) {
269 span.recordException(storageError as Error)
270 span.setAttribute(`result`, `outbox_failure`)
271 this.offlineExecutor.rejectTransaction(
272 transaction.id,
273 terminalError,
274 )
275 throw storageError
276 }
277
278 span.setAttribute(
279 `result`,
280 hookFailure ? `retry_decision_failure` : `permanent_failure`,
281 )
282 this.offlineExecutor.rejectTransaction(transaction.id, terminalError)
283 return
284 }
285
286 const delay = Math.max(
287 0,
288 this.retryPolicy.calculateDelay(transaction.retryCount),
289 )
290 const updatedTransaction: OfflineTransaction = {
291 ...transaction,
292 retryCount: transaction.retryCount + 1,
293 nextAttemptAt: Date.now() + delay,
294 lastError: {
295 name: error.name,
296 message: error.message,
297 stack: error.stack,
298 },
299 }
300
301 span.setAttribute(`retryDelay`, delay)
302 span.setAttribute(`nextRetryCount`, updatedTransaction.retryCount)
303
304 this.scheduler.updateTransaction(updatedTransaction)
305
306 try {
307 await this.outbox.update(transaction.id, updatedTransaction)
308 span.setAttribute(`result`, `scheduled_retry`)
309 } catch (persistError) {
310 // Public removal can finish while the named mutation function runs.
311 // The removed row cannot be retried, but its caller still owns the
312 // provider failure and later queued work must be able to run.
313 if (persistError instanceof OutboxTransactionNotFoundError) {
314 this.scheduler.markCompleted(transaction)
315 span.setAttribute(`result`, `outbox_removed`)
316 this.offlineExecutor.rejectTransaction(transaction.id, error)
317 return
318 }
319 span.recordException(persistError as Error)
320 span.setAttribute(`result`, `persist_failed`)
321 throw persistError
322 } finally {
323 this.scheduler.markFailed(transaction)
324 }
325 },
326 )
327 }
328
329 private async removeSettledTransaction(
330 transaction: OfflineTransaction,
331 persistMarker: boolean,
332 ): Promise<void> {
333 try {
334 if (persistMarker) {
335 try {
336 await this.outbox.update(transaction.id, transaction)
337 } catch (error) {
338 // A public removal can finish while the provider is still running.
339 if (!(error instanceof OutboxTransactionNotFoundError)) throw error
340 }
341 }
342 await this.outbox.remove(transaction.id)
343 this.scheduler.markCompleted(transaction)
344 } catch (error) {
345 const storageError = toError(error)
346 this.fatalError = storageError
347 this.scheduler.markFailed(transaction)
348 this.clearRetryTimer()
349 throw storageError
350 }
351 }
352
353 assertHealthy(): void {
354 if (this.fatalError) throw this.fatalError
355 }
356
357 private getTerminalError(transaction: OfflineTransaction): Error {
358 if (!transaction.lastError)
359 throw new Error(`Terminal transaction ${transaction.id} has no error`)
360 const { name, message, stack } = transaction.lastError
361 const error =
362 name === `NonRetriableError`
363 ? new NonRetriableError(message)
364 : new Error(message)
365 error.name = name
366 error.stack = stack
367 return error
368 }
369
370 async loadPendingTransactions(): Promise<void> {
371 let removedIds: Array<string> = []
372 await this.outbox.withAll((transactions) => {
373 const { isOfflineEnabled } = this.offlineExecutor
374 if (!isOfflineEnabled) return
375 const providerPending = transactions.filter(
376 (transaction) => !transaction.outboxPhase,
377 )
378 const selectedProvider = this.config.beforeRetry
379 ? this.config.beforeRetry(providerPending)
380 : providerPending
381 const selectedById = new Map(selectedProvider.map((tx) => [tx.id, tx]))
382 const filteredTransactions = transactions.flatMap((transaction) => {
383 if (transaction.outboxPhase) return [transaction]
384 const selected = selectedById.get(transaction.id)
385 return selected ? [selected] : []
386 })
387
388 // The retry hook is user code and may synchronously revoke replay rights.
389 if (!this.offlineExecutor.isOfflineEnabled) return
390
391 const newlyLoaded = filteredTransactions.filter((transaction) =>
392 this.scheduler.schedule(transaction),
393 )
394
395 removedIds = providerPending
396 .filter(({ id }) => !selectedById.has(id))
397 .map(({ id }) => id)
398 removedIds = this.scheduler.removePendingTransactions(removedIds)
399
400 // Restore optimistic state for loaded transactions
401 // This ensures the UI shows the optimistic data while transactions are pending
402 this.restoreOptimisticState(
403 newlyLoaded.filter(
404 (transaction) => transaction.outboxPhase !== `rejection-pending`,
405 ),
406 )
407
408 // Reset retry delays for all loaded transactions so they can run immediately
409 this.resetRetryDelays()
410
411 // Schedule retry timer for loaded transactions
412 this.scheduleNextRetry()
413 })
414
415 if (removedIds.length > 0) {
416 const error = new NonRetriableError(`Transaction excluded by beforeRetry`)
417 await Promise.all(
418 removedIds.map(async (id) => {
419 try {
420 await this.outbox.remove(id)
421 this.offlineExecutor.rejectTransaction(id, error)
422 } catch (cleanupError) {
423 console.warn(
424 `Failed to remove transaction excluded by beforeRetry:`,
425 id,
426 cleanupError,
427 )
428 }
429 }),
430 )
431 }
432 }
433
434 /**
435 * Restore optimistic state from loaded transactions.
436 * Creates internal transactions to hold the mutations so the collection's
437 * state manager can show optimistic data while waiting for sync.
438 */
439 private restoreOptimisticState(
440 transactions: Array<OfflineTransaction>,
441 ): void {
442 for (const offlineTx of transactions) {
443 if (offlineTx.mutations.length === 0) {
444 continue
445 }
446
447 try {
448 // Create a restoration transaction that holds mutations for optimistic state display.
449 // It will never commit - the real mutation is handled by the offline executor.
450 const restorationTx = createTransaction({
451 id: offlineTx.id,
452 autoCommit: false,
453 mutationFn: async () => {},
454 })
455
456 restorationTx.applyMutations(offlineTx.mutations)
457
458 // Register with each affected Collection. If one cannot track it, roll
459 // it back, so no Collection keeps a restoration nothing will settle.
460 try {
461 // The serializer resolves every mutation's Collection or throws.
462 for (const mutation of offlineTx.mutations) {
463 if (mutation.collection._state.trackTransaction(restorationTx))
464 mutation.collection._state.recomputeOptimisticState(true)
465 }
466 } catch (error) {
467 // A secondary rollback settles only this restoration; it must not
468 // roll back the user's live transactions on the same keys.
469 restorationTx.rollback({ isSecondaryRollback: true })
470 throw error
471 }
472
473 this.offlineExecutor.registerRestorationTransaction(
474 offlineTx.id,
475 restorationTx,
476 )
477 } catch (error) {
478 // A live transaction with this id already shows these mutations.
479 if (error instanceof DuplicateTransactionIdError) continue
480 console.warn(
481 `Failed to restore optimistic state for transaction ${offlineTx.id}:`,
482 error,
483 )
484 }
485 }
486 }
487
488 clear(): void {
489 this.scheduler.clear()
490 this.clearRetryTimer()
491 }
492
493 pause(): void {
494 // Retain queued work and let the issued call finish its acknowledgment.
495 this.clearRetryTimer()
496 }
497
498 getPendingCount(): number {
499 return this.scheduler.getPendingCount()
500 }
501
502 private scheduleNextRetry(): void {
503 // Clear existing timer
504 this.clearRetryTimer()
505
506 if (this.fatalError || !this.canExecute()) {
507 return
508 }
509
510 const nextRetryTime = this.getNextRetryTime()
511
512 if (nextRetryTime === null) {
513 return // No transactions pending retry
514 }
515
516 const delay = Math.max(0, nextRetryTime - Date.now())
517
518 this.retryTimer = setTimeout(() => {
519 this.executeAll().catch((error) => {
520 console.warn(`Failed to execute retry batch:`, error)
521 })
522 }, delay)
523 }
524
525 private getNextRetryTime(): number | null {
526 const allTransactions = this.scheduler.getAllPendingTransactions()
527
528 if (allTransactions.length === 0) {
529 return null
530 }
531
532 // Later transactions cannot overtake the FIFO head, even if they are ready.
533 return allTransactions[0]!.nextAttemptAt
534 }
535
536 private clearRetryTimer(): void {
537 if (this.retryTimer) {
538 clearTimeout(this.retryTimer)
539 this.retryTimer = null
540 }
541 }
542
543 private canExecute(): boolean {
544 return (
545 this.offlineExecutor.isOfflineEnabled && this.offlineExecutor.isOnline()
546 )
547 }
548
549 getRunningCount(): number {
550 return this.scheduler.getRunningCount()
551 }
552
553 resetRetryDelays(): void {
554 const allTransactions = this.scheduler.getAllPendingTransactions()
555 const updatedTransactions = allTransactions.map((transaction) => ({
556 ...transaction,
557 nextAttemptAt: Date.now(),
558 }))
559
560 this.scheduler.updateTransactions(updatedTransactions)
561 }
562}