Open History to see the last commit
1import { DiffTriggerOperation, LogLevels, sanitizeSQL } from '@powersync/common'
2import { or, withCollectionConfigFactory } from '@tanstack/db'
3import { compileSQLite } from './sqlite-compiler'
4import { PendingOperationStore } from './PendingOperationStore'
5import { PowerSyncTransactor } from './PowerSyncTransactor'
6import { DEFAULT_BATCH_SIZE } from './definitions'
7import { asPowerSyncRecord, mapOperation } from './helpers'
8import { convertTableToSchema } from './schema'
9import { serializeForSQLite } from './serialization'
10import type {
11 CleanupFn,
12 LoadSubsetOptions,
13 OperationType,
14 SyncAppliedReceipt,
15 SyncConfig,
16} from '@tanstack/db'
17import type {
18 AnyTableColumnType,
19 ExtractedTable,
20 OptionalExtractedTable,
21} from './helpers'
22import type {
23 BasePowerSyncCollectionConfig,
24 ConfigWithArbitraryCollectionTypes,
25 ConfigWithSQLiteInputType,
26 ConfigWithSQLiteTypes,
27 EnhancedPowerSyncCollectionConfig,
28 InferPowerSyncOutputType,
29 PowerSyncCollectionConfig,
30 PowerSyncCollectionUtils,
31} from './definitions'
32import type { PendingOperation } from './PendingOperationStore'
33import type { StandardSchemaV1 } from '@standard-schema/spec'
34import type { LockContext, Table, TriggerDiffRecord } from '@powersync/common'
35
36function isPromiseLike(value: unknown): value is PromiseLike<unknown> {
37 return (
38 !!value &&
39 (typeof value === `object` || typeof value === `function`) &&
40 typeof (value as { then?: unknown }).then === `function`
41 )
42}
43
44/**
45 * Creates PowerSync collection options for use with a standard Collection.
46 *
47 * @template TTable - The SQLite-based typing
48 * @template TSchema - The validation schema type (optionally supports a custom input type)
49 * @param config - Configuration options for the PowerSync collection
50 * @returns Collection options with utilities
51 */
52
53// Overload 1: No schema is provided
54
55/**
56 * Creates a PowerSync collection configuration with basic default validation.
57 * Input and Output types are the SQLite column types.
58 *
59 * @example
60 * ```typescript
61 * const APP_SCHEMA = new Schema({
62 * documents: new Table({
63 * name: column.text,
64 * }),
65 * })
66 *
67 * type Document = (typeof APP_SCHEMA)["types"]["documents"]
68 *
69 * const db = new PowerSyncDatabase({
70 * database: {
71 * dbFilename: "test.sqlite",
72 * },
73 * schema: APP_SCHEMA,
74 * })
75 *
76 * const collection = createCollection(
77 * powerSyncCollectionOptions({
78 * database: db,
79 * table: APP_SCHEMA.props.documents
80 * })
81 * )
82 * ```
83 */
84export function powerSyncCollectionOptions<TTable extends Table = Table>(
85 config: BasePowerSyncCollectionConfig<TTable, never> & ConfigWithSQLiteTypes,
86): EnhancedPowerSyncCollectionConfig<
87 TTable,
88 OptionalExtractedTable<TTable>,
89 never
90>
91
92// Overload 2: Schema is provided and the TInput matches SQLite types.
93
94/**
95 * Creates a PowerSync collection configuration with schema validation.
96 *
97 * The input types satisfy the SQLite column types.
98 *
99 * The output types are defined by the provided schema. This schema can enforce additional
100 * validation or type transforms.
101 * Arbitrary output typed mutations are encoded to SQLite for persistence. We provide a basic standard
102 * serialization implementation to serialize column values. Custom or advanced types require providing additional
103 * serializer specifications. Partial column overrides can be supplied to `serializer`.
104 *
105 * @example
106 * ```typescript
107 * import { z } from "zod"
108 *
109 * // The PowerSync SQLite schema
110 * const APP_SCHEMA = new Schema({
111 * documents: new Table({
112 * name: column.text,
113 * // Dates are stored as ISO date strings in SQLite
114 * created_at: column.text
115 * }),
116 * })
117 *
118 * // Advanced Zod validations. The output type of this schema
119 * // is constrained to the SQLite schema of APP_SCHEMA
120 * const schema = z.object({
121 * id: z.string(),
122 * // Notice that `name` is not nullable (is required) here and it has additional validation
123 * name: z.string().min(3, { message: "Should be at least 3 characters" }).nullable(),
124 * // The input type is still the SQLite string type. While collections will output smart Date instances.
125 * created_at: z.string().transform(val => new Date(val))
126 * })
127 *
128 * const collection = createCollection(
129 * powerSyncCollectionOptions({
130 * database: db,
131 * table: APP_SCHEMA.props.documents,
132 * schema,
133 * serializer: {
134 * // The default is toISOString, this is just to demonstrate custom overrides
135 * created_at: (outputValue) => outputValue.toISOString(),
136 * },
137 * })
138 * )
139 * ```
140 */
141export function powerSyncCollectionOptions<
142 TTable extends Table,
143 TSchema extends StandardSchemaV1<
144 // TInput is the SQLite types. We can use the supplied schema to validate sync input
145 OptionalExtractedTable<TTable>,
146 AnyTableColumnType<TTable>
147 >,
148>(
149 config: BasePowerSyncCollectionConfig<TTable, TSchema> &
150 ConfigWithSQLiteInputType<TTable, TSchema>,
151): EnhancedPowerSyncCollectionConfig<
152 TTable,
153 InferPowerSyncOutputType<TTable, TSchema>,
154 TSchema
155> & {
156 schema: TSchema
157}
158
159// Overload 3: Schema is provided with arbitrary TInput and TOutput
160/**
161 * Creates a PowerSync collection configuration with schema validation.
162 *
163 * The input types are not linked to the internal SQLite table types. This can
164 * give greater flexibility, e.g. by accepting rich types as input for `insert` or `update` operations.
165 * An additional `deserializationSchema` is required in order to process incoming SQLite updates to the output type.
166 *
167 * The output types are defined by the provided schema. This schema can enforce additional
168 * validation or type transforms.
169 * Arbitrary output typed mutations are encoded to SQLite for persistence. We provide a basic standard
170 * serialization implementation to serialize column values. Custom or advanced types require providing additional
171 * serializer specifications. Partial column overrides can be supplied to `serializer`.
172 *
173 * @example
174 * ```typescript
175 * import { z } from "zod"
176 *
177 * // The PowerSync SQLite schema
178 * const APP_SCHEMA = new Schema({
179 * documents: new Table({
180 * name: column.text,
181 * // Booleans are represented as integers in SQLite
182 * is_active: column.integer
183 * }),
184 * })
185 *
186 * // Advanced Zod validations.
187 * // We accept boolean values as input for operations and expose Booleans in query results
188 * const schema = z.object({
189 * id: z.string(),
190 * isActive: z.boolean(), // TInput and TOutput are boolean
191 * })
192 *
193 * // The deserializationSchema converts the SQLite synced INTEGER (0/1) values to booleans.
194 * const deserializationSchema = z.object({
195 * id: z.string(),
196 * isActive: z.number().nullable().transform((val) => val == null ? true : val > 0),
197 * })
198 *
199 * const collection = createCollection(
200 * powerSyncCollectionOptions({
201 * database: db,
202 * table: APP_SCHEMA.props.documents,
203 * schema,
204 * deserializationSchema,
205 * })
206 * )
207 * ```
208 */
209export function powerSyncCollectionOptions<
210 TTable extends Table,
211 TSchema extends StandardSchemaV1<
212 // The input and output must have the same keys, the value types can be arbitrary
213 AnyTableColumnType<TTable>,
214 AnyTableColumnType<TTable>
215 >,
216>(
217 config: BasePowerSyncCollectionConfig<TTable, TSchema> &
218 ConfigWithArbitraryCollectionTypes<TTable, TSchema>,
219): EnhancedPowerSyncCollectionConfig<
220 TTable,
221 InferPowerSyncOutputType<TTable, TSchema>,
222 TSchema
223> & {
224 utils: PowerSyncCollectionUtils<TTable>
225 schema: TSchema
226}
227
228/**
229 * Implementation of powerSyncCollectionOptions that handles both schema and non-schema configurations.
230 */
231
232export function powerSyncCollectionOptions<
233 TTable extends Table,
234 TSchema extends StandardSchemaV1<any> = never,
235>(
236 config: PowerSyncCollectionConfig<TTable, TSchema>,
237): ReturnType<typeof createPowerSyncCollectionConfig<TTable, TSchema>> {
238 const outputConfig = createPowerSyncCollectionConfig(config)
239 return withCollectionConfigFactory(outputConfig, () =>
240 createPowerSyncCollectionConfig(config),
241 )
242}
243
244function createPowerSyncCollectionConfig<
245 TTable extends Table,
246 TSchema extends StandardSchemaV1<any> = never,
247>(config: PowerSyncCollectionConfig<TTable, TSchema>) {
248 const {
249 database,
250 table,
251 schema: inputSchema,
252 syncBatchSize = DEFAULT_BATCH_SIZE,
253 syncMode = 'eager',
254 ...restConfig
255 } = config
256
257 const deserializationSchema =
258 `deserializationSchema` in config ? config.deserializationSchema : null
259 const serializer = `serializer` in config ? config.serializer : undefined
260 const onDeserializationError =
261 `onDeserializationError` in config
262 ? config.onDeserializationError
263 : undefined
264
265 // The SQLite table type
266 type TableType = ExtractedTable<TTable>
267
268 // The collection output type
269 type OutputType = InferPowerSyncOutputType<TTable, TSchema>
270
271 const { viewName, trackMetadata: metadataIsTracked } = table
272
273 /**
274 * Deserializes data from the incoming sync stream
275 */
276 const deserializeSyncRow = (value: TableType): OutputType => {
277 const validationSchema = deserializationSchema || schema
278 const validation = validationSchema[`~standard`].validate(value)
279 if (`value` in validation) {
280 return validation.value
281 } else if (`issues` in validation) {
282 const issueMessage = `Failed to validate incoming data for ${viewName}. Issues: ${validation.issues.map((issue) => `${issue.path} - ${issue.message}`)}`
283 database.logger.log({ level: LogLevels.error, message: issueMessage })
284 onDeserializationError!(validation)
285 throw new Error(issueMessage)
286 } else {
287 const unknownErrorMessage = `Unknown deserialization error for ${viewName}`
288 database.logger.log({
289 level: LogLevels.error,
290 message: unknownErrorMessage,
291 })
292 onDeserializationError!({ issues: [{ message: unknownErrorMessage }] })
293 throw new Error(unknownErrorMessage)
294 }
295 }
296
297 // We can do basic runtime validations for columns if not explicit schema has been provided
298 const schema = inputSchema ?? (convertTableToSchema(table) as TSchema)
299 /**
300 * The onInsert, onUpdate, and onDelete handlers should only return
301 * after we have written the changes to TanStack DB.
302 * We currently only write to TanStack DB from a diff trigger.
303 * We wait for the diff trigger to observe the change,
304 * and only then return from the on[X] handlers.
305 * This ensures that when the transaction is reported as
306 * complete to the caller, the in-memory state is already
307 * consistent with the database.
308 */
309 const pendingOperationStore = PendingOperationStore.GLOBAL
310 // Keep the tracked table unique in case of multiple tabs.
311 const trackedTableName = `__${viewName}_tracking_${Math.floor(
312 Math.random() * 0xffffffff,
313 )
314 .toString(16)
315 .padStart(8, `0`)}`
316
317 const transactor = new PowerSyncTransactor({
318 database,
319 })
320
321 /**
322 * "sync"
323 * Notice that this describes the Sync between the local SQLite table
324 * and the in-memory tanstack-db collection.
325 */
326 const sync: SyncConfig<OutputType, string> = {
327 sync: (params) => {
328 const { begin, write, collection, commit, markReady, markError } = params
329 const abortController = new AbortController()
330
331 let disposeTracking:
332 ((options?: { context?: LockContext }) => Promise<void>) | null = null
333 let trackingSetup: Promise<void> | null = null
334 let activeTrackingDisposal: Promise<void> | null = null
335 let abortTrackingDisposal: Promise<void> | null = null
336
337 if (syncMode === `eager`) {
338 return runEagerSync()
339 } else {
340 return runOnDemandSync()
341 }
342
343 /**
344 * Disposes the current diff trigger, if one is active, and clears the
345 * tracking state.
346 */
347 async function safelyDisposeTracking(
348 context?: LockContext,
349 ): Promise<void> {
350 // Cleanup can race trigger creation. Wait until the disposer has been
351 // published so an abort cannot strand a freshly-created trigger.
352 const setup = trackingSetup
353 if (setup) {
354 await setup.catch(() => undefined)
355 }
356
357 const dispose = disposeTracking
358 if (!dispose) {
359 // Reconciliation may have claimed the disposer after this caller
360 // observed setup. Join that invocation instead of settling early.
361 await activeTrackingDisposal
362 return
363 }
364
365 disposeTracking = null
366 let disposal: Promise<void>
367 try {
368 disposal = Promise.resolve(dispose(context ? { context } : undefined))
369 } catch (error) {
370 disposal = Promise.reject(error)
371 }
372 activeTrackingDisposal = disposal
373 try {
374 await disposal
375 } finally {
376 if (activeTrackingDisposal === disposal) {
377 activeTrackingDisposal = null
378 }
379 }
380 }
381
382 function disposeTrackingAfterAbort(): Promise<void> {
383 const disposal = (abortTrackingDisposal ??= safelyDisposeTracking())
384 // Abort event dispatch does not observe a listener's returned promise.
385 // The eager cleanup callback awaits this same promise below.
386 void disposal.catch(() => undefined)
387 return disposal
388 }
389
390 async function establishTracking(
391 options: Parameters<typeof createDiffTrigger>[0],
392 appliedReceipts: Array<SyncAppliedReceipt>,
393 ): Promise<void> {
394 const setup = (async () => {
395 const dispose = await createDiffTrigger(options, appliedReceipts)
396 disposeTracking = dispose
397 })()
398 trackingSetup = setup
399
400 try {
401 await setup
402 } finally {
403 if (trackingSetup === setup) {
404 trackingSetup = null
405 }
406 }
407 }
408
409 async function createDiffTrigger(
410 options: {
411 setupContext?: LockContext
412 immediate?: boolean
413 when: Record<DiffTriggerOperation, string>
414 writeType: (rowId: string) => OperationType
415 batchQuery: (
416 lockContext: LockContext,
417 batchSize: number,
418 cursor: number,
419 ) => Promise<Array<TableType>>
420 },
421 appliedReceipts: Array<SyncAppliedReceipt>,
422 ) {
423 const { setupContext, immediate, when, writeType, batchQuery } = options
424
425 return await database.triggers.createDiffTrigger({
426 source: viewName,
427 destination: trackedTableName,
428 columns: table.columns.map((column) => column.name),
429 setupContext,
430 when,
431 hooks: {
432 beforeCreate: async (context) => {
433 let currentBatchCount = syncBatchSize
434 let cursor = 0
435 while (currentBatchCount == syncBatchSize) {
436 begin(immediate ? { immediate: true } : undefined)
437
438 const batchItems = await batchQuery(
439 context,
440 syncBatchSize,
441 cursor,
442 )
443 currentBatchCount = batchItems.length
444 cursor += currentBatchCount
445 for (const row of batchItems) {
446 write({
447 type: writeType(row.id),
448 value: deserializeSyncRow(row),
449 })
450 }
451 appliedReceipts.push(commit())
452 }
453 database.logger.log({
454 level: LogLevels.info,
455 message: `Sync is ready for ${viewName} into ${trackedTableName}`,
456 })
457 },
458 },
459 })
460 }
461
462 async function flushDiffRecords(): Promise<void> {
463 // PowerSync can notify after creating the tracking table but before its
464 // create call returns. Preserve that notification until the disposer,
465 // which proves the trigger is usable, has been published.
466 const setup = trackingSetup
467 if (setup) {
468 await setup.catch(() => undefined)
469 }
470 if (!disposeTracking) {
471 return
472 }
473
474 const ignoredReceipts: Array<SyncAppliedReceipt> = []
475 await database
476 .writeTransaction(async (context) => {
477 await flushDiffRecordsWithContext(context, ignoredReceipts)
478 })
479 .catch((error) => {
480 database.logger.log({
481 level: LogLevels.error,
482 message: `An error has been detected in the sync handler`,
483 error,
484 })
485 })
486 }
487
488 // We can use this directly if we want to pair a flush with dispose+recreate diff trigger.
489 async function flushDiffRecordsWithContext(
490 context: LockContext,
491 appliedReceipts: Array<SyncAppliedReceipt>,
492 ): Promise<void> {
493 // There is nothing to flush if no tracking table is currently active.
494 if (!disposeTracking) {
495 return
496 }
497
498 try {
499 begin()
500 const operations = await context.getAll<TriggerDiffRecord>(
501 `SELECT * FROM ${trackedTableName} ORDER BY operation_id ASC`,
502 )
503 const pendingOperations: Array<PendingOperation> = []
504
505 for (const op of operations) {
506 const { id, operation, timestamp, value } = op
507 const parsedValue = deserializeSyncRow({
508 id,
509 ...JSON.parse(value),
510 })
511 const parsedPreviousValue =
512 op.operation == DiffTriggerOperation.UPDATE
513 ? deserializeSyncRow({
514 id,
515 ...JSON.parse(op.previous_value),
516 })
517 : undefined
518 write({
519 type: mapOperation(operation),
520 value: parsedValue,
521 previousValue: parsedPreviousValue,
522 })
523 pendingOperations.push({
524 id,
525 operation,
526 timestamp,
527 tableName: viewName,
528 })
529 }
530
531 // clear the current operations
532 await context.execute(`DELETE FROM ${trackedTableName}`)
533
534 const applied = commit()
535 appliedReceipts.push(applied)
536 // Mutation persistence is what releases the Collection's FIFO gate.
537 // Confirm these local operations after the sync transaction is
538 // staged; waiting for its applied receipt would deadlock the user
539 // transaction that currently parks it.
540 pendingOperationStore.resolvePendingFor(pendingOperations)
541 } catch (error) {
542 database.logger.log({
543 level: LogLevels.error,
544 message: `An error has been detected in the sync handler`,
545 error,
546 })
547 }
548 }
549
550 // The sync function needs to be synchronous.
551 async function start(afterOnChangeRegistered?: () => Promise<void>) {
552 database.logger.log({
553 level: LogLevels.info,
554 message: `Sync is starting for ${viewName} into ${trackedTableName}`,
555 })
556 database.onChangeWithCallback(
557 {
558 onChange: async () => {
559 await flushDiffRecords()
560 },
561 },
562 {
563 signal: abortController.signal,
564 triggerImmediate: false,
565 tables: [trackedTableName],
566 },
567 )
568
569 await afterOnChangeRegistered?.()
570
571 // If the abort controller was aborted while processing the request above
572 if (abortController.signal.aborted) {
573 await disposeTrackingAfterAbort()
574 } else {
575 abortController.signal.addEventListener(
576 `abort`,
577 () => {
578 void disposeTrackingAfterAbort()
579 },
580 { once: true },
581 )
582 }
583 }
584
585 // Eager mode.
586 // Registers a diff trigger for the entire table.
587 function runEagerSync() {
588 let onUnload: CleanupFn | void | null = null
589 let onUnloadStarted = false
590
591 const startup = start(async () => {
592 const cleanup = await restConfig.onLoad?.()
593 onUnload = cleanup
594 if (abortController.signal.aborted) {
595 return
596 }
597
598 const appliedReceipts: Array<SyncAppliedReceipt> = []
599 await establishTracking(
600 {
601 // Initial eager hydration must make the source usable before
602 // PowerSync can persist a mutation queued during startup.
603 immediate: true,
604 when: {
605 [DiffTriggerOperation.INSERT]: `TRUE`,
606 [DiffTriggerOperation.UPDATE]: `TRUE`,
607 [DiffTriggerOperation.DELETE]: `TRUE`,
608 },
609 writeType: (_rowId: string) => `insert`,
610 batchQuery: (
611 lockContext: LockContext,
612 batchSize: number,
613 cursor: number,
614 ) =>
615 lockContext.getAll<TableType>(
616 sanitizeSQL`SELECT * FROM ${viewName} LIMIT ? OFFSET ?`,
617 [batchSize, cursor],
618 ),
619 },
620 appliedReceipts,
621 )
622 await Promise.all(appliedReceipts)
623 markReady()
624 }).catch((error) => {
625 database.logger.log({
626 level: LogLevels.error,
627 message: `Could not start syncing process for ${viewName} into ${trackedTableName}`,
628 error,
629 })
630 if (collection.status === `loading`) {
631 markError(error)
632 }
633 })
634
635 const invokeOnUnload = (): void | Promise<void> => {
636 if (onUnloadStarted || !onUnload) return
637 onUnloadStarted = true
638 const cleanup = onUnload
639 onUnload = null
640 return cleanup()
641 }
642
643 return async () => {
644 database.logger.log({
645 level: LogLevels.info,
646 message: `Sync has been stopped for ${viewName} into ${trackedTableName}`,
647 })
648 abortController.abort()
649 const trackingDisposal = disposeTrackingAfterAbort()
650
651 let onUnloadFailure: { error: unknown } | undefined
652 const settleOnUnload = async () => {
653 try {
654 await invokeOnUnload()
655 } catch (error) {
656 onUnloadFailure ??= { error }
657 }
658 }
659
660 // Invoke an already-installed callback in this stack, but also wait
661 // for startup to publish a callback acquired concurrently with abort.
662 const settlements = await Promise.allSettled([
663 startup,
664 trackingDisposal,
665 settleOnUnload(),
666 ])
667 const disposalSettlement = settlements[1]
668 await settleOnUnload()
669 if (disposalSettlement.status === `rejected` && onUnloadFailure) {
670 throw new AggregateError(
671 [disposalSettlement.reason, onUnloadFailure.error],
672 `PowerSync tracking disposal and load-hook cleanup both failed`,
673 { cause: disposalSettlement.reason },
674 )
675 }
676 if (disposalSettlement.status === `rejected`) {
677 throw disposalSettlement.reason
678 }
679 if (onUnloadFailure) throw onUnloadFailure.error
680 }
681 }
682
683 // On-demand mode.
684 // Registers a diff trigger for the active WHERE expressions.
685 function runOnDemandSync() {
686 type DemandRecord = {
687 options: LoadSubsetOptions
688 active: boolean
689 cleanup?: CleanupFn
690 }
691 type PendingRelease = {
692 options: LoadSubsetOptions
693 failures: number
694 }
695 type DemandCleanupTask = {
696 order: number
697 promise: Promise<unknown>
698 }
699 type DemandCleanupFailure = {
700 order: number
701 error: unknown
702 }
703 type DemandCleanupCollector = {
704 tasks: Array<DemandCleanupTask>
705 ownedTasks: Set<Promise<unknown>>
706 failures: Array<DemandCleanupFailure>
707 nextOrder: number
708 }
709
710 const demands = new Map<LoadSubsetOptions, DemandRecord>()
711 const releasedSubsets = new WeakSet<LoadSubsetOptions>()
712 const pendingReleases: Array<PendingRelease> = []
713 const pendingDemandLoads = new Set<Promise<void>>()
714 const pendingDemandCleanups = new Set<Promise<unknown>>()
715 let stopped = false
716 let trackingRevision = 0
717 let reconciledTrackingRevision = 0
718 let rebuildPromise: Promise<void> | null = null
719 let drainingReleases = false
720 let releaseRetryTimer: ReturnType<typeof setTimeout> | undefined
721 let demandCleanupCollector: DemandCleanupCollector | null = null
722 const startup = start()
723 void startup.catch((error) =>
724 database.logger.log({
725 level: LogLevels.error,
726 message: `Could not start syncing process for ${viewName} into ${trackedTableName}`,
727 error,
728 }),
729 )
730
731 const activeWhereExpressions = () =>
732 Array.from(demands.values())
733 .filter((demand) => demand.active)
734 .map((demand) => demand.options.where)
735
736 // One reconciliation owns every queued revision so callers cannot
737 // settle against a stale trigger configuration.
738 const reconcileTracking = async (): Promise<void> => {
739 while (!stopped && reconciledTrackingRevision !== trackingRevision) {
740 const revision = trackingRevision
741 const isCurrent = () => !stopped && trackingRevision === revision
742 const appliedReceipts: Array<SyncAppliedReceipt> = []
743
744 await database.writeLock(async (ctx) => {
745 if (!isCurrent()) return
746 await flushDiffRecordsWithContext(ctx, appliedReceipts)
747 if (!isCurrent()) return
748 await safelyDisposeTracking(ctx)
749 if (!isCurrent()) return
750
751 const active = activeWhereExpressions()
752 // Tracking was absent during an error. Positive baseline rows
753 // alone cannot reveal deletes or predicate exits from that gap.
754 const missing =
755 collection.status === `error`
756 ? new Set(collection.keys())
757 : undefined
758 if (active.length > 0) {
759 const combinedWhere =
760 active.length === 1
761 ? active[0]
762 : or(active[0], active[1], ...active.slice(2))
763 const compiledNewData = compileSQLite(
764 { where: combinedWhere },
765 { jsonColumn: 'NEW.data' },
766 )
767 const compiledOldData = compileSQLite(
768 { where: combinedWhere },
769 { jsonColumn: 'OLD.data' },
770 )
771 const compiledView = compileSQLite({ where: combinedWhere })
772 const newDataWhenClause = toInlinedWhereClause(compiledNewData)
773 const oldDataWhenClause = toInlinedWhereClause(compiledOldData)
774 const viewWhereClause = toInlinedWhereClause(compiledView)
775 await establishTracking(
776 {
777 setupContext: ctx,
778 when: {
779 [DiffTriggerOperation.INSERT]: newDataWhenClause,
780 [DiffTriggerOperation.UPDATE]: `(${newDataWhenClause}) OR (${oldDataWhenClause})`,
781 [DiffTriggerOperation.DELETE]: oldDataWhenClause,
782 },
783 writeType: (rowId: string) =>
784 collection.has(rowId) ? `update` : `insert`,
785 batchQuery: (
786 lockContext: LockContext,
787 batchSize: number,
788 cursor: number,
789 ) =>
790 lockContext
791 .getAll<TableType>(
792 `SELECT * FROM ${viewName} WHERE ${viewWhereClause} LIMIT ? OFFSET ?`,
793 [batchSize, cursor],
794 )
795 .then((rows) => {
796 for (const row of rows) missing?.delete(row.id)
797 return rows
798 }),
799 },
800 appliedReceipts,
801 )
802 }
803 if (!isCurrent()) await safelyDisposeTracking(ctx)
804 else if (missing?.size) {
805 begin()
806 for (const key of missing) write({ type: `delete`, key })
807 appliedReceipts.push(commit())
808 }
809 })
810 await Promise.all(appliedReceipts)
811 if (isCurrent()) {
812 reconciledTrackingRevision = revision
813 // Replacing the trigger alone is not recovery: its baseline
814 // writes must also be applied before the source is ready again.
815 if (collection.status === `error`) markReady()
816 }
817 }
818 }
819
820 const rebuildTracking = async (): Promise<void> => {
821 // New demand can join after reconciliation exits but before its
822 // shared promise clears. Each waiter must check its revision again.
823 while (!stopped && reconciledTrackingRevision !== trackingRevision) {
824 await (rebuildPromise ??= reconcileTracking()
825 .catch((error) => {
826 // A rebuild may already have removed every active diff trigger.
827 // Do not leave healthy consumers ready against a stale source.
828 if (!stopped) markError(error)
829 throw error
830 })
831 .finally(() => {
832 rebuildPromise = null
833 }))
834 }
835 }
836
837 const loadSubset = async (
838 options: LoadSubsetOptions,
839 ): Promise<void> => {
840 if (stopped) return
841 // Never create a trigger that has no observer to drain its diff table.
842 await startup
843 if (
844 // Cleanup can run while startup is pending.
845 // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
846 stopped ||
847 releasedSubsets.has(options) ||
848 options.signal?.aborted
849 ) {
850 return
851 }
852
853 const demand: DemandRecord = { options, active: false }
854 demands.set(options, demand)
855 try {
856 const cleanup = await restConfig.onLoadSubset?.(options)
857 if (cleanup) demand.cleanup = cleanup
858 } catch (error) {
859 demands.delete(options)
860 const collector = demandCleanupCollector
861 if (collector) {
862 collector.failures.push({
863 order: collector.nextOrder++,
864 error,
865 })
866 }
867 throw error
868 }
869
870 if (
871 // The user hook can reenter cleanup.
872 // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
873 stopped ||
874 releasedSubsets.has(options) ||
875 options.signal?.aborted ||
876 demands.get(options) !== demand
877 ) {
878 cleanupDemand(demand)
879 return
880 }
881
882 demand.active = true
883 trackingRevision++
884 await rebuildTracking()
885 }
886
887 const trackLoadSubset = (options: LoadSubsetOptions): Promise<void> => {
888 const task = loadSubset(options)
889 pendingDemandLoads.add(task)
890 const finish = () => pendingDemandLoads.delete(task)
891 void task.then(finish, finish)
892 return task
893 }
894
895 const toInlinedWhereClause = (compiled: {
896 where?: string
897 params: Array<unknown>
898 }): string => {
899 if (!compiled.where) return 'TRUE'
900 const sqlParts = compiled.where.split('?')
901 return sanitizeSQL(
902 sqlParts as unknown as TemplateStringsArray,
903 ...compiled.params,
904 )
905 }
906
907 const reportDemandCleanupFailure = (error: unknown): void => {
908 database.logger.log({
909 level: LogLevels.error,
910 message: `Could not clean up subset hook for ${viewName}`,
911 error,
912 })
913 }
914
915 const cleanupDemand = (demand: DemandRecord): void => {
916 demands.delete(demand.options)
917 const collector = demandCleanupCollector
918 const order = collector ? collector.nextOrder++ : undefined
919 try {
920 const cleanup = demand.cleanup
921 demand.cleanup = undefined
922 const result: unknown = cleanup?.()
923 if (!isPromiseLike(result)) return
924
925 const task = Promise.resolve(result)
926 pendingDemandCleanups.add(task)
927 const finish = () => pendingDemandCleanups.delete(task)
928 void task.then(finish, finish)
929 if (collector) {
930 collector.tasks.push({ order: order!, promise: task })
931 collector.ownedTasks.add(task)
932 void task.catch(() => undefined)
933 } else {
934 void task.catch((error) => {
935 if (!demandCleanupCollector?.ownedTasks.has(task)) {
936 reportDemandCleanupFailure(error)
937 }
938 })
939 }
940 } catch (error) {
941 if (collector) {
942 collector.failures.push({ order: order!, error })
943 } else {
944 reportDemandCleanupFailure(error)
945 }
946 }
947 }
948
949 const throwDemandCleanupFailures = (
950 collector: DemandCleanupCollector,
951 primaryFailures: ReadonlyArray<unknown> = [],
952 ): void => {
953 const errors = [
954 ...primaryFailures,
955 ...collector.failures
956 .slice()
957 .sort((left, right) => left.order - right.order)
958 .map(({ error }) => error),
959 ]
960 if (errors.length === 1) throw errors[0]
961 if (errors.length > 1) {
962 throw new AggregateError(errors, `PowerSync cleanup failed`, {
963 cause: errors[0],
964 })
965 }
966 }
967
968 const performPhysicalRelease = async (
969 options: LoadSubsetOptions,
970 ): Promise<void> => {
971 const compiledDeparting = compileSQLite({ where: options.where })
972 const departingWhereSQL = toInlinedWhereClause(compiledDeparting)
973 let rowsToEvict: Array<{ id: string }>
974 for (;;) {
975 if (stopped) return
976 const revision = trackingRevision
977 const active = activeWhereExpressions()
978 let evictionSQL: string
979 if (active.length === 0) {
980 evictionSQL = `SELECT id FROM ${viewName} WHERE ${departingWhereSQL}`
981 } else {
982 const combinedRemaining =
983 active.length === 1
984 ? active[0]!
985 : or(active[0], active[1], ...active.slice(2))
986 const compiledRemaining = compileSQLite({
987 where: combinedRemaining,
988 })
989 const remainingWhereSQL = toInlinedWhereClause(compiledRemaining)
990 evictionSQL = `SELECT id FROM ${viewName} WHERE (${departingWhereSQL}) AND NOT (${remainingWhereSQL})`
991 }
992
993 rowsToEvict = await database.getAll<{ id: string }>(evictionSQL)
994 // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- cleanup can run during the query
995 if (stopped) return
996 if (trackingRevision === revision) break
997 }
998 if (rowsToEvict.length > 0) {
999 begin()
1000 for (const { id } of rowsToEvict) {
1001 write({ type: `delete`, key: id })
1002 }
1003 void commit()
1004 }
1005 await rebuildTracking()
1006 }
1007
1008 function scheduleReleaseDrain(delay = 0): void {
1009 if (stopped || drainingReleases || releaseRetryTimer) return
1010 if (delay > 0) {
1011 releaseRetryTimer = setTimeout(() => {
1012 releaseRetryTimer = undefined
1013 void drainReleases()
1014 }, delay)
1015 return
1016 }
1017 void drainReleases()
1018 }
1019
1020 async function drainReleases(): Promise<void> {
1021 if (stopped || drainingReleases) return
1022 drainingReleases = true
1023 let retryDelay = 0
1024 try {
1025 const attempts = pendingReleases.length
1026 // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- each release can reenter cleanup
1027 for (let index = 0; !stopped && index < attempts; index++) {
1028 const pending = pendingReleases.shift()!
1029 try {
1030 await performPhysicalRelease(pending.options)
1031 } catch (error) {
1032 pending.failures++
1033 pendingReleases.push(pending)
1034 const delay = Math.min(
1035 1000 * 2 ** (pending.failures - 1),
1036 30000,
1037 )
1038 retryDelay =
1039 retryDelay === 0 ? delay : Math.min(retryDelay, delay)
1040 database.logger.log({
1041 level: LogLevels.error,
1042 message: `Could not release subset tracking for ${viewName}; retrying`,
1043 error,
1044 })
1045 }
1046 }
1047 } finally {
1048 drainingReleases = false
1049 }
1050 if (pendingReleases.length > 0) scheduleReleaseDrain(retryDelay)
1051 }
1052
1053 const unloadSubset = (options: LoadSubsetOptions): void => {
1054 releasedSubsets.add(options)
1055 const demand = demands.get(options)
1056 if (!demand) return
1057
1058 const wasActive = demand.active
1059 if (wasActive) trackingRevision++
1060 cleanupDemand(demand)
1061
1062 if (wasActive) {
1063 pendingReleases.push({ options, failures: 0 })
1064 // New work must not wait for another release's backoff.
1065 clearTimeout(releaseRetryTimer)
1066 releaseRetryTimer = undefined
1067 scheduleReleaseDrain()
1068 }
1069 }
1070
1071 markReady()
1072
1073 return {
1074 cleanup: () => {
1075 stopped = true
1076 clearTimeout(releaseRetryTimer)
1077 releaseRetryTimer = undefined
1078 database.logger.log({
1079 level: LogLevels.info,
1080 message: `Sync has been stopped for ${viewName} into ${trackedTableName}`,
1081 })
1082 abortController.abort()
1083 const trackingDisposal = disposeTrackingAfterAbort()
1084 const pendingLoads = [...pendingDemandLoads]
1085 const pendingTasks = [...pendingDemandCleanups]
1086 const collector: DemandCleanupCollector = {
1087 tasks: pendingTasks.map((promise, order) => ({ order, promise })),
1088 ownedTasks: new Set(pendingTasks),
1089 failures: [],
1090 nextOrder: pendingTasks.length,
1091 }
1092 demandCleanupCollector = collector
1093 for (const demand of demands.values()) {
1094 cleanupDemand(demand)
1095 }
1096 pendingReleases.length = 0
1097
1098 return (async () => {
1099 const ownershipSettlements = await Promise.allSettled([
1100 trackingDisposal,
1101 ...pendingLoads,
1102 ])
1103 const disposalSettlement = ownershipSettlements[0]
1104
1105 let settledTasks = 0
1106 while (settledTasks < collector.tasks.length) {
1107 const tasks = collector.tasks.slice(settledTasks)
1108 settledTasks = collector.tasks.length
1109 const outcomes = await Promise.allSettled(
1110 tasks.map(({ promise }) => promise),
1111 )
1112 for (const [index, outcome] of outcomes.entries()) {
1113 if (outcome.status === `rejected`) {
1114 collector.failures.push({
1115 order: tasks[index]!.order,
1116 error: outcome.reason,
1117 })
1118 }
1119 }
1120 }
1121 throwDemandCleanupFailures(
1122 collector,
1123 disposalSettlement.status === `rejected`
1124 ? [disposalSettlement.reason]
1125 : [],
1126 )
1127 })().finally(() => {
1128 if (demandCleanupCollector === collector) {
1129 demandCleanupCollector = null
1130 }
1131 })
1132 },
1133 loadSubset: trackLoadSubset,
1134 unloadSubset,
1135 }
1136 }
1137 },
1138 // Expose the getSyncMetadata function
1139 getSyncMetadata: undefined,
1140 }
1141
1142 const getKey = (record: OutputType) => asPowerSyncRecord(record).id
1143
1144 const outputConfig: EnhancedPowerSyncCollectionConfig<
1145 TTable,
1146 OutputType,
1147 TSchema
1148 > = {
1149 ...restConfig,
1150 schema,
1151 getKey,
1152 // Syncing should start immediately since we need to monitor the changes for mutations
1153 startSync: true,
1154 syncMode,
1155 sync,
1156 onInsert: async (params) => {
1157 // The transaction here should only ever contain a single insert mutation
1158 return await transactor.applyTransaction(params.transaction)
1159 },
1160 onUpdate: async (params) => {
1161 // The transaction here should only ever contain a single update mutation
1162 return await transactor.applyTransaction(params.transaction)
1163 },
1164 onDelete: async (params) => {
1165 // The transaction here should only ever contain a single delete mutation
1166 return await transactor.applyTransaction(params.transaction)
1167 },
1168 utils: {
1169 getMeta: () => ({
1170 tableName: viewName,
1171 trackedTableName,
1172 metadataIsTracked,
1173 serializeValue: (value) =>
1174 serializeForSQLite<TTable>(value, table, serializer),
1175 }),
1176 },
1177 }
1178 return outputConfig
1179}