Skip to content
Source
Sign inSign up
TanStack/db
Open History to see the last commit

powersync.ts

1,192 lines41.2 KB

1import { DiffTriggerOperation, LogLevels, sanitizeSQL } from '@powersync/common'
2import { or, whenSyncAccepted, withCollectionConfigFactory } from '@tanstack/db'
3import { compileSQLite } from './sqlite-compiler'
4import { PendingOperationStore } from './PendingOperationStore'
5import { PowerSyncTransactor } from './PowerSyncTransactor'
6import { DEFAULT_BATCH_SIZE } from './definitions'
7import { asPowerSyncRecord, mapOperation } from './helpers'
8import { convertTableToSchema } from './schema'
9import { serializeForSQLite } from './serialization'
10import type {
11 CleanupFn,
12 LoadSubsetOptions,
13 OperationType,
14 SyncAppliedReceipt,
15 SyncConfig,
16} from '@tanstack/db'
17import type {
18 AnyTableColumnType,
19 ExtractedTable,
20 OptionalExtractedTable,
21} from './helpers'
22import type {
23 BasePowerSyncCollectionConfig,
24 ConfigWithArbitraryCollectionTypes,
25 ConfigWithSQLiteInputType,
26 ConfigWithSQLiteTypes,
27 EnhancedPowerSyncCollectionConfig,
28 InferPowerSyncOutputType,
29 PowerSyncCollectionConfig,
30 PowerSyncCollectionUtils,
31} from './definitions'
32import type { PendingOperation } from './PendingOperationStore'
33import type { StandardSchemaV1 } from '@standard-schema/spec'
34import type { LockContext, Table, TriggerDiffRecord } from '@powersync/common'
35​
36function isPromiseLike(value: unknown): value is PromiseLike<unknown> {
37 return (
38 !!value &&
39 (typeof value === `object` || typeof value === `function`) &&
40 typeof (value as { then?: unknown }).then === `function`
41 )
43​
44/**
45 * Creates PowerSync collection options for use with a standard Collection.
46 *
47 * @template TTable - The SQLite-based typing
48 * @template TSchema - The validation schema type (optionally supports a custom input type)
49 * @param config - Configuration options for the PowerSync collection
50 * @returns Collection options with utilities
51 */
52​
53// Overload 1: No schema is provided
54​
55/**
56 * Creates a PowerSync collection configuration with basic default validation.
57 * Input and Output types are the SQLite column types.
58 *
59 * @example
60 * ```typescript
61 * const APP_SCHEMA = new Schema({
62 * documents: new Table({
63 * name: column.text,
64 * }),
65 * })
66 *
67 * type Document = (typeof APP_SCHEMA)["types"]["documents"]
68 *
69 * const db = new PowerSyncDatabase({
70 * database: {
71 * dbFilename: "test.sqlite",
72 * },
73 * schema: APP_SCHEMA,
74 * })
75 *
76 * const collection = createCollection(
77 * powerSyncCollectionOptions({
78 * database: db,
79 * table: APP_SCHEMA.props.documents
80 * })
81 * )
82 * ```
83 */
84export function powerSyncCollectionOptions<TTable extends Table = Table>(
85 config: BasePowerSyncCollectionConfig<TTable, never> & ConfigWithSQLiteTypes,
86): EnhancedPowerSyncCollectionConfig<
87 TTable,
88 OptionalExtractedTable<TTable>,
89 never
91​
92// Overload 2: Schema is provided and the TInput matches SQLite types.
93​
94/**
95 * Creates a PowerSync collection configuration with schema validation.
96 *
97 * The input types satisfy the SQLite column types.
98 *
99 * The output types are defined by the provided schema. This schema can enforce additional
100 * validation or type transforms.
101 * Arbitrary output typed mutations are encoded to SQLite for persistence. We provide a basic standard
102 * serialization implementation to serialize column values. Custom or advanced types require providing additional
103 * serializer specifications. Partial column overrides can be supplied to `serializer`.
104 *
105 * @example
106 * ```typescript
107 * import { z } from "zod"
108 *
109 * // The PowerSync SQLite schema
110 * const APP_SCHEMA = new Schema({
111 * documents: new Table({
112 * name: column.text,
113 * // Dates are stored as ISO date strings in SQLite
114 * created_at: column.text
115 * }),
116 * })
117 *
118 * // Advanced Zod validations. The output type of this schema
119 * // is constrained to the SQLite schema of APP_SCHEMA
120 * const schema = z.object({
121 * id: z.string(),
122 * // Notice that `name` is not nullable (is required) here and it has additional validation
123 * name: z.string().min(3, { message: "Should be at least 3 characters" }).nullable(),
124 * // The input type is still the SQLite string type. While collections will output smart Date instances.
125 * created_at: z.string().transform(val => new Date(val))
126 * })
127 *
128 * const collection = createCollection(
129 * powerSyncCollectionOptions({
130 * database: db,
131 * table: APP_SCHEMA.props.documents,
132 * schema,
133 * serializer: {
134 * // The default is toISOString, this is just to demonstrate custom overrides
135 * created_at: (outputValue) => outputValue.toISOString(),
136 * },
137 * })
138 * )
139 * ```
140 */
141export function powerSyncCollectionOptions<
142 TTable extends Table,
143 TSchema extends StandardSchemaV1<
144 // TInput is the SQLite types. We can use the supplied schema to validate sync input
145 OptionalExtractedTable<TTable>,
146 AnyTableColumnType<TTable>
147 >,
148>(
149 config: BasePowerSyncCollectionConfig<TTable, TSchema> &
150 ConfigWithSQLiteInputType<TTable, TSchema>,
151): EnhancedPowerSyncCollectionConfig<
152 TTable,
153 InferPowerSyncOutputType<TTable, TSchema>,
154 TSchema
155> & {
156 schema: TSchema
158​
159// Overload 3: Schema is provided with arbitrary TInput and TOutput
160/**
161 * Creates a PowerSync collection configuration with schema validation.
162 *
163 * The input types are not linked to the internal SQLite table types. This can
164 * give greater flexibility, e.g. by accepting rich types as input for `insert` or `update` operations.
165 * An additional `deserializationSchema` is required in order to process incoming SQLite updates to the output type.
166 *
167 * The output types are defined by the provided schema. This schema can enforce additional
168 * validation or type transforms.
169 * Arbitrary output typed mutations are encoded to SQLite for persistence. We provide a basic standard
170 * serialization implementation to serialize column values. Custom or advanced types require providing additional
171 * serializer specifications. Partial column overrides can be supplied to `serializer`.
172 *
173 * @example
174 * ```typescript
175 * import { z } from "zod"
176 *
177 * // The PowerSync SQLite schema
178 * const APP_SCHEMA = new Schema({
179 * documents: new Table({
180 * name: column.text,
181 * // Booleans are represented as integers in SQLite
182 * is_active: column.integer
183 * }),
184 * })
185 *
186 * // Advanced Zod validations.
187 * // We accept boolean values as input for operations and expose Booleans in query results
188 * const schema = z.object({
189 * id: z.string(),
190 * isActive: z.boolean(), // TInput and TOutput are boolean
191 * })
192 *
193 * // The deserializationSchema converts the SQLite synced INTEGER (0/1) values to booleans.
194 * const deserializationSchema = z.object({
195 * id: z.string(),
196 * isActive: z.number().nullable().transform((val) => val == null ? true : val > 0),
197 * })
198 *
199 * const collection = createCollection(
200 * powerSyncCollectionOptions({
201 * database: db,
202 * table: APP_SCHEMA.props.documents,
203 * schema,
204 * deserializationSchema,
205 * })
206 * )
207 * ```
208 */
209export function powerSyncCollectionOptions<
210 TTable extends Table,
211 TSchema extends StandardSchemaV1<
212 // The input and output must have the same keys, the value types can be arbitrary
213 AnyTableColumnType<TTable>,
214 AnyTableColumnType<TTable>
215 >,
216>(
217 config: BasePowerSyncCollectionConfig<TTable, TSchema> &
218 ConfigWithArbitraryCollectionTypes<TTable, TSchema>,
219): EnhancedPowerSyncCollectionConfig<
220 TTable,
221 InferPowerSyncOutputType<TTable, TSchema>,
222 TSchema
223> & {
224 utils: PowerSyncCollectionUtils<TTable>
225 schema: TSchema
227​
228/**
229 * Implementation of powerSyncCollectionOptions that handles both schema and non-schema configurations.
230 */
231​
232export function powerSyncCollectionOptions<
233 TTable extends Table,
234 TSchema extends StandardSchemaV1<any> = never,
235>(
236 config: PowerSyncCollectionConfig<TTable, TSchema>,
237): ReturnType<typeof createPowerSyncCollectionConfig<TTable, TSchema>> {
238 const outputConfig = createPowerSyncCollectionConfig(config)
239 return withCollectionConfigFactory(outputConfig, () =>
240 createPowerSyncCollectionConfig(config),
241 )
243​
244function createPowerSyncCollectionConfig<
245 TTable extends Table,
246 TSchema extends StandardSchemaV1<any> = never,
247>(config: PowerSyncCollectionConfig<TTable, TSchema>) {
248 const {
249 database,
250 table,
251 schema: inputSchema,
252 syncBatchSize = DEFAULT_BATCH_SIZE,
253 syncMode = 'eager',
254 ...restConfig
255 } = config
256​
257 const deserializationSchema =
258 `deserializationSchema` in config ? config.deserializationSchema : null
259 const serializer = `serializer` in config ? config.serializer : undefined
260 const onDeserializationError =
261 `onDeserializationError` in config
262 ? config.onDeserializationError
263 : undefined
264​
265 // The SQLite table type
266 type TableType = ExtractedTable<TTable>
267​
268 // The collection output type
269 type OutputType = InferPowerSyncOutputType<TTable, TSchema>
270​
271 const { viewName, trackMetadata: metadataIsTracked } = table
272​
273 /**
274 * Deserializes data from the incoming sync stream
275 */
276 const deserializeSyncRow = (value: TableType): OutputType => {
277 const validationSchema = deserializationSchema || schema
278 const validation = validationSchema[`~standard`].validate(value)
279 if (`value` in validation) {
280 return validation.value
281 } else if (`issues` in validation) {
282 const issueMessage = `Failed to validate incoming data for ${viewName}. Issues: ${validation.issues.map((issue) => `${issue.path} - ${issue.message}`)}`
283 database.logger.log({ level: LogLevels.error, message: issueMessage })
284 onDeserializationError!(validation)
285 throw new Error(issueMessage)
286 } else {
287 const unknownErrorMessage = `Unknown deserialization error for ${viewName}`
288 database.logger.log({
289 level: LogLevels.error,
290 message: unknownErrorMessage,
291 })
292 onDeserializationError!({ issues: [{ message: unknownErrorMessage }] })
293 throw new Error(unknownErrorMessage)
294 }
295 }
296​
297 // We can do basic runtime validations for columns if not explicit schema has been provided
298 const schema = inputSchema ?? (convertTableToSchema(table) as TSchema)
299 /**
300 * The onInsert, onUpdate, and onDelete handlers should only return
301 * after we have written the changes to TanStack DB.
302 * We currently only write to TanStack DB from a diff trigger.
303 * We wait for the diff trigger to observe the change,
304 * and only then return from the on[X] handlers.
305 * This ensures that when the transaction is reported as
306 * complete to the caller, the in-memory state is already
307 * consistent with the database.
308 */
309 const pendingOperationStore = PendingOperationStore.GLOBAL
310 // Keep the tracked table unique in case of multiple tabs.
311 const trackedTableName = `__${viewName}_tracking_${Math.floor(
312 Math.random() * 0xffffffff,
313 )
314 .toString(16)
315 .padStart(8, `0`)}`
316​
317 const transactor = new PowerSyncTransactor({
318 database,
319 })
320​
321 /**
322 * "sync"
323 * Notice that this describes the Sync between the local SQLite table
324 * and the in-memory tanstack-db collection.
325 */
326 const sync: SyncConfig<OutputType, string> = {
327 sync: (params) => {
328 const { begin, write, collection, commit, markReady, markError } = params
329 const abortController = new AbortController()
330​
331 let disposeTracking:
332 ((options?: { context?: LockContext }) => Promise<void>) | null = null
333 let trackingSetup: Promise<void> | null = null
334 let activeTrackingDisposal: Promise<void> | null = null
335 let abortTrackingDisposal: Promise<void> | null = null
336​
337 if (syncMode === `eager`) {
338 return runEagerSync()
339 } else {
340 return runOnDemandSync()
341 }
342​
343 /**
344 * Disposes the current diff trigger, if one is active, and clears the
345 * tracking state.
346 */
347 async function safelyDisposeTracking(
348 context?: LockContext,
349 ): Promise<void> {
350 // Cleanup can race trigger creation. Wait until the disposer has been
351 // published so an abort cannot strand a freshly-created trigger.
352 const setup = trackingSetup
353 if (setup) {
354 await setup.catch(() => undefined)
355 }
356​
357 const dispose = disposeTracking
358 if (!dispose) {
359 // Reconciliation may have claimed the disposer after this caller
360 // observed setup. Join that invocation instead of settling early.
361 await activeTrackingDisposal
362 return
363 }
364​
365 disposeTracking = null
366 let disposal: Promise<void>
367 try {
368 disposal = Promise.resolve(dispose(context ? { context } : undefined))
369 } catch (error) {
370 disposal = Promise.reject(error)
371 }
372 activeTrackingDisposal = disposal
373 try {
374 await disposal
375 } finally {
376 if (activeTrackingDisposal === disposal) {
377 activeTrackingDisposal = null
378 }
379 }
380 }
381​
382 function disposeTrackingAfterAbort(): Promise<void> {
383 const disposal = (abortTrackingDisposal ??= safelyDisposeTracking())
384 // Abort event dispatch does not observe a listener's returned promise.
385 // The eager cleanup callback awaits this same promise below.
386 void disposal.catch(() => undefined)
387 return disposal
388 }
389​
390 async function establishTracking(
391 options: Parameters<typeof createDiffTrigger>[0],
392 appliedReceipts: Array<SyncAppliedReceipt>,
393 ): Promise<void> {
394 const setup = (async () => {
395 const dispose = await createDiffTrigger(options, appliedReceipts)
396 disposeTracking = dispose
397 })()
398 trackingSetup = setup
399​
400 try {
401 await setup
402 } finally {
403 if (trackingSetup === setup) {
404 trackingSetup = null
405 }
406 }
407 }
408​
409 async function createDiffTrigger(
410 options: {
411 setupContext?: LockContext
412 when: Record<DiffTriggerOperation, string>
413 writeType: (rowId: string) => OperationType
414 batchQuery: (
415 lockContext: LockContext,
416 batchSize: number,
417 cursor: number,
418 ) => Promise<Array<TableType>>
419 },
420 appliedReceipts: Array<SyncAppliedReceipt>,
421 ) {
422 const { setupContext, when, writeType, batchQuery } = options
423​
424 return await database.triggers.createDiffTrigger({
425 source: viewName,
426 destination: trackedTableName,
427 columns: table.columns.map((column) => column.name),
428 setupContext,
429 when,
430 hooks: {
431 beforeCreate: async (context) => {
432 let currentBatchCount = syncBatchSize
433 let cursor = 0
434 while (currentBatchCount == syncBatchSize) {
435 begin()
436​
437 const batchItems = await batchQuery(
438 context,
439 syncBatchSize,
440 cursor,
441 )
442 currentBatchCount = batchItems.length
443 cursor += currentBatchCount
444 for (const row of batchItems) {
445 write({
446 type: writeType(row.id),
447 value: deserializeSyncRow(row),
448 })
449 }
450 appliedReceipts.push(commit())
451 }
452 database.logger.log({
453 level: LogLevels.info,
454 message: `Sync is ready for ${viewName} into ${trackedTableName}`,
455 })
456 },
457 },
458 })
459 }
460​
461 async function flushDiffRecords(): Promise<void> {
462 // PowerSync can notify after creating the tracking table but before its
463 // create call returns. Preserve that notification until the disposer,
464 // which proves the trigger is usable, has been published.
465 const setup = trackingSetup
466 if (setup) {
467 await setup.catch(() => undefined)
468 }
469 if (!disposeTracking) {
470 return
471 }
472​
473 const ignoredReceipts: Array<SyncAppliedReceipt> = []
474 await database
475 .writeTransaction(async (context) => {
476 await flushDiffRecordsWithContext(context, ignoredReceipts)
477 })
478 .catch((error) => {
479 database.logger.log({
480 level: LogLevels.error,
481 message: `An error has been detected in the sync handler`,
482 error,
483 })
484 })
485 }
486​
487 // We can use this directly if we want to pair a flush with dispose+recreate diff trigger.
488 async function flushDiffRecordsWithContext(
489 context: LockContext,
490 appliedReceipts: Array<SyncAppliedReceipt>,
491 ): Promise<void> {
492 // There is nothing to flush if no tracking table is currently active.
493 if (!disposeTracking) {
494 return
495 }
496​
497 try {
498 begin()
499 const operations = await context.getAll<TriggerDiffRecord>(
500 `SELECT * FROM ${trackedTableName} ORDER BY operation_id ASC`,
501 )
502 const pendingOperations: Array<PendingOperation> = []
503​
504 for (const op of operations) {
505 const { id, operation, timestamp, value } = op
506 const parsedValue = deserializeSyncRow({
507 id,
508 ...JSON.parse(value),
509 })
510 const parsedPreviousValue =
511 op.operation == DiffTriggerOperation.UPDATE
512 ? deserializeSyncRow({
513 id,
514 ...JSON.parse(op.previous_value),
515 })
516 : undefined
517 write({
518 type: mapOperation(operation),
519 value: parsedValue,
520 previousValue: parsedPreviousValue,
521 })
522 pendingOperations.push({
523 id,
524 operation,
525 timestamp,
526 tableName: viewName,
527 })
528 }
529​
530 // clear the current operations
531 await context.execute(`DELETE FROM ${trackedTableName}`)
532​
533 const applied = commit()
534 appliedReceipts.push(applied)
535 // Mutation persistence is what releases the Collection's FIFO gate.
536 // Confirm these local operations after the sync transaction is
537 // staged; waiting for its applied receipt would deadlock the user
538 // transaction that currently parks it.
539 pendingOperationStore.resolvePendingFor(pendingOperations)
540 } catch (error) {
541 database.logger.log({
542 level: LogLevels.error,
543 message: `An error has been detected in the sync handler`,
544 error,
545 })
546 }
547 }
548​
549 // The sync function needs to be synchronous.
550 async function start(afterOnChangeRegistered?: () => Promise<void>) {
551 database.logger.log({
552 level: LogLevels.info,
553 message: `Sync is starting for ${viewName} into ${trackedTableName}`,
554 })
555 database.onChangeWithCallback(
556 {
557 onChange: async () => {
558 await flushDiffRecords()
559 },
560 },
561 {
562 signal: abortController.signal,
563 triggerImmediate: false,
564 tables: [trackedTableName],
565 },
566 )
567​
568 await afterOnChangeRegistered?.()
569​
570 // If the abort controller was aborted while processing the request above
571 if (abortController.signal.aborted) {
572 await disposeTrackingAfterAbort()
573 } else {
574 abortController.signal.addEventListener(
575 `abort`,
576 () => {
577 void disposeTrackingAfterAbort()
578 },
579 { once: true },
580 )
581 }
582 }
583​
584 // Eager mode.
585 // Registers a diff trigger for the entire table.
586 function runEagerSync() {
587 let onUnload: CleanupFn | void | null = null
588 let onUnloadStarted = false
589​
590 const startup = start(async () => {
591 const cleanup = await restConfig.onLoad?.()
592 onUnload = cleanup
593 if (abortController.signal.aborted) {
594 return
595 }
596​
597 const appliedReceipts: Array<SyncAppliedReceipt> = []
598 await establishTracking(
599 {
600 when: {
601 [DiffTriggerOperation.INSERT]: `TRUE`,
602 [DiffTriggerOperation.UPDATE]: `TRUE`,
603 [DiffTriggerOperation.DELETE]: `TRUE`,
604 },
605 writeType: (_rowId: string) => `insert`,
606 batchQuery: (
607 lockContext: LockContext,
608 batchSize: number,
609 cursor: number,
610 ) =>
611 lockContext.getAll<TableType>(
612 sanitizeSQL`SELECT * FROM ${viewName} LIMIT ? OFFSET ?`,
613 [batchSize, cursor],
614 ),
615 },
616 appliedReceipts,
617 )
618 // Readiness counts accepted rows. A mutation handler waits for it,
619 // and its own persisting transaction holds the startup rows.
620 await Promise.all(appliedReceipts.map(whenSyncAccepted))
621 markReady()
622 }).catch((error) => {
623 database.logger.log({
624 level: LogLevels.error,
625 message: `Could not start syncing process for ${viewName} into ${trackedTableName}`,
626 error,
627 })
628 if (collection.status === `loading`) {
629 markError(error)
630 }
631 })
632​
633 const invokeOnUnload = (): void | Promise<void> => {
634 if (onUnloadStarted || !onUnload) return
635 onUnloadStarted = true
636 const cleanup = onUnload
637 onUnload = null
638 return cleanup()
639 }
640​
641 return async () => {
642 database.logger.log({
643 level: LogLevels.info,
644 message: `Sync has been stopped for ${viewName} into ${trackedTableName}`,
645 })
646 abortController.abort()
647 const trackingDisposal = disposeTrackingAfterAbort()
648​
649 let onUnloadFailure: { error: unknown } | undefined
650 const settleOnUnload = async () => {
651 try {
652 await invokeOnUnload()
653 } catch (error) {
654 onUnloadFailure ??= { error }
655 }
656 }
657​
658 // Invoke an already-installed callback in this stack, but also wait
659 // for startup to publish a callback acquired concurrently with abort.
660 const settlements = await Promise.allSettled([
661 startup,
662 trackingDisposal,
663 settleOnUnload(),
664 ])
665 const disposalSettlement = settlements[1]
666 await settleOnUnload()
667 if (disposalSettlement.status === `rejected` && onUnloadFailure) {
668 throw new AggregateError(
669 [disposalSettlement.reason, onUnloadFailure.error],
670 `PowerSync tracking disposal and load-hook cleanup both failed`,
671 { cause: disposalSettlement.reason },
672 )
673 }
674 if (disposalSettlement.status === `rejected`) {
675 throw disposalSettlement.reason
676 }
677 if (onUnloadFailure) throw onUnloadFailure.error
678 }
679 }
680​
681 // On-demand mode.
682 // Registers a diff trigger for the active WHERE expressions.
683 function runOnDemandSync() {
684 type DemandRecord = {
685 options: LoadSubsetOptions
686 active: boolean
687 cleanup?: CleanupFn
688 }
689 type PendingRelease = {
690 options: LoadSubsetOptions
691 failures: number
692 }
693 type DemandCleanupTask = {
694 order: number
695 promise: Promise<unknown>
696 }
697 type DemandCleanupFailure = {
698 order: number
699 error: unknown
700 }
701 type DemandCleanupCollector = {
702 tasks: Array<DemandCleanupTask>
703 ownedTasks: Set<Promise<unknown>>
704 failures: Array<DemandCleanupFailure>
705 nextOrder: number
706 }
707​
708 const demands = new Map<LoadSubsetOptions, DemandRecord>()
709 const releasedSubsets = new WeakSet<LoadSubsetOptions>()
710 const pendingReleases: Array<PendingRelease> = []
711 const pendingDemandLoads = new Set<Promise<void>>()
712 const pendingDemandCleanups = new Set<Promise<unknown>>()
713 let stopped = false
714 let trackingRevision = 0
715 let reconciledTrackingRevision = 0
716 let rebuildPromise: Promise<void> | null = null
717 let drainingReleases = false
718 let releaseRetryTimer: ReturnType<typeof setTimeout> | undefined
719 let demandCleanupCollector: DemandCleanupCollector | null = null
720 const startup = start()
721 void startup.catch((error) =>
722 database.logger.log({
723 level: LogLevels.error,
724 message: `Could not start syncing process for ${viewName} into ${trackedTableName}`,
725 error,
726 }),
727 )
728​
729 const activeWhereExpressions = () =>
730 Array.from(demands.values())
731 .filter((demand) => demand.active)
732 .map((demand) => demand.options.where)
733​
734 // One reconciliation owns every queued revision so callers cannot
735 // settle against a stale trigger configuration.
736 const reconcileTracking = async (): Promise<void> => {
737 while (!stopped && reconciledTrackingRevision !== trackingRevision) {
738 const revision = trackingRevision
739 const isCurrent = () => !stopped && trackingRevision === revision
740 const appliedReceipts: Array<SyncAppliedReceipt> = []
741​
742 await database.writeLock(async (ctx) => {
743 if (!isCurrent()) return
744 await flushDiffRecordsWithContext(ctx, appliedReceipts)
745 if (!isCurrent()) return
746 await safelyDisposeTracking(ctx)
747 if (!isCurrent()) return
748​
749 const active = activeWhereExpressions()
750 // Tracking was absent during an error. Positive baseline rows
751 // alone cannot reveal deletes or predicate exits from that gap.
752 const missing =
753 collection.status === `error`
754 ? new Set(collection.keys())
755 : undefined
756 if (active.length > 0) {
757 const combinedWhere =
758 active.length === 1
759 ? active[0]
760 : or(active[0], active[1], ...active.slice(2))
761 const compiledNewData = compileSQLite(
762 { where: combinedWhere },
763 { jsonColumn: 'NEW.data' },
764 )
765 const compiledOldData = compileSQLite(
766 { where: combinedWhere },
767 { jsonColumn: 'OLD.data' },
768 )
769 const compiledView = compileSQLite({ where: combinedWhere })
770 const newDataWhenClause = toInlinedWhereClause(compiledNewData)
771 const oldDataWhenClause = toInlinedWhereClause(compiledOldData)
772 const viewWhereClause = toInlinedWhereClause(compiledView)
773 await establishTracking(
774 {
775 setupContext: ctx,
776 when: {
777 [DiffTriggerOperation.INSERT]: newDataWhenClause,
778 [DiffTriggerOperation.UPDATE]: `(${newDataWhenClause}) OR (${oldDataWhenClause})`,
779 [DiffTriggerOperation.DELETE]: oldDataWhenClause,
780 },
781 writeType: (rowId: string) =>
782 collection.has(rowId) ? `update` : `insert`,
783 batchQuery: (
784 lockContext: LockContext,
785 batchSize: number,
786 cursor: number,
787 ) =>
788 lockContext
789 .getAll<TableType>(
790 `SELECT * FROM ${viewName} WHERE ${viewWhereClause} LIMIT ? OFFSET ?`,
791 [batchSize, cursor],
792 )
793 .then((rows) => {
794 for (const row of rows) missing?.delete(row.id)
795 return rows
796 }),
797 },
798 appliedReceipts,
799 )
800 }
801 if (!isCurrent()) await safelyDisposeTracking(ctx)
802 else if (missing?.size) {
803 begin()
804 for (const key of missing) write({ type: `delete`, key })
805 appliedReceipts.push(commit())
806 }
807 })
808 await Promise.all(appliedReceipts)
809 if (isCurrent()) {
810 reconciledTrackingRevision = revision
811 // Replacing the trigger alone is not recovery: its baseline
812 // writes must also be applied before the source is ready again.
813 if (collection.status === `error`) markReady()
814 }
815 }
816 }
817​
818 const rebuildTracking = async (): Promise<void> => {
819 // New demand can join after reconciliation exits but before its
820 // shared promise clears. Each waiter must check its revision again.
821 while (!stopped && reconciledTrackingRevision !== trackingRevision) {
822 await (rebuildPromise ??= reconcileTracking()
823 .catch((error) => {
824 // A rebuild may already have removed every active diff trigger.
825 // Do not leave healthy consumers ready against a stale source.
826 if (!stopped) markError(error)
827 throw error
828 })
829 .finally(() => {
830 rebuildPromise = null
831 }))
832 }
833 }
834​
835 const loadSubset = async (
836 options: LoadSubsetOptions,
837 ): Promise<void> => {
838 if (stopped) return
839 // Never create a trigger that has no observer to drain its diff table.
840 await startup
841 if (
842 // Cleanup can run while startup is pending.
843 // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
844 stopped ||
845 releasedSubsets.has(options) ||
846 options.signal?.aborted
847 ) {
848 return
849 }
850​
851 const demand: DemandRecord = { options, active: false }
852 demands.set(options, demand)
853 try {
854 const cleanup = await restConfig.onLoadSubset?.(options)
855 if (cleanup) demand.cleanup = cleanup
856 } catch (error) {
857 demands.delete(options)
858 const collector = demandCleanupCollector
859 if (collector) {
860 collector.failures.push({
861 order: collector.nextOrder++,
862 error,
863 })
864 }
865 throw error
866 }
867​
868 if (
869 // The user hook can reenter cleanup.
870 // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
871 stopped ||
872 releasedSubsets.has(options) ||
873 options.signal?.aborted ||
874 demands.get(options) !== demand
875 ) {
876 cleanupDemand(demand)
877 return
878 }
879​
880 demand.active = true
881 trackingRevision++
882 await rebuildTracking()
883 }
884​
885 const trackLoadSubset = (options: LoadSubsetOptions): Promise<void> => {
886 const task = loadSubset(options)
887 pendingDemandLoads.add(task)
888 const finish = () => pendingDemandLoads.delete(task)
889 void task.then(finish, finish)
890 return task
891 }
892​
893 const toInlinedWhereClause = (compiled: {
894 where?: string
895 params: Array<unknown>
896 }): string => {
897 if (!compiled.where) return 'TRUE'
898 const sqlParts = compiled.where.split('?')
899 return sanitizeSQL(
900 sqlParts as unknown as TemplateStringsArray,
901 ...compiled.params,
902 )
903 }
904​
905 const reportDemandCleanupFailure = (error: unknown): void => {
906 database.logger.log({
907 level: LogLevels.error,
908 message: `Could not clean up subset hook for ${viewName}`,
909 error,
910 })
911 }
912​
913 const cleanupDemand = (demand: DemandRecord): void => {
914 demands.delete(demand.options)
915 const collector = demandCleanupCollector
916 const order = collector ? collector.nextOrder++ : undefined
917 try {
918 const cleanup = demand.cleanup
919 demand.cleanup = undefined
920 const result: unknown = cleanup?.()
921 if (!isPromiseLike(result)) return
922​
923 const task = Promise.resolve(result)
924 pendingDemandCleanups.add(task)
925 const finish = () => pendingDemandCleanups.delete(task)
926 void task.then(finish, finish)
927 if (collector) {
928 collector.tasks.push({ order: order!, promise: task })
929 collector.ownedTasks.add(task)
930 void task.catch(() => undefined)
931 } else {
932 void task.catch((error) => {
933 if (!demandCleanupCollector?.ownedTasks.has(task)) {
934 reportDemandCleanupFailure(error)
935 }
936 })
937 }
938 } catch (error) {
939 if (collector) {
940 collector.failures.push({ order: order!, error })
941 } else {
942 reportDemandCleanupFailure(error)
943 }
944 }
945 }
946​
947 const throwDemandCleanupFailures = (
948 collector: DemandCleanupCollector,
949 primaryFailures: ReadonlyArray<unknown> = [],
950 ): void => {
951 const errors = [
952 ...primaryFailures,
953 ...collector.failures
954 .slice()
955 .sort((left, right) => left.order - right.order)
956 .map(({ error }) => error),
957 ]
958 if (errors.length === 1) throw errors[0]
959 if (errors.length > 1) {
960 throw new AggregateError(errors, `PowerSync cleanup failed`, {
961 cause: errors[0],
962 })
963 }
964 }
965​
966 const performPhysicalRelease = async (
967 options: LoadSubsetOptions,
968 ): Promise<void> => {
969 const compiledDeparting = compileSQLite({ where: options.where })
970 const departingWhereSQL = toInlinedWhereClause(compiledDeparting)
971 let rowsToEvict: Array<{ id: string }>
972 for (;;) {
973 if (stopped) return
974 const revision = trackingRevision
975 const active = activeWhereExpressions()
976 let evictionSQL: string
977 if (active.length === 0) {
978 evictionSQL = `SELECT id FROM ${viewName} WHERE ${departingWhereSQL}`
979 } else {
980 const combinedRemaining =
981 active.length === 1
982 ? active[0]!
983 : or(active[0], active[1], ...active.slice(2))
984 const compiledRemaining = compileSQLite({
985 where: combinedRemaining,
986 })
987 const remainingWhereSQL = toInlinedWhereClause(compiledRemaining)
988 evictionSQL = `SELECT id FROM ${viewName} WHERE (${departingWhereSQL}) AND NOT (${remainingWhereSQL})`
989 }
990​
991 rowsToEvict = await database.getAll<{ id: string }>(evictionSQL)
992 // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- cleanup can run during the query
993 if (stopped) return
994 if (trackingRevision === revision) break
995 }
996 if (rowsToEvict.length > 0) {
997 begin()
998 for (const { id } of rowsToEvict) {
999 write({ type: `delete`, key: id })
1001 void commit()
1003 await rebuildTracking()
1005​
1006 function scheduleReleaseDrain(delay = 0): void {
1007 if (stopped || drainingReleases || releaseRetryTimer) return
1008 if (delay > 0) {
1009 releaseRetryTimer = setTimeout(() => {
1010 releaseRetryTimer = undefined
1011 void drainReleases()
1012 }, delay)
1013 return
1015 void drainReleases()
1017​
1018 async function drainReleases(): Promise<void> {
1019 if (stopped || drainingReleases) return
1020 drainingReleases = true
1021 let retryDelay = 0
1022 try {
1023 const attempts = pendingReleases.length
1024 // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- each release can reenter cleanup
1025 for (let index = 0; !stopped && index < attempts; index++) {
1026 const pending = pendingReleases.shift()!
1027 try {
1028 await performPhysicalRelease(pending.options)
1029 } catch (error) {
1030 pending.failures++
1031 pendingReleases.push(pending)
1032 const delay = Math.min(
1033 1000 * 2 ** (pending.failures - 1),
1034 30000,
1036 retryDelay =
1037 retryDelay === 0 ? delay : Math.min(retryDelay, delay)
1038 database.logger.log({
1039 level: LogLevels.error,
1040 message: `Could not release subset tracking for ${viewName}; retrying`,
1041 error,
1042 })
1045 } finally {
1046 drainingReleases = false
1048 if (pendingReleases.length > 0) scheduleReleaseDrain(retryDelay)
1050​
1051 const unloadSubset = (options: LoadSubsetOptions): void => {
1052 releasedSubsets.add(options)
1053 const demand = demands.get(options)
1054 if (!demand) return
1055​
1056 const wasActive = demand.active
1057 if (wasActive) trackingRevision++
1058 cleanupDemand(demand)
1059​
1060 if (wasActive) {
1061 pendingReleases.push({ options, failures: 0 })
1062 // New work must not wait for another release's backoff.
1063 clearTimeout(releaseRetryTimer)
1064 releaseRetryTimer = undefined
1065 scheduleReleaseDrain()
1068​
1069 markReady()
1070​
1071 return {
1072 cleanup: () => {
1073 stopped = true
1074 clearTimeout(releaseRetryTimer)
1075 releaseRetryTimer = undefined
1076 database.logger.log({
1077 level: LogLevels.info,
1078 message: `Sync has been stopped for ${viewName} into ${trackedTableName}`,
1079 })
1080 abortController.abort()
1081 const trackingDisposal = disposeTrackingAfterAbort()
1082 const pendingLoads = [...pendingDemandLoads]
1083 const pendingTasks = [...pendingDemandCleanups]
1084 const collector: DemandCleanupCollector = {
1085 tasks: pendingTasks.map((promise, order) => ({ order, promise })),
1086 ownedTasks: new Set(pendingTasks),
1087 failures: [],
1088 nextOrder: pendingTasks.length,
1090 demandCleanupCollector = collector
1091 for (const demand of demands.values()) {
1092 cleanupDemand(demand)
1094 pendingReleases.length = 0
1095​
1096 return (async () => {
1097 const ownershipSettlements = await Promise.allSettled([
1098 trackingDisposal,
1099 ...pendingLoads,
1100 ])
1101 const disposalSettlement = ownershipSettlements[0]
1102​
1103 let settledTasks = 0
1104 while (settledTasks < collector.tasks.length) {
1105 const tasks = collector.tasks.slice(settledTasks)
1106 settledTasks = collector.tasks.length
1107 const outcomes = await Promise.allSettled(
1108 tasks.map(({ promise }) => promise),
1110 for (const [index, outcome] of outcomes.entries()) {
1111 if (outcome.status === `rejected`) {
1112 collector.failures.push({
1113 order: tasks[index]!.order,
1114 error: outcome.reason,
1115 })
1119 throwDemandCleanupFailures(
1120 collector,
1121 disposalSettlement.status === `rejected`
1122 ? [disposalSettlement.reason]
1123 : [],
1125 })().finally(() => {
1126 if (demandCleanupCollector === collector) {
1127 demandCleanupCollector = null
1129 })
1130 },
1131 loadSubset: trackLoadSubset,
1132 unloadSubset,
1135 },
1136 // Expose the getSyncMetadata function
1137 getSyncMetadata: undefined,
1139​
1140 const getKey = (record: OutputType) => asPowerSyncRecord(record).id
1141​
1142 // Keep adapter hooks and serialization options out of the core config.
1143 const {
1144 onLoad: _onLoad,
1145 onLoadSubset: _onLoadSubset,
1146 deserializationSchema: _deserializationSchema,
1147 serializer: _serializer,
1148 onDeserializationError: _onDeserializationError,
1149 ...collectionConfig
1150 } = {
1151 deserializationSchema,
1152 serializer,
1153 onDeserializationError,
1154 ...restConfig,
1156​
1157 const outputConfig: EnhancedPowerSyncCollectionConfig<
1158 TTable,
1159 OutputType,
1160 TSchema
1161 > = {
1162 ...collectionConfig,
1163 schema,
1164 getKey,
1165 // Syncing should start immediately since we need to monitor the changes for mutations
1166 startSync: true,
1167 syncMode,
1168 sync,
1169 onInsert: async (params) => {
1170 // The transaction here should only ever contain a single insert mutation
1171 return await transactor.applyTransaction(params.transaction)
1172 },
1173 onUpdate: async (params) => {
1174 // The transaction here should only ever contain a single update mutation
1175 return await transactor.applyTransaction(params.transaction)
1176 },
1177 onDelete: async (params) => {
1178 // The transaction here should only ever contain a single delete mutation
1179 return await transactor.applyTransaction(params.transaction)
1180 },
1181 utils: {
1182 getMeta: () => ({
1183 tableName: viewName,
1184 trackedTableName,
1185 metadataIsTracked,
1186 serializeValue: (value) =>
1187 serializeForSQLite<TTable>(value, table, serializer),
1188 }),
1189 },
1191 return outputConfig