Open History to see the last commit
1import { DiffTriggerOperation, sanitizeSQL } from '@powersync/common'
2import { or } from '@tanstack/db'
3import { compileSQLite } from './sqlite-compiler'
4import { PendingOperationStore } from './PendingOperationStore'
5import { PowerSyncTransactor } from './PowerSyncTransactor'
6import { DEFAULT_BATCH_SIZE } from './definitions'
7import { asPowerSyncRecord, mapOperation } from './helpers'
8import { convertTableToSchema } from './schema'
9import { serializeForSQLite } from './serialization'
10import type {
11 CleanupFn,
12 LoadSubsetOptions,
13 OperationType,
14 SyncConfig,
15} from '@tanstack/db'
16import type {
17 AnyTableColumnType,
18 ExtractedTable,
19 ExtractedTableColumns,
20 MapBaseColumnType,
21 OptionalExtractedTable,
22} from './helpers'
23import type {
24 BasePowerSyncCollectionConfig,
25 ConfigWithArbitraryCollectionTypes,
26 ConfigWithSQLiteInputType,
27 ConfigWithSQLiteTypes,
28 CustomSQLiteSerializer,
29 EnhancedPowerSyncCollectionConfig,
30 InferPowerSyncOutputType,
31 PowerSyncCollectionConfig,
32 PowerSyncCollectionUtils,
33} from './definitions'
34import type { PendingOperation } from './PendingOperationStore'
35import type { StandardSchemaV1 } from '@standard-schema/spec'
36import type { LockContext, Table, TriggerDiffRecord } from '@powersync/common'
37
38/**
39 * Creates PowerSync collection options for use with a standard Collection.
40 *
41 * @template TTable - The SQLite-based typing
42 * @template TSchema - The validation schema type (optionally supports a custom input type)
43 * @param config - Configuration options for the PowerSync collection
44 * @returns Collection options with utilities
45 */
46
47// Overload 1: No schema is provided
48
49/**
50 * Creates a PowerSync collection configuration with basic default validation.
51 * Input and Output types are the SQLite column types.
52 *
53 * @example
54 * ```typescript
55 * const APP_SCHEMA = new Schema({
56 * documents: new Table({
57 * name: column.text,
58 * }),
59 * })
60 *
61 * type Document = (typeof APP_SCHEMA)["types"]["documents"]
62 *
63 * const db = new PowerSyncDatabase({
64 * database: {
65 * dbFilename: "test.sqlite",
66 * },
67 * schema: APP_SCHEMA,
68 * })
69 *
70 * const collection = createCollection(
71 * powerSyncCollectionOptions({
72 * database: db,
73 * table: APP_SCHEMA.props.documents
74 * })
75 * )
76 * ```
77 */
78export function powerSyncCollectionOptions<TTable extends Table = Table>(
79 config: BasePowerSyncCollectionConfig<TTable, never> & ConfigWithSQLiteTypes,
80): EnhancedPowerSyncCollectionConfig<
81 TTable,
82 OptionalExtractedTable<TTable>,
83 never
84>
85
86// Overload 2: Schema is provided and the TInput matches SQLite types.
87
88/**
89 * Creates a PowerSync collection configuration with schema validation.
90 *
91 * The input types satisfy the SQLite column types.
92 *
93 * The output types are defined by the provided schema. This schema can enforce additional
94 * validation or type transforms.
95 * Arbitrary output typed mutations are encoded to SQLite for persistence. We provide a basic standard
96 * serialization implementation to serialize column values. Custom or advanced types require providing additional
97 * serializer specifications. Partial column overrides can be supplied to `serializer`.
98 *
99 * @example
100 * ```typescript
101 * import { z } from "zod"
102 *
103 * // The PowerSync SQLite schema
104 * const APP_SCHEMA = new Schema({
105 * documents: new Table({
106 * name: column.text,
107 * // Dates are stored as ISO date strings in SQLite
108 * created_at: column.text
109 * }),
110 * })
111 *
112 * // Advanced Zod validations. The output type of this schema
113 * // is constrained to the SQLite schema of APP_SCHEMA
114 * const schema = z.object({
115 * id: z.string(),
116 * // Notice that `name` is not nullable (is required) here and it has additional validation
117 * name: z.string().min(3, { message: "Should be at least 3 characters" }).nullable(),
118 * // The input type is still the SQLite string type. While collections will output smart Date instances.
119 * created_at: z.string().transform(val => new Date(val))
120 * })
121 *
122 * const collection = createCollection(
123 * powerSyncCollectionOptions({
124 * database: db,
125 * table: APP_SCHEMA.props.documents,
126 * schema,
127 * serializer: {
128 * // The default is toISOString, this is just to demonstrate custom overrides
129 * created_at: (outputValue) => outputValue.toISOString(),
130 * },
131 * })
132 * )
133 * ```
134 */
135export function powerSyncCollectionOptions<
136 TTable extends Table,
137 TSchema extends StandardSchemaV1<
138 // TInput is the SQLite types. We can use the supplied schema to validate sync input
139 OptionalExtractedTable<TTable>,
140 AnyTableColumnType<TTable>
141 >,
142>(
143 config: BasePowerSyncCollectionConfig<TTable, TSchema> &
144 ConfigWithSQLiteInputType<TTable, TSchema>,
145): EnhancedPowerSyncCollectionConfig<
146 TTable,
147 InferPowerSyncOutputType<TTable, TSchema>,
148 TSchema
149> & {
150 schema: TSchema
151}
152
153// Overload 3: Schema is provided with arbitrary TInput and TOutput
154/**
155 * Creates a PowerSync collection configuration with schema validation.
156 *
157 * The input types are not linked to the internal SQLite table types. This can
158 * give greater flexibility, e.g. by accepting rich types as input for `insert` or `update` operations.
159 * An additional `deserializationSchema` is required in order to process incoming SQLite updates to the output type.
160 *
161 * The output types are defined by the provided schema. This schema can enforce additional
162 * validation or type transforms.
163 * Arbitrary output typed mutations are encoded to SQLite for persistence. We provide a basic standard
164 * serialization implementation to serialize column values. Custom or advanced types require providing additional
165 * serializer specifications. Partial column overrides can be supplied to `serializer`.
166 *
167 * @example
168 * ```typescript
169 * import { z } from "zod"
170 *
171 * // The PowerSync SQLite schema
172 * const APP_SCHEMA = new Schema({
173 * documents: new Table({
174 * name: column.text,
175 * // Booleans are represented as integers in SQLite
176 * is_active: column.integer
177 * }),
178 * })
179 *
180 * // Advanced Zod validations.
181 * // We accept boolean values as input for operations and expose Booleans in query results
182 * const schema = z.object({
183 * id: z.string(),
184 * isActive: z.boolean(), // TInput and TOutput are boolean
185 * })
186 *
187 * // The deserializationSchema converts the SQLite synced INTEGER (0/1) values to booleans.
188 * const deserializationSchema = z.object({
189 * id: z.string(),
190 * isActive: z.number().nullable().transform((val) => val == null ? true : val > 0),
191 * })
192 *
193 * const collection = createCollection(
194 * powerSyncCollectionOptions({
195 * database: db,
196 * table: APP_SCHEMA.props.documents,
197 * schema,
198 * deserializationSchema,
199 * })
200 * )
201 * ```
202 */
203export function powerSyncCollectionOptions<
204 TTable extends Table,
205 TSchema extends StandardSchemaV1<
206 // The input and output must have the same keys, the value types can be arbitrary
207 AnyTableColumnType<TTable>,
208 AnyTableColumnType<TTable>
209 >,
210>(
211 config: BasePowerSyncCollectionConfig<TTable, TSchema> &
212 ConfigWithArbitraryCollectionTypes<TTable, TSchema>,
213): EnhancedPowerSyncCollectionConfig<
214 TTable,
215 InferPowerSyncOutputType<TTable, TSchema>,
216 TSchema
217> & {
218 utils: PowerSyncCollectionUtils<TTable>
219 schema: TSchema
220}
221
222/**
223 * Implementation of powerSyncCollectionOptions that handles both schema and non-schema configurations.
224 */
225
226export function powerSyncCollectionOptions<
227 TTable extends Table,
228 TSchema extends StandardSchemaV1<any> = never,
229>(config: PowerSyncCollectionConfig<TTable, TSchema>) {
230 const {
231 database,
232 table,
233 schema: inputSchema,
234 syncBatchSize = DEFAULT_BATCH_SIZE,
235 syncMode = 'eager',
236 ...restConfig
237 } = config
238
239 const deserializationSchema =
240 `deserializationSchema` in config ? config.deserializationSchema : null
241 const serializer = `serializer` in config ? config.serializer : undefined
242 const onDeserializationError =
243 `onDeserializationError` in config
244 ? config.onDeserializationError
245 : undefined
246
247 // The SQLite table type
248 type TableType = ExtractedTable<TTable>
249
250 // The collection output type
251 type OutputType = InferPowerSyncOutputType<TTable, TSchema>
252
253 const { viewName, trackMetadata: metadataIsTracked } = table
254
255 /**
256 * Deserializes data from the incoming sync stream
257 */
258 const deserializeSyncRow = (value: TableType): OutputType => {
259 const validationSchema = deserializationSchema || schema
260 const validation = validationSchema[`~standard`].validate(value)
261 if (`value` in validation) {
262 return validation.value
263 } else if (`issues` in validation) {
264 const issueMessage = `Failed to validate incoming data for ${viewName}. Issues: ${validation.issues.map((issue) => `${issue.path} - ${issue.message}`)}`
265 database.logger.error(issueMessage)
266 onDeserializationError!(validation)
267 throw new Error(issueMessage)
268 } else {
269 const unknownErrorMessage = `Unknown deserialization error for ${viewName}`
270 database.logger.error(unknownErrorMessage)
271 onDeserializationError!({ issues: [{ message: unknownErrorMessage }] })
272 throw new Error(unknownErrorMessage)
273 }
274 }
275
276 // We can do basic runtime validations for columns if not explicit schema has been provided
277 const schema = inputSchema ?? (convertTableToSchema(table) as TSchema)
278 /**
279 * The onInsert, onUpdate, and onDelete handlers should only return
280 * after we have written the changes to TanStack DB.
281 * We currently only write to TanStack DB from a diff trigger.
282 * We wait for the diff trigger to observe the change,
283 * and only then return from the on[X] handlers.
284 * This ensures that when the transaction is reported as
285 * complete to the caller, the in-memory state is already
286 * consistent with the database.
287 */
288 const pendingOperationStore = PendingOperationStore.GLOBAL
289 // Keep the tracked table unique in case of multiple tabs.
290 const trackedTableName = `__${viewName}_tracking_${Math.floor(
291 Math.random() * 0xffffffff,
292 )
293 .toString(16)
294 .padStart(8, `0`)}`
295
296 const transactor = new PowerSyncTransactor({
297 database,
298 })
299
300 /**
301 * "sync"
302 * Notice that this describes the Sync between the local SQLite table
303 * and the in-memory tanstack-db collection.
304 */
305 const sync: SyncConfig<OutputType, string> = {
306 sync: (params) => {
307 const { begin, write, collection, commit, markReady } = params
308 const abortController = new AbortController()
309
310 let disposeTracking:
311 | ((options?: { context?: LockContext }) => Promise<void>)
312 | null = null
313
314 if (syncMode === `eager`) {
315 return runEagerSync()
316 } else {
317 return runOnDemandSync()
318 }
319
320 /**
321 * Disposes the current diff trigger, if one is active, and clears the
322 * tracking state.
323 */
324 async function safelyDisposeTracking(
325 context?: LockContext,
326 ): Promise<void> {
327 const dispose = disposeTracking
328 if (!dispose) {
329 return
330 }
331
332 disposeTracking = null
333 await dispose(context ? { context } : undefined)
334 }
335
336 async function createDiffTrigger(options: {
337 setupContext?: LockContext
338 when: Record<DiffTriggerOperation, string>
339 writeType: (rowId: string) => OperationType
340 batchQuery: (
341 lockContext: LockContext,
342 batchSize: number,
343 cursor: number,
344 ) => Promise<Array<TableType>>
345 onReady: () => void
346 }) {
347 const { setupContext, when, writeType, batchQuery, onReady } = options
348
349 return await database.triggers.createDiffTrigger({
350 source: viewName,
351 destination: trackedTableName,
352 setupContext,
353 when,
354 hooks: {
355 beforeCreate: async (context) => {
356 let currentBatchCount = syncBatchSize
357 let cursor = 0
358 while (currentBatchCount == syncBatchSize) {
359 begin()
360
361 const batchItems = await batchQuery(
362 context,
363 syncBatchSize,
364 cursor,
365 )
366 currentBatchCount = batchItems.length
367 cursor += currentBatchCount
368 for (const row of batchItems) {
369 write({
370 type: writeType(row.id),
371 value: deserializeSyncRow(row),
372 })
373 }
374 commit()
375 }
376 onReady()
377 database.logger.info(
378 `Sync is ready for ${viewName} into ${trackedTableName}`,
379 )
380 },
381 },
382 })
383 }
384
385 async function flushDiffRecords(): Promise<void> {
386 await database
387 .writeTransaction(async (context) => {
388 await flushDiffRecordsWithContext(context)
389 })
390 .catch((error) => {
391 database.logger.error(
392 `An error has been detected in the sync handler`,
393 error,
394 )
395 })
396 }
397
398 // We can use this directly if we want to pair a flush with dispose+recreate diff trigger.
399 async function flushDiffRecordsWithContext(
400 context: LockContext,
401 ): Promise<void> {
402 // There is nothing to flush if no tracking table is currently active.
403 if (!disposeTracking) {
404 return
405 }
406
407 try {
408 begin()
409 const operations = await context.getAll<TriggerDiffRecord>(
410 `SELECT * FROM ${trackedTableName} ORDER BY operation_id ASC`,
411 )
412 const pendingOperations: Array<PendingOperation> = []
413
414 for (const op of operations) {
415 const { id, operation, timestamp, value } = op
416 const parsedValue = deserializeSyncRow({
417 id,
418 ...JSON.parse(value),
419 })
420 const parsedPreviousValue =
421 op.operation == DiffTriggerOperation.UPDATE
422 ? deserializeSyncRow({
423 id,
424 ...JSON.parse(op.previous_value),
425 })
426 : undefined
427 write({
428 type: mapOperation(operation),
429 value: parsedValue,
430 previousValue: parsedPreviousValue,
431 })
432 pendingOperations.push({
433 id,
434 operation,
435 timestamp,
436 tableName: viewName,
437 })
438 }
439
440 // clear the current operations
441 await context.execute(`DELETE FROM ${trackedTableName}`)
442
443 commit()
444 pendingOperationStore.resolvePendingFor(pendingOperations)
445 } catch (error) {
446 database.logger.error(
447 `An error has been detected in the sync handler`,
448 error,
449 )
450 }
451 }
452
453 // The sync function needs to be synchronous.
454 async function start(afterOnChangeRegistered?: () => Promise<void>) {
455 database.logger.info(
456 `Sync is starting for ${viewName} into ${trackedTableName}`,
457 )
458 database.onChangeWithCallback(
459 {
460 onChange: async () => {
461 await flushDiffRecords()
462 },
463 },
464 {
465 signal: abortController.signal,
466 triggerImmediate: false,
467 tables: [trackedTableName],
468 },
469 )
470
471 await afterOnChangeRegistered?.()
472
473 // If the abort controller was aborted while processing the request above
474 if (abortController.signal.aborted) {
475 await safelyDisposeTracking()
476 } else {
477 abortController.signal.addEventListener(
478 `abort`,
479 async () => {
480 await safelyDisposeTracking()
481 },
482 { once: true },
483 )
484 }
485 }
486
487 // Eager mode.
488 // Registers a diff trigger for the entire table.
489 function runEagerSync() {
490 let onUnload: CleanupFn | void | null = null
491
492 start(async () => {
493 onUnload = await restConfig.onLoad?.()
494
495 disposeTracking = await createDiffTrigger({
496 when: {
497 [DiffTriggerOperation.INSERT]: `TRUE`,
498 [DiffTriggerOperation.UPDATE]: `TRUE`,
499 [DiffTriggerOperation.DELETE]: `TRUE`,
500 },
501 writeType: (_rowId: string) => `insert`,
502 batchQuery: (
503 lockContext: LockContext,
504 batchSize: number,
505 cursor: number,
506 ) =>
507 lockContext.getAll<TableType>(
508 sanitizeSQL`SELECT * FROM ${viewName} LIMIT ? OFFSET ?`,
509 [batchSize, cursor],
510 ),
511 onReady: () => markReady(),
512 })
513 }).catch((error) =>
514 database.logger.error(
515 `Could not start syncing process for ${viewName} into ${trackedTableName}`,
516 error,
517 ),
518 )
519
520 return () => {
521 database.logger.info(
522 `Sync has been stopped for ${viewName} into ${trackedTableName}`,
523 )
524 abortController.abort()
525 onUnload?.()
526 }
527 }
528
529 // On-demand mode.
530 // Registers a diff trigger for the active WHERE expressions.
531 function runOnDemandSync() {
532 let onUnloadSubset: CleanupFn | void | null = null
533
534 start().catch((error) =>
535 database.logger.error(
536 `Could not start syncing process for ${viewName} into ${trackedTableName}`,
537 error,
538 ),
539 )
540
541 // Tracks all active WHERE expressions for on-demand sync filtering.
542 // Each loadSubset call pushes its predicate; unloadSubset removes it.
543 const activeWhereExpressions: Array<LoadSubsetOptions['where']> = []
544
545 const loadSubset = async (
546 options?: LoadSubsetOptions,
547 ): Promise<void> => {
548 if (options) {
549 activeWhereExpressions.push(options.where)
550 onUnloadSubset = await restConfig.onLoadSubset?.(options)
551 }
552
553 // No predicates remain, so stop tracking entirely. Both calls are no-ops
554 // when no tracking table is currently active.
555 if (activeWhereExpressions.length === 0) {
556 await database.writeLock(async (ctx) => {
557 await flushDiffRecordsWithContext(ctx)
558 await safelyDisposeTracking(ctx)
559 })
560 return
561 }
562
563 const combinedWhere =
564 activeWhereExpressions.length === 1
565 ? activeWhereExpressions[0]
566 : or(
567 activeWhereExpressions[0],
568 activeWhereExpressions[1],
569 ...activeWhereExpressions.slice(2),
570 )
571
572 const compiledNewData = compileSQLite(
573 { where: combinedWhere },
574 { jsonColumn: 'NEW.data' },
575 )
576
577 const compiledOldData = compileSQLite(
578 { where: combinedWhere },
579 { jsonColumn: 'OLD.data' },
580 )
581
582 const compiledView = compileSQLite({ where: combinedWhere })
583
584 const newDataWhenClause = toInlinedWhereClause(compiledNewData)
585 const oldDataWhenClause = toInlinedWhereClause(compiledOldData)
586 const viewWhereClause = toInlinedWhereClause(compiledView)
587
588 await database.writeLock(async (ctx) => {
589 // Replace any active tracking with one covering the new set of
590 // predicates.
591 await flushDiffRecordsWithContext(ctx)
592 await safelyDisposeTracking(ctx)
593
594 disposeTracking = await createDiffTrigger({
595 setupContext: ctx,
596 when: {
597 [DiffTriggerOperation.INSERT]: newDataWhenClause,
598 [DiffTriggerOperation.UPDATE]: `(${newDataWhenClause}) OR (${oldDataWhenClause})`,
599 [DiffTriggerOperation.DELETE]: oldDataWhenClause,
600 },
601 writeType: (rowId: string) =>
602 collection.has(rowId) ? `update` : `insert`,
603 batchQuery: (
604 lockContext: LockContext,
605 batchSize: number,
606 cursor: number,
607 ) =>
608 lockContext.getAll<TableType>(
609 `SELECT * FROM ${viewName} WHERE ${viewWhereClause} LIMIT ? OFFSET ?`,
610 [batchSize, cursor],
611 ),
612 onReady: () => {},
613 })
614 })
615 }
616
617 const toInlinedWhereClause = (compiled: {
618 where?: string
619 params: Array<unknown>
620 }): string => {
621 if (!compiled.where) return 'TRUE'
622 const sqlParts = compiled.where.split('?')
623 return sanitizeSQL(
624 sqlParts as unknown as TemplateStringsArray,
625 ...compiled.params,
626 )
627 }
628
629 const unloadSubset = async (options: LoadSubsetOptions) => {
630 onUnloadSubset?.()
631
632 const idx = activeWhereExpressions.indexOf(options.where)
633 if (idx !== -1) {
634 activeWhereExpressions.splice(idx, 1)
635 }
636
637 // Evict rows that were exclusively loaded by the departing predicate.
638 // These are rows matching the departing WHERE that are no longer covered
639 // by any remaining active predicate.
640 const compiledDeparting = compileSQLite({ where: options.where })
641 const departingWhereSQL = toInlinedWhereClause(compiledDeparting)
642
643 let evictionSQL: string
644 if (activeWhereExpressions.length === 0) {
645 evictionSQL = `SELECT id FROM ${viewName} WHERE ${departingWhereSQL}`
646 } else {
647 const combinedRemaining =
648 activeWhereExpressions.length === 1
649 ? activeWhereExpressions[0]!
650 : or(
651 activeWhereExpressions[0],
652 activeWhereExpressions[1],
653 ...activeWhereExpressions.slice(2),
654 )
655 const compiledRemaining = compileSQLite({
656 where: combinedRemaining,
657 })
658 const remainingWhereSQL = toInlinedWhereClause(compiledRemaining)
659 evictionSQL = `SELECT id FROM ${viewName} WHERE (${departingWhereSQL}) AND NOT (${remainingWhereSQL})`
660 }
661
662 const rowsToEvict = await database.getAll<{ id: string }>(evictionSQL)
663 if (rowsToEvict.length > 0) {
664 begin()
665 for (const { id } of rowsToEvict) {
666 write({ type: `delete`, key: id })
667 }
668 commit()
669 }
670
671 // Recreate the diff trigger for the remaining active WHERE expressions.
672 await loadSubset()
673 }
674
675 markReady()
676
677 return {
678 cleanup: () => {
679 database.logger.info(
680 `Sync has been stopped for ${viewName} into ${trackedTableName}`,
681 )
682 abortController.abort()
683 },
684 loadSubset: (options: LoadSubsetOptions) => loadSubset(options),
685 unloadSubset: (options: LoadSubsetOptions) => unloadSubset(options),
686 }
687 }
688 },
689 // Expose the getSyncMetadata function
690 getSyncMetadata: undefined,
691 }
692
693 const getKey = (record: OutputType) => asPowerSyncRecord(record).id
694
695 const outputConfig: EnhancedPowerSyncCollectionConfig<
696 TTable,
697 OutputType,
698 TSchema
699 > = {
700 ...restConfig,
701 schema,
702 getKey,
703 // Syncing should start immediately since we need to monitor the changes for mutations
704 startSync: true,
705 syncMode,
706 sync,
707 onInsert: async (params) => {
708 // The transaction here should only ever contain a single insert mutation
709 return await transactor.applyTransaction(params.transaction)
710 },
711 onUpdate: async (params) => {
712 // The transaction here should only ever contain a single update mutation
713 return await transactor.applyTransaction(params.transaction)
714 },
715 onDelete: async (params) => {
716 // The transaction here should only ever contain a single delete mutation
717 return await transactor.applyTransaction(params.transaction)
718 },
719 utils: {
720 getMeta: () => ({
721 tableName: viewName,
722 trackedTableName,
723 metadataIsTracked,
724 serializeValue: (value) =>
725 serializeForSQLite(
726 value,
727 // This is required by the input generic
728 table as Table<
729 MapBaseColumnType<InferPowerSyncOutputType<TTable, TSchema>>
730 >,
731 // Coerce serializer to the shape that corresponds to the Table constructed from OutputType
732 serializer as CustomSQLiteSerializer<
733 OutputType,
734 ExtractedTableColumns<Table<MapBaseColumnType<OutputType>>>
735 >,
736 ),
737 }),
738 },
739 }
740 return outputConfig
741}