Open History to see the last commit
1import { DiffTriggerOperation, sanitizeSQL } from '@powersync/common'
2import { or } from '@tanstack/db'
3import { compileSQLite } from './sqlite-compiler'
4import { PendingOperationStore } from './PendingOperationStore'
5import { PowerSyncTransactor } from './PowerSyncTransactor'
6import { DEFAULT_BATCH_SIZE } from './definitions'
7import { asPowerSyncRecord, mapOperation } from './helpers'
8import { convertTableToSchema } from './schema'
9import { serializeForSQLite } from './serialization'
10import type {
11 CleanupFn,
12 LoadSubsetOptions,
13 OperationType,
14 SyncConfig,
15} from '@tanstack/db'
16import type {
17 AnyTableColumnType,
18 ExtractedTable,
19 ExtractedTableColumns,
20 MapBaseColumnType,
21 OptionalExtractedTable,
22} from './helpers'
23import type {
24 BasePowerSyncCollectionConfig,
25 ConfigWithArbitraryCollectionTypes,
26 ConfigWithSQLiteInputType,
27 ConfigWithSQLiteTypes,
28 CustomSQLiteSerializer,
29 EnhancedPowerSyncCollectionConfig,
30 InferPowerSyncOutputType,
31 PowerSyncCollectionConfig,
32 PowerSyncCollectionUtils,
33} from './definitions'
34import type { PendingOperation } from './PendingOperationStore'
35import type { StandardSchemaV1 } from '@standard-schema/spec'
36import type { LockContext, Table, TriggerDiffRecord } from '@powersync/common'
37
38/**
39 * Creates PowerSync collection options for use with a standard Collection.
40 *
41 * @template TTable - The SQLite-based typing
42 * @template TSchema - The validation schema type (optionally supports a custom input type)
43 * @param config - Configuration options for the PowerSync collection
44 * @returns Collection options with utilities
45 */
46
47// Overload 1: No schema is provided
48
49/**
50 * Creates a PowerSync collection configuration with basic default validation.
51 * Input and Output types are the SQLite column types.
52 *
53 * @example
54 * ```typescript
55 * const APP_SCHEMA = new Schema({
56 * documents: new Table({
57 * name: column.text,
58 * }),
59 * })
60 *
61 * type Document = (typeof APP_SCHEMA)["types"]["documents"]
62 *
63 * const db = new PowerSyncDatabase({
64 * database: {
65 * dbFilename: "test.sqlite",
66 * },
67 * schema: APP_SCHEMA,
68 * })
69 *
70 * const collection = createCollection(
71 * powerSyncCollectionOptions({
72 * database: db,
73 * table: APP_SCHEMA.props.documents
74 * })
75 * )
76 * ```
77 */
78export function powerSyncCollectionOptions<TTable extends Table = Table>(
79 config: BasePowerSyncCollectionConfig<TTable, never> & ConfigWithSQLiteTypes,
80): EnhancedPowerSyncCollectionConfig<
81 TTable,
82 OptionalExtractedTable<TTable>,
83 never
84>
85
86// Overload 2: Schema is provided and the TInput matches SQLite types.
87
88/**
89 * Creates a PowerSync collection configuration with schema validation.
90 *
91 * The input types satisfy the SQLite column types.
92 *
93 * The output types are defined by the provided schema. This schema can enforce additional
94 * validation or type transforms.
95 * Arbitrary output typed mutations are encoded to SQLite for persistence. We provide a basic standard
96 * serialization implementation to serialize column values. Custom or advanced types require providing additional
97 * serializer specifications. Partial column overrides can be supplied to `serializer`.
98 *
99 * @example
100 * ```typescript
101 * import { z } from "zod"
102 *
103 * // The PowerSync SQLite schema
104 * const APP_SCHEMA = new Schema({
105 * documents: new Table({
106 * name: column.text,
107 * // Dates are stored as ISO date strings in SQLite
108 * created_at: column.text
109 * }),
110 * })
111 *
112 * // Advanced Zod validations. The output type of this schema
113 * // is constrained to the SQLite schema of APP_SCHEMA
114 * const schema = z.object({
115 * id: z.string(),
116 * // Notice that `name` is not nullable (is required) here and it has additional validation
117 * name: z.string().min(3, { message: "Should be at least 3 characters" }).nullable(),
118 * // The input type is still the SQLite string type. While collections will output smart Date instances.
119 * created_at: z.string().transform(val => new Date(val))
120 * })
121 *
122 * const collection = createCollection(
123 * powerSyncCollectionOptions({
124 * database: db,
125 * table: APP_SCHEMA.props.documents,
126 * schema,
127 * serializer: {
128 * // The default is toISOString, this is just to demonstrate custom overrides
129 * created_at: (outputValue) => outputValue.toISOString(),
130 * },
131 * })
132 * )
133 * ```
134 */
135export function powerSyncCollectionOptions<
136 TTable extends Table,
137 TSchema extends StandardSchemaV1<
138 // TInput is the SQLite types. We can use the supplied schema to validate sync input
139 OptionalExtractedTable<TTable>,
140 AnyTableColumnType<TTable>
141 >,
142>(
143 config: BasePowerSyncCollectionConfig<TTable, TSchema> &
144 ConfigWithSQLiteInputType<TTable, TSchema>,
145): EnhancedPowerSyncCollectionConfig<
146 TTable,
147 InferPowerSyncOutputType<TTable, TSchema>,
148 TSchema
149> & {
150 schema: TSchema
151}
152
153// Overload 3: Schema is provided with arbitrary TInput and TOutput
154/**
155 * Creates a PowerSync collection configuration with schema validation.
156 *
157 * The input types are not linked to the internal SQLite table types. This can
158 * give greater flexibility, e.g. by accepting rich types as input for `insert` or `update` operations.
159 * An additional `deserializationSchema` is required in order to process incoming SQLite updates to the output type.
160 *
161 * The output types are defined by the provided schema. This schema can enforce additional
162 * validation or type transforms.
163 * Arbitrary output typed mutations are encoded to SQLite for persistence. We provide a basic standard
164 * serialization implementation to serialize column values. Custom or advanced types require providing additional
165 * serializer specifications. Partial column overrides can be supplied to `serializer`.
166 *
167 * @example
168 * ```typescript
169 * import { z } from "zod"
170 *
171 * // The PowerSync SQLite schema
172 * const APP_SCHEMA = new Schema({
173 * documents: new Table({
174 * name: column.text,
175 * // Booleans are represented as integers in SQLite
176 * is_active: column.integer
177 * }),
178 * })
179 *
180 * // Advanced Zod validations.
181 * // We accept boolean values as input for operations and expose Booleans in query results
182 * const schema = z.object({
183 * id: z.string(),
184 * isActive: z.boolean(), // TInput and TOutput are boolean
185 * })
186 *
187 * // The deserializationSchema converts the SQLite synced INTEGER (0/1) values to booleans.
188 * const deserializationSchema = z.object({
189 * id: z.string(),
190 * isActive: z.number().nullable().transform((val) => val == null ? true : val > 0),
191 * })
192 *
193 * const collection = createCollection(
194 * powerSyncCollectionOptions({
195 * database: db,
196 * table: APP_SCHEMA.props.documents,
197 * schema,
198 * deserializationSchema,
199 * })
200 * )
201 * ```
202 */
203export function powerSyncCollectionOptions<
204 TTable extends Table,
205 TSchema extends StandardSchemaV1<
206 // The input and output must have the same keys, the value types can be arbitrary
207 AnyTableColumnType<TTable>,
208 AnyTableColumnType<TTable>
209 >,
210>(
211 config: BasePowerSyncCollectionConfig<TTable, TSchema> &
212 ConfigWithArbitraryCollectionTypes<TTable, TSchema>,
213): EnhancedPowerSyncCollectionConfig<
214 TTable,
215 InferPowerSyncOutputType<TTable, TSchema>,
216 TSchema
217> & {
218 utils: PowerSyncCollectionUtils<TTable>
219 schema: TSchema
220}
221
222/**
223 * Implementation of powerSyncCollectionOptions that handles both schema and non-schema configurations.
224 */
225
226export function powerSyncCollectionOptions<
227 TTable extends Table,
228 TSchema extends StandardSchemaV1<any> = never,
229>(config: PowerSyncCollectionConfig<TTable, TSchema>) {
230 const {
231 database,
232 table,
233 schema: inputSchema,
234 syncBatchSize = DEFAULT_BATCH_SIZE,
235 syncMode = 'eager',
236 ...restConfig
237 } = config
238
239 const deserializationSchema =
240 `deserializationSchema` in config ? config.deserializationSchema : null
241 const serializer = `serializer` in config ? config.serializer : undefined
242 const onDeserializationError =
243 `onDeserializationError` in config
244 ? config.onDeserializationError
245 : undefined
246
247 // The SQLite table type
248 type TableType = ExtractedTable<TTable>
249
250 // The collection output type
251 type OutputType = InferPowerSyncOutputType<TTable, TSchema>
252
253 const { viewName, trackMetadata: metadataIsTracked } = table
254
255 /**
256 * Deserializes data from the incoming sync stream
257 */
258 const deserializeSyncRow = (value: TableType): OutputType => {
259 const validationSchema = deserializationSchema || schema
260 const validation = validationSchema[`~standard`].validate(value)
261 if (`value` in validation) {
262 return validation.value
263 } else if (`issues` in validation) {
264 const issueMessage = `Failed to validate incoming data for ${viewName}. Issues: ${validation.issues.map((issue) => `${issue.path} - ${issue.message}`)}`
265 database.logger.error(issueMessage)
266 onDeserializationError!(validation)
267 throw new Error(issueMessage)
268 } else {
269 const unknownErrorMessage = `Unknown deserialization error for ${viewName}`
270 database.logger.error(unknownErrorMessage)
271 onDeserializationError!({ issues: [{ message: unknownErrorMessage }] })
272 throw new Error(unknownErrorMessage)
273 }
274 }
275
276 // We can do basic runtime validations for columns if not explicit schema has been provided
277 const schema = inputSchema ?? (convertTableToSchema(table) as TSchema)
278 /**
279 * The onInsert, onUpdate, and onDelete handlers should only return
280 * after we have written the changes to TanStack DB.
281 * We currently only write to TanStack DB from a diff trigger.
282 * We wait for the diff trigger to observe the change,
283 * and only then return from the on[X] handlers.
284 * This ensures that when the transaction is reported as
285 * complete to the caller, the in-memory state is already
286 * consistent with the database.
287 */
288 const pendingOperationStore = PendingOperationStore.GLOBAL
289 // Keep the tracked table unique in case of multiple tabs.
290 const trackedTableName = `__${viewName}_tracking_${Math.floor(
291 Math.random() * 0xffffffff,
292 )
293 .toString(16)
294 .padStart(8, `0`)}`
295
296 const transactor = new PowerSyncTransactor({
297 database,
298 })
299
300 /**
301 * "sync"
302 * Notice that this describes the Sync between the local SQLite table
303 * and the in-memory tanstack-db collection.
304 */
305 const sync: SyncConfig<OutputType, string> = {
306 sync: (params) => {
307 const { begin, write, collection, commit, markReady } = params
308 const abortController = new AbortController()
309
310 let disposeTracking:
311 | ((options?: { context?: LockContext }) => Promise<void>)
312 | null = null
313
314 if (syncMode === `eager`) {
315 return runEagerSync()
316 } else {
317 return runOnDemandSync()
318 }
319
320 async function createDiffTrigger(options: {
321 setupContext?: LockContext
322 when: Record<DiffTriggerOperation, string>
323 writeType: (rowId: string) => OperationType
324 batchQuery: (
325 lockContext: LockContext,
326 batchSize: number,
327 cursor: number,
328 ) => Promise<Array<TableType>>
329 onReady: () => void
330 }) {
331 const { setupContext, when, writeType, batchQuery, onReady } = options
332
333 return await database.triggers.createDiffTrigger({
334 source: viewName,
335 destination: trackedTableName,
336 setupContext,
337 when,
338 hooks: {
339 beforeCreate: async (context) => {
340 let currentBatchCount = syncBatchSize
341 let cursor = 0
342 while (currentBatchCount == syncBatchSize) {
343 begin()
344
345 const batchItems = await batchQuery(
346 context,
347 syncBatchSize,
348 cursor,
349 )
350 currentBatchCount = batchItems.length
351 cursor += currentBatchCount
352 for (const row of batchItems) {
353 write({
354 type: writeType(row.id),
355 value: deserializeSyncRow(row),
356 })
357 }
358 commit()
359 }
360 onReady()
361 database.logger.info(
362 `Sync is ready for ${viewName} into ${trackedTableName}`,
363 )
364 },
365 },
366 })
367 }
368
369 async function flushDiffRecords(): Promise<void> {
370 await database
371 .writeTransaction(async (context) => {
372 await flushDiffRecordsWithContext(context)
373 })
374 .catch((error) => {
375 database.logger.error(
376 `An error has been detected in the sync handler`,
377 error,
378 )
379 })
380 }
381
382 // We can use this directly if we want to pair a flush with dispose+recreate diff trigger.
383 async function flushDiffRecordsWithContext(
384 context: LockContext,
385 ): Promise<void> {
386 try {
387 begin()
388 const operations = await context.getAll<TriggerDiffRecord>(
389 `SELECT * FROM ${trackedTableName} ORDER BY operation_id ASC`,
390 )
391 const pendingOperations: Array<PendingOperation> = []
392
393 for (const op of operations) {
394 const { id, operation, timestamp, value } = op
395 const parsedValue = deserializeSyncRow({
396 id,
397 ...JSON.parse(value),
398 })
399 const parsedPreviousValue =
400 op.operation == DiffTriggerOperation.UPDATE
401 ? deserializeSyncRow({
402 id,
403 ...JSON.parse(op.previous_value),
404 })
405 : undefined
406 write({
407 type: mapOperation(operation),
408 value: parsedValue,
409 previousValue: parsedPreviousValue,
410 })
411 pendingOperations.push({
412 id,
413 operation,
414 timestamp,
415 tableName: viewName,
416 })
417 }
418
419 // clear the current operations
420 await context.execute(`DELETE FROM ${trackedTableName}`)
421
422 commit()
423 pendingOperationStore.resolvePendingFor(pendingOperations)
424 } catch (error) {
425 database.logger.error(
426 `An error has been detected in the sync handler`,
427 error,
428 )
429 }
430 }
431
432 // The sync function needs to be synchronous.
433 async function start(afterOnChangeRegistered?: () => Promise<void>) {
434 database.logger.info(
435 `Sync is starting for ${viewName} into ${trackedTableName}`,
436 )
437 database.onChangeWithCallback(
438 {
439 onChange: async () => {
440 await flushDiffRecords()
441 },
442 },
443 {
444 signal: abortController.signal,
445 triggerImmediate: false,
446 tables: [trackedTableName],
447 },
448 )
449
450 await afterOnChangeRegistered?.()
451
452 // If the abort controller was aborted while processing the request above
453 if (abortController.signal.aborted) {
454 await disposeTracking?.()
455 } else {
456 abortController.signal.addEventListener(
457 `abort`,
458 async () => {
459 await disposeTracking?.()
460 },
461 { once: true },
462 )
463 }
464 }
465
466 // Eager mode.
467 // Registers a diff trigger for the entire table.
468 function runEagerSync() {
469 let onUnload: CleanupFn | void | null = null
470
471 start(async () => {
472 onUnload = await restConfig.onLoad?.()
473
474 disposeTracking = await createDiffTrigger({
475 when: {
476 [DiffTriggerOperation.INSERT]: `TRUE`,
477 [DiffTriggerOperation.UPDATE]: `TRUE`,
478 [DiffTriggerOperation.DELETE]: `TRUE`,
479 },
480 writeType: (_rowId: string) => `insert`,
481 batchQuery: (
482 lockContext: LockContext,
483 batchSize: number,
484 cursor: number,
485 ) =>
486 lockContext.getAll<TableType>(
487 sanitizeSQL`SELECT * FROM ${viewName} LIMIT ? OFFSET ?`,
488 [batchSize, cursor],
489 ),
490 onReady: () => markReady(),
491 })
492 }).catch((error) =>
493 database.logger.error(
494 `Could not start syncing process for ${viewName} into ${trackedTableName}`,
495 error,
496 ),
497 )
498
499 return () => {
500 database.logger.info(
501 `Sync has been stopped for ${viewName} into ${trackedTableName}`,
502 )
503 abortController.abort()
504 onUnload?.()
505 }
506 }
507
508 // On-demand mode.
509 // Registers a diff trigger for the active WHERE expressions.
510 function runOnDemandSync() {
511 let onUnloadSubset: CleanupFn | void | null = null
512
513 start().catch((error) =>
514 database.logger.error(
515 `Could not start syncing process for ${viewName} into ${trackedTableName}`,
516 error,
517 ),
518 )
519
520 // Tracks all active WHERE expressions for on-demand sync filtering.
521 // Each loadSubset call pushes its predicate; unloadSubset removes it.
522 const activeWhereExpressions: Array<LoadSubsetOptions['where']> = []
523
524 const loadSubset = async (
525 options?: LoadSubsetOptions,
526 ): Promise<void> => {
527 if (options) {
528 activeWhereExpressions.push(options.where)
529 onUnloadSubset = await restConfig.onLoadSubset?.(options)
530 }
531
532 if (activeWhereExpressions.length === 0) {
533 await database.writeLock(async (ctx) => {
534 await flushDiffRecordsWithContext(ctx)
535 await disposeTracking?.({ context: ctx })
536 })
537 return
538 }
539
540 const combinedWhere =
541 activeWhereExpressions.length === 1
542 ? activeWhereExpressions[0]
543 : or(
544 activeWhereExpressions[0],
545 activeWhereExpressions[1],
546 ...activeWhereExpressions.slice(2),
547 )
548
549 const compiledNewData = compileSQLite(
550 { where: combinedWhere },
551 { jsonColumn: 'NEW.data' },
552 )
553
554 const compiledOldData = compileSQLite(
555 { where: combinedWhere },
556 { jsonColumn: 'OLD.data' },
557 )
558
559 const compiledView = compileSQLite({ where: combinedWhere })
560
561 const newDataWhenClause = toInlinedWhereClause(compiledNewData)
562 const oldDataWhenClause = toInlinedWhereClause(compiledOldData)
563 const viewWhereClause = toInlinedWhereClause(compiledView)
564
565 await database.writeLock(async (ctx) => {
566 await flushDiffRecordsWithContext(ctx)
567 await disposeTracking?.({ context: ctx })
568
569 disposeTracking = await createDiffTrigger({
570 setupContext: ctx,
571 when: {
572 [DiffTriggerOperation.INSERT]: newDataWhenClause,
573 [DiffTriggerOperation.UPDATE]: `(${newDataWhenClause}) OR (${oldDataWhenClause})`,
574 [DiffTriggerOperation.DELETE]: oldDataWhenClause,
575 },
576 writeType: (rowId: string) =>
577 collection.has(rowId) ? `update` : `insert`,
578 batchQuery: (
579 lockContext: LockContext,
580 batchSize: number,
581 cursor: number,
582 ) =>
583 lockContext.getAll<TableType>(
584 `SELECT * FROM ${viewName} WHERE ${viewWhereClause} LIMIT ? OFFSET ?`,
585 [batchSize, cursor],
586 ),
587 onReady: () => {},
588 })
589 })
590 }
591
592 const toInlinedWhereClause = (compiled: {
593 where?: string
594 params: Array<unknown>
595 }): string => {
596 if (!compiled.where) return 'TRUE'
597 const sqlParts = compiled.where.split('?')
598 return sanitizeSQL(
599 sqlParts as unknown as TemplateStringsArray,
600 ...compiled.params,
601 )
602 }
603
604 const unloadSubset = async (options: LoadSubsetOptions) => {
605 onUnloadSubset?.()
606
607 const idx = activeWhereExpressions.indexOf(options.where)
608 if (idx !== -1) {
609 activeWhereExpressions.splice(idx, 1)
610 }
611
612 // Evict rows that were exclusively loaded by the departing predicate.
613 // These are rows matching the departing WHERE that are no longer covered
614 // by any remaining active predicate.
615 const compiledDeparting = compileSQLite({ where: options.where })
616 const departingWhereSQL = toInlinedWhereClause(compiledDeparting)
617
618 let evictionSQL: string
619 if (activeWhereExpressions.length === 0) {
620 evictionSQL = `SELECT id FROM ${viewName} WHERE ${departingWhereSQL}`
621 } else {
622 const combinedRemaining =
623 activeWhereExpressions.length === 1
624 ? activeWhereExpressions[0]!
625 : or(
626 activeWhereExpressions[0],
627 activeWhereExpressions[1],
628 ...activeWhereExpressions.slice(2),
629 )
630 const compiledRemaining = compileSQLite({
631 where: combinedRemaining,
632 })
633 const remainingWhereSQL = toInlinedWhereClause(compiledRemaining)
634 evictionSQL = `SELECT id FROM ${viewName} WHERE (${departingWhereSQL}) AND NOT (${remainingWhereSQL})`
635 }
636
637 const rowsToEvict = await database.getAll<{ id: string }>(evictionSQL)
638 if (rowsToEvict.length > 0) {
639 begin()
640 for (const { id } of rowsToEvict) {
641 write({ type: `delete`, key: id })
642 }
643 commit()
644 }
645
646 // Recreate the diff trigger for the remaining active WHERE expressions.
647 await loadSubset()
648 }
649
650 markReady()
651
652 return {
653 cleanup: () => {
654 database.logger.info(
655 `Sync has been stopped for ${viewName} into ${trackedTableName}`,
656 )
657 abortController.abort()
658 },
659 loadSubset: (options: LoadSubsetOptions) => loadSubset(options),
660 unloadSubset: (options: LoadSubsetOptions) => unloadSubset(options),
661 }
662 }
663 },
664 // Expose the getSyncMetadata function
665 getSyncMetadata: undefined,
666 }
667
668 const getKey = (record: OutputType) => asPowerSyncRecord(record).id
669
670 const outputConfig: EnhancedPowerSyncCollectionConfig<
671 TTable,
672 OutputType,
673 TSchema
674 > = {
675 ...restConfig,
676 schema,
677 getKey,
678 // Syncing should start immediately since we need to monitor the changes for mutations
679 startSync: true,
680 syncMode,
681 sync,
682 onInsert: async (params) => {
683 // The transaction here should only ever contain a single insert mutation
684 return await transactor.applyTransaction(params.transaction)
685 },
686 onUpdate: async (params) => {
687 // The transaction here should only ever contain a single update mutation
688 return await transactor.applyTransaction(params.transaction)
689 },
690 onDelete: async (params) => {
691 // The transaction here should only ever contain a single delete mutation
692 return await transactor.applyTransaction(params.transaction)
693 },
694 utils: {
695 getMeta: () => ({
696 tableName: viewName,
697 trackedTableName,
698 metadataIsTracked,
699 serializeValue: (value) =>
700 serializeForSQLite(
701 value,
702 // This is required by the input generic
703 table as Table<
704 MapBaseColumnType<InferPowerSyncOutputType<TTable, TSchema>>
705 >,
706 // Coerce serializer to the shape that corresponds to the Table constructed from OutputType
707 serializer as CustomSQLiteSerializer<
708 OutputType,
709 ExtractedTableColumns<Table<MapBaseColumnType<OutputType>>>
710 >,
711 ),
712 }),
713 },
714 }
715 return outputConfig
716}