Skip to content
Source
Sign inSign up
TanStack/db
Open History to see the last commit

join.ts

374 lines12.1 KB

1/**
2 * # Direct Join Algorithms for Incremental View Maintenance
3 *
4 * High-performance join operations implementing all join types (inner, left, right, full, anti)
5 * with minimal state and optimized performance.
6 *
7 * ## Algorithm
8 *
9 * For each tick, the algorithm processes incoming changes (deltas) and emits join results:
10 *
11 * 1. **Build deltas**: Create delta indexes from input messages using `Index.fromMultiSet()`
12 * 2. **Inner results**: Emit `ΔA⋈B_old + A_old⋈ΔB + ΔA⋈ΔB` (matched pairs)
13 * 3. **Outer results**: For unmatched rows, emit null-extended tuples:
14 * - New unmatched rows from deltas (when opposite side empty)
15 * - Presence transitions: when key goes `0→>0` (retract nulls) or `>0→0` (emit nulls)
16 * 4. **Update state**: Append deltas to indexes (consolidated multiplicity tracking automatic)
17 *
18 * **Consolidated multiplicity tracking** enables O(1) presence checks instead of scanning index buckets.
19 *
20 * ## State
21 *
22 * **Indexes** store the actual data:
23 * - `indexA: Index<K, V1>` - all left-side rows accumulated over time
24 * - `indexB: Index<K, V2>` - all right-side rows accumulated over time
25 *
26 * **Consolidated multiplicity tracking** (built into Index):
27 * - Each Index maintains sum of multiplicities per key internally
28 * - Provides O(1) presence checks: `index.hasPresence(key)` and `index.getConsolidatedMultiplicity(key)`
29 * - Avoids scanning entire index buckets just to check if key has any rows
30 *
31 * ## Join Types
32 *
33 * - **Inner**: Standard delta terms only
34 * - **Outer**: Inner results + null-extended unmatched rows with transition handling
35 * - **Anti**: Unmatched rows only (no inner results)
36 *
37 * ## Key Optimizations
38 *
39 * - **No temp copying**: Uses `(A⊎ΔA)⋈ΔB = A⋈ΔB ⊎ ΔA⋈ΔB` distributive property
40 * - **Early-out checks**: Skip phases when no deltas present
41 * - **Zero-entry pruning**: Keep maps compact, O(distinct keys) memory
42 * - **Final presence logic**: Avoid emit→retract churn within same tick
43 *
44 * ## Correctness
45 *
46 * - **Ordering**: Pre-append snapshots for emissions, post-emit state updates
47 * - **Presence**: Key matched iff mass ≠ 0, transitions trigger null handling
48 * - **Bag semantics**: Proper multiplicity handling including negatives
49 */
50​
51import { BinaryOperator, DifferenceStreamWriter } from '../graph.js'
52import { StreamBuilder } from '../d2.js'
53import { MultiSet } from '../multiset.js'
54import { Index } from '../indexes.js'
55import type { DifferenceStreamReader } from '../graph.js'
56import type { IStreamBuilder, KeyValue, PipedOperator } from '../types.js'
57​
58/**
59 * Type of join to perform
60 */
61export type JoinType = `inner` | `left` | `right` | `full` | `anti`
62​
63/**
64 * Operator that joins two input streams using direct join algorithms
65 */
66export class JoinOperator<K, V1, V2> extends BinaryOperator<
67 [K, V1] | [K, V2] | [K, [V1, V2]] | [K, [V1 | null, V2 | null]]
68> {
69 #indexA = new Index<K, V1>()
70 #indexB = new Index<K, V2>()
71 #mode: JoinType
72​
73 constructor(
74 id: number,
75 inputA: DifferenceStreamReader<[K, V1]>,
76 inputB: DifferenceStreamReader<[K, V2]>,
77 output: DifferenceStreamWriter<any>,
78 mode: JoinType = `inner`,
79 ) {
80 super(id, inputA, inputB, output)
81 this.#mode = mode
82 }
83​
84 run(): void {
85 // Build deltas from input messages
86 const deltaA = Index.fromMultiSets<K, V1>(
87 this.inputAMessages() as Array<MultiSet<[K, V1]>>,
88 )
89 const deltaB = Index.fromMultiSets<K, V2>(
90 this.inputBMessages() as Array<MultiSet<[K, V2]>>,
91 )
92​
93 // Early-out if nothing changed
94 if (deltaA.size === 0 && deltaB.size === 0) return
95​
96 const results = new MultiSet<any>()
97​
98 // Emit inner results (all modes except anti)
99 if (this.#mode !== `anti`) {
100 this.emitInnerResults(deltaA, deltaB, results)
101 }
102​
103 // Emit left outer/anti results
104 if (
105 this.#mode === `left` ||
106 this.#mode === `full` ||
107 this.#mode === `anti`
108 ) {
109 this.emitLeftOuterResults(deltaA, deltaB, results)
110 }
111​
112 // Emit right outer results
113 if (this.#mode === `right` || this.#mode === `full`) {
114 this.emitRightOuterResults(deltaA, deltaB, results)
115 }
116​
117 // Update state and send results
118 // IMPORTANT: All emissions use pre-append snapshots of indexA/indexB.
119 // Now append ALL deltas to indices - this happens unconditionally for every key,
120 // regardless of whether presence flipped. Consolidated multiplicity tracking is automatic.
121 this.#indexA.append(deltaA)
122 this.#indexB.append(deltaB)
123​
124 // Send results
125 if (results.getInner().length > 0) {
126 this.output.sendData(results)
127 }
128 }
129​
130 private emitInnerResults(
131 deltaA: Index<K, V1>,
132 deltaB: Index<K, V2>,
133 results: MultiSet<any>,
134 ): void {
135 // Emit the three standard delta terms: ΔA⋈B_old, A_old⋈ΔB, ΔA⋈ΔB
136 if (deltaA.size > 0) results.extend(deltaA.join(this.#indexB))
137 if (deltaB.size > 0) results.extend(this.#indexA.join(deltaB))
138 if (deltaA.size > 0 && deltaB.size > 0) results.extend(deltaA.join(deltaB))
139 }
140​
141 private emitLeftOuterResults(
142 deltaA: Index<K, V1>,
143 deltaB: Index<K, V2>,
144 results: MultiSet<any>,
145 ): void {
146 // Emit unmatched left rows from deltaA
147 if (deltaA.size > 0) {
148 for (const [key, valueIterator] of deltaA.entriesIterators()) {
149 const currentMultiplicityB =
150 this.#indexB.getConsolidatedMultiplicity(key)
151 const deltaMultiplicityB = deltaB.getConsolidatedMultiplicity(key)
152 const finalMultiplicityB = currentMultiplicityB + deltaMultiplicityB
153​
154 if (finalMultiplicityB === 0) {
155 for (const [value, multiplicity] of valueIterator) {
156 if (multiplicity !== 0) {
157 results.add([key, [value, null]], multiplicity)
158 }
159 }
160 }
161 }
162 }
163​
164 // Handle presence transitions from right side changes
165 if (deltaB.size > 0) {
166 for (const key of deltaB.getPresenceKeys()) {
167 const before = this.#indexB.getConsolidatedMultiplicity(key)
168 const deltaMult = deltaB.getConsolidatedMultiplicity(key)
169 if (deltaMult === 0) continue
170 const after = before + deltaMult
171​
172 // Skip transition handling if presence doesn't flip (both zero or both non-zero)
173 // Note: Index updates happen later regardless - we're only skipping null-extension emissions here
174 if ((before === 0) === (after === 0)) continue
175​
176 // Determine the type of transition:
177 // - 0 → non-zero: Right becomes non-empty, left rows transition from unmatched to matched
178 // → RETRACT previously emitted null-extended rows (emit with negative multiplicity)
179 // - non-zero → 0: Right becomes empty, left rows transition from matched to unmatched
180 // → EMIT new null-extended rows (emit with positive multiplicity)
181 const transitioningToMatched = before === 0
182​
183 for (const [value, multiplicity] of this.#indexA.getIterator(key)) {
184 if (multiplicity !== 0) {
185 results.add(
186 [key, [value, null]],
187 transitioningToMatched ? -multiplicity : +multiplicity,
188 )
189 }
190 }
191 }
192 }
193 }
194​
195 private emitRightOuterResults(
196 deltaA: Index<K, V1>,
197 deltaB: Index<K, V2>,
198 results: MultiSet<any>,
199 ): void {
200 // Emit unmatched right rows from deltaB
201 if (deltaB.size > 0) {
202 for (const [key, valueIterator] of deltaB.entriesIterators()) {
203 const currentMultiplicityA =
204 this.#indexA.getConsolidatedMultiplicity(key)
205 const deltaMultiplicityA = deltaA.getConsolidatedMultiplicity(key)
206 const finalMultiplicityA = currentMultiplicityA + deltaMultiplicityA
207​
208 if (finalMultiplicityA === 0) {
209 for (const [value, multiplicity] of valueIterator) {
210 if (multiplicity !== 0) {
211 results.add([key, [null, value]], multiplicity)
212 }
213 }
214 }
215 }
216 }
217​
218 // Handle presence transitions from left side changes
219 if (deltaA.size > 0) {
220 for (const key of deltaA.getPresenceKeys()) {
221 const before = this.#indexA.getConsolidatedMultiplicity(key)
222 const deltaMult = deltaA.getConsolidatedMultiplicity(key)
223 if (deltaMult === 0) continue
224 const after = before + deltaMult
225​
226 // Skip transition handling if presence doesn't flip (both zero or both non-zero)
227 // Note: Index updates happen later regardless - we're only skipping null-extension emissions here
228 if ((before === 0) === (after === 0)) continue
229​
230 // Determine the type of transition:
231 // - 0 → non-zero: Left becomes non-empty, right rows transition from unmatched to matched
232 // → RETRACT previously emitted null-extended rows (emit with negative multiplicity)
233 // - non-zero → 0: Left becomes empty, right rows transition from matched to unmatched
234 // → EMIT new null-extended rows (emit with positive multiplicity)
235 const transitioningToMatched = before === 0
236​
237 for (const [value, multiplicity] of this.#indexB.getIterator(key)) {
238 if (multiplicity !== 0) {
239 results.add(
240 [key, [null, value]],
241 transitioningToMatched ? -multiplicity : +multiplicity,
242 )
243 }
244 }
245 }
246 }
247 }
249​
250/**
251 * Joins two input streams
252 * @param other - The other stream to join with
253 * @param type - The type of join to perform
254 */
255export function join<
256 K,
257 V1 extends T extends KeyValue<infer _KT, infer VT> ? VT : never,
258 V2,
259 T,
260>(
261 other: IStreamBuilder<KeyValue<K, V2>>,
262 type: JoinType = `inner`,
263): PipedOperator<T, KeyValue<K, [V1 | null, V2 | null]>> {
264 return (
265 stream: IStreamBuilder<T>,
266 ): IStreamBuilder<KeyValue<K, [V1 | null, V2 | null]>> => {
267 if (stream.graph !== other.graph) {
268 throw new Error(`Cannot join streams from different graphs`)
269 }
270 const output = new StreamBuilder<KeyValue<K, [V1 | null, V2 | null]>>(
271 stream.graph,
272 new DifferenceStreamWriter<KeyValue<K, [V1 | null, V2 | null]>>(),
273 )
274 const operator = new JoinOperator<K, V1, V2>(
275 stream.graph.getNextOperatorId(),
276 stream.connectReader() as DifferenceStreamReader<KeyValue<K, V1>>,
277 other.connectReader(),
278 output.writer,
279 type,
280 )
281 stream.graph.addOperator(operator)
282 return output
283 }
285​
286/**
287 * Joins two input streams (inner join)
288 * @param other - The other stream to join with
289 */
290export function innerJoin<
291 K,
292 V1 extends T extends KeyValue<infer _KT, infer VT> ? VT : never,
293 V2,
294 T,
295>(
296 other: IStreamBuilder<KeyValue<K, V2>>,
297): PipedOperator<T, KeyValue<K, [V1, V2]>> {
298 return join(other, `inner`) as unknown as PipedOperator<
299 T,
300 KeyValue<K, [V1, V2]>
301 >
303​
304/**
305 * Joins two input streams (anti join)
306 * @param other - The other stream to join with
307 */
308export function antiJoin<
309 K,
310 V1 extends T extends KeyValue<infer _KT, infer VT> ? VT : never,
311 V2,
312 T,
313>(
314 other: IStreamBuilder<KeyValue<K, V2>>,
315): PipedOperator<T, KeyValue<K, [V1, null]>> {
316 return join(other, `anti`) as unknown as PipedOperator<
317 T,
318 KeyValue<K, [V1, null]>
319 >
321​
322/**
323 * Joins two input streams (left join)
324 * @param other - The other stream to join with
325 */
326export function leftJoin<
327 K,
328 V1 extends T extends KeyValue<infer _KT, infer VT> ? VT : never,
329 V2,
330 T,
331>(
332 other: IStreamBuilder<KeyValue<K, V2>>,
333): PipedOperator<T, KeyValue<K, [V1, V2 | null]>> {
334 return join(other, `left`) as unknown as PipedOperator<
335 T,
336 KeyValue<K, [V1, V2 | null]>
337 >
339​
340/**
341 * Joins two input streams (right join)
342 * @param other - The other stream to join with
343 */
344export function rightJoin<
345 K,
346 V1 extends T extends KeyValue<infer _KT, infer VT> ? VT : never,
347 V2,
348 T,
349>(
350 other: IStreamBuilder<KeyValue<K, V2>>,
351): PipedOperator<T, KeyValue<K, [V1 | null, V2]>> {
352 return join(other, `right`) as unknown as PipedOperator<
353 T,
354 KeyValue<K, [V1 | null, V2]>
355 >
357​
358/**
359 * Joins two input streams (full join)
360 * @param other - The other stream to join with
361 */
362export function fullJoin<
363 K,
364 V1 extends T extends KeyValue<infer _KT, infer VT> ? VT : never,
365 V2,
366 T,
367>(
368 other: IStreamBuilder<KeyValue<K, V2>>,
369): PipedOperator<T, KeyValue<K, [V1 | null, V2 | null]>> {
370 return join(other, `full`) as unknown as PipedOperator<
371 T,
372 KeyValue<K, [V1 | null, V2 | null]>
373 >