From 5ae960620de46891cb094fc98c6c1b46e00e1f47 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Tue, 24 Feb 2026 14:06:42 +0100 Subject: [PATCH 01/32] Add support for subqueries in select --- packages/db/src/query/builder/index.ts | 122 ++++++- packages/db/src/query/compiler/group-by.ts | 8 +- packages/db/src/query/compiler/index.ts | 183 +++++++++- packages/db/src/query/compiler/select.ts | 9 + packages/db/src/query/ir.ts | 17 +- .../query/live/collection-config-builder.ts | 322 +++++++++++++++++- packages/db/src/query/live/utils.ts | 49 +++ 7 files changed, 688 insertions(+), 22 deletions(-) diff --git a/packages/db/src/query/builder/index.ts b/packages/db/src/query/builder/index.ts index c6b129c53..a3bdf1340 100644 --- a/packages/db/src/query/builder/index.ts +++ b/packages/db/src/query/builder/index.ts @@ -3,6 +3,7 @@ import { Aggregate as AggregateExpr, CollectionRef, Func as FuncExpr, + IncludesSubquery, PropRef, QueryRef, Value as ValueExpr, @@ -491,7 +492,7 @@ export class BaseQueryBuilder { const aliases = this._getCurrentAliases() const refProxy = createRefProxy(aliases) as RefsForContext const selectObject = callback(refProxy) - const select = buildNestedSelect(selectObject) + const select = buildNestedSelect(selectObject, aliases) return new BaseQueryBuilder({ ...this.query, @@ -867,7 +868,7 @@ function isPlainObject(value: any): value is Record { ) } -function buildNestedSelect(obj: any): any { +function buildNestedSelect(obj: any, parentAliases: Array = []): any { if (!isPlainObject(obj)) return toExpr(obj) const out: Record = {} for (const [k, v] of Object.entries(obj)) { @@ -876,11 +877,126 @@ function buildNestedSelect(obj: any): any { out[k] = v continue } - out[k] = buildNestedSelect(v) + if (v instanceof BaseQueryBuilder) { + out[k] = buildIncludesSubquery(v, k, parentAliases) + continue + } + out[k] = buildNestedSelect(v, parentAliases) } return out } +/** + * Builds an IncludesSubquery IR node from a child query builder. + * Extracts the correlation condition from the child's WHERE clauses by finding + * an eq() predicate that references both a parent alias and a child alias. + */ +function buildIncludesSubquery( + childBuilder: BaseQueryBuilder, + fieldName: string, + parentAliases: Array, +): IncludesSubquery { + const childQuery = childBuilder._getQuery() + + // Collect child's own aliases + const childAliases: Array = [childQuery.from.alias] + if (childQuery.join) { + for (const j of childQuery.join) { + childAliases.push(j.from.alias) + } + } + + // Walk child's WHERE clauses to find the correlation condition + let parentRef: PropRef | undefined + let childRef: PropRef | undefined + let correlationWhereIndex = -1 + + if (childQuery.where) { + for (let i = 0; i < childQuery.where.length; i++) { + const where = childQuery.where[i]! + const expr = + typeof where === `object` && `expression` in where + ? where.expression + : where + + // Look for eq(a, b) where one side references parent and other references child + if ( + expr.type === `func` && + expr.name === `eq` && + expr.args.length === 2 + ) { + const [argA, argB] = expr.args + const result = extractCorrelation( + argA!, + argB!, + parentAliases, + childAliases, + ) + if (result) { + parentRef = result.parentRef + childRef = result.childRef + correlationWhereIndex = i + break + } + } + } + } + + if (!parentRef || !childRef || correlationWhereIndex === -1) { + throw new Error( + `Includes subquery for "${fieldName}" must have a WHERE clause with an eq() condition ` + + `that correlates a parent field with a child field. ` + + `Example: .where(({child}) => eq(child.parentId, parent.id))`, + ) + } + + // Remove the correlation WHERE from the child query + const modifiedWhere = [...childQuery.where!] + modifiedWhere.splice(correlationWhereIndex, 1) + const modifiedQuery: QueryIR = { + ...childQuery, + where: modifiedWhere.length > 0 ? modifiedWhere : undefined, + } + + return new IncludesSubquery(modifiedQuery, parentRef, childRef, fieldName) +} + +/** + * Checks if two eq() arguments form a parent-child correlation. + * Returns the parent and child PropRefs if found, undefined otherwise. + */ +function extractCorrelation( + argA: BasicExpression, + argB: BasicExpression, + parentAliases: Array, + childAliases: Array, +): { parentRef: PropRef; childRef: PropRef } | undefined { + if (argA.type === `ref` && argB.type === `ref`) { + const aAlias = argA.path[0] + const bAlias = argB.path[0] + + if ( + aAlias && + bAlias && + parentAliases.includes(aAlias) && + childAliases.includes(bAlias) + ) { + return { parentRef: argA, childRef: argB } + } + + if ( + aAlias && + bAlias && + parentAliases.includes(bAlias) && + childAliases.includes(aAlias) + ) { + return { parentRef: argB, childRef: argA } + } + } + + return undefined +} + // Internal function to build a query from a callback // used by liveQueryCollectionOptions.query export function buildQuery( diff --git a/packages/db/src/query/compiler/group-by.ts b/packages/db/src/query/compiler/group-by.ts index 10d83a11b..595d277ae 100644 --- a/packages/db/src/query/compiler/group-by.ts +++ b/packages/db/src/query/compiler/group-by.ts @@ -519,7 +519,7 @@ function evaluateWrappedAggregates( * contain an Aggregate. Safely returns false for nested Select objects. */ export function containsAggregate( - expr: BasicExpression | Aggregate | Select, + expr: BasicExpression | Aggregate | Select | { type: string }, ): boolean { if (!isExpressionLike(expr)) { return false @@ -527,9 +527,9 @@ export function containsAggregate( if (expr.type === `agg`) { return true } - if (expr.type === `func`) { - return expr.args.some((arg: BasicExpression | Aggregate) => - containsAggregate(arg), + if (expr.type === `func` && `args` in expr) { + return (expr.args as Array).some( + (arg: BasicExpression | Aggregate) => containsAggregate(arg), ) } return false diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index 885a7eaa6..f39ee8e37 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -1,4 +1,9 @@ -import { distinct, filter, map } from '@tanstack/db-ivm' +import { + distinct, + filter, + join as joinOperator, + map, +} from '@tanstack/db-ivm' import { optimizeQuery } from '../optimizer.js' import { CollectionInputNotFoundError, @@ -9,7 +14,12 @@ import { LimitOffsetRequireOrderByError, UnsupportedFromTypeError, } from '../../errors.js' -import { PropRef, Value as ValClass, getWhereExpression } from '../ir.js' +import { + IncludesSubquery, + PropRef, + Value as ValClass, + getWhereExpression, +} from '../ir.js' import { compileExpression, toBooleanPredicate } from './evaluators.js' import { processJoins } from './joins.js' import { containsAggregate, processGroupBy } from './group-by.js' @@ -34,6 +44,23 @@ import type { QueryCache, QueryMapping, WindowOptions } from './types.js' export type { WindowOptions } from './types.js' +/** + * Result of compiling an includes subquery, including the child pipeline + * and metadata needed to route child results to parent-scoped Collections. + */ +export interface IncludesCompilationResult { + /** Filtered child pipeline (post inner-join with parent keys) */ + pipeline: ResultStream + /** Result field name on parent (e.g., "issues") */ + fieldName: string + /** Parent-side correlation ref (e.g., project.id) */ + correlationField: PropRef + /** Child-side correlation ref (e.g., issue.projectId) */ + childCorrelationField: PropRef + /** Full compilation result for the child query (for nested includes + alias tracking) */ + childCompilationResult: CompilationResult +} + /** * Result of query compilation including both the pipeline and source-specific WHERE clauses */ @@ -68,6 +95,9 @@ export interface CompilationResult { * the inner aliases where collection subscriptions were created. */ aliasRemapping: Record + + /** Child pipelines for includes subqueries */ + includes?: Array } /** @@ -94,6 +124,9 @@ export function compileQuery( setWindowFn: (windowFn: (options: WindowOptions) => void) => void, cache: QueryCache = new WeakMap(), queryMapping: QueryMapping = new WeakMap(), + // For includes: parent key stream to inner-join with this query's FROM + parentKeyStream?: KeyedStream, + childCorrelationField?: PropRef, ): CompilationResult { // Check if the original raw query has already been compiled const cachedResult = cache.get(rawQuery) @@ -153,8 +186,44 @@ export function compileQuery( ) sources[mainSource] = mainInput + // If this is an includes child query, inner-join the raw input with parent keys. + // This filters the child collection to only rows matching parents in the result set. + // The inner join happens BEFORE namespace wrapping / WHERE / SELECT / ORDER BY, + // so the child pipeline only processes rows that match parents. + let filteredMainInput = mainInput + if (parentKeyStream && childCorrelationField) { + // Re-key child input by correlation field: [correlationValue, [childKey, childRow]] + const childFieldPath = childCorrelationField.path.slice(1) // remove alias prefix + const childRekeyed = mainInput.pipe( + map(([key, row]: [unknown, any]) => { + const correlationValue = getNestedValue(row, childFieldPath) + return [correlationValue, [key, row]] as [unknown, [unknown, any]] + }), + ) + + // Inner join: only children whose correlation key exists in parent keys pass through + const joined = childRekeyed.pipe( + joinOperator(parentKeyStream, `inner`), + ) + + // Extract: [correlationValue, [[childKey, childRow], null]] → [childKey, childRow] + // Tag the row with __correlationKey for output routing + filteredMainInput = joined.pipe( + filter(([_correlationValue, [childSide]]: any) => { + return childSide != null + }), + map(([correlationValue, [childSide, _parentSide]]: any) => { + const [childKey, childRow] = childSide + return [childKey, { ...childRow, __correlationKey: correlationValue }] + }), + ) + + // Update sources so the rest of the pipeline uses the filtered input + sources[mainSource] = filteredMainInput + } + // Prepare the initial pipeline with the main source wrapped in its alias - let pipeline: NamespacedAndKeyedStream = mainInput.pipe( + let pipeline: NamespacedAndKeyedStream = filteredMainInput.pipe( map(([key, row]) => { // Initialize the record with a nested structure const ret = [key, { [mainSource]: row }] as [ @@ -215,6 +284,52 @@ export function compileQuery( } } + // Extract includes from SELECT, compile child pipelines, and replace with placeholders. + // This must happen AFTER WHERE (so parent pipeline is filtered) but BEFORE processSelect + // (so IncludesSubquery nodes are stripped before select compilation). + const includesResults: Array = [] + if (query.select) { + const includesEntries = extractIncludesFromSelect(query.select) + for (const { key, subquery } of includesEntries) { + // Branch parent pipeline: map to [correlationValue, null] + const compiledCorrelation = compileExpression(subquery.correlationField) + const parentKeys = pipeline.pipe( + map(([_key, nsRow]: any) => [compiledCorrelation(nsRow), null] as any), + ) + + // Recursively compile child query WITH the parent key stream + const childResult = compileQuery( + subquery.query, + allInputs, + collections, + subscriptions, + callbacks, + lazySources, + optimizableOrderByCollections, + setWindowFn, + cache, + queryMapping, + parentKeys, + subquery.childCorrelationField, + ) + + // Merge child's alias metadata into parent's + Object.assign(aliasToCollectionId, childResult.aliasToCollectionId) + Object.assign(aliasRemapping, childResult.aliasRemapping) + + includesResults.push({ + pipeline: childResult.pipeline, + fieldName: subquery.fieldName, + correlationField: subquery.correlationField, + childCorrelationField: subquery.childCorrelationField, + childCompilationResult: childResult, + }) + + // Replace includes entry in select with a null placeholder + replaceIncludesInSelect(query.select, key) + } + } + if (query.distinct && !query.fnSelect && !query.select) { throw new DistinctRequiresSelectError() } @@ -335,23 +450,30 @@ export function compileQuery( ) // Final step: extract the $selected and include orderBy index - const resultPipeline = orderedPipeline.pipe( + const resultPipeline: ResultStream = orderedPipeline.pipe( map(([key, [row, orderByIndex]]) => { // Extract the final results from $selected and include orderBy index const raw = (row as any).$selected const finalResults = unwrapValue(raw) + // When in includes mode, embed the correlation key as third element + if (parentKeyStream) { + const correlationKey = (row as any)[mainSource]?.__correlationKey + return [key, [finalResults, orderByIndex, correlationKey]] as any + } return [key, [finalResults, orderByIndex]] as [unknown, [any, string]] }), - ) + ) as ResultStream const result = resultPipeline // Cache the result before returning (use original query as key) - const compilationResult = { + const compilationResult: CompilationResult = { collectionId: mainCollectionId, pipeline: result, sourceWhereClauses, aliasToCollectionId, aliasRemapping, + includes: + includesResults.length > 0 ? includesResults : undefined, } cache.set(rawQuery, compilationResult) @@ -367,6 +489,11 @@ export function compileQuery( // Extract the final results from $selected and return [key, [results, undefined]] const raw = (row as any).$selected const finalResults = unwrapValue(raw) + // When in includes mode, embed the correlation key as third element + if (parentKeyStream) { + const correlationKey = (row as any)[mainSource]?.__correlationKey + return [key, [finalResults, undefined, correlationKey]] as any + } return [key, [finalResults, undefined]] as [ unknown, [any, string | undefined], @@ -376,12 +503,14 @@ export function compileQuery( const result = resultPipeline // Cache the result before returning (use original query as key) - const compilationResult = { + const compilationResult: CompilationResult = { collectionId: mainCollectionId, pipeline: result, sourceWhereClauses, aliasToCollectionId, aliasRemapping, + includes: + includesResults.length > 0 ? includesResults : undefined, } cache.set(rawQuery, compilationResult) @@ -709,4 +838,44 @@ export function followRef( } } +/** + * Walks a Select object to find IncludesSubquery entries. + * Returns array of {key, subquery} for each found includes. + */ +function extractIncludesFromSelect( + select: Record, +): Array<{ key: string; subquery: IncludesSubquery }> { + const results: Array<{ key: string; subquery: IncludesSubquery }> = [] + for (const [key, value] of Object.entries(select)) { + if (value instanceof IncludesSubquery) { + results.push({ key, subquery: value }) + } + } + return results +} + +/** + * Replaces an IncludesSubquery entry in the select object with a null Value placeholder. + * This ensures processSelect() doesn't encounter it. + */ +function replaceIncludesInSelect( + select: Record, + key: string, +): void { + select[key] = new ValClass(null) +} + +/** + * Gets a nested value from an object by path segments. + * For v1 with single-level correlation fields (e.g., `projectId`), it's just `obj[path[0]]`. + */ +function getNestedValue(obj: any, path: Array): any { + let value = obj + for (const segment of path) { + if (value == null) return value + value = value[segment] + } + return value +} + export type CompileQueryFn = typeof compileQuery diff --git a/packages/db/src/query/compiler/select.ts b/packages/db/src/query/compiler/select.ts index c5baccb68..6eb866ce8 100644 --- a/packages/db/src/query/compiler/select.ts +++ b/packages/db/src/query/compiler/select.ts @@ -221,6 +221,15 @@ function addFromObject( } const expression = value as any + if (expression && expression.type === `includesSubquery`) { + // Placeholder — field will be set to a child Collection by the output layer + ops.push({ + kind: `field`, + alias: [...prefixPath, key].join(`.`), + compiled: () => null, + }) + continue + } if (isNestedSelectObject(expression)) { // Nested selection object addFromObject([...prefixPath, key], expression, ops) diff --git a/packages/db/src/query/ir.ts b/packages/db/src/query/ir.ts index b1e3d1e07..64ddd22c7 100644 --- a/packages/db/src/query/ir.ts +++ b/packages/db/src/query/ir.ts @@ -28,7 +28,7 @@ export interface QueryIR { export type From = CollectionRef | QueryRef export type Select = { - [alias: string]: BasicExpression | Aggregate | Select + [alias: string]: BasicExpression | Aggregate | Select | IncludesSubquery } export type Join = Array @@ -132,6 +132,18 @@ export class Aggregate extends BaseExpression { } } +export class IncludesSubquery extends BaseExpression { + public type = `includesSubquery` as const + constructor( + public query: QueryIR, // Child query (correlation WHERE removed) + public correlationField: PropRef, // Parent-side ref (e.g., project.id) + public childCorrelationField: PropRef, // Child-side ref (e.g., issue.projectId) + public fieldName: string, // Result field name (e.g., "issues") + ) { + super() + } +} + /** * Runtime helper to detect IR expression-like objects. * Prefer this over ad-hoc local implementations to keep behavior consistent. @@ -141,7 +153,8 @@ export function isExpressionLike(value: any): boolean { value instanceof Aggregate || value instanceof Func || value instanceof PropRef || - value instanceof Value + value instanceof Value || + value instanceof IncludesSubquery ) } diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 6ce17ff53..0da80e7c6 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -1,5 +1,6 @@ import { D2, output } from '@tanstack/db-ivm' import { compileQuery } from '../compiler/index.js' +import { createCollection } from '../../collection/index.js' import { MissingAliasInputsError, SetWindowRequiresOrderByError, @@ -16,7 +17,10 @@ import { extractCollectionsFromQuery, } from './utils.js' import type { LiveQueryInternalUtils } from './internal.js' -import type { WindowOptions } from '../compiler/index.js' +import type { + IncludesCompilationResult, + WindowOptions, +} from '../compiler/index.js' import type { SchedulerContextId } from '../../scheduler.js' import type { CollectionSubscription } from '../../collection/subscription.js' import type { RootStreamBuilder } from '@tanstack/db-ivm' @@ -31,7 +35,7 @@ import type { UtilsRecord, } from '../../types.js' import type { Context, GetResult } from '../builder/types.js' -import type { BasicExpression, QueryIR } from '../ir.js' +import type { BasicExpression, PropRef, QueryIR } from '../ir.js' import type { LazyCollectionCallbacks } from '../compiler/joins.js' import type { Changes, @@ -140,6 +144,7 @@ export class CollectionConfigBuilder< public sourceWhereClausesCache: | Map> | undefined + private includesCache: Array | undefined // Map of source alias to subscription readonly subscriptions: Record = {} @@ -632,6 +637,7 @@ export class CollectionConfigBuilder< this.inputsCache = undefined this.pipelineCache = undefined this.sourceWhereClausesCache = undefined + this.includesCache = undefined // Reset lazy source alias state this.lazySources.clear() @@ -680,6 +686,7 @@ export class CollectionConfigBuilder< this.pipelineCache = compilation.pipeline this.sourceWhereClausesCache = compilation.sourceWhereClauses this.compiledAliasToCollectionId = compilation.aliasToCollectionId + this.includesCache = compilation.includes // Defensive check: verify all compiled aliases have corresponding inputs // This should never happen since all aliases come from user declarations, @@ -727,10 +734,18 @@ export class CollectionConfigBuilder< }), ) + // Set up includes output routing and child collection lifecycle + const includesState = this.setupIncludesOutput(this.includesCache, syncState) + // Flush pending changes and reset the accumulator. // Called at the end of each graph run to commit all accumulated changes. syncState.flushPendingChanges = () => { - if (pendingChanges.size === 0) { + const hasParentChanges = pendingChanges.size > 0 + const hasChildChanges = includesState.some( + (s) => s.pendingChildChanges.size > 0, + ) + + if (!hasParentChanges && !hasChildChanges) { return } @@ -762,10 +777,21 @@ export class CollectionConfigBuilder< changesToApply = merged } - begin() - changesToApply.forEach(this.applyChanges.bind(this, config)) - commit() + // 1. Flush parent changes + if (hasParentChanges) { + begin() + changesToApply.forEach(this.applyChanges.bind(this, config)) + commit() + } pendingChanges = new Map() + + // 2. Process includes: create/dispose child Collections, route child changes + flushIncludesState( + includesState, + config.collection, + this.id, + hasParentChanges ? changesToApply : null, + ) } graph.finalize() @@ -778,6 +804,79 @@ export class CollectionConfigBuilder< return syncState as FullSyncState } + /** + * Sets up output callbacks for includes child pipelines. + * Each includes entry gets its own output callback that accumulates child changes, + * and a child registry that maps correlation key → child Collection. + */ + private setupIncludesOutput( + includesEntries: Array | undefined, + syncState: SyncState, + ): Array { + if (!includesEntries || includesEntries.length === 0) { + return [] + } + + return includesEntries.map((entry) => { + const state: IncludesOutputState = { + fieldName: entry.fieldName, + correlationField: entry.correlationField, + childCorrelationField: entry.childCorrelationField, + childRegistry: new Map(), + pendingChildChanges: new Map(), + } + + // Attach output callback on the child pipeline + entry.pipeline.pipe( + output((data) => { + const messages = data.getInner() + syncState.messagesCount += messages.length + + for (const [ + [childKey, tupleData], + multiplicity, + ] of messages) { + const [childResult, _orderByIndex, correlationKey] = + tupleData as unknown as [any, string | undefined, unknown] + + // Accumulate by [correlationKey, childKey] + let byChild = state.pendingChildChanges.get(correlationKey) + if (!byChild) { + byChild = new Map() + state.pendingChildChanges.set(correlationKey, byChild) + } + + const existing = byChild.get(childKey) || { + deletes: 0, + inserts: 0, + value: childResult, + orderByIndex: _orderByIndex, + } + + if (multiplicity < 0) { + existing.deletes += Math.abs(multiplicity) + } else if (multiplicity > 0) { + existing.inserts += multiplicity + existing.value = childResult + } + + byChild.set(childKey, existing) + } + }), + ) + + // Recursively set up nested includes (e.g., comments inside issues) + if (entry.childCompilationResult.includes) { + state.nestedIncludesState = this.setupIncludesOutput( + entry.childCompilationResult.includes, + syncState, + ) + } + + return state + }) + } + private applyChanges( config: SyncMethods, changes: { @@ -1013,6 +1112,217 @@ function createOrderByComparator( } } +/** + * State tracked per includes entry for output routing and child lifecycle + */ +type IncludesOutputState = { + fieldName: string + correlationField: PropRef + childCorrelationField: PropRef + /** Maps correlation key value → child Collection entry */ + childRegistry: Map + /** Pending child changes: correlationKey → Map */ + pendingChildChanges: Map>> + /** Nested includes state (for projects → issues → comments) */ + nestedIncludesState?: Array +} + +type ChildCollectionEntry = { + collection: Collection + syncMethods: SyncMethods | null + resultKeys: WeakMap +} + +/** + * Creates a child Collection entry for includes subqueries. + * The child Collection is a full-fledged Collection instance that starts syncing immediately. + */ +function createChildCollectionEntry( + parentId: string, + fieldName: string, + correlationKey: unknown, +): ChildCollectionEntry { + const resultKeys = new WeakMap() + let syncMethods: SyncMethods | null = null + + const collection = createCollection({ + id: `${parentId}-${fieldName}-${String(correlationKey)}`, + getKey: (item: any) => resultKeys.get(item) as string | number, + sync: { + rowUpdateMode: `full`, + sync: (methods) => { + syncMethods = methods + return () => { + syncMethods = null + } + }, + }, + startSync: true, + }) + + return { collection, get syncMethods() { return syncMethods }, resultKeys } +} + +/** + * Recursively flushes includes state, processing child changes and creating + * child Collections. Handles nested includes (e.g., comments inside issues) + * by recursing into nested state after flushing each level. + */ +function flushIncludesState( + includesState: Array, + parentCollection: Collection, + parentId: string, + parentChanges: Map> | null, +): void { + for (const state of includesState) { + // For parent INSERTs: ensure a child Collection exists for every parent, + // even those with no children (produces an empty child Collection). + if (parentChanges) { + const fieldPath = state.correlationField.path.slice(1) // remove alias prefix + for (const [_key, changes] of parentChanges) { + if (changes.inserts > 0) { + const parentResult = changes.value + // Extract the correlation key value from the parent result + let correlationKey: unknown = parentResult + for (const segment of fieldPath) { + if (correlationKey == null) break + correlationKey = (correlationKey as any)[segment] + } + + if (correlationKey != null) { + // Ensure child Collection exists for this correlation key + if (!state.childRegistry.has(correlationKey)) { + const entry = createChildCollectionEntry( + parentId, + state.fieldName, + correlationKey, + ) + state.childRegistry.set(correlationKey, entry) + } + // Attach child Collection to the parent result + parentResult[state.fieldName] = + state.childRegistry.get(correlationKey)!.collection + } + } + } + } + + // Flush child changes: route to correct child Collections + if (state.pendingChildChanges.size > 0) { + for (const [ + correlationKey, + childChanges, + ] of state.pendingChildChanges) { + // Ensure child Collection exists for this correlation key + let entry = state.childRegistry.get(correlationKey) + if (!entry) { + entry = createChildCollectionEntry( + parentId, + state.fieldName, + correlationKey, + ) + state.childRegistry.set(correlationKey, entry) + } + + // Attach the child Collection to ANY parent that has this correlation key + // by scanning the parent result collection + attachChildCollectionToParent( + parentCollection, + state.fieldName, + correlationKey, + state.correlationField, + entry.collection, + ) + + // Apply child changes to the child Collection + if (entry.syncMethods) { + entry.syncMethods.begin() + for (const [childKey, change] of childChanges) { + entry.resultKeys.set(change.value, childKey) + if (change.inserts > 0 && change.deletes === 0) { + entry.syncMethods.write({ value: change.value, type: `insert` }) + } else if ( + change.inserts > change.deletes || + (change.inserts === change.deletes && + entry.syncMethods.collection.has( + entry.syncMethods.collection.getKeyFromItem(change.value), + )) + ) { + entry.syncMethods.write({ value: change.value, type: `update` }) + } else if (change.deletes > 0) { + entry.syncMethods.write({ value: change.value, type: `delete` }) + } + } + entry.syncMethods.commit() + } + + // Recursively process nested includes (e.g., comments inside issues) + if (state.nestedIncludesState) { + flushIncludesState( + state.nestedIncludesState, + entry.collection, + entry.collection.id, + childChanges, + ) + } + } + state.pendingChildChanges.clear() + } + + // For parent DELETEs: dispose child Collections so re-added parents + // get a fresh empty child Collection instead of reusing stale data. + if (parentChanges) { + const fieldPath = state.correlationField.path.slice(1) + for (const [_key, changes] of parentChanges) { + if (changes.deletes > 0 && changes.inserts === 0) { + let correlationKey: unknown = changes.value + for (const segment of fieldPath) { + if (correlationKey == null) break + correlationKey = (correlationKey as any)[segment] + } + if (correlationKey != null) { + state.childRegistry.delete(correlationKey) + } + } + } + } + } +} + +/** + * Attaches a child Collection to parent rows that match a given correlation key. + * Scans the parent collection to find matching parents and sets the field. + */ +function attachChildCollectionToParent( + parentCollection: Collection, + fieldName: string, + correlationKey: unknown, + correlationField: PropRef, + childCollection: Collection, +): void { + // Walk the parent collection's items to find those matching this correlation key + // The correlation field path has the alias prefix (e.g., ['project', 'id']), + // but at this point the parent result is the selected object, not namespaced. + // We need to find parents by their correlation value. + // Since the parent correlation field is e.g. project.id, and the selected result + // might have 'id' as a field, we use the correlation field path (minus alias). + const fieldPath = correlationField.path.slice(1) // remove alias prefix + + for (const [_key, item] of parentCollection) { + // Navigate to the correlation value on the parent result + let value: any = item + for (const segment of fieldPath) { + if (value == null) break + value = value[segment] + } + + if (value === correlationKey) { + // Set the child Collection on this parent row + ;(item)[fieldName] = childCollection + } + } +} + function accumulateChanges( acc: Map>, [[key, tupleData], multiplicity]: [ diff --git a/packages/db/src/query/live/utils.ts b/packages/db/src/query/live/utils.ts index 07e68d774..765356606 100644 --- a/packages/db/src/query/live/utils.ts +++ b/packages/db/src/query/live/utils.ts @@ -1,6 +1,7 @@ import { MultiSet, serializeValue } from '@tanstack/db-ivm' import { normalizeOrderByPaths } from '../compiler/expressions.js' import { buildQuery, getQueryIR } from '../builder/index.js' +import { IncludesSubquery } from '../ir.js' import type { MultiSetArray, RootStreamBuilder } from '@tanstack/db-ivm' import type { Collection } from '../../collection/index.js' import type { ChangeMessage } from '../../types.js' @@ -44,6 +45,24 @@ export function extractCollectionsFromQuery( } } } + + // Extract from SELECT (for IncludesSubquery) + if (q.select) { + extractFromSelect(q.select) + } + } + + function extractFromSelect(select: any) { + for (const [key, value] of Object.entries(select)) { + if (typeof key === `string` && key.startsWith(`__SPREAD_SENTINEL__`)) { + continue + } + if (value instanceof IncludesSubquery) { + extractFromQuery(value.query) + } else if (isNestedSelectObject(value)) { + extractFromSelect(value) + } + } } // Start extraction from the root query @@ -117,6 +136,19 @@ export function extractCollectionAliases( } } + function traverseSelect(select: any) { + for (const [key, value] of Object.entries(select)) { + if (typeof key === `string` && key.startsWith(`__SPREAD_SENTINEL__`)) { + continue + } + if (value instanceof IncludesSubquery) { + traverse(value.query) + } else if (isNestedSelectObject(value)) { + traverseSelect(value) + } + } + } + function traverse(q?: QueryIR) { if (!q) return @@ -127,6 +159,10 @@ export function extractCollectionAliases( recordAlias(joinClause.from) } } + + if (q.select) { + traverseSelect(q.select) + } } traverse(query) @@ -134,6 +170,19 @@ export function extractCollectionAliases( return aliasesById } +/** + * Check if a value is a nested select object (plain object, not an expression) + */ +function isNestedSelectObject(obj: any): boolean { + if (obj === null || typeof obj !== `object`) return false + if (obj instanceof IncludesSubquery) return false + // Expression-like objects have a .type property + if (`type` in obj && typeof obj.type === `string`) return false + // Ref proxies from spread operations + if (obj.__refProxy) return false + return true +} + /** * Builds a query IR from a config object that contains either a query builder * function or a QueryBuilder instance. From aefe4bc63f25d50fe97383a0647a046ffbdfeceb Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Tue, 24 Feb 2026 14:07:04 +0100 Subject: [PATCH 02/32] Unit tests for includes --- packages/db/tests/query/includes.test.ts | 365 +++++++++++++++++++++++ 1 file changed, 365 insertions(+) create mode 100644 packages/db/tests/query/includes.test.ts diff --git a/packages/db/tests/query/includes.test.ts b/packages/db/tests/query/includes.test.ts new file mode 100644 index 000000000..c09a0ca9c --- /dev/null +++ b/packages/db/tests/query/includes.test.ts @@ -0,0 +1,365 @@ +import { beforeEach, describe, expect, it } from 'vitest' +import { createLiveQueryCollection, eq } from '../../src/query/index.js' +import { createCollection } from '../../src/collection/index.js' +import { mockSyncCollectionOptions } from '../utils.js' + +type Project = { + id: number + name: string +} + +type Issue = { + id: number + projectId: number + title: string +} + +type Comment = { + id: number + issueId: number + body: string +} + +const sampleProjects: Array = [ + { id: 1, name: `Alpha` }, + { id: 2, name: `Beta` }, + { id: 3, name: `Gamma` }, +] + +const sampleIssues: Array = [ + { id: 10, projectId: 1, title: `Bug in Alpha` }, + { id: 11, projectId: 1, title: `Feature for Alpha` }, + { id: 20, projectId: 2, title: `Bug in Beta` }, + // No issues for project 3 +] + +const sampleComments: Array = [ + { id: 100, issueId: 10, body: `Looks bad` }, + { id: 101, issueId: 10, body: `Fixed it` }, + { id: 200, issueId: 20, body: `Same bug` }, + // No comments for issue 11 +] + +function createProjectsCollection() { + return createCollection( + mockSyncCollectionOptions({ + id: `includes-projects`, + getKey: (p) => p.id, + initialData: sampleProjects, + }), + ) +} + +function createIssuesCollection() { + return createCollection( + mockSyncCollectionOptions({ + id: `includes-issues`, + getKey: (i) => i.id, + initialData: sampleIssues, + }), + ) +} + +function createCommentsCollection() { + return createCollection( + mockSyncCollectionOptions({ + id: `includes-comments`, + getKey: (c) => c.id, + initialData: sampleComments, + }), + ) +} + +/** + * Extracts child collection items as a sorted plain array for comparison. + */ +function childItems(collection: any, sortKey = `id`): Array { + return [...collection.toArray].sort( + (a: any, b: any) => a[sortKey] - b[sortKey], + ) +} + +/** + * Recursively converts a live query collection (or child Collection) into a + * plain sorted array, turning any nested child Collections into nested arrays. + * This lets tests compare the full hierarchical result as a single literal. + */ +function toTree(collection: any, sortKey = `id`): Array { + const rows = [...collection.toArray].sort( + (a: any, b: any) => a[sortKey] - b[sortKey], + ) + return rows.map((row: any) => { + const out: Record = {} + for (const [key, value] of Object.entries(row)) { + out[key] = + value && typeof value === `object` && `toArray` in (value as any) + ? toTree(value, sortKey) + : value + } + return out + }) +} + +describe(`includes subqueries`, () => { + let projects: ReturnType + let issues: ReturnType + let comments: ReturnType + + beforeEach(() => { + projects = createProjectsCollection() + issues = createIssuesCollection() + comments = createCommentsCollection() + }) + + function buildIncludesQuery() { + return createLiveQueryCollection((q) => + q + .from({ p: projects }) + .select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + })), + ) + } + + describe(`basic includes`, () => { + it(`produces child Collections on parent rows`, async () => { + const collection = buildIncludesQuery() + await collection.preload() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { id: 10, title: `Bug in Alpha` }, + { id: 11, title: `Feature for Alpha` }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [{ id: 20, title: `Bug in Beta` }], + }, + { + id: 3, + name: `Gamma`, + issues: [], + }, + ]) + }) + +}) + + describe(`reactivity`, () => { + it(`adding a child updates the parent's child collection`, async () => { + const collection = buildIncludesQuery() + await collection.preload() + + expect(childItems((collection.get(1) as any).issues)).toHaveLength(2) + + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 12, projectId: 1, title: `New Alpha issue` }, + }) + issues.utils.commit() + + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 10, title: `Bug in Alpha` }, + { id: 11, title: `Feature for Alpha` }, + { id: 12, title: `New Alpha issue` }, + ]) + }) + + it(`removing a child updates the parent's child collection`, async () => { + const collection = buildIncludesQuery() + await collection.preload() + + expect(childItems((collection.get(1) as any).issues)).toHaveLength(2) + + issues.utils.begin() + issues.utils.write({ + type: `delete`, + value: sampleIssues.find((i) => i.id === 10)!, + }) + issues.utils.commit() + + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 11, title: `Feature for Alpha` }, + ]) + }) + + it(`removing and re-adding a parent resets its child collection`, async () => { + const collection = buildIncludesQuery() + await collection.preload() + + expect(childItems((collection.get(1) as any).issues)).toHaveLength(2) + + // Remove project Alpha + projects.utils.begin() + projects.utils.write({ + type: `delete`, + value: sampleProjects.find((p) => p.id === 1)!, + }) + projects.utils.commit() + + expect(collection.get(1)).toBeUndefined() + + // Re-add project Alpha — should get a fresh child collection + projects.utils.begin() + projects.utils.write({ + type: `insert`, + value: { id: 1, name: `Alpha Reborn` }, + }) + projects.utils.commit() + + const alpha = collection.get(1) as any + expect(alpha).toMatchObject({ id: 1, name: `Alpha Reborn` }) + expect(childItems(alpha.issues)).toEqual([ + { id: 10, title: `Bug in Alpha` }, + { id: 11, title: `Feature for Alpha` }, + ]) + + // New children should flow into the child collection + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 99, projectId: 1, title: `Post-rebirth issue` }, + }) + issues.utils.commit() + + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 10, title: `Bug in Alpha` }, + { id: 11, title: `Feature for Alpha` }, + { id: 99, title: `Post-rebirth issue` }, + ]) + }) + + it(`adding a child to a previously empty parent works`, async () => { + const collection = buildIncludesQuery() + await collection.preload() + + expect(childItems((collection.get(3) as any).issues)).toEqual([]) + + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 30, projectId: 3, title: `Gamma issue` }, + }) + issues.utils.commit() + + expect(childItems((collection.get(3) as any).issues)).toEqual([ + { id: 30, title: `Gamma issue` }, + ]) + }) + }) + + describe(`inner join filtering`, () => { + it(`only shows children for parents matching a WHERE clause`, async () => { + const collection = createLiveQueryCollection((q) => + q + .from({ p: projects }) + .where(({ p }) => eq(p.name, `Alpha`)) + .select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + })), + ) + + await collection.preload() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { id: 10, title: `Bug in Alpha` }, + { id: 11, title: `Feature for Alpha` }, + ], + }, + ]) + }) + }) + + describe(`nested includes`, () => { + it(`supports two levels of includes`, async () => { + const collection = createLiveQueryCollection((q) => + q + .from({ p: projects }) + .select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + comments: q + .from({ c: comments }) + .where(({ c }) => eq(c.issueId, i.id)) + .select(({ c }) => ({ + id: c.id, + body: c.body, + })), + })), + })), + ) + + await collection.preload() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { + id: 10, + title: `Bug in Alpha`, + comments: [ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ], + }, + { + id: 11, + title: `Feature for Alpha`, + comments: [], + }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [ + { + id: 20, + title: `Bug in Beta`, + comments: [{ id: 200, body: `Same bug` }], + }, + ], + }, + { + id: 3, + name: `Gamma`, + issues: [], + }, + ]) + }) + }) +}) From e23321921d267204369fabd040dd88027e3f3bee Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Tue, 24 Feb 2026 15:51:46 +0100 Subject: [PATCH 03/32] Unit tests for ordered subqueries --- packages/db/tests/query/includes.test.ts | 83 ++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/packages/db/tests/query/includes.test.ts b/packages/db/tests/query/includes.test.ts index c09a0ca9c..a214ed13e 100644 --- a/packages/db/tests/query/includes.test.ts +++ b/packages/db/tests/query/includes.test.ts @@ -296,6 +296,89 @@ describe(`includes subqueries`, () => { }) }) + describe(`ordered child queries`, () => { + it(`child collection respects orderBy on the child query`, async () => { + const collection = createLiveQueryCollection((q) => + q + .from({ p: projects }) + .select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .orderBy(({ i }) => i.title, `desc`) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + })), + ) + + await collection.preload() + + // Alpha's issues should be sorted by title descending: + // "Feature for Alpha" before "Bug in Alpha" + const alpha = collection.get(1) as any + const alphaIssues = [...alpha.issues.toArray] + expect(alphaIssues).toEqual([ + { id: 11, title: `Feature for Alpha` }, + { id: 10, title: `Bug in Alpha` }, + ]) + + // Beta has one issue, order doesn't matter but it should still work + const beta = collection.get(2) as any + const betaIssues = [...beta.issues.toArray] + expect(betaIssues).toEqual([{ id: 20, title: `Bug in Beta` }]) + + // Gamma has no issues + const gamma = collection.get(3) as any + expect([...gamma.issues.toArray]).toEqual([]) + }) + + it(`newly inserted children appear in the correct order`, async () => { + const collection = createLiveQueryCollection((q) => + q + .from({ p: projects }) + .select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .orderBy(({ i }) => i.title, `asc`) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + })), + ) + + await collection.preload() + + // Alpha issues sorted ascending: "Bug in Alpha", "Feature for Alpha" + expect([...(collection.get(1) as any).issues.toArray]).toEqual([ + { id: 10, title: `Bug in Alpha` }, + { id: 11, title: `Feature for Alpha` }, + ]) + + // Insert an issue that should appear between the existing two + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 12, projectId: 1, title: `Docs for Alpha` }, + }) + issues.utils.commit() + + // Should maintain ascending order: Bug, Docs, Feature + expect([...(collection.get(1) as any).issues.toArray]).toEqual([ + { id: 10, title: `Bug in Alpha` }, + { id: 12, title: `Docs for Alpha` }, + { id: 11, title: `Feature for Alpha` }, + ]) + }) + }) + describe(`nested includes`, () => { it(`supports two levels of includes`, async () => { const collection = createLiveQueryCollection((q) => From 6347116ca5a5945e0691de72bd91802ca70da62a Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Tue, 24 Feb 2026 16:01:47 +0100 Subject: [PATCH 04/32] Add support for ordered subquery --- packages/db/src/query/compiler/index.ts | 3 +++ .../query/live/collection-config-builder.ts | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index f39ee8e37..743b8f15a 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -57,6 +57,8 @@ export interface IncludesCompilationResult { correlationField: PropRef /** Child-side correlation ref (e.g., issue.projectId) */ childCorrelationField: PropRef + /** Whether the child query has an ORDER BY clause */ + hasOrderBy: boolean /** Full compilation result for the child query (for nested includes + alias tracking) */ childCompilationResult: CompilationResult } @@ -322,6 +324,7 @@ export function compileQuery( fieldName: subquery.fieldName, correlationField: subquery.correlationField, childCorrelationField: subquery.childCorrelationField, + hasOrderBy: !!(subquery.query.orderBy && subquery.query.orderBy.length > 0), childCompilationResult: childResult, }) diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 0da80e7c6..df00daca6 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -822,6 +822,7 @@ export class CollectionConfigBuilder< fieldName: entry.fieldName, correlationField: entry.correlationField, childCorrelationField: entry.childCorrelationField, + hasOrderBy: entry.hasOrderBy, childRegistry: new Map(), pendingChildChanges: new Map(), } @@ -1119,6 +1120,8 @@ type IncludesOutputState = { fieldName: string correlationField: PropRef childCorrelationField: PropRef + /** Whether the child query has an ORDER BY clause */ + hasOrderBy: boolean /** Maps correlation key value → child Collection entry */ childRegistry: Map /** Pending child changes: correlationKey → Map */ @@ -1131,6 +1134,7 @@ type ChildCollectionEntry = { collection: Collection syncMethods: SyncMethods | null resultKeys: WeakMap + orderByIndices: WeakMap | null } /** @@ -1141,13 +1145,20 @@ function createChildCollectionEntry( parentId: string, fieldName: string, correlationKey: unknown, + hasOrderBy: boolean, ): ChildCollectionEntry { const resultKeys = new WeakMap() + const orderByIndices = hasOrderBy ? new WeakMap() : null let syncMethods: SyncMethods | null = null + const compare = orderByIndices + ? createOrderByComparator(orderByIndices) + : undefined + const collection = createCollection({ id: `${parentId}-${fieldName}-${String(correlationKey)}`, getKey: (item: any) => resultKeys.get(item) as string | number, + compare, sync: { rowUpdateMode: `full`, sync: (methods) => { @@ -1160,7 +1171,7 @@ function createChildCollectionEntry( startSync: true, }) - return { collection, get syncMethods() { return syncMethods }, resultKeys } + return { collection, get syncMethods() { return syncMethods }, resultKeys, orderByIndices } } /** @@ -1196,6 +1207,7 @@ function flushIncludesState( parentId, state.fieldName, correlationKey, + state.hasOrderBy, ) state.childRegistry.set(correlationKey, entry) } @@ -1220,6 +1232,7 @@ function flushIncludesState( parentId, state.fieldName, correlationKey, + state.hasOrderBy, ) state.childRegistry.set(correlationKey, entry) } @@ -1239,6 +1252,9 @@ function flushIncludesState( entry.syncMethods.begin() for (const [childKey, change] of childChanges) { entry.resultKeys.set(change.value, childKey) + if (entry.orderByIndices && change.orderByIndex !== undefined) { + entry.orderByIndices.set(change.value, change.orderByIndex) + } if (change.inserts > 0 && change.deletes === 0) { entry.syncMethods.write({ value: change.value, type: `insert` }) } else if ( From 9ff5e169b3b5c2591b9e139bd81e68bd84328204 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Tue, 24 Feb 2026 16:16:41 +0100 Subject: [PATCH 05/32] Unit tests for subqueries with limit --- packages/db/tests/query/includes.test.ts | 87 ++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/packages/db/tests/query/includes.test.ts b/packages/db/tests/query/includes.test.ts index a214ed13e..248c15ecd 100644 --- a/packages/db/tests/query/includes.test.ts +++ b/packages/db/tests/query/includes.test.ts @@ -379,6 +379,93 @@ describe(`includes subqueries`, () => { }) }) + describe(`ordered child queries with limit`, () => { + it(`limits child collection to N items per parent`, async () => { + const collection = createLiveQueryCollection((q) => + q + .from({ p: projects }) + .select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .orderBy(({ i }) => i.title, `asc`) + .limit(1) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + })), + ) + + await collection.preload() + + // Alpha has 2 issues; limit(1) with asc title should keep only "Bug in Alpha" + const alpha = collection.get(1) as any + expect([...alpha.issues.toArray]).toEqual([ + { id: 10, title: `Bug in Alpha` }, + ]) + + // Beta has 1 issue; limit(1) keeps it + const beta = collection.get(2) as any + expect([...beta.issues.toArray]).toEqual([ + { id: 20, title: `Bug in Beta` }, + ]) + + // Gamma has 0 issues; limit(1) still empty + const gamma = collection.get(3) as any + expect([...gamma.issues.toArray]).toEqual([]) + }) + + it(`inserting a child that displaces an existing one respects the limit`, async () => { + const collection = createLiveQueryCollection((q) => + q + .from({ p: projects }) + .select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .orderBy(({ i }) => i.title, `asc`) + .limit(1) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + })), + ) + + await collection.preload() + + // Alpha should have exactly 1 issue (limit 1): "Bug in Alpha" + const alphaIssues = [...(collection.get(1) as any).issues.toArray] + expect(alphaIssues).toHaveLength(1) + expect(alphaIssues).toEqual([ + { id: 10, title: `Bug in Alpha` }, + ]) + + // Insert an issue that comes before "Bug" alphabetically + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 12, projectId: 1, title: `Alpha priority issue` }, + }) + issues.utils.commit() + + // The new issue should displace "Bug in Alpha" since it sorts first + expect([...(collection.get(1) as any).issues.toArray]).toEqual([ + { id: 12, title: `Alpha priority issue` }, + ]) + + // Beta should still have its 1 issue (limit is per-parent) + expect([...(collection.get(2) as any).issues.toArray]).toEqual([ + { id: 20, title: `Bug in Beta` }, + ]) + }) + }) + describe(`nested includes`, () => { it(`supports two levels of includes`, async () => { const collection = createLiveQueryCollection((q) => From 86f563c71a33da5c771aed2642fe38d748cb59d0 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Tue, 24 Feb 2026 16:33:05 +0100 Subject: [PATCH 06/32] Support LIMIT and OFFSET in subqueries --- packages/db/src/query/compiler/index.ts | 9 ++++++ packages/db/src/query/compiler/order-by.ts | 36 ++++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index 743b8f15a..7fcd184cc 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -440,6 +440,14 @@ export function compileQuery( // Process orderBy parameter if it exists if (query.orderBy && query.orderBy.length > 0) { + // When in includes mode with limit/offset, use grouped ordering so that + // the limit is applied per parent (per correlation key), not globally. + const includesGroupKeyFn = + parentKeyStream && (query.limit !== undefined || query.offset !== undefined) + ? (_key: unknown, row: unknown) => + (row as any)?.[mainSource]?.__correlationKey + : undefined + const orderedPipeline = processOrderBy( rawQuery, pipeline, @@ -450,6 +458,7 @@ export function compileQuery( setWindowFn, query.limit, query.offset, + includesGroupKeyFn, ) // Final step: extract the $selected and include orderBy index diff --git a/packages/db/src/query/compiler/order-by.ts b/packages/db/src/query/compiler/order-by.ts index 0ced0081c..2107413b1 100644 --- a/packages/db/src/query/compiler/order-by.ts +++ b/packages/db/src/query/compiler/order-by.ts @@ -1,4 +1,4 @@ -import { orderByWithFractionalIndex } from '@tanstack/db-ivm' +import { groupedOrderByWithFractionalIndex, orderByWithFractionalIndex } from '@tanstack/db-ivm' import { defaultComparator, makeComparator } from '../../utils/comparison.js' import { PropRef, followRef } from '../ir.js' import { ensureIndexForField } from '../../indexes/auto-index.js' @@ -51,6 +51,7 @@ export function processOrderBy( setWindowFn: (windowFn: (options: WindowOptions) => void) => void, limit?: number, offset?: number, + groupKeyFn?: (key: unknown, value: unknown) => unknown, ): IStreamBuilder> { // Pre-compile all order by expressions const compiledOrderBy = orderByClause.map((clause) => { @@ -126,7 +127,9 @@ export function processOrderBy( // to loadSubset so the sync layer can optimize the query. // We try to use an index on the FIRST orderBy column for lazy loading, // even for multi-column orderBy (using wider bounds on first column). - if (limit) { + // Skip this optimization when using grouped ordering (includes with limit), + // because the limit is per-group, not global — the child collection needs all data loaded. + if (limit && !groupKeyFn) { let index: IndexInterface | undefined let followRefCollection: Collection | undefined let firstColumnValueExtractor: CompiledSingleRowExpression | undefined @@ -290,6 +293,35 @@ export function processOrderBy( } } + // Use grouped ordering when a groupKeyFn is provided (includes with limit/offset), + // otherwise use the standard global ordering operator. + if (groupKeyFn) { + return pipeline.pipe( + groupedOrderByWithFractionalIndex(valueExtractor, { + limit, + offset, + comparator: compare, + setSizeCallback, + groupKeyFn, + setWindowFn: ( + windowFn: (options: { offset?: number; limit?: number }) => void, + ) => { + setWindowFn( + (options) => { + windowFn(options) + if (orderByOptimizationInfo) { + orderByOptimizationInfo.offset = + options.offset ?? orderByOptimizationInfo.offset + orderByOptimizationInfo.limit = + options.limit ?? orderByOptimizationInfo.limit + } + }, + ) + }, + }), + ) + } + // Use fractional indexing and return the tuple [value, index] return pipeline.pipe( orderByWithFractionalIndex(valueExtractor, { From f20be0bc9f75ded66c1366b57a03e8e2ebb1474c Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Wed, 25 Feb 2026 08:46:01 +0000 Subject: [PATCH 07/32] ci: apply automated fixes --- packages/db/src/query/compiler/index.ts | 24 +-- packages/db/src/query/compiler/order-by.ts | 25 +-- .../query/live/collection-config-builder.ts | 26 +-- packages/db/tests/query/includes.test.ts | 177 ++++++++---------- 4 files changed, 118 insertions(+), 134 deletions(-) diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index 7fcd184cc..1635280d2 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -1,9 +1,4 @@ -import { - distinct, - filter, - join as joinOperator, - map, -} from '@tanstack/db-ivm' +import { distinct, filter, join as joinOperator, map } from '@tanstack/db-ivm' import { optimizeQuery } from '../optimizer.js' import { CollectionInputNotFoundError, @@ -204,9 +199,7 @@ export function compileQuery( ) // Inner join: only children whose correlation key exists in parent keys pass through - const joined = childRekeyed.pipe( - joinOperator(parentKeyStream, `inner`), - ) + const joined = childRekeyed.pipe(joinOperator(parentKeyStream, `inner`)) // Extract: [correlationValue, [[childKey, childRow], null]] → [childKey, childRow] // Tag the row with __correlationKey for output routing @@ -324,7 +317,9 @@ export function compileQuery( fieldName: subquery.fieldName, correlationField: subquery.correlationField, childCorrelationField: subquery.childCorrelationField, - hasOrderBy: !!(subquery.query.orderBy && subquery.query.orderBy.length > 0), + hasOrderBy: !!( + subquery.query.orderBy && subquery.query.orderBy.length > 0 + ), childCompilationResult: childResult, }) @@ -443,7 +438,8 @@ export function compileQuery( // When in includes mode with limit/offset, use grouped ordering so that // the limit is applied per parent (per correlation key), not globally. const includesGroupKeyFn = - parentKeyStream && (query.limit !== undefined || query.offset !== undefined) + parentKeyStream && + (query.limit !== undefined || query.offset !== undefined) ? (_key: unknown, row: unknown) => (row as any)?.[mainSource]?.__correlationKey : undefined @@ -484,8 +480,7 @@ export function compileQuery( sourceWhereClauses, aliasToCollectionId, aliasRemapping, - includes: - includesResults.length > 0 ? includesResults : undefined, + includes: includesResults.length > 0 ? includesResults : undefined, } cache.set(rawQuery, compilationResult) @@ -521,8 +516,7 @@ export function compileQuery( sourceWhereClauses, aliasToCollectionId, aliasRemapping, - includes: - includesResults.length > 0 ? includesResults : undefined, + includes: includesResults.length > 0 ? includesResults : undefined, } cache.set(rawQuery, compilationResult) diff --git a/packages/db/src/query/compiler/order-by.ts b/packages/db/src/query/compiler/order-by.ts index 2107413b1..6ed5f958c 100644 --- a/packages/db/src/query/compiler/order-by.ts +++ b/packages/db/src/query/compiler/order-by.ts @@ -1,4 +1,7 @@ -import { groupedOrderByWithFractionalIndex, orderByWithFractionalIndex } from '@tanstack/db-ivm' +import { + groupedOrderByWithFractionalIndex, + orderByWithFractionalIndex, +} from '@tanstack/db-ivm' import { defaultComparator, makeComparator } from '../../utils/comparison.js' import { PropRef, followRef } from '../ir.js' import { ensureIndexForField } from '../../indexes/auto-index.js' @@ -306,17 +309,15 @@ export function processOrderBy( setWindowFn: ( windowFn: (options: { offset?: number; limit?: number }) => void, ) => { - setWindowFn( - (options) => { - windowFn(options) - if (orderByOptimizationInfo) { - orderByOptimizationInfo.offset = - options.offset ?? orderByOptimizationInfo.offset - orderByOptimizationInfo.limit = - options.limit ?? orderByOptimizationInfo.limit - } - }, - ) + setWindowFn((options) => { + windowFn(options) + if (orderByOptimizationInfo) { + orderByOptimizationInfo.offset = + options.offset ?? orderByOptimizationInfo.offset + orderByOptimizationInfo.limit = + options.limit ?? orderByOptimizationInfo.limit + } + }) }, }), ) diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index df00daca6..b09e420b5 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -735,7 +735,10 @@ export class CollectionConfigBuilder< ) // Set up includes output routing and child collection lifecycle - const includesState = this.setupIncludesOutput(this.includesCache, syncState) + const includesState = this.setupIncludesOutput( + this.includesCache, + syncState, + ) // Flush pending changes and reset the accumulator. // Called at the end of each graph run to commit all accumulated changes. @@ -833,10 +836,7 @@ export class CollectionConfigBuilder< const messages = data.getInner() syncState.messagesCount += messages.length - for (const [ - [childKey, tupleData], - multiplicity, - ] of messages) { + for (const [[childKey, tupleData], multiplicity] of messages) { const [childResult, _orderByIndex, correlationKey] = tupleData as unknown as [any, string | undefined, unknown] @@ -1171,7 +1171,14 @@ function createChildCollectionEntry( startSync: true, }) - return { collection, get syncMethods() { return syncMethods }, resultKeys, orderByIndices } + return { + collection, + get syncMethods() { + return syncMethods + }, + resultKeys, + orderByIndices, + } } /** @@ -1221,10 +1228,7 @@ function flushIncludesState( // Flush child changes: route to correct child Collections if (state.pendingChildChanges.size > 0) { - for (const [ - correlationKey, - childChanges, - ] of state.pendingChildChanges) { + for (const [correlationKey, childChanges] of state.pendingChildChanges) { // Ensure child Collection exists for this correlation key let entry = state.childRegistry.get(correlationKey) if (!entry) { @@ -1334,7 +1338,7 @@ function attachChildCollectionToParent( if (value === correlationKey) { // Set the child Collection on this parent row - ;(item)[fieldName] = childCollection + item[fieldName] = childCollection } } } diff --git a/packages/db/tests/query/includes.test.ts b/packages/db/tests/query/includes.test.ts index 248c15ecd..8d95b65f9 100644 --- a/packages/db/tests/query/includes.test.ts +++ b/packages/db/tests/query/includes.test.ts @@ -113,19 +113,17 @@ describe(`includes subqueries`, () => { function buildIncludesQuery() { return createLiveQueryCollection((q) => - q - .from({ p: projects }) - .select(({ p }) => ({ - id: p.id, - name: p.name, - issues: q - .from({ i: issues }) - .where(({ i }) => eq(i.projectId, p.id)) - .select(({ i }) => ({ - id: i.id, - title: i.title, - })), - })), + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + })), ) } @@ -155,8 +153,7 @@ describe(`includes subqueries`, () => { }, ]) }) - -}) + }) describe(`reactivity`, () => { it(`adding a child updates the parent's child collection`, async () => { @@ -299,20 +296,18 @@ describe(`includes subqueries`, () => { describe(`ordered child queries`, () => { it(`child collection respects orderBy on the child query`, async () => { const collection = createLiveQueryCollection((q) => - q - .from({ p: projects }) - .select(({ p }) => ({ - id: p.id, - name: p.name, - issues: q - .from({ i: issues }) - .where(({ i }) => eq(i.projectId, p.id)) - .orderBy(({ i }) => i.title, `desc`) - .select(({ i }) => ({ - id: i.id, - title: i.title, - })), - })), + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .orderBy(({ i }) => i.title, `desc`) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + })), ) await collection.preload() @@ -338,20 +333,18 @@ describe(`includes subqueries`, () => { it(`newly inserted children appear in the correct order`, async () => { const collection = createLiveQueryCollection((q) => - q - .from({ p: projects }) - .select(({ p }) => ({ - id: p.id, - name: p.name, - issues: q - .from({ i: issues }) - .where(({ i }) => eq(i.projectId, p.id)) - .orderBy(({ i }) => i.title, `asc`) - .select(({ i }) => ({ - id: i.id, - title: i.title, - })), - })), + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .orderBy(({ i }) => i.title, `asc`) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + })), ) await collection.preload() @@ -382,21 +375,19 @@ describe(`includes subqueries`, () => { describe(`ordered child queries with limit`, () => { it(`limits child collection to N items per parent`, async () => { const collection = createLiveQueryCollection((q) => - q - .from({ p: projects }) - .select(({ p }) => ({ - id: p.id, - name: p.name, - issues: q - .from({ i: issues }) - .where(({ i }) => eq(i.projectId, p.id)) - .orderBy(({ i }) => i.title, `asc`) - .limit(1) - .select(({ i }) => ({ - id: i.id, - title: i.title, - })), - })), + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .orderBy(({ i }) => i.title, `asc`) + .limit(1) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + })), ) await collection.preload() @@ -420,21 +411,19 @@ describe(`includes subqueries`, () => { it(`inserting a child that displaces an existing one respects the limit`, async () => { const collection = createLiveQueryCollection((q) => - q - .from({ p: projects }) - .select(({ p }) => ({ - id: p.id, - name: p.name, - issues: q - .from({ i: issues }) - .where(({ i }) => eq(i.projectId, p.id)) - .orderBy(({ i }) => i.title, `asc`) - .limit(1) - .select(({ i }) => ({ - id: i.id, - title: i.title, - })), - })), + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .orderBy(({ i }) => i.title, `asc`) + .limit(1) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + })), ) await collection.preload() @@ -442,9 +431,7 @@ describe(`includes subqueries`, () => { // Alpha should have exactly 1 issue (limit 1): "Bug in Alpha" const alphaIssues = [...(collection.get(1) as any).issues.toArray] expect(alphaIssues).toHaveLength(1) - expect(alphaIssues).toEqual([ - { id: 10, title: `Bug in Alpha` }, - ]) + expect(alphaIssues).toEqual([{ id: 10, title: `Bug in Alpha` }]) // Insert an issue that comes before "Bug" alphabetically issues.utils.begin() @@ -469,26 +456,24 @@ describe(`includes subqueries`, () => { describe(`nested includes`, () => { it(`supports two levels of includes`, async () => { const collection = createLiveQueryCollection((q) => - q - .from({ p: projects }) - .select(({ p }) => ({ - id: p.id, - name: p.name, - issues: q - .from({ i: issues }) - .where(({ i }) => eq(i.projectId, p.id)) - .select(({ i }) => ({ - id: i.id, - title: i.title, - comments: q - .from({ c: comments }) - .where(({ c }) => eq(c.issueId, i.id)) - .select(({ c }) => ({ - id: c.id, - body: c.body, - })), - })), - })), + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + comments: q + .from({ c: comments }) + .where(({ c }) => eq(c.issueId, i.id)) + .select(({ c }) => ({ + id: c.id, + body: c.body, + })), + })), + })), ) await collection.preload() From be4c5f90b326dd58f6a1a44f40c78825923157a7 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Wed, 25 Feb 2026 10:44:20 +0100 Subject: [PATCH 08/32] Add changeset for includes subqueries Co-Authored-By: Claude Opus 4.6 --- .changeset/includes-subqueries.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/includes-subqueries.md diff --git a/.changeset/includes-subqueries.md b/.changeset/includes-subqueries.md new file mode 100644 index 000000000..caa9f44d4 --- /dev/null +++ b/.changeset/includes-subqueries.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': minor +--- + +feat: support for subqueries for including hierarchical data in live queries From eacc17ab0b2aa016da95d98e5b73acaf640e28a5 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Wed, 25 Feb 2026 11:16:07 +0100 Subject: [PATCH 09/32] Use reverse index for parent lookup in includes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace O(n) parent collection scans with a reverse index (correlationKey → Set) for attaching child Collections to parent rows. The index is populated during parent INSERTs and cleaned up on parent DELETEs. Co-Authored-By: Claude Opus 4.6 --- .../query/live/collection-config-builder.ts | 57 +++++++++++-------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index b09e420b5..5c6535473 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -828,6 +828,7 @@ export class CollectionConfigBuilder< hasOrderBy: entry.hasOrderBy, childRegistry: new Map(), pendingChildChanges: new Map(), + correlationToParentKeys: new Map(), } // Attach output callback on the child pipeline @@ -1126,6 +1127,8 @@ type IncludesOutputState = { childRegistry: Map /** Pending child changes: correlationKey → Map */ pendingChildChanges: Map>> + /** Reverse index: correlation key → Set of parent collection keys */ + correlationToParentKeys: Map> /** Nested includes state (for projects → issues → comments) */ nestedIncludesState?: Array } @@ -1197,7 +1200,7 @@ function flushIncludesState( // even those with no children (produces an empty child Collection). if (parentChanges) { const fieldPath = state.correlationField.path.slice(1) // remove alias prefix - for (const [_key, changes] of parentChanges) { + for (const [parentKey, changes] of parentChanges) { if (changes.inserts > 0) { const parentResult = changes.value // Extract the correlation key value from the parent result @@ -1218,6 +1221,14 @@ function flushIncludesState( ) state.childRegistry.set(correlationKey, entry) } + // Update reverse index: correlation key → parent keys + let parentKeys = state.correlationToParentKeys.get(correlationKey) + if (!parentKeys) { + parentKeys = new Set() + state.correlationToParentKeys.set(correlationKey, parentKeys) + } + parentKeys.add(parentKey) + // Attach child Collection to the parent result parentResult[state.fieldName] = state.childRegistry.get(correlationKey)!.collection @@ -1242,12 +1253,11 @@ function flushIncludesState( } // Attach the child Collection to ANY parent that has this correlation key - // by scanning the parent result collection attachChildCollectionToParent( parentCollection, state.fieldName, correlationKey, - state.correlationField, + state.correlationToParentKeys, entry.collection, ) @@ -1289,11 +1299,11 @@ function flushIncludesState( state.pendingChildChanges.clear() } - // For parent DELETEs: dispose child Collections so re-added parents - // get a fresh empty child Collection instead of reusing stale data. + // For parent DELETEs: dispose child Collections and clean up reverse index + // so re-added parents get a fresh empty child Collection instead of reusing stale data. if (parentChanges) { const fieldPath = state.correlationField.path.slice(1) - for (const [_key, changes] of parentChanges) { + for (const [parentKey, changes] of parentChanges) { if (changes.deletes > 0 && changes.inserts === 0) { let correlationKey: unknown = changes.value for (const segment of fieldPath) { @@ -1302,6 +1312,15 @@ function flushIncludesState( } if (correlationKey != null) { state.childRegistry.delete(correlationKey) + // Clean up reverse index + const parentKeys = + state.correlationToParentKeys.get(correlationKey) + if (parentKeys) { + parentKeys.delete(parentKey) + if (parentKeys.size === 0) { + state.correlationToParentKeys.delete(correlationKey) + } + } } } } @@ -1311,33 +1330,21 @@ function flushIncludesState( /** * Attaches a child Collection to parent rows that match a given correlation key. - * Scans the parent collection to find matching parents and sets the field. + * Uses the reverse index to look up parent keys directly instead of scanning. */ function attachChildCollectionToParent( parentCollection: Collection, fieldName: string, correlationKey: unknown, - correlationField: PropRef, + correlationToParentKeys: Map>, childCollection: Collection, ): void { - // Walk the parent collection's items to find those matching this correlation key - // The correlation field path has the alias prefix (e.g., ['project', 'id']), - // but at this point the parent result is the selected object, not namespaced. - // We need to find parents by their correlation value. - // Since the parent correlation field is e.g. project.id, and the selected result - // might have 'id' as a field, we use the correlation field path (minus alias). - const fieldPath = correlationField.path.slice(1) // remove alias prefix - - for (const [_key, item] of parentCollection) { - // Navigate to the correlation value on the parent result - let value: any = item - for (const segment of fieldPath) { - if (value == null) break - value = value[segment] - } + const parentKeys = correlationToParentKeys.get(correlationKey) + if (!parentKeys) return - if (value === correlationKey) { - // Set the child Collection on this parent row + for (const parentKey of parentKeys) { + const item = parentCollection.get(parentKey as any) + if (item) { item[fieldName] = childCollection } } From 32754bbe9789a0aaa026aac6351f3aec49f33342 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Wed, 25 Feb 2026 10:17:06 +0000 Subject: [PATCH 10/32] ci: apply automated fixes --- packages/db/src/query/live/collection-config-builder.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 5c6535473..51c47cfeb 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -1313,8 +1313,7 @@ function flushIncludesState( if (correlationKey != null) { state.childRegistry.delete(correlationKey) // Clean up reverse index - const parentKeys = - state.correlationToParentKeys.get(correlationKey) + const parentKeys = state.correlationToParentKeys.get(correlationKey) if (parentKeys) { parentKeys.delete(parentKey) if (parentKeys.size === 0) { From 1cbd6d77aeadade96430eb64fe9001fb8d0ef1ad Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Wed, 25 Feb 2026 13:20:13 +0100 Subject: [PATCH 11/32] Unit tests for changes to deeply nested collections --- packages/db/tests/query/includes.test.ts | 52 +++++++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/packages/db/tests/query/includes.test.ts b/packages/db/tests/query/includes.test.ts index 8d95b65f9..07a76a5c8 100644 --- a/packages/db/tests/query/includes.test.ts +++ b/packages/db/tests/query/includes.test.ts @@ -454,8 +454,8 @@ describe(`includes subqueries`, () => { }) describe(`nested includes`, () => { - it(`supports two levels of includes`, async () => { - const collection = createLiveQueryCollection((q) => + function buildNestedQuery() { + return createLiveQueryCollection((q) => q.from({ p: projects }).select(({ p }) => ({ id: p.id, name: p.name, @@ -475,7 +475,10 @@ describe(`includes subqueries`, () => { })), })), ) + } + it(`supports two levels of includes`, async () => { + const collection = buildNestedQuery() await collection.preload() expect(toTree(collection)).toEqual([ @@ -516,5 +519,50 @@ describe(`includes subqueries`, () => { }, ]) }) + + it(`adding a grandchild (comment) updates the nested child collection`, async () => { + const collection = buildNestedQuery() + await collection.preload() + + // Issue 11 (Feature for Alpha) has no comments initially + const alpha = collection.get(1) as any + const issue11 = alpha.issues.get(11) + expect(childItems(issue11.comments)).toEqual([]) + + // Add a comment to issue 11 — no issue or project changes + comments.utils.begin() + comments.utils.write({ + type: `insert`, + value: { id: 110, issueId: 11, body: `Great feature` }, + }) + comments.utils.commit() + + const issue11After = (collection.get(1) as any).issues.get(11) + expect(childItems(issue11After.comments)).toEqual([ + { id: 110, body: `Great feature` }, + ]) + }) + + it(`removing a grandchild (comment) updates the nested child collection`, async () => { + const collection = buildNestedQuery() + await collection.preload() + + // Issue 10 (Bug in Alpha) has 2 comments + const issue10 = (collection.get(1) as any).issues.get(10) + expect(childItems(issue10.comments)).toHaveLength(2) + + // Remove one comment + comments.utils.begin() + comments.utils.write({ + type: `delete`, + value: sampleComments.find((c) => c.id === 100)!, + }) + comments.utils.commit() + + const issue10After = (collection.get(1) as any).issues.get(10) + expect(childItems(issue10After.comments)).toEqual([ + { id: 101, body: `Fixed it` }, + ]) + }) }) }) From 8b8ed47d1fe69a0c338a144c4e882e9d23266388 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Wed, 25 Feb 2026 13:20:55 +0100 Subject: [PATCH 12/32] Move fro top-down to bottom-up approach for flushing changes to nested collections. --- .../query/live/collection-config-builder.ts | 364 ++++++++++++++++-- 1 file changed, 339 insertions(+), 25 deletions(-) diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 51c47cfeb..4d9133324 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -744,9 +744,7 @@ export class CollectionConfigBuilder< // Called at the end of each graph run to commit all accumulated changes. syncState.flushPendingChanges = () => { const hasParentChanges = pendingChanges.size > 0 - const hasChildChanges = includesState.some( - (s) => s.pendingChildChanges.size > 0, - ) + const hasChildChanges = hasPendingIncludesChanges(includesState) if (!hasParentChanges && !hasChildChanges) { return @@ -867,12 +865,14 @@ export class CollectionConfigBuilder< }), ) - // Recursively set up nested includes (e.g., comments inside issues) + // Set up shared buffers for nested includes (e.g., comments inside issues) if (entry.childCompilationResult.includes) { - state.nestedIncludesState = this.setupIncludesOutput( + state.nestedSetups = setupNestedPipelines( entry.childCompilationResult.includes, syncState, ) + state.nestedRoutingIndex = new Map() + state.nestedRoutingReverseIndex = new Map() } return state @@ -1114,6 +1114,19 @@ function createOrderByComparator( } } +/** + * Shared buffer setup for a single nested includes level. + * Pipeline output writes into the buffer; during flush the buffer is drained + * into per-entry states via the routing index. + */ +type NestedIncludesSetup = { + compilationResult: IncludesCompilationResult + /** Shared buffer: nestedCorrelationKey → Map */ + buffer: Map>> + /** For 3+ levels of nesting */ + nestedSetups?: Array +} + /** * State tracked per includes entry for output routing and child lifecycle */ @@ -1129,8 +1142,12 @@ type IncludesOutputState = { pendingChildChanges: Map>> /** Reverse index: correlation key → Set of parent collection keys */ correlationToParentKeys: Map> - /** Nested includes state (for projects → issues → comments) */ - nestedIncludesState?: Array + /** Shared nested pipeline setups (one per nested includes level) */ + nestedSetups?: Array + /** nestedCorrelationKey → parentCorrelationKey */ + nestedRoutingIndex?: Map + /** parentCorrelationKey → Set */ + nestedRoutingReverseIndex?: Map> } type ChildCollectionEntry = { @@ -1138,6 +1155,250 @@ type ChildCollectionEntry = { syncMethods: SyncMethods | null resultKeys: WeakMap orderByIndices: WeakMap | null + /** Per-entry nested includes states (one per nested includes level) */ + includesStates?: Array +} + +/** + * Sets up shared buffers for nested includes pipelines. + * Instead of writing directly into a single shared IncludesOutputState, + * each nested pipeline writes into a buffer that is later drained per-entry. + */ +function setupNestedPipelines( + includes: Array, + syncState: SyncState, +): Array { + return includes.map((entry) => { + const buffer: Map>> = new Map() + + // Attach output callback that writes into the shared buffer + entry.pipeline.pipe( + output((data) => { + const messages = data.getInner() + syncState.messagesCount += messages.length + + for (const [[childKey, tupleData], multiplicity] of messages) { + const [childResult, _orderByIndex, correlationKey] = + tupleData as unknown as [any, string | undefined, unknown] + + let byChild = buffer.get(correlationKey) + if (!byChild) { + byChild = new Map() + buffer.set(correlationKey, byChild) + } + + const existing = byChild.get(childKey) || { + deletes: 0, + inserts: 0, + value: childResult, + orderByIndex: _orderByIndex, + } + + if (multiplicity < 0) { + existing.deletes += Math.abs(multiplicity) + } else if (multiplicity > 0) { + existing.inserts += multiplicity + existing.value = childResult + } + + byChild.set(childKey, existing) + } + }), + ) + + const setup: NestedIncludesSetup = { + compilationResult: entry, + buffer, + } + + // Recursively set up deeper levels + if (entry.childCompilationResult.includes) { + setup.nestedSetups = setupNestedPipelines( + entry.childCompilationResult.includes, + syncState, + ) + } + + return setup + }) +} + +/** + * Creates fresh per-entry IncludesOutputState array from NestedIncludesSetup array. + * Each entry gets its own isolated state for nested includes. + */ +function createPerEntryIncludesStates( + setups: Array, +): Array { + return setups.map((setup) => { + const state: IncludesOutputState = { + fieldName: setup.compilationResult.fieldName, + correlationField: setup.compilationResult.correlationField, + childCorrelationField: setup.compilationResult.childCorrelationField, + hasOrderBy: setup.compilationResult.hasOrderBy, + childRegistry: new Map(), + pendingChildChanges: new Map(), + correlationToParentKeys: new Map(), + } + + if (setup.nestedSetups) { + state.nestedSetups = setup.nestedSetups + state.nestedRoutingIndex = new Map() + state.nestedRoutingReverseIndex = new Map() + } + + return state + }) +} + +/** + * Drains shared buffers into per-entry states using the routing index. + * Returns the set of parent correlation keys that had changes routed to them. + */ +function drainNestedBuffers( + state: IncludesOutputState, +): Set { + const dirtyCorrelationKeys = new Set() + + if (!state.nestedSetups) return dirtyCorrelationKeys + + for (let i = 0; i < state.nestedSetups.length; i++) { + const setup = state.nestedSetups[i]! + const toDelete: Array = [] + + for (const [nestedCorrelationKey, childChanges] of setup.buffer) { + const parentCorrelationKey = state.nestedRoutingIndex!.get(nestedCorrelationKey) + if (parentCorrelationKey === undefined) { + // Unroutable — parent not yet seen; keep in buffer + continue + } + + const entry = state.childRegistry.get(parentCorrelationKey) + if (!entry || !entry.includesStates) { + continue + } + + // Route changes into this entry's per-entry state at position i + const entryState = entry.includesStates[i]! + for (const [childKey, changes] of childChanges) { + let byChild = entryState.pendingChildChanges.get(nestedCorrelationKey) + if (!byChild) { + byChild = new Map() + entryState.pendingChildChanges.set(nestedCorrelationKey, byChild) + } + const existing = byChild.get(childKey) + if (existing) { + existing.inserts += changes.inserts + existing.deletes += changes.deletes + if (changes.inserts > 0) { + existing.value = changes.value + if (changes.orderByIndex !== undefined) { + existing.orderByIndex = changes.orderByIndex + } + } + } else { + byChild.set(childKey, { ...changes }) + } + } + + dirtyCorrelationKeys.add(parentCorrelationKey) + toDelete.push(nestedCorrelationKey) + } + + for (const key of toDelete) { + setup.buffer.delete(key) + } + } + + return dirtyCorrelationKeys +} + +/** + * Updates the routing index after processing child changes. + * Maps nested correlation keys to parent correlation keys so that + * grandchild changes can be routed to the correct per-entry state. + */ +function updateRoutingIndex( + state: IncludesOutputState, + correlationKey: unknown, + childChanges: Map>, +): void { + if (!state.nestedSetups) return + + for (const setup of state.nestedSetups) { + const nestedFieldPath = setup.compilationResult.correlationField.path.slice(1) + + for (const [, change] of childChanges) { + if (change.inserts > 0) { + // Extract nested correlation key from child result + let nestedCorrelationKey: unknown = change.value + for (const segment of nestedFieldPath) { + if (nestedCorrelationKey == null) break + nestedCorrelationKey = (nestedCorrelationKey as any)[segment] + } + + if (nestedCorrelationKey != null) { + state.nestedRoutingIndex!.set(nestedCorrelationKey, correlationKey) + let reverseSet = state.nestedRoutingReverseIndex!.get(correlationKey) + if (!reverseSet) { + reverseSet = new Set() + state.nestedRoutingReverseIndex!.set(correlationKey, reverseSet) + } + reverseSet.add(nestedCorrelationKey) + } + } else if (change.deletes > 0 && change.inserts === 0) { + // Remove from routing index + let nestedCorrelationKey: unknown = change.value + for (const segment of nestedFieldPath) { + if (nestedCorrelationKey == null) break + nestedCorrelationKey = (nestedCorrelationKey as any)[segment] + } + + if (nestedCorrelationKey != null) { + state.nestedRoutingIndex!.delete(nestedCorrelationKey) + const reverseSet = state.nestedRoutingReverseIndex!.get(correlationKey) + if (reverseSet) { + reverseSet.delete(nestedCorrelationKey) + if (reverseSet.size === 0) { + state.nestedRoutingReverseIndex!.delete(correlationKey) + } + } + } + } + } + } +} + +/** + * Cleans routing index entries when a parent is deleted. + * Uses the reverse index to find and remove all nested routing entries. + */ +function cleanRoutingIndexOnDelete( + state: IncludesOutputState, + correlationKey: unknown, +): void { + if (!state.nestedRoutingReverseIndex) return + + const nestedKeys = state.nestedRoutingReverseIndex.get(correlationKey) + if (nestedKeys) { + for (const nestedKey of nestedKeys) { + state.nestedRoutingIndex!.delete(nestedKey) + } + state.nestedRoutingReverseIndex.delete(correlationKey) + } +} + +/** + * Recursively checks whether any nested buffer has pending changes. + */ +function hasNestedBufferChanges( + setups: Array, +): boolean { + for (const setup of setups) { + if (setup.buffer.size > 0) return true + if (setup.nestedSetups && hasNestedBufferChanges(setup.nestedSetups)) return true + } + return false } /** @@ -1149,6 +1410,7 @@ function createChildCollectionEntry( fieldName: string, correlationKey: unknown, hasOrderBy: boolean, + nestedSetups?: Array, ): ChildCollectionEntry { const resultKeys = new WeakMap() const orderByIndices = hasOrderBy ? new WeakMap() : null @@ -1174,7 +1436,7 @@ function createChildCollectionEntry( startSync: true, }) - return { + const entry: ChildCollectionEntry = { collection, get syncMethods() { return syncMethods @@ -1182,12 +1444,22 @@ function createChildCollectionEntry( resultKeys, orderByIndices, } + + if (nestedSetups) { + entry.includesStates = createPerEntryIncludesStates(nestedSetups) + } + + return entry } /** - * Recursively flushes includes state, processing child changes and creating - * child Collections. Handles nested includes (e.g., comments inside issues) - * by recursing into nested state after flushing each level. + * Flushes includes state using a bottom-up per-entry approach. + * Five phases ensure correct ordering: + * 1. Parent INSERTs — create child entries with per-entry nested states + * 2. Child changes — apply to child Collections, update routing index + * 3. Drain nested buffers — route buffered grandchild changes to per-entry states + * 4. Flush per-entry states — recursively flush nested includes on each entry + * 5. Parent DELETEs — clean up child entries and routing index */ function flushIncludesState( includesState: Array, @@ -1196,8 +1468,7 @@ function flushIncludesState( parentChanges: Map> | null, ): void { for (const state of includesState) { - // For parent INSERTs: ensure a child Collection exists for every parent, - // even those with no children (produces an empty child Collection). + // Phase 1: Parent INSERTs — ensure a child Collection exists for every parent if (parentChanges) { const fieldPath = state.correlationField.path.slice(1) // remove alias prefix for (const [parentKey, changes] of parentChanges) { @@ -1218,6 +1489,7 @@ function flushIncludesState( state.fieldName, correlationKey, state.hasOrderBy, + state.nestedSetups, ) state.childRegistry.set(correlationKey, entry) } @@ -1237,7 +1509,10 @@ function flushIncludesState( } } - // Flush child changes: route to correct child Collections + // Phase 2: Child changes — apply to child Collections + // Track which entries had child changes and capture their childChanges maps + const entriesWithChildChanges = new Map> }>() + if (state.pendingChildChanges.size > 0) { for (const [correlationKey, childChanges] of state.pendingChildChanges) { // Ensure child Collection exists for this correlation key @@ -1248,6 +1523,7 @@ function flushIncludesState( state.fieldName, correlationKey, state.hasOrderBy, + state.nestedSetups, ) state.childRegistry.set(correlationKey, entry) } @@ -1286,21 +1562,44 @@ function flushIncludesState( entry.syncMethods.commit() } - // Recursively process nested includes (e.g., comments inside issues) - if (state.nestedIncludesState) { - flushIncludesState( - state.nestedIncludesState, - entry.collection, - entry.collection.id, - childChanges, - ) - } + // Update routing index for nested includes + updateRoutingIndex(state, correlationKey, childChanges) + + entriesWithChildChanges.set(correlationKey, { entry, childChanges }) } state.pendingChildChanges.clear() } - // For parent DELETEs: dispose child Collections and clean up reverse index - // so re-added parents get a fresh empty child Collection instead of reusing stale data. + // Phase 3: Drain nested buffers — route buffered grandchild changes to per-entry states + const dirtyFromBuffers = drainNestedBuffers(state) + + // Phase 4: Flush per-entry states + // First: entries that had child changes in Phase 2 + for (const [, { entry, childChanges }] of entriesWithChildChanges) { + if (entry.includesStates) { + flushIncludesState( + entry.includesStates, + entry.collection, + entry.collection.id, + childChanges, + ) + } + } + // Then: entries that only had buffer-routed changes (no child changes at this level) + for (const correlationKey of dirtyFromBuffers) { + if (entriesWithChildChanges.has(correlationKey)) continue + const entry = state.childRegistry.get(correlationKey) + if (entry?.includesStates) { + flushIncludesState( + entry.includesStates, + entry.collection, + entry.collection.id, + null, + ) + } + } + + // Phase 5: Parent DELETEs — dispose child Collections and clean up if (parentChanges) { const fieldPath = state.correlationField.path.slice(1) for (const [parentKey, changes] of parentChanges) { @@ -1311,6 +1610,7 @@ function flushIncludesState( correlationKey = (correlationKey as any)[segment] } if (correlationKey != null) { + cleanRoutingIndexOnDelete(state, correlationKey) state.childRegistry.delete(correlationKey) // Clean up reverse index const parentKeys = state.correlationToParentKeys.get(correlationKey) @@ -1327,6 +1627,20 @@ function flushIncludesState( } } +/** + * Checks whether any includes state has pending changes that need to be flushed. + * Checks direct pending child changes and shared nested buffers. + */ +function hasPendingIncludesChanges( + states: Array, +): boolean { + for (const state of states) { + if (state.pendingChildChanges.size > 0) return true + if (state.nestedSetups && hasNestedBufferChanges(state.nestedSetups)) return true + } + return false +} + /** * Attaches a child Collection to parent rows that match a given correlation key. * Uses the reverse index to look up parent keys directly instead of scanning. From 50f9426ee3b6e98c982fd64febeb8df01d2f3043 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Wed, 25 Feb 2026 12:25:15 +0000 Subject: [PATCH 13/32] ci: apply automated fixes --- .../query/live/collection-config-builder.ts | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 4d9133324..cf43d88b2 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -1255,9 +1255,7 @@ function createPerEntryIncludesStates( * Drains shared buffers into per-entry states using the routing index. * Returns the set of parent correlation keys that had changes routed to them. */ -function drainNestedBuffers( - state: IncludesOutputState, -): Set { +function drainNestedBuffers(state: IncludesOutputState): Set { const dirtyCorrelationKeys = new Set() if (!state.nestedSetups) return dirtyCorrelationKeys @@ -1267,7 +1265,8 @@ function drainNestedBuffers( const toDelete: Array = [] for (const [nestedCorrelationKey, childChanges] of setup.buffer) { - const parentCorrelationKey = state.nestedRoutingIndex!.get(nestedCorrelationKey) + const parentCorrelationKey = + state.nestedRoutingIndex!.get(nestedCorrelationKey) if (parentCorrelationKey === undefined) { // Unroutable — parent not yet seen; keep in buffer continue @@ -1326,7 +1325,8 @@ function updateRoutingIndex( if (!state.nestedSetups) return for (const setup of state.nestedSetups) { - const nestedFieldPath = setup.compilationResult.correlationField.path.slice(1) + const nestedFieldPath = + setup.compilationResult.correlationField.path.slice(1) for (const [, change] of childChanges) { if (change.inserts > 0) { @@ -1356,7 +1356,8 @@ function updateRoutingIndex( if (nestedCorrelationKey != null) { state.nestedRoutingIndex!.delete(nestedCorrelationKey) - const reverseSet = state.nestedRoutingReverseIndex!.get(correlationKey) + const reverseSet = + state.nestedRoutingReverseIndex!.get(correlationKey) if (reverseSet) { reverseSet.delete(nestedCorrelationKey) if (reverseSet.size === 0) { @@ -1391,12 +1392,11 @@ function cleanRoutingIndexOnDelete( /** * Recursively checks whether any nested buffer has pending changes. */ -function hasNestedBufferChanges( - setups: Array, -): boolean { +function hasNestedBufferChanges(setups: Array): boolean { for (const setup of setups) { if (setup.buffer.size > 0) return true - if (setup.nestedSetups && hasNestedBufferChanges(setup.nestedSetups)) return true + if (setup.nestedSetups && hasNestedBufferChanges(setup.nestedSetups)) + return true } return false } @@ -1511,7 +1511,10 @@ function flushIncludesState( // Phase 2: Child changes — apply to child Collections // Track which entries had child changes and capture their childChanges maps - const entriesWithChildChanges = new Map> }>() + const entriesWithChildChanges = new Map< + unknown, + { entry: ChildCollectionEntry; childChanges: Map> } + >() if (state.pendingChildChanges.size > 0) { for (const [correlationKey, childChanges] of state.pendingChildChanges) { @@ -1636,7 +1639,8 @@ function hasPendingIncludesChanges( ): boolean { for (const state of states) { if (state.pendingChildChanges.size > 0) return true - if (state.nestedSetups && hasNestedBufferChanges(state.nestedSetups)) return true + if (state.nestedSetups && hasNestedBufferChanges(state.nestedSetups)) + return true } return false } From b90632927ba2c9cfd0451d2347440630c7fd01e9 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 26 Feb 2026 14:45:52 +0100 Subject: [PATCH 14/32] Prefix child collection names to avoid clashes --- packages/db/src/query/live/collection-config-builder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index cf43d88b2..2bdd49896 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -1421,7 +1421,7 @@ function createChildCollectionEntry( : undefined const collection = createCollection({ - id: `${parentId}-${fieldName}-${String(correlationKey)}`, + id: `__child-collection:${parentId}-${fieldName}-${String(correlationKey)}`, getKey: (item: any) => resultKeys.get(item) as string | number, compare, sync: { From 02d372b4890d2ee95ce47478ce5b4e8322e5624d Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 26 Feb 2026 15:30:21 +0100 Subject: [PATCH 15/32] Properly serialize correlation key before using it in collection ID --- packages/db/src/query/live/collection-config-builder.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 2bdd49896..e23fc665c 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -1,4 +1,4 @@ -import { D2, output } from '@tanstack/db-ivm' +import { D2, output, serializeValue } from '@tanstack/db-ivm' import { compileQuery } from '../compiler/index.js' import { createCollection } from '../../collection/index.js' import { @@ -1421,7 +1421,7 @@ function createChildCollectionEntry( : undefined const collection = createCollection({ - id: `__child-collection:${parentId}-${fieldName}-${String(correlationKey)}`, + id: `__child-collection:${parentId}-${fieldName}-${serializeValue(correlationKey)}`, getKey: (item: any) => resultKeys.get(item) as string | number, compare, sync: { From b7b755b970c59ffbd3d79b6cec31c8a7d6a21128 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 26 Feb 2026 15:46:51 +0100 Subject: [PATCH 16/32] Additional test as suggested by Codex review --- packages/db/tests/query/includes.test.ts | 136 +++++++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/packages/db/tests/query/includes.test.ts b/packages/db/tests/query/includes.test.ts index 07a76a5c8..ff37e863d 100644 --- a/packages/db/tests/query/includes.test.ts +++ b/packages/db/tests/query/includes.test.ts @@ -453,6 +453,142 @@ describe(`includes subqueries`, () => { }) }) + describe(`shared correlation key`, () => { + // Multiple parents share the same correlationKey value. + // e.g., two teams in the same department — both should see the same department members. + type Team = { id: number; name: string; departmentId: number } + type Member = { id: number; departmentId: number; name: string } + + const sampleTeams: Array = [ + { id: 1, name: `Frontend`, departmentId: 100 }, + { id: 2, name: `Backend`, departmentId: 100 }, + { id: 3, name: `Marketing`, departmentId: 200 }, + ] + + const sampleMembers: Array = [ + { id: 10, departmentId: 100, name: `Alice` }, + { id: 11, departmentId: 100, name: `Bob` }, + { id: 20, departmentId: 200, name: `Charlie` }, + ] + + function createTeamsCollection() { + return createCollection( + mockSyncCollectionOptions({ + id: `includes-teams`, + getKey: (t) => t.id, + initialData: sampleTeams, + }), + ) + } + + function createMembersCollection() { + return createCollection( + mockSyncCollectionOptions({ + id: `includes-members`, + getKey: (m) => m.id, + initialData: sampleMembers, + }), + ) + } + + it(`multiple parents with the same correlationKey each get the shared children`, async () => { + const teams = createTeamsCollection() + const members = createMembersCollection() + + const collection = createLiveQueryCollection((q) => + q.from({ t: teams }).select(({ t }) => ({ + id: t.id, + name: t.name, + departmentId: t.departmentId, + members: q + .from({ m: members }) + .where(({ m }) => eq(m.departmentId, t.departmentId)) + .select(({ m }) => ({ + id: m.id, + name: m.name, + })), + })), + ) + + await collection.preload() + + // Both Frontend and Backend teams share departmentId 100 + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Frontend`, + departmentId: 100, + members: [ + { id: 10, name: `Alice` }, + { id: 11, name: `Bob` }, + ], + }, + { + id: 2, + name: `Backend`, + departmentId: 100, + members: [ + { id: 10, name: `Alice` }, + { id: 11, name: `Bob` }, + ], + }, + { + id: 3, + name: `Marketing`, + departmentId: 200, + members: [{ id: 20, name: `Charlie` }], + }, + ]) + }) + + it(`adding a child updates all parents that share the correlation key`, async () => { + const teams = createTeamsCollection() + const members = createMembersCollection() + + const collection = createLiveQueryCollection((q) => + q.from({ t: teams }).select(({ t }) => ({ + id: t.id, + name: t.name, + departmentId: t.departmentId, + members: q + .from({ m: members }) + .where(({ m }) => eq(m.departmentId, t.departmentId)) + .select(({ m }) => ({ + id: m.id, + name: m.name, + })), + })), + ) + + await collection.preload() + + // Add a new member to department 100 + members.utils.begin() + members.utils.write({ + type: `insert`, + value: { id: 12, departmentId: 100, name: `Dave` }, + }) + members.utils.commit() + + // Both Frontend and Backend should see the new member + expect(childItems((collection.get(1) as any).members)).toEqual([ + { id: 10, name: `Alice` }, + { id: 11, name: `Bob` }, + { id: 12, name: `Dave` }, + ]) + expect(childItems((collection.get(2) as any).members)).toEqual([ + { id: 10, name: `Alice` }, + { id: 11, name: `Bob` }, + { id: 12, name: `Dave` }, + ]) + + // Marketing unaffected + expect(childItems((collection.get(3) as any).members)).toEqual([ + { id: 20, name: `Charlie` }, + ]) + }) + }) + describe(`nested includes`, () => { function buildNestedQuery() { return createLiveQueryCollection((q) => From d44bf04764f2210718628dc074644929849a2b81 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 26 Feb 2026 15:51:10 +0100 Subject: [PATCH 17/32] Unit test to ensure that correlation field does not need to be in the parent select --- packages/db/tests/query/includes.test.ts | 46 ++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/packages/db/tests/query/includes.test.ts b/packages/db/tests/query/includes.test.ts index ff37e863d..221ff6b0c 100644 --- a/packages/db/tests/query/includes.test.ts +++ b/packages/db/tests/query/includes.test.ts @@ -587,6 +587,52 @@ describe(`includes subqueries`, () => { { id: 20, name: `Charlie` }, ]) }) + + it(`correlation field does not need to be in the parent select`, async () => { + const teams = createTeamsCollection() + const members = createMembersCollection() + + // departmentId is used for correlation but NOT selected in the parent output + const collection = createLiveQueryCollection((q) => + q.from({ t: teams }).select(({ t }) => ({ + id: t.id, + name: t.name, + members: q + .from({ m: members }) + .where(({ m }) => eq(m.departmentId, t.departmentId)) + .select(({ m }) => ({ + id: m.id, + name: m.name, + })), + })), + ) + + await collection.preload() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Frontend`, + members: [ + { id: 10, name: `Alice` }, + { id: 11, name: `Bob` }, + ], + }, + { + id: 2, + name: `Backend`, + members: [ + { id: 10, name: `Alice` }, + { id: 11, name: `Bob` }, + ], + }, + { + id: 3, + name: `Marketing`, + members: [{ id: 20, name: `Charlie` }], + }, + ]) + }) }) describe(`nested includes`, () => { From 8c928aba102dd7b21c3b3f397713d8d8cc171cbf Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 26 Feb 2026 16:05:26 +0100 Subject: [PATCH 18/32] Stamp __includesCorrelationKeys on the result before output, and flushIncludesState reads from that stamp. The stamp is cleaned up at the end of flush so it never leaks to the user --- packages/db/src/query/compiler/index.ts | 29 +++++++++++ .../query/live/collection-config-builder.ts | 51 ++++++++----------- 2 files changed, 50 insertions(+), 30 deletions(-) diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index 1635280d2..ec8bd6c37 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -326,6 +326,25 @@ export function compileQuery( // Replace includes entry in select with a null placeholder replaceIncludesInSelect(query.select, key) } + + // Stamp correlation key values onto the namespaced row so they survive + // select extraction. This allows flushIncludesState to read them directly + // without requiring the correlation field to be in the user's select. + if (includesEntries.length > 0) { + const compiledCorrelations = includesEntries.map(({ subquery }) => ({ + fieldName: subquery.fieldName, + compiled: compileExpression(subquery.correlationField), + })) + pipeline = pipeline.pipe( + map(([key, nsRow]: any) => { + const correlationKeys: Record = {} + for (const { fieldName: fn, compiled } of compiledCorrelations) { + correlationKeys[fn] = compiled(nsRow) + } + return [key, { ...nsRow, __includesCorrelationKeys: correlationKeys }] + }), + ) + } } if (query.distinct && !query.fnSelect && !query.select) { @@ -463,6 +482,11 @@ export function compileQuery( // Extract the final results from $selected and include orderBy index const raw = (row as any).$selected const finalResults = unwrapValue(raw) + // Stamp includes correlation keys onto the result for child routing + if ((row as any).__includesCorrelationKeys) { + finalResults.__includesCorrelationKeys = + (row as any).__includesCorrelationKeys + } // When in includes mode, embed the correlation key as third element if (parentKeyStream) { const correlationKey = (row as any)[mainSource]?.__correlationKey @@ -496,6 +520,11 @@ export function compileQuery( // Extract the final results from $selected and return [key, [results, undefined]] const raw = (row as any).$selected const finalResults = unwrapValue(raw) + // Stamp includes correlation keys onto the result for child routing + if ((row as any).__includesCorrelationKeys) { + finalResults.__includesCorrelationKeys = + (row as any).__includesCorrelationKeys + } // When in includes mode, embed the correlation key as third element if (parentKeyStream) { const correlationKey = (row as any)[mainSource]?.__correlationKey diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index e23fc665c..36a2cb2eb 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -821,7 +821,6 @@ export class CollectionConfigBuilder< return includesEntries.map((entry) => { const state: IncludesOutputState = { fieldName: entry.fieldName, - correlationField: entry.correlationField, childCorrelationField: entry.childCorrelationField, hasOrderBy: entry.hasOrderBy, childRegistry: new Map(), @@ -1132,7 +1131,6 @@ type NestedIncludesSetup = { */ type IncludesOutputState = { fieldName: string - correlationField: PropRef childCorrelationField: PropRef /** Whether the child query has an ORDER BY clause */ hasOrderBy: boolean @@ -1233,7 +1231,6 @@ function createPerEntryIncludesStates( return setups.map((setup) => { const state: IncludesOutputState = { fieldName: setup.compilationResult.fieldName, - correlationField: setup.compilationResult.correlationField, childCorrelationField: setup.compilationResult.childCorrelationField, hasOrderBy: setup.compilationResult.hasOrderBy, childRegistry: new Map(), @@ -1325,17 +1322,13 @@ function updateRoutingIndex( if (!state.nestedSetups) return for (const setup of state.nestedSetups) { - const nestedFieldPath = - setup.compilationResult.correlationField.path.slice(1) - for (const [, change] of childChanges) { if (change.inserts > 0) { - // Extract nested correlation key from child result - let nestedCorrelationKey: unknown = change.value - for (const segment of nestedFieldPath) { - if (nestedCorrelationKey == null) break - nestedCorrelationKey = (nestedCorrelationKey as any)[segment] - } + // Read the pre-computed nested correlation key from the compiler stamp + const nestedCorrelationKey = + (change.value).__includesCorrelationKeys?.[ + setup.compilationResult.fieldName + ] if (nestedCorrelationKey != null) { state.nestedRoutingIndex!.set(nestedCorrelationKey, correlationKey) @@ -1348,11 +1341,10 @@ function updateRoutingIndex( } } else if (change.deletes > 0 && change.inserts === 0) { // Remove from routing index - let nestedCorrelationKey: unknown = change.value - for (const segment of nestedFieldPath) { - if (nestedCorrelationKey == null) break - nestedCorrelationKey = (nestedCorrelationKey as any)[segment] - } + const nestedCorrelationKey = + (change.value).__includesCorrelationKeys?.[ + setup.compilationResult.fieldName + ] if (nestedCorrelationKey != null) { state.nestedRoutingIndex!.delete(nestedCorrelationKey) @@ -1470,16 +1462,12 @@ function flushIncludesState( for (const state of includesState) { // Phase 1: Parent INSERTs — ensure a child Collection exists for every parent if (parentChanges) { - const fieldPath = state.correlationField.path.slice(1) // remove alias prefix for (const [parentKey, changes] of parentChanges) { if (changes.inserts > 0) { const parentResult = changes.value - // Extract the correlation key value from the parent result - let correlationKey: unknown = parentResult - for (const segment of fieldPath) { - if (correlationKey == null) break - correlationKey = (correlationKey as any)[segment] - } + // Read the pre-computed correlation key from the compiler stamp + const correlationKey = + (parentResult).__includesCorrelationKeys?.[state.fieldName] if (correlationKey != null) { // Ensure child Collection exists for this correlation key @@ -1604,14 +1592,10 @@ function flushIncludesState( // Phase 5: Parent DELETEs — dispose child Collections and clean up if (parentChanges) { - const fieldPath = state.correlationField.path.slice(1) for (const [parentKey, changes] of parentChanges) { if (changes.deletes > 0 && changes.inserts === 0) { - let correlationKey: unknown = changes.value - for (const segment of fieldPath) { - if (correlationKey == null) break - correlationKey = (correlationKey as any)[segment] - } + const correlationKey = + (changes.value).__includesCorrelationKeys?.[state.fieldName] if (correlationKey != null) { cleanRoutingIndexOnDelete(state, correlationKey) state.childRegistry.delete(correlationKey) @@ -1628,6 +1612,13 @@ function flushIncludesState( } } } + + // Clean up the internal stamp from parent/child results so it doesn't leak to the user + if (parentChanges) { + for (const [, changes] of parentChanges) { + delete (changes.value).__includesCorrelationKeys + } + } } /** From e284a353055e2850bea9a97404ae976f641d73ec Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Thu, 26 Feb 2026 15:06:38 +0000 Subject: [PATCH 19/32] ci: apply automated fixes --- packages/db/src/query/compiler/index.ts | 10 ++++++---- .../db/src/query/live/collection-config-builder.ts | 10 +++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index ec8bd6c37..f97a57697 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -484,8 +484,9 @@ export function compileQuery( const finalResults = unwrapValue(raw) // Stamp includes correlation keys onto the result for child routing if ((row as any).__includesCorrelationKeys) { - finalResults.__includesCorrelationKeys = - (row as any).__includesCorrelationKeys + finalResults.__includesCorrelationKeys = ( + row as any + ).__includesCorrelationKeys } // When in includes mode, embed the correlation key as third element if (parentKeyStream) { @@ -522,8 +523,9 @@ export function compileQuery( const finalResults = unwrapValue(raw) // Stamp includes correlation keys onto the result for child routing if ((row as any).__includesCorrelationKeys) { - finalResults.__includesCorrelationKeys = - (row as any).__includesCorrelationKeys + finalResults.__includesCorrelationKeys = ( + row as any + ).__includesCorrelationKeys } // When in includes mode, embed the correlation key as third element if (parentKeyStream) { diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 36a2cb2eb..07f8f40dc 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -1326,7 +1326,7 @@ function updateRoutingIndex( if (change.inserts > 0) { // Read the pre-computed nested correlation key from the compiler stamp const nestedCorrelationKey = - (change.value).__includesCorrelationKeys?.[ + change.value.__includesCorrelationKeys?.[ setup.compilationResult.fieldName ] @@ -1342,7 +1342,7 @@ function updateRoutingIndex( } else if (change.deletes > 0 && change.inserts === 0) { // Remove from routing index const nestedCorrelationKey = - (change.value).__includesCorrelationKeys?.[ + change.value.__includesCorrelationKeys?.[ setup.compilationResult.fieldName ] @@ -1467,7 +1467,7 @@ function flushIncludesState( const parentResult = changes.value // Read the pre-computed correlation key from the compiler stamp const correlationKey = - (parentResult).__includesCorrelationKeys?.[state.fieldName] + parentResult.__includesCorrelationKeys?.[state.fieldName] if (correlationKey != null) { // Ensure child Collection exists for this correlation key @@ -1595,7 +1595,7 @@ function flushIncludesState( for (const [parentKey, changes] of parentChanges) { if (changes.deletes > 0 && changes.inserts === 0) { const correlationKey = - (changes.value).__includesCorrelationKeys?.[state.fieldName] + changes.value.__includesCorrelationKeys?.[state.fieldName] if (correlationKey != null) { cleanRoutingIndexOnDelete(state, correlationKey) state.childRegistry.delete(correlationKey) @@ -1616,7 +1616,7 @@ function flushIncludesState( // Clean up the internal stamp from parent/child results so it doesn't leak to the user if (parentChanges) { for (const [, changes] of parentChanges) { - delete (changes.value).__includesCorrelationKeys + delete changes.value.__includesCorrelationKeys } } } From c6d27592abca06089bf369037ca5e1e376991752 Mon Sep 17 00:00:00 2001 From: Kevin Date: Thu, 12 Mar 2026 13:06:22 +0100 Subject: [PATCH 20/32] feat: add toArray() for includes subqueries (#1295) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add toArray() for includes subqueries toArray() wraps an includes subquery so the parent row contains Array instead of Collection. When children change, the parent row is re-emitted with a fresh array snapshot. - Add ToArrayWrapper class and toArray() function - Add materializeAsArray flag to IncludesSubquery IR node - Detect ToArrayWrapper in builder, pass flag through compiler - Re-emit parent rows on child changes for toArray entries - Add SelectValue type support for ToArrayWrapper - Add tests for basic toArray, reactivity, ordering, and limits Co-Authored-By: Claude Opus 4.6 * Removed obsolete test * Small fix * Tests for changes to deeply nested queries * Fix changes being emitted on deeply nested collections * ci: apply automated fixes * Changeset * Add type-level tests for toArray() includes subqueries Co-Authored-By: Claude Opus 4.6 * ci: apply automated fixes * Rename Expected types in includes type tests to descriptive names Co-Authored-By: Claude Opus 4.6 * Fix toArray() type inference in includes subqueries Make ToArrayWrapper generic so it carries the child query result type, and add a ToArrayWrapper branch in ResultTypeFromSelect to unwrap it to Array. Co-Authored-By: Claude Opus 4.6 * Fix toArray re-emit to emit change events for subscribers The toArray re-emit in flushIncludesState mutated parent items in-place before writing them through parentSyncMethods.begin/write/commit. Since commitPendingTransactions captures "previous visible state" by reading syncedData.get(key) — which returns the already-mutated object — deepEquals always returned true and suppressed the change event. Replace the sync methods pattern with direct event emission: capture a shallow copy before mutation (for previousValue), mutate in-place (so collection.get() works), and emit UPDATE events directly via the parent collection's changes manager. Co-Authored-By: Claude Opus 4.6 * Add change propagation tests for includes subqueries Test the reactive model difference between Collection and toArray includes: - Collection includes: child change does NOT re-emit the parent row (the child Collection updates in place) - toArray includes: child change DOES re-emit the parent row (the parent row is re-emitted with the updated array snapshot) Co-Authored-By: Claude Opus 4.6 * ci: apply automated fixes --------- Co-authored-by: Claude Opus 4.6 Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- .changeset/includes-to-array.md | 5 + packages/db/src/query/builder/functions.ts | 14 +- packages/db/src/query/builder/index.ts | 19 +- packages/db/src/query/builder/types.ts | 72 +- packages/db/src/query/compiler/index.ts | 3 + packages/db/src/query/index.ts | 2 + packages/db/src/query/ir.ts | 1 + .../query/live/collection-config-builder.ts | 92 +- packages/db/tests/query/includes.test-d.ts | 153 +++ packages/db/tests/query/includes.test.ts | 1056 ++++++++++++++++- 10 files changed, 1357 insertions(+), 60 deletions(-) create mode 100644 .changeset/includes-to-array.md create mode 100644 packages/db/tests/query/includes.test-d.ts diff --git a/.changeset/includes-to-array.md b/.changeset/includes-to-array.md new file mode 100644 index 000000000..fd59eaea3 --- /dev/null +++ b/.changeset/includes-to-array.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +feat: add `toArray()` wrapper for includes subqueries to materialize child results as plain arrays instead of live Collections diff --git a/packages/db/src/query/builder/functions.ts b/packages/db/src/query/builder/functions.ts index 887c1468d..82b806192 100644 --- a/packages/db/src/query/builder/functions.ts +++ b/packages/db/src/query/builder/functions.ts @@ -2,7 +2,8 @@ import { Aggregate, Func } from '../ir' import { toExpression } from './ref-proxy.js' import type { BasicExpression } from '../ir' import type { RefProxy } from './ref-proxy.js' -import type { RefLeaf } from './types.js' +import type { Context, GetResult, RefLeaf } from './types.js' +import type { QueryBuilder } from './index.js' type StringRef = | RefLeaf @@ -376,3 +377,14 @@ export const operators = [ ] as const export type OperatorName = (typeof operators)[number] + +export class ToArrayWrapper { + declare readonly _type: T + constructor(public readonly query: QueryBuilder) {} +} + +export function toArray( + query: QueryBuilder, +): ToArrayWrapper> { + return new ToArrayWrapper(query) +} diff --git a/packages/db/src/query/builder/index.ts b/packages/db/src/query/builder/index.ts index a3bdf1340..7351cbfa5 100644 --- a/packages/db/src/query/builder/index.ts +++ b/packages/db/src/query/builder/index.ts @@ -24,6 +24,7 @@ import { isRefProxy, toExpression, } from './ref-proxy.js' +import { ToArrayWrapper } from './functions.js' import type { NamespacedRow, SingleResult } from '../../types.js' import type { Aggregate, @@ -878,7 +879,14 @@ function buildNestedSelect(obj: any, parentAliases: Array = []): any { continue } if (v instanceof BaseQueryBuilder) { - out[k] = buildIncludesSubquery(v, k, parentAliases) + out[k] = buildIncludesSubquery(v, k, parentAliases, false) + continue + } + if (v instanceof ToArrayWrapper) { + if (!(v.query instanceof BaseQueryBuilder)) { + throw new Error(`toArray() must wrap a subquery builder`) + } + out[k] = buildIncludesSubquery(v.query, k, parentAliases, true) continue } out[k] = buildNestedSelect(v, parentAliases) @@ -895,6 +903,7 @@ function buildIncludesSubquery( childBuilder: BaseQueryBuilder, fieldName: string, parentAliases: Array, + materializeAsArray: boolean, ): IncludesSubquery { const childQuery = childBuilder._getQuery() @@ -958,7 +967,13 @@ function buildIncludesSubquery( where: modifiedWhere.length > 0 ? modifiedWhere : undefined, } - return new IncludesSubquery(modifiedQuery, parentRef, childRef, fieldName) + return new IncludesSubquery( + modifiedQuery, + parentRef, + childRef, + fieldName, + materializeAsArray, + ) } /** diff --git a/packages/db/src/query/builder/types.ts b/packages/db/src/query/builder/types.ts index 6dce531f8..8877b696f 100644 --- a/packages/db/src/query/builder/types.ts +++ b/packages/db/src/query/builder/types.ts @@ -9,6 +9,7 @@ import type { Value, } from '../ir.js' import type { QueryBuilder } from './index.js' +import type { ToArrayWrapper } from './functions.js' /** * Context - The central state container for query builder operations @@ -174,6 +175,7 @@ type SelectValue = | undefined // Optional values | { [key: string]: SelectValue } | Array> + | ToArrayWrapper // toArray() wrapped subquery // Recursive shape for select objects allowing nested projections type SelectShape = { [key: string]: SelectValue | SelectShape } @@ -227,40 +229,42 @@ export type ResultTypeFromSelect = WithoutRefBrand< Prettify<{ [K in keyof TSelectObject]: NeedsExtraction extends true ? ExtractExpressionType - : // Ref (full object ref or spread with RefBrand) - recursively process properties - TSelectObject[K] extends Ref - ? ExtractRef - : // RefLeaf (simple property ref like user.name) - TSelectObject[K] extends RefLeaf - ? IsNullableRef extends true - ? T | undefined - : T - : // RefLeaf | undefined (schema-optional field) - TSelectObject[K] extends RefLeaf | undefined - ? T | undefined - : // RefLeaf | null (schema-nullable field) - TSelectObject[K] extends RefLeaf | null - ? IsNullableRef> extends true - ? T | null | undefined - : T | null - : // Ref | undefined (optional object-type schema field) - TSelectObject[K] extends Ref | undefined - ? ExtractRef> | undefined - : // Ref | null (nullable object-type schema field) - TSelectObject[K] extends Ref | null - ? ExtractRef> | null - : TSelectObject[K] extends Aggregate - ? T - : TSelectObject[K] extends - | string - | number - | boolean - | null - | undefined - ? TSelectObject[K] - : TSelectObject[K] extends Record - ? ResultTypeFromSelect - : never + : TSelectObject[K] extends ToArrayWrapper + ? Array + : // Ref (full object ref or spread with RefBrand) - recursively process properties + TSelectObject[K] extends Ref + ? ExtractRef + : // RefLeaf (simple property ref like user.name) + TSelectObject[K] extends RefLeaf + ? IsNullableRef extends true + ? T | undefined + : T + : // RefLeaf | undefined (schema-optional field) + TSelectObject[K] extends RefLeaf | undefined + ? T | undefined + : // RefLeaf | null (schema-nullable field) + TSelectObject[K] extends RefLeaf | null + ? IsNullableRef> extends true + ? T | null | undefined + : T | null + : // Ref | undefined (optional object-type schema field) + TSelectObject[K] extends Ref | undefined + ? ExtractRef> | undefined + : // Ref | null (nullable object-type schema field) + TSelectObject[K] extends Ref | null + ? ExtractRef> | null + : TSelectObject[K] extends Aggregate + ? T + : TSelectObject[K] extends + | string + | number + | boolean + | null + | undefined + ? TSelectObject[K] + : TSelectObject[K] extends Record + ? ResultTypeFromSelect + : never }> > diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index f97a57697..8dc030387 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -56,6 +56,8 @@ export interface IncludesCompilationResult { hasOrderBy: boolean /** Full compilation result for the child query (for nested includes + alias tracking) */ childCompilationResult: CompilationResult + /** When true, the output layer materializes children as Array instead of Collection */ + materializeAsArray: boolean } /** @@ -321,6 +323,7 @@ export function compileQuery( subquery.query.orderBy && subquery.query.orderBy.length > 0 ), childCompilationResult: childResult, + materializeAsArray: subquery.materializeAsArray, }) // Replace includes entry in select with a null placeholder diff --git a/packages/db/src/query/index.ts b/packages/db/src/query/index.ts index bceb1648d..3bc1c32c8 100644 --- a/packages/db/src/query/index.ts +++ b/packages/db/src/query/index.ts @@ -60,6 +60,8 @@ export { sum, min, max, + // Includes helpers + toArray, } from './builder/functions.js' // Ref proxy utilities diff --git a/packages/db/src/query/ir.ts b/packages/db/src/query/ir.ts index 64ddd22c7..04c28afd7 100644 --- a/packages/db/src/query/ir.ts +++ b/packages/db/src/query/ir.ts @@ -139,6 +139,7 @@ export class IncludesSubquery extends BaseExpression { public correlationField: PropRef, // Parent-side ref (e.g., project.id) public childCorrelationField: PropRef, // Child-side ref (e.g., issue.projectId) public fieldName: string, // Result field name (e.g., "issues") + public materializeAsArray: boolean = false, // When true, parent gets Array instead of Collection ) { super() } diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 07f8f40dc..f0f89ad5d 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -27,6 +27,7 @@ import type { RootStreamBuilder } from '@tanstack/db-ivm' import type { OrderByOptimizationInfo } from '../compiler/order-by.js' import type { Collection } from '../../collection/index.js' import type { + ChangeMessage, CollectionConfigSingleRowOption, KeyedStream, ResultStream, @@ -792,6 +793,7 @@ export class CollectionConfigBuilder< config.collection, this.id, hasParentChanges ? changesToApply : null, + config, ) } @@ -823,6 +825,7 @@ export class CollectionConfigBuilder< fieldName: entry.fieldName, childCorrelationField: entry.childCorrelationField, hasOrderBy: entry.hasOrderBy, + materializeAsArray: entry.materializeAsArray, childRegistry: new Map(), pendingChildChanges: new Map(), correlationToParentKeys: new Map(), @@ -1134,6 +1137,8 @@ type IncludesOutputState = { childCorrelationField: PropRef /** Whether the child query has an ORDER BY clause */ hasOrderBy: boolean + /** When true, parent gets Array instead of Collection */ + materializeAsArray: boolean /** Maps correlation key value → child Collection entry */ childRegistry: Map /** Pending child changes: correlationKey → Map */ @@ -1233,6 +1238,7 @@ function createPerEntryIncludesStates( fieldName: setup.compilationResult.fieldName, childCorrelationField: setup.compilationResult.childCorrelationField, hasOrderBy: setup.compilationResult.hasOrderBy, + materializeAsArray: setup.compilationResult.materializeAsArray, childRegistry: new Map(), pendingChildChanges: new Map(), correlationToParentKeys: new Map(), @@ -1458,6 +1464,7 @@ function flushIncludesState( parentCollection: Collection, parentId: string, parentChanges: Map> | null, + parentSyncMethods: SyncMethods | null, ): void { for (const state of includesState) { // Phase 1: Parent INSERTs — ensure a child Collection exists for every parent @@ -1489,21 +1496,31 @@ function flushIncludesState( } parentKeys.add(parentKey) - // Attach child Collection to the parent result - parentResult[state.fieldName] = - state.childRegistry.get(correlationKey)!.collection + // Attach child Collection (or array snapshot for toArray) to the parent result + if (state.materializeAsArray) { + parentResult[state.fieldName] = [ + ...state.childRegistry.get(correlationKey)!.collection.toArray, + ] + } else { + parentResult[state.fieldName] = + state.childRegistry.get(correlationKey)!.collection + } } } } } + // Track affected correlation keys for toArray re-emit (before clearing pendingChildChanges) + const affectedCorrelationKeys = state.materializeAsArray + ? new Set(state.pendingChildChanges.keys()) + : null + // Phase 2: Child changes — apply to child Collections // Track which entries had child changes and capture their childChanges maps const entriesWithChildChanges = new Map< unknown, { entry: ChildCollectionEntry; childChanges: Map> } >() - if (state.pendingChildChanges.size > 0) { for (const [correlationKey, childChanges] of state.pendingChildChanges) { // Ensure child Collection exists for this correlation key @@ -1519,14 +1536,17 @@ function flushIncludesState( state.childRegistry.set(correlationKey, entry) } - // Attach the child Collection to ANY parent that has this correlation key - attachChildCollectionToParent( - parentCollection, - state.fieldName, - correlationKey, - state.correlationToParentKeys, - entry.collection, - ) + // For non-toArray: attach the child Collection to ANY parent that has this correlation key + // For toArray: skip — the array snapshot is set during re-emit below + if (!state.materializeAsArray) { + attachChildCollectionToParent( + parentCollection, + state.fieldName, + correlationKey, + state.correlationToParentKeys, + entry.collection, + ) + } // Apply child changes to the child Collection if (entry.syncMethods) { @@ -1573,6 +1593,7 @@ function flushIncludesState( entry.collection, entry.collection.id, childChanges, + entry.syncMethods, ) } } @@ -1586,10 +1607,57 @@ function flushIncludesState( entry.collection, entry.collection.id, null, + entry.syncMethods, ) } } + // For toArray entries: re-emit affected parents with updated array snapshots. + // We mutate items in-place (so collection.get() reflects changes immediately) + // and emit UPDATE events directly. We bypass the sync methods because + // commitPendingTransactions compares previous vs new visible state using + // deepEquals, but in-place mutation means both sides reference the same + // object, so the comparison always returns true and suppresses the event. + const toArrayReEmitKeys = state.materializeAsArray + ? new Set([...(affectedCorrelationKeys || []), ...dirtyFromBuffers]) + : null + if (parentSyncMethods && toArrayReEmitKeys && toArrayReEmitKeys.size > 0) { + const events: Array> = [] + for (const correlationKey of toArrayReEmitKeys) { + const parentKeys = state.correlationToParentKeys.get(correlationKey) + if (!parentKeys) continue + const entry = state.childRegistry.get(correlationKey) + for (const parentKey of parentKeys) { + const item = parentCollection.get(parentKey as any) + if (item) { + const key = parentSyncMethods.collection.getKeyFromItem(item) + // Capture previous value before in-place mutation + const previousValue = { ...item } + if (entry) { + item[state.fieldName] = [...entry.collection.toArray] + } + events.push({ + type: `update`, + key, + value: item, + previousValue, + }) + } + } + } + if (events.length > 0) { + // Emit directly — the in-place mutation already updated the data in + // syncedData, so we only need to notify subscribers. + const changesManager = (parentCollection as any)._changes as { + emitEvents: ( + changes: Array>, + forceEmit?: boolean, + ) => void + } + changesManager.emitEvents(events, true) + } + } + // Phase 5: Parent DELETEs — dispose child Collections and clean up if (parentChanges) { for (const [parentKey, changes] of parentChanges) { diff --git a/packages/db/tests/query/includes.test-d.ts b/packages/db/tests/query/includes.test-d.ts new file mode 100644 index 000000000..dbcdb4a45 --- /dev/null +++ b/packages/db/tests/query/includes.test-d.ts @@ -0,0 +1,153 @@ +import { describe, expectTypeOf, test } from 'vitest' +import { + createLiveQueryCollection, + eq, + toArray, +} from '../../src/query/index.js' +import { createCollection } from '../../src/collection/index.js' +import { mockSyncCollectionOptions } from '../utils.js' + +type Project = { + id: number + name: string +} + +type Issue = { + id: number + projectId: number + title: string +} + +type Comment = { + id: number + issueId: number + body: string +} + +function createProjectsCollection() { + return createCollection( + mockSyncCollectionOptions({ + id: `includes-type-projects`, + getKey: (p) => p.id, + initialData: [], + }), + ) +} + +function createIssuesCollection() { + return createCollection( + mockSyncCollectionOptions({ + id: `includes-type-issues`, + getKey: (i) => i.id, + initialData: [], + }), + ) +} + +function createCommentsCollection() { + return createCollection( + mockSyncCollectionOptions({ + id: `includes-type-comments`, + getKey: (c) => c.id, + initialData: [], + }), + ) +} + +describe(`includes subquery types`, () => { + const projects = createProjectsCollection() + const issues = createIssuesCollection() + const comments = createCommentsCollection() + + describe(`toArray`, () => { + test(`toArray includes infers child result as Array`, () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: toArray( + q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + ), + })), + ) + + type ProjectWithIssueArray = { + id: number + name: string + issues: Array<{ + id: number + title: string + }> + } + + const result = collection.toArray[0]! + expectTypeOf(result).toEqualTypeOf() + }) + + test(`toArray includes without select infers child type`, () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + issues: toArray( + q.from({ i: issues }).where(({ i }) => eq(i.projectId, p.id)), + ), + })), + ) + + type ProjectWithFullIssueArray = { + id: number + issues: Array + } + + const result = collection.toArray[0]! + expectTypeOf(result).toEqualTypeOf() + }) + + test(`nested toArray infers nested arrays`, () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + issues: toArray( + q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + comments: toArray( + q + .from({ c: comments }) + .where(({ c }) => eq(c.issueId, i.id)) + .select(({ c }) => ({ + id: c.id, + body: c.body, + })), + ), + })), + ), + })), + ) + + type ProjectWithNestedArrays = { + id: number + issues: Array<{ + id: number + title: string + comments: Array<{ + id: number + body: string + }> + }> + } + + const result = collection.toArray[0]! + expectTypeOf(result).toEqualTypeOf() + }) + }) +}) diff --git a/packages/db/tests/query/includes.test.ts b/packages/db/tests/query/includes.test.ts index 221ff6b0c..9c2816067 100644 --- a/packages/db/tests/query/includes.test.ts +++ b/packages/db/tests/query/includes.test.ts @@ -1,5 +1,9 @@ -import { beforeEach, describe, expect, it } from 'vitest' -import { createLiveQueryCollection, eq } from '../../src/query/index.js' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { + createLiveQueryCollection, + eq, + toArray, +} from '../../src/query/index.js' import { createCollection } from '../../src/collection/index.js' import { mockSyncCollectionOptions } from '../utils.js' @@ -84,17 +88,27 @@ function childItems(collection: any, sortKey = `id`): Array { * plain sorted array, turning any nested child Collections into nested arrays. * This lets tests compare the full hierarchical result as a single literal. */ -function toTree(collection: any, sortKey = `id`): Array { - const rows = [...collection.toArray].sort( - (a: any, b: any) => a[sortKey] - b[sortKey], - ) +function toTree(collectionOrArray: any, sortKey = `id`): Array { + const rows = ( + Array.isArray(collectionOrArray) + ? [...collectionOrArray] + : [...collectionOrArray.toArray] + ).sort((a: any, b: any) => a[sortKey] - b[sortKey]) return rows.map((row: any) => { + if (typeof row !== `object` || row === null) return row const out: Record = {} for (const [key, value] of Object.entries(row)) { - out[key] = - value && typeof value === `object` && `toArray` in (value as any) - ? toTree(value, sortKey) - : value + if (Array.isArray(value)) { + out[key] = toTree(value, sortKey) + } else if ( + value && + typeof value === `object` && + `toArray` in (value as any) + ) { + out[key] = toTree(value, sortKey) + } else { + out[key] = value + } } return out }) @@ -259,6 +273,93 @@ describe(`includes subqueries`, () => { }) }) + describe(`change propagation`, () => { + it(`Collection includes: child change does not re-emit the parent row`, async () => { + const collection = buildIncludesQuery() + await collection.preload() + + const changeCallback = vi.fn() + const subscription = collection.subscribeChanges(changeCallback, { + includeInitialState: false, + }) + changeCallback.mockClear() + + // Add a child issue to project Alpha + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 12, projectId: 1, title: `New Alpha issue` }, + }) + issues.utils.commit() + + // Wait for async change propagation + await new Promise((resolve) => setTimeout(resolve, 10)) + + // The child Collection updates in place — the parent row should NOT be re-emitted + expect(changeCallback).not.toHaveBeenCalled() + + // But the child data is there + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 10, title: `Bug in Alpha` }, + { id: 11, title: `Feature for Alpha` }, + { id: 12, title: `New Alpha issue` }, + ]) + + subscription.unsubscribe() + }) + + it(`toArray includes: child change re-emits the parent row`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: toArray( + q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + ), + })), + ) + + await collection.preload() + + const changeCallback = vi.fn() + const subscription = collection.subscribeChanges(changeCallback, { + includeInitialState: false, + }) + changeCallback.mockClear() + + // Add a child issue to project Alpha + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 12, projectId: 1, title: `New Alpha issue` }, + }) + issues.utils.commit() + + // Wait for async change propagation + await new Promise((resolve) => setTimeout(resolve, 10)) + + // The parent row SHOULD be re-emitted with the updated array + expect(changeCallback).toHaveBeenCalled() + + // Verify the parent row has the updated array + const alpha = collection.get(1) as any + expect(Array.isArray(alpha.issues)).toBe(true) + expect(alpha.issues.sort((a: any, b: any) => a.id - b.id)).toEqual([ + { id: 10, title: `Bug in Alpha` }, + { id: 11, title: `Feature for Alpha` }, + { id: 12, title: `New Alpha issue` }, + ]) + + subscription.unsubscribe() + }) + }) + describe(`inner join filtering`, () => { it(`only shows children for parents matching a WHERE clause`, async () => { const collection = createLiveQueryCollection((q) => @@ -635,7 +736,14 @@ describe(`includes subqueries`, () => { }) }) - describe(`nested includes`, () => { + // Nested includes: two-level parent → child → grandchild (Project → Issue → Comment). + // Each level (Issue/Comment) can be materialized as a live Collection or a plain array (via toArray). + // We test all four combinations: + // Collection → Collection — both levels are live Collections + // Collection → toArray — issues are Collections, comments are arrays + // toArray → Collection — issues are arrays, comments are Collections + // toArray → toArray — both levels are plain arrays + describe(`nested includes: Collection → Collection`, () => { function buildNestedQuery() { return createLiveQueryCollection((q) => q.from({ p: projects }).select(({ p }) => ({ @@ -746,5 +854,931 @@ describe(`includes subqueries`, () => { { id: 101, body: `Fixed it` }, ]) }) + + it(`adding an issue (middle-level insert) creates a child with empty comments`, async () => { + const collection = buildNestedQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 30, projectId: 3, title: `Gamma issue` }, + }) + issues.utils.commit() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { + id: 10, + title: `Bug in Alpha`, + comments: [ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ], + }, + { id: 11, title: `Feature for Alpha`, comments: [] }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [ + { + id: 20, + title: `Bug in Beta`, + comments: [{ id: 200, body: `Same bug` }], + }, + ], + }, + { + id: 3, + name: `Gamma`, + issues: [{ id: 30, title: `Gamma issue`, comments: [] }], + }, + ]) + }) + + it(`removing an issue (middle-level delete) removes it from the parent`, async () => { + const collection = buildNestedQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `delete`, + value: sampleIssues.find((i) => i.id === 11)!, + }) + issues.utils.commit() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { + id: 10, + title: `Bug in Alpha`, + comments: [ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ], + }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [ + { + id: 20, + title: `Bug in Beta`, + comments: [{ id: 200, body: `Same bug` }], + }, + ], + }, + { + id: 3, + name: `Gamma`, + issues: [], + }, + ]) + }) + + it(`updating an issue title (middle-level update) reflects in the parent`, async () => { + const collection = buildNestedQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `update`, + value: { id: 10, projectId: 1, title: `Renamed Bug` }, + }) + issues.utils.commit() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { + id: 10, + title: `Renamed Bug`, + comments: [ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ], + }, + { id: 11, title: `Feature for Alpha`, comments: [] }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [ + { + id: 20, + title: `Bug in Beta`, + comments: [{ id: 200, body: `Same bug` }], + }, + ], + }, + { + id: 3, + name: `Gamma`, + issues: [], + }, + ]) + }) + }) + + describe(`toArray`, () => { + function buildToArrayQuery() { + return createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: toArray( + q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + ), + })), + ) + } + + it(`produces arrays on parent rows, not Collections`, async () => { + const collection = buildToArrayQuery() + await collection.preload() + + const alpha = collection.get(1) as any + expect(Array.isArray(alpha.issues)).toBe(true) + expect(alpha.issues.sort((a: any, b: any) => a.id - b.id)).toEqual([ + { id: 10, title: `Bug in Alpha` }, + { id: 11, title: `Feature for Alpha` }, + ]) + + const beta = collection.get(2) as any + expect(Array.isArray(beta.issues)).toBe(true) + expect(beta.issues).toEqual([{ id: 20, title: `Bug in Beta` }]) + }) + + it(`empty parents get empty arrays`, async () => { + const collection = buildToArrayQuery() + await collection.preload() + + const gamma = collection.get(3) as any + expect(Array.isArray(gamma.issues)).toBe(true) + expect(gamma.issues).toEqual([]) + }) + + it(`adding a child re-emits the parent with updated array`, async () => { + const collection = buildToArrayQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 12, projectId: 1, title: `New Alpha issue` }, + }) + issues.utils.commit() + + const alpha = collection.get(1) as any + expect(Array.isArray(alpha.issues)).toBe(true) + expect(alpha.issues.sort((a: any, b: any) => a.id - b.id)).toEqual([ + { id: 10, title: `Bug in Alpha` }, + { id: 11, title: `Feature for Alpha` }, + { id: 12, title: `New Alpha issue` }, + ]) + }) + + it(`removing a child re-emits the parent with updated array`, async () => { + const collection = buildToArrayQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `delete`, + value: sampleIssues.find((i) => i.id === 10)!, + }) + issues.utils.commit() + + const alpha = collection.get(1) as any + expect(alpha.issues).toEqual([{ id: 11, title: `Feature for Alpha` }]) + }) + + it(`array respects ORDER BY`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: toArray( + q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .orderBy(({ i }) => i.title, `asc`) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + ), + })), + ) + + await collection.preload() + + const alpha = collection.get(1) as any + expect(alpha.issues).toEqual([ + { id: 10, title: `Bug in Alpha` }, + { id: 11, title: `Feature for Alpha` }, + ]) + }) + + it(`ordered toArray with limit applied per parent`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: toArray( + q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .orderBy(({ i }) => i.title, `asc`) + .limit(1) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + ), + })), + ) + + await collection.preload() + + const alpha = collection.get(1) as any + expect(alpha.issues).toEqual([{ id: 10, title: `Bug in Alpha` }]) + + const beta = collection.get(2) as any + expect(beta.issues).toEqual([{ id: 20, title: `Bug in Beta` }]) + + const gamma = collection.get(3) as any + expect(gamma.issues).toEqual([]) + }) + }) + + describe(`nested includes: Collection → toArray`, () => { + function buildCollectionToArrayQuery() { + return createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + comments: toArray( + q + .from({ c: comments }) + .where(({ c }) => eq(c.issueId, i.id)) + .select(({ c }) => ({ + id: c.id, + body: c.body, + })), + ), + })), + })), + ) + } + + it(`initial load: issues are Collections, comments are arrays`, async () => { + const collection = buildCollectionToArrayQuery() + await collection.preload() + + const alpha = collection.get(1) as any + // issues should be a Collection + expect(alpha.issues.toArray).toBeDefined() + + const issue10 = alpha.issues.get(10) + // comments should be an array + expect(Array.isArray(issue10.comments)).toBe(true) + expect(issue10.comments.sort((a: any, b: any) => a.id - b.id)).toEqual([ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ]) + + const issue11 = alpha.issues.get(11) + expect(Array.isArray(issue11.comments)).toBe(true) + expect(issue11.comments).toEqual([]) + }) + + it(`adding a comment (grandchild-only change) updates the issue's comments array`, async () => { + const collection = buildCollectionToArrayQuery() + await collection.preload() + + const issue11Before = (collection.get(1) as any).issues.get(11) + expect(issue11Before.comments).toEqual([]) + + comments.utils.begin() + comments.utils.write({ + type: `insert`, + value: { id: 110, issueId: 11, body: `Great feature` }, + }) + comments.utils.commit() + + const issue11After = (collection.get(1) as any).issues.get(11) + expect(Array.isArray(issue11After.comments)).toBe(true) + expect(issue11After.comments).toEqual([ + { id: 110, body: `Great feature` }, + ]) + }) + + it(`removing a comment (grandchild-only change) updates the issue's comments array`, async () => { + const collection = buildCollectionToArrayQuery() + await collection.preload() + + const issue10Before = (collection.get(1) as any).issues.get(10) + expect(issue10Before.comments).toHaveLength(2) + + comments.utils.begin() + comments.utils.write({ + type: `delete`, + value: sampleComments.find((c) => c.id === 100)!, + }) + comments.utils.commit() + + const issue10After = (collection.get(1) as any).issues.get(10) + expect(issue10After.comments).toEqual([{ id: 101, body: `Fixed it` }]) + }) + + it(`adding an issue (middle-level insert) creates a child with empty comments array`, async () => { + const collection = buildCollectionToArrayQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 30, projectId: 3, title: `Gamma issue` }, + }) + issues.utils.commit() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { + id: 10, + title: `Bug in Alpha`, + comments: [ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ], + }, + { id: 11, title: `Feature for Alpha`, comments: [] }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [ + { + id: 20, + title: `Bug in Beta`, + comments: [{ id: 200, body: `Same bug` }], + }, + ], + }, + { + id: 3, + name: `Gamma`, + issues: [{ id: 30, title: `Gamma issue`, comments: [] }], + }, + ]) + }) + + it(`removing an issue (middle-level delete) removes it from the parent`, async () => { + const collection = buildCollectionToArrayQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `delete`, + value: sampleIssues.find((i) => i.id === 11)!, + }) + issues.utils.commit() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { + id: 10, + title: `Bug in Alpha`, + comments: [ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ], + }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [ + { + id: 20, + title: `Bug in Beta`, + comments: [{ id: 200, body: `Same bug` }], + }, + ], + }, + { + id: 3, + name: `Gamma`, + issues: [], + }, + ]) + }) + + it(`updating an issue title (middle-level update) reflects in the parent`, async () => { + const collection = buildCollectionToArrayQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `update`, + value: { id: 10, projectId: 1, title: `Renamed Bug` }, + }) + issues.utils.commit() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { + id: 10, + title: `Renamed Bug`, + comments: [ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ], + }, + { id: 11, title: `Feature for Alpha`, comments: [] }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [ + { + id: 20, + title: `Bug in Beta`, + comments: [{ id: 200, body: `Same bug` }], + }, + ], + }, + { + id: 3, + name: `Gamma`, + issues: [], + }, + ]) + }) + }) + + describe(`nested includes: toArray → Collection`, () => { + function buildToArrayToCollectionQuery() { + return createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: toArray( + q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + comments: q + .from({ c: comments }) + .where(({ c }) => eq(c.issueId, i.id)) + .select(({ c }) => ({ + id: c.id, + body: c.body, + })), + })), + ), + })), + ) + } + + it(`initial load: issues are arrays, comments are Collections`, async () => { + const collection = buildToArrayToCollectionQuery() + await collection.preload() + + const alpha = collection.get(1) as any + expect(Array.isArray(alpha.issues)).toBe(true) + + const sortedIssues = alpha.issues.sort((a: any, b: any) => a.id - b.id) + // comments should be Collections + expect(sortedIssues[0].comments.toArray).toBeDefined() + expect(childItems(sortedIssues[0].comments)).toEqual([ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ]) + + expect(sortedIssues[1].comments.toArray).toBeDefined() + expect(childItems(sortedIssues[1].comments)).toEqual([]) + }) + + it(`adding a comment updates the nested Collection (live reference)`, async () => { + const collection = buildToArrayToCollectionQuery() + await collection.preload() + + const alpha = collection.get(1) as any + const issue11 = alpha.issues.find((i: any) => i.id === 11) + expect(childItems(issue11.comments)).toEqual([]) + + comments.utils.begin() + comments.utils.write({ + type: `insert`, + value: { id: 110, issueId: 11, body: `Great feature` }, + }) + comments.utils.commit() + + // The Collection reference on the issue object is live + expect(childItems(issue11.comments)).toEqual([ + { id: 110, body: `Great feature` }, + ]) + }) + + it(`adding an issue re-emits the parent with updated array including nested Collection`, async () => { + const collection = buildToArrayToCollectionQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 30, projectId: 3, title: `Gamma issue` }, + }) + issues.utils.commit() + + const gamma = collection.get(3) as any + expect(Array.isArray(gamma.issues)).toBe(true) + expect(gamma.issues).toHaveLength(1) + expect(gamma.issues[0].id).toBe(30) + expect(gamma.issues[0].comments.toArray).toBeDefined() + expect(childItems(gamma.issues[0].comments)).toEqual([]) + }) + + it(`removing an issue re-emits the parent with updated array`, async () => { + const collection = buildToArrayToCollectionQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `delete`, + value: sampleIssues.find((i) => i.id === 11)!, + }) + issues.utils.commit() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { + id: 10, + title: `Bug in Alpha`, + comments: [ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ], + }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [ + { + id: 20, + title: `Bug in Beta`, + comments: [{ id: 200, body: `Same bug` }], + }, + ], + }, + { + id: 3, + name: `Gamma`, + issues: [], + }, + ]) + }) + + it(`updating an issue title re-emits the parent with updated array`, async () => { + const collection = buildToArrayToCollectionQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `update`, + value: { id: 10, projectId: 1, title: `Renamed Bug` }, + }) + issues.utils.commit() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { + id: 10, + title: `Renamed Bug`, + comments: [ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ], + }, + { id: 11, title: `Feature for Alpha`, comments: [] }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [ + { + id: 20, + title: `Bug in Beta`, + comments: [{ id: 200, body: `Same bug` }], + }, + ], + }, + { + id: 3, + name: `Gamma`, + issues: [], + }, + ]) + }) + }) + + describe(`nested includes: toArray → toArray`, () => { + function buildToArrayToArrayQuery() { + return createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: toArray( + q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + comments: toArray( + q + .from({ c: comments }) + .where(({ c }) => eq(c.issueId, i.id)) + .select(({ c }) => ({ + id: c.id, + body: c.body, + })), + ), + })), + ), + })), + ) + } + + it(`initial load: both levels are arrays`, async () => { + const collection = buildToArrayToArrayQuery() + await collection.preload() + + const alpha = collection.get(1) as any + expect(Array.isArray(alpha.issues)).toBe(true) + + const sortedIssues = alpha.issues.sort((a: any, b: any) => a.id - b.id) + expect(Array.isArray(sortedIssues[0].comments)).toBe(true) + expect( + sortedIssues[0].comments.sort((a: any, b: any) => a.id - b.id), + ).toEqual([ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ]) + + expect(sortedIssues[1].comments).toEqual([]) + }) + + it(`adding a comment (grandchild-only change) updates both array levels`, async () => { + const collection = buildToArrayToArrayQuery() + await collection.preload() + + comments.utils.begin() + comments.utils.write({ + type: `insert`, + value: { id: 110, issueId: 11, body: `Great feature` }, + }) + comments.utils.commit() + + const alpha = collection.get(1) as any + expect(Array.isArray(alpha.issues)).toBe(true) + const issue11 = alpha.issues.find((i: any) => i.id === 11) + expect(Array.isArray(issue11.comments)).toBe(true) + expect(issue11.comments).toEqual([{ id: 110, body: `Great feature` }]) + }) + + it(`removing a comment (grandchild-only change) updates both array levels`, async () => { + const collection = buildToArrayToArrayQuery() + await collection.preload() + + comments.utils.begin() + comments.utils.write({ + type: `delete`, + value: sampleComments.find((c) => c.id === 100)!, + }) + comments.utils.commit() + + const alpha = collection.get(1) as any + const issue10 = alpha.issues.find((i: any) => i.id === 10) + expect(issue10.comments).toEqual([{ id: 101, body: `Fixed it` }]) + }) + + it(`adding an issue (middle-level insert) re-emits parent with updated array`, async () => { + const collection = buildToArrayToArrayQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 30, projectId: 3, title: `Gamma issue` }, + }) + issues.utils.commit() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { + id: 10, + title: `Bug in Alpha`, + comments: [ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ], + }, + { id: 11, title: `Feature for Alpha`, comments: [] }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [ + { + id: 20, + title: `Bug in Beta`, + comments: [{ id: 200, body: `Same bug` }], + }, + ], + }, + { + id: 3, + name: `Gamma`, + issues: [{ id: 30, title: `Gamma issue`, comments: [] }], + }, + ]) + }) + + it(`removing an issue (middle-level delete) re-emits parent with updated array`, async () => { + const collection = buildToArrayToArrayQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `delete`, + value: sampleIssues.find((i) => i.id === 11)!, + }) + issues.utils.commit() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { + id: 10, + title: `Bug in Alpha`, + comments: [ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ], + }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [ + { + id: 20, + title: `Bug in Beta`, + comments: [{ id: 200, body: `Same bug` }], + }, + ], + }, + { + id: 3, + name: `Gamma`, + issues: [], + }, + ]) + }) + + it(`updating an issue title (middle-level update) re-emits parent with updated array`, async () => { + const collection = buildToArrayToArrayQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `update`, + value: { id: 10, projectId: 1, title: `Renamed Bug` }, + }) + issues.utils.commit() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { + id: 10, + title: `Renamed Bug`, + comments: [ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ], + }, + { id: 11, title: `Feature for Alpha`, comments: [] }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [ + { + id: 20, + title: `Bug in Beta`, + comments: [{ id: 200, body: `Same bug` }], + }, + ], + }, + { + id: 3, + name: `Gamma`, + issues: [], + }, + ]) + }) + + it(`concurrent child + grandchild changes in the same transaction`, async () => { + const collection = buildToArrayToArrayQuery() + await collection.preload() + + // Add a new issue AND a comment on an existing issue in one transaction + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 30, projectId: 3, title: `Gamma issue` }, + }) + issues.utils.commit() + + comments.utils.begin() + comments.utils.write({ + type: `insert`, + value: { id: 110, issueId: 11, body: `Great feature` }, + }) + comments.utils.commit() + + // Gamma should have the new issue with empty comments + const gamma = collection.get(3) as any + expect(gamma.issues).toHaveLength(1) + expect(gamma.issues[0].id).toBe(30) + expect(gamma.issues[0].comments).toEqual([]) + + // Alpha's issue 11 should have the new comment + const alpha = collection.get(1) as any + const issue11 = alpha.issues.find((i: any) => i.id === 11) + expect(issue11.comments).toEqual([{ id: 110, body: `Great feature` }]) + }) }) }) From 28d4daf935bf4d733cebf9c790ead077f4c307ac Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 12 Mar 2026 13:29:27 +0100 Subject: [PATCH 21/32] fix: strip __correlationKey from child results when child omits select() When a child includes query has no explicit .select(), the raw row (including the internal __correlationKey stamp) becomes the final result. Strip this internal property before returning so it doesn't leak to users. Co-Authored-By: Claude Opus 4.6 --- packages/db/src/query/compiler/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index 8dc030387..a904f55de 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -492,8 +492,10 @@ export function compileQuery( ).__includesCorrelationKeys } // When in includes mode, embed the correlation key as third element + // and strip the internal __correlationKey stamp so it doesn't leak to the user if (parentKeyStream) { const correlationKey = (row as any)[mainSource]?.__correlationKey + delete finalResults.__correlationKey return [key, [finalResults, orderByIndex, correlationKey]] as any } return [key, [finalResults, orderByIndex]] as [unknown, [any, string]] From a2e9dd54cdf876bc84905305f94c1201773a1417 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 12 Mar 2026 13:31:03 +0100 Subject: [PATCH 22/32] fix: shallow-clone select before mutating for includes placeholders replaceIncludesInSelect mutates query.select in-place, but the optimizer copies select by reference, so rawQuery.select === query.select. This violates the immutable-IR convention. Shallow-clone select when includes entries are found so the original IR is preserved. Co-Authored-By: Claude Opus 4.6 --- packages/db/src/query/compiler/index.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index a904f55de..8cd63b523 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -139,7 +139,9 @@ export function compileQuery( validateQueryStructure(rawQuery) // Optimize the query before compilation - const { optimizedQuery: query, sourceWhereClauses } = optimizeQuery(rawQuery) + const { optimizedQuery, sourceWhereClauses } = optimizeQuery(rawQuery) + // Use a mutable binding so we can shallow-clone select before includes mutation + let query = optimizedQuery // Create mapping from optimized query to original for caching queryMapping.set(query, rawQuery) @@ -287,6 +289,11 @@ export function compileQuery( const includesResults: Array = [] if (query.select) { const includesEntries = extractIncludesFromSelect(query.select) + // Shallow-clone select before mutating so we don't modify the shared IR + // (the optimizer copies select by reference, so rawQuery.select === query.select) + if (includesEntries.length > 0) { + query = { ...query, select: { ...query.select } } + } for (const { key, subquery } of includesEntries) { // Branch parent pipeline: map to [correlationValue, null] const compiledCorrelation = compileExpression(subquery.correlationField) From b18c6a6787643cedc3d392c49065295dd44b0ead Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 12 Mar 2026 13:32:32 +0100 Subject: [PATCH 23/32] fix: throw on nested IncludesSubquery in select extractIncludesFromSelect only checked top-level select entries. If a user placed an includes subquery inside a nested select object (e.g. select({ info: { issues: childQuery } })), the IncludesSubquery would never be extracted and the child pipeline would never compile, silently producing null. Now recursively checks nested objects and throws a clear error when a nested includes is detected. Co-Authored-By: Claude Opus 4.6 --- packages/db/src/query/compiler/index.ts | 35 +++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index 8cd63b523..f8c8ebee3 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -888,8 +888,9 @@ export function followRef( } /** - * Walks a Select object to find IncludesSubquery entries. - * Returns array of {key, subquery} for each found includes. + * Walks a Select object to find IncludesSubquery entries at the top level. + * Throws if an IncludesSubquery is found nested inside a sub-object, since + * the compiler only supports includes at the top level of a select. */ function extractIncludesFromSelect( select: Record, @@ -898,11 +899,41 @@ function extractIncludesFromSelect( for (const [key, value] of Object.entries(select)) { if (value instanceof IncludesSubquery) { results.push({ key, subquery: value }) + } else if (isNestedSelectObject(value)) { + // Check nested objects for IncludesSubquery — not supported yet + assertNoNestedIncludes(value, key) } } return results } +/** Check if a value is a nested plain object in a select (not an IR expression node) */ +function isNestedSelectObject(value: any): value is Record { + return ( + value != null && + typeof value === `object` && + !Array.isArray(value) && + typeof value.type !== `string` + ) +} + +function assertNoNestedIncludes( + obj: Record, + parentPath: string, +): void { + for (const [key, value] of Object.entries(obj)) { + if (value instanceof IncludesSubquery) { + throw new Error( + `Includes subqueries must be at the top level of select(). ` + + `Found nested includes at "${parentPath}.${key}".`, + ) + } + if (isNestedSelectObject(value)) { + assertNoNestedIncludes(value, `${parentPath}.${key}`) + } + } +} + /** * Replaces an IncludesSubquery entry in the select object with a null Value placeholder. * This ensures processSelect() doesn't encounter it. From 9dc9506a4b85236074a4eff2321e92695b292ea6 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 12 Mar 2026 13:33:04 +0100 Subject: [PATCH 24/32] test: add test for updating an existing child row in includes The update branch in flushIncludesState was untested. This test verifies that updating a child's title is reflected in the parent's child collection. Co-Authored-By: Claude Opus 4.6 --- packages/db/tests/query/includes.test.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/packages/db/tests/query/includes.test.ts b/packages/db/tests/query/includes.test.ts index 9c2816067..fb1713020 100644 --- a/packages/db/tests/query/includes.test.ts +++ b/packages/db/tests/query/includes.test.ts @@ -208,6 +208,29 @@ describe(`includes subqueries`, () => { ]) }) + it(`updating a child reflects the change in the parent's child collection`, async () => { + const collection = buildIncludesQuery() + await collection.preload() + + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 10, title: `Bug in Alpha` }, + { id: 11, title: `Feature for Alpha` }, + ]) + + // Update an existing child's title + issues.utils.begin() + issues.utils.write({ + type: `update`, + value: { id: 10, projectId: 1, title: `Bug in Alpha (fixed)` }, + }) + issues.utils.commit() + + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 10, title: `Bug in Alpha (fixed)` }, + { id: 11, title: `Feature for Alpha` }, + ]) + }) + it(`removing and re-adding a parent resets its child collection`, async () => { const collection = buildIncludesQuery() await collection.preload() From 4b48c4315fa59be7213a7c58c7269bab456fb10c Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 12 Mar 2026 13:34:16 +0100 Subject: [PATCH 25/32] test: add tests for validation errors, updates, and sibling includes - Test updating an existing child row (exercises the update branch in flushIncludesState) - Test error on missing WHERE, non-eq WHERE, and self-referencing eq - Test multiple sibling includes (issues + milestones on same parent) verifying independent child collections and independent reactivity Co-Authored-By: Claude Opus 4.6 --- packages/db/tests/query/includes.test.ts | 172 +++++++++++++++++++++++ 1 file changed, 172 insertions(+) diff --git a/packages/db/tests/query/includes.test.ts b/packages/db/tests/query/includes.test.ts index fb1713020..aee735cdd 100644 --- a/packages/db/tests/query/includes.test.ts +++ b/packages/db/tests/query/includes.test.ts @@ -1804,4 +1804,176 @@ describe(`includes subqueries`, () => { expect(issue11.comments).toEqual([{ id: 110, body: `Great feature` }]) }) }) + + describe(`validation errors`, () => { + it(`throws when child query has no WHERE clause`, () => { + expect(() => + createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + issues: q + .from({ i: issues }) + .select(({ i }) => ({ id: i.id, title: i.title })), + })), + ), + ).toThrow(/must have a WHERE clause with an eq\(\) condition/) + }) + + it(`throws when child WHERE has no eq() correlation`, () => { + expect(() => + createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + issues: q + .from({ i: issues }) + .where(({ i }) => i.projectId) + .select(({ i }) => ({ id: i.id, title: i.title })), + })), + ), + ).toThrow(/must have a WHERE clause with an eq\(\) condition/) + }) + + it(`throws when eq() references two child-side aliases`, () => { + expect(() => + createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, i.id)) + .select(({ i }) => ({ id: i.id, title: i.title })), + })), + ), + ).toThrow(/must have a WHERE clause with an eq\(\) condition/) + }) + }) + + describe(`multiple sibling includes`, () => { + type Milestone = { + id: number + projectId: number + name: string + } + + const sampleMilestones: Array = [ + { id: 1, projectId: 1, name: `v1.0` }, + { id: 2, projectId: 1, name: `v2.0` }, + { id: 3, projectId: 2, name: `Beta release` }, + ] + + function createMilestonesCollection() { + return createCollection( + mockSyncCollectionOptions({ + id: `includes-milestones`, + getKey: (m) => m.id, + initialData: sampleMilestones, + }), + ) + } + + it(`parent with two sibling includes produces independent child collections`, async () => { + const milestones = createMilestonesCollection() + + const collection = createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ id: i.id, title: i.title })), + milestones: q + .from({ m: milestones }) + .where(({ m }) => eq(m.projectId, p.id)) + .select(({ m }) => ({ id: m.id, name: m.name })), + })), + ) + + await collection.preload() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { id: 10, title: `Bug in Alpha` }, + { id: 11, title: `Feature for Alpha` }, + ], + milestones: [ + { id: 1, name: `v1.0` }, + { id: 2, name: `v2.0` }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [{ id: 20, title: `Bug in Beta` }], + milestones: [{ id: 3, name: `Beta release` }], + }, + { + id: 3, + name: `Gamma`, + issues: [], + milestones: [], + }, + ]) + }) + + it(`adding a child to one sibling does not affect the other`, async () => { + const milestones = createMilestonesCollection() + + const collection = createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ id: i.id, title: i.title })), + milestones: q + .from({ m: milestones }) + .where(({ m }) => eq(m.projectId, p.id)) + .select(({ m }) => ({ id: m.id, name: m.name })), + })), + ) + + await collection.preload() + + // Add an issue to Alpha — milestones should be unaffected + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 12, projectId: 1, title: `New Alpha issue` }, + }) + issues.utils.commit() + + const alpha = collection.get(1) as any + expect(childItems(alpha.issues)).toEqual([ + { id: 10, title: `Bug in Alpha` }, + { id: 11, title: `Feature for Alpha` }, + { id: 12, title: `New Alpha issue` }, + ]) + expect(childItems(alpha.milestones)).toEqual([ + { id: 1, name: `v1.0` }, + { id: 2, name: `v2.0` }, + ]) + + // Add a milestone to Beta — issues should be unaffected + milestones.utils.begin() + milestones.utils.write({ + type: `insert`, + value: { id: 4, projectId: 2, name: `Beta v2` }, + }) + milestones.utils.commit() + + const beta = collection.get(2) as any + expect(childItems(beta.issues)).toEqual([ + { id: 20, title: `Bug in Beta` }, + ]) + expect(childItems(beta.milestones)).toEqual([ + { id: 3, name: `Beta release` }, + { id: 4, name: `Beta v2` }, + ]) + }) + }) }) From 3dc4518aa84699b04876f42c8e2509030b767549 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 12 Mar 2026 13:37:29 +0100 Subject: [PATCH 26/32] fix: handle spread sentinels and type narrowing in includes extraction Skip __SPREAD_SENTINEL__ entries when checking for nested includes to avoid infinite recursion on RefProxy objects. Add non-null assertion for query.select after shallow clone (TypeScript loses narrowing after reassignment). Co-Authored-By: Claude Opus 4.6 --- packages/db/src/query/compiler/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index f8c8ebee3..b220a72d4 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -334,7 +334,7 @@ export function compileQuery( }) // Replace includes entry in select with a null placeholder - replaceIncludesInSelect(query.select, key) + replaceIncludesInSelect(query.select!, key) } // Stamp correlation key values onto the namespaced row so they survive @@ -897,6 +897,7 @@ function extractIncludesFromSelect( ): Array<{ key: string; subquery: IncludesSubquery }> { const results: Array<{ key: string; subquery: IncludesSubquery }> = [] for (const [key, value] of Object.entries(select)) { + if (key.startsWith(`__SPREAD_SENTINEL__`)) continue if (value instanceof IncludesSubquery) { results.push({ key, subquery: value }) } else if (isNestedSelectObject(value)) { @@ -922,6 +923,7 @@ function assertNoNestedIncludes( parentPath: string, ): void { for (const [key, value] of Object.entries(obj)) { + if (key.startsWith(`__SPREAD_SENTINEL__`)) continue if (value instanceof IncludesSubquery) { throw new Error( `Includes subqueries must be at the top level of select(). ` + From d03ace4ce6da9b0ace2eb7638e7f5801d3e4489d Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 12 Mar 2026 14:06:16 +0100 Subject: [PATCH 27/32] test: add type tests for Collection-based includes (currently failing) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add 4 type tests that verify the expected types for non-toArray includes: - includes with select → Collection<{id, title}> - includes without select → Collection - multiple sibling includes → independent Collection types - nested includes → Collection<{..., comments: Collection<{...}>}> These tests currently fail because SelectValue doesn't include QueryBuilder and ResultTypeFromSelect has no branch for it, so includes fields resolve to `never`. Co-Authored-By: Claude Opus 4.6 --- packages/db/tests/query/includes.test-d.ts | 102 +++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/packages/db/tests/query/includes.test-d.ts b/packages/db/tests/query/includes.test-d.ts index dbcdb4a45..4a928ff3c 100644 --- a/packages/db/tests/query/includes.test-d.ts +++ b/packages/db/tests/query/includes.test-d.ts @@ -6,6 +6,7 @@ import { } from '../../src/query/index.js' import { createCollection } from '../../src/collection/index.js' import { mockSyncCollectionOptions } from '../utils.js' +import type { Collection } from '../../src/collection/index.js' type Project = { id: number @@ -59,6 +60,107 @@ describe(`includes subquery types`, () => { const issues = createIssuesCollection() const comments = createCommentsCollection() + describe(`Collection includes`, () => { + test(`includes with select infers child result as Collection`, () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + })), + ) + + const result = collection.toArray[0]! + expectTypeOf(result.id).toEqualTypeOf() + expectTypeOf(result.name).toEqualTypeOf() + expectTypeOf(result.issues).toMatchTypeOf< + Collection<{ id: number; title: string }> + >() + }) + + test(`includes without select infers full child type as Collection`, () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)), + })), + ) + + const result = collection.toArray[0]! + expectTypeOf(result.id).toEqualTypeOf() + expectTypeOf(result.issues).toMatchTypeOf>() + }) + + test(`multiple sibling includes each infer their own Collection type`, () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + comments: q + .from({ c: comments }) + .where(({ c }) => eq(c.issueId, p.id)) + .select(({ c }) => ({ + id: c.id, + body: c.body, + })), + })), + ) + + const result = collection.toArray[0]! + expectTypeOf(result.issues).toMatchTypeOf< + Collection<{ id: number; title: string }> + >() + expectTypeOf(result.comments).toMatchTypeOf< + Collection<{ id: number; body: string }> + >() + }) + + test(`nested Collection includes infer correctly`, () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + comments: q + .from({ c: comments }) + .where(({ c }) => eq(c.issueId, i.id)) + .select(({ c }) => ({ + id: c.id, + body: c.body, + })), + })), + })), + ) + + const result = collection.toArray[0]! + expectTypeOf(result.issues).toMatchTypeOf< + Collection<{ + id: number + title: string + comments: Collection<{ id: number; body: string }> + }> + >() + }) + }) + describe(`toArray`, () => { test(`toArray includes infers child result as Array`, () => { const collection = createLiveQueryCollection((q) => From a0c6893688cc98a3c88eb1ee6f7db5b174af8a1d Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Thu, 12 Mar 2026 14:12:49 +0100 Subject: [PATCH 28/32] feat: add type support for Collection-based includes in select Add QueryBuilder to the SelectValue union so TypeScript accepts bare QueryBuilder instances in select callbacks. Add a branch in ResultTypeFromSelect that maps QueryBuilder to Collection>, giving users proper autocomplete and type safety on child collection fields. Co-Authored-By: Claude Opus 4.6 --- packages/db/src/query/builder/types.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/db/src/query/builder/types.ts b/packages/db/src/query/builder/types.ts index 8877b696f..6e61d6597 100644 --- a/packages/db/src/query/builder/types.ts +++ b/packages/db/src/query/builder/types.ts @@ -1,4 +1,4 @@ -import type { CollectionImpl } from '../../collection/index.js' +import type { Collection, CollectionImpl } from '../../collection/index.js' import type { SingleResult, StringCollationConfig } from '../../types.js' import type { Aggregate, @@ -176,6 +176,7 @@ type SelectValue = | { [key: string]: SelectValue } | Array> | ToArrayWrapper // toArray() wrapped subquery + | QueryBuilder // includes subquery (produces a child Collection) // Recursive shape for select objects allowing nested projections type SelectShape = { [key: string]: SelectValue | SelectShape } @@ -231,7 +232,10 @@ export type ResultTypeFromSelect = WithoutRefBrand< ? ExtractExpressionType : TSelectObject[K] extends ToArrayWrapper ? Array - : // Ref (full object ref or spread with RefBrand) - recursively process properties + : // includes subquery (bare QueryBuilder) — produces a child Collection + TSelectObject[K] extends QueryBuilder + ? Collection> + : // Ref (full object ref or spread with RefBrand) - recursively process properties TSelectObject[K] extends Ref ? ExtractRef : // RefLeaf (simple property ref like user.name) From 8dee11d8896b29f52c1965ed9cecdb3ff8ba54d4 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Thu, 12 Mar 2026 13:14:02 +0000 Subject: [PATCH 29/32] ci: apply automated fixes --- packages/db/src/query/builder/types.ts | 68 +++++++++++----------- packages/db/tests/query/includes.test-d.ts | 4 +- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/packages/db/src/query/builder/types.ts b/packages/db/src/query/builder/types.ts index 6e61d6597..dbb295a22 100644 --- a/packages/db/src/query/builder/types.ts +++ b/packages/db/src/query/builder/types.ts @@ -236,39 +236,41 @@ export type ResultTypeFromSelect = WithoutRefBrand< TSelectObject[K] extends QueryBuilder ? Collection> : // Ref (full object ref or spread with RefBrand) - recursively process properties - TSelectObject[K] extends Ref - ? ExtractRef - : // RefLeaf (simple property ref like user.name) - TSelectObject[K] extends RefLeaf - ? IsNullableRef extends true - ? T | undefined - : T - : // RefLeaf | undefined (schema-optional field) - TSelectObject[K] extends RefLeaf | undefined - ? T | undefined - : // RefLeaf | null (schema-nullable field) - TSelectObject[K] extends RefLeaf | null - ? IsNullableRef> extends true - ? T | null | undefined - : T | null - : // Ref | undefined (optional object-type schema field) - TSelectObject[K] extends Ref | undefined - ? ExtractRef> | undefined - : // Ref | null (nullable object-type schema field) - TSelectObject[K] extends Ref | null - ? ExtractRef> | null - : TSelectObject[K] extends Aggregate - ? T - : TSelectObject[K] extends - | string - | number - | boolean - | null - | undefined - ? TSelectObject[K] - : TSelectObject[K] extends Record - ? ResultTypeFromSelect - : never + TSelectObject[K] extends Ref + ? ExtractRef + : // RefLeaf (simple property ref like user.name) + TSelectObject[K] extends RefLeaf + ? IsNullableRef extends true + ? T | undefined + : T + : // RefLeaf | undefined (schema-optional field) + TSelectObject[K] extends RefLeaf | undefined + ? T | undefined + : // RefLeaf | null (schema-nullable field) + TSelectObject[K] extends RefLeaf | null + ? IsNullableRef> extends true + ? T | null | undefined + : T | null + : // Ref | undefined (optional object-type schema field) + TSelectObject[K] extends Ref | undefined + ? + | ExtractRef> + | undefined + : // Ref | null (nullable object-type schema field) + TSelectObject[K] extends Ref | null + ? ExtractRef> | null + : TSelectObject[K] extends Aggregate + ? T + : TSelectObject[K] extends + | string + | number + | boolean + | null + | undefined + ? TSelectObject[K] + : TSelectObject[K] extends Record + ? ResultTypeFromSelect + : never }> > diff --git a/packages/db/tests/query/includes.test-d.ts b/packages/db/tests/query/includes.test-d.ts index 4a928ff3c..95a053c29 100644 --- a/packages/db/tests/query/includes.test-d.ts +++ b/packages/db/tests/query/includes.test-d.ts @@ -88,9 +88,7 @@ describe(`includes subquery types`, () => { const collection = createLiveQueryCollection((q) => q.from({ p: projects }).select(({ p }) => ({ id: p.id, - issues: q - .from({ i: issues }) - .where(({ i }) => eq(i.projectId, p.id)), + issues: q.from({ i: issues }).where(({ i }) => eq(i.projectId, p.id)), })), ) From 2271ca14ca691695eb122480b392912224f53d73 Mon Sep 17 00:00:00 2001 From: Kevin Date: Thu, 12 Mar 2026 14:27:21 +0100 Subject: [PATCH 30/32] Support parent-referencing WHERE filters in includes child queries (#1307) * Unit tests for filtering on parent fields in child query * ci: apply automated fixes * Support parent-referencing WHERE filters in includes child queries Allow child queries to have additional WHERE clauses that reference parent fields (e.g., eq(i.createdBy, p.createdBy)) beyond the single correlation eq(). Parent-referencing WHEREs are detected in the builder, parent fields are projected into the key stream, and filters are re-injected into the child query where parent context is available. When no parent-referencing filters exist, behavior is unchanged. Co-Authored-By: Claude Opus 4.6 * ci: apply automated fixes * Extract correlation condition from inside and() WHERE clauses When users write a single .where() with and(eq(i.projectId, p.id), ...), the correlation eq() is now found and extracted from inside the and(). The remaining args stay as WHERE clauses. This means users don't need to know that the correlation must be a separate .where() call. Co-Authored-By: Claude Opus 4.6 * ci: apply automated fixes * Some more tests * changeset * Add failing test for shared correlation key with distinct parent filter values Two parents share the same correlation key (groupId) but have different values for a parent-referenced filter field (createdBy). The test verifies that each parent receives its own filtered child set rather than a shared union. Co-Authored-By: Claude Opus 4.6 * ci: apply automated fixes * Key child collections by composite routing key to fix shared correlation key collision When multiple parents share the same correlation key but have different parent-referenced filter values, child collections were incorrectly shared. Fix by keying child collections by (correlationKey, parentFilterValues) composite, and using composite child keys in the D2 stream to prevent collisions. Co-Authored-By: Claude Opus 4.6 * Add failing test for shared correlation key with orderBy + limit Reproduces the bug where grouped ordering for limit uses the raw correlation key instead of the composite routing key, causing parents that share a correlation key but differ on parent-referenced filters to have their children merged before the limit is applied. Co-Authored-By: Claude Opus 4.6 * ci: apply automated fixes * Use composite routing key for grouped ordering with limit/offset The includesGroupKeyFn for orderBy + limit/offset was grouping by raw correlationKey, causing parents sharing a correlation key but differing on parent-referenced filters to have their children merged before the limit was applied. Use the same composite key as the routing layer. Co-Authored-By: Claude Opus 4.6 * Add failing test for nested includes with parent-referencing filters at both levels When both the child and grandchild includes use parent-referencing filters, the grandchild collection comes back empty because the nested routing index uses a different key than the nested buffer. Co-Authored-By: Claude Opus 4.6 * Use composite routing key in nested routing index to match nested buffer keys The nested routing index was keyed by raw correlationKey while nested buffers use computeRoutingKey(correlationKey, parentContext). This mismatch caused drainNestedBuffers lookups to fail, leaving grandchild collections empty when parent-referencing filters exist at both levels. Co-Authored-By: Claude Opus 4.6 * Add test for three levels of nested includes with parent-referencing filters Verifies that composite routing keys work at arbitrary nesting depth, not just the first two levels. Co-Authored-By: Claude Opus 4.6 * ci: apply automated fixes * Add test for deleting one parent preserving sibling parent's child collection Co-Authored-By: Claude Opus 4.6 * Fix shared correlation key: deduplicate parentKeyStream and defer child cleanup Two fixes for when multiple parents share the same correlation key: 1. Add reduce operator on parentKeyStream to clamp multiplicities to 1, preventing the inner join from producing duplicate child entries that cause incorrect deletions when one parent is removed. 2. In Phase 5, only delete child registry entry when the last parent referencing it is removed. Co-Authored-By: Claude Opus 4.6 * Add test for spread select on child not leaking internal properties Co-Authored-By: Claude Opus 4.6 * Strip internal __correlationKey and __parentContext from child results These routing properties leak into user-visible results when the child query uses a spread select (e.g. { ...i }). Co-Authored-By: Claude Opus 4.6 * style: fix prettier formatting in compiler Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 --- .../includes-parent-referencing-filters.md | 5 + packages/db/src/query/builder/index.ts | 144 +++- packages/db/src/query/compiler/index.ts | 232 ++++-- packages/db/src/query/ir.ts | 2 + .../query/live/collection-config-builder.ts | 130 ++- packages/db/tests/query/includes.test.ts | 779 ++++++++++++++++++ 6 files changed, 1190 insertions(+), 102 deletions(-) create mode 100644 .changeset/includes-parent-referencing-filters.md diff --git a/.changeset/includes-parent-referencing-filters.md b/.changeset/includes-parent-referencing-filters.md new file mode 100644 index 000000000..78d209ad5 --- /dev/null +++ b/.changeset/includes-parent-referencing-filters.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +feat: support parent-referencing WHERE filters in includes child queries diff --git a/packages/db/src/query/builder/index.ts b/packages/db/src/query/builder/index.ts index 7351cbfa5..ac1bc1582 100644 --- a/packages/db/src/query/builder/index.ts +++ b/packages/db/src/query/builder/index.ts @@ -33,6 +33,7 @@ import type { OrderBy, OrderByDirection, QueryIR, + Where, } from '../ir.js' import type { CompareOptions, @@ -894,6 +895,39 @@ function buildNestedSelect(obj: any, parentAliases: Array = []): any { return out } +/** + * Recursively collects all PropRef nodes from an expression tree. + */ +function collectRefsFromExpression(expr: BasicExpression): Array { + const refs: Array = [] + switch (expr.type) { + case `ref`: + refs.push(expr) + break + case `func`: + for (const arg of (expr as any).args ?? []) { + refs.push(...collectRefsFromExpression(arg)) + } + break + default: + break + } + return refs +} + +/** + * Checks whether a WHERE clause references any parent alias. + */ +function referencesParent(where: Where, parentAliases: Array): boolean { + const expr = + typeof where === `object` && `expression` in where + ? where.expression + : where + return collectRefsFromExpression(expr).some( + (ref) => ref.path[0] != null && parentAliases.includes(ref.path[0]), + ) +} + /** * Builds an IncludesSubquery IR node from a child query builder. * Extracts the correlation condition from the child's WHERE clauses by finding @@ -915,10 +949,12 @@ function buildIncludesSubquery( } } - // Walk child's WHERE clauses to find the correlation condition + // Walk child's WHERE clauses to find the correlation condition. + // The correlation eq() may be a standalone WHERE or nested inside a top-level and(). let parentRef: PropRef | undefined let childRef: PropRef | undefined let correlationWhereIndex = -1 + let correlationAndArgIndex = -1 // >= 0 when found inside an and() if (childQuery.where) { for (let i = 0; i < childQuery.where.length; i++) { @@ -928,16 +964,15 @@ function buildIncludesSubquery( ? where.expression : where - // Look for eq(a, b) where one side references parent and other references child + // Try standalone eq() if ( expr.type === `func` && expr.name === `eq` && expr.args.length === 2 ) { - const [argA, argB] = expr.args const result = extractCorrelation( - argA!, - argB!, + expr.args[0]!, + expr.args[1]!, parentAliases, childAliases, ) @@ -948,6 +983,37 @@ function buildIncludesSubquery( break } } + + // Try inside top-level and() + if ( + expr.type === `func` && + expr.name === `and` && + expr.args.length >= 2 + ) { + for (let j = 0; j < expr.args.length; j++) { + const arg = expr.args[j]! + if ( + arg.type === `func` && + arg.name === `eq` && + arg.args.length === 2 + ) { + const result = extractCorrelation( + arg.args[0]!, + arg.args[1]!, + parentAliases, + childAliases, + ) + if (result) { + parentRef = result.parentRef + childRef = result.childRef + correlationWhereIndex = i + correlationAndArgIndex = j + break + } + } + } + if (parentRef) break + } } } @@ -959,12 +1025,72 @@ function buildIncludesSubquery( ) } - // Remove the correlation WHERE from the child query + // Remove the correlation eq() from the child query's WHERE clauses. + // If it was inside an and(), remove just that arg (collapsing the and() if needed). const modifiedWhere = [...childQuery.where!] - modifiedWhere.splice(correlationWhereIndex, 1) + if (correlationAndArgIndex >= 0) { + const where = modifiedWhere[correlationWhereIndex]! + const expr = + typeof where === `object` && `expression` in where + ? where.expression + : where + const remainingArgs = (expr as any).args.filter( + (_: any, idx: number) => idx !== correlationAndArgIndex, + ) + if (remainingArgs.length === 1) { + // Collapse and() with single remaining arg to just that expression + const isResidual = + typeof where === `object` && `expression` in where && where.residual + modifiedWhere[correlationWhereIndex] = isResidual + ? { expression: remainingArgs[0], residual: true } + : remainingArgs[0] + } else { + // Rebuild and() without the extracted arg + const newAnd = new FuncExpr(`and`, remainingArgs) + const isResidual = + typeof where === `object` && `expression` in where && where.residual + modifiedWhere[correlationWhereIndex] = isResidual + ? { expression: newAnd, residual: true } + : newAnd + } + } else { + modifiedWhere.splice(correlationWhereIndex, 1) + } + + // Separate remaining WHEREs into pure-child vs parent-referencing + const pureChildWhere: Array = [] + const parentFilters: Array = [] + for (const w of modifiedWhere) { + if (referencesParent(w, parentAliases)) { + parentFilters.push(w) + } else { + pureChildWhere.push(w) + } + } + + // Collect distinct parent PropRefs from parent-referencing filters + let parentProjection: Array | undefined + if (parentFilters.length > 0) { + const seen = new Set() + parentProjection = [] + for (const w of parentFilters) { + const expr = typeof w === `object` && `expression` in w ? w.expression : w + for (const ref of collectRefsFromExpression(expr)) { + if ( + ref.path[0] != null && + parentAliases.includes(ref.path[0]) && + !seen.has(ref.path.join(`.`)) + ) { + seen.add(ref.path.join(`.`)) + parentProjection.push(ref) + } + } + } + } + const modifiedQuery: QueryIR = { ...childQuery, - where: modifiedWhere.length > 0 ? modifiedWhere : undefined, + where: pureChildWhere.length > 0 ? pureChildWhere : undefined, } return new IncludesSubquery( @@ -972,6 +1098,8 @@ function buildIncludesSubquery( parentRef, childRef, fieldName, + parentFilters.length > 0 ? parentFilters : undefined, + parentProjection, materializeAsArray, ) } diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index b220a72d4..f33837024 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -1,4 +1,10 @@ -import { distinct, filter, join as joinOperator, map } from '@tanstack/db-ivm' +import { + distinct, + filter, + join as joinOperator, + map, + reduce, +} from '@tanstack/db-ivm' import { optimizeQuery } from '../optimizer.js' import { CollectionInputNotFoundError, @@ -39,6 +45,9 @@ import type { QueryCache, QueryMapping, WindowOptions } from './types.js' export type { WindowOptions } from './types.js' +/** Symbol used to tag parent $selected with routing metadata for includes */ +export const INCLUDES_ROUTING = Symbol(`includesRouting`) + /** * Result of compiling an includes subquery, including the child pipeline * and metadata needed to route child results to parent-scoped Collections. @@ -56,6 +65,8 @@ export interface IncludesCompilationResult { hasOrderBy: boolean /** Full compilation result for the child query (for nested includes + alias tracking) */ childCompilationResult: CompilationResult + /** Parent-side projection refs for parent-referencing filters */ + parentProjection?: Array /** When true, the output layer materializes children as Array instead of Collection */ materializeAsArray: boolean } @@ -205,15 +216,24 @@ export function compileQuery( // Inner join: only children whose correlation key exists in parent keys pass through const joined = childRekeyed.pipe(joinOperator(parentKeyStream, `inner`)) - // Extract: [correlationValue, [[childKey, childRow], null]] → [childKey, childRow] + // Extract: [correlationValue, [[childKey, childRow], parentContext]] → [childKey, childRow] // Tag the row with __correlationKey for output routing + // If parentSide is non-null (parent context projected), attach as __parentContext filteredMainInput = joined.pipe( filter(([_correlationValue, [childSide]]: any) => { return childSide != null }), - map(([correlationValue, [childSide, _parentSide]]: any) => { + map(([correlationValue, [childSide, parentSide]]: any) => { const [childKey, childRow] = childSide - return [childKey, { ...childRow, __correlationKey: correlationValue }] + const tagged: any = { ...childRow, __correlationKey: correlationValue } + if (parentSide != null) { + tagged.__parentContext = parentSide + } + const effectiveKey = + parentSide != null + ? `${String(childKey)}::${JSON.stringify(parentSide)}` + : childKey + return [effectiveKey, tagged] }), ) @@ -225,10 +245,15 @@ export function compileQuery( let pipeline: NamespacedAndKeyedStream = filteredMainInput.pipe( map(([key, row]) => { // Initialize the record with a nested structure - const ret = [key, { [mainSource]: row }] as [ - string, - Record, - ] + // If __parentContext exists (from parent-referencing includes), merge parent + // aliases into the namespaced row so WHERE can resolve parent refs + const { __parentContext, ...cleanRow } = row as any + const nsRow: Record = { [mainSource]: cleanRow } + if (__parentContext) { + Object.assign(nsRow, __parentContext) + ;(nsRow as any).__parentContext = __parentContext + } + const ret = [key, nsRow] as [string, Record] return ret }), ) @@ -287,6 +312,13 @@ export function compileQuery( // This must happen AFTER WHERE (so parent pipeline is filtered) but BEFORE processSelect // (so IncludesSubquery nodes are stripped before select compilation). const includesResults: Array = [] + const includesRoutingFns: Array<{ + fieldName: string + getRouting: (nsRow: any) => { + correlationKey: unknown + parentContext: Record | null + } + }> = [] if (query.select) { const includesEntries = extractIncludesFromSelect(query.select) // Shallow-clone select before mutating so we don't modify the shared IR @@ -295,15 +327,69 @@ export function compileQuery( query = { ...query, select: { ...query.select } } } for (const { key, subquery } of includesEntries) { - // Branch parent pipeline: map to [correlationValue, null] + // Branch parent pipeline: map to [correlationValue, parentContext] + // When parentProjection exists, project referenced parent fields; otherwise null (zero overhead) const compiledCorrelation = compileExpression(subquery.correlationField) - const parentKeys = pipeline.pipe( - map(([_key, nsRow]: any) => [compiledCorrelation(nsRow), null] as any), + let parentKeys: any + if (subquery.parentProjection && subquery.parentProjection.length > 0) { + const compiledProjections = subquery.parentProjection.map((ref) => ({ + alias: ref.path[0]!, + field: ref.path.slice(1), + compiled: compileExpression(ref), + })) + parentKeys = pipeline.pipe( + map(([_key, nsRow]: any) => { + const parentContext: Record> = {} + for (const proj of compiledProjections) { + if (!parentContext[proj.alias]) { + parentContext[proj.alias] = {} + } + const value = proj.compiled(nsRow) + // Set nested field in the alias namespace + let target = parentContext[proj.alias]! + for (let i = 0; i < proj.field.length - 1; i++) { + if (!target[proj.field[i]!]) { + target[proj.field[i]!] = {} + } + target = target[proj.field[i]!] + } + target[proj.field[proj.field.length - 1]!] = value + } + return [compiledCorrelation(nsRow), parentContext] as any + }), + ) + } else { + parentKeys = pipeline.pipe( + map( + ([_key, nsRow]: any) => [compiledCorrelation(nsRow), null] as any, + ), + ) + } + + // Deduplicate: when multiple parents share the same correlation key (and + // parentContext), clamp multiplicity to 1 so the inner join doesn't + // produce duplicate child entries that cause incorrect deletions. + parentKeys = parentKeys.pipe( + reduce((values: Array<[any, number]>) => + values.map(([v, mult]) => [v, mult > 0 ? 1 : 0] as [any, number]), + ), ) + // If parent filters exist, append them to the child query's WHERE + const childQuery = + subquery.parentFilters && subquery.parentFilters.length > 0 + ? { + ...subquery.query, + where: [ + ...(subquery.query.where || []), + ...subquery.parentFilters, + ], + } + : subquery.query + // Recursively compile child query WITH the parent key stream const childResult = compileQuery( - subquery.query, + childQuery, allInputs, collections, subscriptions, @@ -330,31 +416,52 @@ export function compileQuery( subquery.query.orderBy && subquery.query.orderBy.length > 0 ), childCompilationResult: childResult, + parentProjection: subquery.parentProjection, materializeAsArray: subquery.materializeAsArray, }) + // Capture routing function for INCLUDES_ROUTING tagging + if (subquery.parentProjection && subquery.parentProjection.length > 0) { + const compiledProjs = subquery.parentProjection.map((ref) => ({ + alias: ref.path[0]!, + field: ref.path.slice(1), + compiled: compileExpression(ref), + })) + const compiledCorr = compiledCorrelation + includesRoutingFns.push({ + fieldName: subquery.fieldName, + getRouting: (nsRow: any) => { + const parentContext: Record> = {} + for (const proj of compiledProjs) { + if (!parentContext[proj.alias]) { + parentContext[proj.alias] = {} + } + const value = proj.compiled(nsRow) + let target = parentContext[proj.alias]! + for (let i = 0; i < proj.field.length - 1; i++) { + if (!target[proj.field[i]!]) { + target[proj.field[i]!] = {} + } + target = target[proj.field[i]!] + } + target[proj.field[proj.field.length - 1]!] = value + } + return { correlationKey: compiledCorr(nsRow), parentContext } + }, + }) + } else { + includesRoutingFns.push({ + fieldName: subquery.fieldName, + getRouting: (nsRow: any) => ({ + correlationKey: compiledCorrelation(nsRow), + parentContext: null, + }), + }) + } + // Replace includes entry in select with a null placeholder replaceIncludesInSelect(query.select!, key) } - - // Stamp correlation key values onto the namespaced row so they survive - // select extraction. This allows flushIncludesState to read them directly - // without requiring the correlation field to be in the user's select. - if (includesEntries.length > 0) { - const compiledCorrelations = includesEntries.map(({ subquery }) => ({ - fieldName: subquery.fieldName, - compiled: compileExpression(subquery.correlationField), - })) - pipeline = pipeline.pipe( - map(([key, nsRow]: any) => { - const correlationKeys: Record = {} - for (const { fieldName: fn, compiled } of compiledCorrelations) { - correlationKeys[fn] = compiled(nsRow) - } - return [key, { ...nsRow, __includesCorrelationKeys: correlationKeys }] - }), - ) - } } if (query.distinct && !query.fnSelect && !query.select) { @@ -403,6 +510,25 @@ export function compileQuery( ) } + // Tag $selected with routing metadata for includes. + // This lets collection-config-builder extract routing info (correlationKey + parentContext) + // from parent results without depending on the user's select. + if (includesRoutingFns.length > 0) { + pipeline = pipeline.pipe( + map(([key, namespacedRow]: any) => { + const routing: Record< + string, + { correlationKey: unknown; parentContext: Record | null } + > = {} + for (const { fieldName, getRouting } of includesRoutingFns) { + routing[fieldName] = getRouting(namespacedRow) + } + namespacedRow.$selected[INCLUDES_ROUTING] = routing + return [key, namespacedRow] + }), + ) + } + // Process the GROUP BY clause if it exists if (query.groupBy && query.groupBy.length > 0) { pipeline = processGroupBy( @@ -469,8 +595,14 @@ export function compileQuery( const includesGroupKeyFn = parentKeyStream && (query.limit !== undefined || query.offset !== undefined) - ? (_key: unknown, row: unknown) => - (row as any)?.[mainSource]?.__correlationKey + ? (_key: unknown, row: unknown) => { + const correlationKey = (row as any)?.[mainSource]?.__correlationKey + const parentContext = (row as any)?.__parentContext + if (parentContext != null) { + return JSON.stringify([correlationKey, parentContext]) + } + return correlationKey + } : undefined const orderedPipeline = processOrderBy( @@ -492,18 +624,17 @@ export function compileQuery( // Extract the final results from $selected and include orderBy index const raw = (row as any).$selected const finalResults = unwrapValue(raw) - // Stamp includes correlation keys onto the result for child routing - if ((row as any).__includesCorrelationKeys) { - finalResults.__includesCorrelationKeys = ( - row as any - ).__includesCorrelationKeys - } - // When in includes mode, embed the correlation key as third element - // and strip the internal __correlationKey stamp so it doesn't leak to the user + // When in includes mode, embed the correlation key and parentContext if (parentKeyStream) { const correlationKey = (row as any)[mainSource]?.__correlationKey + const parentContext = (row as any).__parentContext ?? null + // Strip internal routing properties that may leak via spread selects delete finalResults.__correlationKey - return [key, [finalResults, orderByIndex, correlationKey]] as any + delete finalResults.__parentContext + return [ + key, + [finalResults, orderByIndex, correlationKey, parentContext], + ] as any } return [key, [finalResults, orderByIndex]] as [unknown, [any, string]] }), @@ -533,16 +664,17 @@ export function compileQuery( // Extract the final results from $selected and return [key, [results, undefined]] const raw = (row as any).$selected const finalResults = unwrapValue(raw) - // Stamp includes correlation keys onto the result for child routing - if ((row as any).__includesCorrelationKeys) { - finalResults.__includesCorrelationKeys = ( - row as any - ).__includesCorrelationKeys - } - // When in includes mode, embed the correlation key as third element + // When in includes mode, embed the correlation key and parentContext if (parentKeyStream) { const correlationKey = (row as any)[mainSource]?.__correlationKey - return [key, [finalResults, undefined, correlationKey]] as any + const parentContext = (row as any).__parentContext ?? null + // Strip internal routing properties that may leak via spread selects + delete finalResults.__correlationKey + delete finalResults.__parentContext + return [ + key, + [finalResults, undefined, correlationKey, parentContext], + ] as any } return [key, [finalResults, undefined]] as [ unknown, diff --git a/packages/db/src/query/ir.ts b/packages/db/src/query/ir.ts index 04c28afd7..c115974b2 100644 --- a/packages/db/src/query/ir.ts +++ b/packages/db/src/query/ir.ts @@ -139,6 +139,8 @@ export class IncludesSubquery extends BaseExpression { public correlationField: PropRef, // Parent-side ref (e.g., project.id) public childCorrelationField: PropRef, // Child-side ref (e.g., issue.projectId) public fieldName: string, // Result field name (e.g., "issues") + public parentFilters?: Array, // WHERE clauses referencing parent aliases (applied post-join) + public parentProjection?: Array, // Parent field refs used by parentFilters public materializeAsArray: boolean = false, // When true, parent gets Array instead of Collection ) { super() diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index f0f89ad5d..b55eefc9d 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -1,5 +1,5 @@ import { D2, output, serializeValue } from '@tanstack/db-ivm' -import { compileQuery } from '../compiler/index.js' +import { INCLUDES_ROUTING, compileQuery } from '../compiler/index.js' import { createCollection } from '../../collection/index.js' import { MissingAliasInputsError, @@ -838,14 +838,21 @@ export class CollectionConfigBuilder< syncState.messagesCount += messages.length for (const [[childKey, tupleData], multiplicity] of messages) { - const [childResult, _orderByIndex, correlationKey] = - tupleData as unknown as [any, string | undefined, unknown] + const [childResult, _orderByIndex, correlationKey, parentContext] = + tupleData as unknown as [ + any, + string | undefined, + unknown, + Record | null, + ] + + const routingKey = computeRoutingKey(correlationKey, parentContext) - // Accumulate by [correlationKey, childKey] - let byChild = state.pendingChildChanges.get(correlationKey) + // Accumulate by [routingKey, childKey] + let byChild = state.pendingChildChanges.get(routingKey) if (!byChild) { byChild = new Map() - state.pendingChildChanges.set(correlationKey, byChild) + state.pendingChildChanges.set(routingKey, byChild) } const existing = byChild.get(childKey) || { @@ -1181,13 +1188,20 @@ function setupNestedPipelines( syncState.messagesCount += messages.length for (const [[childKey, tupleData], multiplicity] of messages) { - const [childResult, _orderByIndex, correlationKey] = - tupleData as unknown as [any, string | undefined, unknown] + const [childResult, _orderByIndex, correlationKey, parentContext] = + tupleData as unknown as [ + any, + string | undefined, + unknown, + Record | null, + ] + + const routingKey = computeRoutingKey(correlationKey, parentContext) - let byChild = buffer.get(correlationKey) + let byChild = buffer.get(routingKey) if (!byChild) { byChild = new Map() - buffer.set(correlationKey, byChild) + buffer.set(routingKey, byChild) } const existing = byChild.get(childKey) || { @@ -1330,34 +1344,44 @@ function updateRoutingIndex( for (const setup of state.nestedSetups) { for (const [, change] of childChanges) { if (change.inserts > 0) { - // Read the pre-computed nested correlation key from the compiler stamp - const nestedCorrelationKey = - change.value.__includesCorrelationKeys?.[ - setup.compilationResult.fieldName - ] + // Read the nested routing key from the INCLUDES_ROUTING stamp. + // Must use the composite routing key (not raw correlationKey) to match + // how nested buffers are keyed by computeRoutingKey. + const nestedRouting = + change.value[INCLUDES_ROUTING]?.[setup.compilationResult.fieldName] + const nestedCorrelationKey = nestedRouting?.correlationKey + const nestedParentContext = nestedRouting?.parentContext ?? null + const nestedRoutingKey = computeRoutingKey( + nestedCorrelationKey, + nestedParentContext, + ) if (nestedCorrelationKey != null) { - state.nestedRoutingIndex!.set(nestedCorrelationKey, correlationKey) + state.nestedRoutingIndex!.set(nestedRoutingKey, correlationKey) let reverseSet = state.nestedRoutingReverseIndex!.get(correlationKey) if (!reverseSet) { reverseSet = new Set() state.nestedRoutingReverseIndex!.set(correlationKey, reverseSet) } - reverseSet.add(nestedCorrelationKey) + reverseSet.add(nestedRoutingKey) } } else if (change.deletes > 0 && change.inserts === 0) { // Remove from routing index - const nestedCorrelationKey = - change.value.__includesCorrelationKeys?.[ - setup.compilationResult.fieldName - ] + const nestedRouting2 = + change.value[INCLUDES_ROUTING]?.[setup.compilationResult.fieldName] + const nestedCorrelationKey = nestedRouting2?.correlationKey + const nestedParentContext2 = nestedRouting2?.parentContext ?? null + const nestedRoutingKey = computeRoutingKey( + nestedCorrelationKey, + nestedParentContext2, + ) if (nestedCorrelationKey != null) { - state.nestedRoutingIndex!.delete(nestedCorrelationKey) + state.nestedRoutingIndex!.delete(nestedRoutingKey) const reverseSet = state.nestedRoutingReverseIndex!.get(correlationKey) if (reverseSet) { - reverseSet.delete(nestedCorrelationKey) + reverseSet.delete(nestedRoutingKey) if (reverseSet.size === 0) { state.nestedRoutingReverseIndex!.delete(correlationKey) } @@ -1399,6 +1423,19 @@ function hasNestedBufferChanges(setups: Array): boolean { return false } +/** + * Computes a composite routing key from correlation key and parent context. + * When parentContext is null (no parent filters), returns the raw correlationKey + * for zero behavioral change on existing queries. + */ +function computeRoutingKey( + correlationKey: unknown, + parentContext: Record | null, +): unknown { + if (parentContext == null) return correlationKey + return JSON.stringify([correlationKey, parentContext]) +} + /** * Creates a child Collection entry for includes subqueries. * The child Collection is a full-fledged Collection instance that starts syncing immediately. @@ -1472,38 +1509,40 @@ function flushIncludesState( for (const [parentKey, changes] of parentChanges) { if (changes.inserts > 0) { const parentResult = changes.value - // Read the pre-computed correlation key from the compiler stamp - const correlationKey = - parentResult.__includesCorrelationKeys?.[state.fieldName] + // Extract routing info from INCLUDES_ROUTING symbol (set by compiler) + const routing = parentResult[INCLUDES_ROUTING]?.[state.fieldName] + const correlationKey = routing?.correlationKey + const parentContext = routing?.parentContext ?? null + const routingKey = computeRoutingKey(correlationKey, parentContext) if (correlationKey != null) { - // Ensure child Collection exists for this correlation key - if (!state.childRegistry.has(correlationKey)) { + // Ensure child Collection exists for this routing key + if (!state.childRegistry.has(routingKey)) { const entry = createChildCollectionEntry( parentId, state.fieldName, - correlationKey, + routingKey, state.hasOrderBy, state.nestedSetups, ) - state.childRegistry.set(correlationKey, entry) + state.childRegistry.set(routingKey, entry) } - // Update reverse index: correlation key → parent keys - let parentKeys = state.correlationToParentKeys.get(correlationKey) + // Update reverse index: routing key → parent keys + let parentKeys = state.correlationToParentKeys.get(routingKey) if (!parentKeys) { parentKeys = new Set() - state.correlationToParentKeys.set(correlationKey, parentKeys) + state.correlationToParentKeys.set(routingKey, parentKeys) } parentKeys.add(parentKey) // Attach child Collection (or array snapshot for toArray) to the parent result if (state.materializeAsArray) { parentResult[state.fieldName] = [ - ...state.childRegistry.get(correlationKey)!.collection.toArray, + ...state.childRegistry.get(routingKey)!.collection.toArray, ] } else { parentResult[state.fieldName] = - state.childRegistry.get(correlationKey)!.collection + state.childRegistry.get(routingKey)!.collection } } } @@ -1662,17 +1701,20 @@ function flushIncludesState( if (parentChanges) { for (const [parentKey, changes] of parentChanges) { if (changes.deletes > 0 && changes.inserts === 0) { - const correlationKey = - changes.value.__includesCorrelationKeys?.[state.fieldName] + const routing = changes.value[INCLUDES_ROUTING]?.[state.fieldName] + const correlationKey = routing?.correlationKey + const parentContext = routing?.parentContext ?? null + const routingKey = computeRoutingKey(correlationKey, parentContext) if (correlationKey != null) { - cleanRoutingIndexOnDelete(state, correlationKey) - state.childRegistry.delete(correlationKey) - // Clean up reverse index - const parentKeys = state.correlationToParentKeys.get(correlationKey) + // Clean up reverse index first, only delete child collection + // when the last parent referencing it is removed + const parentKeys = state.correlationToParentKeys.get(routingKey) if (parentKeys) { parentKeys.delete(parentKey) if (parentKeys.size === 0) { - state.correlationToParentKeys.delete(correlationKey) + cleanRoutingIndexOnDelete(state, routingKey) + state.childRegistry.delete(routingKey) + state.correlationToParentKeys.delete(routingKey) } } } @@ -1681,10 +1723,10 @@ function flushIncludesState( } } - // Clean up the internal stamp from parent/child results so it doesn't leak to the user + // Clean up the internal routing stamp from parent/child results if (parentChanges) { for (const [, changes] of parentChanges) { - delete changes.value.__includesCorrelationKeys + delete changes.value[INCLUDES_ROUTING] } } } diff --git a/packages/db/tests/query/includes.test.ts b/packages/db/tests/query/includes.test.ts index aee735cdd..12fa9e0df 100644 --- a/packages/db/tests/query/includes.test.ts +++ b/packages/db/tests/query/includes.test.ts @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { + and, createLiveQueryCollection, eq, toArray, @@ -294,6 +295,34 @@ describe(`includes subqueries`, () => { { id: 30, title: `Gamma issue` }, ]) }) + + it(`spread select on child does not leak internal properties`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + ...i, + })), + })), + ) + + await collection.preload() + + const alpha = collection.get(1) as any + const childIssues = childItems(alpha.issues) + // Should contain only the real Issue fields, no internal __correlationKey + expect(childIssues[0]).toEqual({ + id: 10, + projectId: 1, + title: `Bug in Alpha`, + }) + expect(childIssues[0]).not.toHaveProperty(`__correlationKey`) + expect(childIssues[0]).not.toHaveProperty(`__parentContext`) + }) }) describe(`change propagation`, () => { @@ -712,6 +741,48 @@ describe(`includes subqueries`, () => { ]) }) + it(`deleting one parent preserves sibling parent's child collection`, async () => { + const teams = createTeamsCollection() + const members = createMembersCollection() + + const collection = createLiveQueryCollection((q) => + q.from({ t: teams }).select(({ t }) => ({ + id: t.id, + name: t.name, + departmentId: t.departmentId, + members: q + .from({ m: members }) + .where(({ m }) => eq(m.departmentId, t.departmentId)) + .select(({ m }) => ({ + id: m.id, + name: m.name, + })), + })), + ) + + await collection.preload() + + // Both Frontend and Backend share departmentId 100 + expect(childItems((collection.get(1) as any).members)).toHaveLength(2) + expect(childItems((collection.get(2) as any).members)).toHaveLength(2) + + // Delete the Frontend team + teams.utils.begin() + teams.utils.write({ + type: `delete`, + value: sampleTeams[0]!, + }) + teams.utils.commit() + + expect(collection.get(1)).toBeUndefined() + + // Backend should still have its child collection with all members + expect(childItems((collection.get(2) as any).members)).toEqual([ + { id: 10, name: `Alice` }, + { id: 11, name: `Bob` }, + ]) + }) + it(`correlation field does not need to be in the parent select`, async () => { const teams = createTeamsCollection() const members = createMembersCollection() @@ -1805,6 +1876,714 @@ describe(`includes subqueries`, () => { }) }) + describe(`parent-referencing filters`, () => { + type ProjectWithCreator = { + id: number + name: string + createdBy: string + } + + type IssueWithCreator = { + id: number + projectId: number + title: string + createdBy: string + } + + const sampleProjectsWithCreator: Array = [ + { id: 1, name: `Alpha`, createdBy: `alice` }, + { id: 2, name: `Beta`, createdBy: `bob` }, + { id: 3, name: `Gamma`, createdBy: `alice` }, + ] + + const sampleIssuesWithCreator: Array = [ + { id: 10, projectId: 1, title: `Bug in Alpha`, createdBy: `alice` }, + { id: 11, projectId: 1, title: `Feature for Alpha`, createdBy: `bob` }, + { id: 20, projectId: 2, title: `Bug in Beta`, createdBy: `bob` }, + { id: 21, projectId: 2, title: `Feature for Beta`, createdBy: `alice` }, + { id: 30, projectId: 3, title: `Bug in Gamma`, createdBy: `alice` }, + ] + + function createProjectsWC() { + return createCollection( + mockSyncCollectionOptions({ + id: `includes-projects-wc`, + getKey: (p) => p.id, + initialData: sampleProjectsWithCreator, + }), + ) + } + + function createIssuesWC() { + return createCollection( + mockSyncCollectionOptions({ + id: `includes-issues-wc`, + getKey: (i) => i.id, + initialData: sampleIssuesWithCreator, + }), + ) + } + + let projectsWC: ReturnType + let issuesWC: ReturnType + + beforeEach(() => { + projectsWC = createProjectsWC() + issuesWC = createIssuesWC() + }) + + it(`filters children by parent-referencing eq()`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projectsWC }).select(({ p }) => ({ + id: p.id, + name: p.name, + createdBy: p.createdBy, + issues: q + .from({ i: issuesWC }) + .where(({ i }) => eq(i.projectId, p.id)) + .where(({ i }) => eq(i.createdBy, p.createdBy)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + createdBy: i.createdBy, + })), + })), + ) + + await collection.preload() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + createdBy: `alice`, + issues: [ + // Only issue 10 (createdBy: alice) matches project 1 (createdBy: alice) + { id: 10, title: `Bug in Alpha`, createdBy: `alice` }, + ], + }, + { + id: 2, + name: `Beta`, + createdBy: `bob`, + issues: [ + // Only issue 20 (createdBy: bob) matches project 2 (createdBy: bob) + { id: 20, title: `Bug in Beta`, createdBy: `bob` }, + ], + }, + { + id: 3, + name: `Gamma`, + createdBy: `alice`, + issues: [ + // Only issue 30 (createdBy: alice) matches project 3 (createdBy: alice) + { id: 30, title: `Bug in Gamma`, createdBy: `alice` }, + ], + }, + ]) + }) + + it(`reacts to parent field change`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projectsWC }).select(({ p }) => ({ + id: p.id, + name: p.name, + createdBy: p.createdBy, + issues: q + .from({ i: issuesWC }) + .where(({ i }) => eq(i.projectId, p.id)) + .where(({ i }) => eq(i.createdBy, p.createdBy)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + createdBy: i.createdBy, + })), + })), + ) + + await collection.preload() + + // Project 1 (createdBy: alice) → only issue 10 (alice) + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 10, title: `Bug in Alpha`, createdBy: `alice` }, + ]) + + // Change project 1 createdBy from alice to bob + projectsWC.utils.begin() + projectsWC.utils.write({ + type: `update`, + value: { id: 1, name: `Alpha`, createdBy: `bob` }, + oldValue: sampleProjectsWithCreator[0]!, + }) + projectsWC.utils.commit() + + // Now issue 11 (createdBy: bob) should match, issue 10 (alice) should not + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 11, title: `Feature for Alpha`, createdBy: `bob` }, + ]) + }) + + it(`reacts to child field change`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projectsWC }).select(({ p }) => ({ + id: p.id, + name: p.name, + createdBy: p.createdBy, + issues: q + .from({ i: issuesWC }) + .where(({ i }) => eq(i.projectId, p.id)) + .where(({ i }) => eq(i.createdBy, p.createdBy)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + createdBy: i.createdBy, + })), + })), + ) + + await collection.preload() + + // Project 1 (alice) → only issue 10 + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 10, title: `Bug in Alpha`, createdBy: `alice` }, + ]) + + // Change issue 11's createdBy from bob to alice → it should now appear + issuesWC.utils.begin() + issuesWC.utils.write({ + type: `update`, + value: { + id: 11, + projectId: 1, + title: `Feature for Alpha`, + createdBy: `alice`, + }, + oldValue: sampleIssuesWithCreator[1]!, + }) + issuesWC.utils.commit() + + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 10, title: `Bug in Alpha`, createdBy: `alice` }, + { id: 11, title: `Feature for Alpha`, createdBy: `alice` }, + ]) + }) + + it(`mixed filters: parent-referencing + pure-child`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projectsWC }).select(({ p }) => ({ + id: p.id, + name: p.name, + createdBy: p.createdBy, + issues: q + .from({ i: issuesWC }) + .where(({ i }) => eq(i.projectId, p.id)) + .where(({ i }) => eq(i.createdBy, p.createdBy)) + .where(({ i }) => eq(i.title, `Bug in Alpha`)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + createdBy: i.createdBy, + })), + })), + ) + + await collection.preload() + + // Project 1 (alice): matching createdBy + title = only issue 10 + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 10, title: `Bug in Alpha`, createdBy: `alice` }, + ]) + + // Project 2 (bob): no issues with title "Bug in Alpha" + expect(childItems((collection.get(2) as any).issues)).toEqual([]) + + // Project 3 (alice): no issues with title "Bug in Alpha" + expect(childItems((collection.get(3) as any).issues)).toEqual([]) + }) + + it(`extracts correlation from and() with a pure-child filter`, async () => { + // and(correlation, childFilter) in a single .where() — no parent ref + const collection = createLiveQueryCollection((q) => + q.from({ p: projectsWC }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issuesWC }) + .where(({ i }) => + and(eq(i.projectId, p.id), eq(i.createdBy, `alice`)), + ) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + })), + ) + + await collection.preload() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [{ id: 10, title: `Bug in Alpha` }], + }, + { + id: 2, + name: `Beta`, + issues: [{ id: 21, title: `Feature for Beta` }], + }, + { + id: 3, + name: `Gamma`, + issues: [{ id: 30, title: `Bug in Gamma` }], + }, + ]) + }) + + it(`reactivity works when correlation is inside and()`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projectsWC }).select(({ p }) => ({ + id: p.id, + name: p.name, + createdBy: p.createdBy, + issues: q + .from({ i: issuesWC }) + .where(({ i }) => + and(eq(i.projectId, p.id), eq(i.createdBy, p.createdBy)), + ) + .select(({ i }) => ({ + id: i.id, + title: i.title, + createdBy: i.createdBy, + })), + })), + ) + + await collection.preload() + + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 10, title: `Bug in Alpha`, createdBy: `alice` }, + ]) + + // Change project 1 createdBy from alice to bob → issue 11 should match instead + projectsWC.utils.begin() + projectsWC.utils.write({ + type: `update`, + value: { id: 1, name: `Alpha`, createdBy: `bob` }, + oldValue: sampleProjectsWithCreator[0]!, + }) + projectsWC.utils.commit() + + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 11, title: `Feature for Alpha`, createdBy: `bob` }, + ]) + }) + + it(`extracts correlation from inside and()`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projectsWC }).select(({ p }) => ({ + id: p.id, + name: p.name, + createdBy: p.createdBy, + issues: q + .from({ i: issuesWC }) + .where(({ i }) => + and(eq(i.projectId, p.id), eq(i.createdBy, p.createdBy)), + ) + .select(({ i }) => ({ + id: i.id, + title: i.title, + createdBy: i.createdBy, + })), + })), + ) + + await collection.preload() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + createdBy: `alice`, + issues: [{ id: 10, title: `Bug in Alpha`, createdBy: `alice` }], + }, + { + id: 2, + name: `Beta`, + createdBy: `bob`, + issues: [{ id: 20, title: `Bug in Beta`, createdBy: `bob` }], + }, + { + id: 3, + name: `Gamma`, + createdBy: `alice`, + issues: [{ id: 30, title: `Bug in Gamma`, createdBy: `alice` }], + }, + ]) + }) + + it(`produces distinct child sets when parents share a correlation key but differ in filtered parent fields`, async () => { + // Two parents share the same groupId (correlation key) but have different + // createdBy values. The parent-referencing filter on createdBy must + // produce separate child results per parent, not a shared union. + type GroupParent = { + id: number + groupId: number + createdBy: string + } + + type GroupChild = { + id: number + groupId: number + createdBy: string + } + + const parents = createCollection( + mockSyncCollectionOptions({ + id: `shared-corr-parents`, + getKey: (p) => p.id, + initialData: [ + { id: 1, groupId: 1, createdBy: `alice` }, + { id: 2, groupId: 1, createdBy: `bob` }, + ], + }), + ) + + const children = createCollection( + mockSyncCollectionOptions({ + id: `shared-corr-children`, + getKey: (c) => c.id, + initialData: [{ id: 10, groupId: 1, createdBy: `alice` }], + }), + ) + + const collection = createLiveQueryCollection((q) => + q.from({ p: parents }).select(({ p }) => ({ + id: p.id, + createdBy: p.createdBy, + items: q + .from({ c: children }) + .where(({ c }) => eq(c.groupId, p.groupId)) + .where(({ c }) => eq(c.createdBy, p.createdBy)) + .select(({ c }) => ({ + id: c.id, + createdBy: c.createdBy, + })), + })), + ) + + await collection.preload() + + expect(toTree(collection)).toEqual([ + { + id: 1, + createdBy: `alice`, + items: [{ id: 10, createdBy: `alice` }], + }, + { + id: 2, + createdBy: `bob`, + items: [], + }, + ]) + }) + + it(`shared correlation key with parent filter + orderBy + limit`, async () => { + // Regression: grouped ordering for limit must use the composite routing + // key, not the raw correlation key. Otherwise two parents that share the + // same correlation key but differ on the parent-referenced filter get + // their children merged before the limit is applied. + type GroupParent = { + id: number + groupId: number + createdBy: string + } + + type GroupChild = { + id: number + groupId: number + createdBy: string + } + + const parents = createCollection( + mockSyncCollectionOptions({ + id: `limit-corr-parents`, + getKey: (p) => p.id, + initialData: [ + { id: 1, groupId: 1, createdBy: `alice` }, + { id: 2, groupId: 1, createdBy: `bob` }, + ], + }), + ) + + const children = createCollection( + mockSyncCollectionOptions({ + id: `limit-corr-children`, + getKey: (c) => c.id, + initialData: [ + { id: 10, groupId: 1, createdBy: `alice` }, + { id: 11, groupId: 1, createdBy: `bob` }, + ], + }), + ) + + const collection = createLiveQueryCollection((q) => + q.from({ p: parents }).select(({ p }) => ({ + id: p.id, + createdBy: p.createdBy, + items: q + .from({ c: children }) + .where(({ c }) => eq(c.groupId, p.groupId)) + .where(({ c }) => eq(c.createdBy, p.createdBy)) + .orderBy(({ c }) => c.id, `asc`) + .limit(1) + .select(({ c }) => ({ + id: c.id, + createdBy: c.createdBy, + })), + })), + ) + + await collection.preload() + + expect(toTree(collection)).toEqual([ + { + id: 1, + createdBy: `alice`, + items: [{ id: 10, createdBy: `alice` }], + }, + { + id: 2, + createdBy: `bob`, + items: [{ id: 11, createdBy: `bob` }], + }, + ]) + }) + + it(`extracts correlation from and() with more than 2 args`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projectsWC }).select(({ p }) => ({ + id: p.id, + name: p.name, + createdBy: p.createdBy, + issues: q + .from({ i: issuesWC }) + .where(({ i }) => + and( + eq(i.projectId, p.id), + eq(i.createdBy, p.createdBy), + eq(i.title, `Bug in Alpha`), + ), + ) + .select(({ i }) => ({ + id: i.id, + title: i.title, + createdBy: i.createdBy, + })), + })), + ) + + await collection.preload() + + // Only project 1 (alice) has an issue matching all three conditions + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 10, title: `Bug in Alpha`, createdBy: `alice` }, + ]) + expect(childItems((collection.get(2) as any).issues)).toEqual([]) + expect(childItems((collection.get(3) as any).issues)).toEqual([]) + }) + + it(`nested includes with parent-referencing filters at both levels`, async () => { + // Regression: nested routing index must use composite routing keys + // (matching the nested buffer keys) so that grandchild changes are + // routed correctly when parent-referencing filters exist at both levels. + type NProject = { id: number; groupId: number; createdBy: string } + type NIssue = { + id: number + groupId: number + createdBy: string + categoryId: number + } + type NComment = { + id: number + categoryId: number + createdBy: string + body: string + } + + const nProjects = createCollection( + mockSyncCollectionOptions({ + id: `nested-pref-projects`, + getKey: (p) => p.id, + initialData: [{ id: 1, groupId: 1, createdBy: `alice` }], + }), + ) + + const nIssues = createCollection( + mockSyncCollectionOptions({ + id: `nested-pref-issues`, + getKey: (i) => i.id, + initialData: [ + { id: 10, groupId: 1, createdBy: `alice`, categoryId: 7 }, + ], + }), + ) + + const nComments = createCollection( + mockSyncCollectionOptions({ + id: `nested-pref-comments`, + getKey: (c) => c.id, + initialData: [ + { id: 100, categoryId: 7, createdBy: `alice`, body: `a` }, + ], + }), + ) + + const collection = createLiveQueryCollection((q) => + q.from({ p: nProjects }).select(({ p }) => ({ + id: p.id, + issues: q + .from({ i: nIssues }) + .where(({ i }) => eq(i.groupId, p.groupId)) + .where(({ i }) => eq(i.createdBy, p.createdBy)) + .select(({ i }) => ({ + id: i.id, + createdBy: i.createdBy, + categoryId: i.categoryId, + comments: q + .from({ c: nComments }) + .where(({ c }) => eq(c.categoryId, i.categoryId)) + .where(({ c }) => eq(c.createdBy, i.createdBy)) + .select(({ c }) => ({ + id: c.id, + createdBy: c.createdBy, + body: c.body, + })), + })), + })), + ) + + await collection.preload() + + expect(toTree(collection)).toEqual([ + { + id: 1, + issues: [ + { + id: 10, + createdBy: `alice`, + categoryId: 7, + comments: [{ id: 100, createdBy: `alice`, body: `a` }], + }, + ], + }, + ]) + }) + + it(`three levels of nested includes with parent-referencing filters`, async () => { + // Verifies that composite routing keys work at arbitrary nesting depth, + // not just the first two levels. + type L0 = { id: number; groupId: number; owner: string } + type L1 = { + id: number + groupId: number + owner: string + tagId: number + } + type L2 = { + id: number + tagId: number + owner: string + flagId: number + } + type L3 = { id: number; flagId: number; owner: string; text: string } + + const l0 = createCollection( + mockSyncCollectionOptions({ + id: `deep-l0`, + getKey: (r) => r.id, + initialData: [{ id: 1, groupId: 1, owner: `alice` }], + }), + ) + const l1 = createCollection( + mockSyncCollectionOptions({ + id: `deep-l1`, + getKey: (r) => r.id, + initialData: [{ id: 10, groupId: 1, owner: `alice`, tagId: 5 }], + }), + ) + const l2 = createCollection( + mockSyncCollectionOptions({ + id: `deep-l2`, + getKey: (r) => r.id, + initialData: [{ id: 100, tagId: 5, owner: `alice`, flagId: 9 }], + }), + ) + const l3 = createCollection( + mockSyncCollectionOptions({ + id: `deep-l3`, + getKey: (r) => r.id, + initialData: [{ id: 1000, flagId: 9, owner: `alice`, text: `deep` }], + }), + ) + + const collection = createLiveQueryCollection((q) => + q.from({ a: l0 }).select(({ a }) => ({ + id: a.id, + children: q + .from({ b: l1 }) + .where(({ b }) => eq(b.groupId, a.groupId)) + .where(({ b }) => eq(b.owner, a.owner)) + .select(({ b }) => ({ + id: b.id, + tagId: b.tagId, + owner: b.owner, + grandchildren: q + .from({ c: l2 }) + .where(({ c }) => eq(c.tagId, b.tagId)) + .where(({ c }) => eq(c.owner, b.owner)) + .select(({ c }) => ({ + id: c.id, + flagId: c.flagId, + owner: c.owner, + leaves: q + .from({ d: l3 }) + .where(({ d }) => eq(d.flagId, c.flagId)) + .where(({ d }) => eq(d.owner, c.owner)) + .select(({ d }) => ({ + id: d.id, + text: d.text, + })), + })), + })), + })), + ) + + await collection.preload() + + expect(toTree(collection)).toEqual([ + { + id: 1, + children: [ + { + id: 10, + tagId: 5, + owner: `alice`, + grandchildren: [ + { + id: 100, + flagId: 9, + owner: `alice`, + leaves: [{ id: 1000, text: `deep` }], + }, + ], + }, + ], + }, + ]) + }) + }) + describe(`validation errors`, () => { it(`throws when child query has no WHERE clause`, () => { expect(() => From b8312bce6920a21dcaa9b7c10c3c6a67c3c5901f Mon Sep 17 00:00:00 2001 From: Kevin Date: Thu, 12 Mar 2026 15:19:49 +0100 Subject: [PATCH 31/32] feat: support aggregates in subqueries (#1298) * feat: add toArray() for includes subqueries toArray() wraps an includes subquery so the parent row contains Array instead of Collection. When children change, the parent row is re-emitted with a fresh array snapshot. - Add ToArrayWrapper class and toArray() function - Add materializeAsArray flag to IncludesSubquery IR node - Detect ToArrayWrapper in builder, pass flag through compiler - Re-emit parent rows on child changes for toArray entries - Add SelectValue type support for ToArrayWrapper - Add tests for basic toArray, reactivity, ordering, and limits Co-Authored-By: Claude Opus 4.6 * Removed obsolete test * Tests for changes to deeply nested queries * Add type-level tests for toArray() includes subqueries Co-Authored-By: Claude Opus 4.6 * Add change propagation tests for includes subqueries Test the reactive model difference between Collection and toArray includes: - Collection includes: child change does NOT re-emit the parent row (the child Collection updates in place) - toArray includes: child change DOES re-emit the parent row (the parent row is re-emitted with the updated array snapshot) Co-Authored-By: Claude Opus 4.6 * Test aggregates inside subqueries * Take into account correlation key when aggregating in subqueries * changeset * ci: apply automated fixes --------- Co-authored-by: Claude Opus 4.6 Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- .changeset/includes-aggregates.md | 5 + packages/db/src/query/compiler/group-by.ts | 89 +- packages/db/src/query/compiler/index.ts | 7 +- packages/db/tests/query/includes.test.ts | 1091 ++++++++++++++++++++ 4 files changed, 1162 insertions(+), 30 deletions(-) create mode 100644 .changeset/includes-aggregates.md diff --git a/.changeset/includes-aggregates.md b/.changeset/includes-aggregates.md new file mode 100644 index 000000000..c846a6a60 --- /dev/null +++ b/.changeset/includes-aggregates.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +fix: support aggregates (e.g. count) in child/includes subqueries with per-parent scoping diff --git a/packages/db/src/query/compiler/group-by.ts b/packages/db/src/query/compiler/group-by.ts index 595d277ae..0309816c8 100644 --- a/packages/db/src/query/compiler/group-by.ts +++ b/packages/db/src/query/compiler/group-by.ts @@ -80,6 +80,7 @@ export function processGroupBy( havingClauses?: Array, selectClause?: Select, fnHavingClauses?: Array<(row: any) => any>, + mainSource?: string, ): NamespacedAndKeyedStream { // Handle empty GROUP BY (single-group aggregation) if (groupByClause.length === 0) { @@ -110,8 +111,15 @@ export function processGroupBy( } } - // Use a constant key for single group - const keyExtractor = () => ({ __singleGroup: true }) + // Use a constant key for single group. + // When mainSource is set (includes mode), include __correlationKey so that + // rows from different parents aggregate separately. + const keyExtractor = mainSource + ? ([, row]: [string, NamespacedRow]) => ({ + __singleGroup: true, + __correlationKey: (row as any)?.[mainSource]?.__correlationKey, + }) + : () => ({ __singleGroup: true }) // Apply the groupBy operator with single group pipeline = pipeline.pipe( @@ -139,14 +147,24 @@ export function processGroupBy( ) } - // Use a single key for the result and update $selected - return [ - `single_group`, - { - ...aggregatedRow, - $selected: finalResults, - }, - ] as [unknown, Record] + // Use a single key for the result and update $selected. + // When in includes mode, restore the namespaced source structure with + // __correlationKey so output extraction can route results per-parent. + const correlationKey = mainSource + ? (aggregatedRow as any).__correlationKey + : undefined + const resultKey = + correlationKey !== undefined + ? `single_group_${serializeValue(correlationKey)}` + : `single_group` + const resultRow: Record = { + ...aggregatedRow, + $selected: finalResults, + } + if (mainSource && correlationKey !== undefined) { + resultRow[mainSource] = { __correlationKey: correlationKey } + } + return [resultKey, resultRow] as [unknown, Record] }), ) @@ -196,7 +214,9 @@ export function processGroupBy( compileExpression(e), ) - // Create a key extractor function using simple __key_X format + // Create a key extractor function using simple __key_X format. + // When mainSource is set (includes mode), include __correlationKey so that + // rows from different parents with the same group key aggregate separately. const keyExtractor = ([, row]: [ string, NamespacedRow & { $selected?: any }, @@ -214,6 +234,10 @@ export function processGroupBy( key[`__key_${i}`] = value } + if (mainSource) { + key.__correlationKey = (row as any)?.[mainSource]?.__correlationKey + } + return key } @@ -278,25 +302,32 @@ export function processGroupBy( } } - // Generate a simple key for the live collection using group values - let finalKey: unknown - if (groupByClause.length === 1) { - finalKey = aggregatedRow[`__key_0`] - } else { - const keyParts: Array = [] - for (let i = 0; i < groupByClause.length; i++) { - keyParts.push(aggregatedRow[`__key_${i}`]) - } - finalKey = serializeValue(keyParts) + // Generate a simple key for the live collection using group values. + // When in includes mode, include the correlation key so that groups + // from different parents don't collide. + const correlationKey = mainSource + ? (aggregatedRow as any).__correlationKey + : undefined + const keyParts: Array = [] + for (let i = 0; i < groupByClause.length; i++) { + keyParts.push(aggregatedRow[`__key_${i}`]) } - - return [ - finalKey, - { - ...aggregatedRow, - $selected: finalResults, - }, - ] as [unknown, Record] + if (correlationKey !== undefined) { + keyParts.push(correlationKey) + } + const finalKey = + keyParts.length === 1 ? keyParts[0] : serializeValue(keyParts) + + // When in includes mode, restore the namespaced source structure with + // __correlationKey so output extraction can route results per-parent. + const resultRow: Record = { + ...aggregatedRow, + $selected: finalResults, + } + if (mainSource && correlationKey !== undefined) { + resultRow[mainSource] = { __correlationKey: correlationKey } + } + return [finalKey, resultRow] as [unknown, Record] }), ) diff --git a/packages/db/src/query/compiler/index.ts b/packages/db/src/query/compiler/index.ts index f33837024..842300981 100644 --- a/packages/db/src/query/compiler/index.ts +++ b/packages/db/src/query/compiler/index.ts @@ -529,7 +529,10 @@ export function compileQuery( ) } - // Process the GROUP BY clause if it exists + // Process the GROUP BY clause if it exists. + // When in includes mode (parentKeyStream), pass mainSource so that groupBy + // preserves __correlationKey for per-parent aggregation. + const groupByMainSource = parentKeyStream ? mainSource : undefined if (query.groupBy && query.groupBy.length > 0) { pipeline = processGroupBy( pipeline, @@ -537,6 +540,7 @@ export function compileQuery( query.having, query.select, query.fnHaving, + groupByMainSource, ) } else if (query.select) { // Check if SELECT contains aggregates but no GROUP BY (implicit single-group aggregation) @@ -551,6 +555,7 @@ export function compileQuery( query.having, query.select, query.fnHaving, + groupByMainSource, ) } } diff --git a/packages/db/tests/query/includes.test.ts b/packages/db/tests/query/includes.test.ts index 12fa9e0df..51f648bf4 100644 --- a/packages/db/tests/query/includes.test.ts +++ b/packages/db/tests/query/includes.test.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { and, + count, createLiveQueryCollection, eq, toArray, @@ -412,6 +413,93 @@ describe(`includes subqueries`, () => { }) }) + describe(`change propagation`, () => { + it(`Collection includes: child change does not re-emit the parent row`, async () => { + const collection = buildIncludesQuery() + await collection.preload() + + const changeCallback = vi.fn() + const subscription = collection.subscribeChanges(changeCallback, { + includeInitialState: false, + }) + changeCallback.mockClear() + + // Add a child issue to project Alpha + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 12, projectId: 1, title: `New Alpha issue` }, + }) + issues.utils.commit() + + // Wait for async change propagation + await new Promise((resolve) => setTimeout(resolve, 10)) + + // The child Collection updates in place — the parent row should NOT be re-emitted + expect(changeCallback).not.toHaveBeenCalled() + + // But the child data is there + expect(childItems((collection.get(1) as any).issues)).toEqual([ + { id: 10, title: `Bug in Alpha` }, + { id: 11, title: `Feature for Alpha` }, + { id: 12, title: `New Alpha issue` }, + ]) + + subscription.unsubscribe() + }) + + it(`toArray includes: child change re-emits the parent row`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: toArray( + q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + ), + })), + ) + + await collection.preload() + + const changeCallback = vi.fn() + const subscription = collection.subscribeChanges(changeCallback, { + includeInitialState: false, + }) + changeCallback.mockClear() + + // Add a child issue to project Alpha + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 12, projectId: 1, title: `New Alpha issue` }, + }) + issues.utils.commit() + + // Wait for async change propagation + await new Promise((resolve) => setTimeout(resolve, 10)) + + // The parent row SHOULD be re-emitted with the updated array + expect(changeCallback).toHaveBeenCalled() + + // Verify the parent row has the updated array + const alpha = collection.get(1) as any + expect(Array.isArray(alpha.issues)).toBe(true) + expect(alpha.issues.sort((a: any, b: any) => a.id - b.id)).toEqual([ + { id: 10, title: `Bug in Alpha` }, + { id: 11, title: `Feature for Alpha` }, + { id: 12, title: `New Alpha issue` }, + ]) + + subscription.unsubscribe() + }) + }) + describe(`inner join filtering`, () => { it(`only shows children for parents matching a WHERE clause`, async () => { const collection = createLiveQueryCollection((q) => @@ -2755,4 +2843,1007 @@ describe(`includes subqueries`, () => { ]) }) }) + + describe(`toArray`, () => { + function buildToArrayQuery() { + return createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: toArray( + q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + ), + })), + ) + } + + it(`produces arrays on parent rows, not Collections`, async () => { + const collection = buildToArrayQuery() + await collection.preload() + + const alpha = collection.get(1) as any + expect(Array.isArray(alpha.issues)).toBe(true) + expect(alpha.issues.sort((a: any, b: any) => a.id - b.id)).toEqual([ + { id: 10, title: `Bug in Alpha` }, + { id: 11, title: `Feature for Alpha` }, + ]) + + const beta = collection.get(2) as any + expect(Array.isArray(beta.issues)).toBe(true) + expect(beta.issues).toEqual([{ id: 20, title: `Bug in Beta` }]) + }) + + it(`empty parents get empty arrays`, async () => { + const collection = buildToArrayQuery() + await collection.preload() + + const gamma = collection.get(3) as any + expect(Array.isArray(gamma.issues)).toBe(true) + expect(gamma.issues).toEqual([]) + }) + + it(`adding a child re-emits the parent with updated array`, async () => { + const collection = buildToArrayQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 12, projectId: 1, title: `New Alpha issue` }, + }) + issues.utils.commit() + + const alpha = collection.get(1) as any + expect(Array.isArray(alpha.issues)).toBe(true) + expect(alpha.issues.sort((a: any, b: any) => a.id - b.id)).toEqual([ + { id: 10, title: `Bug in Alpha` }, + { id: 11, title: `Feature for Alpha` }, + { id: 12, title: `New Alpha issue` }, + ]) + }) + + it(`removing a child re-emits the parent with updated array`, async () => { + const collection = buildToArrayQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `delete`, + value: sampleIssues.find((i) => i.id === 10)!, + }) + issues.utils.commit() + + const alpha = collection.get(1) as any + expect(alpha.issues).toEqual([{ id: 11, title: `Feature for Alpha` }]) + }) + + it(`array respects ORDER BY`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: toArray( + q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .orderBy(({ i }) => i.title, `asc`) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + ), + })), + ) + + await collection.preload() + + const alpha = collection.get(1) as any + expect(alpha.issues).toEqual([ + { id: 10, title: `Bug in Alpha` }, + { id: 11, title: `Feature for Alpha` }, + ]) + }) + + it(`ordered toArray with limit applied per parent`, async () => { + const collection = createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: toArray( + q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .orderBy(({ i }) => i.title, `asc`) + .limit(1) + .select(({ i }) => ({ + id: i.id, + title: i.title, + })), + ), + })), + ) + + await collection.preload() + + const alpha = collection.get(1) as any + expect(alpha.issues).toEqual([{ id: 10, title: `Bug in Alpha` }]) + + const beta = collection.get(2) as any + expect(beta.issues).toEqual([{ id: 20, title: `Bug in Beta` }]) + + const gamma = collection.get(3) as any + expect(gamma.issues).toEqual([]) + }) + }) + + describe(`nested includes: Collection → toArray`, () => { + function buildCollectionToArrayQuery() { + return createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + comments: toArray( + q + .from({ c: comments }) + .where(({ c }) => eq(c.issueId, i.id)) + .select(({ c }) => ({ + id: c.id, + body: c.body, + })), + ), + })), + })), + ) + } + + it(`initial load: issues are Collections, comments are arrays`, async () => { + const collection = buildCollectionToArrayQuery() + await collection.preload() + + const alpha = collection.get(1) as any + // issues should be a Collection + expect(alpha.issues.toArray).toBeDefined() + + const issue10 = alpha.issues.get(10) + // comments should be an array + expect(Array.isArray(issue10.comments)).toBe(true) + expect(issue10.comments.sort((a: any, b: any) => a.id - b.id)).toEqual([ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ]) + + const issue11 = alpha.issues.get(11) + expect(Array.isArray(issue11.comments)).toBe(true) + expect(issue11.comments).toEqual([]) + }) + + it(`adding a comment (grandchild-only change) updates the issue's comments array`, async () => { + const collection = buildCollectionToArrayQuery() + await collection.preload() + + const issue11Before = (collection.get(1) as any).issues.get(11) + expect(issue11Before.comments).toEqual([]) + + comments.utils.begin() + comments.utils.write({ + type: `insert`, + value: { id: 110, issueId: 11, body: `Great feature` }, + }) + comments.utils.commit() + + const issue11After = (collection.get(1) as any).issues.get(11) + expect(Array.isArray(issue11After.comments)).toBe(true) + expect(issue11After.comments).toEqual([ + { id: 110, body: `Great feature` }, + ]) + }) + + it(`removing a comment (grandchild-only change) updates the issue's comments array`, async () => { + const collection = buildCollectionToArrayQuery() + await collection.preload() + + const issue10Before = (collection.get(1) as any).issues.get(10) + expect(issue10Before.comments).toHaveLength(2) + + comments.utils.begin() + comments.utils.write({ + type: `delete`, + value: sampleComments.find((c) => c.id === 100)!, + }) + comments.utils.commit() + + const issue10After = (collection.get(1) as any).issues.get(10) + expect(issue10After.comments).toEqual([{ id: 101, body: `Fixed it` }]) + }) + + it(`adding an issue (middle-level insert) creates a child with empty comments array`, async () => { + const collection = buildCollectionToArrayQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 30, projectId: 3, title: `Gamma issue` }, + }) + issues.utils.commit() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { + id: 10, + title: `Bug in Alpha`, + comments: [ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ], + }, + { id: 11, title: `Feature for Alpha`, comments: [] }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [ + { + id: 20, + title: `Bug in Beta`, + comments: [{ id: 200, body: `Same bug` }], + }, + ], + }, + { + id: 3, + name: `Gamma`, + issues: [{ id: 30, title: `Gamma issue`, comments: [] }], + }, + ]) + }) + + it(`removing an issue (middle-level delete) removes it from the parent`, async () => { + const collection = buildCollectionToArrayQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `delete`, + value: sampleIssues.find((i) => i.id === 11)!, + }) + issues.utils.commit() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { + id: 10, + title: `Bug in Alpha`, + comments: [ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ], + }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [ + { + id: 20, + title: `Bug in Beta`, + comments: [{ id: 200, body: `Same bug` }], + }, + ], + }, + { + id: 3, + name: `Gamma`, + issues: [], + }, + ]) + }) + + it(`updating an issue title (middle-level update) reflects in the parent`, async () => { + const collection = buildCollectionToArrayQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `update`, + value: { id: 10, projectId: 1, title: `Renamed Bug` }, + }) + issues.utils.commit() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { + id: 10, + title: `Renamed Bug`, + comments: [ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ], + }, + { id: 11, title: `Feature for Alpha`, comments: [] }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [ + { + id: 20, + title: `Bug in Beta`, + comments: [{ id: 200, body: `Same bug` }], + }, + ], + }, + { + id: 3, + name: `Gamma`, + issues: [], + }, + ]) + }) + }) + + describe(`nested includes: toArray → Collection`, () => { + function buildToArrayToCollectionQuery() { + return createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: toArray( + q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + comments: q + .from({ c: comments }) + .where(({ c }) => eq(c.issueId, i.id)) + .select(({ c }) => ({ + id: c.id, + body: c.body, + })), + })), + ), + })), + ) + } + + it(`initial load: issues are arrays, comments are Collections`, async () => { + const collection = buildToArrayToCollectionQuery() + await collection.preload() + + const alpha = collection.get(1) as any + expect(Array.isArray(alpha.issues)).toBe(true) + + const sortedIssues = alpha.issues.sort((a: any, b: any) => a.id - b.id) + // comments should be Collections + expect(sortedIssues[0].comments.toArray).toBeDefined() + expect(childItems(sortedIssues[0].comments)).toEqual([ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ]) + + expect(sortedIssues[1].comments.toArray).toBeDefined() + expect(childItems(sortedIssues[1].comments)).toEqual([]) + }) + + it(`adding a comment updates the nested Collection (live reference)`, async () => { + const collection = buildToArrayToCollectionQuery() + await collection.preload() + + const alpha = collection.get(1) as any + const issue11 = alpha.issues.find((i: any) => i.id === 11) + expect(childItems(issue11.comments)).toEqual([]) + + comments.utils.begin() + comments.utils.write({ + type: `insert`, + value: { id: 110, issueId: 11, body: `Great feature` }, + }) + comments.utils.commit() + + // The Collection reference on the issue object is live + expect(childItems(issue11.comments)).toEqual([ + { id: 110, body: `Great feature` }, + ]) + }) + + it(`adding an issue re-emits the parent with updated array including nested Collection`, async () => { + const collection = buildToArrayToCollectionQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 30, projectId: 3, title: `Gamma issue` }, + }) + issues.utils.commit() + + const gamma = collection.get(3) as any + expect(Array.isArray(gamma.issues)).toBe(true) + expect(gamma.issues).toHaveLength(1) + expect(gamma.issues[0].id).toBe(30) + expect(gamma.issues[0].comments.toArray).toBeDefined() + expect(childItems(gamma.issues[0].comments)).toEqual([]) + }) + + it(`removing an issue re-emits the parent with updated array`, async () => { + const collection = buildToArrayToCollectionQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `delete`, + value: sampleIssues.find((i) => i.id === 11)!, + }) + issues.utils.commit() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { + id: 10, + title: `Bug in Alpha`, + comments: [ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ], + }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [ + { + id: 20, + title: `Bug in Beta`, + comments: [{ id: 200, body: `Same bug` }], + }, + ], + }, + { + id: 3, + name: `Gamma`, + issues: [], + }, + ]) + }) + + it(`updating an issue title re-emits the parent with updated array`, async () => { + const collection = buildToArrayToCollectionQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `update`, + value: { id: 10, projectId: 1, title: `Renamed Bug` }, + }) + issues.utils.commit() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { + id: 10, + title: `Renamed Bug`, + comments: [ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ], + }, + { id: 11, title: `Feature for Alpha`, comments: [] }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [ + { + id: 20, + title: `Bug in Beta`, + comments: [{ id: 200, body: `Same bug` }], + }, + ], + }, + { + id: 3, + name: `Gamma`, + issues: [], + }, + ]) + }) + }) + + describe(`nested includes: toArray → toArray`, () => { + function buildToArrayToArrayQuery() { + return createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: toArray( + q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + comments: toArray( + q + .from({ c: comments }) + .where(({ c }) => eq(c.issueId, i.id)) + .select(({ c }) => ({ + id: c.id, + body: c.body, + })), + ), + })), + ), + })), + ) + } + + it(`initial load: both levels are arrays`, async () => { + const collection = buildToArrayToArrayQuery() + await collection.preload() + + const alpha = collection.get(1) as any + expect(Array.isArray(alpha.issues)).toBe(true) + + const sortedIssues = alpha.issues.sort((a: any, b: any) => a.id - b.id) + expect(Array.isArray(sortedIssues[0].comments)).toBe(true) + expect( + sortedIssues[0].comments.sort((a: any, b: any) => a.id - b.id), + ).toEqual([ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ]) + + expect(sortedIssues[1].comments).toEqual([]) + }) + + it(`adding a comment (grandchild-only change) updates both array levels`, async () => { + const collection = buildToArrayToArrayQuery() + await collection.preload() + + comments.utils.begin() + comments.utils.write({ + type: `insert`, + value: { id: 110, issueId: 11, body: `Great feature` }, + }) + comments.utils.commit() + + const alpha = collection.get(1) as any + expect(Array.isArray(alpha.issues)).toBe(true) + const issue11 = alpha.issues.find((i: any) => i.id === 11) + expect(Array.isArray(issue11.comments)).toBe(true) + expect(issue11.comments).toEqual([{ id: 110, body: `Great feature` }]) + }) + + it(`removing a comment (grandchild-only change) updates both array levels`, async () => { + const collection = buildToArrayToArrayQuery() + await collection.preload() + + comments.utils.begin() + comments.utils.write({ + type: `delete`, + value: sampleComments.find((c) => c.id === 100)!, + }) + comments.utils.commit() + + const alpha = collection.get(1) as any + const issue10 = alpha.issues.find((i: any) => i.id === 10) + expect(issue10.comments).toEqual([{ id: 101, body: `Fixed it` }]) + }) + + it(`adding an issue (middle-level insert) re-emits parent with updated array`, async () => { + const collection = buildToArrayToArrayQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 30, projectId: 3, title: `Gamma issue` }, + }) + issues.utils.commit() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { + id: 10, + title: `Bug in Alpha`, + comments: [ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ], + }, + { id: 11, title: `Feature for Alpha`, comments: [] }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [ + { + id: 20, + title: `Bug in Beta`, + comments: [{ id: 200, body: `Same bug` }], + }, + ], + }, + { + id: 3, + name: `Gamma`, + issues: [{ id: 30, title: `Gamma issue`, comments: [] }], + }, + ]) + }) + + it(`removing an issue (middle-level delete) re-emits parent with updated array`, async () => { + const collection = buildToArrayToArrayQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `delete`, + value: sampleIssues.find((i) => i.id === 11)!, + }) + issues.utils.commit() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { + id: 10, + title: `Bug in Alpha`, + comments: [ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ], + }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [ + { + id: 20, + title: `Bug in Beta`, + comments: [{ id: 200, body: `Same bug` }], + }, + ], + }, + { + id: 3, + name: `Gamma`, + issues: [], + }, + ]) + }) + + it(`updating an issue title (middle-level update) re-emits parent with updated array`, async () => { + const collection = buildToArrayToArrayQuery() + await collection.preload() + + issues.utils.begin() + issues.utils.write({ + type: `update`, + value: { id: 10, projectId: 1, title: `Renamed Bug` }, + }) + issues.utils.commit() + + expect(toTree(collection)).toEqual([ + { + id: 1, + name: `Alpha`, + issues: [ + { + id: 10, + title: `Renamed Bug`, + comments: [ + { id: 100, body: `Looks bad` }, + { id: 101, body: `Fixed it` }, + ], + }, + { id: 11, title: `Feature for Alpha`, comments: [] }, + ], + }, + { + id: 2, + name: `Beta`, + issues: [ + { + id: 20, + title: `Bug in Beta`, + comments: [{ id: 200, body: `Same bug` }], + }, + ], + }, + { + id: 3, + name: `Gamma`, + issues: [], + }, + ]) + }) + + it(`concurrent child + grandchild changes in the same transaction`, async () => { + const collection = buildToArrayToArrayQuery() + await collection.preload() + + // Add a new issue AND a comment on an existing issue in one transaction + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 30, projectId: 3, title: `Gamma issue` }, + }) + issues.utils.commit() + + comments.utils.begin() + comments.utils.write({ + type: `insert`, + value: { id: 110, issueId: 11, body: `Great feature` }, + }) + comments.utils.commit() + + // Gamma should have the new issue with empty comments + const gamma = collection.get(3) as any + expect(gamma.issues).toHaveLength(1) + expect(gamma.issues[0].id).toBe(30) + expect(gamma.issues[0].comments).toEqual([]) + + // Alpha's issue 11 should have the new comment + const alpha = collection.get(1) as any + const issue11 = alpha.issues.find((i: any) => i.id === 11) + expect(issue11.comments).toEqual([{ id: 110, body: `Great feature` }]) + }) + }) + + // Aggregates in child queries: the aggregate (e.g. count) should be computed + // per-parent, not globally across all parents. Currently, the correlation key + // is lost after GROUP BY, causing all child rows to aggregate into a single + // global result rather than per-parent results. + describe(`aggregates in child queries`, () => { + describe(`single-group aggregate: count issues per project (as Collection)`, () => { + function buildAggregateQuery() { + return createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issueCount: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ total: count(i.id) })), + })), + ) + } + + it(`each project gets its own aggregate result`, async () => { + const collection = buildAggregateQuery() + await collection.preload() + + // Alpha has 2 issues + const alpha = collection.get(1) as any + expect(childItems(alpha.issueCount, `total`)).toEqual([{ total: 2 }]) + + // Beta has 1 issue + const beta = collection.get(2) as any + expect(childItems(beta.issueCount, `total`)).toEqual([{ total: 1 }]) + + // Gamma has 0 issues — no matching rows means empty Collection + const gamma = collection.get(3) as any + expect(childItems(gamma.issueCount, `total`)).toEqual([]) + }) + + it(`adding an issue updates the count for that parent`, async () => { + const collection = buildAggregateQuery() + await collection.preload() + + // Gamma starts with 0 issues + expect( + childItems((collection.get(3) as any).issueCount, `total`), + ).toEqual([]) + + issues.utils.begin() + issues.utils.write({ + type: `insert`, + value: { id: 30, projectId: 3, title: `Gamma issue` }, + }) + issues.utils.commit() + + // Gamma now has 1 issue + expect( + childItems((collection.get(3) as any).issueCount, `total`), + ).toEqual([{ total: 1 }]) + + // Alpha should still have 2 + expect( + childItems((collection.get(1) as any).issueCount, `total`), + ).toEqual([{ total: 2 }]) + }) + + it(`removing an issue updates the count for that parent`, async () => { + const collection = buildAggregateQuery() + await collection.preload() + + // Alpha starts with 2 issues + expect( + childItems((collection.get(1) as any).issueCount, `total`), + ).toEqual([{ total: 2 }]) + + issues.utils.begin() + issues.utils.write({ + type: `delete`, + value: sampleIssues.find((i) => i.id === 10)!, + }) + issues.utils.commit() + + // Alpha now has 1 issue + expect( + childItems((collection.get(1) as any).issueCount, `total`), + ).toEqual([{ total: 1 }]) + + // Beta should still have 1 + expect( + childItems((collection.get(2) as any).issueCount, `total`), + ).toEqual([{ total: 1 }]) + }) + }) + + describe(`single-group aggregate: count issues per project (as toArray)`, () => { + function buildAggregateToArrayQuery() { + return createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issueCount: toArray( + q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ total: count(i.id) })), + ), + })), + ) + } + + it(`each project gets its own aggregate result as an array`, async () => { + const collection = buildAggregateToArrayQuery() + await collection.preload() + + // Alpha has 2 issues + const alpha = collection.get(1) as any + expect(alpha.issueCount).toEqual([{ total: 2 }]) + + // Beta has 1 issue + const beta = collection.get(2) as any + expect(beta.issueCount).toEqual([{ total: 1 }]) + + // Gamma has 0 issues — empty array + const gamma = collection.get(3) as any + expect(gamma.issueCount).toEqual([]) + }) + }) + + describe(`nested aggregate: count comments per issue (as Collection)`, () => { + function buildNestedAggregateQuery() { + return createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + commentCount: q + .from({ c: comments }) + .where(({ c }) => eq(c.issueId, i.id)) + .select(({ c }) => ({ total: count(c.id) })), + })), + })), + ) + } + + it(`each issue gets its own comment count`, async () => { + const collection = buildNestedAggregateQuery() + await collection.preload() + + // Alpha's issues + const alpha = collection.get(1) as any + const issue10 = alpha.issues.get(10) + expect(childItems(issue10.commentCount, `total`)).toEqual([ + { total: 2 }, + ]) + + const issue11 = alpha.issues.get(11) + // Issue 11 has 0 comments — empty Collection + expect(childItems(issue11.commentCount, `total`)).toEqual([]) + + // Beta's issue + const beta = collection.get(2) as any + const issue20 = beta.issues.get(20) + expect(childItems(issue20.commentCount, `total`)).toEqual([ + { total: 1 }, + ]) + }) + }) + + describe(`nested aggregate: count comments per issue (as toArray)`, () => { + function buildNestedAggregateToArrayQuery() { + return createLiveQueryCollection((q) => + q.from({ p: projects }).select(({ p }) => ({ + id: p.id, + name: p.name, + issues: q + .from({ i: issues }) + .where(({ i }) => eq(i.projectId, p.id)) + .select(({ i }) => ({ + id: i.id, + title: i.title, + commentCount: toArray( + q + .from({ c: comments }) + .where(({ c }) => eq(c.issueId, i.id)) + .select(({ c }) => ({ total: count(c.id) })), + ), + })), + })), + ) + } + + it(`each issue gets its own comment count as an array`, async () => { + const collection = buildNestedAggregateToArrayQuery() + await collection.preload() + + // Alpha's issues + const alpha = collection.get(1) as any + const issue10 = alpha.issues.get(10) + expect(issue10.commentCount).toEqual([{ total: 2 }]) + + const issue11 = alpha.issues.get(11) + // Issue 11 has 0 comments — empty array + expect(issue11.commentCount).toEqual([]) + + // Beta's issue + const beta = collection.get(2) as any + const issue20 = beta.issues.get(20) + expect(issue20.commentCount).toEqual([{ total: 1 }]) + }) + }) + }) }) From 45d87e859d98f7a116798fe02483e37f1743e1bf Mon Sep 17 00:00:00 2001 From: Kevin Date: Mon, 16 Mar 2026 11:22:37 +0100 Subject: [PATCH 32/32] Update changeset to patch release Co-authored-by: Sam Willis --- .changeset/includes-subqueries.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/includes-subqueries.md b/.changeset/includes-subqueries.md index caa9f44d4..950d94d52 100644 --- a/.changeset/includes-subqueries.md +++ b/.changeset/includes-subqueries.md @@ -1,5 +1,5 @@ --- -'@tanstack/db': minor +'@tanstack/db': patch --- feat: support for subqueries for including hierarchical data in live queries