fix(db): ensure deterministic iteration order for collections and indexes

- SortedMap: add key-based tie-breaking for deterministic ordering
- SortedMap: optimize to skip value comparison when no comparator provided
- BTreeIndex: sort keys within same indexed value for deterministic order
- BTreeIndex: add fast paths for empty/single-key sets
- CollectionStateManager: always use SortedMap for deterministic iteration
- Extract compareKeys utility to utils/comparison.ts
- Add comprehensive tests for deterministic ordering behavior

Author: samwillis

.changeset/deterministic-collection-ordering.md

@@ -0,0 +1,23 @@
+---
+"@tanstack/db": patch
+---
+
+Ensure deterministic iteration order for collections and indexes.
+
+**SortedMap improvements:**
+
+- Added key-based tie-breaking when values compare as equal, ensuring deterministic ordering
+- Optimized to skip value comparison entirely when no comparator is provided (key-only sorting)
+- Extracted `compareKeys` utility to `utils/comparison.ts` for reuse
+
+**BTreeIndex improvements:**
+
+- Keys within the same indexed value are now returned in deterministic sorted order
+- Optimized with fast paths for empty sets and single-key sets to avoid unnecessary allocations
+
+**CollectionStateManager changes:**
+
+- Collections now always use `SortedMap` for `syncedData`, ensuring deterministic iteration order
+- When no `compare` function is provided, entries are sorted by key only
+
+This ensures that live queries with `orderBy` and `limit` produce stable, deterministic results even when multiple rows have equal sort values.

packages/db/src/SortedMap.ts

@@ -1,62 +1,81 @@
+import { compareKeys } from "./utils/comparison.js"
+
 /**
  * A Map implementation that keeps its entries sorted based on a comparator function
- * @template TKey - The type of keys in the map
+ * @template TKey - The type of keys in the map (must be string | number)
  * @template TValue - The type of values in the map
  */
-export class SortedMap<TKey, TValue> {
+export class SortedMap<TKey extends string | number, TValue> {
   private map: Map<TKey, TValue>
   private sortedKeys: Array<TKey>
-  private comparator: (a: TValue, b: TValue) => number
+  private comparator: ((a: TValue, b: TValue) => number) | undefined
 
   /**
    * Creates a new SortedMap instance
    *
-   * @param comparator - Optional function to compare values for sorting
+   * @param comparator - Optional function to compare values for sorting.
+   *                     If not provided, entries are sorted by key only.
    */
   constructor(comparator?: (a: TValue, b: TValue) => number) {
     this.map = new Map<TKey, TValue>()
     this.sortedKeys = []
-    this.comparator = comparator || this.defaultComparator
-  }
-
-  /**
-   * Default comparator function used when none is provided
-   *
-   * @param a - First value to compare
-   * @param b - Second value to compare
-   * @returns -1 if a < b, 1 if a > b, 0 if equal
-   */
-  private defaultComparator(a: TValue, b: TValue): number {
-    if (a < b) return -1
-    if (a > b) return 1
-    return 0
+    this.comparator = comparator
   }
 
   /**
    * Finds the index where a key-value pair should be inserted to maintain sort order.
-   * Uses binary search to find the correct position based on the value.
-   * Hence, it is in O(log n) time.
+   * Uses binary search to find the correct position based on the value (if comparator provided),
+   * with key-based tie-breaking for deterministic ordering when values compare as equal.
+   * If no comparator is provided, sorts by key only.
+   * Runs in O(log n) time.
    *
-   * @param key - The key to find position for
-   * @param value - The value to compare against
+   * @param key - The key to find position for (used as tie-breaker or primary sort when no comparator)
+   * @param value - The value to compare against (only used if comparator is provided)
    * @returns The index where the key should be inserted
    */
-  private indexOf(value: TValue): number {
+  private indexOf(key: TKey, value: TValue): number {
     let left = 0
     let right = this.sortedKeys.length
 
+    // Fast path: no comparator means sort by key only
+    if (!this.comparator) {
+      while (left < right) {
+        const mid = Math.floor((left + right) / 2)
+        const midKey = this.sortedKeys[mid]!
+        const keyComparison = compareKeys(key, midKey)
+        if (keyComparison < 0) {
+          right = mid
+        } else if (keyComparison > 0) {
+          left = mid + 1
+        } else {
+          return mid
+        }
+      }
+      return left
+    }
+
+    // With comparator: sort by value first, then key as tie-breaker
     while (left < right) {
       const mid = Math.floor((left + right) / 2)
       const midKey = this.sortedKeys[mid]!
       const midValue = this.map.get(midKey)!
-      const comparison = this.comparator(value, midValue)
+      const valueComparison = this.comparator(value, midValue)
 
-      if (comparison < 0) {
+      if (valueComparison < 0) {
         right = mid
-      } else if (comparison > 0) {
+      } else if (valueComparison > 0) {
         left = mid + 1
       } else {
-        return mid
+        // Values are equal, use key as tie-breaker for deterministic ordering
+        const keyComparison = compareKeys(key, midKey)
+        if (keyComparison < 0) {
+          right = mid
+        } else if (keyComparison > 0) {
+          left = mid + 1
+        } else {
+          // Same key (shouldn't happen during insert, but handle for lookups)
+          return mid
+        }
       }
     }
 
@@ -74,12 +93,12 @@ export class SortedMap<TKey, TValue> {
     if (this.map.has(key)) {
       // Need to remove the old key from the sorted keys array
       const oldValue = this.map.get(key)!
-      const oldIndex = this.indexOf(oldValue)
+      const oldIndex = this.indexOf(key, oldValue)
       this.sortedKeys.splice(oldIndex, 1)
     }
 
     // Insert the new key at the correct position
-    const index = this.indexOf(value)
+    const index = this.indexOf(key, value)
     this.sortedKeys.splice(index, 0, key)
 
     this.map.set(key, value)
@@ -106,7 +125,7 @@ export class SortedMap<TKey, TValue> {
   delete(key: TKey): boolean {
     if (this.map.has(key)) {
       const oldValue = this.map.get(key)
-      const index = this.indexOf(oldValue!)
+      const index = this.indexOf(key, oldValue!)
       this.sortedKeys.splice(index, 1)
       return this.map.delete(key)
     }

packages/db/src/collection/state.ts

@@ -43,7 +43,7 @@ export class CollectionStateManager<
   public pendingSyncedTransactions: Array<
     PendingSyncedTransaction<TOutput, TKey>
   > = []
-  public syncedData: Map<TKey, TOutput> | SortedMap<TKey, TOutput>
+  public syncedData: SortedMap<TKey, TOutput>
   public syncedMetadata = new Map<TKey, unknown>()
 
   // Optimistic state tracking - make public for testing
@@ -69,12 +69,9 @@ export class CollectionStateManager<
       a.compareCreatedAt(b)
     )
 
-    // Set up data storage with optional comparison function
-    if (config.compare) {
-      this.syncedData = new SortedMap<TKey, TOutput>(config.compare)
-    } else {
-      this.syncedData = new Map<TKey, TOutput>()
-    }
+    // Set up data storage - always use SortedMap for deterministic iteration.
+    // If a custom compare function is provided, use it; otherwise entries are sorted by key only.
+    this.syncedData = new SortedMap<TKey, TOutput>(config.compare)
   }
 
   setDeps(deps: {

packages/db/src/indexes/btree-index.ts

@@ -1,5 +1,9 @@
 import { BTree } from "../utils/btree.js"
-import { defaultComparator, normalizeValue } from "../utils/comparison.js"
+import {
+  compareKeys,
+  defaultComparator,
+  normalizeValue,
+} from "../utils/comparison.js"
 import { BaseIndex } from "./base-index.js"
 import type { CompareOptions } from "../query/builder/types.js"
 import type { BasicExpression } from "../query/ir.js"
@@ -261,7 +265,8 @@ export class BTreeIndex<
     n: number,
     nextPair: (k?: any) => [any, any] | undefined,
     from?: any,
-    filterFn?: (key: TKey) => boolean
+    filterFn?: (key: TKey) => boolean,
+    reversed: boolean = false
   ): Array<TKey> {
     const keysInResult: Set<TKey> = new Set()
     const result: Array<TKey> = []
@@ -271,14 +276,25 @@ export class BTreeIndex<
     while ((pair = nextPair(key)) !== undefined && result.length < n) {
       key = pair[0]
       const keys = this.valueMap.get(key)
-      if (keys) {
-        const it = keys.values()
-        let ks: TKey | undefined
-        while (result.length < n && (ks = it.next().value)) {
+      if (keys && keys.size > 0) {
+        // Fast path: single key doesn't need sorting
+        if (keys.size === 1) {
+          const ks = keys.values().next().value as TKey
           if (!keysInResult.has(ks) && (filterFn?.(ks) ?? true)) {
             result.push(ks)
             keysInResult.add(ks)
           }
+        } else {
+          // Sort keys for deterministic order, reverse if needed
+          const sorted = Array.from(keys).sort(compareKeys)
+          if (reversed) sorted.reverse()
+          for (const ks of sorted) {
+            if (result.length >= n) break
+            if (!keysInResult.has(ks) && (filterFn?.(ks) ?? true)) {
+              result.push(ks)
+              keysInResult.add(ks)
+            }
+          }
         }
       }
     }
@@ -309,7 +325,7 @@ export class BTreeIndex<
     filterFn?: (key: TKey) => boolean
   ): Array<TKey> {
     const nextPair = (k?: any) => this.orderedEntries.nextLowerPair(k)
-    return this.takeInternal(n, nextPair, from, filterFn)
+    return this.takeInternal(n, nextPair, from, filterFn, true)
   }
 
   /**

packages/db/src/utils/comparison.ts

@@ -189,3 +189,21 @@ export function areValuesEqual(a: any, b: any): boolean {
   // Different types or not Uint8Arrays
   return false
 }
+
+/**
+ * Compares two keys (string | number) for deterministic ordering.
+ * Strings come before numbers, then sorted within type.
+ */
+export function compareKeys<TKey extends string | number>(
+  a: TKey,
+  b: TKey
+): number {
+  // Same type: compare directly
+  if (typeof a === typeof b) {
+    if (a < b) return -1
+    if (a > b) return 1
+    return 0
+  }
+  // Different types: strings come before numbers
+  return typeof a === `string` ? -1 : 1
+}