From 9f8905159d1f782021caf64c116b982328449cf1 Mon Sep 17 00:00:00 2001 From: HashEngineering Date: Thu, 5 Nov 2026 10:26:45 -0800 Subject: [PATCH 1/6] feat(kotlin-sdk): port #4638 to Android, as in #5767 Credit verdicts at save time and the post-sync TXO reconcile, with the reconcile policy in Kotlin. Taken from #5767 at 093a8e2ba6 as the starting point for moving that policy onto the Rust reconciler. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 3 + .../dashsdk/ffi/NativePersistenceBridge.kt | 25 + .../dashsdk/ffi/WalletManagerNative.kt | 83 + .../PlatformWalletPersistenceHandler.kt | 1626 ++++++++++++- .../dashsdk/persistence/dao/TxoDao.kt | 21 + .../dashsdk/wallet/PlatformWalletManager.kt | 214 +- .../dashsdk/wallet/TxoReconcile.kt | 213 ++ .../PlatformWalletPersistenceHandlerTest.kt | 2083 +++++++++++++++++ .../dashsdk/wallet/TxoReconcileTest.kt | 285 +++ .../golden/txo_reconcile_transport_v1.json | 123 + packages/rs-unified-sdk-jni/Cargo.toml | 8 + .../rs-unified-sdk-jni/src/persistence.rs | 110 +- .../rs-unified-sdk-jni/src/wallet_manager.rs | 932 ++++++++ 13 files changed, 5613 insertions(+), 113 deletions(-) create mode 100644 packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcile.kt create mode 100644 packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcileTest.kt create mode 100644 packages/kotlin-sdk/sdk/src/test/resources/golden/txo_reconcile_transport_v1.json diff --git a/Cargo.lock b/Cargo.lock index 7399dfdebe6..7fa50b4e1ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6689,10 +6689,13 @@ dependencies = [ "dashcore", "dashpay-contract", "dpp", + "hex", "jni 0.21.1", "log", "platform-wallet-ffi", "rs-sdk-ffi", + "serde", + "serde_json", "zeroize", ] diff --git a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/ffi/NativePersistenceBridge.kt b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/ffi/NativePersistenceBridge.kt index ae4d7946395..6f84b27ce94 100644 --- a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/ffi/NativePersistenceBridge.kt +++ b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/ffi/NativePersistenceBridge.kt @@ -522,6 +522,31 @@ abstract class NativePersistenceBridge { clearedIdentities: ByteArray, ): Int = 0 + /** + * The engine's credit verdicts for the round (dashpay/platform#4638): + * every `Received` / `Change` output of the round's records that the + * engine did NOT credit to the owning account. Fired inside the + * begin/end bracket BEFORE the changeset callback, and only on rounds + * that carry at least one verdict, so an implementation has them in + * hand when the round's [onWalletChangesetUtxoAdded] entries arrive. + * Descriptor `([B[B[B[II)I`. + * + * [outpoints] is [count] 36-byte keys (32-byte wire-order txid plus a + * little-endian vout) packed into one array; [verdicts] holds one + * `UTXO_CREDIT_VERDICT_*` code per outpoint (1 observed spent, 2 + * doomed, 3 uncredited) and [spentAtHeights] the observed spending + * block's height for code 1, 0 otherwise. Purely additive: a host that + * ignores the verdicts writes exactly what it wrote before the slot + * existed. + */ + open fun onWalletChangesetUtxoVerdicts( + walletId: ByteArray, + outpoints: ByteArray, + verdicts: ByteArray, + spentAtHeights: IntArray, + count: Int, + ): Int = 0 + // ── Identities ──────────────────────────────────────────────────── /** diff --git a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/ffi/WalletManagerNative.kt b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/ffi/WalletManagerNative.kt index 8b2d0dc619f..f22f6877742 100644 --- a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/ffi/WalletManagerNative.kt +++ b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/ffi/WalletManagerNative.kt @@ -155,6 +155,89 @@ internal object WalletManagerNative { gapLimit: Int, ) + /** + * One bounded page of the engine's UTXO inventory for one wallet, + * across every funds account, as JSON + * `{"utxos":[...],"cursor":,"hasMore":}` — the + * source of truth the TXO-store reconciler + * ([PlatformWalletManager.reconcileTxoStore]) diffs against the Room + * `txos` mirror. A serialization shim over one Rust call + * (`platform_wallet_wallet_utxos_page`): account ordering, cursor + * semantics, the page bound and the address rendering all live in + * `platform-wallet`, so the Swift host walks the identical inventory. + * + * Paged, not swept whole: a wallet's UTXO count is chain-controlled + * (anyone who knows a watched address can keep sending dust to it), so + * a full-inventory read would let a remote party decide how much this + * process allocates on every steady-state transition and every + * 30-minute pass. Pass [cursor] `null` to start, then hand back the returned + * `cursor` verbatim while `hasMore` is true — a cursor this export did + * not produce throws. [limit] caps the rows in one page; non-positive + * asks for the engine's own default, and the engine clamps its own + * maximum, so no bound is mirrored on this side. + * + * Contact watch-only chains (`DashpayExternalAccount`) are absent by + * construction — those coins are the contact's money, not this + * wallet's — so the inventory never offers one to heal. + * + * Each `utxos` row is a + * [org.dashfoundation.dashsdk.persistence.PlatformWalletPersistenceHandler.EngineUtxoRow]: + * the owning account tuple, the txid hex in the same byte order the + * changeset path hands the handler (so hex→bytes reproduces the + * `txos.txid` blob), vout, amount (duffs), the engine's own + * Base58Check address (empty when the script has no address form), + * scriptHex, height, and the engine's own `isConfirmed` / + * `isInstantLocked` / `isCoinbase` / `isLocked` flags. + */ + external fun walletManagerUtxosPageJson( + managerHandle: Long, + walletId: ByteArray, + cursor: String?, + limit: Int, + ): String? + + /** + * Classify a batch of store rows against the engine's live state: the + * reverse half of the reconcile transport, and the reason the paged + * inventory above carries no spent-outpoint list. The caller pages its + * OWN mirror rows and asks about them a batch at a time, so neither + * side ever builds a set over the whole engine inventory. + * + * [queriesJson] is a JSON array of + * [org.dashfoundation.dashsdk.persistence.PlatformWalletPersistenceHandler.OutpointQuery]: + * for each row, the account tuple the STORE files the coin under, its + * txid hex, vout, and the scriptHex the store recorded. The account + * and script are the store's claim — the engine checks both against + * its own pools rather than trusting them, which is why a bare + * outpoint is not enough to ask the question. + * + * Returns one verdict byte per query, positionally — see + * `OUTPOINT_CLASS_*` on + * [org.dashfoundation.dashsdk.persistence.PlatformWalletPersistenceHandler]: + * 0 unknown, 1 unspent, 2 known-uncredited, 3 not-owned. A query whose + * account tag this build cannot map keeps `unknown`; the rest are + * still answered. + */ + external fun walletManagerClassifyOutpoints( + managerHandle: Long, + walletId: ByteArray, + queriesJson: String, + ): ByteArray? + + /** + * The wallet's core scan watermark as the ENGINE holds it + * (`platform_wallet_core_wallet_state`'s `synced_height`) — the value the + * TXO-store reconcile gates on, as Swift's does. The persisted + * `syncedHeight` in Room trails it by up to one persistence round, which + * is exactly the window the SYNCED transition lands in. Throws + * [org.dashfoundation.dashsdk.errors.DashSdkError] `NotFound` when the + * manager does not hold the wallet. + */ + external fun walletManagerCoreWalletSyncedHeight( + managerHandle: Long, + walletId: ByteArray, + ): Long + // ── Core transaction builder (1:1 over `core_wallet_tx_builder_*`) ─ // // Each step is a thin extern (one export = one FFI call, per diff --git a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandler.kt b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandler.kt index b33f99a467d..6d54a69a5df 100644 --- a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandler.kt +++ b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandler.kt @@ -13,6 +13,8 @@ import kotlinx.coroutines.flow.update import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json import org.dashfoundation.dashsdk.errors.DashSdkError import org.dashfoundation.dashsdk.ffi.AccountSpecData import org.dashfoundation.dashsdk.ffi.ContactProfileRestoreData @@ -233,6 +235,47 @@ class PlatformWalletPersistenceHandler( * [collectFinalizedSweptTombstones] once, last. */ var finalityAdvanced: Boolean = false + + /** + * The engine's credit verdicts for this round (dashpay/platform#4638) + * — every `Received` / `Change` output the round's records carry that + * the engine did NOT credit to the owning account — keyed by the hex + * of the 36-byte outpoint, valued by the `UTXO_CREDIT_VERDICT_*` + * code. Delivered by [onWalletChangesetUtxoVerdicts] BEFORE the + * changeset callback, so [onWalletChangesetUtxoAdded] can hand each + * row its verdict as it stages it: a row for a coin the engine never + * held would otherwise be written unspent, restored into the engine + * at the next launch, and show as a phantom balance + * (rust-dashcore#992). An outpoint absent here is credited — the + * ordinary case. Dies with the buffer, so a rolled-back or abandoned + * round leaves nothing behind. + */ + val utxoCreditVerdicts: HashMap = HashMap() + + /** How [upsertUtxoRow] applied [utxoCreditVerdicts] — logged once, as + * counts, by [onChangesetEnd]. */ + val utxoCreditTally = UtxoCreditVerdictTally() + } + + /** + * Per-round tally of how [upsertUtxoRow] applied the engine's credit + * verdicts. Logged as counts only: the outpoints themselves are wallet + * history and never leave the store. + */ + internal class UtxoCreditVerdictTally { + /** Rows written spent on an observed-spent verdict. */ + var observedSpent = 0 + + /** Rows written spent on a doomed verdict. */ + var doomed = 0 + + /** Rows that were already spent when their verdict arrived. */ + var alreadySpent = 0 + + /** Context-free verdicts, which only vetoed the recovery clear. */ + var uncredited = 0 + + val total: Int get() = observedSpent + doomed + alreadySpent + uncredited } /** Open rounds keyed by walletId hex (a round is per-walletId). */ @@ -420,9 +463,51 @@ class PlatformWalletPersistenceHandler( runBlockingCatching { database.withTransaction { op(database) } } + // A standalone write commits store state exactly like a round + // does, so it stales a reconcile verdict read before it too. + committedRoundGeneration++ } } + /** + * Writes committed by persistence callbacks since this handler was + * built: every successful [onChangesetEnd] commit, and every standalone + * [stage] write. The TXO reconcile reads it together with each store + * page and asks the engine about that page OUTSIDE [callbackExclusion]; + * a round that commits in that gap may have re-credited one of the + * page's coins (a reorg of its spender hands it back in `utxos_added`), + * so the verdicts are applied only while the count is unchanged and no + * round is open for the wallet — see [classifyStoreRows]. Read and + * written only under [callbackExclusion]. + */ + private var committedRoundGeneration = 0L + + /** + * Where each wallet's next [reconcileTxos] run picks up (keyed by wallet + * id hex), so a run that stops on its page budget, or for any other + * reason, does not make the next one walk the pages it already finished. + * Written after every finished page (never for a page whose step failed + * or was cancelled) and removed when the classify pass reaches its end, + * which makes the run after that start a new cycle. + * + * Each entry carries the scope it was written under (see + * [ReconcileCheckpoint.scope]); a run passing a different scope ignores + * it and starts a new cycle. The manager passes its reconcile epoch, + * which moves on every wallet removal and on close, so a place left by + * a removed wallet is never resumed by a wallet added again under the + * same id. A new manager builds a new handler, so a place never + * outlives the engine whose cursor it holds. + * + * In memory on purpose: a fresh process starts a new cycle, which is + * safe (the passes are idempotent) and costs at most one budget per run. + * Dropped with the wallet's other in-memory state by + * [deleteWalletDataLocked]. Read and written only under + * [callbackExclusion]; a write re-checks the run's cancellation under + * it, so a run that a wallet deletion cancelled cannot leave an entry + * behind after the deletion dropped it. + */ + private val txoReconcileCheckpoints = HashMap() + /** The open round for [walletId], or null on the standalone-callback path. */ private fun openRound(walletId: ByteArray): ChangesetBuffer? = buffers[walletId.toHex()] @@ -476,18 +561,37 @@ class PlatformWalletPersistenceHandler( } // Rows committed — the aliases are discoverable the normal way, // and the round's pending-key state changes are now true. + committedRoundGeneration++ pendingRoundAliases.remove(key) publishPendingKeyDeltas(buffer) + logUtxoCreditTally(buffer.utxoCreditTally, roundSuccess = true) } catch (t: Throwable) { // Commit failed: the staged rows never landed, so the round's // aliases are orphans exactly like the !success branch — and its // pending-key deltas are equally void (discarded with the buffer). scrubPendingAliases(key) + logUtxoCreditTally(buffer.utxoCreditTally, roundSuccess = false) throw t } 0 } + /** + * One line per round that applied a credit verdict — the Kotlin twin of + * Swift's `persistence_txo_credit_verdicts` event. Counts only: no + * txid, outpoint, address or script. A rolled-back round never ran its + * staged writes, so it has nothing to report. + */ + private fun logUtxoCreditTally(tally: UtxoCreditVerdictTally, roundSuccess: Boolean) { + if (tally.total == 0) return + Log.i( + TAG, + "txo credit verdicts: observedSpent=${tally.observedSpent} " + + "doomed=${tally.doomed} alreadySpent=${tally.alreadySpent} " + + "uncredited=${tally.uncredited} roundSuccess=$roundSuccess", + ) + } + // on_store_fn / on_flush_fn have no Swift analog — they are // FFIPersister-level notifications. The begin/end bracket is the only @@ -1116,6 +1220,40 @@ class PlatformWalletPersistenceHandler( 0 } + /** + * Port of `PlatformWalletPersistenceHandler.swift`'s + * `persistWalletChangesetUtxoVerdicts` (dashpay/platform#4638 Part 1): + * stage the round's credit verdicts (see + * [ChangesetBuffer.utxoCreditVerdicts]). Fired by Rust inside the + * begin/end bracket BEFORE the changeset callback, only on rounds that + * carry at least one verdict. Nothing is written here — the verdicts + * are applied by [upsertUtxoRow] when the round's `utxos_added` entries + * arrive, and a verdict for an outpoint no entry names is simply dropped + * with the round. A malformed batch fails the round rather than losing + * a verdict: a dropped verdict is a phantom coin the store hands back + * to the engine at the next load. + */ + override fun onWalletChangesetUtxoVerdicts( + walletId: ByteArray, + outpoints: ByteArray, + verdicts: ByteArray, + spentAtHeights: IntArray, + count: Int, + ): Int = guarded { + require( + count >= 0 && outpoints.size == count * OUTPOINT_BYTES && + verdicts.size == count && spentAtHeights.size == count, + ) { "credit-verdict batch does not match its count ($count)" } + // Rust fires this slot only inside a round; with no round open there + // is no `utxos_added` delivery for the verdicts to scope. + val round = openRound(walletId) ?: return@guarded 0 + for (i in 0 until count) { + val outpoint = outpoints.copyOfRange(i * OUTPOINT_BYTES, (i + 1) * OUTPOINT_BYTES) + round.utxoCreditVerdicts[outpoint.toHex()] = verdicts[i] + } + 0 + } + override fun onWalletChangesetUtxoAdded( walletId: ByteArray, txid: ByteArray, @@ -1129,112 +1267,1220 @@ class PlatformWalletPersistenceHandler( isInstantLocked: Boolean, isLocked: Boolean, ): Int = guarded { + // The round's verdicts all arrived before this callback, so the + // engine's verdict on this output (if it did NOT credit it) is + // final here and travels with the staged write. + val round = openRound(walletId) + val creditVerdict = round?.utxoCreditVerdicts?.takeIf { it.isNotEmpty() } + ?.get(makeOutpoint(txid, vout).toHex()) stage(walletId) { db -> - val outpoint = makeOutpoint(txid, vout) - // Ensure a parent transaction row exists (stub if missing, so - // the TXO FK holds; the real tx upsert overwrites it later). - if (db.transactionDao().getByTxid(txid) == null) { - db.transactionDao().upsert( - TransactionEntity(txid = txid, transactionData = ByteArray(0)), + upsertUtxoRow( + db, walletId, txid, vout, amount, address, scriptPubKey, + height, isCoinbase, isConfirmed, isInstantLocked, isLocked, + creditVerdict = creditVerdict, + creditTally = round?.utxoCreditTally, + ) + } + 0 + } + + /** + * The single TXO-insert discipline, shared by the changeset callback + * ([onWalletChangesetUtxoAdded]) and the reconcile sweep + * ([reconcileTxos]): stub the parent transaction row so the FK holds, + * upsert the TXO preserving any existing spend linkage, then drain + * pending-input rows staged before this funding TXO existed. + * + * Returns the row's final `isSpent` as written, after the drain. + */ + private suspend fun upsertUtxoRow( + db: DashDatabase, + walletId: ByteArray, + txid: ByteArray, + vout: Int, + amount: Long, + address: String, + scriptPubKey: ByteArray, + height: Int, + isCoinbase: Boolean, + isConfirmed: Boolean, + isInstantLocked: Boolean, + isLocked: Boolean, + // The Room account this output belongs to, when the CALLER could + // resolve it (the reconcile resolves it from the engine inventory's + // account tags). Stamped on the row so ownership survives even when + // the address projection is absent — a heal into a store that lost + // BOTH the TXO and its address row must not produce a row the + // restore loader cannot attribute (it would be skipped at the next + // mirror-reload, recreating the fund loss the heal repaired). An + // existing row's accountId always wins; changeset callbacks pass + // null and keep their address-projection behavior. + resolvedAccountId: Long? = null, + // The engine's credit verdict on this output for the round + // delivering it (see [ChangesetBuffer.utxoCreditVerdicts]), or null + // when the engine credited it. Only the changeset callback passes + // one; the reconcile heal inserts coins the engine HOLDS. + creditVerdict: Byte? = null, + creditTally: UtxoCreditVerdictTally? = null, + ): Boolean { + val outpoint = makeOutpoint(txid, vout) + // Ensure a parent transaction row exists (stub if missing, so + // the TXO FK holds; the real tx upsert overwrites it later). + if (db.transactionDao().getByTxid(txid) == null) { + db.transactionDao().upsert( + TransactionEntity(txid = txid, transactionData = ByteArray(0)), + ) + } + val existing = db.txoDao().getByOutpoint(outpoint) + val coreAddressId = if (address.isNotEmpty()) address else null + // A materialised coin the wallet re-delivers unspent follows the + // wallet — the mirror of the SQLite store's upsert valve, which + // holds only never-materialised placeholders. The wallet knows + // this coin, and any network-final spender of a coin it knows is + // wallet-relevant by BIP158 prevout matching, so its own scan + // re-discovers the spend; refusing the re-delivery would lock a + // real coin out forever after a reorg of the winner, and on this + // side of the FFI a row at `isSpent = true` is never restored to + // Rust again. So an UNLINKED row — a sweep hold with its stamp, or + // a legacy flag with nothing behind it — is cleared, stamp + // included. A LINKED row keeps its flag and stamp: the link is + // this store's recorded spend attribution, the pending drain + // below and the sweep pass own that transition, and a spender + // that reached a block is confirmed evidence a re-delivery never + // displaces. + // + // Any credit verdict vetoes that clear (dashpay/platform#4638): the + // wallet is not handing this coin back as unspent — its record + // merely still names the output as ours. An observed-spent or + // doomed verdict is block-context evidence the coin is not + // spendable (a block spent it before the output was recognised — + // the spender may never have been recorded, rust-dashcore#992 — or + // the record can never confirm), so the row is written spent with + // no spender link. `isSpent` is monotonic: a row already spent is + // left as it is. An uncredited verdict carries no context and only + // keeps the clear from resurrecting the row. + val linked = existing?.spendingTxid != null + val keepSpentState = linked || (existing != null && creditVerdict != null) + val alreadySpent = keepSpentState && existing!!.isSpent + val bornSpent = creditVerdict == UTXO_CREDIT_VERDICT_OBSERVED_SPENT || + creditVerdict == UTXO_CREDIT_VERDICT_DOOMED + if (creditVerdict != null && creditTally != null) { + when { + !bornSpent -> creditTally.uncredited++ + alreadySpent -> creditTally.alreadySpent++ + creditVerdict == UTXO_CREDIT_VERDICT_OBSERVED_SPENT -> creditTally.observedSpent++ + else -> creditTally.doomed++ + } + } + val row = TxoEntity( + outpoint = outpoint, + vout = vout, + amount = amount, + address = address, + scriptPubKey = scriptPubKey, + height = height, + isCoinbase = isCoinbase, + isConfirmed = isConfirmed, + isInstantLocked = isInstantLocked, + isLocked = isLocked, + isSpent = alreadySpent || bornSpent, + walletId = walletId, + txid = txid, + spendingTxid = existing?.spendingTxid, + spendingInputIndex = existing?.spendingInputIndex, + accountId = existing?.accountId ?: resolvedAccountId, + coreAddressId = existing?.coreAddressId ?: coreAddressIdIfPresent(db, coreAddressId), + createdAt = existing?.createdAt ?: java.util.Date(), + lastUpdated = now(), + supersededByTxid = if (keepSpentState) existing!!.supersededByTxid else null, + ) + db.txoDao().upsert(row) + var written = row + // Drain any pending-input rows staged before this funding TXO + // existed — a port of the Swift `upsertUtxo` drain + // (PlatformWalletPersistenceHandler.swift). A spend that arrived + // first was deferred (see onWalletChangesetTransaction); now that + // the funding output is here, resolve the claim and clear the rows + // so the UTXO-restore path won't hand this consumed output back to + // Rust as spendable. + val pending = db.documentDao().getPendingInputsByOutpoint(outpoint) + if (pending.isNotEmpty()) { + // A tombstone outranks every ordinary row regardless of age: + // ordinary rows are competing *observations*, a tombstone is + // the sweep's settled verdict that its winner consumed this + // coin. Prefer the tombstone tagged with the delivering + // wallet; failing that any tombstone on the outpoint still + // holds — the stamp is a txid fact, not a per-wallet one. + val tombstones = pending.filter { it.isSweptTombstone } + val tombstone = tombstones.filter { it.walletId.contentEquals(walletId) } + .maxByOrNull { it.createdAt } + ?: tombstones.maxByOrNull { it.createdAt } + if (tombstone != null) { + // A drained tombstone STAMPS, it never mints a spender + // link: the winner need not have its own `transactions` + // row, and a link would make the coin non-releasable + // (the release pass frees stamped, unlinked rows) when a + // later sweep proves the winner never took it. The + // existing link, if any, is carried as it was. + written = row.copy( + isSpent = true, + supersededByTxid = tombstone.spendingTxid, + lastUpdated = now(), ) + db.txoDao().upsert(written) + } else { + // Competing ordinary observations: a network-final spender + // outranks a newer mempool one (its row is the settled + // claim the link guard protects); among equals the newest + // wins, as before (reorg / double-spend: newest wins). + val ranked = pending.map { p -> p to db.transactionDao().getByTxid(p.spendingTxid) } + val (chosen, spending) = ranked.maxWithOrNull( + compareBy>( + { it.second?.context ?: 0 }, + { it.first.createdAt }, + ), + )!! + val spendingContext = spending?.context ?: 0 + val keepExistingLink = + keepSettledSpenderLink(db, row, chosen.spendingTxid, spendingContext) + written = linkSpender(row, chosen.spendingTxid, chosen.inputIndex, spendingContext, keepExistingLink) + db.txoDao().upsert(written) } - val existing = db.txoDao().getByOutpoint(outpoint) - val coreAddressId = if (address.isNotEmpty()) address else null - // A materialised coin the wallet re-delivers unspent follows the - // wallet — the mirror of the SQLite store's upsert valve, which - // holds only never-materialised placeholders. The wallet knows - // this coin, and any network-final spender of a coin it knows is - // wallet-relevant by BIP158 prevout matching, so its own scan - // re-discovers the spend; refusing the re-delivery would lock a - // real coin out forever after a reorg of the winner, and on this - // side of the FFI a row at `isSpent = true` is never restored to - // Rust again. So an UNLINKED row — a sweep hold with its stamp, or - // a legacy flag with nothing behind it — is cleared, stamp - // included. A LINKED row keeps its flag and stamp: the link is - // this store's recorded spend attribution, the pending drain - // below and the sweep pass own that transition, and a spender - // that reached a block is confirmed evidence a re-delivery never - // displaces. - val linked = existing?.spendingTxid != null - val row = TxoEntity( - outpoint = outpoint, - vout = vout, - amount = amount, - address = address, - scriptPubKey = scriptPubKey, - height = height, - isCoinbase = isCoinbase, - isConfirmed = isConfirmed, - isInstantLocked = isInstantLocked, - isLocked = isLocked, - isSpent = linked && existing!!.isSpent, - walletId = walletId, - txid = txid, - spendingTxid = existing?.spendingTxid, - spendingInputIndex = existing?.spendingInputIndex, - accountId = existing?.accountId, - coreAddressId = existing?.coreAddressId ?: coreAddressIdIfPresent(db, coreAddressId), - createdAt = existing?.createdAt ?: java.util.Date(), - lastUpdated = now(), - supersededByTxid = if (linked) existing!!.supersededByTxid else null, + for (p in pending) db.documentDao().deletePendingInput(p) + } + return written.isSpent + } + + /** + * One row of the engine's paged UTXO inventory, exactly as + * `walletManagerUtxosPageJson` serializes it (rs-unified-sdk-jni + * `UtxoPageRow`, serde on that side, kotlinx on this one). The contract + * is typed on BOTH ends so a renamed or missing key fails the decode + * loudly instead of healing a row with a defaulted owner or a zero + * amount into the mirror the engine reloads from. The account tuple + * (`typeTag`…`friendIdentityId`) is the routing context + * [findReconcileAccount] resolves the owning Room account by, so every + * field of it is required except the DashPay identity halves, which are + * absent on every non-DashPay account. Every other field is required + * too: the address, the script and the engine's flags are written into + * a healed row as given, so a missing one must not decode as an empty + * address, an empty script or a guessed flag. + */ + @Serializable + data class EngineUtxoRow( + val typeTag: Int, + val standardTag: Int, + val index: Int, + val registrationIndex: Int, + val keyClass: Int, + val userIdentityId: String? = null, + val friendIdentityId: String? = null, + val txid: String, + val vout: Int, + val amount: Long, + val address: String, + val scriptHex: String, + val height: Int, + /** The engine's own flags for the coin. Stamped onto a healed row + * as given: guessing them (a coinbase row filed as non-coinbase) + * would misstate maturity in the mirror the engine reloads from. */ + val isConfirmed: Boolean, + val isInstantlocked: Boolean, + val isCoinbase: Boolean, + val isLocked: Boolean, + ) + + /** + * One store row handed to `walletManagerClassifyOutpoints`: the account + * tuple the STORE files the coin under, its outpoint, and the script + * the store recorded for it. The account and script are the store's + * claim — the engine checks both against its own pools rather than + * trusting them, which is why a bare outpoint cannot ask the question. + * Serialized against the same field names the Rust `OutpointQuery` + * deserializes. + * + * Only the identity halves have defaults. The Rust tuple requires every + * other account field, and kotlinx's default `Json` omits a property + * equal to its default — so a defaulted `standardTag`/`index`/… went + * out missing for the common BIP44 account-0 row and the native side + * rejected the whole batch. + */ + @Serializable + data class OutpointQuery( + val typeTag: Int, + val standardTag: Int, + val index: Int, + val registrationIndex: Int, + val keyClass: Int, + val userIdentityId: String? = null, + val friendIdentityId: String? = null, + val txid: String, + val vout: Int, + val scriptHex: String, + ) + + /** + * One page of the engine's UTXO inventory: the rows, the opaque resume + * cursor (null on the last page) and whether more pages follow. All + * three keys are required, and a page that says more follow without a + * cursor to resume from fails the decode: either would otherwise read + * as the last page and end the heal walk early, leaving later engine + * coins unrepaired. + */ + @Serializable + data class EngineUtxoPage( + val utxos: List, + val cursor: String?, + val hasMore: Boolean, + ) { + init { + require(!hasMore || cursor != null) { "an inventory page with more to come carries no cursor" } + } + } + + /** + * What one [reconcileTxos] run did — the Kotlin twin of Swift's + * `CoreTxoReconcileReport` (dashpay/platform#4638). Counts only: no + * outpoint, address or amount of an individual coin leaves the store + * through this type or the log lines built from it. + */ + data class TxoReconcileReport( + // Heal pass (engine → store, insert-only). + /** Engine inventory rows walked. */ + val engineRows: Int = 0, + /** Engine coins the store already held. */ + val alreadyPresent: Int = 0, + /** Engine coins inserted unspent. A non-zero value after a completed + * sync means a changeset failed to deliver an owned output (the + * CoinJoin-funded-send change-drop class), which would have become + * a fund-loss on the next engine reload from this store. */ + val inserted: Int = 0, + val insertedDuffs: Long = 0, + /** Engine coins the store lacked whose row the pending-input drain + * wrote spent on insert: the divergence is recorded, not repaired. */ + val healedSpent: Int = 0, + /** Engine coins the engine does not call confirmed, or below the + * confirmation gate — they age into a later run. */ + val skippedImmature: Int = 0, + /** Engine coins whose account has no store row to file them under. + * Never inserted unowned: the restore loader routes by account, so + * an unowned row would be dropped at the next launch. */ + val skippedUnresolvedAccount: Int = 0, + /** Engine coins whose address has no saved row under the coin's + * account. A cold restore rebuilds each account's address pool from + * the rows filed under it, so without one the coin has no derivation + * path and no build could sign for it. Healed by a later run once + * the address is persisted under that account. */ + val skippedMissingAddress: Int = 0, + /** Engine coins the store could not validate (txid not 32 bytes, no + * script, no address). */ + val skippedInvalid: Int = 0, + /** Watch-only DIP-15 contact coins excluded on both sides — engine + * coins the heal pass refused to file as ours, and unspent store + * rows the classify pass did not ask about (the engine's own + * accounts never report them, so their absence is expected). */ + val skippedForeign: Int = 0, + // Classify pass (store → engine, flip-only). + /** Unspent store rows of this wallet handed to the engine. */ + val storeRows: Int = 0, + /** Store rows the engine reported unspent — consistent. */ + val unspent: Int = 0, + /** Store rows marked spent on the engine's positive + * [OUTPOINT_CLASS_KNOWN_UNCREDITED] verdict. */ + val flipped: Int = 0, + val flippedDuffs: Long = 0, + /** Store rows the engine has no opinion about — left untouched. */ + val unknown: Int = 0, + /** Store rows filed under an account that does not monitor their + * script — left untouched, reported. */ + val notOwned: Int = 0, + /** Unspent store rows whose owning account resolves through neither + * the explicit link nor the address projection. They carry no claim + * for the engine to check, so they are counted, not asked about. */ + val unresolvedAccount: Int = 0, + // Run shape. + /** Pages classified again because a persistence round committed + * between their read and their apply, or was still open at it. */ + val staleRetries: Int = 0, + /** Engine reads that failed (an inventory page, the first one + * included, or a classification batch); the run stops at the first. */ + val transportFailures: Int = 0, + /** Store steps that failed and rolled back; the run stops at the first. */ + val storeFailures: Int = 0, + /** + * `false` when the run stopped before the end of the classify pass: + * cancelled, a failed read or write, too many stale pages, or the + * page budget ([stoppedOnBudget]). What landed before the stop stays. + * + * A budget stop is not a fault, but it still reads `false`, for two + * reasons. The summary line calls a run with `completed` and zero + * [mutations] "mirror consistent", and a run that stopped on its + * budget has not looked at the rest of the inventory, so it cannot + * say that. And Swift's `completed` is `false` for every run that + * stopped before the end, whatever the cause. Read [stoppedOnBudget] + * to tell the budget stop apart from a failure. + * + * `true` means this run reached the end of the cycle. For a run that + * [resumedFromCheckpoint], that is the end of the part an earlier + * run left over, not a walk of the whole inventory in this run. + */ + val completed: Boolean = true, + /** The run used its whole page budget + * ([TXO_RECONCILE_PAGE_BUDGET]) before the end of the cycle and + * stopped cleanly between pages. The wallet's next run picks up at + * the first page this one did not finish; [completed] is `false`. */ + val stoppedOnBudget: Boolean = false, + /** The run picked up where an earlier run of this wallet stopped, + * rather than at the first heal page. Its counts cover only the + * pages it walked itself. */ + val resumedFromCheckpoint: Boolean = false, + ) { + /** Rows this run wrote: healed, flipped, and healed rows the drain + * wrote spent on insert — the last is a divergence recorded, not + * repaired, but it is a row the store did not have before. */ + val mutations: Int get() = inserted + healedSpent + flipped + } + + /** The mutable half of a [TxoReconcileReport] while the run is in flight. */ + private class ReconcileTally { + var engineRows = 0 + var alreadyPresent = 0 + var inserted = 0 + var insertedDuffs = 0L + var healedSpent = 0 + var skippedImmature = 0 + var skippedUnresolvedAccount = 0 + var skippedMissingAddress = 0 + var skippedInvalid = 0 + var skippedForeign = 0 + var storeRows = 0 + var unspent = 0 + var flipped = 0 + var flippedDuffs = 0L + var unknown = 0 + var notOwned = 0 + var unresolvedAccount = 0 + var staleRetries = 0 + var transportFailures = 0 + var storeFailures = 0 + /** Set only where the classify pass reaches its end; every other + * exit leaves the run incomplete. */ + var completed = false + var stoppedOnBudget = false + var resumedFromCheckpoint = false + + /** Pages this run may still read, across both passes. Not reported: + * it is the run's allowance, not something the run found. */ + var pagesLeft = 0 + + /** + * Take one page from the budget before reading it. `false` when the + * run is cancelled, or when the budget is spent: the run is then + * marked as stopped on its budget. Either way the caller stops + * before the read. Checked between pages only, so a page that was + * read always gets its store step. + */ + fun takePage(isCancelled: () -> Boolean): Boolean { + if (isCancelled()) return false + if (pagesLeft <= 0) { + stoppedOnBudget = true + return false + } + pagesLeft-- + return true + } + + fun toReport() = TxoReconcileReport( + engineRows = engineRows, + alreadyPresent = alreadyPresent, + inserted = inserted, + insertedDuffs = insertedDuffs, + healedSpent = healedSpent, + skippedImmature = skippedImmature, + skippedUnresolvedAccount = skippedUnresolvedAccount, + skippedMissingAddress = skippedMissingAddress, + skippedInvalid = skippedInvalid, + skippedForeign = skippedForeign, + storeRows = storeRows, + unspent = unspent, + flipped = flipped, + flippedDuffs = flippedDuffs, + unknown = unknown, + notOwned = notOwned, + unresolvedAccount = unresolvedAccount, + staleRetries = staleRetries, + transportFailures = transportFailures, + storeFailures = storeFailures, + completed = completed, + stoppedOnBudget = stoppedOnBudget, + resumedFromCheckpoint = resumedFromCheckpoint, + ) + } + + /** + * Where the next [reconcileTxos] run of a wallet picks up — see + * [txoReconcileCheckpoints]. No entry means the next run starts a new + * cycle at the first heal page. + */ + private sealed class ReconcileCheckpoint { + /** The `checkpointScope` of the run that wrote it. Only a run with + * the same scope resumes from it. */ + abstract val scope: Long + + /** In the heal pass, at the page after the engine cursor [cursor] + * (the cursor the last finished page returned). */ + class Heal(override val scope: Long, val cursor: String) : ReconcileCheckpoint() + + /** In the classify pass, at the first unspent row after the + * outpoint [after]; an empty [after] is the first row. */ + class Classify(override val scope: Long, val after: ByteArray) : ReconcileCheckpoint() + } + + /** Counts from one heal step; merged into the run only once it commits. */ + private class HealCounts { + var alreadyPresent = 0 + var inserted = 0 + var insertedDuffs = 0L + var healedSpent = 0 + var skippedImmature = 0 + var skippedUnresolvedAccount = 0 + var skippedMissingAddress = 0 + var skippedInvalid = 0 + var skippedForeign = 0 + val healedOutpoints = ArrayList() + val healedSpentOutpoints = ArrayList() + } + + /** + * One page of the wallet's unspent store rows for the classify pass: + * the rows the engine is asked about with their queries, what the page + * set aside, the cursor to continue from, and [committedRoundGeneration] + * as read together with the rows. + */ + private class UnspentPage( + val rows: List, + val queries: List, + /** Rows the page read before setting any aside. */ + val fetched: Int, + val lastOutpoint: ByteArray?, + val unresolvedAccount: Int, + val skippedForeign: Int, + val generation: Long, + ) + + /** Counts from one classify-apply step. */ + private class FlipCounts { + var flipped = 0 + var flippedDuffs = 0L + var unspent = 0 + var unknown = 0 + var notOwned = 0 + + /** A round committed between the read and the apply, or one is + * open at the apply: nothing was written and the page is + * classified again. */ + var staleGeneration = false + val flippedOutpoints = ArrayList() + } + + /** + * Reconcile the Room `txos` mirror against the engine — the Kotlin port + * of Swift's `runCoreTxoReconcile` (dashpay/platform#4638 Part 2). The + * mirror is write-behind and is itself the source the engine is + * restored from at every launch ([buildUtxoRestoreData] hands back every + * `isSpent = 0` row), so two failure classes outlive a full scan: a coin + * the engine credited whose row the store never got (the job-flower + * 106.43→86.33 restart drop — rescan dropping the change outputs of + * CoinJoin-funded sends), and a row the store holds unspent for a coin + * the engine never credited (rust-dashcore#992: the spender had no + * wallet-owned output and was discarded before the coin was known). The + * restore hands the second back, and the engine shows a balance it had + * just corrected (dashpay/platform#4575). This repairs exactly what + * positive evidence supports: + * + * * [healMissingTxos] pages the ENGINE ([engineUtxoPage]: `cursor` null + * to start, then the cursor the previous page returned while its + * `hasMore` is true) and inserts each coin the store lacks — validated, + * owned, confirmed by the engine, at least [minConfirmations] deep, and + * at an address the store has a saved row for under the coin's account. + * * [classifyStoreRows] pages the wallet's UNSPENT store rows and asks + * [classifyOutpoints] about each page. Only + * [OUTPOINT_CLASS_KNOWN_UNCREDITED] — the owning account recorded the + * funding transaction, owns the script, does not hold the coin, AND a + * funds account holds a mined record spending it — marks a row spent. + * + * It never deletes a row, never un-marks a spent row, and never acts on + * absence alone. It is idempotent (a consistent store yields a report + * with zero [TxoReconcileReport.mutations]) and bounded in both + * directions: a wallet's UTXO count is chain-controlled, so a pass that + * materialized a whole inventory would let anyone who knows a watched + * address decide how much a phone allocates. + * + * Each page bounds memory; [pageBudget] bounds the work of the whole run. + * Anyone can send dust to a watched address, so the inventory can grow + * without the owner doing anything, and a run that always walked all of + * it would turn that growth into CPU, battery and callback work on every + * automatic run. So a run reads at most [pageBudget] pages, counting the + * heal pass's engine pages and the classify pass's store pages together + * (a stale page read again counts again). When the budget runs out the + * run stops between pages, after the last page's store step, and + * reports [TxoReconcileReport.stoppedOnBudget] (with `completed = + * false`). + * + * The run remembers where it stopped ([txoReconcileCheckpoints]), and the + * wallet's next run with the same [checkpointScope] picks up there + * instead of at the first heal page: in the heal pass, at the engine + * cursor the last finished page returned; in the classify pass, after + * the last finished page's last outpoint. The heal pass reaching its end + * moves the checkpoint to the start of the classify pass, and the + * classify pass reaching its end removes it, so the run after that + * starts a new cycle. The classify pass of a cycle therefore never + * starts before that cycle's heal pass has reached its end, however many + * runs the cycle takes. The checkpoint only ever moves past a page whose + * step finished: a failed, cancelled or stale-aborted page, or one that + * does not decode, is read again by the next run. A large inventory is + * therefore covered over several runs, one budget per run at the + * manager's cadence, instead of all at once on every run. Coins that + * land behind the cursor while a cycle is in progress are picked up by + * the next cycle; both cursors are keys, not offsets, so resuming never + * skips or repeats a row that was there all along. + * + * Engine reads run OUTSIDE [callbackExclusion] — they park on the + * engine's wallet lock, which a persistence callback may hold while it + * waits for ours. Every store step runs under it, one Room transaction + * each, after re-checking [isCancelled]: the manager bumps its epoch on + * close and on wallet deletion, and the run stops between pages, + * reporting `completed = false`. + * + * A failed engine read stops the run and is counted, the first page + * included (Swift's contract: the report says the run did not + * complete, rather than there being no report). A page that does not + * decode as an [EngineUtxoPage] is a contract violation between the JNI + * emitter and this reader, and it throws rather than being absorbed. + * + * `netAmount` is never repaired: a record born blind to one of its own + * outputs persisted a net short by exactly that output's value, but the + * record may equally have been corrected already by a callback racing + * this run, and blind addition double-credits. + * + * Logs one summary line of counts per run (see [logReconcileSummary]). + * Must NOT be called from the handler's own [dispatcher] (it takes + * [callbackExclusion] and runs Room transactions). + */ + suspend fun reconcileTxos( + walletId: ByteArray, + tipHeight: Int, + minConfirmations: Int = TXO_RECONCILE_MIN_CONFIRMATIONS, + pageSize: Int = TXO_RECONCILE_PAGE_SIZE, + pageBudget: Int = TXO_RECONCILE_PAGE_BUDGET, + checkpointScope: Long = 0L, + isCancelled: () -> Boolean = { false }, + engineUtxoPage: suspend (cursor: String?, limit: Int) -> String?, + classifyOutpoints: suspend (queriesJson: String) -> ByteArray?, + ): TxoReconcileReport { + val limit = pageSize.coerceAtLeast(1) + val walletRef = logReference(walletId) + val tally = ReconcileTally().apply { pagesLeft = pageBudget.coerceAtLeast(1) } + // A place left under another scope (a removed wallet's, or one from + // before an epoch move) is not this run's to resume. + val checkpoint = callbackExclusion.withLock { txoReconcileCheckpoints[walletId.toHex()] } + ?.takeIf { it.scope == checkpointScope } + tally.resumedFromCheckpoint = checkpoint != null + + // Watch-only DIP-15 contact (external) accounts, resolved once up + // front because BOTH passes need the exclusion and the account set + // does not move under a run. The engine tracks these accounts' coins + // to show payments TO contacts, but they are the CONTACT's money and + // must never be healed into the store as ours. Before the check + // lived on the insert pass, a fresh restore's post-backfill reconcile + // healed every contact-payment coin into the store (12 rows / + // 0.05692493 tDASH on the large-wallet validation run of 2026-08-25). + val foreignAccountIds = reconcileStoreStep(tally, walletRef, "accounts", isCancelled) { + database.accountDao() + .observeByWalletAndType(walletId, ACCOUNT_TYPE_TAG_DASHPAY_EXTERNAL).first() + .map { it.id } + .toSet() + } + if (foreignAccountIds != null) { + suspend fun heal(from: String?) = healMissingTxos( + walletId, from, checkpointScope, tipHeight, minConfirmations, limit, + foreignAccountIds, isCancelled, engineUtxoPage, tally, walletRef, ) - db.txoDao().upsert(row) - // Drain any pending-input rows staged before this funding TXO - // existed — a port of the Swift `upsertUtxo` drain - // (PlatformWalletPersistenceHandler.swift). A spend that arrived - // first was deferred (see onWalletChangesetTransaction); now that - // the funding output is here, resolve the claim and clear the rows - // so the UTXO-restore path won't hand this consumed output back to - // Rust as spendable. - val pending = db.documentDao().getPendingInputsByOutpoint(outpoint) - if (pending.isNotEmpty()) { - // A tombstone outranks every ordinary row regardless of age: - // ordinary rows are competing *observations*, a tombstone is - // the sweep's settled verdict that its winner consumed this - // coin. Prefer the tombstone tagged with the delivering - // wallet; failing that any tombstone on the outpoint still - // holds — the stamp is a txid fact, not a per-wallet one. - val tombstones = pending.filter { it.isSweptTombstone } - val tombstone = tombstones.filter { it.walletId.contentEquals(walletId) } - .maxByOrNull { it.createdAt } - ?: tombstones.maxByOrNull { it.createdAt } - if (tombstone != null) { - // A drained tombstone STAMPS, it never mints a spender - // link: the winner need not have its own `transactions` - // row, and a link would make the coin non-releasable - // (the release pass frees stamped, unlinked rows) when a - // later sweep proves the winner never took it. The - // existing link, if any, is carried as it was. - db.txoDao().upsert( - row.copy( - isSpent = true, - supersededByTxid = tombstone.spendingTxid, - lastUpdated = now(), - ), - ) - } else { - // Competing ordinary observations: a network-final spender - // outranks a newer mempool one (its row is the settled - // claim the link guard protects); among equals the newest - // wins, as before (reorg / double-spend: newest wins). - val ranked = pending.map { p -> p to db.transactionDao().getByTxid(p.spendingTxid) } - val (chosen, spending) = ranked.maxWithOrNull( - compareBy>( - { it.second?.context ?: 0 }, - { it.first.createdAt }, - ), - )!! - val spendingContext = spending?.context ?: 0 - val keepExistingLink = - keepSettledSpenderLink(db, row, chosen.spendingTxid, spendingContext) - db.txoDao().upsert( - linkSpender(row, chosen.spendingTxid, chosen.inputIndex, spendingContext, keepExistingLink), - ) + val healDone = when (checkpoint) { + // The heal pass of this cycle already reached its end. + is ReconcileCheckpoint.Classify -> true + is ReconcileCheckpoint.Heal -> heal(checkpoint.cursor) + null -> heal(null) + } + if (healDone) { + // An empty BLOB sorts before every real outpoint, so a cycle + // whose heal pass just ended starts at the first row. + classifyStoreRows( + walletId, (checkpoint as? ReconcileCheckpoint.Classify)?.after ?: ByteArray(0), checkpointScope, + limit, foreignAccountIds, isCancelled, classifyOutpoints, tally, walletRef, + ) + } + } + val report = tally.toReport() + logReconcileSummary(report, walletRef) + return report + } + + /** + * Heal pass of [reconcileTxos]: one engine page at a time, one store + * step each. Each page is fetched OUTSIDE the exclusion lock — the + * fetch is a native call into the engine, and the lock exists to keep + * changeset callbacks out of our writes, not out of the engine. Returns + * whether the run may go on to the classify pass. + * + * Starts after [from] (null: the first page) and stops before a page + * the run's budget does not cover. Each finished page moves the + * wallet's checkpoint to the cursor it returned; the last one moves it + * to the start of the classify pass. + */ + private suspend fun healMissingTxos( + walletId: ByteArray, + from: String?, + scope: Long, + tipHeight: Int, + minConfirmations: Int, + limit: Int, + foreignAccountIds: Set, + isCancelled: () -> Boolean, + engineUtxoPage: suspend (cursor: String?, limit: Int) -> String?, + tally: ReconcileTally, + walletRef: String, + ): Boolean { + var cursor: String? = from + while (true) { + if (!tally.takePage(isCancelled)) return false + val pageJson = engineUtxoPage(cursor, limit) + if (pageJson == null) { + // The transport died. Everything already applied stands + // (insert-only, idempotent); the rest waits for the next run. + tally.transportFailures++ + return false + } + // Strict decode: an unknown key, a missing required field or a + // mistyped value throws — the row would otherwise be healed with + // a defaulted owner or a zero amount. + val page = decodeEngineUtxoPage(pageJson) + tally.engineRows += page.utxos.size + + if (page.utxos.isNotEmpty()) { + val counts = reconcileStoreStep(tally, walletRef, "heal", isCancelled) { + database.withTransaction { + val c = HealCounts() + // Address -> its saved row (null when absent), for this page only. + val addressRows = HashMap() + for (row in page.utxos) { + healEngineRow( + walletId, row, tipHeight, minConfirmations, foreignAccountIds, c, addressRows, + ) + } + c + } + } ?: return false + tally.alreadyPresent += counts.alreadyPresent + tally.inserted += counts.inserted + tally.insertedDuffs += counts.insertedDuffs + tally.healedSpent += counts.healedSpent + tally.skippedImmature += counts.skippedImmature + tally.skippedUnresolvedAccount += counts.skippedUnresolvedAccount + tally.skippedMissingAddress += counts.skippedMissingAddress + tally.skippedInvalid += counts.skippedInvalid + tally.skippedForeign += counts.skippedForeign + // Logged once the step committed, so a rolled-back step + // never claims a row it did not write. + for (outpoint in counts.healedOutpoints) logReconcileItem("healed", outpoint, walletRef) + for (outpoint in counts.healedSpentOutpoints) logReconcileItem("healed_spent", outpoint, walletRef) + } + // The page is finished: record it, so the next run does not + // read it again. + val next = page.cursor + if (!page.hasMore || next == null) { + saveReconcileCheckpoint(walletId, ReconcileCheckpoint.Classify(scope, ByteArray(0)), isCancelled) + return true + } + saveReconcileCheckpoint(walletId, ReconcileCheckpoint.Heal(scope, next), isCancelled) + cursor = next + } + } + + /** + * Apply one engine inventory row to the store (inside the caller's + * exclusion lock and Room transaction) — Swift's + * `reconcileHealMissingTxos` gates, in its order: + * + * 1. malformed (txid not 32 bytes, no script, no address) → invalid; + * 2. the engine does not call it confirmed, or it is not + * [minConfirmations] deep at [tipHeight] → immature (a fresh coin can + * still reorg or, for coinbase, be immature; it ages into a later + * run); + * 3. the store already holds the outpoint → already present; + * 4. the seven-field account tuple names no store account → unresolved, + * NOT inserted: the restore loader routes by account, and an unowned + * row would be dropped at the next launch, recreating the loss; + * 5. the address has no saved `core_addresses` row filed under that + * account → missing address (Kotlin-only), NOT inserted: a cold + * restore rebuilds each account's address pool from the rows filed + * under it, so the coin would come back into the balance with no + * derivation path and no build could sign for it. A later run heals + * it once the address is persisted under the account. + * + * Between 2 and 3 sits the Kotlin-only contact exclusion (see + * [reconcileTxos]). A row that passes is inserted through + * [upsertUtxoRow] — the changeset path's own insert, so both writers + * honour the same rules (stub parent transaction, account stamp, + * address link, pending-input drain) — with `isConfirmed = true` (the + * gate guarantees it) and the engine's other flags as given. A row the + * drain wrote spent on the spot is counted as [HealCounts.healedSpent]: + * the engine holds the coin and the store now says spent, which is not + * a repair of the divergence the engine reported. + */ + private suspend fun healEngineRow( + walletId: ByteArray, + row: EngineUtxoRow, + tipHeight: Int, + minConfirmations: Int, + foreignAccountIds: Set, + counts: HealCounts, + addressRows: MutableMap, + ) { + val txid = row.txid.hexToByteArrayOrNull() + val script = row.scriptHex.hexToByteArrayOrNull() + val userIdentityId = row.userIdentityId?.hexToByteArrayOrNull() + val friendIdentityId = row.friendIdentityId?.hexToByteArrayOrNull() + if (txid == null || txid.size != 32 || script == null || script.isEmpty() || + row.address.isEmpty() || + (row.userIdentityId != null && userIdentityId == null) || + (row.friendIdentityId != null && friendIdentityId == null) + ) { + counts.skippedInvalid++ + return + } + if (!row.isConfirmed || row.height <= 0 || tipHeight < row.height || + tipHeight - row.height + 1 < minConfirmations + ) { + counts.skippedImmature++ + return + } + // The inventory tags every coin with its owning account tuple. The + // tag is the authoritative foreign check — a watch-only external + // account's coin is the CONTACT's money whether or not its address + // row survived persistence. The address-based check (through + // core_addresses.accountId, the same second path the classify pass + // resolves store rows by) stays as a fallback. + if (row.typeTag == ACCOUNT_TYPE_TAG_DASHPAY_EXTERNAL) { + counts.skippedForeign++ + return + } + val addressRow = addressRows.memo(row.address) { + database.coreAddressDao().getByAddress(row.address) + } + val addressOwner = addressRow?.accountId + if (addressOwner != null && addressOwner in foreignAccountIds) { + counts.skippedForeign++ + return + } + val outpoint = makeOutpoint(txid, row.vout) + if (database.txoDao().getByOutpoint(outpoint) != null) { + counts.alreadyPresent++ + return + } + // Absent identity halves are the non-DashPay case (no identity). + val account = findReconcileAccount( + walletId, row.typeTag, row.index, row.standardTag, row.registrationIndex, row.keyClass, + userIdentityId ?: ByteArray(0), friendIdentityId ?: ByteArray(0), + ) + if (account == null) { + counts.skippedUnresolvedAccount++ + return + } + // A cold restore rebuilds each account's address pool from the rows + // filed under it, so an address with no row, a row with no account, + // or a row under another account leaves the coin with no derivation + // path: it would be restored into the balance, but no build could + // sign for it. Leave it out until the address is persisted under + // this account; a later run heals it. + if (addressRow?.accountId != account.id) { + counts.skippedMissingAddress++ + return + } + val writtenSpent = upsertUtxoRow( + database, walletId, txid, row.vout, row.amount, row.address, script, row.height, + isCoinbase = row.isCoinbase, + isConfirmed = true, + isInstantLocked = row.isInstantlocked, + isLocked = row.isLocked, + resolvedAccountId = account.id, + ) + // netAmount is NOT mutated here. The record's net may already be + // correct (a corrective record callback can land while its TXO + // delivery races this run), and adding the healed amount to an + // already-corrected net double-credits. + if (writtenSpent) { + // A pending-input claim or a swept tombstone already covered + // this outpoint, so the drain wrote the row spent on the spot. + counts.healedSpent++ + counts.healedSpentOutpoints += outpoint + } else { + counts.inserted++ + counts.insertedDuffs += row.amount + counts.healedOutpoints += outpoint + } + } + + /** + * Non-creating lookup of the store account an engine row's seven-field + * tuple names — [fetchAccount]'s match, except that an empty identity + * column and 32 zero bytes both mean "no identity" (the account + * registration path stores whatever the FFI handed it, the inventory + * projects zeros), as Swift's `findAccountRow` treats them. + */ + private suspend fun findReconcileAccount( + walletId: ByteArray, + accountType: Int, + accountIndex: Int, + standardTag: Int, + registrationIndex: Int, + keyClass: Int, + userIdentityId: ByteArray, + friendIdentityId: ByteArray, + ): AccountEntity? = fetchAccount( + database, walletId, accountType, accountIndex, standardTag, registrationIndex, keyClass, + userIdentityId, friendIdentityId, + normaliseIdentity = { if (it.isEmpty()) ByteArray(32) else it }, + ) + + /** + * Classify pass of [reconcileTxos]: the wallet's `isSpent = 0` rows, a + * page at a time, each page asked about in one batch. Spent rows are + * never asked about — nothing here un-marks a row, so their verdict + * could change nothing. + * + * Each page is read under [callbackExclusion] together with + * [committedRoundGeneration], the engine is asked OUTSIDE the lock, and + * the verdicts are applied back under it in one Room transaction — only + * if no round committed in between and none is open for the wallet. A + * round that did may have re-credited one of the page's coins, so then + * nothing is written and the same page is read and classified again, at + * most [TXO_RECONCILE_MAX_STALE_RETRIES] times in a row before the run + * stops incomplete. An open round counts too: it has not moved + * [committedRoundGeneration] yet, but it may already have staged that + * re-credit, and a flip landing first would survive it — a credited + * re-delivery keeps the flag of a row with a spender link (see + * [upsertUtxoRow]), and no later run looks at a spent row again. + * + * The walk is keyed by outpoint (`outpoint > after`, the primary-key + * order), not by offset, so the flips themselves never disturb it: a + * flipped row leaves the `isSpent = 0` predicate behind the cursor, + * never ahead of it. (Swift walks by offset and has to step back by the + * number of rows it flipped.) The same key is what lets a later run + * resume after [from]: rows inserted or flipped between runs never + * shift the rows after it. + * + * Stops before a page the run's budget does not cover. Each finished + * page moves the wallet's checkpoint past its last outpoint; reaching + * the end removes it. A stale page that is read again is not finished, + * so it does not move the checkpoint. + */ + private suspend fun classifyStoreRows( + walletId: ByteArray, + from: ByteArray, + scope: Long, + limit: Int, + foreignAccountIds: Set, + isCancelled: () -> Boolean, + classifyOutpoints: suspend (queriesJson: String) -> ByteArray?, + tally: ReconcileTally, + walletRef: String, + ) { + var after = from + var staleInARow = 0 + while (true) { + if (!tally.takePage(isCancelled)) return + val page = reconcileStoreStep(tally, walletRef, "unspent_page", isCancelled) { + readUnspentPage(walletId, after, limit, foreignAccountIds) + } ?: return + if (page.fetched == 0) { + saveReconcileCheckpoint(walletId, null, isCancelled) + tally.completed = true + return + } + tally.storeRows += page.rows.size + tally.unresolvedAccount += page.unresolvedAccount + tally.skippedForeign += page.skippedForeign + + if (page.rows.isNotEmpty()) { + val verdicts = classifyOutpoints(encodeOutpointQueries(page.queries)) + if (verdicts == null || verdicts.size != page.rows.size) { + // No verdicts, no classification: stop rather than guess. + tally.transportFailures++ + return + } + val counts = reconcileStoreStep(tally, walletRef, "flip", isCancelled) { + if (committedRoundGeneration != page.generation || openRound(walletId) != null) { + FlipCounts().apply { staleGeneration = true } + } else { + database.withTransaction { applyEngineClasses(page.rows, verdicts) } + } + } ?: return + if (counts.staleGeneration) { + staleInARow++ + tally.staleRetries++ + if (staleInARow > TXO_RECONCILE_MAX_STALE_RETRIES) return + // Same cursor: nothing was written, so the page is read + // again and classified against the store as it is now. + tally.storeRows -= page.rows.size + tally.unresolvedAccount -= page.unresolvedAccount + tally.skippedForeign -= page.skippedForeign + continue } - for (p in pending) db.documentDao().deletePendingInput(p) + staleInARow = 0 + tally.flipped += counts.flipped + tally.flippedDuffs += counts.flippedDuffs + tally.unspent += counts.unspent + tally.unknown += counts.unknown + tally.notOwned += counts.notOwned + for (outpoint in counts.flippedOutpoints) logReconcileItem("flipped_spent", outpoint, walletRef) + } + // The page is finished. A short page is the last one: the cycle + // is over and the next run starts anew. + val last = page.lastOutpoint + if (last == null || page.fetched < limit) { + saveReconcileCheckpoint(walletId, null, isCancelled) + tally.completed = true + return + } + saveReconcileCheckpoint(walletId, ReconcileCheckpoint.Classify(scope, last), isCancelled) + after = last + } + } + + /** + * Record where [walletId]'s next reconcile run picks up (null: a new + * cycle) — see [txoReconcileCheckpoints]. Under [callbackExclusion], + * with cancellation re-checked under it, so it is ordered against a + * wallet deletion's cascade the same way a store step is. + */ + private suspend fun saveReconcileCheckpoint( + walletId: ByteArray, + checkpoint: ReconcileCheckpoint?, + isCancelled: () -> Boolean, + ) { + callbackExclusion.withLock { + if (isCancelled()) return@withLock + val key = walletId.toHex() + if (checkpoint == null) { + txoReconcileCheckpoints.remove(key) + } else { + txoReconcileCheckpoints[key] = checkpoint } } - 0 + } + + /** + * Read half of one classify step (inside the caller's exclusion lock): + * one page of the wallet's unspent rows from [after], and the query the + * engine classifies each by — the account tuple the STORE files the + * coin under and the script it recorded. Those are the store's claim; + * the engine checks both against its own pools rather than trusting + * them, which is why a row whose owning account cannot be resolved is + * counted rather than asked about, and why a bare outpoint cannot ask + * the question. A row still mid-insert (no 32-byte txid) is not + * classifiable at all. + * + * Watch-only DIP-15 contact rows are excluded via [foreignAccountIds]. + * Production changeset writes leave txos.accountId null and route + * ownership through the address projection, so the owner resolves + * BOTH paths ([resolveTxoOwnerAccount]) — an accountId-only check + * silently classifies every contact row. + */ + private suspend fun readUnspentPage( + walletId: ByteArray, + after: ByteArray, + limit: Int, + foreignAccountIds: Set, + ): UnspentPage { + val fetched = database.txoDao().pageUnspentByWallet(walletId, after, limit) + val rows = ArrayList(fetched.size) + val queries = ArrayList(fetched.size) + var unresolvedAccount = 0 + var skippedForeign = 0 + val accountByAddress = HashMap() + val accountById = HashMap() + for (row in fetched) { + val txid = row.txid + if (txid == null || txid.size != 32 || row.outpoint.size != OUTPOINT_BYTES) continue + val owner = resolveTxoOwnerAccount(row, accountByAddress, accountById) + if (owner == null) { + unresolvedAccount++ + continue + } + if (owner.id in foreignAccountIds) { + skippedForeign++ + continue + } + rows.add(row) + queries.add(outpointQueryFor(owner, txid, row.vout, row.scriptPubKey)) + } + return UnspentPage( + rows = rows, + queries = queries, + fetched = fetched.size, + lastOutpoint = fetched.lastOrNull()?.outpoint, + unresolvedAccount = unresolvedAccount, + skippedForeign = skippedForeign, + generation = committedRoundGeneration, + ) + } + + /** + * Owning Room account of a `txos` row — the one resolution the restore + * loader ([buildUtxoRestoreData]) and the reconcile ([readUnspentPage]) + * share, so every row the loader hands back to Rust is a row the + * reconcile can ask the engine about. The explicit link first, then the + * address projection production writes use, keyed by the row's own + * address: [upsertUtxoRow] leaves `coreAddressId` null when the address + * row is not there yet, and the address-pool callback that inserts it + * later does not backfill the TXO, so `coreAddressId` alone misses rows + * the loader restores. [accountByAddress] and [accountById] memoise + * the address leg and the account reads across one walk (a wallet's + * coins cluster on few addresses and accounts). + */ + private suspend fun resolveTxoOwnerAccount( + txo: TxoEntity, + accountByAddress: MutableMap, + accountById: MutableMap, + ): AccountEntity? { + suspend fun account(id: Long) = accountById.memo(id) { database.accountDao().getById(id) } + txo.accountId?.let { id -> account(id)?.let { return it } } + val address = txo.coreAddressId ?: txo.address.takeIf { it.isNotEmpty() } ?: return null + return accountByAddress.memo(address) { + database.coreAddressDao().getByAddress(address)?.accountId?.let { account(it) } + } + } + + /** + * Write half of one classify step (inside the caller's exclusion lock + * and Room transaction, the generation already checked): apply the + * engine's verdicts to the rows they were asked about. Only + * [OUTPOINT_CLASS_KNOWN_UNCREDITED] writes — the row is marked spent + * with no spender link (the spender may never have been recorded — + * rust-dashcore#992), its `lastUpdated` bumped, and every pending-input + * row on the outpoint dropped (Swift's `removePendingInputs`), so a + * claim on a settled coin does not linger. `isSpent` is monotonic: a + * row already spent, or gone, by now is stale and left alone. Unspent, + * unknown and not-owned verdicts are counted, never acted on: absence of + * a coin from the engine proves nothing. + */ + private suspend fun applyEngineClasses(rows: List, verdicts: ByteArray): FlipCounts { + val counts = FlipCounts() + for ((i, row) in rows.withIndex()) { + when (verdicts[i]) { + OUTPOINT_CLASS_UNSPENT -> counts.unspent++ + OUTPOINT_CLASS_NOT_OWNED -> counts.notOwned++ + OUTPOINT_CLASS_KNOWN_UNCREDITED -> { + val current = database.txoDao().getByOutpoint(row.outpoint) + // A row already spent, or gone, changed under the walk. + if (current == null || current.isSpent || + database.txoDao().markSpentByOutpoint(row.outpoint, now()) == 0 + ) { + continue + } + val claims = database.documentDao().getPendingInputsByOutpoint(row.outpoint) + if (claims.isNotEmpty()) { + database.documentDao().deletePendingInputsByIds(claims.map { it.id }) + } + counts.flipped++ + counts.flippedDuffs += current.amount + counts.flippedOutpoints += row.outpoint + } + // OUTPOINT_CLASS_UNKNOWN, and any code this build does not + // know: the engine has no opinion. After a restart its + // finalized set is empty, so this is the ordinary answer + // rather than the alarming one. + else -> counts.unknown++ + } + } + return counts + } + + /** + * Run one reconcile store step under [callbackExclusion], so no + * changeset round commits inside it. Returns null when the run must + * stop — cancelled, or failed. + * Cancellation is re-checked UNDER the lock: wallet deletion bumps the + * manager's epoch before it takes the lock for its cascade, so a step + * that gets the lock after the cascade writes nothing for the deleted + * wallet. A step that throws rolled back its Room transaction, so + * nothing it staged survives; it is logged by operation and exception + * type only. + */ + private suspend fun reconcileStoreStep( + tally: ReconcileTally, + walletRef: String, + operation: String, + isCancelled: () -> Boolean, + step: suspend () -> T, + ): T? = + try { + callbackExclusion.withLock { if (isCancelled()) null else step() } + } catch (cancellation: kotlin.coroutines.cancellation.CancellationException) { + throw cancellation + } catch (t: Throwable) { + tally.storeFailures++ + Log.e( + TAG, + "txos reconcile: store step failed operation=$operation " + + "error=${t.javaClass.simpleName} wallet=$walletRef", + ) + null + } + + /** One line per healed / healed-spent / flipped row: the action and + * keyed references only ([logReference]) — never the outpoint, + * address or amount. */ + private fun logReconcileItem(action: String, outpoint: ByteArray, walletRef: String) { + Log.i(TAG, "txos reconcile: item action=$action outpoint=${logReference(outpoint)} wallet=$walletRef") + } + + /** + * The one summary line per run: counts, `completed`, the budget and + * resume flags, and the wallet's reference digest. No amount: the report's duff totals stay with the + * host, since a logged total would expose the wallet's activity in + * value to logcat and anything that collects it. A completed run with + * zero mutations starts with `txos reconcile: mirror consistent`; a run + * that stopped on its budget with zero mutations starts with + * `txos reconcile: page budget used`; anything else is a warning. + * Device tooling greps for `txos reconcile`. + */ + private fun logReconcileSummary(report: TxoReconcileReport, walletRef: String) { + val counts = "engineRows=${report.engineRows} storeRows=${report.storeRows} " + + "inserted=${report.inserted} healedSpent=${report.healedSpent} " + + "flipped=${report.flipped} " + + "alreadyPresent=${report.alreadyPresent} unspent=${report.unspent} " + + "unknown=${report.unknown} notOwned=${report.notOwned} " + + "skippedImmature=${report.skippedImmature} " + + "skippedUnresolvedAccount=${report.skippedUnresolvedAccount} " + + "skippedMissingAddress=${report.skippedMissingAddress} " + + "skippedInvalid=${report.skippedInvalid} skippedForeign=${report.skippedForeign} " + + "unresolvedAccount=${report.unresolvedAccount} staleRetries=${report.staleRetries} " + + "transportFailures=${report.transportFailures} storeFailures=${report.storeFailures} " + + "completed=${report.completed} stoppedOnBudget=${report.stoppedOnBudget} " + + "resumedFromCheckpoint=${report.resumedFromCheckpoint} wallet=$walletRef" + when { + report.completed && report.mutations == 0 -> + Log.i(TAG, "txos reconcile: mirror consistent ($counts)") + // The budget stop is the expected shape for a large inventory: + // nothing found so far, and the next run carries on. A failed + // read or write stops the run before the budget can, so a + // budget stop never hides one. + report.stoppedOnBudget && report.mutations == 0 -> + Log.i(TAG, "txos reconcile: page budget used, resuming next run ($counts)") + else -> Log.w(TAG, "txos reconcile: $counts") + } } override fun onWalletChangesetUtxoSpent( @@ -3315,6 +4561,7 @@ class PlatformWalletPersistenceHandler( // Per-call memo of address → account row (a wallet's UTXOs // cluster on few addresses/accounts; avoids N duplicate joins). val accountByAddress = HashMap() + val accountById = HashMap() for (txo in rows) { // Stale-flag guard (see doc): consumed-but-unflipped rows // must not rehydrate as spendable. @@ -3345,11 +4592,7 @@ class PlatformWalletPersistenceHandler( continue } } - val account = txo.accountId?.let { database.accountDao().getById(it) } - ?: accountByAddress.getOrPut(txo.address) { - database.coreAddressDao().getByAddress(txo.address) - ?.accountId?.let { database.accountDao().getById(it) } - } + val account = resolveTxoOwnerAccount(txo, accountByAddress, accountById) if (account == null) { Log.w(TAG, "load: skipping UTXO with no resolvable account (address=${txo.address})") continue @@ -3618,14 +4861,20 @@ class PlatformWalletPersistenceHandler( keyClass: Int, userIdentityId: ByteArray, friendIdentityId: ByteArray, - ): AccountEntity? = - db.accountDao().getByKey(walletId, accountType, accountIndex).firstOrNull { c -> + // Maps an identity column to its canonical form before comparing, + // on both sides. The persister's own lookups compare as stored. + normaliseIdentity: (ByteArray) -> ByteArray = { it }, + ): AccountEntity? { + val user = normaliseIdentity(userIdentityId) + val friend = normaliseIdentity(friendIdentityId) + return db.accountDao().getByKey(walletId, accountType, accountIndex).firstOrNull { c -> c.standardTag == standardTag && c.registrationIndex == registrationIndex && c.keyClass == keyClass && - c.userIdentityId.contentEquals(userIdentityId) && - c.friendIdentityId.contentEquals(friendIdentityId) + normaliseIdentity(c.userIdentityId).contentEquals(user) && + normaliseIdentity(c.friendIdentityId).contentEquals(friend) } + } private suspend fun coreAddressIdIfPresent(db: DashDatabase, address: String?): String? { if (address == null) return null @@ -3691,6 +4940,9 @@ class PlatformWalletPersistenceHandler( buffers.remove(walletId.toHex()) pendingRoundAliases.remove(walletId.toHex()) orphanedAliases.remove(walletId.toHex()) + // A wallet added again under the same id starts a new reconcile + // cycle rather than resuming the deleted wallet's. + txoReconcileCheckpoints.remove(walletId.toHex()) database.withTransaction { val walletRow = database.walletDao().getByWalletId(walletId) val walletNetwork = walletRow?.networkRaw @@ -4188,6 +5440,123 @@ class PlatformWalletPersistenceHandler( runBlocking(dispatcher) { block() } companion object { + /** `AccountTypeTagFFI::DashpayExternalAccount` — watch-only DIP-15 + * contact accounts the engine's inventories never report. */ + internal const val ACCOUNT_TYPE_TAG_DASHPAY_EXTERNAL = 13 + + /** Bytes of a `txos.outpoint` key: 32-byte txid (wire order) plus + * the vout as a little-endian `Int`. Also the wire format of the + * reconcile's outpoint-classification batch. */ + internal const val OUTPOINT_BYTES = 36 + + /** `UTXO_CREDIT_VERDICT_*` codes of the credit-verdict slot + * (rs-platform-wallet-ffi `core_wallet_types.rs`). Observed spent: + * a block spent the outpoint before the output was recognised. */ + internal const val UTXO_CREDIT_VERDICT_OBSERVED_SPENT: Byte = 1 + + /** The record is unconfirmed and a block already spent one of its + * inputs; nothing it created was credited. */ + internal const val UTXO_CREDIT_VERDICT_DOOMED: Byte = 2 + + /** Not credited for a reason the bridge cannot name; no context. */ + internal const val UTXO_CREDIT_VERDICT_UNCREDITED: Byte = 3 + + /** + * Rows per page in both directions of [reconcileTxos] — engine + * UTXOs coming in, store rows going out for classification. + * + * The number itself is not delicate; that there IS one is the + * point. Inventory size is chain-controlled, so an unpaged sweep + * would let anyone who knows a watched address decide how much a + * phone allocates at every SYNCED transition and cadence tick. + */ + const val TXO_RECONCILE_PAGE_SIZE = 512 + + /** + * Pages one [reconcileTxos] run may read, across both passes + * together: heal pages from the engine plus classify pages from the + * store, a stale page read again counting again. A run that reaches + * it stops between pages and the wallet's next run resumes there. + * + * The page size bounds what a run holds at once; this bounds what a + * run does. Without it every automatic run walks the whole + * inventory, and anyone who sends dust to a watched address decides + * how long that is. 16 pages of [TXO_RECONCILE_PAGE_SIZE] is 8192 + * rows per run, so a wallet with up to about 4000 unspent coins + * still finishes a whole cycle (every coin healed and classified) + * in one run, as before; only an inventory past that is spread over + * several runs, one budget per run at the manager's cadence. + */ + const val TXO_RECONCILE_PAGE_BUDGET = 16 + + /** Confirmations an engine coin needs before [reconcileTxos] inserts + * it: coinbase maturity, and well past any plausible reorg. */ + const val TXO_RECONCILE_MIN_CONFIRMATIONS = 100 + + /** Times in a row one classify page is classified again because a + * persistence round committed between its read and its apply, + * before [reconcileTxos] gives up on the run. */ + internal const val TXO_RECONCILE_MAX_STALE_RETRIES = 5 + + /** + * Decoder for the engine inventory transport. Deliberately the + * strict default (`ignoreUnknownKeys = false`, no coercion): the JNI + * emitter and [EngineUtxoPage] are one contract, and a drift between + * them must fail the decode, not heal a defaulted row. + */ + private val engineJson = Json + + /** One page of `walletManagerUtxosPageJson`, through [engineJson]. */ + internal fun decodeEngineUtxoPage(pageJson: String): EngineUtxoPage = + engineJson.decodeFromString(EngineUtxoPage.serializer(), pageJson) + + /** One classification batch, as `walletManagerClassifyOutpoints` + * deserializes it (rs-unified-sdk-jni `OutpointQuery`). */ + internal fun encodeOutpointQueries(queries: List): String = + engineJson.encodeToString( + kotlinx.serialization.builtins.ListSerializer(OutpointQuery.serializer()), + queries, + ) + + /** The query for one store coin filed under [owner]: the account + * tuple as the store holds it (an identity half only when it is a + * set 32-byte id, as the native side emits them) plus the + * outpoint and the script the store recorded. */ + internal fun outpointQueryFor( + owner: AccountEntity, + txid: ByteArray, + vout: Int, + scriptPubKey: ByteArray, + ): OutpointQuery { + fun setIdentityHex(id: ByteArray) = id.takeIf { it.size == 32 && it.any { b -> b != 0.toByte() } }?.toHex() + return OutpointQuery( + typeTag = owner.accountType, + standardTag = owner.standardTag, + index = owner.accountIndex, + registrationIndex = owner.registrationIndex, + keyClass = owner.keyClass, + userIdentityId = setIdentityHex(owner.userIdentityId), + friendIdentityId = setIdentityHex(owner.friendIdentityId), + txid = txid.toHex(), + vout = vout, + scriptHex = scriptPubKey.toHex(), + ) + } + + /** [reconcileTxos] classification verdicts, mirroring + * `platform_wallet::manager::accessors::OutpointClass`. Only + * [OUTPOINT_CLASS_KNOWN_UNCREDITED] is positive evidence: the + * owning account knows the funding txid, recognises the script, + * does not hold the coin, AND a funds account holds a MINED + * record that spends the outpoint. Plain absence from the live + * UTXO set is NOT evidence — a mempool spend that never confirms + * and a conflict sweep both leave a coin absent with its funding + * known — so those arrive as [OUTPOINT_CLASS_UNKNOWN]. */ + internal const val OUTPOINT_CLASS_UNKNOWN: Byte = 0 + internal const val OUTPOINT_CLASS_UNSPENT: Byte = 1 + internal const val OUTPOINT_CLASS_KNOWN_UNCREDITED: Byte = 2 + internal const val OUTPOINT_CLASS_NOT_OWNED: Byte = 3 + internal const val PERSISTENCE_CAPABILITIES_VERSION: Int = 1 internal const val CAPABILITY_ATOMIC_CHANGESETS: Long = 0x01 internal const val CAPABILITY_INVITATIONS: Long = 0x02 @@ -4388,6 +5757,10 @@ class NativeStoredTransactionInputs( * the `contactAcceptedAccounts` column. Empty → null (matches the * absent-optional convention of the other nullable columns). */ +/** [getOrPut] that also remembers a null result, so a miss is looked up once. */ +private inline fun MutableMap.memo(key: K, load: () -> V?): V? = + if (containsKey(key)) get(key) else load().also { put(key, it) } + internal fun encodeAcceptedAccounts(accounts: IntArray): ByteArray? { if (accounts.isEmpty()) return null val buffer = java.nio.ByteBuffer.allocate(accounts.size * 4) // big-endian by default @@ -4413,6 +5786,43 @@ internal fun ByteArray.toHex(): String { return String(out) } +/** [hexToByteArray], or null when this is not well-formed hex. */ +internal fun String.hexToByteArrayOrNull(): ByteArray? = + try { + hexToByteArray() + } catch (_: IllegalArgumentException) { + null + } + +/** + * A log reference for [value]: the first 12 hex characters of its + * HMAC-SHA256 under [logReferenceKey]. Lets log lines from one process run + * be correlated (same wallet, same coin) without carrying the wallet id, + * txid or outpoint itself. + * + * Keyed, unlike Swift's `SDKLogger.reference` (a plain SHA-256 prefix): + * outpoints are public chain data, so an unkeyed digest lets anyone holding + * the logs hash candidate outpoints and match them, and the wallet + * reference on the same line then groups the matches into one wallet's + * history. The key is random per process and never logged, so a reference + * matches no digest computable outside the process and does not correlate + * across runs. + */ +internal fun logReference(value: ByteArray): String = + javax.crypto.Mac.getInstance(LOG_REFERENCE_MAC).run { + init(logReferenceKey) + doFinal(value) + }.copyOf(6).toHex() + +private const val LOG_REFERENCE_MAC = "HmacSHA256" + +/** The [logReference] key: 32 random bytes, drawn once per process on + * first use and held only in memory. */ +private val logReferenceKey: javax.crypto.spec.SecretKeySpec by lazy { + val key = ByteArray(32).also { java.security.SecureRandom().nextBytes(it) } + javax.crypto.spec.SecretKeySpec(key, LOG_REFERENCE_MAC) +} + /** Inverse of [toHex]; requires an even-length lowercase/uppercase hex string. */ internal fun String.hexToByteArray(): ByteArray { require(length % 2 == 0) { "hex string must have even length, got $length" } diff --git a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/dao/TxoDao.kt b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/dao/TxoDao.kt index ac484917836..5b7ee10fc39 100644 --- a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/dao/TxoDao.kt +++ b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/dao/TxoDao.kt @@ -21,6 +21,27 @@ interface TxoDao { @Query("SELECT * FROM txos WHERE walletId = :walletId") fun observeByWallet(walletId: ByteArray): Flow> + /** + * One outpoint-ordered page of a wallet's `isSpent = 0` TXOs, for the + * store reconcile's classify pass, which must not hold the whole table + * at once and only ever asks the engine about the rows the restore + * would hand back. Pass an empty [after] to start (an empty BLOB sorts + * before every real 36-byte outpoint), then the previous page's last + * `outpoint` to continue. + * + * Not cheap per page: SQLite finds the wallet's rows through the + * `walletId` index, so each page reads all of them (spent rows are + * never deleted) and sorts the unspent ones past [after]. The cursor + * is an outpoint, not an offset, so it is exact: no row is visited + * twice or skipped because another was inserted, deleted or marked + * spent mid-sweep. + */ + @Query( + "SELECT * FROM txos WHERE walletId = :walletId AND isSpent = 0 AND outpoint > :after " + + "ORDER BY outpoint LIMIT :limit", + ) + suspend fun pageUnspentByWallet(walletId: ByteArray, after: ByteArray, limit: Int): List + /** WalletMemoryExplorer: `txo.walletId == walletId && txo.isSpent == false`. */ @Query("SELECT * FROM txos WHERE walletId = :walletId AND isSpent = 0") fun observeUnspentByWallet(walletId: ByteArray): Flow> diff --git a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/PlatformWalletManager.kt b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/PlatformWalletManager.kt index ab4207b1213..f35fbbb97d6 100644 --- a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/PlatformWalletManager.kt +++ b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/PlatformWalletManager.kt @@ -33,6 +33,7 @@ import org.dashfoundation.dashsdk.persistence.DashDatabase import org.dashfoundation.dashsdk.persistence.PlatformWalletPersistenceHandler import org.dashfoundation.dashsdk.persistence.entities.DashpayPaymentEntity import org.dashfoundation.dashsdk.persistence.hexToByteArray +import org.dashfoundation.dashsdk.persistence.logReference import org.dashfoundation.dashsdk.persistence.toBase58String import org.dashfoundation.dashsdk.persistence.toHex import org.json.JSONArray @@ -1178,6 +1179,12 @@ class PlatformWalletManager( _wallets.value[key]?.let { runCatching { it.close() } } _wallets.update { it - key } _dashPayUnlockStatus.update { it - key } + // A TXO-store reconcile in flight for any wallet stops at its next + // page or store step (each re-checks under the callback gate, so + // none writes after the cascade below). Coarse on purpose — the + // cadence re-runs the others. + txoReconcile.bumpEpoch() + txoReconcile.forget(key) // 3. Snapshot → atomic secret delete → Room cascade, with both // persistence callbacks and app-side key writers excluded. @@ -1412,6 +1419,149 @@ class PlatformWalletManager( mapNativeErrors { DashpayNative.walletManagerAccountBalances(managerHandle, walletId) } } + /** + * Reconcile the Room `txos` mirror of [walletId] against the engine, + * once the SPV scan has reached a trustworthy steady state — the Kotlin + * port of Swift's `reconcileCoreTxoStore(for:)` (dashpay/platform#4638 + * Part 2). The mirror is what the engine is restored from at every + * launch, so this inserts engine coins the store lacks and marks spent + * the rows the engine proves spent; it never deletes a row, never + * un-marks one, and never acts on absence. See + * [PlatformWalletPersistenceHandler.reconcileTxos] for the passes. + * + * Gates, each a [TxoReconcileOutcome.Skipped] rather than an error, in + * order: the manager not closed; the wallet loaded; no run for it + * already in flight; SPV in steady state + * ([TxoReconcileGates.isSteadySyncState]); a scan tip + * ([TxoReconcileGates.scanTipHeight]); no latched sync fault + * ([syncFaultDetected] — a rejected round means rows are missing by + * design and a rescan is pending); and the wallet's own scan watermark + * as the engine holds it + * ([WalletManagerNative.walletManagerCoreWalletSyncedHeight], Swift's + * `platform_wallet_core_wallet_state`) within + * [TxoReconcileGates.TIP_MARGIN] blocks of the tip: the tip says how far + * the CLIENT got, and a wallet added behind the tip is still being + * scanned. Not the persisted Room `syncedHeight`: that copy trails the + * engine by up to one persistence round, and the SYNCED transition lands + * inside that window — gating on it skipped the run at the end of every + * restore. + * + * The run holds a [teardownGate] borrow, so the native manager is not + * destroyed under an engine read. [closeSuspending] and [removeWallet] + * bump the reconcile epoch first, and the run stops between pages, + * reporting `completed = false`. + * + * One run reads at most + * [PlatformWalletPersistenceHandler.TXO_RECONCILE_PAGE_BUDGET] pages; a + * larger inventory is finished by the wallet's later runs, each picking + * up where the previous one stopped (the report's `stoppedOnBudget` and + * `resumedFromCheckpoint`). The place is scoped to the reconcile epoch + * the run started in, so a removal (of any wallet) or a close makes the + * next run start a new cycle: a wallet removed and added again never + * resumes the removed wallet's place. + * + * Runs automatically on the steady-state transition of the SPV progress + * poll and every 30 minutes while it lasts (see + * [maybeReconcileTxoStores]); hosts may also call it. Logs one skip + * line (reason and wallet reference) or the handler's summary line — + * counts and reference digests only. + */ + suspend fun reconcileTxoStore(walletId: ByteArray): TxoReconcileOutcome { + val walletRef = logReference(walletId) + fun skip(reason: TxoReconcileSkipReason): TxoReconcileOutcome { + android.util.Log.d( + "PlatformWalletManager", + "txos reconcile: skipped reason=${reason.name} wallet=$walletRef", + ) + return TxoReconcileOutcome.Skipped(reason) + } + if (isClosed) return skip(TxoReconcileSkipReason.MANAGER_CLOSED) + val key = walletId.toHex() + if (walletId.size != 32 || _wallets.value[key] == null) { + return skip(TxoReconcileSkipReason.WALLET_UNKNOWN) + } + if (!txoReconcile.tryBegin(key)) return skip(TxoReconcileSkipReason.ALREADY_RUNNING) + try { + val generation = txoReconcile.currentEpoch + val progress = _spvProgress.value + if (!TxoReconcileGates.isSteadySyncState(progress)) { + return skip(TxoReconcileSkipReason.NOT_STEADY_STATE) + } + val tipHeight = TxoReconcileGates.scanTipHeight(progress) + ?: return skip(TxoReconcileSkipReason.TIP_UNAVAILABLE) + return try { + teardownGate.withOp { + if (syncFaultDetected()) return@withOp skip(TxoReconcileSkipReason.SYNC_FAULT_DETECTED) + val walletSyncedHeight = try { + withContext(Dispatchers.IO) { + mapNativeErrors { + WalletManagerNative.walletManagerCoreWalletSyncedHeight(managerHandle, walletId) + } + } + } catch (notFound: DashSdkError.PlatformWallet.NotFound) { + return@withOp skip(TxoReconcileSkipReason.WALLET_UNKNOWN) + } + if (!TxoReconcileGates.walletCaughtUp(walletSyncedHeight, tipHeight)) { + return@withOp skip(TxoReconcileSkipReason.WALLET_BEHIND_TIP) + } + val isCancelled = { isClosed || txoReconcile.currentEpoch != generation } + if (isCancelled()) return@withOp skip(TxoReconcileSkipReason.MANAGER_CLOSED) + val report = persistenceHandler.reconcileTxos( + walletId = walletId, + tipHeight = tipHeight, + checkpointScope = generation, + isCancelled = isCancelled, + engineUtxoPage = { cursor, limit -> + reconcileEngineRead("engine inventory page", walletRef) { + WalletManagerNative.walletManagerUtxosPageJson( + managerHandle, walletId, cursor, limit, + ) + } + }, + classifyOutpoints = { queriesJson -> + reconcileEngineRead("outpoint classification", walletRef) { + WalletManagerNative.walletManagerClassifyOutpoints( + managerHandle, walletId, queriesJson, + ) + } + }, + ) + // Not for a run the epoch move overtook: its wallet may + // be a re-added one whose cadence the removal cleared, + // and which is due at once (see markRun). + txoReconcile.markRun(key, monotonicNowMs(), generation) + TxoReconcileOutcome.Reconciled(report) + } + } catch (closing: IllegalStateException) { + // The teardown gate refuses new borrows once close began. + if (isClosed) skip(TxoReconcileSkipReason.MANAGER_CLOSED) else throw closing + } + } finally { + txoReconcile.end(key) + } + } + + /** + * One engine read of the reconcile, off the caller's thread. A native + * failure is logged by kind and exception type only — the message of a + * native error can echo the input it rejected — and becomes the null + * the handler's stop-and-report contract is written against. + */ + private suspend fun reconcileEngineRead(what: String, walletRef: String, read: () -> T): T? = + withContext(Dispatchers.IO) { + try { + mapNativeErrors { read() } + } catch (cancellation: kotlin.coroutines.cancellation.CancellationException) { + throw cancellation + } catch (t: Throwable) { + android.util.Log.w( + "PlatformWalletManager", + "txos reconcile: $what failed error=${t.javaClass.simpleName} wallet=$walletRef", + ) + null + } + } + /** * Refresh the persisted DashPay payment history for one identity: * one FFI read (`managed_identity_get_dashpay_payments`) + one Room @@ -2375,6 +2525,10 @@ class PlatformWalletManager( suspend fun stopSpv() = withContext(Dispatchers.IO) { mapNativeErrors { WalletManagerNative.spvStop(managerHandle) } + // Arm the reconcile's next steady-state edge here as well as on the + // poll's next stopped read: a restart inside one poll interval + // would otherwise never be seen as a stop. + txoReconcile.spvStopped() } /** Clear all persisted SPV storage (headers, filters, state). */ @@ -2463,19 +2617,69 @@ class PlatformWalletManager( if (running) { runCatching { spvSyncProgress() }.getOrNull()?.let { next -> if (next != _spvProgress.value) _spvProgress.value = next + maybeReconcileTxoStores(next) } runCatching { spvTipUnixSeconds() }.getOrNull()?.let { tip -> if (tip != _spvTipUnixSeconds.value) _spvTipUnixSeconds.value = tip } - } else if (_spvProgress.value != SpvSyncProgressData.EMPTY) { - // SPV stopped — reset so the overlay clears. - _spvProgress.value = SpvSyncProgressData.EMPTY + } else { + // SPV stopped — the next steady read after a restart is + // a new steady state for the reconcile. + txoReconcile.spvStopped() + if (_spvProgress.value != SpvSyncProgressData.EMPTY) { + // Reset so the overlay clears. + _spvProgress.value = SpvSyncProgressData.EMPTY + } } delay(POLL_INTERVAL_MS) } } } + /** In-flight runs, per-wallet cadence, the steady-state edge and the + * cancellation epoch of the TXO-store reconcile. */ + private val txoReconcile = TxoReconcileCoordinator() + + /** + * SDK-internal trigger for [reconcileTxoStore], fed by every accepted + * read of the SPV progress poll: run every loaded wallet on the + * transition into steady state, and again every 30 minutes per wallet + * while it lasts (each wallet's last run is stamped when it is + * scheduled) — never while a run for that wallet is in flight or a + * launched run of it has not ended, and never inline: the poll tick + * stays cheap. Lives here rather than in + * the host apps so every host gets the repair without wiring anything. + * A run that throws is logged (exception type and wallet reference + * only) and retried on the next cadence tick — never allowed to kill + * the progress poll. + */ + private fun maybeReconcileTxoStores(progress: SpvSyncProgressData) { + if (isClosed) return + val loaded = wallets.value + val due = txoReconcile.walletsDue(progress, loaded.keys, monotonicNowMs()) + for (key in due) { + // `due` is a subset of `loaded.keys`. + val walletId = loaded.getValue(key).walletId + // Released on completion, not in a `finally`: a job cancelled + // before it starts never runs its body, but still completes. + scope.launch { + try { + reconcileTxoStore(walletId) + } catch (cancellation: kotlin.coroutines.cancellation.CancellationException) { + throw cancellation + } catch (t: Throwable) { + android.util.Log.w( + "PlatformWalletManager", + "txos reconcile: run failed error=${t.javaClass.simpleName} " + + "wallet=${logReference(walletId)}", + ) + } + }.invokeOnCompletion { txoReconcile.launchEnded(key) } + } + } + + private fun monotonicNowMs(): Long = System.nanoTime() / 1_000_000 + // ── DashPay sync + seedless unlock ──────────────────────────────── // // Port of `PlatformWalletManagerDashPaySync.swift` + the unlock flow @@ -2902,6 +3106,10 @@ class PlatformWalletManager( * teardown hops to [Dispatchers.IO]). */ suspend fun closeSuspending() { + // Stale any TXO-store reconcile in flight first: it stops at its next + // page or store step instead of holding teardown behind the rest of + // its walk. + txoReconcile.bumpEpoch() val bundle = bundleRef.getAndSet(0) if (bundle == 0L) return // NonCancellable from the moment the bundle is claimed: bundleRef diff --git a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcile.kt b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcile.kt new file mode 100644 index 00000000000..ea0c5be95a2 --- /dev/null +++ b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcile.kt @@ -0,0 +1,213 @@ +package org.dashfoundation.dashsdk.wallet + +import org.dashfoundation.dashsdk.persistence.PlatformWalletPersistenceHandler +import java.util.concurrent.atomic.AtomicLong + +/** + * What [PlatformWalletManager.reconcileTxoStore] did — the Kotlin twin of + * Swift's `CoreTxoReconcileOutcome` (dashpay/platform#4638). + */ +sealed class TxoReconcileOutcome { + /** A gate refused the run; nothing was read or written. */ + data class Skipped(val reason: TxoReconcileSkipReason) : TxoReconcileOutcome() + + /** The run happened; [report] says how far it got and what it changed. */ + data class Reconciled( + val report: PlatformWalletPersistenceHandler.TxoReconcileReport, + ) : TxoReconcileOutcome() +} + +/** Why [PlatformWalletManager.reconcileTxoStore] did not run. */ +enum class TxoReconcileSkipReason { + /** The manager is closed or closing. */ + MANAGER_CLOSED, + + /** The wallet is not loaded in this manager (or has no Room row). */ + WALLET_UNKNOWN, + + /** A run for this wallet is already in flight. */ + ALREADY_RUNNING, + + /** SPV is not in its steady state (see [TxoReconcileGates.isSteadySyncState]). */ + NOT_STEADY_STATE, + + /** The progress carries no scan tip height. */ + TIP_UNAVAILABLE, + + /** The native manager latched a sync fault: rows are missing by design + * and a rescan is pending ([PlatformWalletManager.syncFaultDetected]). */ + SYNC_FAULT_DETECTED, + + /** The engine's scan watermark for the wallet trails the scan tip by + * more than [TxoReconcileGates.TIP_MARGIN] blocks — it is still being + * scanned. */ + WALLET_BEHIND_TIP, +} + +/** The pure gates of the TXO-store reconcile, testable without a manager. */ +internal object TxoReconcileGates { + /** How far the engine's scan watermark for the wallet may trail the + * scan tip for the scan to count as complete for this wallet. */ + const val TIP_MARGIN = 6 + + /** Cadence of the automatic run while SPV stays in steady state. */ + const val CADENCE_MS = 30 * 60 * 1_000L + + /** + * Whether the wallet's own scan watermark (the engine's, see + * [org.dashfoundation.dashsdk.ffi.WalletManagerNative.walletManagerCoreWalletSyncedHeight]) + * is within [TIP_MARGIN] blocks of the scan tip — Swift's + * `synced_height + margin >= tip`. The tip says how far the CLIENT got; + * a wallet added behind it is still being scanned. + */ + fun walletCaughtUp(walletSyncedHeight: Long, tipHeight: Int): Boolean = + walletSyncedHeight >= 0 && walletSyncedHeight + TIP_MARGIN >= tipHeight + + /** + * dash-spv's steady state for a fully synced client is + * [SpvSyncState.WAIT_FOR_EVENTS] with the filter phase at its target + * height; [SpvSyncState.SYNCED] is the transient window before it. + * Either counts — anything else is a scan still running (or not + * running at all, which the poll publishes as + * [SpvSyncProgressData.EMPTY]: waiting, with no filter phase). + */ + fun isSteadySyncState(progress: SpvSyncProgressData): Boolean = + when (progress.overallState) { + SpvSyncState.SYNCED -> true + SpvSyncState.WAIT_FOR_EVENTS -> { + val filters = progress.filters + filters != null && filters.targetHeight > 0 && filters.currentHeight >= filters.targetHeight + } + SpvSyncState.WAITING_FOR_CONNECTIONS, SpvSyncState.SYNCING, SpvSyncState.ERROR -> false + } + + /** + * The scan tip: the filter phase's height (the wallet-relevant one), + * falling back to the header tip when the filter phase is absent; null + * when neither carries a height. + */ + fun scanTipHeight(progress: SpvSyncProgressData): Int? { + val height = progress.filters?.currentHeight?.takeIf { it > 0 } + ?: progress.headers?.currentHeight?.takeIf { it > 0 } + ?: return null + return height.coerceAtMost(Int.MAX_VALUE.toLong()).toInt() + } +} + +/** + * The manager-side state of the TXO-store reconcile — the Kotlin twin of + * Swift's `coreTxoReconcileInFlight` / `coreTxoReconcileLastRunAt` / + * `coreTxoReconcileWasSteady` / `coreTxoReconcileEpoch`. Thread-safe: the + * progress poll, scheduled runs and host calls all reach it. + */ +internal class TxoReconcileCoordinator(private val cadenceMs: Long = TxoReconcileGates.CADENCE_MS) { + private val lock = Any() + private var wasSteady = false + private val inFlight = HashSet() + + /** + * Wallets [walletsDue] handed out whose launched run has not ended yet + * ([launchEnded]). A launched run claims the wallet ([tryBegin]) only + * once its coroutine starts, so [inFlight] alone leaves a gap: a rising + * edge inside it (a new block takes the client out of steady state and + * back) would schedule the wallet again, and the second run would + * either skip or run straight after the first. Swift keeps the same set + * (`coreTxoReconcileLaunched`). + */ + private val launched = HashSet() + private val lastRunAtMs = HashMap() + private val epoch = AtomicLong() + + /** + * Bumped by manager close and by wallet deletion; a run captures it at + * start and stops between pages once it moves. + */ + val currentEpoch: Long get() = epoch.get() + + fun bumpEpoch() { + epoch.incrementAndGet() + } + + /** + * Feed one progress read (every accepted read, not only a changed one: + * it is the reconcile's only clock, and a quiet steady-state wallet's + * progress does not change for the whole cadence). Returns the wallets + * to run now — all of [walletKeys] on the rising edge into steady + * state, and afterwards each wallet whose last run is at least the + * cadence old (or that never ran) — never one already in flight, and + * never one whose launched run has not ended. Each returned wallet is + * stamped NOW, at schedule time, so the next tick does not schedule it + * again while this run is still gating, and is recorded as launched + * until the caller reports [launchEnded]. + */ + fun walletsDue(progress: SpvSyncProgressData, walletKeys: Collection, nowMs: Long): List = + synchronized(lock) { + val steady = TxoReconcileGates.isSteadySyncState(progress) + val rising = steady && !wasSteady + wasSteady = steady + if (!steady) return@synchronized emptyList() + walletKeys.filter { key -> + if (key in inFlight || key in launched) return@filter false + val last = lastRunAtMs[key] + val due = rising || last == null || nowMs - last >= cadenceMs + if (due) { + lastRunAtMs[key] = nowMs + launched.add(key) + } + due + } + } + + /** + * SPV stopped: arm the next rising edge. The poll publishes no progress + * read while SPV is down, so without this a client that restarts + * straight into steady state would never leave it as far as + * [walletsDue] can tell, and every wallet would wait out the cadence. + */ + fun spvStopped() { + synchronized(lock) { wasSteady = false } + } + + /** + * The run [walletsDue] handed out for [walletKey] has ended, or was + * never started. Must be called exactly once per wallet returned, even + * for a launch cancelled before its body ran. + */ + fun launchEnded(walletKey: String) { + synchronized(lock) { launched.remove(walletKey) } + } + + /** Claim the wallet for one run; false when a run is already in flight. */ + fun tryBegin(walletKey: String): Boolean = synchronized(lock) { inFlight.add(walletKey) } + + /** Release the claim [tryBegin] took. */ + fun end(walletKey: String) { + synchronized(lock) { inFlight.remove(walletKey) } + } + + /** + * Record a finished run of the wallet for the cadence, but only while + * [runEpoch] (the epoch the run captured at its start) is still + * current. A run that a close or a wallet removal overtook stopped + * early, and its wallet may since have been removed and added again + * under the same id: [forget] already cleared that wallet's cadence, and + * stamping it now would hold the re-added wallet off for a whole + * cadence. Returns whether the stamp was written. + * + * The epoch is compared under the same lock [forget] takes, and + * [bumpEpoch] always precedes [forget] in a removal, so a stamp either + * lands before the removal's [forget] (which then clears it) or sees + * the moved epoch and is not written. + */ + fun markRun(walletKey: String, nowMs: Long, runEpoch: Long): Boolean = + synchronized(lock) { + if (epoch.get() != runEpoch) return@synchronized false + lastRunAtMs[walletKey] = nowMs + true + } + + /** Forget a deleted wallet's cadence. */ + fun forget(walletKey: String) { + synchronized(lock) { lastRunAtMs.remove(walletKey) } + } +} diff --git a/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandlerTest.kt b/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandlerTest.kt index 6a40c939b72..c3b8918556e 100644 --- a/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandlerTest.kt +++ b/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandlerTest.kt @@ -13,6 +13,10 @@ import androidx.test.core.app.ApplicationProvider import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.first import kotlinx.coroutines.test.runTest +import kotlinx.serialization.json.int +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive import org.dashfoundation.dashsdk.Network import org.dashfoundation.dashsdk.errors.DashSdkError import org.dashfoundation.dashsdk.ffi.NativePersistenceBridge @@ -6380,6 +6384,48 @@ class PlatformWalletPersistenceHandlerTest { assertTrue(row.isSweptTombstone) } + // ── TXO-store reconcile (the job-flower change-drop repair) ─────── + + private val changeTxid = ByteArray(32) { 7 } + private val reconcileTip = 1_536_950 + + @Test + fun reconcileHealsMissingChangeTxoAndLeavesNetAmountAlone() = runTest { + // A send record born blind to its own change output: netAmount + // persisted as the full input value (the job-flower 6cef55ab… + // shape) and NO txos row for the change. + db.transactionDao().upsert( + org.dashfoundation.dashsdk.persistence.entities.TransactionEntity( + txid = changeTxid, + transactionData = byteArrayOf(1, 2, 3), + netAmount = -1_000_010_000L, + ), + ) + + saveReconcileAddress(attributeUnownedTxos(walletId)) + val report = handler.reconcileInventory( + inventoryJson(rows = listOf(engineRowJson(changeTxid, vout = 1, amount = 989_009_773L))), + ) + + assertEquals(1, report.inserted) + assertEquals(989_009_773L, report.insertedDuffs) + + val row = db.txoDao().getByOutpoint(makeOutpoint(changeTxid, 1)) + assertNotNull(row) + assertFalse(row!!.isSpent) + assertEquals(989_009_773L, row.amount) + assertTrue(row.isConfirmed) + + // The stored netAmount is NOT mutated: the record may already carry + // the corrected net (a corrective callback racing this sweep), and + // blind addition double-credits. The event pipeline owns net + // correctness. + assertEquals( + -1_000_010_000L, + db.transactionDao().getByTxid(changeTxid)!!.netAmount, + ) + } + @Test fun aRepointedTombstoneIsRestampedToTheLaterSweep() = runTest { // A chained sweep that re-points a still-unfunded claim to a new @@ -7034,6 +7080,2043 @@ class PlatformWalletPersistenceHandlerTest { assertTrue(covered.contentEquals(after.dashPayBackfillCovered)) } + /** One engine inventory row, every field the transport carries. */ + private fun engineRowJson( + txid: ByteArray, + vout: Int = 0, + amount: Long = 19_549L, + address: String = reconcileAddress, + scriptHex: String = reconcileScript.toHex(), + height: Int = 1_534_921, + isConfirmed: Boolean = true, + isCoinbase: Boolean = false, + typeTag: Int = 0, + txidHex: String = txid.toHex(), + /** The DashPay identity halves, as hex; absent for every other account. */ + identities: Pair? = null, + ): String = + """{"typeTag":$typeTag,"standardTag":0,"index":0,"registrationIndex":0,"keyClass":0,""" + + identities.let { if (it == null) "" else """"userIdentityId":"${it.first}","friendIdentityId":"${it.second}",""" } + + """"txid":"$txidHex","vout":$vout,""" + + """"amount":$amount,"address":"$address","scriptHex":"$scriptHex",""" + + """"height":$height,"isConfirmed":$isConfirmed,"isInstantlocked":false,""" + + """"isCoinbase":$isCoinbase,"isLocked":false}""" + + /** A whole inventory for [FakeEngine]: the engine's coins, the outpoints + * it proves spent, and the ones it says the store misfiled. */ + private fun inventoryJson( + rows: List = emptyList(), + spent: List> = emptyList(), + notOwned: List> = emptyList(), + ): String { + fun keys(list: List>) = + list.joinToString(",") { (txid, vout) -> """{"txid":"${txid.toHex()}","vout":$vout}""" } + return """{"utxos":[${rows.joinToString(",")}],"spent":[${keys(spent)}],""" + + """"notOwned":[${keys(notOwned)}]}""" + } + + private val reconcileAddress = "yStxXHHzhAx58JhaPBNhn3xsH93UwBM2nd" + private val reconcileScript = byteArrayOf(0x76, 0xa9.toByte(), 0x14) + ByteArray(20) { 0x5b } + + byteArrayOf(0x88.toByte(), 0xac.toByte()) + + /** A wallet whose BIP44 account owns [reconcileAddress] — what a real + * wallet load leaves behind, and what the restore loader needs. */ + private suspend fun seedReconcileWallet(wallet: ByteArray = walletId, address: String = reconcileAddress) { + seedWalletWithAddress(wallet, address) + } + + /** Save the `core_addresses` row for [address] under [accountId], as a + * real wallet load leaves it. The heal pass refuses a coin whose + * address has no saved row. */ + private suspend fun saveReconcileAddress(accountId: Long, address: String = reconcileAddress) { + db.coreAddressDao().upsert( + CoreAddressEntity( + address = address, + poolTypeTag = 0, + addressIndex = 0, + derivationPath = "m/44'/1'/0'/0/0", + accountId = accountId, + ), + ) + } + + /** An unspent, confirmed row of [wallet] for `txid:vout`, owned through + * the address projection exactly as the changeset path writes it. */ + private fun seedUnspentTxo( + txid: ByteArray, + vout: Int = 0, + amount: Long = 19_549L, + wallet: ByteArray = walletId, + address: String = reconcileAddress, + height: Int = 2_391_743, + ) { + assertEquals( + 0, + handler.onWalletChangesetUtxoAdded( + wallet, txid, vout, amount, address, reconcileScript, height, + false, true, false, false, + ), + ) + } + + /** Mark the stored row for `txid:vout` spent, as a settled spend leaves it. */ + private suspend fun markStoredSpent(txid: ByteArray, vout: Int = 0) { + db.txoDao().upsert(db.txoDao().getByOutpoint(makeOutpoint(txid, vout))!!.copy(isSpent = true)) + } + + private suspend fun storedIsSpent(txid: ByteArray, vout: Int = 0): Boolean = + db.txoDao().getByOutpoint(makeOutpoint(txid, vout))!!.isSpent + + private suspend fun pendingClaims(txid: ByteArray, vout: Int = 0) = + db.documentDao().getPendingInputsByOutpoint(makeOutpoint(txid, vout)).size + + private fun restoredUtxoCount(h: PlatformWalletPersistenceHandler = handler): Int = + h.onLoadWalletList().single { it.walletId.contentEquals(walletId) }.utxos.size + + /** A reconcile of [walletId] at [reconcileTip] against [engine]; either + * transport half can be replaced to fail, stall or race a round. */ + private suspend fun PlatformWalletPersistenceHandler.reconcileWith( + engine: FakeEngine = FakeEngine(inventoryJson()), + pageSize: Int = 2, + pageBudget: Int = PlatformWalletPersistenceHandler.TXO_RECONCILE_PAGE_BUDGET, + checkpointScope: Long = 0L, + isCancelled: () -> Boolean = { false }, + engineUtxoPage: suspend (String?, Int) -> String? = engine::page, + classifyOutpoints: suspend (String) -> ByteArray? = engine::classify, + ): PlatformWalletPersistenceHandler.TxoReconcileReport = + reconcileTxos( + walletId = walletId, + tipHeight = reconcileTip, + pageSize = pageSize, + pageBudget = pageBudget, + checkpointScope = checkpointScope, + isCancelled = isCancelled, + engineUtxoPage = engineUtxoPage, + classifyOutpoints = classifyOutpoints, + ) + + /** Every TXO row of [wallet], spent or not — what these tests count + * and attribute, through the DAO's whole-wallet query. */ + private suspend fun walletTxos(wallet: ByteArray = walletId): List = + db.txoDao().observeByWallet(wallet).first() + + /** A round that opens and commits in the gap between an engine read and its apply. */ + private fun commitARound() { + handler.onChangesetBegin(walletId) + handler.onChangesetEnd(walletId, success = true) + } + + // ── 1. Positive engine evidence marks a local unspent row spent ─── + + @Test + fun aPositiveEngineVerdictFlipsTheRowAndKeepsItOutOfTheRestore() = runTest { + // The lost-spend class (dashpay/platform#4425, rust-dashcore#992): + // a store row unspent for a coin the engine proves spent — the + // owning account recorded the funding, owns the script, does not + // hold the coin, and a funds account holds a mined spender. The row + // is marked spent with no spender invented, and the claims on it + // are dropped. + seedReconcileWallet() + val txid = ByteArray(32) { 0x71 } + seedUnspentTxo(txid) + db.documentDao().insertPendingInputs( + listOf( + PendingInputEntity( + outpoint = makeOutpoint(txid, 0), + inputIndex = 0, + spendingTxid = ByteArray(32) { 0x7f }, + walletId = walletId, + ), + ), + ) + assertEquals(1, restoredUtxoCount()) + val engine = FakeEngine(inventoryJson(spent = listOf(txid to 0))) + + val report = handler.reconcileWith(engine) + + assertTrue(report.completed) + assertEquals(1, report.storeRows) + assertEquals(1, report.flipped) + assertEquals(19_549L, report.flippedDuffs) + assertEquals(0, report.inserted) + assertEquals(1, report.mutations) + val coin = db.txoDao().getByOutpoint(makeOutpoint(txid, 0))!! + assertTrue(coin.isSpent) + assertNull("no spender is invented", coin.spendingTxid) + assertNull(coin.supersededByTxid) + assertEquals("claims on a settled coin are dropped", 0, pendingClaims(txid)) + assertEquals(0, restoredUtxoCount()) + // The engine was asked about exactly this coin, with the store's + // own account and script — the ownership it checks. + val query = engine.queries.single() + assertEquals(0, query["typeTag"]!!.jsonPrimitive.int) + assertEquals(0, query["index"]!!.jsonPrimitive.int) + assertEquals(reconcileScript.toHex(), query["scriptHex"]!!.jsonPrimitive.content) + } + + @Test + fun aVerdictReadBeforeAnInterveningRoundIsRefusedAndThePageReclassified() = runTest { + // The engine is asked outside the exclusion lock and the verdict is + // applied under it; a round that opens AND commits in that gap can + // re-credit the very coin (a reorg of its spender hands it back in + // `utxos_added`). The apply refuses a verdict read before that + // round, and the page is classified again against the store as it + // is now — here the engine holds the coin again, so nothing flips. + seedReconcileWallet() + val txid = ByteArray(32) { 0x72 } + seedUnspentTxo(txid) + var classifyCalls = 0 + + val report = handler.reconcileWith( + classifyOutpoints = { + classifyCalls++ + if (classifyCalls == 1) { + commitARound() + byteArrayOf(PlatformWalletPersistenceHandler.OUTPOINT_CLASS_KNOWN_UNCREDITED) + } else { + byteArrayOf(PlatformWalletPersistenceHandler.OUTPOINT_CLASS_UNSPENT) + } + }, + ) + + assertTrue(report.completed) + assertEquals("the first verdict was read before the round and refused", 1, report.staleRetries) + assertEquals("the page is classified again after the refusal", 2, classifyCalls) + assertEquals(0, report.flipped) + assertEquals(1, report.unspent) + assertEquals(1, report.storeRows) + assertFalse( + "a coin the engine re-credited in the gap stays unspent", + db.txoDao().getByOutpoint(makeOutpoint(txid, 0))!!.isSpent, + ) + } + + @Test + fun aPageThatKeepsGoingStaleStopsTheRunIncomplete() = runTest { + // A round commits between every read and apply: the page is + // re-classified a bounded number of times, then the run gives up + // without having written anything. + seedReconcileWallet() + val txid = ByteArray(32) { 0x73 } + seedUnspentTxo(txid) + var classifyCalls = 0 + + val report = handler.reconcileWith( + classifyOutpoints = { + classifyCalls++ + commitARound() + byteArrayOf(PlatformWalletPersistenceHandler.OUTPOINT_CLASS_KNOWN_UNCREDITED) + }, + ) + + val limit = PlatformWalletPersistenceHandler.TXO_RECONCILE_MAX_STALE_RETRIES + 1 + assertFalse(report.completed) + assertEquals(limit, report.staleRetries) + assertEquals(limit, classifyCalls) + assertEquals(0, report.flipped) + assertFalse(db.txoDao().getByOutpoint(makeOutpoint(txid, 0))!!.isSpent) + } + + @Test + fun aVerdictIsNotAppliedWhileARoundIsOpenEvenBeforeItCommits() = runTest { + // A round that has BEGUN but not yet committed in the gap between + // the engine read and the apply has not moved the commit count, yet + // it may already have staged the re-credit of the very coins the + // verdict condemns. A flip landing first would outlive the round on + // a row with a mempool spender link: the credited re-delivery keeps + // a linked row's flag, the restore drops it while the engine holds + // the coin, and no later run looks at a spent row again. The apply + // refuses while the round is open; the run stops incomplete and the + // rows are classified again once it has closed. + seedReconcileWallet() + val plain = ByteArray(32) { 0x74 } + val linked = ByteArray(32) { 0x75 } + val spender = ByteArray(32) { 0x76 } + seedUnspentTxo(plain) + seedUnspentTxo(linked) + recordMempoolSpend(spender, makeOutpoint(linked, 0)) + val linkedBefore = db.txoDao().getByOutpoint(makeOutpoint(linked, 0))!! + assertTrue("sanity: the mempool spender is linked", spender.contentEquals(linkedBefore.spendingTxid)) + assertFalse("sanity: a mempool spender does not mark the coin spent", linkedBefore.isSpent) + assertEquals(2, restoredUtxoCount()) + var classifyCalls = 0 + + val report = handler.reconcileWith( + classifyOutpoints = { + classifyCalls++ + if (classifyCalls == 1) { + // A reorg of the spenders hands both coins back: the + // round opens and stages the credited re-delivery, and + // stays open past the apply. + handler.onChangesetBegin(walletId) + for (txid in listOf(plain, linked)) { + handler.onWalletChangesetUtxoAdded( + walletId, txid, 0, 19_549L, reconcileAddress, reconcileScript, 2_391_743, + false, true, false, false, + ) + } + } + ByteArray(2) { PlatformWalletPersistenceHandler.OUTPOINT_CLASS_KNOWN_UNCREDITED } + }, + ) + handler.onChangesetEnd(walletId, success = true) + + for (txid in listOf(plain, linked)) { + assertFalse( + "a coin re-credited by the open round stays unspent", + db.txoDao().getByOutpoint(makeOutpoint(txid, 0))!!.isSpent, + ) + } + assertTrue( + "the mempool spender link is untouched", + spender.contentEquals(db.txoDao().getByOutpoint(makeOutpoint(linked, 0))!!.spendingTxid), + ) + assertEquals("both coins are still handed back at the next load", 2, restoredUtxoCount()) + val limit = PlatformWalletPersistenceHandler.TXO_RECONCILE_MAX_STALE_RETRIES + 1 + assertFalse("the run stops rather than apply under an open round", report.completed) + assertEquals(limit, report.staleRetries) + assertEquals(0, report.flipped) + + // Once the round has closed, the next run classifies both rows + // again against the engine as it is now. + val after = handler.reconcileWith( + classifyOutpoints = { ByteArray(2) { PlatformWalletPersistenceHandler.OUTPOINT_CLASS_UNSPENT } }, + ) + assertTrue(after.completed) + assertEquals(0, after.staleRetries) + assertEquals(2, after.unspent) + assertEquals(0, after.flipped) + assertEquals(2, restoredUtxoCount()) + } + + // ── 2. Absence and no-opinion verdicts change nothing ───────────── + + @Test + fun rowsTheEngineGivesNoVerdictOnAreCountedAndLeftAlone() = runTest { + // Three store rows the reconcile has no grounds to touch, each + // counted under its own reason: + // - absent from BOTH engine inventories (vout 4): residue of a + // swept/abandoned transaction, or a funding record this session + // never processed (after a restart the engine's finalized set is + // empty). Absence proves nothing. + // - NOT_OWNED (vout 21): the account the store filed the coin under + // does not monitor its script, so the engine could never have + // credited it there — a store attribution error, not a missing coin. + // - no resolvable owner (vout 22): a query names the account the + // store filed the coin under, because the engine checks that claim + // against its own pools. A row whose owner resolves through neither + // the explicit link nor the core_addresses projection carries no + // claim to check, so it is counted rather than asked about. + db.coreAddressDao().upsert( + CoreAddressEntity( + address = "yOrphanAddr", + publicKey = ByteArray(33), + poolTypeTag = 0, + addressIndex = 0, + derivationPath = "m/44'/1'/0'/0/7", + isUsed = true, + accountId = null, + ), + ) + val rows = listOf(Triple(4, 250_000L, "yTestAddr"), Triple(21, 777_000L, "yTestAddr"), Triple(22, 31_000L, "yOrphanAddr")) + for ((vout, amount, address) in rows) { + handler.onWalletChangesetUtxoAdded( + walletId, changeTxid, vout, amount, address, byteArrayOf(0x51), 1_400_000, + false, true, false, false, + ) + } + assertEquals( + "the orphan row must hang off the address projection for this case to mean anything", + "yOrphanAddr", + db.txoDao().getByOutpoint(makeOutpoint(changeTxid, 22))!!.coreAddressId, + ) + + val report = handler.reconcileInventory(inventoryJson(notOwned = listOf(changeTxid to 21))) + + assertTrue(report.completed) + assertEquals("an unasked row is not handed to the engine", 2, report.storeRows) + assertEquals("a not-owned or unasked row is not also counted as unknown", 1, report.unknown) + assertEquals(1, report.notOwned) + assertEquals(1, report.unresolvedAccount) + assertEquals(0, report.flipped) + assertEquals(0, report.mutations) + for ((vout, amount, _) in rows) { + val row = db.txoDao().getByOutpoint(makeOutpoint(changeTxid, vout))!! + assertFalse(row.isSpent) + assertEquals(amount, row.amount) + } + } + + @Test + fun aRowWhoseAddressArrivedAfterItResolvesItsOwnerTheWayTheRestoreDoes() = runTest { + // The changeset write links `coreAddressId` only when the address + // row already exists, and the address-pool callback that inserts it + // later does not backfill the TXO. The restore loader still routes + // such a row to its account through the row's own address, so it is + // handed back to the engine at every launch; the reconcile must + // resolve it the same way, or a coin the engine proves spent is + // counted unresolved on every run and restored forever. + handler.onPersistWalletMetadata(walletId, testnet, groupId, 0) + handler.onPersistAccountRegistration( + walletId, 0, 0, 0, 0, 0, ByteArray(0), ByteArray(0), ByteArray(78) { 30 }, + ) + val txid = ByteArray(32) { 0x74 } + seedUnspentTxo(txid) + val account = db.accountDao().observeByWallet(walletId).first().single() + db.coreAddressDao().upsert( + CoreAddressEntity( + address = reconcileAddress, + poolTypeTag = 0, + addressIndex = 0, + derivationPath = "m/44'/1'/0'/0/0", + accountId = account.id, + ), + ) + val stored = db.txoDao().getByOutpoint(makeOutpoint(txid, 0))!! + assertNull("the address row arrived after the TXO and nothing backfilled it", stored.coreAddressId) + assertNull(stored.accountId) + assertEquals("the restore routes the row through its own address", 1, restoredUtxoCount()) + val engine = FakeEngine(inventoryJson(spent = listOf(txid to 0))) + + val report = handler.reconcileWith(engine) + + assertTrue(report.completed) + assertEquals("the owner resolves through the row's address", 0, report.unresolvedAccount) + assertEquals(1, report.storeRows) + assertEquals(1, report.flipped) + assertTrue(db.txoDao().getByOutpoint(makeOutpoint(txid, 0))!!.isSpent) + assertEquals("the flipped coin leaves the restore set", 0, restoredUtxoCount()) + // Asked under the account the restore would have routed it to. + val query = engine.queries.single() + assertEquals(account.accountType, query["typeTag"]!!.jsonPrimitive.int) + assertEquals(account.accountIndex, query["index"]!!.jsonPrimitive.int) + } + + // ── 3. A consistent store stays untouched ───────────────────────── + + @Test + fun aConsistentStoreYieldsZeroMutationsAndSpentRowsAreNeverAsked() = runTest { + // Rows the engine also holds unspent — including a YOUNG coin the + // heal pass skips as immature — are consistent, not divergence; a + // spent row is never even asked about; and the run logs one + // "mirror consistent" summary line and nothing else. + seedReconcileWallet() + val unspent = ByteArray(32) { 0xb1.toByte() } + val spent = ByteArray(32) { 0xb2.toByte() } + val young = ByteArray(32) { 0xb3.toByte() } + seedUnspentTxo(unspent) + seedUnspentTxo(spent) + seedUnspentTxo(young, height = reconcileTip - 3) + markStoredSpent(spent) + // The engine holds exactly the store's unspent coins, and says so. + val engine = FakeEngine( + inventoryJson(rows = listOf(engineRowJson(unspent), engineRowJson(young, height = reconcileTip - 3))), + ) + org.robolectric.shadows.ShadowLog.clear() + + val report = handler.reconcileWith(engine) + + assertTrue(report.completed) + assertEquals(0, report.mutations) + assertEquals(1, report.alreadyPresent) + assertEquals(1, report.skippedImmature) + assertEquals(2, report.unspent) + assertEquals(0, report.unknown) + assertEquals("spent rows are never even asked about", 2, report.storeRows) + assertEquals(2, engine.classified) + assertFalse(storedIsSpent(unspent)) + assertFalse(storedIsSpent(young)) + assertTrue("never un-marked", storedIsSpent(spent)) + val summary = org.robolectric.shadows.ShadowLog.getLogs().map { it.msg } + .single { it.startsWith("txos reconcile:") } + assertTrue(summary, summary.startsWith("txos reconcile: mirror consistent (")) + assertTrue(summary, summary.contains("engineRows=2")) + assertTrue(summary, summary.contains("completed=true")) + } + + /** A watch-only DIP-15 contact (DashPay external) account of [walletId], + * owning [address] through the core_addresses projection. */ + private suspend fun seedContactAccount(index: Int, address: String): Long { + db.walletDao().upsert(WalletEntity(walletId, networkRaw = Network.TESTNET.ffiValue)) + val accountId = db.accountDao().insert( + org.dashfoundation.dashsdk.persistence.entities.AccountEntity( + walletId = walletId, + accountType = PlatformWalletPersistenceHandler.ACCOUNT_TYPE_TAG_DASHPAY_EXTERNAL, + accountIndex = index, + accountTypeName = "DashpayExternalAccount", + ), + ) + db.coreAddressDao().upsert( + CoreAddressEntity( + address = address, + publicKey = ByteArray(33), + poolTypeTag = 0, + addressIndex = 0, + derivationPath = "m/9'/1'/15'/0'/x/y/$index", + isUsed = true, + accountId = accountId, + ), + ) + return accountId + } + + @Test + fun watchOnlyContactRowsAreExcludedFromTheClassifyPassHoweverTheyResolve() = runTest { + // Watch-only DIP-15 contact rows are never in the engine's + // inventories; asking about them would be a false positive on every + // wallet with contact payments. Vout 6 is filed under the contact + // account directly. Vout 8 is the production shape: changeset + // writes leave txos.accountId null and route ownership through + // coreAddressId -> core_addresses.accountId, and the exclusion must + // resolve that path too, or every contact row gets classified. + val contactAccountId = seedContactAccount(index = 0, address = "yContactRouted") + for ((vout, address) in listOf(6 to "yContactAddr", 8 to "yContactRouted")) { + handler.onWalletChangesetUtxoAdded( + walletId, changeTxid, vout, 1_230_000L, address, byteArrayOf(0x51), 1_400_000, + false, true, false, false, + ) + } + db.txoDao().upsert(db.txoDao().getByOutpoint(makeOutpoint(changeTxid, 6))!!.copy(accountId = contactAccountId)) + assertNull( + "production shape: accountId is null", + db.txoDao().getByOutpoint(makeOutpoint(changeTxid, 8))!!.accountId, + ) + + val report = handler.reconcileInventory(inventoryJson()) + + assertEquals(2, report.skippedForeign) + assertEquals(0, report.storeRows) + assertEquals(0, report.unknown) + assertFalse(storedIsSpent(changeTxid, 6)) + assertFalse(storedIsSpent(changeTxid, 8)) + } + + // ── 4. Heal pass: a missing engine coin is inserted, validated ──── + + @Test + fun anEngineCoinMissingFromTheStoreIsInsertedWithStubParentAccountAndAddress() = runTest { + // The inventory carries the engine's own isCoinbase / isConfirmed / + // isInstantLocked. They are stamped as given, not assumed: a + // coinbase output filed as an ordinary one misstates maturity in + // the mirror the engine reloads from. + seedReconcileWallet() + val account = db.accountDao().observeByWallet(walletId).first().single() + val txid = ByteArray(32) { 0x74 } + val engine = FakeEngine(inventoryJson(rows = listOf(engineRowJson(txid, height = 1_534_921, isCoinbase = true)))) + + val report = handler.reconcileWith(engine) + + assertTrue(report.completed) + assertEquals(1, report.engineRows) + assertEquals(1, report.inserted) + assertEquals(19_549L, report.insertedDuffs) + val coin = db.txoDao().getByOutpoint(makeOutpoint(txid, 0))!! + assertFalse(coin.isSpent) + assertTrue(coin.isConfirmed) + assertTrue("a coinbase output must be filed as one", coin.isCoinbase) + assertEquals(19_549L, coin.amount) + assertEquals(reconcileAddress, coin.address) + assertTrue(reconcileScript.contentEquals(coin.scriptPubKey)) + assertEquals(1_534_921, coin.height) + assertTrue(walletId.contentEquals(coin.walletId)) + assertEquals("filed under the account the engine named", account.id, coin.accountId) + assertEquals("linked to its address row", reconcileAddress, coin.coreAddressId) + val parent = db.transactionDao().getByTxid(txid) + assertNotNull("a stub parent row holds the relationship", parent) + assertEquals(0, parent!!.transactionData.size) + assertEquals(1, restoredUtxoCount()) + } + + @Test + fun theHealPassRefusesImmatureUnconfirmedUnresolvedAndMalformedCoins() = runTest { + seedReconcileWallet() // BIP44 only: no CoinJoin account row + fun txid(b: Int) = ByteArray(32) { b.toByte() } + val engine = FakeEngine( + inventoryJson( + rows = listOf( + engineRowJson(txid(0x76), height = reconcileTip - 50), // immature + engineRowJson(txid(0x77), height = reconcileTip - 99), // exactly 100 confirmations + // Deep enough, but the engine itself does not call it confirmed. + engineRowJson(txid(0x78), isConfirmed = false), + engineRowJson(txid(0x79), typeTag = 1), // CoinJoin: no store account + engineRowJson(txid(0x7a), scriptHex = ""), // no script + engineRowJson(txid(0x7b), address = ""), // no address + engineRowJson(txid(0x7c), height = 0), // unconfirmed height + engineRowJson(txid(0x7d), txidHex = "7d".repeat(31)), // 31-byte txid + ), + ), + ) + + val report = handler.reconcileWith(engine) + + assertTrue(report.completed) + assertEquals(8, report.engineRows) + assertEquals(1, report.inserted) + assertEquals(3, report.skippedImmature) + assertEquals(1, report.skippedUnresolvedAccount) + assertEquals(3, report.skippedInvalid) + assertNotNull(db.txoDao().getByOutpoint(makeOutpoint(txid(0x77), 0))) + for (b in listOf(0x76, 0x78, 0x79, 0x7a, 0x7b, 0x7c)) { + assertNull("coin $b must not be healed", db.txoDao().getByOutpoint(makeOutpoint(txid(b), 0))) + } + assertEquals(1, walletTxos().size) + } + + @Test + fun aHealTheDrainWritesSpentIsCountedAsHealedSpent() = runTest { + // The spender is already a chainlocked row of the store; only the + // coin it consumed is missing (the funding record never reached the + // store), so its input claim is still pending. The heal inserts the + // row and the drain writes it spent on the spot. Nothing was + // repaired — the engine still holds the coin — but a row was + // written, so it is a mutation rather than a clean pass. + seedReconcileWallet() + val funding = ByteArray(32) { 0x7d } + val spender = ByteArray(32) { 0x7e } + db.transactionDao().upsert( + org.dashfoundation.dashsdk.persistence.entities.TransactionEntity( + txid = spender, + transactionData = ByteArray(10) { 5 }, + context = 3, + blockHeight = 2_391_800, + netAmount = -19_549, + ), + ) + db.documentDao().insertPendingInputs( + listOf( + PendingInputEntity( + outpoint = makeOutpoint(funding, 0), + inputIndex = 0, + spendingTxid = spender, + spendingTransactionTxid = spender, + walletId = walletId, + ), + ), + ) + val engine = FakeEngine(inventoryJson(rows = listOf(engineRowJson(funding)))) + + val report = handler.reconcileWith(engine) + + assertTrue(report.completed) + assertEquals(1, report.engineRows) + assertEquals(1, report.healedSpent) + assertEquals(0, report.inserted) + assertEquals(0, report.flipped) + assertEquals("a row was written, even though nothing was repaired", 1, report.mutations) + assertTrue(db.txoDao().getByOutpoint(makeOutpoint(funding, 0))!!.isSpent) + assertEquals("the claim was consumed by the drain", 0, pendingClaims(funding)) + assertEquals(0, restoredUtxoCount()) + } + + @Test + fun theHealPassNeverHealsContactAccountCoins() = runTest { + // The engine's inventory carries the watch-only DIP-15 external + // accounts' coins — it tracks them to show payments TO contacts, but + // they are the CONTACT's money (12 rows / 5,692,493 duffs healed into + // the store on the 2026-08-25 large-wallet validation run before + // this exclusion). Such a coin is skipped and counted as foreign, + // not healed. The inventory tag is the authoritative check: vout 13 + // is skipped even though its address row never survived + // persistence. The address is the fallback: vout 9's resolves to an + // external account. + seedContactAccount(index = 2, address = "yContactPaid") + val report = handler.reconcileInventory( + inventoryJson( + rows = listOf( + engineRowJson(changeTxid, vout = 9, amount = 10_000L, address = "yContactPaid", height = 1_400_000), + engineRowJson( + changeTxid, vout = 13, amount = 10_000L, address = "yContactNoRow", height = 1_400_000, + typeTag = 13, identities = "11".repeat(32) to "22".repeat(32), + ), + ), + ), + ) + + assertEquals(0, report.inserted) + assertEquals(0L, report.insertedDuffs) + assertEquals(2, report.skippedForeign) + for (vout in listOf(9, 13)) { + assertNull( + "the contact's coin must not enter the store", + db.txoDao().getByOutpoint(makeOutpoint(changeTxid, vout)), + ) + } + } + + @Test + fun reconcileStampsResolvedAccountOnHealedRows() = runTest { + // Persistence lost the TXO. The heal resolves the owning Room + // account from the inventory's account tuple and stamps it on the + // inserted row — a healed row with neither accountId nor a + // resolvable address is skipped by the restore loader at the next + // mirror-reload. + db.walletDao().upsert(WalletEntity(walletId, networkRaw = Network.TESTNET.ffiValue)) + // Production shape: onPersistAccountRegistration stores the FFI's + // 32-zero-byte identity ids verbatim. + val bip44Id = db.accountDao().insert( + org.dashfoundation.dashsdk.persistence.entities.AccountEntity( + walletId = walletId, + accountType = 0, + accountIndex = 0, + accountTypeName = "standardBip44", + userIdentityId = ByteArray(32), + friendIdentityId = ByteArray(32), + ), + ) + // The address row is saved under the account the tuple resolves to: + // the heal refuses a coin whose address is not filed under its + // account, since a cold restore could not sign for it. + saveReconcileAddress(bip44Id, address = "yOrphanAddr") + val report = handler.reconcileInventory( + inventoryJson(rows = listOf(engineRowJson(changeTxid, vout = 11, amount = 70_000L, address = "yOrphanAddr"))), + ) + assertEquals(1, report.inserted) + assertEquals(0, report.skippedUnresolvedAccount) + assertEquals( + "the healed row must carry the account resolved from the inventory tuple", + bip44Id, db.txoDao().getByOutpoint(makeOutpoint(changeTxid, 11))!!.accountId, + ) + } + + @Test + fun reconcileSkipsHealWhenAddressRowMissing() = runTest { + // The engine holds a confirmed, deep-enough coin the store lacks, + // and persistence lost its address row too. A cold restore rebuilds + // the address pools from saved rows only, so a healed coin would + // come back into the balance with no derivation path, and selecting + // it would fail the build. The heal leaves it out and counts it. + val ownerId = attributeUnownedTxos(walletId) + val engine = FakeEngine( + inventoryJson(rows = listOf(engineRowJson(changeTxid, vout = 12, amount = 70_000L))), + ) + org.robolectric.shadows.ShadowLog.clear() + + val first = handler.reconcileWith(engine) + assertTrue(first.completed) + assertEquals(1, first.skippedMissingAddress) + assertEquals(0, first.inserted) + assertEquals(0, first.mutations) + assertNull(db.txoDao().getByOutpoint(makeOutpoint(changeTxid, 12))) + val summary = org.robolectric.shadows.ShadowLog.getLogs() + .single { it.msg.startsWith("txos reconcile: ") && "completed=" in it.msg } + assertTrue(summary.msg, summary.msg.contains("skippedMissingAddress=1 ")) + assertFalse("counts only, no address", summary.msg.contains(reconcileAddress)) + + // Once the address is persisted, a later run heals the coin. + saveReconcileAddress(ownerId) + val second = handler.reconcileWith(engine) + assertTrue(second.completed) + assertEquals(0, second.skippedMissingAddress) + assertEquals(1, second.inserted) + assertNotNull(db.txoDao().getByOutpoint(makeOutpoint(changeTxid, 12))) + } + + @Test + fun reconcileSkipsHealWhenAddressRowUnderAnotherAccount() = runTest { + // A cold restore rebuilds each account's address pool from the rows + // filed under it, so a row with no account, or one under another + // account than the coin's, leaves the coin as unsignable as no row + // at all. Only a row under the tuple's account lets the heal in. + val ownerId = attributeUnownedTxos(walletId) + val otherId = db.accountDao().insert( + org.dashfoundation.dashsdk.persistence.entities.AccountEntity( + walletId = walletId, + accountType = 0, + accountIndex = 1, + accountTypeName = "Standard", + ), + ) + val engine = FakeEngine( + inventoryJson(rows = listOf(engineRowJson(changeTxid, vout = 13, amount = 70_000L))), + ) + + db.coreAddressDao().upsert( + CoreAddressEntity( + address = reconcileAddress, + poolTypeTag = 0, + addressIndex = 0, + derivationPath = "m/44'/1'/0'/0/0", + accountId = null, + ), + ) + val noAccount = handler.reconcileWith(engine) + assertTrue(noAccount.completed) + assertEquals("a row with no account", 1, noAccount.skippedMissingAddress) + assertEquals(0, noAccount.skippedUnresolvedAccount) + assertEquals(0, noAccount.inserted) + assertNull(db.txoDao().getByOutpoint(makeOutpoint(changeTxid, 13))) + + saveReconcileAddress(otherId) + val otherAccount = handler.reconcileWith(engine) + assertTrue(otherAccount.completed) + assertEquals("a row under another account", 1, otherAccount.skippedMissingAddress) + assertEquals(0, otherAccount.skippedForeign) + assertEquals(0, otherAccount.inserted) + assertNull(db.txoDao().getByOutpoint(makeOutpoint(changeTxid, 13))) + + saveReconcileAddress(ownerId) + val healed = handler.reconcileWith(engine) + assertTrue(healed.completed) + assertEquals(0, healed.skippedMissingAddress) + assertEquals(1, healed.inserted) + assertEquals(ownerId, db.txoDao().getByOutpoint(makeOutpoint(changeTxid, 13))?.accountId) + } + + // ── 5. Never un-mark, never delete; idempotent; wallet-scoped ───── + + @Test + fun reconcileNeverUnmarksSpentRowsEvenWhenEngineDisagrees() = runTest { + // A row marked spent while the engine lists the coin unspent: a live + // spend racing the engine is indistinguishable from lost residue, + // and un-marking mid-payment would let the wallet double-spend. + handler.onWalletChangesetUtxoAdded( + walletId, changeTxid, 7, 77_000L, "yTestAddr", byteArrayOf(0x51), 1_400_000, + false, true, false, false, + ) + markStoredSpent(changeTxid, 7) + + val report = handler.reconcileInventory( + inventoryJson(rows = listOf(engineRowJson(changeTxid, vout = 7, amount = 77_000L))), + ) + assertEquals(1, report.alreadyPresent) + assertEquals(0, report.storeRows) + assertEquals(0, report.mutations) + assertTrue("the row must stay spent — un-marking is never done by reconciliation", storedIsSpent(changeTxid, 7)) + } + + @Test + fun aSecondRunChangesNothing() = runTest { + seedReconcileWallet() + val flippedTxid = ByteArray(32) { 0x81.toByte() } + val healedTxid = ByteArray(32) { 0x82.toByte() } + seedUnspentTxo(flippedTxid) + val engine = FakeEngine( + inventoryJson(rows = listOf(engineRowJson(healedTxid)), spent = listOf(flippedTxid to 0)), + ) + + assertEquals(2, handler.reconcileWith(engine).mutations) + + val second = handler.reconcileWith(engine) + assertTrue(second.completed) + assertEquals(0, second.mutations) + assertEquals(1, second.alreadyPresent) + assertEquals("the healed coin is the only unspent row left", 1, second.storeRows) + assertEquals(1, second.unspent) + assertEquals(2, walletTxos().size) + } + + @Test + fun theReconcileTouchesOnlyTheWalletItWasAskedAbout() = runTest { + val otherWalletId = ByteArray(32) { 9 } + seedReconcileWallet() + seedWalletWithAddress(otherWalletId, "yOtherWalletAddr", xpubFill = 31) + val mine = ByteArray(32) { 0x91.toByte() } + val theirs = ByteArray(32) { 0x92.toByte() } + seedUnspentTxo(mine) + seedUnspentTxo(theirs, wallet = otherWalletId, address = "yOtherWalletAddr") + // The fake would flip both if asked; only one may be asked. + val engine = FakeEngine(inventoryJson(spent = listOf(mine to 0, theirs to 0))) + + val report = handler.reconcileWith(engine) + + assertEquals(1, report.storeRows) + assertEquals(1, report.flipped) + assertTrue(db.txoDao().getByOutpoint(makeOutpoint(mine, 0))!!.isSpent) + assertFalse( + "the other wallet's coin is untouched", + db.txoDao().getByOutpoint(makeOutpoint(theirs, 0))!!.isSpent, + ) + assertEquals(mine.toHex(), engine.queries.single()["txid"]!!.jsonPrimitive.content) + } + + /** + * The relaunch shape over a real ON-DISK store [name]: [firstLaunch] + * runs against a fresh handler whose wallet's BIP44 account owns + * [address]; the store is then closed, and reopened twice, each time + * under a fresh handler that [relaunch] checks. + */ + private suspend fun acrossRelaunches( + name: String, + address: String, + firstLaunch: suspend (PlatformWalletPersistenceHandler, DashDatabase) -> Unit, + relaunch: suspend (PlatformWalletPersistenceHandler, DashDatabase) -> Unit, + ) { + val context = ApplicationProvider.getApplicationContext() + suspend fun launch(body: suspend (PlatformWalletPersistenceHandler, DashDatabase) -> Unit) { + val store = Room.databaseBuilder(context, DashDatabase::class.java, name) + .allowMainThreadQueries() + .build() + try { + body(PlatformWalletPersistenceHandler(store, Dispatchers.Unconfined, null, storedTransactionInputs = storedInputs), store) + } finally { + store.close() + } + } + context.deleteDatabase(name) + try { + launch { h, store -> + h.onPersistWalletMetadata(walletId, testnet, groupId, 0) + h.onPersistAccountRegistration( + walletId, 0, 0, 0, 0, 0, ByteArray(0), ByteArray(0), ByteArray(78) { 30 }, + ) + val account = store.accountDao().observeByWallet(walletId).first().single() + store.coreAddressDao().upsert( + CoreAddressEntity( + address = address, + poolTypeTag = 0, + addressIndex = 0, + derivationPath = "m/44'/1'/0'/0/0", + accountId = account.id, + ), + ) + firstLaunch(h, store) + } + repeat(2) { launch(relaunch) } + } finally { + context.deleteDatabase(name) + } + } + + @Test + fun repeatedRelaunchesRestoreNothingForAFlippedCoin() = runTest { + // The acceptance shape: after the run that flipped the phantom, a + // relaunch — a fresh handler over the same ON-DISK store — restores + // nothing for it, and so does the relaunch after that. + val txid = ByteArray(32) { 0xa1.toByte() } + acrossRelaunches( + "txo-reconcile-restart.db", + reconcileAddress, + firstLaunch = { h, _ -> + h.onWalletChangesetUtxoAdded( + walletId, txid, 0, 19_549, reconcileAddress, reconcileScript, 2_391_743, + false, true, false, false, + ) + assertEquals("the phantom the engine would be handed", 1, restoredUtxoCount(h)) + val report = h.reconcileWith(FakeEngine(inventoryJson(spent = listOf(txid to 0)))) + assertEquals(1, report.flipped) + assertEquals(0, restoredUtxoCount(h)) + }, + relaunch = { h, store -> + assertTrue(store.txoDao().getByOutpoint(makeOutpoint(txid, 0))!!.isSpent) + assertEquals("a relaunch without a rescan restores nothing", 0, restoredUtxoCount(h)) + }, + ) + } + + // ── 6. Run shape: paging, failures, cancellation ────────────────── + + @Test + fun theClassifyPassWalksEveryUnspentRowAcrossFlips() = runTest { + // Flipped rows leave the `isSpent = 0` predicate; the outpoint + // cursor sits past them, so the next page still starts at the first + // row this walk has not seen. + seedReconcileWallet() + val txids = (0 until 5).map { i -> ByteArray(32) { (0xf0 + i).toByte() } } + txids.forEach { seedUnspentTxo(it) } + val engine = FakeEngine(inventoryJson(spent = txids.map { it to 0 })) + + val report = handler.reconcileWith(engine, pageSize = 2) + + assertTrue(report.completed) + assertEquals(5, report.storeRows) + assertEquals("flipping rows out of the page does not skip the next ones", 5, report.flipped) + assertEquals(3, engine.batches) + assertEquals(0, restoredUtxoCount()) + } + + /** An engine of five confirmed coins on [changeTxid] from [firstVout]. */ + private fun fiveCoinEngine(firstVout: Int) = FakeEngine( + inventoryJson(rows = (0 until 5).map { engineRowJson(changeTxid, vout = firstVout + it, amount = 1_000L) }), + ) + + @Test + fun aRunStopsAtItsPageBudgetAndTheNextRunsResumeWhereItStopped() = runTest { + // Dust sent to a watched address grows the inventory with no action + // from the owner, so one run reads a bounded number of pages and the + // wallet's later runs carry on from there instead of starting over. + // 5 coins at 2 per page: 3 heal pages, then 3 classify pages. + saveReconcileAddress(attributeUnownedTxos(walletId)) + val engine = fiveCoinEngine(firstVout = 60) + + // Run 1: two heal pages, then the budget is spent mid-pass. + val first = handler.reconcileWith(engine, pageSize = 2, pageBudget = 2) + assertTrue(first.stoppedOnBudget) + assertFalse("a budget stop has not reached the end of the cycle", first.completed) + assertFalse(first.resumedFromCheckpoint) + assertEquals(0, first.transportFailures + first.storeFailures) + assertEquals(listOf(null, "2"), engine.cursors) + assertEquals(4, first.engineRows) + assertEquals("both finished pages landed", 4, first.inserted) + assertEquals("the classify pass waits for the heal pass", 0, engine.batches) + + // Run 2: the heal pass resumes at the saved engine cursor, reaches + // its end, and the classify pass starts at the first row. + val second = handler.reconcileWith(engine, pageSize = 2, pageBudget = 2) + assertTrue(second.resumedFromCheckpoint) + assertTrue(second.stoppedOnBudget) + assertFalse(second.completed) + assertEquals("only the page run 1 did not read", listOf(null, "2", "4"), engine.cursors) + assertEquals(1, second.engineRows) + assertEquals(1, second.inserted) + assertEquals(2, second.storeRows) + assertEquals(1, engine.batches) + assertEquals(listOf(60, 61), engine.queries.map { it["vout"]!!.jsonPrimitive.int }) + + // Run 3: the classify pass resumes after the saved outpoint and + // reaches its end. Every row is classified once across the runs. + val third = handler.reconcileWith(engine, pageSize = 2, pageBudget = 2) + assertTrue(third.resumedFromCheckpoint) + assertFalse(third.stoppedOnBudget) + assertTrue(third.completed) + assertEquals("the heal pass is not read again in this cycle", 3, engine.pages) + assertEquals(3, third.storeRows) + assertEquals(3, third.unspent) + assertEquals((60 until 65).toList(), engine.queries.map { it["vout"]!!.jsonPrimitive.int }) + + // Run 4: the finished cycle cleared its checkpoint, so this run + // starts a new one at the first heal page and walks everything. + val fourth = handler.reconcileWith(engine, pageSize = 2) + assertFalse(fourth.resumedFromCheckpoint) + assertTrue(fourth.completed) + assertEquals(null, engine.cursors[3]) + assertEquals(5, fourth.engineRows) + assertEquals(5, fourth.alreadyPresent) + assertEquals(5, fourth.storeRows) + assertEquals(0, fourth.mutations) + + // Run 5: the budget alone still bounds a run that starts over. + val fifth = handler.reconcileWith(engine, pageSize = 2, pageBudget = 1) + assertFalse(fifth.resumedFromCheckpoint) + assertTrue(fifth.stoppedOnBudget) + assertEquals("one page read", 7, engine.pages) + } + + @Test + fun aBudgetStopWithNothingFoundLogsAsInformationNotAWarning() = runTest { + saveReconcileAddress(attributeUnownedTxos(walletId)) + val engine = fiveCoinEngine(firstVout = 90) + assertEquals(5, handler.reconcileWith(engine, pageSize = 8).inserted) + org.robolectric.shadows.ShadowLog.clear() + + val report = handler.reconcileWith(engine, pageSize = 2, pageBudget = 1) + assertTrue(report.stoppedOnBudget) + assertEquals(0, report.mutations) + val summary = org.robolectric.shadows.ShadowLog.getLogs() + .single { it.msg.startsWith("txos reconcile: ") && "completed=" in it.msg } + assertEquals(android.util.Log.INFO, summary.type) + assertTrue(summary.msg, summary.msg.startsWith("txos reconcile: page budget used, resuming next run (")) + assertTrue(summary.msg, summary.msg.contains("stoppedOnBudget=true resumedFromCheckpoint=false")) + } + + @Test + fun aFailedHealPageDoesNotMoveTheCheckpointPastIt() = runTest { + // The next run must read the page that failed, not the one after it, + // and the classify pass must not start until the heal pass ends. + saveReconcileAddress(attributeUnownedTxos(walletId)) + val engine = fiveCoinEngine(firstVout = 70) + val failed = handler.reconcileWith( + engine, + engineUtxoPage = { cursor, limit -> if (cursor == null) engine.page(cursor, limit) else null }, + ) + assertFalse(failed.completed) + assertFalse("a failure is not a budget stop", failed.stoppedOnBudget) + assertEquals(1, failed.transportFailures) + assertEquals(2, failed.inserted) + assertEquals(0, engine.batches) + + val next = handler.reconcileWith(engine, pageSize = 2) + assertTrue(next.resumedFromCheckpoint) + assertTrue(next.completed) + assertEquals("resumed at the failed page, after the finished one", listOf(null, "2", "4"), engine.cursors) + assertEquals(3, next.inserted) + assertEquals("the classify pass covered every row once", 5, next.storeRows) + for (vout in 70 until 75) { + assertNotNull(db.txoDao().getByOutpoint(makeOutpoint(changeTxid, vout))) + } + } + + @Test + fun aResumedHealPageThatDoesNotDecodeIsReadAgainByTheNextRun() = runTest { + // The strict page decode throws; the page that threw is not + // finished, so the place stays at the cursor before it. A page that + // says more follow with no cursor is refused by the decode, so it + // can never leave a place that ends the walk early. + saveReconcileAddress(attributeUnownedTxos(walletId)) + val engine = fiveCoinEngine(firstVout = 100) + assertTrue(handler.reconcileWith(engine, pageSize = 2, pageBudget = 1).stoppedOnBudget) + + for ( + malformed in listOf( + """{"utxos":[],"cursor":null}""", + """{"utxos":[],"cursor":null,"hasMore":true}""", + ) + ) { + val asked = ArrayList() + val thrown = runCatching { + handler.reconcileWith( + engine, + engineUtxoPage = { cursor, _ -> + asked += cursor + malformed + }, + ) + }.exceptionOrNull() + assertTrue("$malformed must fail the decode, got $thrown", thrown is IllegalArgumentException) + assertEquals("the resumed run asked for the saved page", listOf("2"), asked) + } + + val next = handler.reconcileWith(engine, pageSize = 2) + assertTrue(next.resumedFromCheckpoint) + assertTrue(next.completed) + assertEquals(listOf(null, "2", "4"), engine.cursors) + assertEquals(3, next.inserted) + assertEquals("the classify pass started only once the heal pass ended", 5, next.storeRows) + } + + @Test + fun aFailedOrStaleClassifyPageDoesNotMoveTheCheckpointPastIt() = runTest { + seedReconcileWallet() + val txids = (0 until 6).map { i -> ByteArray(32) { (0xa0 + i).toByte() } } + txids.forEach { seedUnspentTxo(it) } + val engine = FakeEngine(inventoryJson()) + fun firstBytes() = engine.queries.map { it["txid"]!!.jsonPrimitive.content.substring(0, 2) } + + // The second classification batch fails. + var batch = 0 + val failed = handler.reconcileWith( + engine, + classifyOutpoints = { queries -> if (++batch == 2) null else engine.classify(queries) }, + ) + assertFalse(failed.completed) + assertEquals(1, failed.transportFailures) + assertEquals(listOf("a0", "a1"), firstBytes()) + + // The second page stays stale past the retry limit: a round + // commits between every read and its apply. + val stale = handler.reconcileWith( + engine, + classifyOutpoints = { queries -> + commitARound() + engine.classify(queries) + }, + ) + assertTrue(stale.resumedFromCheckpoint) + assertFalse(stale.completed) + assertFalse(stale.stoppedOnBudget) + assertEquals(PlatformWalletPersistenceHandler.TXO_RECONCILE_MAX_STALE_RETRIES + 1, stale.staleRetries) + assertEquals("the heal pass already ended in this cycle", 1, engine.pages) + + // A clean run picks up at the page that never finished. + val next = handler.reconcileWith(engine, pageSize = 2) + assertTrue(next.resumedFromCheckpoint) + assertTrue(next.completed) + assertEquals(4, next.storeRows) + val staleReads = List(PlatformWalletPersistenceHandler.TXO_RECONCILE_MAX_STALE_RETRIES + 1) { + listOf("a2", "a3") + }.flatten() + assertEquals( + "a0, a1 from the first run, the stale page's a2, a3 each try, then a2 to a5", + listOf("a0", "a1") + staleReads + listOf("a2", "a3", "a4", "a5"), + firstBytes(), + ) + } + + @Test + fun deletingTheWalletDropsItsReconcileCheckpoint() = runTest { + saveReconcileAddress(attributeUnownedTxos(walletId)) + val engine = fiveCoinEngine(firstVout = 80) + assertTrue(handler.reconcileWith(engine, pageSize = 2, pageBudget = 1).stoppedOnBudget) + + handler.deleteWalletData(walletId) + saveReconcileAddress(attributeUnownedTxos(walletId)) + + val report = handler.reconcileWith(engine, pageSize = 2) + assertFalse("a re-added wallet starts a new cycle", report.resumedFromCheckpoint) + assertTrue(report.completed) + assertEquals(listOf(null, null, "2", "4"), engine.cursors) + assertEquals(5, report.inserted) + } + + @Test + fun aCheckpointIsResumedOnlyUnderTheScopeThatWroteIt() = runTest { + // The manager scopes a place to its reconcile epoch, which moves on + // every wallet removal and on close: a wallet removed and added + // again (or any place left from before the move) starts a new cycle + // even if the handler never saw the deletion. + saveReconcileAddress(attributeUnownedTxos(walletId)) + val engine = fiveCoinEngine(firstVout = 110) + assertTrue(handler.reconcileWith(engine, pageSize = 2, pageBudget = 1, checkpointScope = 1L).stoppedOnBudget) + + val otherScope = handler.reconcileWith(engine, pageSize = 2, pageBudget = 1, checkpointScope = 2L) + assertFalse("another scope's place is not resumed", otherScope.resumedFromCheckpoint) + assertEquals(listOf(null, null), engine.cursors) + + // That run replaced the place with its own; the old scope no longer + // finds one, and the new scope resumes its own. + val sameScope = handler.reconcileWith(engine, pageSize = 2, pageBudget = 1, checkpointScope = 2L) + assertTrue(sameScope.resumedFromCheckpoint) + assertEquals(listOf(null, null, "2"), engine.cursors) + val oldScope = handler.reconcileWith(engine, pageSize = 2, pageBudget = 1, checkpointScope = 1L) + assertFalse(oldScope.resumedFromCheckpoint) + assertEquals(listOf(null, null, "2", null), engine.cursors) + } + + @Test + fun aCancelledRunLeavesNoCheckpoint() = runTest { + // A run that a wallet deletion cancelled must not record a place + // after the deletion dropped the wallet's: the save re-checks the + // cancellation under the callback lock. + saveReconcileAddress(attributeUnownedTxos(walletId)) + val engine = fiveCoinEngine(firstVout = 120) + var cancelled = false + val report = handler.reconcileWith( + engine, + isCancelled = { cancelled }, + engineUtxoPage = { cursor, limit -> engine.page(cursor, limit).also { cancelled = true } }, + ) + assertFalse(report.completed) + assertEquals("the cancelled step wrote nothing", 0, report.inserted) + + val next = handler.reconcileWith(engine, pageSize = 2) + assertFalse(next.resumedFromCheckpoint) + assertEquals(listOf(null, null, "2", "4"), engine.cursors) + } + + @Test + fun reconcileStopsWhenAClassificationBatchFails() = runTest { + // No verdicts, no classification: the classify pass stops rather + // than guessing at rows it could not ask the engine about. + seedReconcileWallet() + for (vout in 50 until 53) seedUnspentTxo(changeTxid, vout) + val report = handler.reconcileWith(pageSize = 1, classifyOutpoints = { null }) + + assertFalse(report.completed) + assertEquals(1, report.transportFailures) + assertEquals(0, report.unknown) + assertEquals(0, report.flipped) + } + + @Test + fun theRunStopsWhenTheHealCannotReadTheStore() = runTest { + // The heal asks the store whether it already holds each engine + // coin. A read that fails there must not read as "absent" (that + // would insert a row the store may well hold): the step fails, it + // rolls back, and the run stops. + val faults = useFaultedDatabase("SELECT * FROM txos WHERE outpoint") + seedReconcileWallet() + val txid = ByteArray(32) { 0x84.toByte() } + faults.armed = true + val report = handler.reconcileWith(FakeEngine(inventoryJson(rows = listOf(engineRowJson(txid))))) + faults.armed = false + + assertFalse(report.completed) + assertEquals(1, report.storeFailures) + assertEquals("a failed read is not a hit either", 0, report.alreadyPresent) + assertEquals(0, report.inserted) + assertNull(db.txoDao().getByOutpoint(makeOutpoint(txid, 0))) + } + + @Test + fun theRunStopsWhenTheFlipLookupCannotRead() = runTest { + // The classify pass re-reads each known-uncredited row before it + // flips it. A read that fails there must not read as "stale": the + // step fails, nothing staged is kept, and the row stays as it was. + val faults = useFaultedDatabase("SELECT * FROM txos WHERE outpoint") + seedReconcileWallet() + val txid = ByteArray(32) { 0x85.toByte() } + seedUnspentTxo(txid) + org.robolectric.shadows.ShadowLog.clear() + faults.armed = true + val report = handler.reconcileWith(FakeEngine(inventoryJson(spent = listOf(txid to 0)))) + faults.armed = false + + assertFalse(report.completed) + assertEquals(1, report.storeFailures) + assertEquals("the page itself was read", 1, report.storeRows) + assertEquals(0, report.flipped) + assertEquals("a failed read is not a stale page either", 0, report.staleRetries) + assertFalse(db.txoDao().getByOutpoint(makeOutpoint(txid, 0))!!.isSpent) + val failure = org.robolectric.shadows.ShadowLog.getLogs() + .map { it.msg } + .single { it.startsWith("txos reconcile: store step failed") } + assertTrue(failure, failure.contains("operation=flip")) + assertFalse(failure, failure.contains(txid.toHex())) + } + + @Test + fun aCancelledRunDoesNothingAndSaysSo() = runTest { + seedReconcileWallet() + val engine = FakeEngine(inventoryJson(rows = listOf(engineRowJson(ByteArray(32) { 0x11 })))) + + val report = handler.reconcileWith(engine, isCancelled = { true }) + + assertFalse(report.completed) + assertEquals(0, report.mutations) + assertEquals("cancellation is checked before the first engine read", 0, engine.pages) + assertEquals(0, walletTxos().size) + } + + @Test + fun aRunCancelledMidWayStopsAtThePageBoundaryAndKeepsWhatLanded() = runTest { + seedReconcileWallet() + val engine = FakeEngine( + inventoryJson(rows = (0 until 6).map { i -> engineRowJson(ByteArray(32) { (0x20 + i).toByte() }) }), + ) + var checks = 0 + + val report = handler.reconcileWith( + engine, + // The run checks once for the account read, once before each + // engine read and once more before each store step: the first + // three admit the account read, page one and its heal; the + // fourth (before page two) reports the epoch bumped. + isCancelled = { ++checks >= 4 }, + ) + + assertFalse(report.completed) + assertEquals(1, engine.pages) + assertEquals("the page that landed stays", 2, report.inserted) + assertEquals(2, walletTxos().size) + } + + @Test + fun aPageReadWhileTheManagerShutsDownNeverReachesTheStore() = runTest { + // Shutdown (or a wallet deletion) bumps the epoch while an engine + // read is parked; the page that read returns must not be written: + // cancellation is re-checked under the exclusion lock, after the + // read and before the write. + seedReconcileWallet() + val engine = FakeEngine( + inventoryJson(rows = listOf(engineRowJson(ByteArray(32) { 0x41 }), engineRowJson(ByteArray(32) { 0x42 }))), + ) + var cancelled = false + + val report = handler.reconcileWith( + engine, + pageSize = 1, + isCancelled = { cancelled }, + engineUtxoPage = { cursor, limit -> engine.page(cursor, limit).also { cancelled = true } }, + ) + + assertFalse(report.completed) + assertEquals("no second engine read may start", 1, engine.pages) + assertEquals("the old page must not reach the store", 0, report.mutations) + assertEquals(0, walletTxos().size) + } + + // ── 7. Logs carry counts and digests only ───────────────────────── + + @Test + fun theReconcileLogsCarryNoWalletHistory() = runTest { + // Every line the reconcile writes, rendered with fixture values + // shaped like the real thing: no address, no txid or outpoint in + // either byte orientation, no script, no wallet id — and, as a + // backstop against any future field, no long hex or address-length + // Base58 run at all. References are 12 hex characters, keyed: the + // plain SHA-256 prefix of a public outpoint (what anyone holding the + // logs could compute and match) never appears. + val address = "yTestPrivacyFixtureAddress12345678" + val healedTxid = ByteArray(32) { (0xa0 + it % 16).toByte() } + val flippedTxid = ByteArray(32) { (0x30 + it % 16).toByte() } + val unknownTxid = ByteArray(32) { (0x50 + it % 16).toByte() } + seedWalletWithAddress(walletId, address) + seedUnspentTxo(flippedTxid, vout = 1, address = address) + seedUnspentTxo(unknownTxid, vout = 2, address = address) + val engine = FakeEngine( + inventoryJson( + rows = listOf( + engineRowJson(healedTxid, amount = 100_001L, address = address, height = 1_434_986), + // Refused (immature), so the skip path renders too. + engineRowJson(ByteArray(32) { 0x66 }, address = address, height = reconcileTip), + ), + spent = listOf(flippedTxid to 1), + ), + ) + org.robolectric.shadows.ShadowLog.clear() + + val report = handler.reconcileWith(engine) + // A second, failing run renders the incomplete summary as well. + handler.reconcileTxos( + walletId = walletId, + tipHeight = reconcileTip, + engineUtxoPage = engine::page, + classifyOutpoints = { null }, + ) + + assertEquals(1, report.inserted) + assertEquals(1, report.flipped) + assertEquals(1, report.unknown) + // The report still carries the totals for the host; only the log + // must not. + assertEquals(100_001L, report.insertedDuffs) + assertEquals(19_549L, report.flippedDuffs) + val lines = org.robolectric.shadows.ShadowLog.getLogs().map { it.msg } + .filter { it.contains("txos reconcile") } + assertTrue(lines.any { it.startsWith("txos reconcile: item action=healed ") }) + assertTrue(lines.any { it.startsWith("txos reconcile: item action=flipped_spent ") }) + assertTrue(lines.any { it.startsWith("txos reconcile: inserted=") || it.startsWith("txos reconcile: engineRows=") }) + assertTrue(lines.any { it.contains("transportFailures=1") && it.contains("completed=false") }) + val walletRef = logReference(walletId) + assertTrue("every summary names the wallet by reference", lines.filter { "completed=" in it }.all { "wallet=$walletRef" in it }) + val flippedRef = logReference(makeOutpoint(flippedTxid, 1)) + assertTrue( + "the flipped coin's line carries its in-process reference", + lines.any { it == "txos reconcile: item action=flipped_spent outpoint=$flippedRef wallet=$walletRef" }, + ) + + val forbidden = listOf( + "address" to address, + "script" to reconcileScript.toHex(), + "healed txid" to healedTxid.toHex(), + "healed txid reversed" to healedTxid.reversedArray().toHex(), + "flipped txid" to flippedTxid.toHex(), + "flipped txid reversed" to flippedTxid.reversedArray().toHex(), + "unknown txid" to unknownTxid.toHex(), + "flipped outpoint" to makeOutpoint(flippedTxid, 1).toHex(), + "wallet id" to walletId.toHex(), + "healed outpoint's unkeyed digest" to unkeyedReference(makeOutpoint(healedTxid, 0)), + "flipped outpoint's unkeyed digest" to unkeyedReference(makeOutpoint(flippedTxid, 1)), + "wallet id's unkeyed digest" to unkeyedReference(walletId), + // As a logged value: a bare digit run could occur in a digest. + "healed amount" to "=100001", + "flipped amount" to "=19549", + "amount total" to "Duffs", + ) + val longHex = Regex("[0-9a-fA-F]{32,}") + val base58Run = Regex("[1-9A-HJ-NP-Za-km-z]{26,}") + for (line in lines) { + for ((label, value) in forbidden) { + assertFalse("the log must not carry the $label: $line", line.contains(value, ignoreCase = true)) + } + assertNull("no 32+ character hex run: $line", longHex.find(line)) + assertNull("no address-length Base58 run: $line", base58Run.find(line)) + } + } + + /** The reference an UNKEYED digest would give [value] — what anyone + * holding the logs could compute for a public outpoint. */ + private fun unkeyedReference(value: ByteArray): String = + java.security.MessageDigest.getInstance("SHA-256").digest(value).copyOf(6).toHex() + + @Test + fun logReferencesAreKeyedPerProcessAndStableWithinIt() { + // Stable within the process, so one run's lines still correlate; + // never the plain SHA-256 prefix, so a public outpoint cannot be + // hashed and matched against the logs. + val outpoint = makeOutpoint(ByteArray(32) { (0x40 + it).toByte() }, 3) + val ref = logReference(outpoint) + assertTrue("12 lower-hex characters: $ref", Regex("[0-9a-f]{12}").matches(ref)) + assertEquals(ref, logReference(outpoint.copyOf())) + assertFalse("distinct values get distinct references", ref == logReference(makeOutpoint(ByteArray(32), 3))) + for (value in listOf(outpoint, walletId, ByteArray(0))) { + assertFalse( + "a reference must not be the unkeyed digest", + logReference(value) == unkeyedReference(value), + ) + } + } + + @Test + fun reconcileRejectsAMalformedInventoryRowInsteadOfHealingIt() = runTest { + // The transport is a typed contract on both ends. A row missing its + // amount (or carrying a key this reader does not know) must fail the + // decode loudly — the alternative is a defaulted row written into + // the mirror the engine reloads from. + val malformed = """{"utxos":[{"typeTag":0,"txid":"${changeTxid.toHex()}",""" + + """"vout":40,"address":"yStxXHHzhAx58JhaPBNhn3xsH93UwBM2nd","height":1400000}],""" + + """"cursor":null,"hasMore":false}""" + val thrown = runCatching { handler.reconcileWith(engineUtxoPage = { _, _ -> malformed }) }.exceptionOrNull() + assertTrue( + "strict decode must throw, got $thrown", + thrown is kotlinx.serialization.SerializationException, + ) + assertNull( + "nothing was healed from the malformed page", + db.txoDao().getByOutpoint(makeOutpoint(changeTxid, 40)), + ) + } + + @Test + fun anInventoryRowMissingAnyRequiredFieldFailsTheDecode() { + // The account tuple is the routing context a healed row is filed + // under. A row missing `index` must not decode as account 0 and be + // healed under the wrong owner; the address, script and flags are + // written into a healed row as given, so a missing one must not + // decode as empty or false. Only the DashPay identity halves may be + // absent. + val complete = kotlinx.serialization.json.Json.parseToJsonElement( + engineRowJson(changeTxid, vout = 41), + ).jsonObject + fun page(row: Map) = + """{"utxos":[${kotlinx.serialization.json.JsonObject(row)}],"cursor":null,"hasMore":false}""" + assertEquals( + 1, + PlatformWalletPersistenceHandler.decodeEngineUtxoPage(page(complete)).utxos.size, + ) + + for ( + key in listOf( + "typeTag", "standardTag", "index", "registrationIndex", "keyClass", "txid", "vout", "amount", + "address", "scriptHex", "height", "isConfirmed", "isInstantlocked", "isCoinbase", "isLocked", + ) + ) { + val thrown = runCatching { + PlatformWalletPersistenceHandler.decodeEngineUtxoPage(page(complete - key)) + }.exceptionOrNull() + assertTrue( + "a row without `$key` must fail the decode, got $thrown", + thrown is kotlinx.serialization.SerializationException, + ) + } + } + + @Test + fun anInventoryPageWithoutItsContinuationFailsTheDecode() { + // A page that lost a key, or says more follow with nowhere to + // resume from, must not read as the last page: the heal walk would + // end early and leave later engine coins unrepaired. + val row = engineRowJson(changeTxid, vout = 42) + fun decodes(page: String) = + runCatching { PlatformWalletPersistenceHandler.decodeEngineUtxoPage(page) } + assertTrue(decodes("""{"utxos":[$row],"cursor":null,"hasMore":false}""").isSuccess) + assertTrue(decodes("""{"utxos":[$row],"cursor":"c","hasMore":true}""").isSuccess) + for ( + (label, page) in listOf( + "no utxos" to """{"cursor":null,"hasMore":false}""", + "no cursor" to """{"utxos":[$row],"hasMore":false}""", + "no hasMore" to """{"utxos":[$row],"cursor":null}""", + "more to come with no cursor" to """{"utxos":[$row],"cursor":null,"hasMore":true}""", + ) + ) { + val thrown = decodes(page).exceptionOrNull() + assertTrue("$label must fail the decode, got $thrown", thrown is IllegalArgumentException) + } + } + + // ── Cross-language golden fixture (shared with rs-unified-sdk-jni) ── + + /** + * The checked-in TXO-reconcile transport payloads. The Rust + * `wallet_manager::tests` read the SAME file (`include_str!` of this + * resource): they serialize the real `UtxoPage` for four accessor + * entries and require `inventoryPage` exactly, and they deserialize + * `outpointQueries` through the real `OutpointQuery` conversion. Here + * the page is decoded by the production decoder and the queries are + * built by the production encoder, so neither codec can drift from the + * other without one of the two suites failing. + */ + private fun reconcileGolden(): kotlinx.serialization.json.JsonObject = + javaClass.getResourceAsStream("/golden/txo_reconcile_transport_v1.json") + .use { requireNotNull(it) { "golden fixture resource missing" }.readBytes() } + .let { kotlinx.serialization.json.Json.parseToJsonElement(it.decodeToString()).jsonObject } + + /** The four golden coins: txid bytes counting up from [first]. */ + private fun goldenTxid(first: Int) = ByteArray(32) { (first + it).toByte() } + private fun goldenScript(fill: Int) = + byteArrayOf(0x76, 0xa9.toByte(), 0x14) + ByteArray(20) { fill.toByte() } + + byteArrayOf(0x88.toByte(), 0xac.toByte()) + private val goldenUserIdentity = ByteArray(32) { 0xa1.toByte() } + private val goldenFriendIdentity = ByteArray(32) { 0xb2.toByte() } + + @Test + fun theInventoryDecoderReadsTheGoldenRustPage() { + val page = PlatformWalletPersistenceHandler.decodeEngineUtxoPage( + reconcileGolden()["inventoryPage"]!!.toString(), + ) + + assertTrue(page.hasMore) + assertNotNull("a page with more to come carries its cursor", page.cursor) + assertEquals( + listOf( + // BIP44 account 0: every account field zero, and present. + PlatformWalletPersistenceHandler.EngineUtxoRow( + 0, 0, 0, 0, 0, null, null, goldenTxid(0x00).toHex(), 0, 19_549L, + reconcileAddress, goldenScript(0x5b).toHex(), 1_534_921, + true, false, false, false, + ), + // Non-default account fields, every flag set, and an amount + // past 32 bits. + PlatformWalletPersistenceHandler.EngineUtxoRow( + 0, 1, 2, 0, 0, null, null, goldenTxid(0x20).toHex(), 7, 5_000_000_000L, + "yGoldenFixtureBip32Addr", goldenScript(0xc3).toHex(), 1_400_000, + true, true, true, true, + ), + // A registration index, and a script with no address form. + PlatformWalletPersistenceHandler.EngineUtxoRow( + 3, 0, 0, 4, 0, null, null, goldenTxid(0x40).toHex(), 1, 100_000L, + "", "", 0, + false, false, false, false, + ), + // DashPay: both identity halves, as lower hex. + PlatformWalletPersistenceHandler.EngineUtxoRow( + 12, 0, 1, 0, 0, goldenUserIdentity.toHex(), goldenFriendIdentity.toHex(), + goldenTxid(0x60).toHex(), 3, 250_000L, + "yGoldenFixtureDashPayAddr", goldenScript(0xd4).toHex(), 1_534_000, + true, true, false, false, + ), + ), + page.utxos, + ) + } + + @Test + fun classificationQueriesMatchTheGoldenRustInput() { + // The same four coins, filed under the store accounts they resolve + // to, through the encoder [readUnspentPage] uses. A zeroed identity + // half on a non-DashPay account goes out absent, as the native side + // emits it. Coin 0 is the common BIP44 account-0 case, every tuple + // field 0 and still present: kotlinx's default `Json` drops a + // property equal to its default, and the native side rejected every + // such batch ("missing field `standardTag`", int26/int27 devices). + fun account(type: Int, index: Int, name: String) = org.dashfoundation.dashsdk.persistence.entities + .AccountEntity(walletId = walletId, accountType = type, accountIndex = index, accountTypeName = name) + val queries = listOf( + PlatformWalletPersistenceHandler.outpointQueryFor( + account(0, 0, "Standard").copy(userIdentityId = ByteArray(32)), + goldenTxid(0x00), 0, goldenScript(0x5b), + ), + PlatformWalletPersistenceHandler.outpointQueryFor( + account(0, 2, "Standard").copy(standardTag = 1), + goldenTxid(0x20), 7, goldenScript(0xc3), + ), + PlatformWalletPersistenceHandler.outpointQueryFor( + account(3, 0, "IdentityTopUp").copy(registrationIndex = 4), + goldenTxid(0x40), 1, ByteArray(0), + ), + PlatformWalletPersistenceHandler.outpointQueryFor( + account(12, 1, "DashpayReceivingFunds").copy( + userIdentityId = goldenUserIdentity, + friendIdentityId = goldenFriendIdentity, + ), + goldenTxid(0x60), 3, goldenScript(0xd4), + ), + ) + + val json = PlatformWalletPersistenceHandler.encodeOutpointQueries(queries) + + assertEquals( + "Kotlin's classification batch drifted from the golden fixture the Rust deserializer is tested against", + reconcileGolden()["outpointQueries"], + kotlinx.serialization.json.Json.parseToJsonElement(json), + ) + } + + /** + * Drive the paged reconcile from one whole-inventory JSON blob — the + * shape these tests describe an engine in — served through [FakeEngine] + * in pages of 2, so a test describing more than a couple of rows walks + * the real cursor loop rather than a single page. Every + * still-unattributed TXO is filed under a default BIP44 account first + * (see [attributeUnownedTxos]), which is also the account the + * inventory's `typeTag 0 / index 0` rows resolve to. + */ + private suspend fun PlatformWalletPersistenceHandler.reconcileInventory( + inventoryJson: String, + ): PlatformWalletPersistenceHandler.TxoReconcileReport { + attributeUnownedTxos(walletId) + return reconcileWith(FakeEngine(inventoryJson)) + } + + /** + * File every still-unattributed TXO of [walletId] under a default + * BIP44 account, seeding that account on first use. + * + * The classify pass asks the engine about a row by naming the account + * the STORE filed it under, so a row with no resolvable owner carries no + * claim and is counted instead of asked about; the heal pass refuses a + * coin whose account tuple names no store account. Production rows + * always resolve one, through the explicit link or the `core_addresses` + * projection; these tests write TXOs straight through the changeset + * callbacks without the account rows a real wallet load creates, so the + * link is seeded here rather than in each test. Rows a test attributed + * itself (the watch-only contact case) keep their own account. + */ + private suspend fun attributeUnownedTxos(walletId: ByteArray): Long { + db.walletDao().upsert(WalletEntity(walletId, networkRaw = Network.TESTNET.ffiValue)) + val existing = db.accountDao().getByKey(walletId, accountType = 0, accountIndex = 0) + val ownerId = existing.firstOrNull()?.id + ?: db.accountDao().insert( + org.dashfoundation.dashsdk.persistence.entities.AccountEntity( + walletId = walletId, + accountType = 0, + accountIndex = 0, + accountTypeName = "Standard", + ), + ) + for (row in walletTxos(walletId)) { + if (row.accountId == null && row.coreAddressId == null) { + db.txoDao().upsert(row.copy(accountId = ownerId)) + } + } + return ownerId + } + + /** + * A stand-in for the engine's paged inventory transport, built from the + * whole-inventory JSON a test writes out. Pages come back behind an + * opaque ordinal cursor — the real cursor is opaque too, the handler + * only ever hands back what it was given — and classification answers + * positionally out of the same inventories: 1 unspent, 2 + * known-uncredited (the blob's `spent` list: the engine holds a mined + * record spending the coin), 3 not-owned (its optional `notOwned` + * list), 0 neither. + */ + private class FakeEngine(inventoryJson: String) { + private val utxos: List + private val unspentKeys: Set + private val spentKeys: Set + private val notOwnedKeys: Set + + /** Inventory pages served, classification batches answered, and + * outpoints classified across those batches. */ + var pages = 0 + private set + var batches = 0 + private set + var classified = 0 + private set + + /** Every classification query received, in order. */ + val queries = ArrayList() + + /** The cursor of every inventory page asked for, in order. */ + val cursors = ArrayList() + + init { + val root = kotlinx.serialization.json.Json + .parseToJsonElement(inventoryJson).jsonObject + fun keys(list: String) = root[list]?.jsonArray.orEmpty().map { it.jsonObject } + .map { key(it["txid"]!!.jsonPrimitive.content, it["vout"]!!.jsonPrimitive.int) }.toSet() + utxos = root["utxos"]?.jsonArray?.map { it.jsonObject } ?: emptyList() + unspentKeys = keys("utxos") + spentKeys = keys("spent") + notOwnedKeys = keys("notOwned") + } + + fun page(cursor: String?, limit: Int): String { + pages++ + cursors += cursor + val start = cursor?.toInt() ?: 0 + val slice = utxos.drop(start).take(limit) + val next = start + slice.size + val hasMore = next < utxos.size + return """{"utxos":[${slice.joinToString(",") { it.toString() }}],""" + + """"cursor":${if (hasMore) "\"$next\"" else "null"},"hasMore":$hasMore}""" + } + + fun classify(queriesJson: String): ByteArray { + batches++ + val batch = kotlinx.serialization.json.Json + .parseToJsonElement(queriesJson).jsonArray + classified += batch.size + val verdicts = ByteArray(batch.size) + for ((i, query) in batch.withIndex()) { + val q = query.jsonObject + queries += q + // Every query must name the account the store filed the + // coin under and the script it recorded — that claim is + // the whole reason the engine can answer NOT_OWNED. The + // real deserializer rejects the WHOLE batch on a missing + // key, so this fake does too rather than answering it. + for (k in RUST_REQUIRED_QUERY_KEYS) { + requireNotNull(q[k]) { "missing field `$k` in classification query $q" } + } + requireNotNull(q["scriptHex"]) { "classification query carries no script" } + val k = key(q["txid"]!!.jsonPrimitive.content, q["vout"]!!.jsonPrimitive.int) + verdicts[i] = when { + k in notOwnedKeys -> 3 + k in unspentKeys -> 1 + k in spentKeys -> 2 + else -> 0 + } + } + return verdicts + } + + private fun key(txidHex: String, vout: Int) = "$txidHex:$vout" + + companion object { + /** The rs-unified-sdk-jni `OutpointQuery` keys with no serde + * default (its flattened `UtxoAccountTuple` plus the outpoint): + * omitting any one fails the whole classification batch with + * `malformed outpoint queries: missing field …`. */ + val RUST_REQUIRED_QUERY_KEYS = listOf( + "typeTag", "standardTag", "index", "registrationIndex", "keyClass", "txid", "vout", + ) + } + } + + // ── Credit verdicts at the changeset seam (dashpay/platform#4638 Part 1) ── + // + // Port of the Swift `BornSpentTxoPersistTests`. The shape the verdicts + // exist for (rust-dashcore#992, dashpay/platform#4575): a coin is spent + // by a transaction with no wallet-owned output (a CoinJoin collateral + // burn — sole `OP_RETURN` output) that the engine processed while the + // coin was not yet in its UTXO set, so the spender matched nothing and + // was discarded. When the funding record is (re)emitted it still + // classifies the output `Received`, a `utxos_added` entry is derived + // from that role, and — without the verdict — the row is written + // UNSPENT for a coin the engine never held. The restore path then hands + // it back to the engine on every launch. With the verdict the row is + // written spent at creation and the restore emits nothing for it. + + private val bornSpentFundingTxid = ByteArray(32) { 0x61 } + private val bornSpentOtherTxid = ByteArray(32) { 0x62 } + private val bornSpentAddress = "yBornSpentFixtureAddr" + + private class CreditVerdict(val txid: ByteArray, val vout: Int, val code: Byte, val height: Int = 0) + + /** The verdict slot alone, inside whatever bracket the caller opened. */ + private fun stageVerdicts( + h: PlatformWalletPersistenceHandler, + vararg verdicts: CreditVerdict, + wallet: ByteArray = walletId, + ): Int { + val outpoints = ByteArray(verdicts.size * 36) + verdicts.forEachIndexed { i, v -> + System.arraycopy(makeOutpoint(v.txid, v.vout), 0, outpoints, i * 36, 36) + } + return h.onWalletChangesetUtxoVerdicts( + wallet, + outpoints, + ByteArray(verdicts.size) { verdicts[it].code }, + IntArray(verdicts.size) { verdicts[it].height }, + verdicts.size, + ) + } + + /** One `utxos_added` entry for `txid:vout`, inside whatever bracket the caller opened. */ + private fun stageBornSpentUtxo(h: PlatformWalletPersistenceHandler, txid: ByteArray, vout: Int = 0): Int = + h.onWalletChangesetUtxoAdded( + walletId, txid, vout, 19_549, bornSpentAddress, ByteArray(25) { 6 }, + 2_391_743, false, true, false, false, + ) + + private fun verdictRound( + h: PlatformWalletPersistenceHandler = handler, + success: Boolean = true, + body: () -> Unit, + ) { + h.onChangesetBegin(walletId) + body() + h.onChangesetEnd(walletId, success) + } + + private val observedSpent = PlatformWalletPersistenceHandler.UTXO_CREDIT_VERDICT_OBSERVED_SPENT + private val doomed = PlatformWalletPersistenceHandler.UTXO_CREDIT_VERDICT_DOOMED + private val uncredited = PlatformWalletPersistenceHandler.UTXO_CREDIT_VERDICT_UNCREDITED + + @Test + fun aVerdictVetoesTheRedeliveryClearAndItsAbsenceDoesNot() = runTest { + // Without a verdict, a redelivery of a spent, unlinked row clears + // the flag (the wallet is handing the coin back as unspent — the + // recovery rule). With ANY verdict the clear is vetoed: the record + // merely still names the output as ours, the engine does not hold it. + seedWalletWithAddress(walletId, bornSpentAddress) + val outpoint = makeOutpoint(bornSpentFundingTxid, 0) + verdictRound { + stageVerdicts(handler, CreditVerdict(bornSpentFundingTxid, 0, observedSpent, 2_402_896)) + stageBornSpentUtxo(handler, bornSpentFundingTxid) + } + assertTrue(db.txoDao().getByOutpoint(outpoint)!!.isSpent) + + // Redelivered with a context-free verdict: stays spent. + verdictRound { + stageVerdicts(handler, CreditVerdict(bornSpentFundingTxid, 0, uncredited)) + stageBornSpentUtxo(handler, bornSpentFundingTxid) + } + assertTrue(db.txoDao().getByOutpoint(outpoint)!!.isSpent) + assertTrue(handler.onLoadWalletList().single().utxos.isEmpty()) + + // Redelivered with no verdict at all: the engine credited it again + // (a reorg of the spender), and the row follows the wallet. + verdictRound { stageBornSpentUtxo(handler, bornSpentFundingTxid) } + assertFalse(db.txoDao().getByOutpoint(outpoint)!!.isSpent) + assertEquals(1, handler.onLoadWalletList().single().utxos.size) + } + + @Test + fun aVerdictKeepsASweepStampOnARedeliveredRow() = runTest { + // The veto covers the whole clear, the stamp included: a sweep-held + // row redelivered with a verdict keeps both its flag and its stamp, + // exactly as Swift leaves `supersededByTxid` alone. + seedWalletWithAddress(walletId, bornSpentAddress) + val outpoint = makeOutpoint(bornSpentFundingTxid, 0) + val winner = ByteArray(32) { 0x63 } + verdictRound { stageBornSpentUtxo(handler, bornSpentFundingTxid) } + db.txoDao().upsert( + db.txoDao().getByOutpoint(outpoint)!!.copy(isSpent = true, supersededByTxid = winner), + ) + verdictRound { + stageVerdicts(handler, CreditVerdict(bornSpentFundingTxid, 0, uncredited)) + stageBornSpentUtxo(handler, bornSpentFundingTxid) + } + val row = db.txoDao().getByOutpoint(outpoint)!! + assertTrue(row.isSpent) + assertTrue(winner.contentEquals(row.supersededByTxid)) + } + + @Test + fun verdictsDoNotOutliveTheirRound() = runTest { + // A verdict for an outpoint the round never delivers is dropped with + // the round, and a later round delivering that outpoint without a + // verdict writes it unspent as ever. + seedWalletWithAddress(walletId, bornSpentAddress) + verdictRound { + stageVerdicts(handler, CreditVerdict(bornSpentOtherTxid, 0, observedSpent, 2_402_896)) + stageBornSpentUtxo(handler, bornSpentFundingTxid) + } + assertFalse(db.txoDao().getByOutpoint(makeOutpoint(bornSpentFundingTxid, 0))!!.isSpent) + assertNull(db.txoDao().getByOutpoint(makeOutpoint(bornSpentOtherTxid, 0))) + + verdictRound { stageBornSpentUtxo(handler, bornSpentOtherTxid) } + assertFalse(db.txoDao().getByOutpoint(makeOutpoint(bornSpentOtherTxid, 0))!!.isSpent) + } + + @Test + fun aVerdictForAnUnknownWalletOrOutsideARoundIsAcceptedAndIgnored() = runTest { + // The same contract as every other per-kind callback for an unknown + // wallet: accepted, nothing written. Rust only fires the slot inside + // a round; outside one there is nothing to scope it to. + val stranger = ByteArray(32) { 0x7e } + assertEquals( + 0, + stageVerdicts(handler, CreditVerdict(bornSpentFundingTxid, 0, observedSpent), wallet = stranger), + ) + handler.onChangesetBegin(stranger) + assertEquals( + 0, + stageVerdicts(handler, CreditVerdict(bornSpentFundingTxid, 0, observedSpent), wallet = stranger), + ) + handler.onChangesetEnd(stranger, success = true) + assertNull(db.txoDao().getByOutpoint(makeOutpoint(bornSpentFundingTxid, 0))) + } + + @Test + fun aMalformedVerdictBatchFailsTheCallback() { + // A verdict silently dropped is a phantom coin the store hands back + // to the engine at the next load, so a batch whose arrays do not + // match its count fails the round instead of being half-read. + handler.onChangesetBegin(walletId) + assertEquals( + 1, + handler.onWalletChangesetUtxoVerdicts(walletId, ByteArray(36), ByteArray(2), IntArray(2), 2), + ) + handler.onChangesetEnd(walletId, success = false) + } + + @Test + fun aRolledBackRoundLeavesNoRow() = runTest { + // A rolled-back round leaves neither the row nor the verdict behind. + seedWalletWithAddress(walletId, bornSpentAddress) + verdictRound(success = false) { + stageVerdicts(handler, CreditVerdict(bornSpentFundingTxid, 0, observedSpent, 2_402_896)) + stageBornSpentUtxo(handler, bornSpentFundingTxid) + } + assertNull(db.txoDao().getByOutpoint(makeOutpoint(bornSpentFundingTxid, 0))) + assertTrue(handler.onLoadWalletList().single().utxos.isEmpty()) + + verdictRound { stageBornSpentUtxo(handler, bornSpentFundingTxid) } + assertFalse( + "the rolled-back verdict did not survive into a later round", + db.txoDao().getByOutpoint(makeOutpoint(bornSpentFundingTxid, 0))!!.isSpent, + ) + } + + @Test + fun eachVerdictDecidesItsNewRowAndTheRoundLogsOneLineOfCounts() = runTest { + // An observed-spent (vout 0) or doomed (vout 1) verdict writes the + // new row spent at creation, with no spender invented, and keeps it + // out of the restore. A context-free uncredited verdict (vout 2) + // never marks the row spent: the engine only said "not credited", + // and the spender's own record or the sweep callback settles it. + // What it does change is the redelivery clear. + seedWalletWithAddress(walletId, bornSpentAddress) + org.robolectric.shadows.ShadowLog.clear() + verdictRound { + assertEquals( + 0, + stageVerdicts( + handler, + CreditVerdict(bornSpentFundingTxid, 0, observedSpent, 2_402_896), + CreditVerdict(bornSpentFundingTxid, 1, doomed), + CreditVerdict(bornSpentFundingTxid, 2, uncredited), + ), + ) + for (vout in 0..2) assertEquals(0, stageBornSpentUtxo(handler, bornSpentFundingTxid, vout)) + } + for (vout in 0..1) { + val coin = db.txoDao().getByOutpoint(makeOutpoint(bornSpentFundingTxid, vout))!! + assertTrue(coin.isSpent) + assertNull("the spender was never recorded", coin.spendingTxid) + assertNull(coin.spendingInputIndex) + assertNull(coin.supersededByTxid) + assertEquals(19_549L, coin.amount) + } + assertFalse(storedIsSpent(bornSpentFundingTxid, 2)) + assertEquals( + "only the uncredited coin is restored", + listOf(2), + handler.onLoadWalletList().single().utxos.map { it.vout }, + ) + val lines = org.robolectric.shadows.ShadowLog.getLogs() + .map { it.msg } + .filter { it.startsWith("txo credit verdicts:") } + assertEquals( + listOf("txo credit verdicts: observedSpent=1 doomed=1 alreadySpent=0 uncredited=1 roundSuccess=true"), + lines, + ) + val txidHex = bornSpentFundingTxid.toHex() + assertTrue( + "no txid, outpoint or address in any persistence log line", + org.robolectric.shadows.ShadowLog.getLogs().none { + it.msg.contains(txidHex) || it.msg.contains(bornSpentAddress) + }, + ) + + // A redelivery of an already-spent row counts as such, not as a new spend. + org.robolectric.shadows.ShadowLog.clear() + verdictRound { + stageVerdicts(handler, CreditVerdict(bornSpentFundingTxid, 0, observedSpent, 2_402_896)) + stageBornSpentUtxo(handler, bornSpentFundingTxid, 0) + } + assertEquals( + listOf("txo credit verdicts: observedSpent=0 doomed=0 alreadySpent=1 uncredited=0 roundSuccess=true"), + org.robolectric.shadows.ShadowLog.getLogs().map { it.msg }.filter { it.startsWith("txo credit verdicts:") }, + ) + + // A round with no verdict logs nothing. + org.robolectric.shadows.ShadowLog.clear() + verdictRound { stageBornSpentUtxo(handler, bornSpentOtherTxid) } + assertTrue( + org.robolectric.shadows.ShadowLog.getLogs().none { it.msg.startsWith("txo credit verdicts:") }, + ) + } + + @Test + fun aRestartRestoresNothingForABornSpentRow() = runTest { + // The acceptance shape: after the round that wrote the row spent, a + // relaunch — a fresh handler over the same ON-DISK store, reopened — + // restores zero coins for the wallet, and so does the relaunch after + // that. The phantom never comes back. + acrossRelaunches( + "born-spent-restart.db", + bornSpentAddress, + firstLaunch = { h, store -> + verdictRound(h) { + stageVerdicts(h, CreditVerdict(bornSpentFundingTxid, 0, observedSpent, 2_402_896)) + stageBornSpentUtxo(h, bornSpentFundingTxid) + } + assertTrue(store.txoDao().getByOutpoint(makeOutpoint(bornSpentFundingTxid, 0))!!.isSpent) + assertTrue(h.onLoadWalletList().single().utxos.isEmpty()) + }, + relaunch = { h, store -> + val restore = h.onLoadWalletList().single() + assertTrue(store.txoDao().getByOutpoint(makeOutpoint(bornSpentFundingTxid, 0))!!.isSpent) + assertTrue("a relaunch restores nothing for the burned coin", restore.utxos.isEmpty()) + }, + ) + } } /** diff --git a/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcileTest.kt b/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcileTest.kt new file mode 100644 index 00000000000..13b872c9dbd --- /dev/null +++ b/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcileTest.kt @@ -0,0 +1,285 @@ +package org.dashfoundation.dashsdk.wallet + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * The manager-side gates and trigger of the TXO-store reconcile — port of + * the steady-state and scheduling cases of Swift's `CoreTxoReconcileTests` + * / `CoreTxoReconcileShutdownTests` (dashpay/platform#4638). Pure: no + * native manager is needed. + */ +class TxoReconcileTest { + + private fun progress( + state: SpvSyncState, + filters: Pair?, + headers: Long = 2_535_898, + ) = SpvSyncProgressData( + overallState = state, + overallPercentage = 0.0, + headers = SpvSubProgress(state, headers, headers, 1.0), + filterHeaders = null, + filters = filters?.let { (current, target) -> SpvSubProgress(state, current, target, 0.0) }, + masternodes = null, + ) + + /** [TxoReconcileCoordinator.walletsDue] for a schedule whose every + * launched run ends before the next tick — the cases that are about + * the edge and the cadence, not about a launch still pending. */ + private fun TxoReconcileCoordinator.dueWithLaunchesEnded( + progress: SpvSyncProgressData, + walletKeys: Collection, + nowMs: Long, + ): List = walletsDue(progress, walletKeys, nowMs).onEach { launchEnded(it) } + + private val steady = progress(SpvSyncState.WAIT_FOR_EVENTS, filters = 2_535_898L to 2_535_898L) + private val syncing = progress(SpvSyncState.SYNCING, filters = 199_999L to 2_535_898L) + + @Test + fun theSteadyStateGateRefusesAnUnfinishedScan() { + assertFalse(TxoReconcileGates.isSteadySyncState(syncing)) + assertFalse(TxoReconcileGates.isSteadySyncState(progress(SpvSyncState.WAITING_FOR_CONNECTIONS, filters = null))) + assertFalse( + TxoReconcileGates.isSteadySyncState(progress(SpvSyncState.ERROR, filters = 2_535_898L to 2_535_898L)), + ) + assertFalse( + "waiting for events with the filter phase behind its target is a scan still running", + TxoReconcileGates.isSteadySyncState(progress(SpvSyncState.WAIT_FOR_EVENTS, filters = 2_535_800L to 2_535_898L)), + ) + assertFalse( + "no filter phase at all proves nothing", + TxoReconcileGates.isSteadySyncState(progress(SpvSyncState.WAIT_FOR_EVENTS, filters = null)), + ) + assertFalse( + "a filter phase with no target proves nothing", + TxoReconcileGates.isSteadySyncState(progress(SpvSyncState.WAIT_FOR_EVENTS, filters = 0L to 0L)), + ) + assertFalse( + "the poll's not-running placeholder is not steady", + TxoReconcileGates.isSteadySyncState(SpvSyncProgressData.EMPTY), + ) + assertTrue(TxoReconcileGates.isSteadySyncState(progress(SpvSyncState.SYNCED, filters = 2_535_898L to 2_535_898L))) + assertTrue("dash-spv's fully synced steady state", TxoReconcileGates.isSteadySyncState(steady)) + } + + @Test + fun theTipIsTheFilterHeightFallingBackToTheHeaderTip() { + assertEquals(2_535_898, TxoReconcileGates.scanTipHeight(steady)) + assertEquals( + "falls back to the header tip", + 2_535_898, + TxoReconcileGates.scanTipHeight(progress(SpvSyncState.SYNCED, filters = null)), + ) + assertEquals( + "a filter phase at height 0 falls back too", + 2_535_000, + TxoReconcileGates.scanTipHeight(progress(SpvSyncState.SYNCED, filters = 0L to 0L, headers = 2_535_000)), + ) + assertNull(TxoReconcileGates.scanTipHeight(SpvSyncProgressData.EMPTY)) + } + + @Test + fun everyLoadedWalletRunsOnTheRisingEdgeIntoSteadyState() { + val coordinator = TxoReconcileCoordinator(cadenceMs = 1_000) + val wallets = listOf("a", "b") + + assertEquals(emptyList(), coordinator.dueWithLaunchesEnded(syncing, wallets, nowMs = 0)) + assertEquals(wallets, coordinator.dueWithLaunchesEnded(steady, wallets, nowMs = 10)) + assertEquals( + "the next tick inside the cadence schedules nothing", + emptyList(), + coordinator.dueWithLaunchesEnded(steady, wallets, nowMs = 20), + ) + } + + @Test + fun aSteadyClientRunsEachWalletAgainOnItsOwnCadence() { + val coordinator = TxoReconcileCoordinator(cadenceMs = 1_000) + assertEquals(listOf("a"), coordinator.dueWithLaunchesEnded(steady, listOf("a"), nowMs = 0)) + // A wallet loaded while steady never ran: it is due at once, on its + // own clock. + assertEquals(listOf("b"), coordinator.dueWithLaunchesEnded(steady, listOf("a", "b"), nowMs = 500)) + assertEquals(listOf("a"), coordinator.dueWithLaunchesEnded(steady, listOf("a", "b"), nowMs = 1_000)) + assertEquals(listOf("b"), coordinator.dueWithLaunchesEnded(steady, listOf("a", "b"), nowMs = 1_500)) + // A completed run restarts that wallet's cadence. + assertTrue(coordinator.markRun("a", nowMs = 1_900, runEpoch = coordinator.currentEpoch)) + assertEquals(emptyList(), coordinator.dueWithLaunchesEnded(steady, listOf("a", "b"), nowMs = 2_000)) + assertEquals(listOf("a", "b"), coordinator.dueWithLaunchesEnded(steady, listOf("a", "b"), nowMs = 2_900)) + } + + @Test + fun aWalletWithARunInFlightIsNeverScheduledOrStartedAgain() { + val coordinator = TxoReconcileCoordinator(cadenceMs = 1_000) + assertTrue(coordinator.tryBegin("a")) + assertFalse("a second run for the same wallet is refused", coordinator.tryBegin("a")) + assertEquals( + "not even on the rising edge", + listOf("b"), + coordinator.walletsDue(steady, listOf("a", "b"), nowMs = 0), + ) + coordinator.end("a") + assertTrue(coordinator.tryBegin("a")) + } + + @Test + fun aLaunchedRunThatHasNotClaimedYetIsNotScheduledAgain() { + // The run is launched but its coroutine has not reached tryBegin + // yet, and a new block takes the client out of steady state and + // back: that rising edge must not launch the wallet a second time. + val coordinator = TxoReconcileCoordinator(cadenceMs = 1_000_000) + assertEquals(listOf("a"), coordinator.walletsDue(steady, listOf("a"), nowMs = 0)) + assertEquals(emptyList(), coordinator.walletsDue(syncing, listOf("a"), nowMs = 10)) + assertEquals( + "a rising edge while the launch is pending skips it", + emptyList(), + coordinator.walletsDue(steady, listOf("a"), nowMs = 20), + ) + + // The launched run starts, claims, and ends. + assertTrue(coordinator.tryBegin("a")) + coordinator.end("a") + coordinator.launchEnded("a") + assertEquals(emptyList(), coordinator.walletsDue(syncing, listOf("a"), nowMs = 30)) + assertEquals( + "the next rising edge after the launch ended schedules it again", + listOf("a"), + coordinator.walletsDue(steady, listOf("a"), nowMs = 40), + ) + } + + @Test + fun aRefusedClaimSkipsAndNeverQueues() { + // A host call racing a scheduled run: the loser's tryBegin is + // refused (reconcileTxoStore then skips with ALREADY_RUNNING, it does + // not wait), and nothing is left behind to run later. + val coordinator = TxoReconcileCoordinator(cadenceMs = 1_000_000) + assertEquals(listOf("a"), coordinator.walletsDue(steady, listOf("a"), nowMs = 0)) + assertTrue("the scheduled run claims", coordinator.tryBegin("a")) + assertFalse("the host call is refused", coordinator.tryBegin("a")) + coordinator.end("a") + coordinator.launchEnded("a") + assertEquals( + "inside the cadence, with no edge, nothing runs again", + emptyList(), + coordinator.walletsDue(steady, listOf("a"), nowMs = 10), + ) + assertTrue("and the wallet can be claimed afresh", coordinator.tryBegin("a")) + } + + @Test + fun leavingSteadyStateArmsTheNextRisingEdge() { + val coordinator = TxoReconcileCoordinator(cadenceMs = 1_000_000) + assertEquals(listOf("a"), coordinator.dueWithLaunchesEnded(steady, listOf("a"), nowMs = 0)) + assertEquals(emptyList(), coordinator.dueWithLaunchesEnded(syncing, listOf("a"), nowMs = 10)) + assertEquals( + "back in steady state: due again, cadence or not", + listOf("a"), + coordinator.dueWithLaunchesEnded(steady, listOf("a"), nowMs = 20), + ) + } + + @Test + fun stoppingSpvArmsTheNextRisingEdgeWithoutASyncingRead() { + // SPV stops and restarts straight into steady state: the poll feeds + // no progress while SPV is down and the first read after the + // restart is already steady, so no syncing read ever lowers the + // edge. The stop itself must, or every wallet waits out the cadence. + val coordinator = TxoReconcileCoordinator(cadenceMs = 1_000_000) + val wallets = listOf("a", "b") + assertEquals(wallets, coordinator.dueWithLaunchesEnded(steady, wallets, nowMs = 0)) + assertEquals(emptyList(), coordinator.dueWithLaunchesEnded(steady, wallets, nowMs = 10)) + + coordinator.spvStopped() + + assertEquals( + "the first steady read after the restart is a new steady state", + wallets, + coordinator.dueWithLaunchesEnded(steady, wallets, nowMs = 20), + ) + assertEquals( + "and only the first", + emptyList(), + coordinator.dueWithLaunchesEnded(steady, wallets, nowMs = 30), + ) + } + + @Test + fun aRunOvertakenByARemovalDoesNotStampTheReaddedWalletsCadence() { + // A run captured the epoch and was still going when its wallet was + // removed (epoch bumped, cadence forgotten) and the same id added + // back. Its finish must not stamp the cadence: the re-added wallet + // has never been reconciled and is due on the next tick. + val coordinator = TxoReconcileCoordinator(cadenceMs = 1_000_000) + assertEquals(listOf("a"), coordinator.walletsDue(steady, listOf("a"), nowMs = 0)) + assertTrue(coordinator.tryBegin("a")) + val runEpoch = coordinator.currentEpoch + + // removeWallet: bump first, then forget; addWallet puts "a" back. + coordinator.bumpEpoch() + coordinator.forget("a") + assertEquals( + "in flight, so not scheduled yet", + emptyList(), + coordinator.walletsDue(steady, listOf("a"), nowMs = 10), + ) + + assertFalse(coordinator.markRun("a", nowMs = 20, runEpoch = runEpoch)) + coordinator.end("a") + coordinator.launchEnded("a") + assertEquals( + "the re-added wallet is due at once, not a cadence later", + listOf("a"), + coordinator.walletsDue(steady, listOf("a"), nowMs = 30), + ) + } + + @Test + fun aRunStampedBeforeTheRemovalIsClearedByIt() { + // The other order: the run finished and stamped before the removal. + // The removal's forget clears that stamp, so the re-added wallet is + // still due at once. + val coordinator = TxoReconcileCoordinator(cadenceMs = 1_000_000) + assertEquals(listOf("a"), coordinator.walletsDue(steady, listOf("a"), nowMs = 0)) + assertTrue(coordinator.markRun("a", nowMs = 5, runEpoch = coordinator.currentEpoch)) + coordinator.launchEnded("a") + coordinator.bumpEpoch() + coordinator.forget("a") + assertEquals(listOf("a"), coordinator.walletsDue(steady, listOf("a"), nowMs = 10)) + } + + @Test + fun closeAndDeletionStaleTheEpochAndDeletionForgetsTheCadence() { + val coordinator = TxoReconcileCoordinator(cadenceMs = 1_000) + val before = coordinator.currentEpoch + coordinator.bumpEpoch() + assertTrue("an in-flight run sees its epoch move", coordinator.currentEpoch != before) + + assertEquals(listOf("a"), coordinator.dueWithLaunchesEnded(steady, listOf("a"), nowMs = 0)) + coordinator.forget("a") + assertEquals( + "a re-added wallet is due at once", + listOf("a"), + coordinator.dueWithLaunchesEnded(steady, listOf("a"), nowMs = 10), + ) + } + + @Test + fun theWalletGateUsesTheEnginesWatermarkWithinTheTipMargin() { + // Swift's `synced_height + margin >= tip`. The value passed is the + // ENGINE's watermark: the persisted Room copy trails it by a round, + // and gating on that skipped the run at the SYNCED edge of a restore + // (2026-10-02 device run: WALLET_BEHIND_TIP at the transition, the + // next attempt 30 minutes later). + assertTrue(TxoReconcileGates.walletCaughtUp(walletSyncedHeight = 1_565_211L, tipHeight = 1_565_211)) + assertTrue(TxoReconcileGates.walletCaughtUp(walletSyncedHeight = 1_565_205L, tipHeight = 1_565_211)) + assertFalse(TxoReconcileGates.walletCaughtUp(walletSyncedHeight = 1_565_204L, tipHeight = 1_565_211)) + // A wallet added behind the tip is still being scanned. + assertFalse(TxoReconcileGates.walletCaughtUp(walletSyncedHeight = 0L, tipHeight = 1_565_211)) + // The native "no value" sentinel never passes. + assertFalse(TxoReconcileGates.walletCaughtUp(walletSyncedHeight = -1L, tipHeight = 3)) + } +} diff --git a/packages/kotlin-sdk/sdk/src/test/resources/golden/txo_reconcile_transport_v1.json b/packages/kotlin-sdk/sdk/src/test/resources/golden/txo_reconcile_transport_v1.json new file mode 100644 index 00000000000..0a6a151d66c --- /dev/null +++ b/packages/kotlin-sdk/sdk/src/test/resources/golden/txo_reconcile_transport_v1.json @@ -0,0 +1,123 @@ +{ + "_about": "Golden TXO-reconcile transport payloads, read by rs-unified-sdk-jni (wallet_manager::tests) and the Kotlin PlatformWalletPersistenceHandlerTest. inventoryPage is exactly what walletManagerUtxosPageJson emits for the four accessor entries the Rust test builds; outpointQueries is exactly what the Kotlin reconcile sends walletManagerClassifyOutpoints for the same four coins. Change both sides together.", + "inventoryPage": { + "utxos": [ + { + "typeTag": 0, + "standardTag": 0, + "index": 0, + "registrationIndex": 0, + "keyClass": 0, + "txid": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "vout": 0, + "amount": 19549, + "address": "yStxXHHzhAx58JhaPBNhn3xsH93UwBM2nd", + "scriptHex": "76a9145b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b88ac", + "height": 1534921, + "isConfirmed": true, + "isInstantlocked": false, + "isCoinbase": false, + "isLocked": false + }, + { + "typeTag": 0, + "standardTag": 1, + "index": 2, + "registrationIndex": 0, + "keyClass": 0, + "txid": "202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f", + "vout": 7, + "amount": 5000000000, + "address": "yGoldenFixtureBip32Addr", + "scriptHex": "76a914c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c388ac", + "height": 1400000, + "isConfirmed": true, + "isInstantlocked": true, + "isCoinbase": true, + "isLocked": true + }, + { + "typeTag": 3, + "standardTag": 0, + "index": 0, + "registrationIndex": 4, + "keyClass": 0, + "txid": "404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f", + "vout": 1, + "amount": 100000, + "address": "", + "scriptHex": "", + "height": 0, + "isConfirmed": false, + "isInstantlocked": false, + "isCoinbase": false, + "isLocked": false + }, + { + "typeTag": 12, + "standardTag": 0, + "index": 1, + "registrationIndex": 0, + "keyClass": 0, + "userIdentityId": "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", + "friendIdentityId": "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2", + "txid": "606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f", + "vout": 3, + "amount": 250000, + "address": "yGoldenFixtureDashPayAddr", + "scriptHex": "76a914d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d488ac", + "height": 1534000, + "isConfirmed": true, + "isInstantlocked": true, + "isCoinbase": false, + "isLocked": false + } + ], + "cursor": "{\"typeTag\":12,\"standardTag\":0,\"index\":1,\"registrationIndex\":0,\"keyClass\":0,\"userIdentityId\":\"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1\",\"friendIdentityId\":\"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2\",\"txid\":\"606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f\",\"vout\":3}", + "hasMore": true + }, + "outpointQueries": [ + { + "typeTag": 0, + "standardTag": 0, + "index": 0, + "registrationIndex": 0, + "keyClass": 0, + "txid": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "vout": 0, + "scriptHex": "76a9145b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b5b88ac" + }, + { + "typeTag": 0, + "standardTag": 1, + "index": 2, + "registrationIndex": 0, + "keyClass": 0, + "txid": "202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f", + "vout": 7, + "scriptHex": "76a914c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c388ac" + }, + { + "typeTag": 3, + "standardTag": 0, + "index": 0, + "registrationIndex": 4, + "keyClass": 0, + "txid": "404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f", + "vout": 1, + "scriptHex": "" + }, + { + "typeTag": 12, + "standardTag": 0, + "index": 1, + "registrationIndex": 0, + "keyClass": 0, + "userIdentityId": "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", + "friendIdentityId": "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2", + "txid": "606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f", + "vout": 3, + "scriptHex": "76a914d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d4d488ac" + } + ] +} diff --git a/packages/rs-unified-sdk-jni/Cargo.toml b/packages/rs-unified-sdk-jni/Cargo.toml index 9f5b2413f06..9b396969e5f 100644 --- a/packages/rs-unified-sdk-jni/Cargo.toml +++ b/packages/rs-unified-sdk-jni/Cargo.toml @@ -16,6 +16,14 @@ jni = "0.21" rs-sdk-ffi = { path = "../rs-sdk-ffi" } platform-wallet-ffi = { path = "../rs-platform-wallet-ffi" } log = "0.4" +# Serialization for the TXO-reconcile transport (the paged wallet +# inventory and the outpoint-classification batch). Both exports are +# shims over one platform-wallet call each: serde shapes the JSON +# against the same field names the Kotlin side decodes, so a renamed +# key fails the decode loudly instead of defaulting a row. +serde = { version = "1", features = ["derive"] } +serde_json = "1" +hex = "0.4" zeroize = "1" [target.'cfg(target_os = "android")'.dependencies] diff --git a/packages/rs-unified-sdk-jni/src/persistence.rs b/packages/rs-unified-sdk-jni/src/persistence.rs index 86a38c3329e..b095e1e4f4c 100644 --- a/packages/rs-unified-sdk-jni/src/persistence.rs +++ b/packages/rs-unified-sdk-jni/src/persistence.rs @@ -60,8 +60,9 @@ use platform_wallet_ffi::{ PaymentRestoreEntryFFI, PersistenceCallbacks, PersistenceCallbacksExtension, PlatformAddressFFI, PrivateContactRequestsPositionFFI, PrivateShieldedAddressFFI, ProviderSpecialTxRestoreEntryFFI, SpentOutPointFFI, SweepBatchFFI, TokenBalanceRemovalFFI, - TokenBalanceUpsertFFI, TransactionRecordFFI, UnresolvedAssetLockTxRecordFFI, UtxoEntryFFI, - UtxoRestoreEntryFFI, WalletChangeSetFFI, WalletRestoreEntryFFI, + TokenBalanceUpsertFFI, TransactionRecordFFI, UnresolvedAssetLockTxRecordFFI, + UtxoCreditVerdictFFI, UtxoEntryFFI, UtxoRestoreEntryFFI, WalletChangeSetFFI, + WalletRestoreEntryFFI, }; use std::ffi::{c_void, CStr, CString}; use std::os::raw::c_char; @@ -363,6 +364,13 @@ fn extension_for(overrides: BridgeOverrides) -> PersistenceCallbacksExtension { on_load_private_contact_scan_state_wallets_free_fn: overrides .private_contact_scan_state .then_some(tramp_load_private_contact_scan_state_wallets_free), + // Wired for every bridge: the base class's no-op body is exactly + // the behaviour of a host without the slot (verdicts only ever + // refine what the changeset callback writes), so there is no + // capability to strip the way the sweep slot has. + on_persist_wallet_changeset_utxo_verdicts_fn: Some( + tramp_persist_wallet_changeset_utxo_verdicts, + ), ..Default::default() } } @@ -1266,6 +1274,62 @@ unsafe extern "C" fn tramp_persist_private_contact_scan_state( }) } +/// Descriptor of `NativePersistenceBridge.onWalletChangesetUtxoVerdicts`: +/// `(walletId, outpoints, verdicts, spentAtHeights, count)`. Outpoints ride +/// as ONE flat `byte[]` of 36-byte keys (`pack_outpoint_key`, the shape the +/// handler stores them in), the verdict codes as one `byte[]` and the +/// observed-spent heights as one `int[]`, all N long — the same flat packing +/// as the sweep batches, since the count is chain-controlled and this runs +/// inside the atomic persistence callback. +const WALLET_CHANGESET_UTXO_VERDICTS_DESCRIPTOR: &str = "([B[B[B[II)I"; + +/// Deliver the round's credit verdicts (see +/// `PersistWalletChangesetUtxoVerdictsFn`): every `Received` / `Change` +/// output of the round's records the engine did NOT credit, with the +/// reason. Fired by `store()` inside the begin/end bracket BEFORE the +/// changeset callback and only on rounds that carry a verdict, so the +/// handler has them in hand when the round's `utxos_added` entries arrive. +/// One bridge call for the whole set, in the slot's outpoint order. +unsafe extern "C" fn tramp_persist_wallet_changeset_utxo_verdicts( + context: *mut c_void, + wallet_id: *const u8, + verdicts: *const UtxoCreditVerdictFFI, + verdicts_count: usize, +) -> i32 { + with_bridge(context, |env, bridge| { + let wid = id32(env, wallet_id)?; + // A null buffer that claims entries fails the round: read as "no + // verdicts", it would let the round's `utxos_added` persist a coin + // without the veto the producer meant to deliver. + let entries = slice_or_fail(verdicts, verdicts_count)?; + let mut packed_outpoints = Vec::with_capacity(entries.len() * 36); + let mut codes = Vec::with_capacity(entries.len()); + let mut heights = Vec::with_capacity(entries.len()); + for entry in entries { + packed_outpoints.extend_from_slice(&pack_outpoint_key(&entry.outpoint)); + codes.push(entry.verdict); + heights.push(jint_height(entry.spent_at_height)?); + } + let outpoints_arr = env.byte_array_from_slice(&packed_outpoints)?; + let codes_arr = env.byte_array_from_slice(&codes)?; + let heights_arr = env.new_int_array(heights.len() as i32)?; + env.set_int_array_region(&heights_arr, 0, &heights)?; + env.call_method( + bridge, + "onWalletChangesetUtxoVerdicts", + WALLET_CHANGESET_UTXO_VERDICTS_DESCRIPTOR, + &[ + (&wid).into(), + (&outpoints_arr).into(), + (&codes_arr).into(), + (&heights_arr).into(), + JValue::Int(entries.len() as i32), + ], + )? + .i() + }) +} + /// Hand Rust a wallet's stored scan state (see /// `LoadPrivateContactScanStateFn`). The rows are staged in owned buffers /// and minted as raw arrays only once the whole read succeeded; a failed @@ -4869,6 +4933,21 @@ unsafe fn slice_or_empty<'a, T>(ptr: *const T, count: usize) -> &'a [T] { } } +/// As [`slice_or_empty`], except that a null pointer with a non-zero +/// count is an error rather than an empty slice — for a slot whose entries +/// must not be dropped silently (`with_bridge` maps the error to +/// `ERR_JNI`, failing the round). +/// +/// # Safety +/// A non-null `ptr` must be readable for `count` elements for the callback +/// window. +unsafe fn slice_or_fail<'a, T>(ptr: *const T, count: usize) -> Result<&'a [T], jni::errors::Error> { + if ptr.is_null() && count > 0 { + return Err(jni::errors::Error::NullPtr("a non-empty callback slice")); + } + Ok(slice_or_empty(ptr, count)) +} + /// Pack an [`OutPointFFI`] into the 36-byte key (raw txid ‖ little-endian /// vout) the Kotlin handler stores outpoints under — byte-identical to /// Kotlin's `makeOutpoint` (and Swift's). This is the join key sweep @@ -5232,6 +5311,13 @@ const BRIDGE_METHOD_TABLE: &[(&str, &str)] = &[ // literal at the `call_method` site in // `tramp_persist_wallet_changeset_chain_lock_height`. ("onWalletChangesetChainLockHeight", "([BI)I"), + // Same drift risk again: this slot fires only on rounds that carry a + // credit verdict. The same constant is bound at the `call_method` site + // in `tramp_persist_wallet_changeset_utxo_verdicts`. + ( + "onWalletChangesetUtxoVerdicts", + WALLET_CHANGESET_UTXO_VERDICTS_DESCRIPTOR, + ), ( "onWalletChangesetDashPayBackfill", WALLET_CHANGESET_DASHPAY_BACKFILL_DESCRIPTOR, @@ -5648,6 +5734,18 @@ mod tests { assert!(jint_height(u32::MAX).is_err()); } + #[test] + fn a_null_verdict_buffer_that_claims_entries_fails_the_round() { + let none: *const UtxoCreditVerdictFFI = ptr::null(); + assert!(unsafe { slice_or_fail(none, 0) }.unwrap().is_empty()); + assert!(unsafe { slice_or_fail(none, 2) }.is_err()); + let values = [7u8, 8]; + assert_eq!( + unsafe { slice_or_fail(values.as_ptr(), 2) }.unwrap(), + &values + ); + } + #[test] fn sweeps_callback_descriptor_ships_flat_arrays_and_an_explicit_height_pair() { // walletId, packed txids + count, winner, packed released outpoints @@ -5656,6 +5754,14 @@ mod tests { assert_eq!(WALLET_CHANGESET_SWEEPS_DESCRIPTOR, "([B[BI[B[BIZI)I"); } + #[test] + fn utxo_verdicts_callback_descriptor_ships_flat_arrays_and_a_count() { + // walletId, packed 36-byte outpoints, verdict codes, observed-spent + // heights, count — must match + // `NativePersistenceBridge.onWalletChangesetUtxoVerdicts`. + assert_eq!(WALLET_CHANGESET_UTXO_VERDICTS_DESCRIPTOR, "([B[B[B[II)I"); + } + #[test] fn vtable_layout_remains_independent_of_capability_declaration() { let callbacks = build_vtable(ptr::null_mut()); diff --git a/packages/rs-unified-sdk-jni/src/wallet_manager.rs b/packages/rs-unified-sdk-jni/src/wallet_manager.rs index b15b2128aea..3897ee89ccc 100644 --- a/packages/rs-unified-sdk-jni/src/wallet_manager.rs +++ b/packages/rs-unified-sdk-jni/src/wallet_manager.rs @@ -3064,6 +3064,43 @@ pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_s }) } +/// The wallet's core scan watermark as the ENGINE holds it +/// (`platform_wallet_core_wallet_state`'s `synced_height`) — the value the +/// TXO-store reconcile gates on, as Swift's does. The persisted copy in the +/// host store trails it by up to one persistence round, which is exactly +/// the window the SYNCED transition lands in. Throws `NotFound` when the +/// manager does not hold the wallet. Backs +/// `PlatformWalletManager.reconcileTxoStore`. +#[no_mangle] +pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_walletManagerCoreWalletSyncedHeight( + mut env: JNIEnv, + _class: JClass, + manager_handle: jlong, + wallet_id: JByteArray, +) -> jlong { + guard(&mut env, -1, |env| { + let Some(wid) = read_id32(env, &wallet_id) else { + return -1; + }; + let mut state = platform_wallet_ffi::CoreWalletStateFFI { + synced_height: 0, + last_processed_height: 0, + monitor_revision: 0, + }; + let result = unsafe { + platform_wallet_ffi::platform_wallet_core_wallet_state( + manager_handle as Handle, + wid.as_ptr(), + &mut state as *mut platform_wallet_ffi::CoreWalletStateFFI, + ) + }; + if take_pwffi_error(env, result) { + return -1; + } + jlong::from(state.synced_height) + }) +} + #[cfg(feature = "shielded")] sync_start_stop!( Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_shieldedSyncStart, @@ -3622,6 +3659,482 @@ fn core_selection_strategy( } } +// ── TXO-store reconcile transport ───────────────────────────────────── +// +// Two exports, each a serialization shim over exactly ONE platform-wallet +// call: `platform_wallet_wallet_utxos_page` and +// `platform_wallet_classify_outpoints`. The account ordering, the cursor +// semantics, the page bound and every classification rule live in +// `platform-wallet`, so the Swift host walks the identical inventory and +// gets the identical verdicts. Nothing here decides anything; it moves +// serde-shaped JSON in and out. + +/// The account tuple that owns one inventory row — the raw `AccountSpecFFI` +/// tag layout minus the xpub, in the field names the Kotlin `EngineUtxoRow` +/// and `PlatformWalletPersistenceHandler.fetchAccount` resolve a Room +/// account by. The DashPay identity halves are emitted only when set +/// (all-zero on every non-DashPay account); the Kotlin side defaults them. +#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +struct UtxoAccountTuple { + type_tag: u8, + standard_tag: u8, + index: u32, + registration_index: u32, + key_class: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + user_identity_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + friend_identity_id: Option, +} + +impl UtxoAccountTuple { + fn from_entry(e: &platform_wallet_ffi::WalletUtxoEntryFFI) -> Self { + let identity = |id: &[u8; 32]| (*id != [0u8; 32]).then(|| hex::encode(id)); + UtxoAccountTuple { + type_tag: e.type_tag as u8, + standard_tag: e.standard_tag as u8, + index: e.index, + registration_index: e.registration_index, + key_class: e.key_class, + user_identity_id: identity(&e.user_identity_id), + friend_identity_id: identity(&e.friend_identity_id), + } + } + + /// The two 32-byte identity halves, or `None` when a hex string the + /// host handed back is malformed — a tuple this layer did not emit. + fn identity_ids(&self) -> Option<([u8; 32], [u8; 32])> { + fn id32(hex_id: &Option) -> Option<[u8; 32]> { + match hex_id { + None => Some([0u8; 32]), + Some(h) => hex32(h), + } + } + Some(( + id32(&self.user_identity_id)?, + id32(&self.friend_identity_id)?, + )) + } + + /// The two identity halves and the FFI outpoint of a row keyed by this + /// tuple and `txid`/`vout`, or the error naming the malformed field. + fn keyed_outpoint( + &self, + txid: &str, + vout: u32, + ) -> Result<([u8; 32], [u8; 32], platform_wallet_ffi::OutPointFFI), &'static str> { + let (user_identity_id, friend_identity_id) = self + .identity_ids() + .ok_or("outpoint query carries a malformed identity id")?; + let txid = hex32(txid).ok_or("outpoint query carries a malformed txid")?; + Ok(( + user_identity_id, + friend_identity_id, + platform_wallet_ffi::OutPointFFI { txid, vout }, + )) + } +} + +/// 32 bytes from their hex form, or `None` when it is not exactly that. +fn hex32(s: &str) -> Option<[u8; 32]> { + hex::decode(s).ok()?.try_into().ok() +} + +/// One row of an inventory page — the typed contract the Kotlin +/// `EngineUtxoRow` decodes, strictly (an unknown or missing key throws +/// there rather than healing a defaulted row). `txid` is lower hex in the +/// same byte order the changeset path hands Kotlin, so hex→bytes +/// reproduces the `txos.txid` blob; `address` is the engine's own +/// Base58Check rendering, empty when the script has no address form. +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +struct UtxoPageRow { + #[serde(flatten)] + account: UtxoAccountTuple, + txid: String, + vout: u32, + amount: u64, + address: String, + script_hex: String, + height: u32, + is_confirmed: bool, + is_instantlocked: bool, + is_coinbase: bool, + is_locked: bool, +} + +impl UtxoPageRow { + /// The row for one accessor entry, carrying the engine's own address + /// and flags. + /// + /// # Safety + /// + /// `e.address` must be null or a valid NUL-terminated string, and + /// `e.script_pubkey` null or valid for `e.script_pubkey_len` bytes — + /// the accessor's contract for every entry it returns. + unsafe fn from_entry(e: &platform_wallet_ffi::WalletUtxoEntryFFI) -> Self { + let script: &[u8] = if e.script_pubkey.is_null() || e.script_pubkey_len == 0 { + &[] + } else { + std::slice::from_raw_parts(e.script_pubkey, e.script_pubkey_len) + }; + // Never null per the accessor's contract; treated as empty if it + // ever is, which the Kotlin heal skips rather than filing a row it + // cannot key an address by. + let address = cstr_opt(e.address).unwrap_or_default(); + UtxoPageRow { + account: UtxoAccountTuple::from_entry(e), + txid: hex::encode(e.outpoint.txid), + vout: e.outpoint.vout, + amount: e.value_duffs, + address, + script_hex: hex::encode(script), + height: e.height, + is_confirmed: e.is_confirmed, + is_instantlocked: e.is_instantlocked, + is_coinbase: e.is_coinbase, + is_locked: e.is_locked, + } + } +} + +/// One page: the rows, the opaque resume cursor (serialized as `null` on +/// the last page, never omitted) and whether more pages follow. The Kotlin +/// `EngineUtxoPage` requires all three keys. +#[derive(serde::Serialize)] +#[serde(rename_all = "camelCase")] +struct UtxoPage { + utxos: Vec, + cursor: Option, + has_more: bool, +} + +impl UtxoPage { + /// The page for `utxos`. The cursor is the last row itself; a page with + /// no rows has nowhere to resume from, and the accessor reports no more + /// in that case. Fails only if the cursor cannot be encoded. + fn new(utxos: Vec, has_more: bool) -> serde_json::Result { + let cursor = match utxos.last() { + Some(last) if has_more => Some(serde_json::to_string(&UtxoPageCursor::after(last))?), + _ => None, + }; + Ok(UtxoPage { + // `cursor` is set exactly when `has_more` is and a row exists. + has_more: cursor.is_some(), + cursor, + utxos, + }) + } +} + +/// Where a paged inventory sweep left off — the last row's account tuple +/// and outpoint, which is exactly what `platform_wallet_wallet_utxos_page` +/// resumes from. Serialized as JSON and handed to the host as an opaque +/// string; the host only ever gives back what a previous page returned. +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +struct UtxoPageCursor { + #[serde(flatten)] + account: UtxoAccountTuple, + txid: String, + vout: u32, +} + +impl UtxoPageCursor { + /// The cursor that resumes the sweep after `last`. + fn after(last: &UtxoPageRow) -> Self { + UtxoPageCursor { + account: last.account.clone(), + txid: last.txid.clone(), + vout: last.vout, + } + } + + /// The resume point a cursor string names, or `None` when it is not + /// one a previous page returned (unparseable, or a malformed identity + /// half or txid). + fn decode(raw: &str) -> Option { + let c = serde_json::from_str::(raw).ok()?; + let (user_identity_id, friend_identity_id, outpoint) = + c.account.keyed_outpoint(&c.txid, c.vout).ok()?; + Some(platform_wallet_ffi::WalletUtxoCursorFFI { + type_tag: c.account.type_tag, + standard_tag: c.account.standard_tag, + index: c.account.index, + registration_index: c.account.registration_index, + key_class: c.account.key_class, + user_identity_id, + friend_identity_id, + outpoint, + }) + } +} + +/// One store row the host asks the engine to classify: the account the +/// STORE files the coin under, the outpoint, and the script the store +/// recorded. The engine checks that claim against its own pools rather +/// than trusting it, which is why the tuple and script travel with every +/// query instead of the outpoint alone. +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +struct OutpointQuery { + #[serde(flatten)] + account: UtxoAccountTuple, + txid: String, + vout: u32, + #[serde(default)] + script_hex: String, +} + +impl OutpointQuery { + /// The FFI query for this row and the script bytes it must point at. + /// The script pointer is left null: the caller fills it once the + /// owned scripts have stopped moving. The error names the malformed + /// field, as the SDK exception the batch fails with. + fn to_ffi( + &self, + ) -> Result<(platform_wallet_ffi::OutpointOwnershipQueryFFI, Vec), &'static str> { + let (user_identity_id, friend_identity_id, outpoint) = + self.account.keyed_outpoint(&self.txid, self.vout)?; + let script = hex::decode(&self.script_hex) + .map_err(|_| "outpoint query carries a malformed script")?; + let query = platform_wallet_ffi::OutpointOwnershipQueryFFI { + type_tag: self.account.type_tag, + standard_tag: self.account.standard_tag, + index: self.account.index, + registration_index: self.account.registration_index, + key_class: self.account.key_class, + user_identity_id, + friend_identity_id, + outpoint, + script_pubkey: ptr::null(), + script_pubkey_len: 0, + }; + Ok((query, script)) + } +} + +/// One bounded page of the engine's UTXO inventory across every funds +/// account of one wallet — the source of truth +/// `PlatformWalletManager.reconcileTxoStore` diffs against the Room `txos` +/// mirror (dropped change outputs of CoinJoin-funded sends leave the +/// mirror short; the engine reloads from that mirror on restart, so an +/// un-reconciled hole becomes a fund loss). +/// +/// Paged rather than swept whole because inventory size is +/// chain-controlled: anyone who knows a watched address can keep sending +/// dust outputs to it, and a periodic full-inventory read would let them +/// decide how much a phone allocates at every SYNCED transition and every +/// 30-minute pass. Nothing bigger than one page is ever formatted, copied +/// across JNI, or parsed. +/// +/// Returns a JSON object `{"utxos":[...],"cursor":,"hasMore":}`. +/// `cursor` is opaque: hand it back verbatim on the next call +/// (`null`/absent starts from the beginning) and keep going while +/// `hasMore` is true. A cursor this export did not produce is rejected +/// with an SDK exception rather than silently restarting the sweep. +/// `limit` caps the rows in one page; non-positive passes 0, which means +/// the engine's own default, and the engine clamps its own maximum — this +/// layer holds no copy of either bound. +/// +/// Rows carry the engine's own address and flags. Nothing is derived +/// here: an address computed in this shim could disagree with the one the +/// engine credited the coin under, and the store keys its address rows by +/// that string. +#[no_mangle] +pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_walletManagerUtxosPageJson( + mut env: JNIEnv, + _class: JClass, + manager_handle: jlong, + wallet_id: JByteArray, + cursor: JString, + limit: jni::sys::jint, +) -> jni::sys::jstring { + guard(&mut env, ptr::null_mut(), |env| { + let Some(wid) = read_id32(env, &wallet_id) else { + return ptr::null_mut(); + }; + // 0 means "the engine's default page"; the engine also clamps its + // own maximum, so a host asking for more simply gets less. + let page_limit = if limit <= 0 { 0usize } else { limit as usize }; + + // Resume point, decoded from the opaque cursor. Kept alive across + // the FFI call — it is passed by pointer. + let resume: Option = if cursor.is_null() { + None + } else { + let raw = match env.get_string(&cursor) { + Ok(s) => String::from(s), + Err(_) => { + throw_sdk_exception(env, 1, "inventory cursor must be a String"); + return ptr::null_mut(); + } + }; + match UtxoPageCursor::decode(&raw) { + Some(r) => Some(r), + None => { + throw_sdk_exception( + env, + 1, + "malformed inventory cursor: only a cursor returned by a previous page may be handed back", + ); + return ptr::null_mut(); + } + } + }; + + let mut entries: *const platform_wallet_ffi::WalletUtxoEntryFFI = ptr::null(); + let mut count: usize = 0; + let mut has_more = false; + let result = unsafe { + platform_wallet_ffi::platform_wallet_wallet_utxos_page( + manager_handle as Handle, + wid.as_ptr(), + resume.as_ref().map_or(ptr::null(), |c| c as *const _), + page_limit, + &mut entries, + &mut count, + &mut has_more, + ) + }; + if take_pwffi_error(env, result) { + return ptr::null_mut(); + } + let mut rows: Vec = Vec::with_capacity(count); + if !entries.is_null() && count > 0 { + let items = unsafe { std::slice::from_raw_parts(entries, count) }; + for e in items { + // SAFETY: every entry the accessor returns satisfies the + // pointer contract `from_entry` documents. + rows.push(unsafe { UtxoPageRow::from_entry(e) }); + } + unsafe { + platform_wallet_ffi::platform_wallet_wallet_utxos_page_free( + entries as *mut platform_wallet_ffi::WalletUtxoEntryFFI, + count, + ) + }; + } + let page = match UtxoPage::new(rows, has_more) { + Ok(p) => p, + Err(e) => { + throw_sdk_exception(env, 1, &format!("inventory cursor encode failed: {e}")); + return ptr::null_mut(); + } + }; + match serde_json::to_string(&page) { + Ok(json) => env + .new_string(json) + .map(|s| s.into_raw()) + .unwrap_or(ptr::null_mut()), + Err(e) => { + throw_sdk_exception(env, 1, &format!("inventory page encode failed: {e}")); + ptr::null_mut() + } + } + }) +} + +/// Classify a batch of store rows against the engine's live state — the +/// reverse half of the reconcile transport, and the reason the paged +/// inventory carries no spent-outpoint list. The host pages its OWN mirror +/// rows and asks about them a batch at a time, so neither side builds a +/// set over the whole engine inventory. +/// +/// `queriesJson` is a JSON array of store rows: the account tuple the +/// store files each coin under, its `txid` (lower hex, wire order) and +/// `vout`, and the `scriptHex` the store recorded for it. The account and +/// script are the store's CLAIM — the engine checks both against its own +/// pools rather than trusting them, which is why a bare outpoint is not +/// enough. +/// +/// Returns one verdict byte per query, positionally: see the +/// `OUTPOINT_CLASS_*` constants (0 unknown, 1 unspent, 2 known-uncredited, +/// 3 not-owned). A query whose account tag this build cannot map keeps +/// `unknown` and the rest are still answered. +#[no_mangle] +pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_walletManagerClassifyOutpoints( + mut env: JNIEnv, + _class: JClass, + manager_handle: jlong, + wallet_id: JByteArray, + queries_json: JString, +) -> jbyteArray { + guard(&mut env, ptr::null_mut(), |env| { + let Some(wid) = read_id32(env, &wallet_id) else { + return ptr::null_mut(); + }; + let raw = match env.get_string(&queries_json) { + Ok(s) => String::from(s), + Err(_) => { + throw_sdk_exception(env, 1, "outpoint queries must be a String"); + return ptr::null_mut(); + } + }; + let parsed: Vec = match serde_json::from_str(&raw) { + Ok(q) => q, + Err(e) => { + throw_sdk_exception(env, 1, &format!("malformed outpoint queries: {e}")); + return ptr::null_mut(); + } + }; + if parsed.is_empty() { + return env + .byte_array_from_slice(&[]) + .map(|a| a.into_raw()) + .unwrap_or(ptr::null_mut()); + } + // Scripts are owned here for the duration of the call — the query + // struct borrows them by pointer. + let mut scripts: Vec> = Vec::with_capacity(parsed.len()); + let mut queries: Vec = + Vec::with_capacity(parsed.len()); + for q in &parsed { + match q.to_ffi() { + Ok((query, script)) => { + scripts.push(script); + // Its script pointer is filled below, once `scripts` + // has stopped reallocating. + queries.push(query); + } + Err(message) => { + throw_sdk_exception(env, 1, message); + return ptr::null_mut(); + } + } + } + for (query, script) in queries.iter_mut().zip(scripts.iter()) { + query.script_pubkey = if script.is_empty() { + ptr::null() + } else { + script.as_ptr() + }; + query.script_pubkey_len = script.len(); + } + + // Caller-allocated answers, one byte per query. + let mut classes = vec![platform_wallet_ffi::OUTPOINT_CLASS_UNKNOWN; queries.len()]; + let result = unsafe { + platform_wallet_ffi::platform_wallet_classify_outpoints( + manager_handle as Handle, + wid.as_ptr(), + queries.as_ptr(), + queries.len(), + classes.as_mut_ptr(), + ) + }; + // `scripts` must outlive the call above; this keeps that explicit + // against a future reorder. + drop(scripts); + if take_pwffi_error(env, result) { + return ptr::null_mut(); + } + env.byte_array_from_slice(&classes) + .map(|a| a.into_raw()) + .unwrap_or(ptr::null_mut()) + }) +} /// Read a 32-byte id from a Java `byte[]`; throws + returns None on the /// wrong length or a JNI error. fn read_id32(env: &mut JNIEnv, arr: &JByteArray) -> Option<[u8; 32]> { @@ -4407,7 +4920,426 @@ pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_s #[cfg(test)] mod tests { + //! The TXO-reconcile transport codecs against the golden payloads the + //! Kotlin `PlatformWalletPersistenceHandlerTest` reads too, so the + //! serde models here and the kotlinx models there cannot drift apart + //! silently. + use super::*; + use platform_wallet_ffi::{AccountTypeTagFFI, StandardAccountTypeTagFFI, WalletUtxoEntryFFI}; + use std::ffi::CString; + + /// The checked-in transport payloads, shared byte-for-byte with the + /// Kotlin tests (`theInventoryDecoderReadsTheGoldenRustPage`, + /// `classificationQueriesMatchTheGoldenRustInput`). Referenced from the + /// single canonical copy in the Kotlin SDK's test resources. + const GOLDEN: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../kotlin-sdk/sdk/src/test/resources/golden/txo_reconcile_transport_v1.json" + )); + + fn golden() -> serde_json::Value { + serde_json::from_str(GOLDEN).expect("golden fixture must parse") + } + + /// A txid whose bytes count up from `first`, so a byte-order slip shows. + fn txid(first: u8) -> [u8; 32] { + std::array::from_fn(|i| first + i as u8) + } + + fn p2pkh(fill: u8) -> Vec { + let mut script = vec![0x76, 0xa9, 0x14]; + script.extend([fill; 20]); + script.extend([0x88, 0xac]); + script + } + + /// One accessor entry plus the buffers it points into, kept alive with it. + struct Coin { + entry: WalletUtxoEntryFFI, + _address: CString, + script: Vec, + } + + struct CoinSpec { + type_tag: AccountTypeTagFFI, + standard_tag: StandardAccountTypeTagFFI, + index: u32, + registration_index: u32, + key_class: u32, + identities: [[u8; 32]; 2], + txid: [u8; 32], + vout: u32, + value: u64, + address: &'static str, + script: Vec, + height: u32, + /// confirmed, instantlocked, coinbase, locked. + flags: [bool; 4], + } + + impl CoinSpec { + fn build(self) -> Coin { + let address = CString::new(self.address).expect("fixture address has no NUL"); + let script = self.script; + let entry = WalletUtxoEntryFFI { + type_tag: self.type_tag, + standard_tag: self.standard_tag, + index: self.index, + registration_index: self.registration_index, + key_class: self.key_class, + user_identity_id: self.identities[0], + friend_identity_id: self.identities[1], + outpoint: platform_wallet_ffi::OutPointFFI { + txid: self.txid, + vout: self.vout, + }, + value_duffs: self.value, + address: address.as_ptr() as *mut c_char, + // Null when empty, as the accessor hands it over. + script_pubkey: if script.is_empty() { + ptr::null_mut() + } else { + script.as_ptr() as *mut u8 + }, + script_pubkey_len: script.len(), + height: self.height, + is_confirmed: self.flags[0], + is_instantlocked: self.flags[1], + is_coinbase: self.flags[2], + is_locked: self.flags[3], + }; + Coin { + entry, + _address: address, + script, + } + } + } + + const NO_IDENTITY: [[u8; 32]; 2] = [[0u8; 32]; 2]; + const DASHPAY_IDENTITIES: [[u8; 32]; 2] = [[0xa1; 32], [0xb2; 32]]; + + /// The four golden coins: a zero-valued BIP44 account, non-default + /// account fields with every flag set, a registration index with no + /// address form, and a DashPay account with both identity halves. + fn golden_coins() -> Vec { + vec![ + CoinSpec { + type_tag: AccountTypeTagFFI::Standard, + standard_tag: StandardAccountTypeTagFFI::Bip44, + index: 0, + registration_index: 0, + key_class: 0, + identities: NO_IDENTITY, + txid: txid(0x00), + vout: 0, + value: 19_549, + address: "yStxXHHzhAx58JhaPBNhn3xsH93UwBM2nd", + script: p2pkh(0x5b), + height: 1_534_921, + flags: [true, false, false, false], + } + .build(), + CoinSpec { + type_tag: AccountTypeTagFFI::Standard, + standard_tag: StandardAccountTypeTagFFI::Bip32, + index: 2, + registration_index: 0, + key_class: 0, + identities: NO_IDENTITY, + txid: txid(0x20), + vout: 7, + value: 5_000_000_000, + address: "yGoldenFixtureBip32Addr", + script: p2pkh(0xc3), + height: 1_400_000, + flags: [true, true, true, true], + } + .build(), + CoinSpec { + type_tag: AccountTypeTagFFI::IdentityTopUp, + standard_tag: StandardAccountTypeTagFFI::Bip44, + index: 0, + registration_index: 4, + key_class: 0, + identities: NO_IDENTITY, + txid: txid(0x40), + vout: 1, + value: 100_000, + address: "", + script: Vec::new(), + height: 0, + flags: [false, false, false, false], + } + .build(), + CoinSpec { + type_tag: AccountTypeTagFFI::DashpayReceivingFunds, + standard_tag: StandardAccountTypeTagFFI::Bip44, + index: 1, + registration_index: 0, + key_class: 0, + identities: DASHPAY_IDENTITIES, + txid: txid(0x60), + vout: 3, + value: 250_000, + address: "yGoldenFixtureDashPayAddr", + script: p2pkh(0xd4), + height: 1_534_000, + flags: [true, true, false, false], + } + .build(), + ] + } + + fn rows(coins: &[Coin]) -> Vec { + coins + .iter() + // SAFETY: every pointer in a `Coin` entry points into that coin's + // own live buffers. + .map(|c| unsafe { UtxoPageRow::from_entry(&c.entry) }) + .collect() + } + + // ── Inventory page (Rust → Kotlin) ──────────────────────────────── + + #[test] + fn inventory_page_serializes_to_the_golden_page() { + let page = UtxoPage::new(rows(&golden_coins()), true).expect("page encodes"); + assert_eq!( + serde_json::to_value(&page).expect("page serializes"), + golden()["inventoryPage"], + "the inventory page drifted from the golden fixture the Kotlin decoder is tested against" + ); + } + + #[test] + fn a_last_or_empty_page_carries_no_cursor() { + let last = UtxoPage::new(rows(&golden_coins()), false).expect("page encodes"); + assert!(!last.has_more); + assert!(last.cursor.is_none()); + let value = serde_json::to_value(&last).expect("page serializes"); + assert_eq!(value["cursor"], serde_json::Value::Null); + assert_eq!(value["hasMore"], serde_json::Value::Bool(false)); + + // An empty page has nowhere to resume from, whatever the accessor said. + let empty = UtxoPage::new(Vec::new(), true).expect("page encodes"); + assert!(!empty.has_more); + assert!(empty.cursor.is_none()); + } + + #[test] + fn the_golden_cursor_resumes_after_the_last_row() { + let coins = golden_coins(); + let last = &coins.last().expect("golden coins").entry; + let raw = golden()["inventoryPage"]["cursor"] + .as_str() + .expect("golden cursor is a string") + .to_owned(); + + let resume = UtxoPageCursor::decode(&raw).expect("golden cursor decodes"); + + assert_eq!(resume.type_tag, last.type_tag as u8); + assert_eq!(resume.standard_tag, last.standard_tag as u8); + assert_eq!(resume.index, last.index); + assert_eq!(resume.registration_index, last.registration_index); + assert_eq!(resume.key_class, last.key_class); + assert_eq!(resume.user_identity_id, last.user_identity_id); + assert_eq!(resume.friend_identity_id, last.friend_identity_id); + assert_eq!(resume.outpoint.txid, last.outpoint.txid); + assert_eq!(resume.outpoint.vout, last.outpoint.vout); + } + + #[test] + fn a_cursor_round_trips_every_account_field() { + let coin = CoinSpec { + type_tag: AccountTypeTagFFI::PlatformPayment, + standard_tag: StandardAccountTypeTagFFI::Bip32, + index: 7, + registration_index: 9, + key_class: 2, + identities: [[0x0c; 32], [0x0d; 32]], + txid: txid(0x80), + vout: 65_535, + value: 1, + address: "yGoldenFixtureCursorAddr", + script: p2pkh(0x01), + height: 2, + flags: [false, true, false, true], + } + .build(); + let row = rows(std::slice::from_ref(&coin)).remove(0); + let raw = serde_json::to_string(&UtxoPageCursor::after(&row)).expect("cursor encodes"); + + let resume = UtxoPageCursor::decode(&raw).expect("own cursor decodes"); + + let e = &coin.entry; + assert_eq!( + ( + resume.type_tag, + resume.standard_tag, + resume.index, + resume.registration_index, + resume.key_class + ), + (14, 1, 7, 9, 2) + ); + assert_eq!(resume.user_identity_id, e.user_identity_id); + assert_eq!(resume.friend_identity_id, e.friend_identity_id); + assert_eq!(resume.outpoint.txid, e.outpoint.txid); + assert_eq!(resume.outpoint.vout, e.outpoint.vout); + } + + #[test] + fn a_cursor_this_layer_did_not_emit_is_rejected() { + let good: serde_json::Value = serde_json::from_str( + golden()["inventoryPage"]["cursor"] + .as_str() + .expect("golden cursor is a string"), + ) + .expect("golden cursor is JSON"); + let tampered = |key: &str, value: serde_json::Value| { + let mut c = good.clone(); + c[key] = value; + c.to_string() + }; + let mut missing_index = good.clone(); + missing_index + .as_object_mut() + .expect("cursor is an object") + .remove("index"); + + for (label, raw) in [ + ("not a cursor object", "7".to_owned()), + ("missing index", missing_index.to_string()), + ("short txid", tampered("txid", "00".repeat(31).into())), + ("non-hex txid", tampered("txid", "zz".repeat(32).into())), + ( + "short identity", + tampered("userIdentityId", "a1".repeat(31).into()), + ), + ( + "odd-length identity", + tampered("friendIdentityId", "b".repeat(63).into()), + ), + ] { + assert!( + UtxoPageCursor::decode(&raw).is_none(), + "{label} must be rejected" + ); + } + } + + // ── Classification queries (Kotlin → Rust) ──────────────────────── + + fn golden_queries() -> serde_json::Value { + golden()["outpointQueries"].clone() + } + + #[test] + fn golden_queries_decode_to_the_inventory_coins() { + let parsed: Vec = + serde_json::from_value(golden_queries()).expect("golden queries deserialize"); + let coins = golden_coins(); + assert_eq!(parsed.len(), coins.len()); + + for (q, coin) in parsed.iter().zip(&coins) { + let e = &coin.entry; + // The tuple the store sends is the tuple the inventory emitted. + assert_eq!(q.account, UtxoAccountTuple::from_entry(e)); + let (ffi, script) = q.to_ffi().expect("golden query converts"); + assert_eq!(ffi.type_tag, e.type_tag as u8); + assert_eq!(ffi.standard_tag, e.standard_tag as u8); + assert_eq!(ffi.index, e.index); + assert_eq!(ffi.registration_index, e.registration_index); + assert_eq!(ffi.key_class, e.key_class); + assert_eq!(ffi.user_identity_id, e.user_identity_id); + assert_eq!(ffi.friend_identity_id, e.friend_identity_id); + assert_eq!( + ffi.outpoint.txid, e.outpoint.txid, + "same byte order both ways" + ); + assert_eq!(ffi.outpoint.vout, e.outpoint.vout); + assert_eq!(script, coin.script); + assert!( + ffi.script_pubkey.is_null(), + "the caller fills the script pointer" + ); + } + } + + #[test] + fn a_query_missing_any_required_field_fails_the_batch() { + // Mirrors Kotlin's `FakeEngine.RUST_REQUIRED_QUERY_KEYS`: every field + // without a serde default, zero-valued ones included (kotlinx drops a + // property equal to its declared default, which once sent BIP44 + // account-0 queries without `standardTag`). + for key in [ + "typeTag", + "standardTag", + "index", + "registrationIndex", + "keyClass", + "txid", + "vout", + ] { + let mut batch = golden_queries(); + batch[0] + .as_object_mut() + .expect("query is an object") + .remove(key); + let err = serde_json::from_value::>(batch) + .err() + .unwrap_or_else(|| panic!("a query without `{key}` must not deserialize")); + assert!( + err.to_string().contains(&format!("missing field `{key}`")), + "{key}: {err}" + ); + } + } + + #[test] + fn malformed_query_fields_are_rejected_by_name() { + let dashpay = 3; + let cases: [(&str, serde_json::Value, &str); 5] = [ + ( + "userIdentityId", + "a1".repeat(31).into(), + "outpoint query carries a malformed identity id", + ), + ( + "friendIdentityId", + "zz".repeat(32).into(), + "outpoint query carries a malformed identity id", + ), + ( + "txid", + "60".repeat(31).into(), + "outpoint query carries a malformed txid", + ), + ( + "txid", + "6".repeat(64 + 1).into(), + "outpoint query carries a malformed txid", + ), + ( + "scriptHex", + "76a".into(), + "outpoint query carries a malformed script", + ), + ]; + for (key, value, expected) in cases { + let mut batch = golden_queries(); + batch[dashpay][key] = value; + let parsed: Vec = + serde_json::from_value(batch).expect("still well-formed JSON"); + assert_eq!( + parsed[dashpay].to_ffi().err(), + Some(expected), + "{key} must be rejected" + ); + } + } #[test] fn should_route_each_fund_scope_to_its_own_ffi() { From 10f2208f80822d35a68f8581595ee2a48ed7c387 Mon Sep 17 00:00:00 2001 From: HashEngineering Date: Thu, 5 Nov 2026 10:57:19 -0800 Subject: [PATCH 2/6] refactor(kotlin-sdk): ask the Rust wallet for every TXO reconcile decision The Kotlin reconcile keeps only its Room reads and writes and the checks that need the store: the heal address check, unresolved-account counting, contact (DIP-15) exclusions, the stale-round refusal, one launched run per wallet, cancellation, and the privacy-safe summary. Scheduling, the gates, heal eligibility, classify verdicts, the page budget and the resume point come from rs-platform-wallet's TxoReconciler, as they do for Swift. JNI: bindings for due wallets, begin and finish, the heal page, classify, the held-verdict release, the applied reports and the stale-page retry rule, ported from #5259 without its chain-truncation code. The utxo page, classify and synced-height bindings this replaces are removed; the inventory golden fixture now carries the heal verdict and budget flag. Each page's engine guard is freed in a non-cancellable `finally` before the page is reported applied, and every run frees its guard and then releases its claim, including on failure, cancellation or an exception. Co-Authored-By: Claude Opus 5.5 --- .../dashsdk/ffi/WalletManagerNative.kt | 203 +++- .../PlatformWalletPersistenceHandler.kt | 831 +++++++------ .../dashsdk/wallet/PlatformWalletManager.kt | 386 +++--- .../dashsdk/wallet/TxoReconcile.kt | 248 ++-- .../PlatformWalletPersistenceHandlerTest.kt | 678 +++++++---- .../dashsdk/wallet/TxoReconcileRunEndTest.kt | 52 + .../dashsdk/wallet/TxoReconcileTest.kt | 336 ++---- .../golden/txo_reconcile_transport_v1.json | 18 +- .../rs-unified-sdk-jni/src/wallet_manager.rs | 1069 ++++++++++++----- 9 files changed, 2308 insertions(+), 1513 deletions(-) create mode 100644 packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcileRunEndTest.kt diff --git a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/ffi/WalletManagerNative.kt b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/ffi/WalletManagerNative.kt index f22f6877742..1becaa273fd 100644 --- a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/ffi/WalletManagerNative.kt +++ b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/ffi/WalletManagerNative.kt @@ -155,26 +155,114 @@ internal object WalletManagerNative { gapLimit: Int, ) + // ── TXO-store reconcile (dashpay/platform#4638) ───────────────────── + // + // One export = one `platform-wallet` call. The schedule, the start + // gates, the heal verdicts, the stale-page retry limit, the page budget + // and where a run resumes all live in + // `platform_wallet::manager::txo_reconcile`; Kotlin pages its own Room + // rows, resolves owners, writes, and reports each page it applied. + + /** [walletManagerTxoReconcileBegin]: the run may start + * (`TXO_RECONCILE_GATE_RUN`). */ + const val TXO_RECONCILE_GATE_RUN = 0 + + /** [walletManagerTxoReconcileBegin]'s `outStart[0]`, mirroring + * rs-platform-wallet-ffi `TXO_RECONCILE_START_*`: a new cycle at the + * heal pass. */ + const val TXO_RECONCILE_START_HEAL = 0 + + /** The heal pass, where the wallet's last run stopped. The native + * reconciler keeps that place; the host passes no cursor. */ + const val TXO_RECONCILE_START_HEAL_RESUMED = 1 + + /** Straight to the classify pass, after the store cursor + * [walletManagerTxoReconcileBegin] hands back in `outStoreAfter[0]`. */ + const val TXO_RECONCILE_START_CLASSIFY = 2 + + /** The longest store cursor + * [walletManagerTxoReconcileClassifyPageApplied] keeps for the + * wallet's next run (rs-platform-wallet-ffi + * `TXO_RECONCILE_STORE_CURSOR_MAX`). */ + const val TXO_RECONCILE_STORE_CURSOR_MAX = 64 + + /** + * The wallets whose reconcile is due now, as concatenated 32-byte ids + * (`platform_wallet_manager_txo_reconcile_due_wallets`). Called on every + * SPV progress poll, running or not: the poll is the schedule's clock, + * and the native side detects the steady-state edge and applies the + * cadence. Reporting a wallet consumes nothing: its cadence restarts + * only when [walletManagerTxoReconcileBegin] claims it, so a wallet the + * host does not launch is reported again on the next poll. + */ + external fun walletManagerTxoReconcileDueWallets(managerHandle: Long): ByteArray? + + /** + * Claim [walletId] for one reconcile run and check that it may start + * (`platform_wallet_manager_txo_reconcile_begin`). Returns + * [TXO_RECONCILE_GATE_RUN] or a `TXO_RECONCILE_SKIP_*` reason (see + * [org.dashfoundation.dashsdk.wallet.TxoReconcileSkipReason.fromGateCode]); + * on [TXO_RECONCILE_GATE_RUN] `outTipHeight[0]` is the scan tip the run + * is measured against (the native claim keeps it), and the claim holds + * until [walletManagerTxoReconcileFinish]. + * + * On [TXO_RECONCILE_GATE_RUN] `outStart[0]` is where the run starts, as + * the native reconciler decides it from the wallet's checkpoint: + * [TXO_RECONCILE_START_HEAL], [TXO_RECONCILE_START_HEAL_RESUMED] or + * [TXO_RECONCILE_START_CLASSIFY]. For the last, `outStoreAfter[0]` is + * the store cursor to start after, the bytes the host last passed to + * [walletManagerTxoReconcileClassifyPageApplied] (empty: the first + * row); otherwise it is null. [outTipHeight] and [outStart] are + * `IntArray(1)`, [outStoreAfter] an `arrayOfNulls(1)`, + * checked before the claim is taken. + */ + external fun walletManagerTxoReconcileBegin( + managerHandle: Long, + walletId: ByteArray, + outTipHeight: IntArray, + outStart: IntArray, + outStoreAfter: Array, + ): Int + + /** + * Release the claim a [TXO_RECONCILE_GATE_RUN] from + * [walletManagerTxoReconcileBegin] granted; [ran] is whether the run went + * ahead, which restarts the wallet's cadence + * (`platform_wallet_manager_txo_reconcile_finish`). + */ + external fun walletManagerTxoReconcileFinish(managerHandle: Long, walletId: ByteArray, ran: Boolean) + + /** + * Whether a reconcile run classifies a page again after it came back + * stale for the [staleInARow]-th time in a row, or stops incomplete + * (`platform_wallet_txo_reconcile_retry_stale_page`, which holds the + * limit). + */ + external fun txoReconcileRetryStalePage(staleInARow: Int): Boolean + /** * One bounded page of the engine's UTXO inventory for one wallet, - * across every funds account, as JSON - * `{"utxos":[...],"cursor":,"hasMore":}` — the - * source of truth the TXO-store reconciler - * ([PlatformWalletManager.reconcileTxoStore]) diffs against the Room - * `txos` mirror. A serialization shim over one Rust call - * (`platform_wallet_wallet_utxos_page`): account ordering, cursor - * semantics, the page bound and the address rendering all live in - * `platform-wallet`, so the Swift host walks the identical inventory. + * across every funds account, each row with the engine's heal verdict + * (measured against the run's own tip, which the native claim keeps), as + * JSON `{"utxos":[...],"hasMore":,"budgetSpent":}`. A + * serialization shim over one Rust call + * (`platform_wallet_txo_reconcile_heal_page`): account ordering, the + * run's place in the inventory, the page bound, the page budget, the + * address rendering and the verdicts all live in `platform-wallet`. * * Paged, not swept whole: a wallet's UTXO count is chain-controlled * (anyone who knows a watched address can keep sending dust to it), so * a full-inventory read would let a remote party decide how much this - * process allocates on every steady-state transition and every - * 30-minute pass. Pass [cursor] `null` to start, then hand back the returned - * `cursor` verbatim while `hasMore` is true — a cursor this export did - * not produce throws. [limit] caps the rows in one page; non-positive - * asks for the engine's own default, and the engine clamps its own - * maximum, so no bound is mirrored on this side. + * process allocates on every steady-state transition and every cadence + * pass. The native side keeps the run's place, so the host passes no + * cursor: each page starts after the last one reported with + * [walletManagerTxoReconcileHealPageApplied] (or where the wallet's last + * run stopped), and a page read again before it is reported is the same + * page. Keep going while `hasMore` is true. Each call is one page of the + * run's budget; `budgetSpent` true means nothing was read and nothing + * is held: the run has used its budget and stops. [limit] caps the rows + * in one page; non-positive asks for the engine's own default, and the + * engine clamps its own maximum, so no bound is mirrored on this side. * * Contact watch-only chains (`DashpayExternalAccount`) are absent by * construction — those coins are the contact's money, not this @@ -186,16 +274,37 @@ internal object WalletManagerNative { * changeset path hands the handler (so hex→bytes reproduces the * `txos.txid` blob), vout, amount (duffs), the engine's own * Base58Check address (empty when the script has no address form), - * scriptHex, height, and the engine's own `isConfirmed` / - * `isInstantLocked` / `isCoinbase` / `isLocked` flags. + * scriptHex, height, the engine's own `isConfirmed` / + * `isInstantLocked` / `isCoinbase` / `isLocked` flags, and its + * `healVerdict` (`TXO_HEAL_VERDICT_*`). + * + * Only for the wallet's claimed reconcile run. The page comes back with + * the engine's wallet lock still held for reading, so no fork can drop a + * coin it offers before the page is written: the guard is written to + * [outHeld] (a `LongArray(1)`; 0 when nothing is held) only together + * with the page, and is released with [txoReconcileHeldVerdictsFree] + * under the rules [walletManagerTxoReconcileClassifyOutpoints] gives. */ - external fun walletManagerUtxosPageJson( + external fun walletManagerTxoHealPageJson( managerHandle: Long, walletId: ByteArray, - cursor: String?, limit: Int, + outHeld: LongArray, ): String? + /** + * Report that the heal page [walletManagerTxoHealPageJson] last returned + * was applied: its store step committed (a page with nothing to write + * counts as applied) and its guard was released. The run's next heal + * page starts after it, and so does the wallet's next run if this one + * stops here (`platform_wallet_txo_reconcile_heal_page_applied`). + * Returns whether the run has budget left for another page. + * + * Resolves [managerHandle] in the native registry, so it is never called + * while a page's guard is held: release first, then report. + */ + external fun walletManagerTxoReconcileHealPageApplied(managerHandle: Long, walletId: ByteArray): Boolean + /** * Classify a batch of store rows against the engine's live state: the * reverse half of the reconcile transport, and the reason the paged @@ -217,26 +326,64 @@ internal object WalletManagerNative { * 0 unknown, 1 unspent, 2 known-uncredited, 3 not-owned. A query whose * account tag this build cannot map keeps `unknown`; the rest are * still answered. + * + * Only for the wallet's claimed reconcile run (between + * [walletManagerTxoReconcileBegin] and [walletManagerTxoReconcileFinish]). + * The answers come back with the engine's wallet lock still held for + * reading, so no fork can drop a spender they name before the page is + * applied. The guard is written to [outHeld] (a `LongArray(1)`; 0 when + * nothing is held) only together with the answers. Release it with + * [txoReconcileHeldVerdictsFree] once the page is applied and before + * the next call that takes [managerHandle]: those resolve the handle in + * the native registry, which must never be waited for while the guard + * is held. Finish releases a guard left behind, as a backstop. Call it + * outside the persistence handler's callback exclusion and never from + * inside a persistence callback: the engine calls those callbacks while + * it holds its wallet lock for writing. + * + * Called for every store page the classify pass reads, one with no + * queries (`"[]"`) too: each call is one page of the run's budget, holds + * a guard like any other, and names the page + * [walletManagerTxoReconcileClassifyPageApplied] reports. Once the run + * has used its budget `outBudgetSpent[0]` is true, every answer is + * unknown and nothing is held: the run stops. [outBudgetSpent] is a + * `BooleanArray(1)`. */ - external fun walletManagerClassifyOutpoints( + external fun walletManagerTxoReconcileClassifyOutpoints( managerHandle: Long, walletId: ByteArray, queriesJson: String, + outBudgetSpent: BooleanArray, + outHeld: LongArray, ): ByteArray? /** - * The wallet's core scan watermark as the ENGINE holds it - * (`platform_wallet_core_wallet_state`'s `synced_height`) — the value the - * TXO-store reconcile gates on, as Swift's does. The persisted - * `syncedHeight` in Room trails it by up to one persistence round, which - * is exactly the window the SYNCED transition lands in. Throws - * [org.dashfoundation.dashsdk.errors.DashSdkError] `NotFound` when the - * manager does not hold the wallet. + * Report that the store page [walletManagerTxoReconcileClassifyOutpoints] + * last classified was applied and its guard released + * (`platform_wallet_txo_reconcile_classify_page_applied`). [nextAfter] is + * the host's store cursor past that page, at most + * [TXO_RECONCILE_STORE_CURSOR_MAX] bytes, which + * [walletManagerTxoReconcileBegin] hands back to the wallet's next run; + * null when that was the store's last page, which ends the cycle (the + * next run starts at the heal pass again). Returns whether the run has + * budget left for another page. + * + * Resolves [managerHandle] in the native registry, so it is never called + * while a page's guard is held: release first, then report. */ - external fun walletManagerCoreWalletSyncedHeight( + external fun walletManagerTxoReconcileClassifyPageApplied( managerHandle: Long, walletId: ByteArray, - ): Long + nextAfter: ByteArray?, + ): Boolean + + /** + * Release the engine guard [walletManagerTxoHealPageJson] or + * [walletManagerTxoReconcileClassifyOutpoints] handed out. Takes no manager handle and touches no native registry, + * so it completes whatever is queued there. 0 is a no-op; release each + * value once. + */ + external fun txoReconcileHeldVerdictsFree(held: Long) // ── Core transaction builder (1:1 over `core_wallet_tx_builder_*`) ─ // diff --git a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandler.kt b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandler.kt index 6d54a69a5df..2615dc7fb94 100644 --- a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandler.kt +++ b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandler.kt @@ -3,6 +3,7 @@ package org.dashfoundation.dashsdk.persistence import android.util.Log import androidx.room.withTransaction import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.NonCancellable import kotlinx.coroutines.ExecutorCoroutineDispatcher import kotlinx.coroutines.asCoroutineDispatcher import kotlinx.coroutines.flow.MutableStateFlow @@ -13,6 +14,7 @@ import kotlinx.coroutines.flow.update import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext import kotlinx.serialization.Serializable import kotlinx.serialization.json.Json import org.dashfoundation.dashsdk.errors.DashSdkError @@ -482,32 +484,6 @@ class PlatformWalletPersistenceHandler( */ private var committedRoundGeneration = 0L - /** - * Where each wallet's next [reconcileTxos] run picks up (keyed by wallet - * id hex), so a run that stops on its page budget, or for any other - * reason, does not make the next one walk the pages it already finished. - * Written after every finished page (never for a page whose step failed - * or was cancelled) and removed when the classify pass reaches its end, - * which makes the run after that start a new cycle. - * - * Each entry carries the scope it was written under (see - * [ReconcileCheckpoint.scope]); a run passing a different scope ignores - * it and starts a new cycle. The manager passes its reconcile epoch, - * which moves on every wallet removal and on close, so a place left by - * a removed wallet is never resumed by a wallet added again under the - * same id. A new manager builds a new handler, so a place never - * outlives the engine whose cursor it holds. - * - * In memory on purpose: a fresh process starts a new cycle, which is - * safe (the passes are idempotent) and costs at most one budget per run. - * Dropped with the wallet's other in-memory state by - * [deleteWalletDataLocked]. Read and written only under - * [callbackExclusion]; a write re-checks the run's cancellation under - * it, so a run that a wallet deletion cancelled cannot leave an entry - * behind after the deletion dropped it. - */ - private val txoReconcileCheckpoints = HashMap() - /** The open round for [walletId], or null on the standalone-callback path. */ private fun openRound(walletId: ByteArray): ChangesetBuffer? = buffers[walletId.toHex()] @@ -1452,19 +1428,22 @@ class PlatformWalletPersistenceHandler( } /** - * One row of the engine's paged UTXO inventory, exactly as - * `walletManagerUtxosPageJson` serializes it (rs-unified-sdk-jni - * `UtxoPageRow`, serde on that side, kotlinx on this one). The contract - * is typed on BOTH ends so a renamed or missing key fails the decode - * loudly instead of healing a row with a defaulted owner or a zero - * amount into the mirror the engine reloads from. The account tuple - * (`typeTag`…`friendIdentityId`) is the routing context - * [findReconcileAccount] resolves the owning Room account by, so every - * field of it is required except the DashPay identity halves, which are - * absent on every non-DashPay account. Every other field is required - * too: the address, the script and the engine's flags are written into - * a healed row as given, so a missing one must not decode as an empty - * address, an empty script or a guessed flag. + * One row of a heal page, exactly as `walletManagerTxoHealPageJson` + * serializes it (rs-unified-sdk-jni `UtxoPageRow`, serde on that side, + * kotlinx on this one). The contract is typed on BOTH ends so a renamed + * or missing key fails the decode loudly instead of healing a row with a + * defaulted owner or a zero amount into the mirror the engine reloads + * from. The account tuple (`typeTag`…`friendIdentityId`) is the routing + * context [findReconcileAccount] resolves the owning Room account by, so + * every field of it is required except the DashPay identity halves, + * which are absent on every non-DashPay account. Every other field is + * required too: the address, the script and the engine's flags are + * written into a healed row as given, so a missing one must not decode + * as an empty address, an empty script or a guessed flag. [healVerdict] + * is the engine's decision whether the store may heal the coin in + * (`TXO_HEAL_VERDICT_*`, `platform_wallet::manager::txo_reconcile`) — + * confirmations, maturity and well-formedness are its call, not this + * side's. */ @Serializable data class EngineUtxoRow( @@ -1488,16 +1467,17 @@ class PlatformWalletPersistenceHandler( val isInstantlocked: Boolean, val isCoinbase: Boolean, val isLocked: Boolean, + val healVerdict: Int, ) /** - * One store row handed to `walletManagerClassifyOutpoints`: the account - * tuple the STORE files the coin under, its outpoint, and the script - * the store recorded for it. The account and script are the store's - * claim — the engine checks both against its own pools rather than - * trusting them, which is why a bare outpoint cannot ask the question. - * Serialized against the same field names the Rust `OutpointQuery` - * deserializes. + * One store row handed to `walletManagerTxoReconcileClassifyOutpoints`: + * the account tuple the STORE files the coin under, its outpoint, and + * the script the store recorded for it. The account and script are the + * store's claim — the engine checks both against its own pools rather + * than trusting them, which is why a bare outpoint cannot ask the + * question. Serialized against the same field names the Rust + * `OutpointQuery` deserializes. * * Only the identity halves have defaults. The Rust tuple requires every * other account field, and kotlinx's default `Json` omits a property @@ -1520,24 +1500,108 @@ class PlatformWalletPersistenceHandler( ) /** - * One page of the engine's UTXO inventory: the rows, the opaque resume - * cursor (null on the last page) and whether more pages follow. All - * three keys are required, and a page that says more follow without a - * cursor to resume from fails the decode: either would otherwise read - * as the last page and end the heal walk early, leaving later engine - * coins unrepaired. + * One heal page: the rows, whether more pages follow, and whether the + * run had already used its page budget (`budgetSpent`), in which case + * nothing was read. The page carries no cursor: the native reconciler + * keeps the run's place and moves it past a page only once the page is + * reported applied. + * + * All three keys are required: a missing `hasMore` would read as the + * last page and end the heal walk early, leaving later engine coins + * unrepaired, and a missing `budgetSpent` would read a budget stop as an + * empty last page. A budget-spent page that carries rows, or says more + * follow, fails the decode too: it would be a read the native side says + * it did not make. */ @Serializable data class EngineUtxoPage( val utxos: List, - val cursor: String?, val hasMore: Boolean, + val budgetSpent: Boolean, ) { init { - require(!hasMore || cursor != null) { "an inventory page with more to come carries no cursor" } + require(!budgetSpent || (utxos.isEmpty() && !hasMore)) { + "an inventory page past the run's budget carries a read" + } } } + /** + * Where one [reconcileTxos] run starts, as the native reconciler decides + * it from the wallet's checkpoint when it grants the run's claim + * (`walletManagerTxoReconcileBegin`'s `TXO_RECONCILE_START_*`). The + * checkpoint is native: it moves only past a page the host reported + * applied, a fresh process starts a new cycle, and the wallet's removal + * drops it. + */ + sealed class TxoReconcileStart { + /** A new cycle: the heal pass from the engine's first page, then + * the classify pass from the store's first row. */ + object NewCycle : TxoReconcileStart() + + /** The heal pass, where the wallet's last run stopped. The native + * reconciler keeps that place, so nothing is passed here. */ + object HealResumed : TxoReconcileStart() + + /** Straight to the classify pass, at the first unspent row after + * the store cursor [after]: the bytes an earlier run reported with + * its last applied classify page (empty: the first row). */ + class Classify(val after: ByteArray) : TxoReconcileStart() + } + + /** + * One classification, as [TxoReconcileEngine.classify] returns it: one + * verdict byte per query, positionally, or [budgetSpent] when the run + * had already used its page budget, in which case nothing was + * classified or held and [classes] means nothing. + */ + class OutpointVerdicts(val classes: ByteArray, val budgetSpent: Boolean = false) + + /** + * The engine side of one claimed [reconcileTxos] run — the Kotlin twin + * of Swift's `CoreTxoReconcileEngine`. Every decision is the engine's + * (`platform_wallet::manager::txo_reconcile`): which coins may be + * healed, which rows are proven spent, how many pages the run may read, + * where its next page starts and how often a stale page is classified + * again. The manager implements it over the native calls; tests over a + * fake. + * + * [healPage] and [classify] answer under a read guard on the engine's + * wallet lock that stays held until [releaseVerdicts], so the page is + * written before any fork can move the engine past it. Neither may be + * called inside [callbackExclusion] or from a persistence callback, and + * [healPageApplied], [classifyPageApplied] and the next read resolve the + * native manager handle, so they are only ever called after + * [releaseVerdicts]. A null answer is a failed call: the run stops and + * counts it. + */ + interface TxoReconcileEngine { + /** One heal page as JSON ([EngineUtxoPage]), held. The engine keeps + * the run's place: the same page until it is reported applied. */ + suspend fun healPage(limit: Int): String? + + /** The heal page last read is applied. Whether the run has budget + * left for another page. */ + suspend fun healPageApplied(): Boolean? + + /** The verdicts on one store page's queries (JSON + * [OutpointQuery] array, `[]` too), held. */ + suspend fun classify(queriesJson: String): OutpointVerdicts? + + /** The store page last classified is applied; [nextAfter] is the + * store cursor past it, null on the store's last page. Whether the + * run has budget left for another page. */ + suspend fun classifyPageApplied(nextAfter: ByteArray?): Boolean? + + /** Release the guard of the last page read, if any. Never resolves + * the native manager handle and never waits. */ + suspend fun releaseVerdicts() + + /** Whether a page that came back stale for the [staleInARow]-th time + * in a row is classified again, or the run stops. */ + fun retryStalePage(staleInARow: Int): Boolean + } + /** * What one [reconcileTxos] run did — the Kotlin twin of Swift's * `CoreTxoReconcileReport` (dashpay/platform#4638). Counts only: no @@ -1559,8 +1623,8 @@ class PlatformWalletPersistenceHandler( /** Engine coins the store lacked whose row the pending-input drain * wrote spent on insert: the divergence is recorded, not repaired. */ val healedSpent: Int = 0, - /** Engine coins the engine does not call confirmed, or below the - * confirmation gate — they age into a later run. */ + /** Engine coins the engine's heal verdict calls immature (not + * confirmed, or not deep enough yet) — they age into a later run. */ val skippedImmature: Int = 0, /** Engine coins whose account has no store row to file them under. * Never inserted unowned: the restore loader routes by account, so @@ -1572,8 +1636,9 @@ class PlatformWalletPersistenceHandler( * path and no build could sign for it. Healed by a later run once * the address is persisted under that account. */ val skippedMissingAddress: Int = 0, - /** Engine coins the store could not validate (txid not 32 bytes, no - * script, no address). */ + /** Engine coins the engine's heal verdict calls invalid, or whose + * fields this store could not decode (a txid that is not 32 bytes + * of hex, a malformed script or identity half). */ val skippedInvalid: Int = 0, /** Watch-only DIP-15 contact coins excluded on both sides — engine * coins the heal pass refused to file as ours, and unspent store @@ -1602,8 +1667,9 @@ class PlatformWalletPersistenceHandler( /** Pages classified again because a persistence round committed * between their read and their apply, or was still open at it. */ val staleRetries: Int = 0, - /** Engine reads that failed (an inventory page, the first one - * included, or a classification batch); the run stops at the first. */ + /** Engine calls that failed (an inventory page, the first one + * included, a classification batch, or a page's applied report); + * the run stops at the first. */ val transportFailures: Int = 0, /** Store steps that failed and rolled back; the run stops at the first. */ val storeFailures: Int = 0, @@ -1625,10 +1691,10 @@ class PlatformWalletPersistenceHandler( * run left over, not a walk of the whole inventory in this run. */ val completed: Boolean = true, - /** The run used its whole page budget - * ([TXO_RECONCILE_PAGE_BUDGET]) before the end of the cycle and - * stopped cleanly between pages. The wallet's next run picks up at - * the first page this one did not finish; [completed] is `false`. */ + /** The run used the page budget the engine gives one run before the + * end of the cycle and stopped cleanly between pages. The wallet's + * next run picks up at the first page this one did not apply; + * [completed] is `false`. */ val stoppedOnBudget: Boolean = false, /** The run picked up where an earlier run of this wallet stopped, * rather than at the first heal page. Its counts cover only the @@ -1663,31 +1729,24 @@ class PlatformWalletPersistenceHandler( var staleRetries = 0 var transportFailures = 0 var storeFailures = 0 - /** Set only where the classify pass reaches its end; every other - * exit leaves the run incomplete. */ + + /** Set only where the classify pass reaches the store's last page; + * every other exit leaves the run incomplete. */ var completed = false var stoppedOnBudget = false var resumedFromCheckpoint = false - /** Pages this run may still read, across both passes. Not reported: - * it is the run's allowance, not something the run found. */ - var pagesLeft = 0 + /** The engine said the run's budget is spent: it stops cleanly + * between pages. */ + fun stopOnBudget() { + stoppedOnBudget = true + completed = false + } - /** - * Take one page from the budget before reading it. `false` when the - * run is cancelled, or when the budget is spent: the run is then - * marked as stopped on its budget. Either way the caller stops - * before the read. Checked between pages only, so a page that was - * read always gets its store step. - */ - fun takePage(isCancelled: () -> Boolean): Boolean { - if (isCancelled()) return false - if (pagesLeft <= 0) { - stoppedOnBudget = true - return false - } - pagesLeft-- - return true + /** An engine call failed: the run stops and says so. */ + fun stopOnTransport() { + transportFailures++ + completed = false } fun toReport() = TxoReconcileReport( @@ -1717,25 +1776,6 @@ class PlatformWalletPersistenceHandler( ) } - /** - * Where the next [reconcileTxos] run of a wallet picks up — see - * [txoReconcileCheckpoints]. No entry means the next run starts a new - * cycle at the first heal page. - */ - private sealed class ReconcileCheckpoint { - /** The `checkpointScope` of the run that wrote it. Only a run with - * the same scope resumes from it. */ - abstract val scope: Long - - /** In the heal pass, at the page after the engine cursor [cursor] - * (the cursor the last finished page returned). */ - class Heal(override val scope: Long, val cursor: String) : ReconcileCheckpoint() - - /** In the classify pass, at the first unspent row after the - * outpoint [after]; an empty [after] is the first row. */ - class Classify(override val scope: Long, val after: ByteArray) : ReconcileCheckpoint() - } - /** Counts from one heal step; merged into the run only once it commits. */ private class HealCounts { var alreadyPresent = 0 @@ -1785,78 +1825,75 @@ class PlatformWalletPersistenceHandler( /** * Reconcile the Room `txos` mirror against the engine — the Kotlin port - * of Swift's `runCoreTxoReconcile` (dashpay/platform#4638 Part 2). The - * mirror is write-behind and is itself the source the engine is - * restored from at every launch ([buildUtxoRestoreData] hands back every - * `isSpent = 0` row), so two failure classes outlive a full scan: a coin - * the engine credited whose row the store never got (the job-flower - * 106.43→86.33 restart drop — rescan dropping the change outputs of - * CoinJoin-funded sends), and a row the store holds unspent for a coin - * the engine never credited (rust-dashcore#992: the spender had no - * wallet-owned output and was discarded before the coin was known). The - * restore hands the second back, and the engine shows a balance it had - * just corrected (dashpay/platform#4575). This repairs exactly what - * positive evidence supports: + * of Swift's `runCoreTxoReconcile` (dashpay/platform#4638 Part 2), for + * one run the native reconciler has granted ([start] is where it says + * the run begins). The mirror is write-behind and is itself the source + * the engine is restored from at every launch ([buildUtxoRestoreData] + * hands back every `isSpent = 0` row), so two failure classes outlive a + * full scan: a coin the engine credited whose row the store never got + * (the job-flower 106.43→86.33 restart drop — rescan dropping the change + * outputs of CoinJoin-funded sends), and a row the store holds unspent + * for a coin the engine never credited (rust-dashcore#992: the spender + * had no wallet-owned output and was discarded before the coin was + * known). The restore hands the second back, and the engine shows a + * balance it had just corrected (dashpay/platform#4575). This repairs + * exactly what positive evidence supports: * - * * [healMissingTxos] pages the ENGINE ([engineUtxoPage]: `cursor` null - * to start, then the cursor the previous page returned while its - * `hasMore` is true) and inserts each coin the store lacks — validated, - * owned, confirmed by the engine, at least [minConfirmations] deep, and - * at an address the store has a saved row for under the coin's account. + * * [healMissingTxos] pages the ENGINE ([TxoReconcileEngine.healPage]) + * and inserts each coin the store lacks that the engine's verdict + * calls eligible, whose owning account the store resolves, and whose + * address the store has a saved row for under that account. * * [classifyStoreRows] pages the wallet's UNSPENT store rows and asks - * [classifyOutpoints] about each page. Only + * the engine about each page ([TxoReconcileEngine.classify]). Only * [OUTPOINT_CLASS_KNOWN_UNCREDITED] — the owning account recorded the * funding transaction, owns the script, does not hold the coin, AND a - * funds account holds a mined record spending it — marks a row spent. + * funds account holds a record spending it in a block the wallet's + * ChainLock covers — marks a row spent. + * + * Every decision is the engine's: which coins may be healed, which rows + * to flip, how many pages one run reads, where the run's next page + * starts and how often a stale page is classified again. This side + * pages its store, resolves owners, applies the host-side checks (the + * DIP-15 contact exclusion, the address row, the round fences) and + * writes. * * It never deletes a row, never un-marks a spent row, and never acts on * absence alone. It is idempotent (a consistent store yields a report * with zero [TxoReconcileReport.mutations]) and bounded in both - * directions: a wallet's UTXO count is chain-controlled, so a pass that - * materialized a whole inventory would let anyone who knows a watched - * address decide how much a phone allocates. + * directions: each page bounds memory, and the engine's page budget + * bounds the work of one run — anyone can send dust to a watched + * address, so a run that always walked the whole inventory would let a + * remote party decide how much a phone does on every automatic run. * - * Each page bounds memory; [pageBudget] bounds the work of the whole run. - * Anyone can send dust to a watched address, so the inventory can grow - * without the owner doing anything, and a run that always walked all of - * it would turn that growth into CPU, battery and callback work on every - * automatic run. So a run reads at most [pageBudget] pages, counting the - * heal pass's engine pages and the classify pass's store pages together - * (a stale page read again counts again). When the budget runs out the - * run stops between pages, after the last page's store step, and - * reports [TxoReconcileReport.stoppedOnBudget] (with `completed = - * false`). + * Each page goes: engine read (held), store step, guard released, then — + * only when the step committed or there was nothing to write — the page + * reported applied, whose answer says whether the run may read another. + * A page that failed, came back stale, was cancelled or did not decode is + * never reported, so the engine serves it again (to this run or the + * wallet's next). The run stops cleanly between pages when a read comes + * back budget-spent or a report leaves no budget before the cycle's end + * ([TxoReconcileReport.stoppedOnBudget]); the classify pass's last store + * page is reported with no cursor, which ends the cycle. * - * The run remembers where it stopped ([txoReconcileCheckpoints]), and the - * wallet's next run with the same [checkpointScope] picks up there - * instead of at the first heal page: in the heal pass, at the engine - * cursor the last finished page returned; in the classify pass, after - * the last finished page's last outpoint. The heal pass reaching its end - * moves the checkpoint to the start of the classify pass, and the - * classify pass reaching its end removes it, so the run after that - * starts a new cycle. The classify pass of a cycle therefore never - * starts before that cycle's heal pass has reached its end, however many - * runs the cycle takes. The checkpoint only ever moves past a page whose - * step finished: a failed, cancelled or stale-aborted page, or one that - * does not decode, is read again by the next run. A large inventory is - * therefore covered over several runs, one budget per run at the - * manager's cadence, instead of all at once on every run. Coins that - * land behind the cursor while a cycle is in progress are picked up by - * the next cycle; both cursors are keys, not offsets, so resuming never - * skips or repeats a row that was there all along. + * Lock order, the engine's own: the native handle registry (read only + * transiently, inside one native call), the engine's wallet lock (held + * for reading from a page's read until [TxoReconcileEngine.releaseVerdicts]), + * [callbackExclusion], the Room transaction. The engine calls the + * persistence callbacks — which take [callbackExclusion] — while it holds + * its wallet lock for writing, so every engine read happens OUTSIDE + * [callbackExclusion] and never from a callback, and each page's guard is + * released after its store step has let go of [callbackExclusion] and + * before anything resolves the manager handle again. The guard is + * released on every path out of a page — a committed step, a stale + * page, a failed or cancelled step, an exception — in a `finally`. * - * Engine reads run OUTSIDE [callbackExclusion] — they park on the - * engine's wallet lock, which a persistence callback may hold while it - * waits for ours. Every store step runs under it, one Room transaction + * Every store step runs under [callbackExclusion], one Room transaction * each, after re-checking [isCancelled]: the manager bumps its epoch on * close and on wallet deletion, and the run stops between pages, - * reporting `completed = false`. - * - * A failed engine read stops the run and is counted, the first page - * included (Swift's contract: the report says the run did not - * complete, rather than there being no report). A page that does not - * decode as an [EngineUtxoPage] is a contract violation between the JNI - * emitter and this reader, and it throws rather than being absorbed. + * reporting `completed = false`. A page that does not decode as an + * [EngineUtxoPage] is a contract violation between the JNI emitter and + * this reader, and it throws (the guard released first) rather than + * being absorbed. * * `netAmount` is never repaired: a record born blind to one of its own * outputs persisted a net short by exactly that output's value, but the @@ -1869,23 +1906,15 @@ class PlatformWalletPersistenceHandler( */ suspend fun reconcileTxos( walletId: ByteArray, - tipHeight: Int, - minConfirmations: Int = TXO_RECONCILE_MIN_CONFIRMATIONS, + start: TxoReconcileStart, + engine: TxoReconcileEngine, pageSize: Int = TXO_RECONCILE_PAGE_SIZE, - pageBudget: Int = TXO_RECONCILE_PAGE_BUDGET, - checkpointScope: Long = 0L, isCancelled: () -> Boolean = { false }, - engineUtxoPage: suspend (cursor: String?, limit: Int) -> String?, - classifyOutpoints: suspend (queriesJson: String) -> ByteArray?, ): TxoReconcileReport { val limit = pageSize.coerceAtLeast(1) val walletRef = logReference(walletId) - val tally = ReconcileTally().apply { pagesLeft = pageBudget.coerceAtLeast(1) } - // A place left under another scope (a removed wallet's, or one from - // before an epoch move) is not this run's to resume. - val checkpoint = callbackExclusion.withLock { txoReconcileCheckpoints[walletId.toHex()] } - ?.takeIf { it.scope == checkpointScope } - tally.resumedFromCheckpoint = checkpoint != null + val tally = ReconcileTally() + tally.resumedFromCheckpoint = start != TxoReconcileStart.NewCycle // Watch-only DIP-15 contact (external) accounts, resolved once up // front because BOTH passes need the exclusion and the account set @@ -1902,22 +1931,18 @@ class PlatformWalletPersistenceHandler( .toSet() } if (foreignAccountIds != null) { - suspend fun heal(from: String?) = healMissingTxos( - walletId, from, checkpointScope, tipHeight, minConfirmations, limit, - foreignAccountIds, isCancelled, engineUtxoPage, tally, walletRef, - ) - val healDone = when (checkpoint) { + val healDone = when (start) { // The heal pass of this cycle already reached its end. - is ReconcileCheckpoint.Classify -> true - is ReconcileCheckpoint.Heal -> heal(checkpoint.cursor) - null -> heal(null) + is TxoReconcileStart.Classify -> true + // A resumed heal pass needs nothing from here: the engine + // serves the page after the last one reported applied. + TxoReconcileStart.NewCycle, TxoReconcileStart.HealResumed -> + healMissingTxos(walletId, limit, foreignAccountIds, isCancelled, engine, tally, walletRef) } if (healDone) { - // An empty BLOB sorts before every real outpoint, so a cycle - // whose heal pass just ended starts at the first row. classifyStoreRows( - walletId, (checkpoint as? ReconcileCheckpoint.Classify)?.after ?: ByteArray(0), checkpointScope, - limit, foreignAccountIds, isCancelled, classifyOutpoints, tally, walletRef, + walletId, classifyStartCursor(start), limit, foreignAccountIds, isCancelled, engine, tally, + walletRef, ) } } @@ -1926,123 +1951,166 @@ class PlatformWalletPersistenceHandler( return report } + /** + * The outpoint the classify pass starts after: the store cursor the + * engine handed back for a [TxoReconcileStart.Classify] start, or the + * empty BLOB, which sorts before every real outpoint (the first row). + * The cursor is this side's own — a 36-byte outpoint it reported with + * an applied page — so anything else (an engine that kept none, or a + * cursor from a build that paged differently) starts at the first row: + * the passes are idempotent, so walking a few rows again costs only + * the walk. + */ + private fun classifyStartCursor(start: TxoReconcileStart): ByteArray = + (start as? TxoReconcileStart.Classify)?.after?.takeIf { it.size == OUTPOINT_BYTES } ?: ByteArray(0) + + /** + * Report the page just applied, after its guard was released (the call + * resolves the native manager handle). Returns whether the run may read + * another page, or null when it must stop: cancelled (the page is left + * unreported, so the engine serves it again; its step is idempotent), + * or the report failed (counted). + */ + private suspend fun reportPageApplied( + tally: ReconcileTally, + isCancelled: () -> Boolean, + applied: suspend () -> Boolean?, + ): Boolean? { + if (isCancelled()) return null + return applied() ?: run { + tally.stopOnTransport() + null + } + } + /** * Heal pass of [reconcileTxos]: one engine page at a time, one store - * step each. Each page is fetched OUTSIDE the exclusion lock — the - * fetch is a native call into the engine, and the lock exists to keep - * changeset callbacks out of our writes, not out of the engine. Returns - * whether the run may go on to the classify pass. + * step each. Each page is read OUTSIDE the exclusion lock — the read is + * a native call into the engine, and the lock exists to keep changeset + * callbacks out of our writes, not out of the engine. Returns whether + * the run may go on to the classify pass: the inventory's end was + * reached with budget left. * - * Starts after [from] (null: the first page) and stops before a page - * the run's budget does not cover. Each finished page moves the - * wallet's checkpoint to the cursor it returned; the last one moves it - * to the start of the classify pass. + * The engine answers each page under a read guard on its wallet lock + * and keeps it until [TxoReconcileEngine.releaseVerdicts]: a fork that + * truncated the engine between the read and the write would leave a + * healed row unspent in the store for a coin the engine no longer owns — + * the classify pass answers unknown for it and leaves it, and the next + * launch restores it as spendable. The guard is released in a `finally`, + * after the page's step, before the page is reported applied. */ private suspend fun healMissingTxos( walletId: ByteArray, - from: String?, - scope: Long, - tipHeight: Int, - minConfirmations: Int, limit: Int, foreignAccountIds: Set, isCancelled: () -> Boolean, - engineUtxoPage: suspend (cursor: String?, limit: Int) -> String?, + engine: TxoReconcileEngine, tally: ReconcileTally, walletRef: String, ): Boolean { - var cursor: String? = from while (true) { - if (!tally.takePage(isCancelled)) return false - val pageJson = engineUtxoPage(cursor, limit) - if (pageJson == null) { - // The transport died. Everything already applied stands - // (insert-only, idempotent); the rest waits for the next run. - tally.transportFailures++ - return false - } - // Strict decode: an unknown key, a missing required field or a - // mistyped value throws — the row would otherwise be healed with - // a defaulted owner or a zero amount. - val page = decodeEngineUtxoPage(pageJson) - tally.engineRows += page.utxos.size - - if (page.utxos.isNotEmpty()) { - val counts = reconcileStoreStep(tally, walletRef, "heal", isCancelled) { - database.withTransaction { - val c = HealCounts() - // Address -> its saved row (null when absent), for this page only. - val addressRows = HashMap() - for (row in page.utxos) { - healEngineRow( - walletId, row, tipHeight, minConfirmations, foreignAccountIds, c, addressRows, - ) + if (isCancelled()) return false + val page = try { + val pageJson = engine.healPage(limit) + if (pageJson == null) { + // The transport died. Everything already applied stands + // (insert-only, idempotent); the rest waits for the next run. + tally.stopOnTransport() + return false + } + // Strict decode: an unknown key, a missing required field or a + // mistyped value throws — the row would otherwise be healed with + // a defaulted owner or a zero amount. + val page = decodeEngineUtxoPage(pageJson) + if (page.budgetSpent) { + // Nothing read, nothing held: the wallet's next run picks + // up at this page. + tally.stopOnBudget() + return false + } + if (page.utxos.isNotEmpty()) { + val counts = reconcileStoreStep(tally, walletRef, "heal", isCancelled) { + database.withTransaction { + val c = HealCounts() + // Address -> its saved row (null when absent), for this page only. + val addressRows = HashMap() + for (row in page.utxos) { + healEngineRow(walletId, row, foreignAccountIds, c, addressRows) + } + c } - c - } - } ?: return false - tally.alreadyPresent += counts.alreadyPresent - tally.inserted += counts.inserted - tally.insertedDuffs += counts.insertedDuffs - tally.healedSpent += counts.healedSpent - tally.skippedImmature += counts.skippedImmature - tally.skippedUnresolvedAccount += counts.skippedUnresolvedAccount - tally.skippedMissingAddress += counts.skippedMissingAddress - tally.skippedInvalid += counts.skippedInvalid - tally.skippedForeign += counts.skippedForeign - // Logged once the step committed, so a rolled-back step - // never claims a row it did not write. - for (outpoint in counts.healedOutpoints) logReconcileItem("healed", outpoint, walletRef) - for (outpoint in counts.healedSpentOutpoints) logReconcileItem("healed_spent", outpoint, walletRef) + } ?: return false + tally.alreadyPresent += counts.alreadyPresent + tally.inserted += counts.inserted + tally.insertedDuffs += counts.insertedDuffs + tally.healedSpent += counts.healedSpent + tally.skippedImmature += counts.skippedImmature + tally.skippedUnresolvedAccount += counts.skippedUnresolvedAccount + tally.skippedMissingAddress += counts.skippedMissingAddress + tally.skippedInvalid += counts.skippedInvalid + tally.skippedForeign += counts.skippedForeign + // Logged once the step committed, so a rolled-back step + // never claims a row it did not write. + for (outpoint in counts.healedOutpoints) logReconcileItem("healed", outpoint, walletRef) + for (outpoint in counts.healedSpentOutpoints) logReconcileItem("healed_spent", outpoint, walletRef) + } + // Counted once the page's step is done, as Swift counts it. + tally.engineRows += page.utxos.size + page + } finally { + withContext(NonCancellable) { engine.releaseVerdicts() } } - // The page is finished: record it, so the next run does not - // read it again. - val next = page.cursor - if (!page.hasMore || next == null) { - saveReconcileCheckpoint(walletId, ReconcileCheckpoint.Classify(scope, ByteArray(0)), isCancelled) - return true + // The page is applied and its guard released: report it, so the + // next page, and the wallet's next run if this one stops here, + // starts after it. + val budgetLeft = reportPageApplied(tally, isCancelled) { engine.healPageApplied() } ?: return false + if (!budgetLeft) { + // After the pass's last page too: the engine moved the + // wallet's next run on to the classify pass, which waits + // for it. + tally.stopOnBudget() + return false } - saveReconcileCheckpoint(walletId, ReconcileCheckpoint.Heal(scope, next), isCancelled) - cursor = next + if (!page.hasMore || page.utxos.isEmpty()) return true } } /** * Apply one engine inventory row to the store (inside the caller's * exclusion lock and Room transaction) — Swift's - * `reconcileHealMissingTxos` gates, in its order: + * `reconcileHealMissingTxos`, in its order: * - * 1. malformed (txid not 32 bytes, no script, no address) → invalid; - * 2. the engine does not call it confirmed, or it is not - * [minConfirmations] deep at [tipHeight] → immature (a fresh coin can - * still reorg or, for coinbase, be immature; it ages into a later - * run); + * 1. fields this store cannot decode (txid not 32 bytes of hex, a + * malformed script or identity half), or the engine's + * [TXO_HEAL_VERDICT_INVALID] (no script or no address), or a verdict + * this build does not know → invalid; + * 2. the engine's [TXO_HEAL_VERDICT_IMMATURE] (not confirmed, or not + * deep enough at the run's tip — a fresh coin can still reorg or, + * for coinbase, be immature) → immature; it ages into a later run; * 3. the store already holds the outpoint → already present; * 4. the seven-field account tuple names no store account → unresolved, * NOT inserted: the restore loader routes by account, and an unowned * row would be dropped at the next launch, recreating the loss; * 5. the address has no saved `core_addresses` row filed under that - * account → missing address (Kotlin-only), NOT inserted: a cold - * restore rebuilds each account's address pool from the rows filed - * under it, so the coin would come back into the balance with no - * derivation path and no build could sign for it. A later run heals - * it once the address is persisted under the account. + * account → missing address, NOT inserted: a cold restore rebuilds + * each account's address pool from the rows filed under it, so the + * coin would come back into the balance with no derivation path and + * no build could sign for it. A later run heals it once the address + * is persisted under the account. * - * Between 2 and 3 sits the Kotlin-only contact exclusion (see - * [reconcileTxos]). A row that passes is inserted through - * [upsertUtxoRow] — the changeset path's own insert, so both writers - * honour the same rules (stub parent transaction, account stamp, - * address link, pending-input drain) — with `isConfirmed = true` (the - * gate guarantees it) and the engine's other flags as given. A row the - * drain wrote spent on the spot is counted as [HealCounts.healedSpent]: - * the engine holds the coin and the store now says spent, which is not - * a repair of the divergence the engine reported. + * Between 2 and 3 sits the contact exclusion (see [reconcileTxos]). A + * row that passes is inserted through [upsertUtxoRow] — the changeset + * path's own insert, so both writers honour the same rules (stub parent + * transaction, account stamp, address link, pending-input drain) — with + * `isConfirmed = true` (an eligible coin is engine-confirmed) and the + * engine's other flags as given. A row the drain wrote spent on the spot + * is counted as [HealCounts.healedSpent]: the engine holds the coin and + * the store now says spent, which is not a repair of the divergence the + * engine reported. */ private suspend fun healEngineRow( walletId: ByteArray, row: EngineUtxoRow, - tipHeight: Int, - minConfirmations: Int, foreignAccountIds: Set, counts: HealCounts, addressRows: MutableMap, @@ -2051,26 +2119,34 @@ class PlatformWalletPersistenceHandler( val script = row.scriptHex.hexToByteArrayOrNull() val userIdentityId = row.userIdentityId?.hexToByteArrayOrNull() val friendIdentityId = row.friendIdentityId?.hexToByteArrayOrNull() - if (txid == null || txid.size != 32 || script == null || script.isEmpty() || - row.address.isEmpty() || + if (txid == null || txid.size != 32 || script == null || (row.userIdentityId != null && userIdentityId == null) || (row.friendIdentityId != null && friendIdentityId == null) ) { counts.skippedInvalid++ return } - if (!row.isConfirmed || row.height <= 0 || tipHeight < row.height || - tipHeight - row.height + 1 < minConfirmations - ) { - counts.skippedImmature++ - return + when (row.healVerdict) { + TXO_HEAL_VERDICT_ELIGIBLE -> Unit + TXO_HEAL_VERDICT_IMMATURE -> { + counts.skippedImmature++ + return + } + // TXO_HEAL_VERDICT_INVALID, and any verdict this build does not + // know: nothing the engine vouched for. + else -> { + counts.skippedInvalid++ + return + } } // The inventory tags every coin with its owning account tuple. The // tag is the authoritative foreign check — a watch-only external // account's coin is the CONTACT's money whether or not its address // row survived persistence. The address-based check (through // core_addresses.accountId, the same second path the classify pass - // resolves store rows by) stays as a fallback. + // resolves store rows by) stays as a fallback. The engine's + // inventory never offers a contact's coin; this side does not rely + // on that alone. if (row.typeTag == ACCOUNT_TYPE_TAG_DASHPAY_EXTERNAL) { counts.skippedForeign++ return @@ -2160,123 +2236,135 @@ class PlatformWalletPersistenceHandler( * could change nothing. * * Each page is read under [callbackExclusion] together with - * [committedRoundGeneration], the engine is asked OUTSIDE the lock, and - * the verdicts are applied back under it in one Room transaction — only - * if no round committed in between and none is open for the wallet. A - * round that did may have re-credited one of the page's coins, so then - * nothing is written and the same page is read and classified again, at - * most [TXO_RECONCILE_MAX_STALE_RETRIES] times in a row before the run - * stops incomplete. An open round counts too: it has not moved + * [committedRoundGeneration], classified OUTSIDE the lock under an engine + * guard, and the verdicts are applied back under the lock in one Room + * transaction — only if no round committed in between and none is open + * for the wallet. A round that did may have re-credited one of the + * page's coins, so then nothing is written, the guard is released, and + * the same page is read and classified again (another page of the + * budget) for as long as the engine's stale-page rule + * ([TxoReconcileEngine.retryStalePage]) allows; then the run stops + * incomplete. An open round counts too: it has not moved * [committedRoundGeneration] yet, but it may already have staged that * re-credit, and a flip landing first would survive it — a credited * re-delivery keeps the flag of a row with a spender link (see * [upsertUtxoRow]), and no later run looks at a spent row again. * + * Those fences see this store's rounds, not the engine, and the engine + * can move with no round at all: dash-spv follows a fork by truncating + * the wallet, which drops the spender and restores the coin it spent, + * and a truncation reaches no persistence round. Flipping the row after + * such a fork would retire a live coin for good. So the engine answers + * under a read guard on its wallet lock, which a truncation needs for + * writing, and keeps it until [TxoReconcileEngine.releaseVerdicts]: the + * verdicts are applied while the spenders they name are still there. + * * The walk is keyed by outpoint (`outpoint > after`, the primary-key * order), not by offset, so the flips themselves never disturb it: a * flipped row leaves the `isSpent = 0` predicate behind the cursor, * never ahead of it. (Swift walks by offset and has to step back by the - * number of rows it flipped.) The same key is what lets a later run - * resume after [from]: rows inserted or flipped between runs never + * number of rows it flipped.) The same key is the store cursor the + * engine keeps between runs: rows inserted or flipped between runs never * shift the rows after it. * - * Stops before a page the run's budget does not cover. Each finished - * page moves the wallet's checkpoint past its last outpoint; reaching - * the end removes it. A stale page that is read again is not finished, - * so it does not move the checkpoint. + * Every store page is classified, one with nothing to ask (an empty + * page, or every row unresolved or a contact's) too: each classification + * is one page of the run's budget, and it names the page + * [TxoReconcileEngine.classifyPageApplied] reports. A page is reported + * once its flips committed (or it had none to write) and its guard is + * released, with its last outpoint as the cursor (36 bytes, within the + * engine's store-cursor bound); the store's last page — a short one — is + * reported with no cursor, which ends the cycle. A stale page is not + * reported. */ private suspend fun classifyStoreRows( walletId: ByteArray, from: ByteArray, - scope: Long, limit: Int, foreignAccountIds: Set, isCancelled: () -> Boolean, - classifyOutpoints: suspend (queriesJson: String) -> ByteArray?, + engine: TxoReconcileEngine, tally: ReconcileTally, walletRef: String, ) { var after = from var staleInARow = 0 while (true) { - if (!tally.takePage(isCancelled)) return + if (isCancelled()) return val page = reconcileStoreStep(tally, walletRef, "unspent_page", isCancelled) { readUnspentPage(walletId, after, limit, foreignAccountIds) } ?: return - if (page.fetched == 0) { - saveReconcileCheckpoint(walletId, null, isCancelled) - tally.completed = true - return - } tally.storeRows += page.rows.size tally.unresolvedAccount += page.unresolvedAccount tally.skippedForeign += page.skippedForeign - if (page.rows.isNotEmpty()) { - val verdicts = classifyOutpoints(encodeOutpointQueries(page.queries)) - if (verdicts == null || verdicts.size != page.rows.size) { + // The engine guard is held from the classification to the + // `finally`, across the apply only — see above. + try { + val answer = engine.classify(encodeOutpointQueries(page.queries)) + if (answer == null) { // No verdicts, no classification: stop rather than guess. - tally.transportFailures++ + tally.stopOnTransport() return } - val counts = reconcileStoreStep(tally, walletRef, "flip", isCancelled) { - if (committedRoundGeneration != page.generation || openRound(walletId) != null) { - FlipCounts().apply { staleGeneration = true } - } else { - database.withTransaction { applyEngineClasses(page.rows, verdicts) } - } - } ?: return - if (counts.staleGeneration) { - staleInARow++ - tally.staleRetries++ - if (staleInARow > TXO_RECONCILE_MAX_STALE_RETRIES) return - // Same cursor: nothing was written, so the page is read - // again and classified against the store as it is now. + if (answer.budgetSpent) { + // Nothing classified, nothing held: this run walked none + // of the page, and the wallet's next run reads it. tally.storeRows -= page.rows.size tally.unresolvedAccount -= page.unresolvedAccount tally.skippedForeign -= page.skippedForeign - continue + tally.stopOnBudget() + return } - staleInARow = 0 - tally.flipped += counts.flipped - tally.flippedDuffs += counts.flippedDuffs - tally.unspent += counts.unspent - tally.unknown += counts.unknown - tally.notOwned += counts.notOwned - for (outpoint in counts.flippedOutpoints) logReconcileItem("flipped_spent", outpoint, walletRef) + if (answer.classes.size != page.rows.size) { + tally.stopOnTransport() + return + } + if (page.rows.isNotEmpty()) { + val counts = reconcileStoreStep(tally, walletRef, "flip", isCancelled) { + if (committedRoundGeneration != page.generation || openRound(walletId) != null) { + FlipCounts().apply { staleGeneration = true } + } else { + database.withTransaction { applyEngineClasses(page.rows, answer.classes) } + } + } ?: return + if (counts.staleGeneration) { + staleInARow++ + tally.staleRetries++ + if (!engine.retryStalePage(staleInARow)) return + // Same cursor, not reported: nothing was written, so + // the page is read again and classified against the + // store as it is now. + tally.storeRows -= page.rows.size + tally.unresolvedAccount -= page.unresolvedAccount + tally.skippedForeign -= page.skippedForeign + continue + } + tally.flipped += counts.flipped + tally.flippedDuffs += counts.flippedDuffs + tally.unspent += counts.unspent + tally.unknown += counts.unknown + tally.notOwned += counts.notOwned + for (outpoint in counts.flippedOutpoints) logReconcileItem("flipped_spent", outpoint, walletRef) + } + } finally { + withContext(NonCancellable) { engine.releaseVerdicts() } } - // The page is finished. A short page is the last one: the cycle - // is over and the next run starts anew. - val last = page.lastOutpoint - if (last == null || page.fetched < limit) { - saveReconcileCheckpoint(walletId, null, isCancelled) + staleInARow = 0 + // The page is applied and its guard released: report it. A short + // page is the store's last, reported with no cursor: the cycle is + // over and the wallet's next run starts at the heal pass again. + val next = if (page.fetched < limit) null else page.lastOutpoint + val budgetLeft = reportPageApplied(tally, isCancelled) { engine.classifyPageApplied(next) } ?: return + if (next == null) { tally.completed = true return } - saveReconcileCheckpoint(walletId, ReconcileCheckpoint.Classify(scope, last), isCancelled) - after = last - } - } - - /** - * Record where [walletId]'s next reconcile run picks up (null: a new - * cycle) — see [txoReconcileCheckpoints]. Under [callbackExclusion], - * with cancellation re-checked under it, so it is ordered against a - * wallet deletion's cascade the same way a store step is. - */ - private suspend fun saveReconcileCheckpoint( - walletId: ByteArray, - checkpoint: ReconcileCheckpoint?, - isCancelled: () -> Boolean, - ) { - callbackExclusion.withLock { - if (isCancelled()) return@withLock - val key = walletId.toHex() - if (checkpoint == null) { - txoReconcileCheckpoints.remove(key) - } else { - txoReconcileCheckpoints[key] = checkpoint + if (!budgetLeft) { + tally.stopOnBudget() + return } + after = next } } @@ -4940,9 +5028,6 @@ class PlatformWalletPersistenceHandler( buffers.remove(walletId.toHex()) pendingRoundAliases.remove(walletId.toHex()) orphanedAliases.remove(walletId.toHex()) - // A wallet added again under the same id starts a new reconcile - // cycle rather than resuming the deleted wallet's. - txoReconcileCheckpoints.remove(walletId.toHex()) database.withTransaction { val walletRow = database.walletDao().getByWalletId(walletId) val walletNetwork = walletRow?.networkRaw @@ -5472,31 +5557,13 @@ class PlatformWalletPersistenceHandler( */ const val TXO_RECONCILE_PAGE_SIZE = 512 - /** - * Pages one [reconcileTxos] run may read, across both passes - * together: heal pages from the engine plus classify pages from the - * store, a stale page read again counting again. A run that reaches - * it stops between pages and the wallet's next run resumes there. - * - * The page size bounds what a run holds at once; this bounds what a - * run does. Without it every automatic run walks the whole - * inventory, and anyone who sends dust to a watched address decides - * how long that is. 16 pages of [TXO_RECONCILE_PAGE_SIZE] is 8192 - * rows per run, so a wallet with up to about 4000 unspent coins - * still finishes a whole cycle (every coin healed and classified) - * in one run, as before; only an inventory past that is spread over - * several runs, one budget per run at the manager's cadence. - */ - const val TXO_RECONCILE_PAGE_BUDGET = 16 - - /** Confirmations an engine coin needs before [reconcileTxos] inserts - * it: coinbase maturity, and well past any plausible reorg. */ - const val TXO_RECONCILE_MIN_CONFIRMATIONS = 100 - - /** Times in a row one classify page is classified again because a - * persistence round committed between its read and its apply, - * before [reconcileTxos] gives up on the run. */ - internal const val TXO_RECONCILE_MAX_STALE_RETRIES = 5 + /** [EngineUtxoRow.healVerdict] codes, mirroring rs-platform-wallet-ffi + * `TXO_HEAL_VERDICT_*` (`platform_wallet::manager::txo_reconcile:: + * TxoHealVerdict`). The engine decides them; only + * [TXO_HEAL_VERDICT_ELIGIBLE] may be inserted. */ + internal const val TXO_HEAL_VERDICT_INVALID = 0 + internal const val TXO_HEAL_VERDICT_IMMATURE = 1 + internal const val TXO_HEAL_VERDICT_ELIGIBLE = 2 /** * Decoder for the engine inventory transport. Deliberately the @@ -5506,11 +5573,11 @@ class PlatformWalletPersistenceHandler( */ private val engineJson = Json - /** One page of `walletManagerUtxosPageJson`, through [engineJson]. */ + /** One page of `walletManagerTxoHealPageJson`, through [engineJson]. */ internal fun decodeEngineUtxoPage(pageJson: String): EngineUtxoPage = engineJson.decodeFromString(EngineUtxoPage.serializer(), pageJson) - /** One classification batch, as `walletManagerClassifyOutpoints` + /** One classification batch, as `walletManagerTxoReconcileClassifyOutpoints` * deserializes it (rs-unified-sdk-jni `OutpointQuery`). */ internal fun encodeOutpointQueries(queries: List): String = engineJson.encodeToString( diff --git a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/PlatformWalletManager.kt b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/PlatformWalletManager.kt index f35fbbb97d6..058ba92ceb1 100644 --- a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/PlatformWalletManager.kt +++ b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/PlatformWalletManager.kt @@ -1182,9 +1182,9 @@ class PlatformWalletManager( // A TXO-store reconcile in flight for any wallet stops at its next // page or store step (each re-checks under the callback gate, so // none writes after the cascade below). Coarse on purpose — the - // cadence re-runs the others. - txoReconcile.bumpEpoch() - txoReconcile.forget(key) + // cadence re-runs the others. The native removal above already + // forgot the wallet's reconcile cadence and place. + txoReconcileEpoch.incrementAndGet() // 3. Snapshot → atomic secret delete → Room cascade, with both // persistence callbacks and app-side key writers excluded. @@ -1429,39 +1429,37 @@ class PlatformWalletManager( * un-marks one, and never acts on absence. See * [PlatformWalletPersistenceHandler.reconcileTxos] for the passes. * - * Gates, each a [TxoReconcileOutcome.Skipped] rather than an error, in - * order: the manager not closed; the wallet loaded; no run for it - * already in flight; SPV in steady state - * ([TxoReconcileGates.isSteadySyncState]); a scan tip - * ([TxoReconcileGates.scanTipHeight]); no latched sync fault - * ([syncFaultDetected] — a rejected round means rows are missing by - * design and a rescan is pending); and the wallet's own scan watermark - * as the engine holds it - * ([WalletManagerNative.walletManagerCoreWalletSyncedHeight], Swift's - * `platform_wallet_core_wallet_state`) within - * [TxoReconcileGates.TIP_MARGIN] blocks of the tip: the tip says how far - * the CLIENT got, and a wallet added behind the tip is still being - * scanned. Not the persisted Room `syncedHeight`: that copy trails the - * engine by up to one persistence round, and the SYNCED transition lands - * inside that window — gating on it skipped the run at the end of every - * restore. + * Whether the run may start is the native start gate's decision + * ([WalletManagerNative.walletManagerTxoReconcileBegin], backed by + * `platform_wallet::manager::txo_reconcile`): one run per wallet at a + * time, SPV in steady state with a scan tip, no latched sync fault + * ([syncFaultDetected]), and the wallet's own scan watermark as the + * engine holds it within the native margin of the tip. Each refusal is a + * [TxoReconcileOutcome.Skipped], not an error; so is a closed manager or + * a wallet this manager has not loaded, checked here first. * - * The run holds a [teardownGate] borrow, so the native manager is not - * destroyed under an engine read. [closeSuspending] and [removeWallet] - * bump the reconcile epoch first, and the run stops between pages, - * reporting `completed = false`. + * How much one run reads, and where it starts, are the native + * reconciler's decisions too: begin says where the run starts, every + * engine read is one page of the run's budget, and the handler reports + * each page it applied, which is what moves the wallet's place. A larger + * inventory is finished by the wallet's later runs, each picking up + * where the previous one stopped (the report's `stoppedOnBudget` and + * `resumedFromCheckpoint`). * - * One run reads at most - * [PlatformWalletPersistenceHandler.TXO_RECONCILE_PAGE_BUDGET] pages; a - * larger inventory is finished by the wallet's later runs, each picking - * up where the previous one stopped (the report's `stoppedOnBudget` and - * `resumedFromCheckpoint`). The place is scoped to the reconcile epoch - * the run started in, so a removal (of any wallet) or a close makes the - * next run start a new cycle: a wallet removed and added again never - * resumes the removed wallet's place. + * The run holds a [teardownGate] borrow, so the native manager is not + * destroyed under an engine read. A granted begin is always followed by + * the release of the last page's engine guard and then the native + * finish ([endTxoReconcileRun]) — after a completed run, a failed page, + * an exception or a cancelled caller alike: a claim left behind would + * refuse every later run of the wallet as + * [TxoReconcileSkipReason.ALREADY_RUNNING], and a guard left behind + * would hold the engine's writers off. [closeSuspending] and + * [removeWallet] move [txoReconcileEpoch] first, and the run stops + * between pages, reporting `completed = false`; such a run does not + * restart the wallet's cadence (the native removal already forgot the + * removed wallet's cadence and place). * - * Runs automatically on the steady-state transition of the SPV progress - * poll and every 30 minutes while it lasts (see + * Runs automatically when the native schedule says so (see * [maybeReconcileTxoStores]); hosts may also call it. Logs one skip * line (reason and wallet reference) or the handler's summary line — * counts and reference digests only. @@ -1476,92 +1474,192 @@ class PlatformWalletManager( return TxoReconcileOutcome.Skipped(reason) } if (isClosed) return skip(TxoReconcileSkipReason.MANAGER_CLOSED) - val key = walletId.toHex() - if (walletId.size != 32 || _wallets.value[key] == null) { + if (walletId.size != 32 || _wallets.value[walletId.toHex()] == null) { return skip(TxoReconcileSkipReason.WALLET_UNKNOWN) } - if (!txoReconcile.tryBegin(key)) return skip(TxoReconcileSkipReason.ALREADY_RUNNING) - try { - val generation = txoReconcile.currentEpoch - val progress = _spvProgress.value - if (!TxoReconcileGates.isSteadySyncState(progress)) { - return skip(TxoReconcileSkipReason.NOT_STEADY_STATE) - } - val tipHeight = TxoReconcileGates.scanTipHeight(progress) - ?: return skip(TxoReconcileSkipReason.TIP_UNAVAILABLE) - return try { - teardownGate.withOp { - if (syncFaultDetected()) return@withOp skip(TxoReconcileSkipReason.SYNC_FAULT_DETECTED) - val walletSyncedHeight = try { - withContext(Dispatchers.IO) { - mapNativeErrors { - WalletManagerNative.walletManagerCoreWalletSyncedHeight(managerHandle, walletId) - } - } - } catch (notFound: DashSdkError.PlatformWallet.NotFound) { - return@withOp skip(TxoReconcileSkipReason.WALLET_UNKNOWN) - } - if (!TxoReconcileGates.walletCaughtUp(walletSyncedHeight, tipHeight)) { - return@withOp skip(TxoReconcileSkipReason.WALLET_BEHIND_TIP) + val generation = txoReconcileEpoch.get() + val isCancelled = { isClosed || txoReconcileEpoch.get() != generation } + // Closed, or a wallet was removed since this run was asked for: a + // removed and re-added wallet must not be claimed by a run of the + // lifecycle before it. + fun stale() = skip( + if (isClosed) TxoReconcileSkipReason.MANAGER_CLOSED else TxoReconcileSkipReason.WALLET_UNKNOWN, + ) + return try { + teardownGate.withOp { + if (isCancelled()) return@withOp stale() + val tipOut = IntArray(1) + val startOut = IntArray(1) + val storeAfterOut = arrayOfNulls(1) + // NonCancellable: a RUN return holds a native claim that only + // the `finally` below releases. A cancellable withContext would + // discard that result and throw before the `try` is entered, + // leaking the claim for the rest of the session. Cancellation + // is still observed at the first suspension inside the `try`. + val gate = withContext(NonCancellable + Dispatchers.IO) { + mapNativeErrors { + WalletManagerNative.walletManagerTxoReconcileBegin( + managerHandle, walletId, tipOut, startOut, storeAfterOut, + ) } - val isCancelled = { isClosed || txoReconcile.currentEpoch != generation } - if (isCancelled()) return@withOp skip(TxoReconcileSkipReason.MANAGER_CLOSED) + } + if (gate != WalletManagerNative.TXO_RECONCILE_GATE_RUN) { + return@withOp skip(TxoReconcileSkipReason.fromGateCode(gate)) + } + val engine = NativeTxoReconcileEngine(managerHandle, walletId, walletRef) + var ran = false + try { + // Inside the `try`, so a start this build cannot read + // still releases the claim. + val start = txoReconcileStart(startOut[0], storeAfterOut[0]) + if (isCancelled()) return@withOp stale() val report = persistenceHandler.reconcileTxos( walletId = walletId, - tipHeight = tipHeight, - checkpointScope = generation, + start = start, + engine = engine, isCancelled = isCancelled, - engineUtxoPage = { cursor, limit -> - reconcileEngineRead("engine inventory page", walletRef) { - WalletManagerNative.walletManagerUtxosPageJson( - managerHandle, walletId, cursor, limit, - ) - } - }, - classifyOutpoints = { queriesJson -> - reconcileEngineRead("outpoint classification", walletRef) { - WalletManagerNative.walletManagerClassifyOutpoints( - managerHandle, walletId, queriesJson, - ) - } - }, ) - // Not for a run the epoch move overtook: its wallet may - // be a re-added one whose cadence the removal cleared, - // and which is due at once (see markRun). - txoReconcile.markRun(key, monotonicNowMs(), generation) + // A run a close or a removal overtook stopped early, and + // does not restart the cadence of a wallet that may be + // gone or added again. + ran = !isCancelled() TxoReconcileOutcome.Reconciled(report) + } finally { + // The teardown borrow is still held, so the handle is live. + withContext(NonCancellable + Dispatchers.IO) { + endTxoReconcileRun( + walletRef = walletRef, + freeHeldVerdicts = engine::freeHeldVerdicts, + finish = { + mapNativeErrors { + WalletManagerNative.walletManagerTxoReconcileFinish(managerHandle, walletId, ran) + } + }, + ) + } } - } catch (closing: IllegalStateException) { - // The teardown gate refuses new borrows once close began. - if (isClosed) skip(TxoReconcileSkipReason.MANAGER_CLOSED) else throw closing } - } finally { - txoReconcile.end(key) - } + } catch (closing: IllegalStateException) { + // The teardown gate refuses new borrows once close began. + if (isClosed) skip(TxoReconcileSkipReason.MANAGER_CLOSED) else throw closing + } + } + + /** + * The handler's view of where a reconcile run starts, from what + * [WalletManagerNative.walletManagerTxoReconcileBegin] wrote to + * `outStart[0]` and `outStoreAfter[0]`. A code this build does not know, + * or a classify start with no store cursor, is a contract violation + * between the JNI layer and this reader and throws. + */ + private fun txoReconcileStart( + code: Int, + storeAfter: ByteArray?, + ): PlatformWalletPersistenceHandler.TxoReconcileStart = when (code) { + WalletManagerNative.TXO_RECONCILE_START_HEAL -> + PlatformWalletPersistenceHandler.TxoReconcileStart.NewCycle + WalletManagerNative.TXO_RECONCILE_START_HEAL_RESUMED -> + PlatformWalletPersistenceHandler.TxoReconcileStart.HealResumed + WalletManagerNative.TXO_RECONCILE_START_CLASSIFY -> + PlatformWalletPersistenceHandler.TxoReconcileStart.Classify( + checkNotNull(storeAfter) { "a classify start carries no store cursor" }, + ) + else -> error("unknown reconcile start $code") } /** - * One engine read of the reconcile, off the caller's thread. A native - * failure is logged by kind and exception type only — the message of a - * native error can echo the input it rejected — and becomes the null - * the handler's stop-and-report contract is written against. - */ - private suspend fun reconcileEngineRead(what: String, walletRef: String, read: () -> T): T? = - withContext(Dispatchers.IO) { + * The engine side of one claimed reconcile run, over the native calls — + * the Kotlin twin of Swift's `NativeCoreTxoReconcileEngine`. Each call + * runs off the caller's thread; a native failure is logged by kind and + * exception type only (the message of a native error can echo the input + * it rejected) and becomes the null the handler's stop-and-report + * contract is written against. + * + * [held] is the engine guard the last heal page or classification is + * held under, 0 when none is. A held read keeps the guard it wrote to + * `outHeld` even when the call failed after writing it, and a guard the + * handler did not release is freed, never overwritten. + */ + private class NativeTxoReconcileEngine( + private val managerHandle: Long, + private val walletId: ByteArray, + private val walletRef: String, + ) : PlatformWalletPersistenceHandler.TxoReconcileEngine { + private val held = AtomicLong(0L) + + override suspend fun healPage(limit: Int): String? = call("engine inventory page") { + val outHeld = LongArray(1) try { - mapNativeErrors { read() } - } catch (cancellation: kotlin.coroutines.cancellation.CancellationException) { - throw cancellation - } catch (t: Throwable) { - android.util.Log.w( - "PlatformWalletManager", - "txos reconcile: $what failed error=${t.javaClass.simpleName} wallet=$walletRef", - ) - null + WalletManagerNative.walletManagerTxoHealPageJson(managerHandle, walletId, limit, outHeld) + } finally { + keepHeld(outHeld) } } + override suspend fun healPageApplied(): Boolean? = call("heal page applied") { + WalletManagerNative.walletManagerTxoReconcileHealPageApplied(managerHandle, walletId) + } + + override suspend fun classify(queriesJson: String): PlatformWalletPersistenceHandler.OutpointVerdicts? = + call("outpoint classification") { + val outHeld = LongArray(1) + val outBudgetSpent = BooleanArray(1) + try { + WalletManagerNative.walletManagerTxoReconcileClassifyOutpoints( + managerHandle, walletId, queriesJson, outBudgetSpent, outHeld, + )?.let { classes -> + PlatformWalletPersistenceHandler.OutpointVerdicts(classes, outBudgetSpent[0]) + } + } finally { + keepHeld(outHeld) + } + } + + override suspend fun classifyPageApplied(nextAfter: ByteArray?): Boolean? = call("classify page applied") { + WalletManagerNative.walletManagerTxoReconcileClassifyPageApplied(managerHandle, walletId, nextAfter) + } + + override suspend fun releaseVerdicts() = freeHeldVerdicts() + + override fun retryStalePage(staleInARow: Int): Boolean = + // A failed call stops the run: retrying is the engine's call, + // and an engine that cannot answer has not allowed it. + runCatching { WalletManagerNative.txoReconcileRetryStalePage(staleInARow) }.getOrDefault(false) + + /** + * Release the guard [held] carries, at most once (the value is taken + * out before it is freed). The native free takes no manager handle + * and never waits: it must complete while a registry writer — a + * manager being created or destroyed — is queued behind a native + * read that waits on the engine behind a writer that waits on this + * guard. + */ + fun freeHeldVerdicts() { + val token = held.getAndSet(0L) + if (token != 0L) WalletManagerNative.txoReconcileHeldVerdictsFree(token) + } + + private fun keepHeld(outHeld: LongArray) { + val previous = held.getAndSet(outHeld[0]) + if (previous != 0L) WalletManagerNative.txoReconcileHeldVerdictsFree(previous) + } + + private suspend fun call(what: String, read: () -> T): T? = + withContext(Dispatchers.IO) { + try { + mapNativeErrors { read() } + } catch (cancellation: kotlin.coroutines.cancellation.CancellationException) { + throw cancellation + } catch (t: Throwable) { + android.util.Log.w( + "PlatformWalletManager", + "txos reconcile: $what failed error=${t.javaClass.simpleName} wallet=$walletRef", + ) + null + } + } + } + /** * Refresh the persisted DashPay payment history for one identity: * one FFI read (`managed_identity_get_dashpay_payments`) + one Room @@ -2525,10 +2623,6 @@ class PlatformWalletManager( suspend fun stopSpv() = withContext(Dispatchers.IO) { mapNativeErrors { WalletManagerNative.spvStop(managerHandle) } - // Arm the reconcile's next steady-state edge here as well as on the - // poll's next stopped read: a restart inside one poll interval - // would otherwise never be seen as a stop. - txoReconcile.spvStopped() } /** Clear all persisted SPV storage (headers, filters, state). */ @@ -2617,52 +2711,64 @@ class PlatformWalletManager( if (running) { runCatching { spvSyncProgress() }.getOrNull()?.let { next -> if (next != _spvProgress.value) _spvProgress.value = next - maybeReconcileTxoStores(next) } runCatching { spvTipUnixSeconds() }.getOrNull()?.let { tip -> if (tip != _spvTipUnixSeconds.value) _spvTipUnixSeconds.value = tip } - } else { - // SPV stopped — the next steady read after a restart is - // a new steady state for the reconcile. - txoReconcile.spvStopped() - if (_spvProgress.value != SpvSyncProgressData.EMPTY) { - // Reset so the overlay clears. - _spvProgress.value = SpvSyncProgressData.EMPTY - } + } else if (_spvProgress.value != SpvSyncProgressData.EMPTY) { + // SPV stopped — reset so the overlay clears. + _spvProgress.value = SpvSyncProgressData.EMPTY } + // Every tick, running or not: the poll is the native + // reconcile schedule's clock, and a stopped client arms its + // next steady-state edge. + maybeReconcileTxoStores() delay(POLL_INTERVAL_MS) } } } - /** In-flight runs, per-wallet cadence, the steady-state edge and the - * cancellation epoch of the TXO-store reconcile. */ - private val txoReconcile = TxoReconcileCoordinator() + /** + * Cancellation generation of the TXO-store reconcile: moved by + * [closeSuspending] and [removeWallet]; a run captures it at start and + * stops between pages once it moves. + */ + private val txoReconcileEpoch = AtomicLong() + + /** The automatic runs [maybeReconcileTxoStores] launched that have not + * ended yet: one per wallet ([TxoReconcileLaunches]). */ + private val txoReconcileLaunches = TxoReconcileLaunches() /** - * SDK-internal trigger for [reconcileTxoStore], fed by every accepted - * read of the SPV progress poll: run every loaded wallet on the - * transition into steady state, and again every 30 minutes per wallet - * while it lasts (each wallet's last run is stamped when it is - * scheduled) — never while a run for that wallet is in flight or a - * launched run of it has not ended, and never inline: the poll tick - * stays cheap. Lives here rather than in - * the host apps so every host gets the repair without wiring anything. - * A run that throws is logged (exception type and wallet reference - * only) and retried on the next cadence tick — never allowed to kill - * the progress poll. + * SDK-internal trigger for [reconcileTxoStore], called on every tick of + * the progress poll, SPV running or not: the poll is the native + * schedule's clock, and a stopped client arms its next steady-state + * edge. Launches a run for each loaded wallet the native schedule + * reports due ([WalletManagerNative.walletManagerTxoReconcileDueWallets], + * which owns the steady-state edge and the cadence) — never inline, so + * the poll tick stays cheap. A wallet the engine already holds but this + * manager has not wrapped yet (creation and restore publish the native + * wallet first) is skipped; being reported spends none of its cadence — + * only the native claim in [reconcileTxoStore] restarts it — so it is + * reported again on the next tick. A wallet whose launched run has not + * ended is not launched again ([txoReconcileLaunches]). Lives here + * rather than in the host apps so every host gets the repair without + * wiring anything. A run that throws is logged (exception type and + * wallet reference only) and retried when the schedule next reports it + * due — never allowed to kill the progress poll. */ - private fun maybeReconcileTxoStores(progress: SpvSyncProgressData) { + private suspend fun maybeReconcileTxoStores() { if (isClosed) return + val due = runCatching { + withContext(Dispatchers.IO) { + mapNativeErrors { WalletManagerNative.walletManagerTxoReconcileDueWallets(managerHandle) } + } + }.getOrNull() ?: return val loaded = wallets.value - val due = txoReconcile.walletsDue(progress, loaded.keys, monotonicNowMs()) - for (key in due) { - // `due` is a subset of `loaded.keys`. - val walletId = loaded.getValue(key).walletId - // Released on completion, not in a `finally`: a job cancelled - // before it starts never runs its body, but still completes. - scope.launch { + for (offset in 0 until due.size - due.size % 32 step 32) { + val key = due.copyOfRange(offset, offset + 32).toHex() + val walletId = loaded[key]?.walletId ?: continue + txoReconcileLaunches.launch(scope, key) { try { reconcileTxoStore(walletId) } catch (cancellation: kotlin.coroutines.cancellation.CancellationException) { @@ -2674,12 +2780,10 @@ class PlatformWalletManager( "wallet=${logReference(walletId)}", ) } - }.invokeOnCompletion { txoReconcile.launchEnded(key) } + } } } - private fun monotonicNowMs(): Long = System.nanoTime() / 1_000_000 - // ── DashPay sync + seedless unlock ──────────────────────────────── // // Port of `PlatformWalletManagerDashPaySync.swift` + the unlock flow @@ -3109,7 +3213,7 @@ class PlatformWalletManager( // Stale any TXO-store reconcile in flight first: it stops at its next // page or store step instead of holding teardown behind the rest of // its walk. - txoReconcile.bumpEpoch() + txoReconcileEpoch.incrementAndGet() val bundle = bundleRef.getAndSet(0) if (bundle == 0L) return // NonCancellable from the moment the bundle is claimed: bundleRef diff --git a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcile.kt b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcile.kt index ea0c5be95a2..476b14300fe 100644 --- a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcile.kt +++ b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcile.kt @@ -1,7 +1,9 @@ package org.dashfoundation.dashsdk.wallet +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.launch import org.dashfoundation.dashsdk.persistence.PlatformWalletPersistenceHandler -import java.util.concurrent.atomic.AtomicLong +import java.util.concurrent.ConcurrentHashMap /** * What [PlatformWalletManager.reconcileTxoStore] did — the Kotlin twin of @@ -17,197 +19,99 @@ sealed class TxoReconcileOutcome { ) : TxoReconcileOutcome() } -/** Why [PlatformWalletManager.reconcileTxoStore] did not run. */ -enum class TxoReconcileSkipReason { +/** + * Why [PlatformWalletManager.reconcileTxoStore] did not run. Every reason + * but [MANAGER_CLOSED] comes from the native start gate + * (`platform_wallet::manager::txo_reconcile::TxoReconcileSkip`), which owns + * the thresholds; [gateCode] is its `TXO_RECONCILE_SKIP_*` code. + * [WALLET_UNKNOWN] is also what a wallet this manager has not loaded gets, + * checked here before the native gate. + */ +enum class TxoReconcileSkipReason(internal val gateCode: Int?) { /** The manager is closed or closing. */ - MANAGER_CLOSED, + MANAGER_CLOSED(null), - /** The wallet is not loaded in this manager (or has no Room row). */ - WALLET_UNKNOWN, + /** The wallet is not loaded in this manager, or the engine does not + * hold it. */ + WALLET_UNKNOWN(1), /** A run for this wallet is already in flight. */ - ALREADY_RUNNING, + ALREADY_RUNNING(2), - /** SPV is not in its steady state (see [TxoReconcileGates.isSteadySyncState]). */ - NOT_STEADY_STATE, + /** SPV is not running, or not in its steady state. */ + NOT_STEADY_STATE(3), /** The progress carries no scan tip height. */ - TIP_UNAVAILABLE, + TIP_UNAVAILABLE(4), /** The native manager latched a sync fault: rows are missing by design * and a rescan is pending ([PlatformWalletManager.syncFaultDetected]). */ - SYNC_FAULT_DETECTED, - - /** The engine's scan watermark for the wallet trails the scan tip by - * more than [TxoReconcileGates.TIP_MARGIN] blocks — it is still being - * scanned. */ - WALLET_BEHIND_TIP, -} - -/** The pure gates of the TXO-store reconcile, testable without a manager. */ -internal object TxoReconcileGates { - /** How far the engine's scan watermark for the wallet may trail the - * scan tip for the scan to count as complete for this wallet. */ - const val TIP_MARGIN = 6 - - /** Cadence of the automatic run while SPV stays in steady state. */ - const val CADENCE_MS = 30 * 60 * 1_000L - - /** - * Whether the wallet's own scan watermark (the engine's, see - * [org.dashfoundation.dashsdk.ffi.WalletManagerNative.walletManagerCoreWalletSyncedHeight]) - * is within [TIP_MARGIN] blocks of the scan tip — Swift's - * `synced_height + margin >= tip`. The tip says how far the CLIENT got; - * a wallet added behind it is still being scanned. - */ - fun walletCaughtUp(walletSyncedHeight: Long, tipHeight: Int): Boolean = - walletSyncedHeight >= 0 && walletSyncedHeight + TIP_MARGIN >= tipHeight - - /** - * dash-spv's steady state for a fully synced client is - * [SpvSyncState.WAIT_FOR_EVENTS] with the filter phase at its target - * height; [SpvSyncState.SYNCED] is the transient window before it. - * Either counts — anything else is a scan still running (or not - * running at all, which the poll publishes as - * [SpvSyncProgressData.EMPTY]: waiting, with no filter phase). - */ - fun isSteadySyncState(progress: SpvSyncProgressData): Boolean = - when (progress.overallState) { - SpvSyncState.SYNCED -> true - SpvSyncState.WAIT_FOR_EVENTS -> { - val filters = progress.filters - filters != null && filters.targetHeight > 0 && filters.currentHeight >= filters.targetHeight - } - SpvSyncState.WAITING_FOR_CONNECTIONS, SpvSyncState.SYNCING, SpvSyncState.ERROR -> false - } - - /** - * The scan tip: the filter phase's height (the wallet-relevant one), - * falling back to the header tip when the filter phase is absent; null - * when neither carries a height. - */ - fun scanTipHeight(progress: SpvSyncProgressData): Int? { - val height = progress.filters?.currentHeight?.takeIf { it > 0 } - ?: progress.headers?.currentHeight?.takeIf { it > 0 } - ?: return null - return height.coerceAtMost(Int.MAX_VALUE.toLong()).toInt() + SYNC_FAULT_DETECTED(5), + + /** The wallet's scan watermark, as the engine holds it, trails the scan + * tip by more than the native margin — it is still being scanned. */ + WALLET_BEHIND_TIP(6), + ; + + internal companion object { + /** The reason a non-run code from the native start gate names. A + * code this build does not know is a contract break with the + * native library it shipped with, not a reason to guess. */ + fun fromGateCode(code: Int): TxoReconcileSkipReason = + entries.firstOrNull { it.gateCode == code } + ?: throw IllegalStateException("unknown TXO reconcile gate code $code") } } /** - * The manager-side state of the TXO-store reconcile — the Kotlin twin of - * Swift's `coreTxoReconcileInFlight` / `coreTxoReconcileLastRunAt` / - * `coreTxoReconcileWasSteady` / `coreTxoReconcileEpoch`. Thread-safe: the - * progress poll, scheduled runs and host calls all reach it. + * End a TXO-store reconcile run: free the engine guard its last page may + * still hold, then release its native claim — Swift's + * `endCoreTxoReconcileRun`. The guard goes first and without the native + * registry: `finish` resolves the manager handle there, which must never be + * waited for while a guard is held. Neither step can skip the other: a + * failure is logged by exception type and [walletRef] only, and a claim + * left behind would refuse every later run of the wallet as + * [TxoReconcileSkipReason.ALREADY_RUNNING]. */ -internal class TxoReconcileCoordinator(private val cadenceMs: Long = TxoReconcileGates.CADENCE_MS) { - private val lock = Any() - private var wasSteady = false - private val inFlight = HashSet() - - /** - * Wallets [walletsDue] handed out whose launched run has not ended yet - * ([launchEnded]). A launched run claims the wallet ([tryBegin]) only - * once its coroutine starts, so [inFlight] alone leaves a gap: a rising - * edge inside it (a new block takes the client out of steady state and - * back) would schedule the wallet again, and the second run would - * either skip or run straight after the first. Swift keeps the same set - * (`coreTxoReconcileLaunched`). - */ - private val launched = HashSet() - private val lastRunAtMs = HashMap() - private val epoch = AtomicLong() - - /** - * Bumped by manager close and by wallet deletion; a run captures it at - * start and stops between pages once it moves. - */ - val currentEpoch: Long get() = epoch.get() - - fun bumpEpoch() { - epoch.incrementAndGet() +internal fun endTxoReconcileRun(walletRef: String, freeHeldVerdicts: () -> Unit, finish: () -> Unit) { + runCatching(freeHeldVerdicts).onFailure { t -> + android.util.Log.w( + "PlatformWalletManager", + "txos reconcile: verdict release failed error=${t.javaClass.simpleName} wallet=$walletRef", + ) } - - /** - * Feed one progress read (every accepted read, not only a changed one: - * it is the reconcile's only clock, and a quiet steady-state wallet's - * progress does not change for the whole cadence). Returns the wallets - * to run now — all of [walletKeys] on the rising edge into steady - * state, and afterwards each wallet whose last run is at least the - * cadence old (or that never ran) — never one already in flight, and - * never one whose launched run has not ended. Each returned wallet is - * stamped NOW, at schedule time, so the next tick does not schedule it - * again while this run is still gating, and is recorded as launched - * until the caller reports [launchEnded]. - */ - fun walletsDue(progress: SpvSyncProgressData, walletKeys: Collection, nowMs: Long): List = - synchronized(lock) { - val steady = TxoReconcileGates.isSteadySyncState(progress) - val rising = steady && !wasSteady - wasSteady = steady - if (!steady) return@synchronized emptyList() - walletKeys.filter { key -> - if (key in inFlight || key in launched) return@filter false - val last = lastRunAtMs[key] - val due = rising || last == null || nowMs - last >= cadenceMs - if (due) { - lastRunAtMs[key] = nowMs - launched.add(key) - } - due - } - } - - /** - * SPV stopped: arm the next rising edge. The poll publishes no progress - * read while SPV is down, so without this a client that restarts - * straight into steady state would never leave it as far as - * [walletsDue] can tell, and every wallet would wait out the cadence. - */ - fun spvStopped() { - synchronized(lock) { wasSteady = false } + runCatching(finish).onFailure { t -> + android.util.Log.w( + "PlatformWalletManager", + "txos reconcile: release failed error=${t.javaClass.simpleName} wallet=$walletRef", + ) } +} - /** - * The run [walletsDue] handed out for [walletKey] has ended, or was - * never started. Must be called exactly once per wallet returned, even - * for a launch cancelled before its body ran. - */ - fun launchEnded(walletKey: String) { - synchronized(lock) { launched.remove(walletKey) } - } - - /** Claim the wallet for one run; false when a run is already in flight. */ - fun tryBegin(walletKey: String): Boolean = synchronized(lock) { inFlight.add(walletKey) } - - /** Release the claim [tryBegin] took. */ - fun end(walletKey: String) { - synchronized(lock) { inFlight.remove(walletKey) } - } +/** + * The automatic reconcile runs a manager has launched and that have not + * ended yet, by wallet (hex id) — Swift's `coreTxoReconcileLaunched`. + * Bookkeeping of the manager's own coroutines only: whether and when a run + * may start is the native schedule's and start gate's. The native schedule + * reports a wallet due until a run claims it, so without this every poll + * tick before a launched run reaches its begin would queue another one, and + * they would run back to back once the first ended. + */ +internal class TxoReconcileLaunches { + private val launched: MutableSet = ConcurrentHashMap.newKeySet() /** - * Record a finished run of the wallet for the cadence, but only while - * [runEpoch] (the epoch the run captured at its start) is still - * current. A run that a close or a wallet removal overtook stopped - * early, and its wallet may since have been removed and added again - * under the same id: [forget] already cleared that wallet's cadence, and - * stamping it now would hold the re-added wallet off for a whole - * cadence. Returns whether the stamp was written. - * - * The epoch is compared under the same lock [forget] takes, and - * [bumpEpoch] always precedes [forget] in a removal, so a stamp either - * lands before the removal's [forget] (which then clears it) or sees - * the moved epoch and is not written. + * Launch [run] for [walletKey] in [scope], unless a launch of the same + * wallet has not ended; returns whether it launched. The wallet is + * released when the job completes — on completion rather than in a + * `finally`, so a job cancelled before its body ran releases it too. */ - fun markRun(walletKey: String, nowMs: Long, runEpoch: Long): Boolean = - synchronized(lock) { - if (epoch.get() != runEpoch) return@synchronized false - lastRunAtMs[walletKey] = nowMs - true - } - - /** Forget a deleted wallet's cadence. */ - fun forget(walletKey: String) { - synchronized(lock) { lastRunAtMs.remove(walletKey) } + fun launch(scope: CoroutineScope, walletKey: String, run: suspend () -> Unit): Boolean { + if (!launched.add(walletKey)) return false + scope.launch { run() }.invokeOnCompletion { launched.remove(walletKey) } + return true } + + /** Whether a launch of [walletKey] has not ended. */ + operator fun contains(walletKey: String): Boolean = walletKey in launched } diff --git a/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandlerTest.kt b/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandlerTest.kt index c3b8918556e..a9a7ca5b2ab 100644 --- a/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandlerTest.kt +++ b/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandlerTest.kt @@ -12,6 +12,7 @@ import androidx.sqlite.db.framework.FrameworkSQLiteOpenHelperFactory import androidx.test.core.app.ApplicationProvider import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch import kotlinx.coroutines.test.runTest import kotlinx.serialization.json.int import kotlinx.serialization.json.jsonArray @@ -7094,13 +7095,19 @@ class PlatformWalletPersistenceHandlerTest { txidHex: String = txid.toHex(), /** The DashPay identity halves, as hex; absent for every other account. */ identities: Pair? = null, + /** The engine's verdict on the row: whether the store may heal it. */ + healVerdict: Int = healEligible, ): String = """{"typeTag":$typeTag,"standardTag":0,"index":0,"registrationIndex":0,"keyClass":0,""" + identities.let { if (it == null) "" else """"userIdentityId":"${it.first}","friendIdentityId":"${it.second}",""" } + """"txid":"$txidHex","vout":$vout,""" + """"amount":$amount,"address":"$address","scriptHex":"$scriptHex",""" + """"height":$height,"isConfirmed":$isConfirmed,"isInstantlocked":false,""" + - """"isCoinbase":$isCoinbase,"isLocked":false}""" + """"isCoinbase":$isCoinbase,"isLocked":false,"healVerdict":$healVerdict}""" + + private val healEligible = PlatformWalletPersistenceHandler.TXO_HEAL_VERDICT_ELIGIBLE + private val healImmature = PlatformWalletPersistenceHandler.TXO_HEAL_VERDICT_IMMATURE + private val healInvalid = PlatformWalletPersistenceHandler.TXO_HEAL_VERDICT_INVALID /** A whole inventory for [FakeEngine]: the engine's coins, the outpoints * it proves spent, and the ones it says the store misfiled. */ @@ -7173,27 +7180,38 @@ class PlatformWalletPersistenceHandlerTest { private fun restoredUtxoCount(h: PlatformWalletPersistenceHandler = handler): Int = h.onLoadWalletList().single { it.walletId.contentEquals(walletId) }.utxos.size - /** A reconcile of [walletId] at [reconcileTip] against [engine]; either - * transport half can be replaced to fail, stall or race a round. */ + /** + * One run of [walletId] against [engine], wired the way the manager + * wires the native side: [FakeEngine.begin] says where the run starts, + * and the reads, the applied reports, the release and the stale-page + * rule are the engine's own. [healPage] / [classifyOutpoints] swap the + * answer of a read (the fake still does its bookkeeping: the budget, + * the page it names, the guard), to fail it, stall it, or race a round + * against it; null leaves the fake's answer. + */ private suspend fun PlatformWalletPersistenceHandler.reconcileWith( engine: FakeEngine = FakeEngine(inventoryJson()), pageSize: Int = 2, - pageBudget: Int = PlatformWalletPersistenceHandler.TXO_RECONCILE_PAGE_BUDGET, - checkpointScope: Long = 0L, isCancelled: () -> Boolean = { false }, - engineUtxoPage: suspend (String?, Int) -> String? = engine::page, - classifyOutpoints: suspend (String) -> ByteArray? = engine::classify, - ): PlatformWalletPersistenceHandler.TxoReconcileReport = - reconcileTxos( - walletId = walletId, - tipHeight = reconcileTip, - pageSize = pageSize, - pageBudget = pageBudget, - checkpointScope = checkpointScope, - isCancelled = isCancelled, - engineUtxoPage = engineUtxoPage, - classifyOutpoints = classifyOutpoints, - ) + healPage: (suspend (served: String) -> String?)? = null, + classifyOutpoints: (suspend (queriesJson: String, verdicts: ByteArray) -> ByteArray?)? = null, + ): PlatformWalletPersistenceHandler.TxoReconcileReport { + engine.healAnswer = healPage + engine.classifyAnswer = classifyOutpoints + return try { + reconcileTxos( + walletId = walletId, + start = engine.begin(), + engine = engine, + pageSize = pageSize, + isCancelled = isCancelled, + ) + } finally { + engine.healAnswer = null + engine.classifyAnswer = null + assertFalse("a run ends holding no engine guard", engine.held) + } + } /** Every TXO row of [wallet], spent or not — what these tests count * and attribute, through the DAO's whole-wallet query. */ @@ -7252,6 +7270,12 @@ class PlatformWalletPersistenceHandlerTest { assertEquals(0, query["typeTag"]!!.jsonPrimitive.int) assertEquals(0, query["index"]!!.jsonPrimitive.int) assertEquals(reconcileScript.toHex(), query["scriptHex"]!!.jsonPrimitive.content) + // Each page's guard is released after its store step and before the + // page is reported applied; the store's last page ends the cycle. + assertEquals( + listOf("heal", "release", "heal applied", "classify", "release", "classify applied null"), + engine.events, + ) } @Test @@ -7266,9 +7290,11 @@ class PlatformWalletPersistenceHandlerTest { val txid = ByteArray(32) { 0x72 } seedUnspentTxo(txid) var classifyCalls = 0 + val engine = FakeEngine(inventoryJson()) val report = handler.reconcileWith( - classifyOutpoints = { + engine, + classifyOutpoints = { _, _ -> classifyCalls++ if (classifyCalls == 1) { commitARound() @@ -7282,6 +7308,12 @@ class PlatformWalletPersistenceHandlerTest { assertTrue(report.completed) assertEquals("the first verdict was read before the round and refused", 1, report.staleRetries) assertEquals("the page is classified again after the refusal", 2, classifyCalls) + assertEquals("the engine's stale-page rule was asked once", listOf(1), engine.staleAsks) + assertEquals( + "the stale page is released and read again, never reported", + listOf("classify", "release", "classify", "release", "classify applied null"), + engine.events.drop(3), + ) assertEquals(0, report.flipped) assertEquals(1, report.unspent) assertEquals(1, report.storeRows) @@ -7294,27 +7326,31 @@ class PlatformWalletPersistenceHandlerTest { @Test fun aPageThatKeepsGoingStaleStopsTheRunIncomplete() = runTest { // A round commits between every read and apply: the page is - // re-classified a bounded number of times, then the run gives up - // without having written anything. + // re-classified for as long as the engine's stale-page rule allows, + // then the run gives up without having written anything. seedReconcileWallet() val txid = ByteArray(32) { 0x73 } seedUnspentTxo(txid) var classifyCalls = 0 + val engine = FakeEngine(inventoryJson()).apply { staleRetryLimit = 2 } val report = handler.reconcileWith( - classifyOutpoints = { + engine, + classifyOutpoints = { _, _ -> classifyCalls++ commitARound() byteArrayOf(PlatformWalletPersistenceHandler.OUTPOINT_CLASS_KNOWN_UNCREDITED) }, ) - val limit = PlatformWalletPersistenceHandler.TXO_RECONCILE_MAX_STALE_RETRIES + 1 assertFalse(report.completed) - assertEquals(limit, report.staleRetries) - assertEquals(limit, classifyCalls) + assertFalse("a stale stop is not a budget stop", report.stoppedOnBudget) + assertEquals("the host stops where the engine says", listOf(1, 2, 3), engine.staleAsks) + assertEquals(3, report.staleRetries) + assertEquals(3, classifyCalls) assertEquals(0, report.flipped) assertFalse(db.txoDao().getByOutpoint(makeOutpoint(txid, 0))!!.isSpent) + assertFalse("a page never applied is never reported", engine.events.any { it.startsWith("classify applied") }) } @Test @@ -7340,9 +7376,11 @@ class PlatformWalletPersistenceHandlerTest { assertFalse("sanity: a mempool spender does not mark the coin spent", linkedBefore.isSpent) assertEquals(2, restoredUtxoCount()) var classifyCalls = 0 + val engine = FakeEngine(inventoryJson()) val report = handler.reconcileWith( - classifyOutpoints = { + engine, + classifyOutpoints = { _, _ -> classifyCalls++ if (classifyCalls == 1) { // A reorg of the spenders hands both coins back: the @@ -7372,15 +7410,17 @@ class PlatformWalletPersistenceHandlerTest { spender.contentEquals(db.txoDao().getByOutpoint(makeOutpoint(linked, 0))!!.spendingTxid), ) assertEquals("both coins are still handed back at the next load", 2, restoredUtxoCount()) - val limit = PlatformWalletPersistenceHandler.TXO_RECONCILE_MAX_STALE_RETRIES + 1 assertFalse("the run stops rather than apply under an open round", report.completed) - assertEquals(limit, report.staleRetries) + assertEquals(engine.staleRetryLimit + 1, report.staleRetries) assertEquals(0, report.flipped) // Once the round has closed, the next run classifies both rows // again against the engine as it is now. val after = handler.reconcileWith( - classifyOutpoints = { ByteArray(2) { PlatformWalletPersistenceHandler.OUTPOINT_CLASS_UNSPENT } }, + engine, + classifyOutpoints = { _, verdicts -> + ByteArray(verdicts.size) { PlatformWalletPersistenceHandler.OUTPOINT_CLASS_UNSPENT } + }, ) assertTrue(after.completed) assertEquals(0, after.staleRetries) @@ -7497,7 +7537,7 @@ class PlatformWalletPersistenceHandlerTest { @Test fun aConsistentStoreYieldsZeroMutationsAndSpentRowsAreNeverAsked() = runTest { // Rows the engine also holds unspent — including a YOUNG coin the - // heal pass skips as immature — are consistent, not divergence; a + // engine's verdict calls immature — are consistent, not divergence; a // spent row is never even asked about; and the run logs one // "mirror consistent" summary line and nothing else. seedReconcileWallet() @@ -7510,7 +7550,12 @@ class PlatformWalletPersistenceHandlerTest { markStoredSpent(spent) // The engine holds exactly the store's unspent coins, and says so. val engine = FakeEngine( - inventoryJson(rows = listOf(engineRowJson(unspent), engineRowJson(young, height = reconcileTip - 3))), + inventoryJson( + rows = listOf( + engineRowJson(unspent), + engineRowJson(young, height = reconcileTip - 3, healVerdict = healImmature), + ), + ), ) org.robolectric.shadows.ShadowLog.clear() @@ -7628,35 +7673,39 @@ class PlatformWalletPersistenceHandlerTest { } @Test - fun theHealPassRefusesImmatureUnconfirmedUnresolvedAndMalformedCoins() = runTest { + fun theHealPassFollowsTheEnginesVerdictAndRefusesUnresolvedAndUndecodableCoins() = runTest { + // Confirmations, maturity and well-formedness are the engine's call + // (its heal verdict); the host only follows it, and adds what needs + // its store: the account the coin is filed under, and fields it can + // decode at all. seedReconcileWallet() // BIP44 only: no CoinJoin account row fun txid(b: Int) = ByteArray(32) { b.toByte() } val engine = FakeEngine( inventoryJson( rows = listOf( - engineRowJson(txid(0x76), height = reconcileTip - 50), // immature - engineRowJson(txid(0x77), height = reconcileTip - 99), // exactly 100 confirmations - // Deep enough, but the engine itself does not call it confirmed. - engineRowJson(txid(0x78), isConfirmed = false), + engineRowJson(txid(0x76), healVerdict = healImmature), + // The host does not second-guess an eligible verdict, even + // for a coin it would have called young. + engineRowJson(txid(0x77), height = reconcileTip - 3), + engineRowJson(txid(0x78), healVerdict = healInvalid), engineRowJson(txid(0x79), typeTag = 1), // CoinJoin: no store account - engineRowJson(txid(0x7a), scriptHex = ""), // no script - engineRowJson(txid(0x7b), address = ""), // no address - engineRowJson(txid(0x7c), height = 0), // unconfirmed height + engineRowJson(txid(0x7a), scriptHex = "76a"), // a script that is not hex + engineRowJson(txid(0x7b), healVerdict = 9), // a verdict this build does not know engineRowJson(txid(0x7d), txidHex = "7d".repeat(31)), // 31-byte txid ), ), ) - val report = handler.reconcileWith(engine) + val report = handler.reconcileWith(engine, pageSize = 8) assertTrue(report.completed) - assertEquals(8, report.engineRows) + assertEquals(7, report.engineRows) assertEquals(1, report.inserted) - assertEquals(3, report.skippedImmature) + assertEquals(1, report.skippedImmature) assertEquals(1, report.skippedUnresolvedAccount) - assertEquals(3, report.skippedInvalid) + assertEquals(4, report.skippedInvalid) assertNotNull(db.txoDao().getByOutpoint(makeOutpoint(txid(0x77), 0))) - for (b in listOf(0x76, 0x78, 0x79, 0x7a, 0x7b, 0x7c)) { + for (b in listOf(0x76, 0x78, 0x79, 0x7a, 0x7b)) { assertNull("coin $b must not be healed", db.txoDao().getByOutpoint(makeOutpoint(txid(b), 0))) } assertEquals(1, walletTxos().size) @@ -8030,41 +8079,50 @@ class PlatformWalletPersistenceHandlerTest { ) @Test - fun aRunStopsAtItsPageBudgetAndTheNextRunsResumeWhereItStopped() = runTest { - // Dust sent to a watched address grows the inventory with no action - // from the owner, so one run reads a bounded number of pages and the - // wallet's later runs carry on from there instead of starting over. - // 5 coins at 2 per page: 3 heal pages, then 3 classify pages. + fun aRunStopsWhereTheEnginesBudgetSaysAndTheNextRunsStartWhereItSays() = runTest { + // How much one run reads and where the next starts are the engine's + // decisions: every read is one page of its budget, each applied + // report answers whether budget is left, and begin says where a run + // starts. The host stops when told and starts where told; it keeps + // no place of its own. 5 coins at 2 per page: 3 heal pages, then 3 + // store pages. saveReconcileAddress(attributeUnownedTxos(walletId)) - val engine = fiveCoinEngine(firstVout = 60) + val engine = fiveCoinEngine(firstVout = 60).apply { budget = 2 } - // Run 1: two heal pages, then the budget is spent mid-pass. - val first = handler.reconcileWith(engine, pageSize = 2, pageBudget = 2) + // Run 1: two heal pages; the second's report leaves no budget. + val first = handler.reconcileWith(engine, pageSize = 2) assertTrue(first.stoppedOnBudget) assertFalse("a budget stop has not reached the end of the cycle", first.completed) assertFalse(first.resumedFromCheckpoint) assertEquals(0, first.transportFailures + first.storeFailures) - assertEquals(listOf(null, "2"), engine.cursors) + assertEquals(listOf(0, 2), engine.healStarts) assertEquals(4, first.engineRows) - assertEquals("both finished pages landed", 4, first.inserted) + assertEquals("both applied pages landed", 4, first.inserted) assertEquals("the classify pass waits for the heal pass", 0, engine.batches) - // Run 2: the heal pass resumes at the saved engine cursor, reaches - // its end, and the classify pass starts at the first row. - val second = handler.reconcileWith(engine, pageSize = 2, pageBudget = 2) + // Run 2: the engine resumes the heal pass after the last applied + // page; it reaches its end, and the classify pass starts at the + // store's first row. + val second = handler.reconcileWith(engine, pageSize = 2) assertTrue(second.resumedFromCheckpoint) assertTrue(second.stoppedOnBudget) assertFalse(second.completed) - assertEquals("only the page run 1 did not read", listOf(null, "2", "4"), engine.cursors) + assertEquals("only the page run 1 did not apply", listOf(0, 2, 4), engine.healStarts) assertEquals(1, second.engineRows) assertEquals(1, second.inserted) assertEquals(2, second.storeRows) assertEquals(1, engine.batches) assertEquals(listOf(60, 61), engine.queries.map { it["vout"]!!.jsonPrimitive.int }) + assertEquals( + "the page was reported with its last outpoint as the store cursor", + "classify applied ${makeOutpoint(changeTxid, 61).toHex()}", + engine.events.last(), + ) - // Run 3: the classify pass resumes after the saved outpoint and - // reaches its end. Every row is classified once across the runs. - val third = handler.reconcileWith(engine, pageSize = 2, pageBudget = 2) + // Run 3: the engine hands that cursor back; the classify pass + // resumes after it and reaches the store's last page. Every row is + // classified once across the runs. + val third = handler.reconcileWith(engine, pageSize = 2) assertTrue(third.resumedFromCheckpoint) assertFalse(third.stoppedOnBudget) assertTrue(third.completed) @@ -8072,23 +8130,18 @@ class PlatformWalletPersistenceHandlerTest { assertEquals(3, third.storeRows) assertEquals(3, third.unspent) assertEquals((60 until 65).toList(), engine.queries.map { it["vout"]!!.jsonPrimitive.int }) + assertEquals("the store's last page ends the cycle", "classify applied null", engine.events.last()) - // Run 4: the finished cycle cleared its checkpoint, so this run - // starts a new one at the first heal page and walks everything. + // Run 4: a new cycle, at the first heal page, walks everything. + engine.budget = 16 val fourth = handler.reconcileWith(engine, pageSize = 2) assertFalse(fourth.resumedFromCheckpoint) assertTrue(fourth.completed) - assertEquals(null, engine.cursors[3]) + assertEquals(0, engine.healStarts[3]) assertEquals(5, fourth.engineRows) assertEquals(5, fourth.alreadyPresent) assertEquals(5, fourth.storeRows) assertEquals(0, fourth.mutations) - - // Run 5: the budget alone still bounds a run that starts over. - val fifth = handler.reconcileWith(engine, pageSize = 2, pageBudget = 1) - assertFalse(fifth.resumedFromCheckpoint) - assertTrue(fifth.stoppedOnBudget) - assertEquals("one page read", 7, engine.pages) } @Test @@ -8098,7 +8151,8 @@ class PlatformWalletPersistenceHandlerTest { assertEquals(5, handler.reconcileWith(engine, pageSize = 8).inserted) org.robolectric.shadows.ShadowLog.clear() - val report = handler.reconcileWith(engine, pageSize = 2, pageBudget = 1) + engine.budget = 1 + val report = handler.reconcileWith(engine, pageSize = 2) assertTrue(report.stoppedOnBudget) assertEquals(0, report.mutations) val summary = org.robolectric.shadows.ShadowLog.getLogs() @@ -8109,25 +8163,78 @@ class PlatformWalletPersistenceHandlerTest { } @Test - fun aFailedHealPageDoesNotMoveTheCheckpointPastIt() = runTest { + fun aStalePageThatOutlastsTheBudgetStopsOnTheBudget() = runTest { + // A stale page is read again, and every read is a page of the + // budget: a read past it comes back budget-spent with nothing held, + // and the run stops there, counting none of the page as walked. + seedReconcileWallet() + val txid = ByteArray(32) { 0x86.toByte() } + seedUnspentTxo(txid) + val engine = FakeEngine(inventoryJson()).apply { + budget = 3 + staleRetryLimit = 10 + } + + val report = handler.reconcileWith( + engine, + classifyOutpoints = { _, verdicts -> + commitARound() + verdicts + }, + ) + + assertTrue(report.stoppedOnBudget) + assertFalse(report.completed) + assertEquals(2, report.staleRetries) + assertEquals("the unclassified page is not counted as walked", 0, report.storeRows) + assertEquals( + listOf( + "heal", "release", "heal applied", + "classify", "release", "classify", "release", "budget spent", + ), + engine.events, + ) + } + + @Test + fun aStorePageWithNothingToAskIsStillClassifiedAndReported() = runTest { + // Each classification is one page of the run's budget and names the + // page the applied report moves the engine past, so a store page + // with nothing to ask — here the empty store — is classified too. + seedReconcileWallet() + val engine = FakeEngine(inventoryJson()) + + val report = handler.reconcileWith(engine) + + assertTrue(report.completed) + assertEquals(0, report.storeRows) + assertEquals( + listOf("heal", "release", "heal applied", "classify []", "release", "classify applied null"), + engine.events, + ) + } + + @Test + fun aFailedHealPageIsNotReportedSoTheNextRunReadsItAgain() = runTest { // The next run must read the page that failed, not the one after it, // and the classify pass must not start until the heal pass ends. saveReconcileAddress(attributeUnownedTxos(walletId)) val engine = fiveCoinEngine(firstVout = 70) val failed = handler.reconcileWith( engine, - engineUtxoPage = { cursor, limit -> if (cursor == null) engine.page(cursor, limit) else null }, + healPage = { served -> if (engine.pages == 1) served else null }, ) assertFalse(failed.completed) assertFalse("a failure is not a budget stop", failed.stoppedOnBudget) assertEquals(1, failed.transportFailures) assertEquals(2, failed.inserted) assertEquals(0, engine.batches) + assertEquals(1, engine.events.count { it == "heal applied" }) val next = handler.reconcileWith(engine, pageSize = 2) assertTrue(next.resumedFromCheckpoint) assertTrue(next.completed) - assertEquals("resumed at the failed page, after the finished one", listOf(null, "2", "4"), engine.cursors) + assertEquals("resumed at the failed page, after the applied one", listOf(0, 2, 2, 4), engine.healStarts) assertEquals(3, next.inserted) assertEquals("the classify pass covered every row once", 5, next.storeRows) for (vout in 70 until 75) { @@ -8136,45 +8243,39 @@ class PlatformWalletPersistenceHandlerTest { } @Test - fun aResumedHealPageThatDoesNotDecodeIsReadAgainByTheNextRun() = runTest { - // The strict page decode throws; the page that threw is not - // finished, so the place stays at the cursor before it. A page that - // says more follow with no cursor is refused by the decode, so it - // can never leave a place that ends the walk early. + fun aHealPageThatDoesNotDecodeIsReleasedAndReadAgainByTheNextRun() = runTest { + // The strict page decode throws; the page's guard is released on the + // way out, and the page is not reported, so the engine serves it + // again. saveReconcileAddress(attributeUnownedTxos(walletId)) - val engine = fiveCoinEngine(firstVout = 100) - assertTrue(handler.reconcileWith(engine, pageSize = 2, pageBudget = 1).stoppedOnBudget) + val engine = fiveCoinEngine(firstVout = 100).apply { budget = 1 } + assertTrue(handler.reconcileWith(engine, pageSize = 2).stoppedOnBudget) + engine.budget = 16 for ( malformed in listOf( - """{"utxos":[],"cursor":null}""", - """{"utxos":[],"cursor":null,"hasMore":true}""", + """{"utxos":[],"hasMore":false}""", + """{"utxos":[],"hasMore":true,"budgetSpent":true}""", ) ) { - val asked = ArrayList() - val thrown = runCatching { - handler.reconcileWith( - engine, - engineUtxoPage = { cursor, _ -> - asked += cursor - malformed - }, - ) - }.exceptionOrNull() + val reads = engine.pages + val thrown = runCatching { handler.reconcileWith(engine, healPage = { malformed }) }.exceptionOrNull() assertTrue("$malformed must fail the decode, got $thrown", thrown is IllegalArgumentException) - assertEquals("the resumed run asked for the saved page", listOf("2"), asked) + assertEquals("the resumed run read the page after the applied one", 2, engine.healStarts.last()) + assertEquals(reads + 1, engine.pages) + assertEquals(listOf("heal", "release"), engine.events.takeLast(2)) } val next = handler.reconcileWith(engine, pageSize = 2) assertTrue(next.resumedFromCheckpoint) assertTrue(next.completed) - assertEquals(listOf(null, "2", "4"), engine.cursors) + assertEquals(listOf(0, 2, 2, 2, 4), engine.healStarts) assertEquals(3, next.inserted) assertEquals("the classify pass started only once the heal pass ended", 5, next.storeRows) } @Test - fun aFailedOrStaleClassifyPageDoesNotMoveTheCheckpointPastIt() = runTest { + fun aFailedOrStaleClassifyPageIsNotReportedSoTheNextRunReadsItAgain() = runTest { seedReconcileWallet() val txids = (0 until 6).map { i -> ByteArray(32) { (0xa0 + i).toByte() } } txids.forEach { seedUnspentTxo(it) } @@ -8185,101 +8286,114 @@ class PlatformWalletPersistenceHandlerTest { var batch = 0 val failed = handler.reconcileWith( engine, - classifyOutpoints = { queries -> if (++batch == 2) null else engine.classify(queries) }, + classifyOutpoints = { _, verdicts -> if (++batch == 2) null else verdicts }, ) assertFalse(failed.completed) assertEquals(1, failed.transportFailures) - assertEquals(listOf("a0", "a1"), firstBytes()) + assertEquals(listOf("a0", "a1", "a2", "a3"), firstBytes()) - // The second page stays stale past the retry limit: a round + // The second page stays stale past the engine's retry limit: a round // commits between every read and its apply. val stale = handler.reconcileWith( engine, - classifyOutpoints = { queries -> + classifyOutpoints = { _, verdicts -> commitARound() - engine.classify(queries) + verdicts }, ) assertTrue(stale.resumedFromCheckpoint) assertFalse(stale.completed) assertFalse(stale.stoppedOnBudget) - assertEquals(PlatformWalletPersistenceHandler.TXO_RECONCILE_MAX_STALE_RETRIES + 1, stale.staleRetries) + assertEquals(engine.staleRetryLimit + 1, stale.staleRetries) assertEquals("the heal pass already ended in this cycle", 1, engine.pages) - // A clean run picks up at the page that never finished. + // A clean run picks up at the page that never applied. val next = handler.reconcileWith(engine, pageSize = 2) assertTrue(next.resumedFromCheckpoint) assertTrue(next.completed) assertEquals(4, next.storeRows) - val staleReads = List(PlatformWalletPersistenceHandler.TXO_RECONCILE_MAX_STALE_RETRIES + 1) { - listOf("a2", "a3") - }.flatten() + val staleReads = List(engine.staleRetryLimit + 1) { listOf("a2", "a3") }.flatten() assertEquals( - "a0, a1 from the first run, the stale page's a2, a3 each try, then a2 to a5", - listOf("a0", "a1") + staleReads + listOf("a2", "a3", "a4", "a5"), + "a0, a1 then the failed a2, a3; the stale page's a2, a3 each try; then a2 to a5", + listOf("a0", "a1", "a2", "a3") + staleReads + listOf("a2", "a3", "a4", "a5"), firstBytes(), ) } @Test - fun deletingTheWalletDropsItsReconcileCheckpoint() = runTest { - saveReconcileAddress(attributeUnownedTxos(walletId)) - val engine = fiveCoinEngine(firstVout = 80) - assertTrue(handler.reconcileWith(engine, pageSize = 2, pageBudget = 1).stoppedOnBudget) + fun aClassifyStartWithACursorThisStoreDidNotWriteStartsAtTheFirstRow() = runTest { + // The store cursor is this side's own 36-byte outpoint, handed back + // as it was given. Anything else starts the walk at the first row + // rather than at an arbitrary BLOB position; the passes are + // idempotent, so that costs only the walk. + seedReconcileWallet() + val txids = (0 until 3).map { i -> ByteArray(32) { (0xc0 + i).toByte() } } + txids.forEach { seedUnspentTxo(it) } + val engine = FakeEngine(inventoryJson()) - handler.deleteWalletData(walletId) - saveReconcileAddress(attributeUnownedTxos(walletId)) + val report = handler.reconcileTxos( + walletId = walletId, + start = PlatformWalletPersistenceHandler.TxoReconcileStart.Classify(ByteArray(8) { 0xff.toByte() }), + engine = engine.also { it.begin() }, + pageSize = 8, + ) - val report = handler.reconcileWith(engine, pageSize = 2) - assertFalse("a re-added wallet starts a new cycle", report.resumedFromCheckpoint) assertTrue(report.completed) - assertEquals(listOf(null, null, "2", "4"), engine.cursors) - assertEquals(5, report.inserted) + assertTrue(report.resumedFromCheckpoint) + assertEquals(3, report.storeRows) + assertEquals("no heal page is read for a classify start", 0, engine.pages) } @Test - fun aCheckpointIsResumedOnlyUnderTheScopeThatWroteIt() = runTest { - // The manager scopes a place to its reconcile epoch, which moves on - // every wallet removal and on close: a wallet removed and added - // again (or any place left from before the move) starts a new cycle - // even if the handler never saw the deletion. - saveReconcileAddress(attributeUnownedTxos(walletId)) - val engine = fiveCoinEngine(firstVout = 110) - assertTrue(handler.reconcileWith(engine, pageSize = 2, pageBudget = 1, checkpointScope = 1L).stoppedOnBudget) - - val otherScope = handler.reconcileWith(engine, pageSize = 2, pageBudget = 1, checkpointScope = 2L) - assertFalse("another scope's place is not resumed", otherScope.resumedFromCheckpoint) - assertEquals(listOf(null, null), engine.cursors) - - // That run replaced the place with its own; the old scope no longer - // finds one, and the new scope resumes its own. - val sameScope = handler.reconcileWith(engine, pageSize = 2, pageBudget = 1, checkpointScope = 2L) - assertTrue(sameScope.resumedFromCheckpoint) - assertEquals(listOf(null, null, "2"), engine.cursors) - val oldScope = handler.reconcileWith(engine, pageSize = 2, pageBudget = 1, checkpointScope = 1L) - assertFalse(oldScope.resumedFromCheckpoint) - assertEquals(listOf(null, null, "2", null), engine.cursors) - } - - @Test - fun aCancelledRunLeavesNoCheckpoint() = runTest { - // A run that a wallet deletion cancelled must not record a place - // after the deletion dropped the wallet's: the save re-checks the - // cancellation under the callback lock. + fun aCancelledPageIsNeverReportedApplied() = runTest { + // A run that a wallet deletion or a close cancelled stops between + // pages; the page it read is not written and not reported, so the + // engine does not move past it. saveReconcileAddress(attributeUnownedTxos(walletId)) val engine = fiveCoinEngine(firstVout = 120) var cancelled = false val report = handler.reconcileWith( engine, isCancelled = { cancelled }, - engineUtxoPage = { cursor, limit -> engine.page(cursor, limit).also { cancelled = true } }, + healPage = { served -> served.also { cancelled = true } }, ) assertFalse(report.completed) assertEquals("the cancelled step wrote nothing", 0, report.inserted) + assertEquals(listOf("heal", "release"), engine.events) val next = handler.reconcileWith(engine, pageSize = 2) assertFalse(next.resumedFromCheckpoint) - assertEquals(listOf(null, null, "2", "4"), engine.cursors) + assertEquals(listOf(0, 0, 2, 4), engine.healStarts) + assertEquals(5, next.inserted) + } + + @Test + fun aCancelledCallerStillReleasesTheHeldPage() = runTest { + // The caller's coroutine is cancelled while a classification it + // holds a guard for is in flight: the guard is released on the way + // out all the same, and the page is not reported. + seedReconcileWallet() + seedUnspentTxo(ByteArray(32) { 0x87.toByte() }) + val engine = FakeEngine(inventoryJson()) + val parked = kotlinx.coroutines.CompletableDeferred() + val run = launch { + handler.reconcileWith( + engine, + classifyOutpoints = { _, _ -> + parked.complete(Unit) + kotlinx.coroutines.awaitCancellation() + }, + ) + } + parked.await() + assertTrue("sanity: the classification holds its guard", engine.held) + + run.cancel() + run.join() + + assertFalse("released despite the cancellation", engine.held) + assertEquals(listOf("classify", "release"), engine.events.takeLast(2)) + assertFalse(engine.events.any { it.startsWith("classify applied") }) } @Test @@ -8288,10 +8402,12 @@ class PlatformWalletPersistenceHandlerTest { // than guessing at rows it could not ask the engine about. seedReconcileWallet() for (vout in 50 until 53) seedUnspentTxo(changeTxid, vout) - val report = handler.reconcileWith(pageSize = 1, classifyOutpoints = { null }) + val engine = FakeEngine(inventoryJson()) + val report = handler.reconcileWith(engine, pageSize = 1, classifyOutpoints = { _, _ -> null }) assertFalse(report.completed) assertEquals(1, report.transportFailures) + assertFalse("a failed batch is never reported", engine.events.any { it.startsWith("classify applied") }) assertEquals(0, report.unknown) assertEquals(0, report.flipped) } @@ -8306,11 +8422,17 @@ class PlatformWalletPersistenceHandlerTest { seedReconcileWallet() val txid = ByteArray(32) { 0x84.toByte() } faults.armed = true - val report = handler.reconcileWith(FakeEngine(inventoryJson(rows = listOf(engineRowJson(txid))))) + val engine = FakeEngine(inventoryJson(rows = listOf(engineRowJson(txid)))) + val report = handler.reconcileWith(engine) faults.armed = false assertFalse(report.completed) assertEquals(1, report.storeFailures) + assertEquals( + "the guard is released and the failed page never reported", + listOf("heal", "release"), + engine.events, + ) assertEquals("a failed read is not a hit either", 0, report.alreadyPresent) assertEquals(0, report.inserted) assertNull(db.txoDao().getByOutpoint(makeOutpoint(txid, 0))) @@ -8366,15 +8488,17 @@ class PlatformWalletPersistenceHandlerTest { val report = handler.reconcileWith( engine, - // The run checks once for the account read, once before each - // engine read and once more before each store step: the first - // three admit the account read, page one and its heal; the - // fourth (before page two) reports the epoch bumped. - isCancelled = { ++checks >= 4 }, + // The run checks once for the account read, then before each + // engine read, before each store step and before each applied + // report: the first four admit the account read, page one, its + // heal and its report; the fifth (before page two) reports the + // epoch bumped. + isCancelled = { ++checks >= 5 }, ) assertFalse(report.completed) assertEquals(1, engine.pages) + assertEquals(listOf("heal", "release", "heal applied"), engine.events) assertEquals("the page that landed stays", 2, report.inserted) assertEquals(2, walletTxos().size) } @@ -8395,12 +8519,13 @@ class PlatformWalletPersistenceHandlerTest { engine, pageSize = 1, isCancelled = { cancelled }, - engineUtxoPage = { cursor, limit -> engine.page(cursor, limit).also { cancelled = true } }, + healPage = { served -> served.also { cancelled = true } }, ) assertFalse(report.completed) assertEquals("no second engine read may start", 1, engine.pages) assertEquals("the old page must not reach the store", 0, report.mutations) + assertEquals("released, and not reported applied", listOf("heal", "release"), engine.events) assertEquals(0, walletTxos().size) } @@ -8427,7 +8552,7 @@ class PlatformWalletPersistenceHandlerTest { rows = listOf( engineRowJson(healedTxid, amount = 100_001L, address = address, height = 1_434_986), // Refused (immature), so the skip path renders too. - engineRowJson(ByteArray(32) { 0x66 }, address = address, height = reconcileTip), + engineRowJson(ByteArray(32) { 0x66 }, address = address, height = reconcileTip, healVerdict = healImmature), ), spent = listOf(flippedTxid to 1), ), @@ -8436,12 +8561,7 @@ class PlatformWalletPersistenceHandlerTest { val report = handler.reconcileWith(engine) // A second, failing run renders the incomplete summary as well. - handler.reconcileTxos( - walletId = walletId, - tipHeight = reconcileTip, - engineUtxoPage = engine::page, - classifyOutpoints = { null }, - ) + handler.reconcileWith(engine, classifyOutpoints = { _, _ -> null }) assertEquals(1, report.inserted) assertEquals(1, report.flipped) @@ -8524,12 +8644,18 @@ class PlatformWalletPersistenceHandlerTest { // the mirror the engine reloads from. val malformed = """{"utxos":[{"typeTag":0,"txid":"${changeTxid.toHex()}",""" + """"vout":40,"address":"yStxXHHzhAx58JhaPBNhn3xsH93UwBM2nd","height":1400000}],""" + - """"cursor":null,"hasMore":false}""" - val thrown = runCatching { handler.reconcileWith(engineUtxoPage = { _, _ -> malformed }) }.exceptionOrNull() + """"hasMore":false,"budgetSpent":false}""" + val engine = FakeEngine(inventoryJson()) + val thrown = runCatching { handler.reconcileWith(engine, healPage = { malformed }) }.exceptionOrNull() assertTrue( "strict decode must throw, got $thrown", thrown is kotlinx.serialization.SerializationException, ) + assertEquals( + "the page's guard is released on the way out, and the page is not reported", + listOf("heal", "release"), + engine.events, + ) assertNull( "nothing was healed from the malformed page", db.txoDao().getByOutpoint(makeOutpoint(changeTxid, 40)), @@ -8548,7 +8674,7 @@ class PlatformWalletPersistenceHandlerTest { engineRowJson(changeTxid, vout = 41), ).jsonObject fun page(row: Map) = - """{"utxos":[${kotlinx.serialization.json.JsonObject(row)}],"cursor":null,"hasMore":false}""" + """{"utxos":[${kotlinx.serialization.json.JsonObject(row)}],"hasMore":false,"budgetSpent":false}""" assertEquals( 1, PlatformWalletPersistenceHandler.decodeEngineUtxoPage(page(complete)).utxos.size, @@ -8558,6 +8684,7 @@ class PlatformWalletPersistenceHandlerTest { key in listOf( "typeTag", "standardTag", "index", "registrationIndex", "keyClass", "txid", "vout", "amount", "address", "scriptHex", "height", "isConfirmed", "isInstantlocked", "isCoinbase", "isLocked", + "healVerdict", ) ) { val thrown = runCatching { @@ -8571,21 +8698,26 @@ class PlatformWalletPersistenceHandlerTest { } @Test - fun anInventoryPageWithoutItsContinuationFailsTheDecode() { - // A page that lost a key, or says more follow with nowhere to - // resume from, must not read as the last page: the heal walk would - // end early and leave later engine coins unrepaired. + fun anInventoryPageMissingAKeyOrClaimingAReadPastItsBudgetFailsTheDecode() { + // A page that lost a key must not read as the last page (the heal + // walk would end early and leave later engine coins unrepaired) or + // as no budget stop; a budget-spent page that carries a read is one + // the native side says it did not make; and the page carries no + // cursor any more — the engine keeps the run's place. val row = engineRowJson(changeTxid, vout = 42) fun decodes(page: String) = runCatching { PlatformWalletPersistenceHandler.decodeEngineUtxoPage(page) } - assertTrue(decodes("""{"utxos":[$row],"cursor":null,"hasMore":false}""").isSuccess) - assertTrue(decodes("""{"utxos":[$row],"cursor":"c","hasMore":true}""").isSuccess) + assertTrue(decodes("""{"utxos":[$row],"hasMore":false,"budgetSpent":false}""").isSuccess) + assertTrue(decodes("""{"utxos":[$row],"hasMore":true,"budgetSpent":false}""").isSuccess) + assertTrue(decodes("""{"utxos":[],"hasMore":false,"budgetSpent":true}""").isSuccess) for ( (label, page) in listOf( - "no utxos" to """{"cursor":null,"hasMore":false}""", - "no cursor" to """{"utxos":[$row],"hasMore":false}""", - "no hasMore" to """{"utxos":[$row],"cursor":null}""", - "more to come with no cursor" to """{"utxos":[$row],"cursor":null,"hasMore":true}""", + "no utxos" to """{"hasMore":false,"budgetSpent":false}""", + "no hasMore" to """{"utxos":[$row],"budgetSpent":false}""", + "no budgetSpent" to """{"utxos":[$row],"hasMore":false}""", + "a budget stop with rows" to """{"utxos":[$row],"hasMore":false,"budgetSpent":true}""", + "a budget stop with more to come" to """{"utxos":[],"hasMore":true,"budgetSpent":true}""", + "a cursor" to """{"utxos":[],"cursor":null,"hasMore":false,"budgetSpent":false}""", ) ) { val thrown = decodes(page).exceptionOrNull() @@ -8598,8 +8730,8 @@ class PlatformWalletPersistenceHandlerTest { /** * The checked-in TXO-reconcile transport payloads. The Rust * `wallet_manager::tests` read the SAME file (`include_str!` of this - * resource): they serialize the real `UtxoPage` for four accessor - * entries and require `inventoryPage` exactly, and they deserialize + * resource): they serialize the real `UtxoPage` for four heal-page + * rows and require `inventoryPage` exactly, and they deserialize * `outpointQueries` through the real `OutpointQuery` conversion. Here * the page is decoded by the production decoder and the queries are * built by the production encoder, so neither codec can drift from the @@ -8625,34 +8757,34 @@ class PlatformWalletPersistenceHandlerTest { ) assertTrue(page.hasMore) - assertNotNull("a page with more to come carries its cursor", page.cursor) + assertFalse(page.budgetSpent) assertEquals( listOf( // BIP44 account 0: every account field zero, and present. PlatformWalletPersistenceHandler.EngineUtxoRow( 0, 0, 0, 0, 0, null, null, goldenTxid(0x00).toHex(), 0, 19_549L, reconcileAddress, goldenScript(0x5b).toHex(), 1_534_921, - true, false, false, false, + true, false, false, false, healEligible, ), // Non-default account fields, every flag set, and an amount // past 32 bits. PlatformWalletPersistenceHandler.EngineUtxoRow( 0, 1, 2, 0, 0, null, null, goldenTxid(0x20).toHex(), 7, 5_000_000_000L, "yGoldenFixtureBip32Addr", goldenScript(0xc3).toHex(), 1_400_000, - true, true, true, true, + true, true, true, true, healEligible, ), // A registration index, and a script with no address form. PlatformWalletPersistenceHandler.EngineUtxoRow( 3, 0, 0, 4, 0, null, null, goldenTxid(0x40).toHex(), 1, 100_000L, "", "", 0, - false, false, false, false, + false, false, false, false, healInvalid, ), // DashPay: both identity halves, as lower hex. PlatformWalletPersistenceHandler.EngineUtxoRow( 12, 0, 1, 0, 0, goldenUserIdentity.toHex(), goldenFriendIdentity.toHex(), goldenTxid(0x60).toHex(), 3, 250_000L, "yGoldenFixtureDashPayAddr", goldenScript(0xd4).toHex(), 1_534_000, - true, true, false, false, + true, true, false, false, healImmature, ), ), page.utxos, @@ -8705,7 +8837,7 @@ class PlatformWalletPersistenceHandlerTest { * Drive the paged reconcile from one whole-inventory JSON blob — the * shape these tests describe an engine in — served through [FakeEngine] * in pages of 2, so a test describing more than a couple of rows walks - * the real cursor loop rather than a single page. Every + * the real page loop rather than a single page. Every * still-unattributed TXO is filed under a default BIP44 account first * (see [attributeUnownedTxos]), which is also the account the * inventory's `typeTag 0 / index 0` rows resolve to. @@ -8752,23 +8884,34 @@ class PlatformWalletPersistenceHandlerTest { } /** - * A stand-in for the engine's paged inventory transport, built from the - * whole-inventory JSON a test writes out. Pages come back behind an - * opaque ordinal cursor — the real cursor is opaque too, the handler - * only ever hands back what it was given — and classification answers - * positionally out of the same inventories: 1 unspent, 2 - * known-uncredited (the blob's `spent` list: the engine holds a mined - * record spending the coin), 3 not-owned (its optional `notOwned` - * list), 0 neither. + * A stand-in for the native reconciler of one wallet, built from the + * whole-inventory JSON a test writes out — the run bookkeeping + * `platform_wallet::manager::txo_reconcile` keeps, reduced to what the + * host can observe: a page budget per run ([budget]), the run's place in + * the heal pass and the wallet's checkpoint (moved only by a page + * reported applied), the page last read until it is reported, and the + * guard each read holds until [releaseVerdicts]. Heal rows carry the + * `healVerdict` the test wrote (the verdict is the engine's, never the + * host's); classifications answer positionally out of the same + * inventory: 1 unspent, 2 known-uncredited (the blob's `spent` list), 3 + * not-owned (its optional `notOwned` list), 0 neither. [retryStalePage] + * answers from [staleRetryLimit], a number the fake picks, so a test + * shows the host follows the engine's answer rather than a constant of + * its own. + * + * The guard rules the host must keep are asserted on every call: no + * read and no applied report while the last page's guard is held. */ - private class FakeEngine(inventoryJson: String) { + private class FakeEngine(inventoryJson: String, var budget: Int = 16) : + PlatformWalletPersistenceHandler.TxoReconcileEngine { private val utxos: List private val unspentKeys: Set private val spentKeys: Set private val notOwnedKeys: Set - /** Inventory pages served, classification batches answered, and - * outpoints classified across those batches. */ + /** Heal pages served, classification batches answered, and + * outpoints classified across those batches. Reads past the budget + * serve nothing and are not counted. */ var pages = 0 private set var batches = 0 @@ -8779,8 +8922,44 @@ class PlatformWalletPersistenceHandlerTest { /** Every classification query received, in order. */ val queries = ArrayList() - /** The cursor of every inventory page asked for, in order. */ - val cursors = ArrayList() + /** The inventory row each heal page started at, in order. */ + val healStarts = ArrayList() + + /** Every read, release and applied report, in order: `heal`, + * `classify []` / `classify`, `budget spent`, `release`, + * `heal applied`, and `classify applied ` (`null` for the + * store's last page). */ + val events = ArrayList() + + /** The stale counts the host asked [retryStalePage] about. */ + val staleAsks = ArrayList() + var staleRetryLimit = 3 + + /** Swap a read's answer: [healAnswer] gets the page the fake would + * serve, [classifyAnswer] the verdicts it would give. Either may + * return null (a failed call, which holds nothing). */ + var healAnswer: (suspend (served: String) -> String?)? = null + var classifyAnswer: (suspend (queriesJson: String, verdicts: ByteArray) -> ByteArray?)? = null + + /** The guard of the last page read, until [releaseVerdicts]. */ + var held = false + private set + + private var checkpoint: Checkpoint? = null + private var healAfter = 0 + private var pagesLeft = 0 + private var pending: Pending? = null + + private sealed class Checkpoint { + class Heal(val after: Int) : Checkpoint() + class Classify(val after: ByteArray) : Checkpoint() + } + + private sealed class Pending { + /** A heal page; the row after it, or null for the last page. */ + class Heal(val next: Int?) : Pending() + object Classify : Pending() + } init { val root = kotlinx.serialization.json.Json @@ -8793,21 +8972,76 @@ class PlatformWalletPersistenceHandlerTest { notOwnedKeys = keys("notOwned") } - fun page(cursor: String?, limit: Int): String { + /** Claim a run: a fresh budget, and where the wallet's checkpoint + * says it starts — what the native begin hands the manager. */ + fun begin(): PlatformWalletPersistenceHandler.TxoReconcileStart { + pagesLeft = budget + pending = null + return when (val at = checkpoint) { + null -> { + healAfter = 0 + PlatformWalletPersistenceHandler.TxoReconcileStart.NewCycle + } + is Checkpoint.Heal -> { + healAfter = at.after + PlatformWalletPersistenceHandler.TxoReconcileStart.HealResumed + } + is Checkpoint.Classify -> PlatformWalletPersistenceHandler.TxoReconcileStart.Classify(at.after) + } + } + + /** Take one page from the run's budget; false past it. */ + private fun takePage(): Boolean { + assertFalse("a page is read while the last page's guard is held", held) + if (pagesLeft == 0) { + events += "budget spent" + return false + } + pagesLeft-- + return true + } + + override suspend fun healPage(limit: Int): String? { + if (!takePage()) return """{"utxos":[],"hasMore":false,"budgetSpent":true}""" pages++ - cursors += cursor - val start = cursor?.toInt() ?: 0 - val slice = utxos.drop(start).take(limit) - val next = start + slice.size - val hasMore = next < utxos.size - return """{"utxos":[${slice.joinToString(",") { it.toString() }}],""" + - """"cursor":${if (hasMore) "\"$next\"" else "null"},"hasMore":$hasMore}""" + healStarts += healAfter + events += "heal" + val slice = utxos.drop(healAfter).take(limit) + val next = healAfter + slice.size + val hasMore = slice.isNotEmpty() && next < utxos.size + pending = Pending.Heal(if (hasMore) next else null) + held = true + val served = """{"utxos":[${slice.joinToString(",") { it.toString() }}],""" + + """"hasMore":$hasMore,"budgetSpent":false}""" + val answer = healAnswer.let { if (it == null) served else it(served) } + if (answer == null) { + // A failed read holds nothing, and names no page. + held = false + pending = null + } + return answer + } + + override suspend fun healPageApplied(): Boolean? { + assertFalse("a heal page is reported while its guard is held", held) + events += "heal applied" + (pending as? Pending.Heal)?.let { page -> + page.next?.let { healAfter = it } + checkpoint = page.next?.let { Checkpoint.Heal(it) } ?: Checkpoint.Classify(ByteArray(0)) + } + pending = null + return pagesLeft > 0 } - fun classify(queriesJson: String): ByteArray { + override suspend fun classify(queriesJson: String): PlatformWalletPersistenceHandler.OutpointVerdicts? { + val batch = kotlinx.serialization.json.Json.parseToJsonElement(queriesJson).jsonArray + if (!takePage()) { + return PlatformWalletPersistenceHandler.OutpointVerdicts(ByteArray(batch.size), budgetSpent = true) + } batches++ - val batch = kotlinx.serialization.json.Json - .parseToJsonElement(queriesJson).jsonArray + events += if (batch.isEmpty()) "classify []" else "classify" + pending = Pending.Classify + held = true classified += batch.size val verdicts = ByteArray(batch.size) for ((i, query) in batch.withIndex()) { @@ -8830,7 +9064,33 @@ class PlatformWalletPersistenceHandlerTest { else -> 0 } } - return verdicts + val answer = classifyAnswer.let { if (it == null) verdicts else it(queriesJson, verdicts) } + if (answer == null) { + held = false + pending = null + return null + } + return PlatformWalletPersistenceHandler.OutpointVerdicts(answer) + } + + override suspend fun classifyPageApplied(nextAfter: ByteArray?): Boolean? { + assertFalse("a store page is reported while its guard is held", held) + events += "classify applied ${nextAfter?.toHex()}" + if (pending == Pending.Classify) { + checkpoint = nextAfter?.let { Checkpoint.Classify(it) } + } + pending = null + return pagesLeft > 0 + } + + override suspend fun releaseVerdicts() { + if (held) events += "release" + held = false + } + + override fun retryStalePage(staleInARow: Int): Boolean { + staleAsks += staleInARow + return staleInARow <= staleRetryLimit } private fun key(txidHex: String, vout: Int) = "$txidHex:$vout" diff --git a/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcileRunEndTest.kt b/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcileRunEndTest.kt new file mode 100644 index 00000000000..a8f770c2486 --- /dev/null +++ b/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcileRunEndTest.kt @@ -0,0 +1,52 @@ +package org.dashfoundation.dashsdk.wallet + +import org.junit.Assert.assertEquals +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +/** + * The end of a TXO-store reconcile run ([endTxoReconcileRun]): it frees + * the held verdicts first, then releases the native claim, and a failure + * of either never skips the other — Swift's `endCoreTxoReconcileRun`. + * Robolectric, for the warning each failure logs. + */ +@RunWith(RobolectricTestRunner::class) +class TxoReconcileRunEndTest { + + @Test + fun theHeldVerdictsAreFreedBeforeTheClaimIsReleased() { + val steps = ArrayList() + endTxoReconcileRun( + walletRef = "ref", + freeHeldVerdicts = { steps += "free" }, + finish = { steps += "finish" }, + ) + assertEquals(listOf("free", "finish"), steps) + } + + @Test + fun aFailedFreeStillReleasesTheClaim() { + val steps = ArrayList() + endTxoReconcileRun( + walletRef = "ref", + freeHeldVerdicts = { + steps += "free" + throw IllegalStateException("native free failed") + }, + finish = { steps += "finish" }, + ) + assertEquals("finish runs after a failed free", listOf("free", "finish"), steps) + } + + @Test + fun aFailedFinishIsLoggedNotThrown() { + var freed = 0 + endTxoReconcileRun( + walletRef = "ref", + freeHeldVerdicts = { freed++ }, + finish = { throw IllegalStateException("handle gone") }, + ) + assertEquals(1, freed) + } +} diff --git a/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcileTest.kt b/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcileTest.kt index 13b872c9dbd..d089a36a02f 100644 --- a/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcileTest.kt +++ b/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcileTest.kt @@ -1,285 +1,101 @@ package org.dashfoundation.dashsdk.wallet +import kotlinx.coroutines.CompletableDeferred +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.test.StandardTestDispatcher +import kotlinx.coroutines.test.TestScope +import kotlinx.coroutines.test.advanceUntilIdle +import kotlinx.coroutines.test.runTest import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse -import org.junit.Assert.assertNull import org.junit.Assert.assertTrue import org.junit.Test /** - * The manager-side gates and trigger of the TXO-store reconcile — port of - * the steady-state and scheduling cases of Swift's `CoreTxoReconcileTests` - * / `CoreTxoReconcileShutdownTests` (dashpay/platform#4638). Pure: no - * native manager is needed. + * The Kotlin side of the TXO-store reconcile's start gate and trigger. The + * gates, the schedule and their thresholds live in + * `platform_wallet::manager::txo_reconcile` and are tested there; what + * Kotlin owns is reading the gate's answer and not launching a second run + * of a wallet while its first has not ended. */ class TxoReconcileTest { - private fun progress( - state: SpvSyncState, - filters: Pair?, - headers: Long = 2_535_898, - ) = SpvSyncProgressData( - overallState = state, - overallPercentage = 0.0, - headers = SpvSubProgress(state, headers, headers, 1.0), - filterHeaders = null, - filters = filters?.let { (current, target) -> SpvSubProgress(state, current, target, 0.0) }, - masternodes = null, - ) - - /** [TxoReconcileCoordinator.walletsDue] for a schedule whose every - * launched run ends before the next tick — the cases that are about - * the edge and the cadence, not about a launch still pending. */ - private fun TxoReconcileCoordinator.dueWithLaunchesEnded( - progress: SpvSyncProgressData, - walletKeys: Collection, - nowMs: Long, - ): List = walletsDue(progress, walletKeys, nowMs).onEach { launchEnded(it) } - - private val steady = progress(SpvSyncState.WAIT_FOR_EVENTS, filters = 2_535_898L to 2_535_898L) - private val syncing = progress(SpvSyncState.SYNCING, filters = 199_999L to 2_535_898L) - - @Test - fun theSteadyStateGateRefusesAnUnfinishedScan() { - assertFalse(TxoReconcileGates.isSteadySyncState(syncing)) - assertFalse(TxoReconcileGates.isSteadySyncState(progress(SpvSyncState.WAITING_FOR_CONNECTIONS, filters = null))) - assertFalse( - TxoReconcileGates.isSteadySyncState(progress(SpvSyncState.ERROR, filters = 2_535_898L to 2_535_898L)), - ) - assertFalse( - "waiting for events with the filter phase behind its target is a scan still running", - TxoReconcileGates.isSteadySyncState(progress(SpvSyncState.WAIT_FOR_EVENTS, filters = 2_535_800L to 2_535_898L)), - ) - assertFalse( - "no filter phase at all proves nothing", - TxoReconcileGates.isSteadySyncState(progress(SpvSyncState.WAIT_FOR_EVENTS, filters = null)), - ) - assertFalse( - "a filter phase with no target proves nothing", - TxoReconcileGates.isSteadySyncState(progress(SpvSyncState.WAIT_FOR_EVENTS, filters = 0L to 0L)), - ) - assertFalse( - "the poll's not-running placeholder is not steady", - TxoReconcileGates.isSteadySyncState(SpvSyncProgressData.EMPTY), - ) - assertTrue(TxoReconcileGates.isSteadySyncState(progress(SpvSyncState.SYNCED, filters = 2_535_898L to 2_535_898L))) - assertTrue("dash-spv's fully synced steady state", TxoReconcileGates.isSteadySyncState(steady)) - } - - @Test - fun theTipIsTheFilterHeightFallingBackToTheHeaderTip() { - assertEquals(2_535_898, TxoReconcileGates.scanTipHeight(steady)) - assertEquals( - "falls back to the header tip", - 2_535_898, - TxoReconcileGates.scanTipHeight(progress(SpvSyncState.SYNCED, filters = null)), - ) - assertEquals( - "a filter phase at height 0 falls back too", - 2_535_000, - TxoReconcileGates.scanTipHeight(progress(SpvSyncState.SYNCED, filters = 0L to 0L, headers = 2_535_000)), - ) - assertNull(TxoReconcileGates.scanTipHeight(SpvSyncProgressData.EMPTY)) - } - - @Test - fun everyLoadedWalletRunsOnTheRisingEdgeIntoSteadyState() { - val coordinator = TxoReconcileCoordinator(cadenceMs = 1_000) - val wallets = listOf("a", "b") - - assertEquals(emptyList(), coordinator.dueWithLaunchesEnded(syncing, wallets, nowMs = 0)) - assertEquals(wallets, coordinator.dueWithLaunchesEnded(steady, wallets, nowMs = 10)) - assertEquals( - "the next tick inside the cadence schedules nothing", - emptyList(), - coordinator.dueWithLaunchesEnded(steady, wallets, nowMs = 20), - ) - } - - @Test - fun aSteadyClientRunsEachWalletAgainOnItsOwnCadence() { - val coordinator = TxoReconcileCoordinator(cadenceMs = 1_000) - assertEquals(listOf("a"), coordinator.dueWithLaunchesEnded(steady, listOf("a"), nowMs = 0)) - // A wallet loaded while steady never ran: it is due at once, on its - // own clock. - assertEquals(listOf("b"), coordinator.dueWithLaunchesEnded(steady, listOf("a", "b"), nowMs = 500)) - assertEquals(listOf("a"), coordinator.dueWithLaunchesEnded(steady, listOf("a", "b"), nowMs = 1_000)) - assertEquals(listOf("b"), coordinator.dueWithLaunchesEnded(steady, listOf("a", "b"), nowMs = 1_500)) - // A completed run restarts that wallet's cadence. - assertTrue(coordinator.markRun("a", nowMs = 1_900, runEpoch = coordinator.currentEpoch)) - assertEquals(emptyList(), coordinator.dueWithLaunchesEnded(steady, listOf("a", "b"), nowMs = 2_000)) - assertEquals(listOf("a", "b"), coordinator.dueWithLaunchesEnded(steady, listOf("a", "b"), nowMs = 2_900)) - } - - @Test - fun aWalletWithARunInFlightIsNeverScheduledOrStartedAgain() { - val coordinator = TxoReconcileCoordinator(cadenceMs = 1_000) - assertTrue(coordinator.tryBegin("a")) - assertFalse("a second run for the same wallet is refused", coordinator.tryBegin("a")) - assertEquals( - "not even on the rising edge", - listOf("b"), - coordinator.walletsDue(steady, listOf("a", "b"), nowMs = 0), - ) - coordinator.end("a") - assertTrue(coordinator.tryBegin("a")) - } - - @Test - fun aLaunchedRunThatHasNotClaimedYetIsNotScheduledAgain() { - // The run is launched but its coroutine has not reached tryBegin - // yet, and a new block takes the client out of steady state and - // back: that rising edge must not launch the wallet a second time. - val coordinator = TxoReconcileCoordinator(cadenceMs = 1_000_000) - assertEquals(listOf("a"), coordinator.walletsDue(steady, listOf("a"), nowMs = 0)) - assertEquals(emptyList(), coordinator.walletsDue(syncing, listOf("a"), nowMs = 10)) - assertEquals( - "a rising edge while the launch is pending skips it", - emptyList(), - coordinator.walletsDue(steady, listOf("a"), nowMs = 20), - ) - - // The launched run starts, claims, and ends. - assertTrue(coordinator.tryBegin("a")) - coordinator.end("a") - coordinator.launchEnded("a") - assertEquals(emptyList(), coordinator.walletsDue(syncing, listOf("a"), nowMs = 30)) - assertEquals( - "the next rising edge after the launch ended schedules it again", - listOf("a"), - coordinator.walletsDue(steady, listOf("a"), nowMs = 40), - ) - } - @Test - fun aRefusedClaimSkipsAndNeverQueues() { - // A host call racing a scheduled run: the loser's tryBegin is - // refused (reconcileTxoStore then skips with ALREADY_RUNNING, it does - // not wait), and nothing is left behind to run later. - val coordinator = TxoReconcileCoordinator(cadenceMs = 1_000_000) - assertEquals(listOf("a"), coordinator.walletsDue(steady, listOf("a"), nowMs = 0)) - assertTrue("the scheduled run claims", coordinator.tryBegin("a")) - assertFalse("the host call is refused", coordinator.tryBegin("a")) - coordinator.end("a") - coordinator.launchEnded("a") + fun everyNativeSkipCodeMapsToItsReason() { + // rs-platform-wallet-ffi `TXO_RECONCILE_SKIP_*`, in code order. + val expected = mapOf( + 1 to TxoReconcileSkipReason.WALLET_UNKNOWN, + 2 to TxoReconcileSkipReason.ALREADY_RUNNING, + 3 to TxoReconcileSkipReason.NOT_STEADY_STATE, + 4 to TxoReconcileSkipReason.TIP_UNAVAILABLE, + 5 to TxoReconcileSkipReason.SYNC_FAULT_DETECTED, + 6 to TxoReconcileSkipReason.WALLET_BEHIND_TIP, + ) + for ((code, reason) in expected) { + assertEquals(reason, TxoReconcileSkipReason.fromGateCode(code)) + } assertEquals( - "inside the cadence, with no edge, nothing runs again", - emptyList(), - coordinator.walletsDue(steady, listOf("a"), nowMs = 10), + "only the manager's own closed check has no native code", + listOf(TxoReconcileSkipReason.MANAGER_CLOSED), + TxoReconcileSkipReason.entries.filter { it.gateCode == null }, ) - assertTrue("and the wallet can be claimed afresh", coordinator.tryBegin("a")) } @Test - fun leavingSteadyStateArmsTheNextRisingEdge() { - val coordinator = TxoReconcileCoordinator(cadenceMs = 1_000_000) - assertEquals(listOf("a"), coordinator.dueWithLaunchesEnded(steady, listOf("a"), nowMs = 0)) - assertEquals(emptyList(), coordinator.dueWithLaunchesEnded(syncing, listOf("a"), nowMs = 10)) - assertEquals( - "back in steady state: due again, cadence or not", - listOf("a"), - coordinator.dueWithLaunchesEnded(steady, listOf("a"), nowMs = 20), - ) + fun aCodeThisBuildDoesNotKnowIsAContractBreakNotASkip() { + for (code in listOf(0, 7, -1)) { + val thrown = runCatching { TxoReconcileSkipReason.fromGateCode(code) }.exceptionOrNull() + assertTrue("code $code must not map to a reason, got $thrown", thrown is IllegalStateException) + } } @Test - fun stoppingSpvArmsTheNextRisingEdgeWithoutASyncingRead() { - // SPV stops and restarts straight into steady state: the poll feeds - // no progress while SPV is down and the first read after the - // restart is already steady, so no syncing read ever lowers the - // edge. The stop itself must, or every wallet waits out the cadence. - val coordinator = TxoReconcileCoordinator(cadenceMs = 1_000_000) - val wallets = listOf("a", "b") - assertEquals(wallets, coordinator.dueWithLaunchesEnded(steady, wallets, nowMs = 0)) - assertEquals(emptyList(), coordinator.dueWithLaunchesEnded(steady, wallets, nowMs = 10)) - - coordinator.spvStopped() - - assertEquals( - "the first steady read after the restart is a new steady state", - wallets, - coordinator.dueWithLaunchesEnded(steady, wallets, nowMs = 20), - ) - assertEquals( - "and only the first", - emptyList(), - coordinator.dueWithLaunchesEnded(steady, wallets, nowMs = 30), - ) - } - - @Test - fun aRunOvertakenByARemovalDoesNotStampTheReaddedWalletsCadence() { - // A run captured the epoch and was still going when its wallet was - // removed (epoch bumped, cadence forgotten) and the same id added - // back. Its finish must not stamp the cadence: the re-added wallet - // has never been reconciled and is due on the next tick. - val coordinator = TxoReconcileCoordinator(cadenceMs = 1_000_000) - assertEquals(listOf("a"), coordinator.walletsDue(steady, listOf("a"), nowMs = 0)) - assertTrue(coordinator.tryBegin("a")) - val runEpoch = coordinator.currentEpoch - - // removeWallet: bump first, then forget; addWallet puts "a" back. - coordinator.bumpEpoch() - coordinator.forget("a") - assertEquals( - "in flight, so not scheduled yet", - emptyList(), - coordinator.walletsDue(steady, listOf("a"), nowMs = 10), - ) - - assertFalse(coordinator.markRun("a", nowMs = 20, runEpoch = runEpoch)) - coordinator.end("a") - coordinator.launchEnded("a") - assertEquals( - "the re-added wallet is due at once, not a cadence later", - listOf("a"), - coordinator.walletsDue(steady, listOf("a"), nowMs = 30), - ) - } - - @Test - fun aRunStampedBeforeTheRemovalIsClearedByIt() { - // The other order: the run finished and stamped before the removal. - // The removal's forget clears that stamp, so the re-added wallet is - // still due at once. - val coordinator = TxoReconcileCoordinator(cadenceMs = 1_000_000) - assertEquals(listOf("a"), coordinator.walletsDue(steady, listOf("a"), nowMs = 0)) - assertTrue(coordinator.markRun("a", nowMs = 5, runEpoch = coordinator.currentEpoch)) - coordinator.launchEnded("a") - coordinator.bumpEpoch() - coordinator.forget("a") - assertEquals(listOf("a"), coordinator.walletsDue(steady, listOf("a"), nowMs = 10)) - } - - @Test - fun closeAndDeletionStaleTheEpochAndDeletionForgetsTheCadence() { - val coordinator = TxoReconcileCoordinator(cadenceMs = 1_000) - val before = coordinator.currentEpoch - coordinator.bumpEpoch() - assertTrue("an in-flight run sees its epoch move", coordinator.currentEpoch != before) - - assertEquals(listOf("a"), coordinator.dueWithLaunchesEnded(steady, listOf("a"), nowMs = 0)) - coordinator.forget("a") - assertEquals( - "a re-added wallet is due at once", - listOf("a"), - coordinator.dueWithLaunchesEnded(steady, listOf("a"), nowMs = 10), - ) + fun aWalletWhoseLaunchedRunHasNotEndedIsNotLaunchedAgain() = runTest { + // The native schedule reports a wallet due on every tick until a run + // claims it; a run still queued must not get a second one behind it. + val launches = TxoReconcileLaunches() + val release = CompletableDeferred() + var runs = 0 + suspend fun run() { + runs++ + release.await() + } + + assertTrue(launches.launch(this, "a") { run() }) + assertFalse("a launch of the same wallet is pending", launches.launch(this, "a") { run() }) + assertTrue("another wallet launches on its own", launches.launch(this, "b") { runs++ }) + advanceUntilIdle() + assertEquals(2, runs) + assertTrue("a" in launches) + assertFalse("b" in launches) + + release.complete(Unit) + advanceUntilIdle() + assertFalse("an ended run releases its wallet", "a" in launches) + assertTrue(launches.launch(this, "a") { runs++ }) + advanceUntilIdle() + assertEquals(3, runs) } @Test - fun theWalletGateUsesTheEnginesWatermarkWithinTheTipMargin() { - // Swift's `synced_height + margin >= tip`. The value passed is the - // ENGINE's watermark: the persisted Room copy trails it by a round, - // and gating on that skipped the run at the SYNCED edge of a restore - // (2026-10-02 device run: WALLET_BEHIND_TIP at the transition, the - // next attempt 30 minutes later). - assertTrue(TxoReconcileGates.walletCaughtUp(walletSyncedHeight = 1_565_211L, tipHeight = 1_565_211)) - assertTrue(TxoReconcileGates.walletCaughtUp(walletSyncedHeight = 1_565_205L, tipHeight = 1_565_211)) - assertFalse(TxoReconcileGates.walletCaughtUp(walletSyncedHeight = 1_565_204L, tipHeight = 1_565_211)) - // A wallet added behind the tip is still being scanned. - assertFalse(TxoReconcileGates.walletCaughtUp(walletSyncedHeight = 0L, tipHeight = 1_565_211)) - // The native "no value" sentinel never passes. - assertFalse(TxoReconcileGates.walletCaughtUp(walletSyncedHeight = -1L, tipHeight = 3)) + fun aRunThatThrowsOrIsCancelledBeforeItStartsStillReleasesItsWallet() { + val launches = TxoReconcileLaunches() + val scope = TestScope(StandardTestDispatcher()) + + // A scope whose job is already cancelled: the body never runs, but + // the job completes, and the wallet is released with it. + val cancelled = CoroutineScope(scope.coroutineContext + Job().apply { cancel() }) + var ran = false + assertTrue(launches.launch(cancelled, "a") { ran = true }) + scope.advanceUntilIdle() + assertFalse(ran) + assertFalse("a launch that never ran releases its wallet", "a" in launches) + + val handler = kotlinx.coroutines.CoroutineExceptionHandler { _, _ -> } + val failing = CoroutineScope(scope.coroutineContext + Job() + handler) + assertTrue(launches.launch(failing, "b") { error("run failed") }) + scope.advanceUntilIdle() + assertFalse("a run that threw releases its wallet", "b" in launches) } } diff --git a/packages/kotlin-sdk/sdk/src/test/resources/golden/txo_reconcile_transport_v1.json b/packages/kotlin-sdk/sdk/src/test/resources/golden/txo_reconcile_transport_v1.json index 0a6a151d66c..7d5b3bc9c65 100644 --- a/packages/kotlin-sdk/sdk/src/test/resources/golden/txo_reconcile_transport_v1.json +++ b/packages/kotlin-sdk/sdk/src/test/resources/golden/txo_reconcile_transport_v1.json @@ -1,5 +1,5 @@ { - "_about": "Golden TXO-reconcile transport payloads, read by rs-unified-sdk-jni (wallet_manager::tests) and the Kotlin PlatformWalletPersistenceHandlerTest. inventoryPage is exactly what walletManagerUtxosPageJson emits for the four accessor entries the Rust test builds; outpointQueries is exactly what the Kotlin reconcile sends walletManagerClassifyOutpoints for the same four coins. Change both sides together.", + "_about": "Golden TXO-reconcile transport payloads, read by rs-unified-sdk-jni (wallet_manager::tests) and the Kotlin PlatformWalletPersistenceHandlerTest. inventoryPage is exactly what walletManagerTxoHealPageJson emits for the four heal-page rows the Rust test builds; outpointQueries is exactly what the Kotlin reconcile sends walletManagerTxoReconcileClassifyOutpoints for the same four coins. Change both sides together.", "inventoryPage": { "utxos": [ { @@ -17,7 +17,8 @@ "isConfirmed": true, "isInstantlocked": false, "isCoinbase": false, - "isLocked": false + "isLocked": false, + "healVerdict": 2 }, { "typeTag": 0, @@ -34,7 +35,8 @@ "isConfirmed": true, "isInstantlocked": true, "isCoinbase": true, - "isLocked": true + "isLocked": true, + "healVerdict": 2 }, { "typeTag": 3, @@ -51,7 +53,8 @@ "isConfirmed": false, "isInstantlocked": false, "isCoinbase": false, - "isLocked": false + "isLocked": false, + "healVerdict": 0 }, { "typeTag": 12, @@ -70,11 +73,12 @@ "isConfirmed": true, "isInstantlocked": true, "isCoinbase": false, - "isLocked": false + "isLocked": false, + "healVerdict": 1 } ], - "cursor": "{\"typeTag\":12,\"standardTag\":0,\"index\":1,\"registrationIndex\":0,\"keyClass\":0,\"userIdentityId\":\"a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1\",\"friendIdentityId\":\"b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2\",\"txid\":\"606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f\",\"vout\":3}", - "hasMore": true + "hasMore": true, + "budgetSpent": false }, "outpointQueries": [ { diff --git a/packages/rs-unified-sdk-jni/src/wallet_manager.rs b/packages/rs-unified-sdk-jni/src/wallet_manager.rs index 3897ee89ccc..263c0481009 100644 --- a/packages/rs-unified-sdk-jni/src/wallet_manager.rs +++ b/packages/rs-unified-sdk-jni/src/wallet_manager.rs @@ -3064,43 +3064,6 @@ pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_s }) } -/// The wallet's core scan watermark as the ENGINE holds it -/// (`platform_wallet_core_wallet_state`'s `synced_height`) — the value the -/// TXO-store reconcile gates on, as Swift's does. The persisted copy in the -/// host store trails it by up to one persistence round, which is exactly -/// the window the SYNCED transition lands in. Throws `NotFound` when the -/// manager does not hold the wallet. Backs -/// `PlatformWalletManager.reconcileTxoStore`. -#[no_mangle] -pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_walletManagerCoreWalletSyncedHeight( - mut env: JNIEnv, - _class: JClass, - manager_handle: jlong, - wallet_id: JByteArray, -) -> jlong { - guard(&mut env, -1, |env| { - let Some(wid) = read_id32(env, &wallet_id) else { - return -1; - }; - let mut state = platform_wallet_ffi::CoreWalletStateFFI { - synced_height: 0, - last_processed_height: 0, - monitor_revision: 0, - }; - let result = unsafe { - platform_wallet_ffi::platform_wallet_core_wallet_state( - manager_handle as Handle, - wid.as_ptr(), - &mut state as *mut platform_wallet_ffi::CoreWalletStateFFI, - ) - }; - if take_pwffi_error(env, result) { - return -1; - } - jlong::from(state.synced_height) - }) -} - #[cfg(feature = "shielded")] sync_start_stop!( Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_shieldedSyncStart, @@ -3659,15 +3622,20 @@ fn core_selection_strategy( } } -// ── TXO-store reconcile transport ───────────────────────────────────── +// ── TXO-store reconcile ─────────────────────────────────────────────── // -// Two exports, each a serialization shim over exactly ONE platform-wallet -// call: `platform_wallet_wallet_utxos_page` and -// `platform_wallet_classify_outpoints`. The account ordering, the cursor -// semantics, the page bound and every classification rule live in -// `platform-wallet`, so the Swift host walks the identical inventory and -// gets the identical verdicts. Nothing here decides anything; it moves -// serde-shaped JSON in and out. +// Each export bridges exactly ONE platform-wallet call over the reconcile +// policy in `platform_wallet::manager::txo_reconcile`: the schedule +// (`platform_wallet_manager_txo_reconcile_due_wallets`), the start gate and +// its release (`…_txo_reconcile_begin` / `…_finish`), the held heal page +// (`platform_wallet_txo_reconcile_heal_page`), the held spend verdicts +// (`platform_wallet_txo_reconcile_classify_outpoints`), the release of the +// engine guard either is held under (`…_held_verdicts_free`), the two +// applied reports (`…_heal_page_applied` / `…_classify_page_applied`) and +// the stale-page retry (`platform_wallet_txo_reconcile_retry_stale_page`). +// The cadence, the gates, the heal verdicts, every classification rule, the +// page budget and where a run resumes live there. Nothing here decides +// anything; it moves values and serde-shaped JSON in and out. /// The account tuple that owns one inventory row — the raw `AccountSpecFFI` /// tag layout minus the xpub, in the field names the Kotlin `EngineUtxoRow` @@ -3741,12 +3709,14 @@ fn hex32(s: &str) -> Option<[u8; 32]> { hex::decode(s).ok()?.try_into().ok() } -/// One row of an inventory page — the typed contract the Kotlin -/// `EngineUtxoRow` decodes, strictly (an unknown or missing key throws -/// there rather than healing a defaulted row). `txid` is lower hex in the -/// same byte order the changeset path hands Kotlin, so hex→bytes -/// reproduces the `txos.txid` blob; `address` is the engine's own -/// Base58Check rendering, empty when the script has no address form. +/// One row of a heal page — the typed contract the Kotlin `EngineUtxoRow` +/// decodes, strictly (an unknown or missing key throws there rather than +/// healing a defaulted row). `txid` is lower hex in the same byte order the +/// changeset path hands Kotlin, so hex→bytes reproduces the `txos.txid` +/// blob; `address` is the engine's own Base58Check rendering, empty when +/// the script has no address form; `healVerdict` is the engine's +/// `TXO_HEAL_VERDICT_*` for the row, the only thing that says whether the +/// store may heal it in. #[derive(serde::Serialize)] #[serde(rename_all = "camelCase")] struct UtxoPageRow { @@ -3762,26 +3732,27 @@ struct UtxoPageRow { is_instantlocked: bool, is_coinbase: bool, is_locked: bool, + heal_verdict: u8, } impl UtxoPageRow { - /// The row for one accessor entry, carrying the engine's own address - /// and flags. + /// The row for one heal-page entry, carrying the engine's own address, + /// flags and verdict. /// /// # Safety /// - /// `e.address` must be null or a valid NUL-terminated string, and - /// `e.script_pubkey` null or valid for `e.script_pubkey_len` bytes — - /// the accessor's contract for every entry it returns. - unsafe fn from_entry(e: &platform_wallet_ffi::WalletUtxoEntryFFI) -> Self { + /// `r.utxo.address` must be null or a valid NUL-terminated string, and + /// `r.utxo.script_pubkey` null or valid for `r.utxo.script_pubkey_len` + /// bytes — the heal page's contract for every row it returns. + unsafe fn from_heal_row(r: &platform_wallet_ffi::TxoHealRowFFI) -> Self { + let e = &r.utxo; let script: &[u8] = if e.script_pubkey.is_null() || e.script_pubkey_len == 0 { &[] } else { std::slice::from_raw_parts(e.script_pubkey, e.script_pubkey_len) }; // Never null per the accessor's contract; treated as empty if it - // ever is, which the Kotlin heal skips rather than filing a row it - // cannot key an address by. + // ever is (the engine's verdict on such a row is `invalid`). let address = cstr_opt(e.address).unwrap_or_default(); UtxoPageRow { account: UtxoAccountTuple::from_entry(e), @@ -3795,79 +3766,41 @@ impl UtxoPageRow { is_instantlocked: e.is_instantlocked, is_coinbase: e.is_coinbase, is_locked: e.is_locked, + heal_verdict: r.verdict, } } } -/// One page: the rows, the opaque resume cursor (serialized as `null` on -/// the last page, never omitted) and whether more pages follow. The Kotlin -/// `EngineUtxoPage` requires all three keys. +/// One heal page: the rows, whether more pages follow, and whether the run +/// had already used its page budget, in which case nothing was read. The +/// run keeps its own place in the inventory, so the page carries no +/// cursor. The Kotlin `EngineUtxoPage` requires all three keys. #[derive(serde::Serialize)] #[serde(rename_all = "camelCase")] struct UtxoPage { utxos: Vec, - cursor: Option, has_more: bool, + budget_spent: bool, } impl UtxoPage { - /// The page for `utxos`. The cursor is the last row itself; a page with - /// no rows has nowhere to resume from, and the accessor reports no more - /// in that case. Fails only if the cursor cannot be encoded. - fn new(utxos: Vec, has_more: bool) -> serde_json::Result { - let cursor = match utxos.last() { - Some(last) if has_more => Some(serde_json::to_string(&UtxoPageCursor::after(last))?), - _ => None, - }; - Ok(UtxoPage { - // `cursor` is set exactly when `has_more` is and a row exists. - has_more: cursor.is_some(), - cursor, + /// The page for `utxos`. A page with no rows reports no more, whatever + /// the accessor said: there is nothing after it to read. + fn new(utxos: Vec, has_more: bool) -> Self { + UtxoPage { + has_more: has_more && !utxos.is_empty(), utxos, - }) - } -} - -/// Where a paged inventory sweep left off — the last row's account tuple -/// and outpoint, which is exactly what `platform_wallet_wallet_utxos_page` -/// resumes from. Serialized as JSON and handed to the host as an opaque -/// string; the host only ever gives back what a previous page returned. -#[derive(serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "camelCase")] -struct UtxoPageCursor { - #[serde(flatten)] - account: UtxoAccountTuple, - txid: String, - vout: u32, -} - -impl UtxoPageCursor { - /// The cursor that resumes the sweep after `last`. - fn after(last: &UtxoPageRow) -> Self { - UtxoPageCursor { - account: last.account.clone(), - txid: last.txid.clone(), - vout: last.vout, + budget_spent: false, } } - /// The resume point a cursor string names, or `None` when it is not - /// one a previous page returned (unparseable, or a malformed identity - /// half or txid). - fn decode(raw: &str) -> Option { - let c = serde_json::from_str::(raw).ok()?; - let (user_identity_id, friend_identity_id, outpoint) = - c.account.keyed_outpoint(&c.txid, c.vout).ok()?; - Some(platform_wallet_ffi::WalletUtxoCursorFFI { - type_tag: c.account.type_tag, - standard_tag: c.account.standard_tag, - index: c.account.index, - registration_index: c.account.registration_index, - key_class: c.account.key_class, - user_identity_id, - friend_identity_id, - outpoint, - }) + /// No page: the run had used its budget. + fn budget_spent() -> Self { + UtxoPage { + utxos: Vec::new(), + has_more: false, + budget_spent: true, + } } } @@ -3915,43 +3848,301 @@ impl OutpointQuery { } } +/// The wallets whose TXO-store reconcile is due now, as a flat `byte[]` of +/// concatenated 32-byte ids. Bridges +/// `platform_wallet_manager_txo_reconcile_due_wallets`: the steady-state +/// edge and the cadence live there. Reporting a wallet consumes none of its +/// cadence, so one the host skips is reported again. Called on every SPV +/// progress poll, running or not. Backs `PlatformWalletManager`'s progress +/// poll. +#[no_mangle] +pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_walletManagerTxoReconcileDueWallets( + mut env: JNIEnv, + _class: JClass, + manager_handle: jlong, +) -> jbyteArray { + guard(&mut env, ptr::null_mut(), |env| { + let mut out = IdentifierArray { + items: ptr::null_mut(), + count: 0, + }; + let result = unsafe { + platform_wallet_ffi::platform_wallet_manager_txo_reconcile_due_wallets( + manager_handle as Handle, + &mut out as *mut IdentifierArray, + ) + }; + if take_pwffi_error(env, result) { + return ptr::null_mut(); + } + identifier_array_to_flat(env, out) + }) +} + +/// Claim one wallet for a TXO-store reconcile run and check that it may +/// start. Returns `TXO_RECONCILE_GATE_RUN` (0) or a `TXO_RECONCILE_SKIP_*` +/// reason; on 0, `outTipHeight[0]` is the scan tip the run is measured +/// against (the claim keeps it; the host only reports it), and the claim +/// holds until `walletManagerTxoReconcileFinish`. Bridges +/// `platform_wallet_manager_txo_reconcile_begin`, where every gate lives. +/// +/// On 0 `outStart[0]` is where the run starts, a `TXO_RECONCILE_START_*` +/// (0 a new cycle at the heal pass, 1 the heal pass where the last run +/// stopped, 2 the classify pass), and for 2 `outStoreAfter[0]` is the +/// store cursor to start after, as the host handed it to +/// `walletManagerTxoReconcileClassifyPageApplied` (empty: the first row); +/// otherwise it is null. The out-arrays are checked before the claim is +/// taken, and a claim this export could not hand to Kotlin is released +/// before it returns. +#[no_mangle] +pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_walletManagerTxoReconcileBegin( + mut env: JNIEnv, + _class: JClass, + manager_handle: jlong, + wallet_id: JByteArray, + out_tip_height: jni::sys::jintArray, + out_start: jni::sys::jintArray, + out_store_after: jni::sys::jobjectArray, +) -> jint { + guard(&mut env, -1, |env| { + let Some(wid) = read_id32(env, &wallet_id) else { + return -1; + }; + const OUTS: &str = + "outTipHeight and outStart must be IntArray(1), outStoreAfter an Array(1)"; + if out_tip_height.is_null() || out_start.is_null() || out_store_after.is_null() { + throw_sdk_exception(env, 1, OUTS); + return -1; + } + // SAFETY: non-null arrays from the Kotlin caller. + let out_tip = unsafe { jni::objects::JIntArray::from_raw(out_tip_height) }; + let out_start = unsafe { jni::objects::JIntArray::from_raw(out_start) }; + let out_store_after = unsafe { JObjectArray::from_raw(out_store_after) }; + // Checked before the claim is taken: a wrong-sized array would + // otherwise fail a write below with the wallet already claimed. + if !has_out_slot(env.get_array_length(&out_tip)) + || !has_out_slot(env.get_array_length(&out_start)) + || !has_out_slot(env.get_array_length(&out_store_after)) + { + throw_sdk_exception(env, 1, OUTS); + return -1; + } + let mut gate = 0u8; + let mut tip_height = 0u32; + let mut start = platform_wallet_ffi::TXO_RECONCILE_START_HEAL; + let mut store_after = [0u8; platform_wallet_ffi::TXO_RECONCILE_STORE_CURSOR_MAX]; + let mut store_after_len = 0usize; + let result = unsafe { + platform_wallet_ffi::platform_wallet_manager_txo_reconcile_begin( + manager_handle as Handle, + wid.as_ptr(), + &mut gate as *mut u8, + &mut tip_height as *mut u32, + &mut start as *mut u8, + store_after.as_mut_ptr(), + &mut store_after_len as *mut usize, + ) + }; + if take_pwffi_error(env, result) { + return -1; + } + if gate != platform_wallet_ffi::TXO_RECONCILE_GATE_RUN { + // A skip holds no claim. + return jint::from(gate); + } + // Kotlin releases the claim only after a RUN reaches it. Until then + // every exit — a failed write, a panic `guard` catches — finishes + // the run as one that did not go ahead. The finish resolves the + // manager in the native handle registry, which is safe here: begin + // holds no engine guard, so this cannot join the registry/engine + // lock cycle a held-verdicts release must stay out of. + let claim = UntilHandedOver::new(move || unsafe { + let _ = platform_wallet_ffi::platform_wallet_manager_txo_reconcile_finish( + manager_handle as Handle, + wid.as_ptr(), + false, + ); + }); + // A tip past i32::MAX cannot cross as a jint; no Dash height is near it. + let tip = jint::try_from(tip_height).unwrap_or(jint::MAX); + if env.set_int_array_region(&out_tip, 0, &[tip]).is_err() + || env + .set_int_array_region(&out_start, 0, &[jint::from(start)]) + .is_err() + { + return -1; + } + let after = if start == platform_wallet_ffi::TXO_RECONCILE_START_CLASSIFY { + let len = store_after_len.min(store_after.len()); + match env.byte_array_from_slice(&store_after[..len]) { + Ok(bytes) => JObject::from(bytes), + Err(_) => return -1, + } + } else { + JObject::null() + }; + if env + .set_object_array_element(&out_store_after, 0, after) + .is_err() + { + return -1; + } + claim.hand_over(); + jint::from(gate) + }) +} + +/// Whether a Kotlin out-array of `length` (as `get_array_length` returned +/// it) has the one slot an export writes. +fn has_out_slot(length: jni::errors::Result) -> bool { + matches!(length, Ok(n) if n >= 1) +} + +/// Something a JNI export obtained natively and has not handed to Kotlin +/// yet. Dropped before [`Self::hand_over`] — an early return, or a panic +/// unwinding into `guard` — it runs `release` once. +struct UntilHandedOver { + release: Option, +} + +impl UntilHandedOver { + fn new(release: F) -> Self { + Self { + release: Some(release), + } + } + + /// Kotlin owns it now. + fn hand_over(mut self) { + self.release = None; + } +} + +impl Drop for UntilHandedOver { + fn drop(&mut self) { + if let Some(release) = self.release.take() { + release(); + } + } +} + +/// Release the claim a 0 from `walletManagerTxoReconcileBegin` granted, +/// and, as a backstop, any engine guard the run still holds; `ran` is +/// whether the run went ahead, which restarts the wallet's cadence. Bridges +/// `platform_wallet_manager_txo_reconcile_finish`. Resolves the manager +/// handle, so Kotlin frees the last page's guard first +/// (`txoReconcileHeldVerdictsFree`). +#[no_mangle] +pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_walletManagerTxoReconcileFinish( + mut env: JNIEnv, + _class: JClass, + manager_handle: jlong, + wallet_id: JByteArray, + ran: jboolean, +) { + guard(&mut env, (), |env| { + let Some(wid) = read_id32(env, &wallet_id) else { + return; + }; + let result = unsafe { + platform_wallet_ffi::platform_wallet_manager_txo_reconcile_finish( + manager_handle as Handle, + wid.as_ptr(), + ran != JNI_FALSE, + ) + }; + let _ = take_pwffi_error(env, result); + }) +} + +/// Release the engine guard `walletManagerTxoHealPageJson` or +/// `walletManagerTxoReconcileClassifyOutpoints` handed out as `outHeld[0]`, +/// once its page is applied. Takes no manager handle and touches no handle +/// registry, so it completes whatever is queued there. 0 is a no-op; each +/// value is released once. Bridges +/// `platform_wallet_txo_reconcile_held_verdicts_free`. +#[no_mangle] +pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_txoReconcileHeldVerdictsFree( + mut env: JNIEnv, + _class: JClass, + held: jlong, +) { + guard(&mut env, (), |_env| unsafe { + platform_wallet_ffi::platform_wallet_txo_reconcile_held_verdicts_free( + held as *mut platform_wallet_ffi::TxoReconcileHeldVerdictsFFI, + ) + }) +} + +/// Whether a TXO-store reconcile run classifies a page again after it came +/// back stale for the `staleInARow`-th time in a row (counting this one). +/// Bridges `platform_wallet_txo_reconcile_retry_stale_page`, which holds the +/// limit. A negative count is no count at all and reads as 0. +#[no_mangle] +pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_txoReconcileRetryStalePage( + mut env: JNIEnv, + _class: JClass, + stale_in_a_row: jint, +) -> jboolean { + guard(&mut env, JNI_FALSE, |_env| { + let count = u32::try_from(stale_in_a_row).unwrap_or(0); + if platform_wallet_ffi::platform_wallet_txo_reconcile_retry_stale_page(count) { + JNI_TRUE + } else { + JNI_FALSE + } + }) +} + /// One bounded page of the engine's UTXO inventory across every funds -/// account of one wallet — the source of truth -/// `PlatformWalletManager.reconcileTxoStore` diffs against the Room `txos` -/// mirror (dropped change outputs of CoinJoin-funded sends leave the -/// mirror short; the engine reloads from that mirror on restart, so an -/// un-reconciled hole becomes a fund loss). +/// account of one wallet, each row with the engine's heal verdict, for the +/// wallet's claimed reconcile run. Bridges +/// `platform_wallet_txo_reconcile_heal_page`. /// /// Paged rather than swept whole because inventory size is /// chain-controlled: anyone who knows a watched address can keep sending /// dust outputs to it, and a periodic full-inventory read would let them -/// decide how much a phone allocates at every SYNCED transition and every -/// 30-minute pass. Nothing bigger than one page is ever formatted, copied -/// across JNI, or parsed. +/// decide how much a phone allocates at every steady-state transition and +/// every cadence pass. Nothing bigger than one page is ever formatted, +/// copied across JNI, or parsed. +/// +/// Returns a JSON object `{"utxos":[...],"hasMore":,"budgetSpent":}`. +/// The run keeps its own place in the inventory: each page starts after +/// the last one reported with `walletManagerTxoReconcileHealPageApplied`, +/// or where the wallet's last run stopped, so the host passes no cursor; +/// a page read again before it is reported is the same page. Keep going +/// while `hasMore` is true. `budgetSpent` is true, with no rows and +/// nothing held, once the run has read its budget of pages: stop the run; +/// the wallet's next run picks up where it stopped. `limit` caps the rows +/// in one page; non-positive passes 0, which means the engine's own +/// default, and the engine clamps its own maximum — this layer holds no +/// copy of either bound. The verdicts are measured against the run's own +/// tip, which the engine keeps; the caller passes no height. /// -/// Returns a JSON object `{"utxos":[...],"cursor":,"hasMore":}`. -/// `cursor` is opaque: hand it back verbatim on the next call -/// (`null`/absent starts from the beginning) and keep going while -/// `hasMore` is true. A cursor this export did not produce is rejected -/// with an SDK exception rather than silently restarting the sweep. -/// `limit` caps the rows in one page; non-positive passes 0, which means -/// the engine's own default, and the engine clamps its own maximum — this -/// layer holds no copy of either bound. +/// The page comes back with the engine's wallet-manager read lock still +/// held, so no fork can drop a coin it offers before the page is written. +/// The guard is `outHeld[0]` (0 when nothing is held), written last, only +/// once the page is ready to return, and released with +/// `txoReconcileHeldVerdictsFree` under the rules +/// `walletManagerTxoReconcileClassifyOutpoints` gives. /// -/// Rows carry the engine's own address and flags. Nothing is derived -/// here: an address computed in this shim could disagree with the one the -/// engine credited the coin under, and the store keys its address rows by -/// that string. +/// Rows carry the engine's own address, flags and verdict. Nothing is +/// derived here: an address computed in this shim could disagree with the +/// one the engine credited the coin under, and the store keys its address +/// rows by that string. #[no_mangle] -pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_walletManagerUtxosPageJson( +pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_walletManagerTxoHealPageJson( mut env: JNIEnv, _class: JClass, manager_handle: jlong, wallet_id: JByteArray, - cursor: JString, - limit: jni::sys::jint, -) -> jni::sys::jstring { + limit: jint, + out_held: jlongArray, +) -> jstring { guard(&mut env, ptr::null_mut(), |env| { + let Some(out) = clear_out_held(env, out_held) else { + return ptr::null_mut(); + }; let Some(wid) = read_id32(env, &wallet_id) else { return ptr::null_mut(); }; @@ -3959,88 +4150,75 @@ pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_w // own maximum, so a host asking for more simply gets less. let page_limit = if limit <= 0 { 0usize } else { limit as usize }; - // Resume point, decoded from the opaque cursor. Kept alive across - // the FFI call — it is passed by pointer. - let resume: Option = if cursor.is_null() { - None - } else { - let raw = match env.get_string(&cursor) { - Ok(s) => String::from(s), - Err(_) => { - throw_sdk_exception(env, 1, "inventory cursor must be a String"); - return ptr::null_mut(); - } - }; - match UtxoPageCursor::decode(&raw) { - Some(r) => Some(r), - None => { - throw_sdk_exception( - env, - 1, - "malformed inventory cursor: only a cursor returned by a previous page may be handed back", - ); - return ptr::null_mut(); - } - } - }; - - let mut entries: *const platform_wallet_ffi::WalletUtxoEntryFFI = ptr::null(); + let mut entries: *const platform_wallet_ffi::TxoHealRowFFI = ptr::null(); let mut count: usize = 0; let mut has_more = false; + let mut budget_spent = false; + let mut held: *mut platform_wallet_ffi::TxoReconcileHeldVerdictsFFI = ptr::null_mut(); let result = unsafe { - platform_wallet_ffi::platform_wallet_wallet_utxos_page( + platform_wallet_ffi::platform_wallet_txo_reconcile_heal_page( manager_handle as Handle, wid.as_ptr(), - resume.as_ref().map_or(ptr::null(), |c| c as *const _), page_limit, &mut entries, &mut count, &mut has_more, + &mut budget_spent, + &mut held, ) }; + // Every exit before the hand-over frees the guard — see + // `walletManagerTxoReconcileClassifyOutpoints`. + let held = UnsentHeldVerdicts::new( + held, + platform_wallet_ffi::platform_wallet_txo_reconcile_held_verdicts_free, + ); if take_pwffi_error(env, result) { return ptr::null_mut(); } let mut rows: Vec = Vec::with_capacity(count); if !entries.is_null() && count > 0 { let items = unsafe { std::slice::from_raw_parts(entries, count) }; - for e in items { - // SAFETY: every entry the accessor returns satisfies the - // pointer contract `from_entry` documents. - rows.push(unsafe { UtxoPageRow::from_entry(e) }); + for r in items { + // SAFETY: every row the heal page returns satisfies the + // pointer contract `from_heal_row` documents. + rows.push(unsafe { UtxoPageRow::from_heal_row(r) }); } unsafe { - platform_wallet_ffi::platform_wallet_wallet_utxos_page_free( - entries as *mut platform_wallet_ffi::WalletUtxoEntryFFI, + platform_wallet_ffi::platform_wallet_txo_reconcile_heal_page_free( + entries as *mut platform_wallet_ffi::TxoHealRowFFI, count, ) }; } - let page = match UtxoPage::new(rows, has_more) { - Ok(p) => p, - Err(e) => { - throw_sdk_exception(env, 1, &format!("inventory cursor encode failed: {e}")); - return ptr::null_mut(); - } + let page = if budget_spent { + UtxoPage::budget_spent() + } else { + UtxoPage::new(rows, has_more) }; - match serde_json::to_string(&page) { - Ok(json) => env - .new_string(json) - .map(|s| s.into_raw()) - .unwrap_or(ptr::null_mut()), + let json = match serde_json::to_string(&page) { + Ok(json) => json, Err(e) => { throw_sdk_exception(env, 1, &format!("inventory page encode failed: {e}")); - ptr::null_mut() + return ptr::null_mut(); } + }; + let Ok(page) = env.new_string(json).map(|s| s.into_raw()) else { + return ptr::null_mut(); + }; + if !held.send(env, &out) { + return ptr::null_mut(); } + page }) } -/// Classify a batch of store rows against the engine's live state — the -/// reverse half of the reconcile transport, and the reason the paged -/// inventory carries no spent-outpoint list. The host pages its OWN mirror -/// rows and asks about them a batch at a time, so neither side builds a -/// set over the whole engine inventory. +/// Classify a batch of store rows against the engine's live state for the +/// wallet's claimed reconcile run — the reverse half of the reconcile +/// transport, and the reason the heal page carries no spent-outpoint list. +/// The host pages its OWN mirror rows and asks about them a batch at a +/// time, so neither side builds a set over the whole engine inventory. +/// Bridges `platform_wallet_txo_reconcile_classify_outpoints`. /// /// `queriesJson` is a JSON array of store rows: the account tuple the /// store files each coin under, its `txid` (lower hex, wire order) and @@ -4053,15 +4231,51 @@ pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_w /// `OUTPOINT_CLASS_*` constants (0 unknown, 1 unspent, 2 known-uncredited, /// 3 not-owned). A query whose account tag this build cannot map keeps /// `unknown` and the rest are still answered. +/// +/// The answers come back with the engine's wallet-manager read lock still +/// held, so no fork can drop a spender they name before the page is +/// applied. The guard is `outHeld[0]` (0 when nothing is held), written +/// last, only once the answers are ready to return; release it with +/// `txoReconcileHeldVerdictsFree` once the page is applied and before the +/// next call that takes this manager handle — those resolve the handle in +/// the native registry, which must never be waited for while the guard is +/// held. Call outside the persistence callback exclusion and never from a +/// persistence callback: the engine calls those under its write lock. +/// +/// Classify every store page the run reads, an empty one (`[]`) too: each +/// call is one page of the run's budget, and names the page +/// `walletManagerTxoReconcileClassifyPageApplied` reports. Once the run +/// has used its budget, `outBudgetSpent[0]` is true, every answer is +/// unknown and nothing is held. #[no_mangle] -pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_walletManagerClassifyOutpoints( +pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_walletManagerTxoReconcileClassifyOutpoints( mut env: JNIEnv, _class: JClass, manager_handle: jlong, wallet_id: JByteArray, queries_json: JString, + out_budget_spent: jni::sys::jbooleanArray, + out_held: jlongArray, ) -> jbyteArray { guard(&mut env, ptr::null_mut(), |env| { + let Some(out) = clear_out_held(env, out_held) else { + return ptr::null_mut(); + }; + const BUDGET_OUT: &str = "outBudgetSpent must be a BooleanArray(1)"; + if out_budget_spent.is_null() { + throw_sdk_exception(env, 1, BUDGET_OUT); + return ptr::null_mut(); + } + // SAFETY: a non-null `jbooleanArray` from the Kotlin caller. + let out_budget_spent = unsafe { jni::objects::JBooleanArray::from_raw(out_budget_spent) }; + if !has_out_slot(env.get_array_length(&out_budget_spent)) + || env + .set_boolean_array_region(&out_budget_spent, 0, &[JNI_FALSE]) + .is_err() + { + throw_sdk_exception(env, 1, BUDGET_OUT); + return ptr::null_mut(); + } let Some(wid) = read_id32(env, &wallet_id) else { return ptr::null_mut(); }; @@ -4079,12 +4293,6 @@ pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_w return ptr::null_mut(); } }; - if parsed.is_empty() { - return env - .byte_array_from_slice(&[]) - .map(|a| a.into_raw()) - .unwrap_or(ptr::null_mut()); - } // Scripts are owned here for the duration of the call — the query // struct borrows them by pointer. let mut scripts: Vec> = Vec::with_capacity(parsed.len()); @@ -4115,26 +4323,203 @@ pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_w // Caller-allocated answers, one byte per query. let mut classes = vec![platform_wallet_ffi::OUTPOINT_CLASS_UNKNOWN; queries.len()]; + let mut budget_spent = false; + let mut held: *mut platform_wallet_ffi::TxoReconcileHeldVerdictsFFI = ptr::null_mut(); let result = unsafe { - platform_wallet_ffi::platform_wallet_classify_outpoints( + platform_wallet_ffi::platform_wallet_txo_reconcile_classify_outpoints( manager_handle as Handle, wid.as_ptr(), queries.as_ptr(), queries.len(), classes.as_mut_ptr(), + &mut budget_spent, + &mut held, ) }; + // From here every exit — an error, a failed write, a panic that + // `guard` catches — frees the guard, until Kotlin owns it. + let held = UnsentHeldVerdicts::new( + held, + platform_wallet_ffi::platform_wallet_txo_reconcile_held_verdicts_free, + ); // `scripts` must outlive the call above; this keeps that explicit // against a future reorder. drop(scripts); if take_pwffi_error(env, result) { return ptr::null_mut(); } - env.byte_array_from_slice(&classes) - .map(|a| a.into_raw()) - .unwrap_or(ptr::null_mut()) + if budget_spent + && env + .set_boolean_array_region(&out_budget_spent, 0, &[JNI_TRUE]) + .is_err() + { + return ptr::null_mut(); + } + let Ok(answers) = env.byte_array_from_slice(&classes).map(|a| a.into_raw()) else { + return ptr::null_mut(); + }; + if !held.send(env, &out) { + return ptr::null_mut(); + } + answers + }) +} + +/// The host has applied (and released the guard of) the heal page it last +/// read: the run's next heal page starts after it, and so does the +/// wallet's next run if this one stops here. Returns whether the run has +/// budget left for another page. Bridges +/// `platform_wallet_txo_reconcile_heal_page_applied`. +#[no_mangle] +pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_walletManagerTxoReconcileHealPageApplied( + mut env: JNIEnv, + _class: JClass, + manager_handle: jlong, + wallet_id: JByteArray, +) -> jboolean { + guard(&mut env, JNI_FALSE, |env| { + let Some(wid) = read_id32(env, &wallet_id) else { + return JNI_FALSE; + }; + let mut may_continue = false; + let result = unsafe { + platform_wallet_ffi::platform_wallet_txo_reconcile_heal_page_applied( + manager_handle as Handle, + wid.as_ptr(), + &mut may_continue, + ) + }; + if take_pwffi_error(env, result) || !may_continue { + return JNI_FALSE; + } + JNI_TRUE }) } + +/// The host has applied (and released the guard of) the store page it last +/// classified. `nextAfter` is the host's own store cursor past that page, +/// at most `TXO_RECONCILE_STORE_CURSOR_MAX` bytes, which +/// `walletManagerTxoReconcileBegin` hands back to the wallet's next run; +/// null when that was the store's last page, which ends the cycle. +/// Returns whether the run has budget left for another page. Bridges +/// `platform_wallet_txo_reconcile_classify_page_applied`. +#[no_mangle] +pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_walletManagerTxoReconcileClassifyPageApplied( + mut env: JNIEnv, + _class: JClass, + manager_handle: jlong, + wallet_id: JByteArray, + next_after: JByteArray, +) -> jboolean { + guard(&mut env, JNI_FALSE, |env| { + let Some(wid) = read_id32(env, &wallet_id) else { + return JNI_FALSE; + }; + let after = if next_after.is_null() { + None + } else { + match env.convert_byte_array(&next_after) { + Ok(bytes) => Some(bytes), + Err(_) => return JNI_FALSE, + } + }; + let mut may_continue = false; + let result = unsafe { + platform_wallet_ffi::platform_wallet_txo_reconcile_classify_page_applied( + manager_handle as Handle, + wid.as_ptr(), + after.as_ref().map_or(ptr::null(), |bytes| bytes.as_ptr()), + after.as_ref().map_or(0, Vec::len), + after.is_none(), + &mut may_continue, + ) + }; + if take_pwffi_error(env, result) || !may_continue { + return JNI_FALSE; + } + JNI_TRUE + }) +} + +/// Validate `outHeld` as a `LongArray(1)` and write "nothing held" to it, +/// before any other check of a held read can return, so a reused array +/// never carries a token from an earlier call into this one, whichever way +/// this call ends. Throws and returns `None` on a null or short array. +fn clear_out_held<'local>( + env: &mut JNIEnv<'local>, + out_held: jlongArray, +) -> Option> { + const HELD_OUT: &str = "outHeld must be a LongArray(1)"; + if out_held.is_null() { + throw_sdk_exception(env, 1, HELD_OUT); + return None; + } + // SAFETY: a non-null `jlongArray` from the Kotlin caller. + let out = unsafe { jni::objects::JLongArray::from_raw(out_held) }; + if !has_out_slot(env.get_array_length(&out)) + || env.set_long_array_region(&out, 0, &[0]).is_err() + { + throw_sdk_exception(env, 1, HELD_OUT); + return None; + } + Some(out) +} + +/// A held-verdicts guard `walletManagerTxoHealPageJson` or +/// `walletManagerTxoReconcileClassifyOutpoints` has not handed to Kotlin +/// yet. Dropped before [`Self::hand_over`] — an early return, or a panic +/// unwinding into `guard` — it frees the guard at once. Left to +/// `walletManagerTxoReconcileFinish` instead, it would be released only by +/// a call that resolves the manager handle in the native registry while +/// the guard is still held, the registry/engine lock cycle the token exists +/// to avoid. +struct UnsentHeldVerdicts { + held: *mut T, + free: unsafe extern "C" fn(*mut T), +} + +impl UnsentHeldVerdicts { + fn new(held: *mut T, free: unsafe extern "C" fn(*mut T)) -> Self { + Self { held, free } + } + + fn as_ptr(&self) -> *mut T { + self.held + } + + /// Kotlin owns the guard now and frees it with + /// `txoReconcileHeldVerdictsFree`. + fn hand_over(mut self) { + self.held = ptr::null_mut(); + } +} + +impl UnsentHeldVerdicts { + /// Write the guard to `outHeld[0]`, last, once the call's answer is + /// ready to return, and hand it to Kotlin. `false` — the guard freed + /// here — when the write failed. + fn send(self, env: &mut JNIEnv, out: &jni::objects::JLongArray) -> bool { + if env + .set_long_array_region(out, 0, &[self.as_ptr() as jlong]) + .is_err() + { + return false; + } + self.hand_over(); + true + } +} + +impl Drop for UnsentHeldVerdicts { + fn drop(&mut self) { + if !self.held.is_null() { + // SAFETY: `held` came from the native held read and was never + // handed over, so nothing else frees it. + unsafe { (self.free)(self.held) }; + } + } +} + /// Read a 32-byte id from a Java `byte[]`; throws + returns None on the /// wrong length or a JNI error. fn read_id32(env: &mut JNIEnv, arr: &JByteArray) -> Option<[u8; 32]> { @@ -4926,7 +5311,10 @@ mod tests { //! silently. use super::*; - use platform_wallet_ffi::{AccountTypeTagFFI, StandardAccountTypeTagFFI, WalletUtxoEntryFFI}; + use platform_wallet_ffi::{ + AccountTypeTagFFI, StandardAccountTypeTagFFI, TxoHealRowFFI, WalletUtxoEntryFFI, + TXO_HEAL_VERDICT_ELIGIBLE, TXO_HEAL_VERDICT_IMMATURE, TXO_HEAL_VERDICT_INVALID, + }; use std::ffi::CString; /// The checked-in transport payloads, shared byte-for-byte with the @@ -4954,13 +5342,19 @@ mod tests { script } - /// One accessor entry plus the buffers it points into, kept alive with it. + /// One heal-page row plus the buffers it points into, kept alive with it. struct Coin { - entry: WalletUtxoEntryFFI, + row: TxoHealRowFFI, _address: CString, script: Vec, } + impl Coin { + fn entry(&self) -> &WalletUtxoEntryFFI { + &self.row.utxo + } + } + struct CoinSpec { type_tag: AccountTypeTagFFI, standard_tag: StandardAccountTypeTagFFI, @@ -4976,6 +5370,7 @@ mod tests { height: u32, /// confirmed, instantlocked, coinbase, locked. flags: [bool; 4], + heal_verdict: u8, } impl CoinSpec { @@ -5010,7 +5405,10 @@ mod tests { is_locked: self.flags[3], }; Coin { - entry, + row: TxoHealRowFFI { + utxo: entry, + verdict: self.heal_verdict, + }, _address: address, script, } @@ -5039,6 +5437,7 @@ mod tests { script: p2pkh(0x5b), height: 1_534_921, flags: [true, false, false, false], + heal_verdict: TXO_HEAL_VERDICT_ELIGIBLE, } .build(), CoinSpec { @@ -5055,6 +5454,7 @@ mod tests { script: p2pkh(0xc3), height: 1_400_000, flags: [true, true, true, true], + heal_verdict: TXO_HEAL_VERDICT_ELIGIBLE, } .build(), CoinSpec { @@ -5071,6 +5471,7 @@ mod tests { script: Vec::new(), height: 0, flags: [false, false, false, false], + heal_verdict: TXO_HEAL_VERDICT_INVALID, } .build(), CoinSpec { @@ -5087,6 +5488,7 @@ mod tests { script: p2pkh(0xd4), height: 1_534_000, flags: [true, true, false, false], + heal_verdict: TXO_HEAL_VERDICT_IMMATURE, } .build(), ] @@ -5097,7 +5499,7 @@ mod tests { .iter() // SAFETY: every pointer in a `Coin` entry points into that coin's // own live buffers. - .map(|c| unsafe { UtxoPageRow::from_entry(&c.entry) }) + .map(|c| unsafe { UtxoPageRow::from_heal_row(&c.row) }) .collect() } @@ -5105,7 +5507,7 @@ mod tests { #[test] fn inventory_page_serializes_to_the_golden_page() { - let page = UtxoPage::new(rows(&golden_coins()), true).expect("page encodes"); + let page = UtxoPage::new(rows(&golden_coins()), true); assert_eq!( serde_json::to_value(&page).expect("page serializes"), golden()["inventoryPage"], @@ -5114,120 +5516,52 @@ mod tests { } #[test] - fn a_last_or_empty_page_carries_no_cursor() { - let last = UtxoPage::new(rows(&golden_coins()), false).expect("page encodes"); - assert!(!last.has_more); - assert!(last.cursor.is_none()); + fn each_row_carries_the_engines_heal_verdict_unchanged() { + let value = serde_json::to_value(UtxoPage::new(rows(&golden_coins()), true)) + .expect("page serializes"); + let verdicts: Vec = value["utxos"] + .as_array() + .expect("utxos is an array") + .iter() + .map(|row| { + row["healVerdict"] + .as_u64() + .expect("healVerdict is a number") + }) + .collect(); + assert_eq!( + verdicts, + [ + TXO_HEAL_VERDICT_ELIGIBLE, + TXO_HEAL_VERDICT_ELIGIBLE, + TXO_HEAL_VERDICT_INVALID, + TXO_HEAL_VERDICT_IMMATURE + ] + .map(u64::from), + "the shim passes the verdict through; it never decides one" + ); + } + + #[test] + fn a_last_or_empty_page_reports_no_more_and_carries_no_cursor() { + let last = UtxoPage::new(rows(&golden_coins()), false); let value = serde_json::to_value(&last).expect("page serializes"); - assert_eq!(value["cursor"], serde_json::Value::Null); assert_eq!(value["hasMore"], serde_json::Value::Bool(false)); + assert_eq!(value["budgetSpent"], serde_json::Value::Bool(false)); + assert!(value.get("cursor").is_none(), "the run keeps its own place"); - // An empty page has nowhere to resume from, whatever the accessor said. - let empty = UtxoPage::new(Vec::new(), true).expect("page encodes"); + // An empty page has nothing after it, whatever the accessor said. + let empty = UtxoPage::new(Vec::new(), true); assert!(!empty.has_more); - assert!(empty.cursor.is_none()); - } - - #[test] - fn the_golden_cursor_resumes_after_the_last_row() { - let coins = golden_coins(); - let last = &coins.last().expect("golden coins").entry; - let raw = golden()["inventoryPage"]["cursor"] - .as_str() - .expect("golden cursor is a string") - .to_owned(); - - let resume = UtxoPageCursor::decode(&raw).expect("golden cursor decodes"); - - assert_eq!(resume.type_tag, last.type_tag as u8); - assert_eq!(resume.standard_tag, last.standard_tag as u8); - assert_eq!(resume.index, last.index); - assert_eq!(resume.registration_index, last.registration_index); - assert_eq!(resume.key_class, last.key_class); - assert_eq!(resume.user_identity_id, last.user_identity_id); - assert_eq!(resume.friend_identity_id, last.friend_identity_id); - assert_eq!(resume.outpoint.txid, last.outpoint.txid); - assert_eq!(resume.outpoint.vout, last.outpoint.vout); } #[test] - fn a_cursor_round_trips_every_account_field() { - let coin = CoinSpec { - type_tag: AccountTypeTagFFI::PlatformPayment, - standard_tag: StandardAccountTypeTagFFI::Bip32, - index: 7, - registration_index: 9, - key_class: 2, - identities: [[0x0c; 32], [0x0d; 32]], - txid: txid(0x80), - vout: 65_535, - value: 1, - address: "yGoldenFixtureCursorAddr", - script: p2pkh(0x01), - height: 2, - flags: [false, true, false, true], - } - .build(); - let row = rows(std::slice::from_ref(&coin)).remove(0); - let raw = serde_json::to_string(&UtxoPageCursor::after(&row)).expect("cursor encodes"); - - let resume = UtxoPageCursor::decode(&raw).expect("own cursor decodes"); - - let e = &coin.entry; + fn a_budget_spent_page_carries_no_rows() { + let value = serde_json::to_value(UtxoPage::budget_spent()).expect("page serializes"); assert_eq!( - ( - resume.type_tag, - resume.standard_tag, - resume.index, - resume.registration_index, - resume.key_class - ), - (14, 1, 7, 9, 2) + value, + serde_json::json!({"utxos": [], "hasMore": false, "budgetSpent": true}) ); - assert_eq!(resume.user_identity_id, e.user_identity_id); - assert_eq!(resume.friend_identity_id, e.friend_identity_id); - assert_eq!(resume.outpoint.txid, e.outpoint.txid); - assert_eq!(resume.outpoint.vout, e.outpoint.vout); - } - - #[test] - fn a_cursor_this_layer_did_not_emit_is_rejected() { - let good: serde_json::Value = serde_json::from_str( - golden()["inventoryPage"]["cursor"] - .as_str() - .expect("golden cursor is a string"), - ) - .expect("golden cursor is JSON"); - let tampered = |key: &str, value: serde_json::Value| { - let mut c = good.clone(); - c[key] = value; - c.to_string() - }; - let mut missing_index = good.clone(); - missing_index - .as_object_mut() - .expect("cursor is an object") - .remove("index"); - - for (label, raw) in [ - ("not a cursor object", "7".to_owned()), - ("missing index", missing_index.to_string()), - ("short txid", tampered("txid", "00".repeat(31).into())), - ("non-hex txid", tampered("txid", "zz".repeat(32).into())), - ( - "short identity", - tampered("userIdentityId", "a1".repeat(31).into()), - ), - ( - "odd-length identity", - tampered("friendIdentityId", "b".repeat(63).into()), - ), - ] { - assert!( - UtxoPageCursor::decode(&raw).is_none(), - "{label} must be rejected" - ); - } } // ── Classification queries (Kotlin → Rust) ──────────────────────── @@ -5244,7 +5578,7 @@ mod tests { assert_eq!(parsed.len(), coins.len()); for (q, coin) in parsed.iter().zip(&coins) { - let e = &coin.entry; + let e = coin.entry(); // The tuple the store sends is the tuple the inventory emitted. assert_eq!(q.account, UtxoAccountTuple::from_entry(e)); let (ffi, script) = q.to_ffi().expect("golden query converts"); @@ -5374,4 +5708,111 @@ mod tests { external as usize ); } + + mod until_handed_over { + use std::cell::Cell; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::super::{has_out_slot, UntilHandedOver}; + + #[test] + fn an_unreturned_claim_is_released_once_on_drop() { + let released = Cell::new(0); + drop(UntilHandedOver::new(|| released.set(released.get() + 1))); + assert_eq!(released.get(), 1); + } + + #[test] + fn a_claim_handed_to_kotlin_is_not_released() { + let released = Cell::new(0); + UntilHandedOver::new(|| released.set(released.get() + 1)).hand_over(); + assert_eq!(released.get(), 0); + } + + #[test] + fn a_panic_before_the_hand_over_releases_the_claim() { + let released = AtomicUsize::new(0); + let unwound = std::panic::catch_unwind(|| { + let _claim = UntilHandedOver::new(|| { + released.fetch_add(1, Ordering::SeqCst); + }); + panic!("writing outTipHeight failed"); + }); + assert!(unwound.is_err()); + assert_eq!( + released.load(Ordering::SeqCst), + 1, + "released while unwinding" + ); + } + + #[test] + fn an_out_array_needs_one_slot() { + assert!(has_out_slot(Ok(1))); + assert!(has_out_slot(Ok(2))); + assert!( + !has_out_slot(Ok(0)), + "an empty array is refused before the claim" + ); + assert!(!has_out_slot(Err(jni::errors::Error::NullPtr("array")))); + } + } + + mod unsent_held_verdicts { + use std::ptr::NonNull; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::super::UnsentHeldVerdicts; + + /// A stand-in guard: the frees below only count, never dereference. + fn token() -> *mut u8 { + NonNull::::dangling().as_ptr() + } + + #[test] + fn an_unsent_guard_is_freed_once_on_drop() { + static FREED: AtomicUsize = AtomicUsize::new(0); + unsafe extern "C" fn free(_: *mut u8) { + FREED.fetch_add(1, Ordering::SeqCst); + } + drop(UnsentHeldVerdicts::new(token(), free)); + assert_eq!(FREED.load(Ordering::SeqCst), 1); + } + + #[test] + fn a_guard_handed_to_kotlin_is_not_freed() { + static FREED: AtomicUsize = AtomicUsize::new(0); + unsafe extern "C" fn free(_: *mut u8) { + FREED.fetch_add(1, Ordering::SeqCst); + } + let held = UnsentHeldVerdicts::new(token(), free); + assert_eq!(held.as_ptr(), token()); + held.hand_over(); + assert_eq!(FREED.load(Ordering::SeqCst), 0); + } + + #[test] + fn a_panic_before_the_hand_over_frees_the_guard() { + static FREED: AtomicUsize = AtomicUsize::new(0); + unsafe extern "C" fn free(_: *mut u8) { + FREED.fetch_add(1, Ordering::SeqCst); + } + let unwound = std::panic::catch_unwind(|| { + let _held = UnsentHeldVerdicts::new(token(), free); + panic!("writing outHeld failed"); + }); + assert!(unwound.is_err()); + assert_eq!(FREED.load(Ordering::SeqCst), 1, "freed while unwinding"); + } + + #[test] + fn nothing_held_is_nothing_freed() { + static FREED: AtomicUsize = AtomicUsize::new(0); + unsafe extern "C" fn free(_: *mut u8) { + FREED.fetch_add(1, Ordering::SeqCst); + } + drop(UnsentHeldVerdicts::new(std::ptr::null_mut(), free)); + assert_eq!(FREED.load(Ordering::SeqCst), 0); + } + } } From d8fea2128735f6e2be445d9f6745ba01e8d5acd4 Mon Sep 17 00:00:00 2001 From: HashEngineering Date: Thu, 5 Nov 2026 14:44:12 -0800 Subject: [PATCH 3/6] refactor(kotlin-sdk): simplify the TXO reconcile host - Follow PR 1's C ABI: `begin` hands over no tip, and an empty heal page always ends the pass, so JNI and Kotlin no longer normalize it. - One `TxoReconcileReport` with a `plus` operator replaces the tally, heal and flip count classes; a classify page is counted once. - Drop Kotlin's copy of the contact (DIP-15) exclusion: Rust already keeps contact coins out of heal pages and gives contact queries no verdict, as Swift relies on. `skippedForeign` goes with it; a contact row in the store is now asked about and counted unknown, and is never flipped. - Heal: one batched presence query and one batched address lookup per page, account lookups memoized by tuple, same skip order. - Flip: the spent-mark's row count decides the flip (no read first); pending inputs are deleted in one batched statement per page. - JNI: one drop guard for held tokens, a shared applied-answer helper, `slice_or_empty` reused, single-use helpers inlined. - One cancellation-safe logging helper; `splitIds` shared; dead constants removed; test fixtures share `saveCoreAddress`. Co-Authored-By: Claude Opus 5.5 --- .../dashsdk/ffi/WalletManagerNative.kt | 26 +- .../PlatformWalletPersistenceHandler.kt | 651 ++++++++---------- .../dashsdk/persistence/dao/CoreAddressDao.kt | 5 + .../dashsdk/persistence/dao/DocumentDao.kt | 5 + .../dashsdk/persistence/dao/TxoDao.kt | 8 + .../dashsdk/wallet/ManagedPlatformWallet.kt | 25 +- .../dashsdk/wallet/PlatformWalletManager.kt | 46 +- .../dashsdk/wallet/TxoReconcile.kt | 2 + .../PlatformWalletPersistenceHandlerTest.kt | 158 +++-- .../rs-unified-sdk-jni/src/persistence.rs | 2 +- .../rs-unified-sdk-jni/src/wallet_manager.rs | 315 +++------ 11 files changed, 557 insertions(+), 686 deletions(-) diff --git a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/ffi/WalletManagerNative.kt b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/ffi/WalletManagerNative.kt index 1becaa273fd..bde6af67013 100644 --- a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/ffi/WalletManagerNative.kt +++ b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/ffi/WalletManagerNative.kt @@ -180,12 +180,6 @@ internal object WalletManagerNative { * [walletManagerTxoReconcileBegin] hands back in `outStoreAfter[0]`. */ const val TXO_RECONCILE_START_CLASSIFY = 2 - /** The longest store cursor - * [walletManagerTxoReconcileClassifyPageApplied] keeps for the - * wallet's next run (rs-platform-wallet-ffi - * `TXO_RECONCILE_STORE_CURSOR_MAX`). */ - const val TXO_RECONCILE_STORE_CURSOR_MAX = 64 - /** * The wallets whose reconcile is due now, as concatenated 32-byte ids * (`platform_wallet_manager_txo_reconcile_due_wallets`). Called on every @@ -202,9 +196,9 @@ internal object WalletManagerNative { * (`platform_wallet_manager_txo_reconcile_begin`). Returns * [TXO_RECONCILE_GATE_RUN] or a `TXO_RECONCILE_SKIP_*` reason (see * [org.dashfoundation.dashsdk.wallet.TxoReconcileSkipReason.fromGateCode]); - * on [TXO_RECONCILE_GATE_RUN] `outTipHeight[0]` is the scan tip the run - * is measured against (the native claim keeps it), and the claim holds - * until [walletManagerTxoReconcileFinish]. + * on [TXO_RECONCILE_GATE_RUN] the claim holds until + * [walletManagerTxoReconcileFinish]. The native claim keeps the scan tip + * the run is measured against; the host never sees it. * * On [TXO_RECONCILE_GATE_RUN] `outStart[0]` is where the run starts, as * the native reconciler decides it from the wallet's checkpoint: @@ -212,14 +206,13 @@ internal object WalletManagerNative { * [TXO_RECONCILE_START_CLASSIFY]. For the last, `outStoreAfter[0]` is * the store cursor to start after, the bytes the host last passed to * [walletManagerTxoReconcileClassifyPageApplied] (empty: the first - * row); otherwise it is null. [outTipHeight] and [outStart] are - * `IntArray(1)`, [outStoreAfter] an `arrayOfNulls(1)`, - * checked before the claim is taken. + * row); otherwise it is null. [outStart] is an `IntArray(1)`, + * [outStoreAfter] an `arrayOfNulls(1)`, both checked before + * the claim is taken. */ external fun walletManagerTxoReconcileBegin( managerHandle: Long, walletId: ByteArray, - outTipHeight: IntArray, outStart: IntArray, outStoreAfter: Array, ): Int @@ -258,7 +251,8 @@ internal object WalletManagerNative { * cursor: each page starts after the last one reported with * [walletManagerTxoReconcileHealPageApplied] (or where the wallet's last * run stopped), and a page read again before it is reported is the same - * page. Keep going while `hasMore` is true. Each call is one page of the + * page. Keep going while `hasMore` is true; it is false on an empty + * page. Each call is one page of the * run's budget; `budgetSpent` true means nothing was read and nothing * is held: the run has used its budget and stops. [limit] caps the rows * in one page; non-positive asks for the engine's own default, and the @@ -361,8 +355,8 @@ internal object WalletManagerNative { * Report that the store page [walletManagerTxoReconcileClassifyOutpoints] * last classified was applied and its guard released * (`platform_wallet_txo_reconcile_classify_page_applied`). [nextAfter] is - * the host's store cursor past that page, at most - * [TXO_RECONCILE_STORE_CURSOR_MAX] bytes, which + * the host's store cursor past that page, at most rs-platform-wallet-ffi + * `TXO_RECONCILE_STORE_CURSOR_MAX` (64) bytes, which * [walletManagerTxoReconcileBegin] hands back to the wallet's next run; * null when that was the store's last page, which ends the cycle (the * next run starts at the heal pass again). Returns whether the run has diff --git a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandler.kt b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandler.kt index 2615dc7fb94..39d8d86bb26 100644 --- a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandler.kt +++ b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandler.kt @@ -1,9 +1,9 @@ package org.dashfoundation.dashsdk.persistence import android.util.Log +import androidx.annotation.VisibleForTesting import androidx.room.withTransaction import kotlinx.coroutines.CoroutineDispatcher -import kotlinx.coroutines.NonCancellable import kotlinx.coroutines.ExecutorCoroutineDispatcher import kotlinx.coroutines.asCoroutineDispatcher import kotlinx.coroutines.flow.MutableStateFlow @@ -14,7 +14,6 @@ import kotlinx.coroutines.flow.update import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -import kotlinx.coroutines.withContext import kotlinx.serialization.Serializable import kotlinx.serialization.json.Json import org.dashfoundation.dashsdk.errors.DashSdkError @@ -1247,8 +1246,7 @@ class PlatformWalletPersistenceHandler( // engine's verdict on this output (if it did NOT credit it) is // final here and travels with the staged write. val round = openRound(walletId) - val creditVerdict = round?.utxoCreditVerdicts?.takeIf { it.isNotEmpty() } - ?.get(makeOutpoint(txid, vout).toHex()) + val creditVerdict = round?.utxoCreditVerdicts?.get(makeOutpoint(txid, vout).toHex()) stage(walletId) { db -> upsertUtxoRow( db, walletId, txid, vout, amount, address, scriptPubKey, @@ -1594,8 +1592,9 @@ class PlatformWalletPersistenceHandler( suspend fun classifyPageApplied(nextAfter: ByteArray?): Boolean? /** Release the guard of the last page read, if any. Never resolves - * the native manager handle and never waits. */ - suspend fun releaseVerdicts() + * the native manager handle, never waits and never suspends, so it + * runs in a cancelled run's `finally` as it is. */ + fun releaseVerdicts() /** Whether a page that came back stale for the [staleInARow]-th time * in a row is classified again, or the run stops. */ @@ -1640,11 +1639,6 @@ class PlatformWalletPersistenceHandler( * fields this store could not decode (a txid that is not 32 bytes * of hex, a malformed script or identity half). */ val skippedInvalid: Int = 0, - /** Watch-only DIP-15 contact coins excluded on both sides — engine - * coins the heal pass refused to file as ours, and unspent store - * rows the classify pass did not ask about (the engine's own - * accounts never report them, so their absence is expected). */ - val skippedForeign: Int = 0, // Classify pass (store → engine, flip-only). /** Unspent store rows of this wallet handed to the engine. */ val storeRows: Int = 0, @@ -1654,7 +1648,9 @@ class PlatformWalletPersistenceHandler( * [OUTPOINT_CLASS_KNOWN_UNCREDITED] verdict. */ val flipped: Int = 0, val flippedDuffs: Long = 0, - /** Store rows the engine has no opinion about — left untouched. */ + /** Store rows the engine has no opinion about — left untouched. A + * watch-only DIP-15 contact row is one: the engine gives a + * contact's coin no verdict. */ val unknown: Int = 0, /** Store rows filed under an account that does not monitor their * script — left untouched, reported. */ @@ -1705,91 +1701,67 @@ class PlatformWalletPersistenceHandler( * wrote spent on insert — the last is a divergence recorded, not * repaired, but it is a row the store did not have before. */ val mutations: Int get() = inserted + healedSpent + flipped + + /** The counts of this report and [other] added together; the + * run-shape flags ([completed], [stoppedOnBudget], + * [resumedFromCheckpoint]) stay this report's. */ + internal operator fun plus(other: TxoReconcileReport) = copy( + engineRows = engineRows + other.engineRows, + alreadyPresent = alreadyPresent + other.alreadyPresent, + inserted = inserted + other.inserted, + insertedDuffs = insertedDuffs + other.insertedDuffs, + healedSpent = healedSpent + other.healedSpent, + skippedImmature = skippedImmature + other.skippedImmature, + skippedUnresolvedAccount = skippedUnresolvedAccount + other.skippedUnresolvedAccount, + skippedMissingAddress = skippedMissingAddress + other.skippedMissingAddress, + skippedInvalid = skippedInvalid + other.skippedInvalid, + storeRows = storeRows + other.storeRows, + unspent = unspent + other.unspent, + flipped = flipped + other.flipped, + flippedDuffs = flippedDuffs + other.flippedDuffs, + unknown = unknown + other.unknown, + notOwned = notOwned + other.notOwned, + unresolvedAccount = unresolvedAccount + other.unresolvedAccount, + staleRetries = staleRetries + other.staleRetries, + transportFailures = transportFailures + other.transportFailures, + storeFailures = storeFailures + other.storeFailures, + ) } - /** The mutable half of a [TxoReconcileReport] while the run is in flight. */ - private class ReconcileTally { - var engineRows = 0 - var alreadyPresent = 0 - var inserted = 0 - var insertedDuffs = 0L - var healedSpent = 0 - var skippedImmature = 0 - var skippedUnresolvedAccount = 0 - var skippedMissingAddress = 0 - var skippedInvalid = 0 - var skippedForeign = 0 - var storeRows = 0 - var unspent = 0 - var flipped = 0 - var flippedDuffs = 0L - var unknown = 0 - var notOwned = 0 - var unresolvedAccount = 0 - var staleRetries = 0 - var transportFailures = 0 - var storeFailures = 0 + /** The report of a [reconcileTxos] run while it is in flight. */ + private class ReconcileTally(resumedFromCheckpoint: Boolean) { + /** `completed` is set only where the classify pass reaches the + * store's last page; every other exit leaves the run incomplete. */ + var report = TxoReconcileReport(completed = false, resumedFromCheckpoint = resumedFromCheckpoint) - /** Set only where the classify pass reaches the store's last page; - * every other exit leaves the run incomplete. */ - var completed = false - var stoppedOnBudget = false - var resumedFromCheckpoint = false + operator fun plusAssign(counts: TxoReconcileReport) { + report += counts + } /** The engine said the run's budget is spent: it stops cleanly * between pages. */ fun stopOnBudget() { - stoppedOnBudget = true - completed = false + report = report.copy(stoppedOnBudget = true) } /** An engine call failed: the run stops and says so. */ fun stopOnTransport() { - transportFailures++ - completed = false + this += TxoReconcileReport(transportFailures = 1) } - - fun toReport() = TxoReconcileReport( - engineRows = engineRows, - alreadyPresent = alreadyPresent, - inserted = inserted, - insertedDuffs = insertedDuffs, - healedSpent = healedSpent, - skippedImmature = skippedImmature, - skippedUnresolvedAccount = skippedUnresolvedAccount, - skippedMissingAddress = skippedMissingAddress, - skippedInvalid = skippedInvalid, - skippedForeign = skippedForeign, - storeRows = storeRows, - unspent = unspent, - flipped = flipped, - flippedDuffs = flippedDuffs, - unknown = unknown, - notOwned = notOwned, - unresolvedAccount = unresolvedAccount, - staleRetries = staleRetries, - transportFailures = transportFailures, - storeFailures = storeFailures, - completed = completed, - stoppedOnBudget = stoppedOnBudget, - resumedFromCheckpoint = resumedFromCheckpoint, - ) } - /** Counts from one heal step; merged into the run only once it commits. */ - private class HealCounts { - var alreadyPresent = 0 - var inserted = 0 - var insertedDuffs = 0L - var healedSpent = 0 - var skippedImmature = 0 - var skippedUnresolvedAccount = 0 - var skippedMissingAddress = 0 - var skippedInvalid = 0 - var skippedForeign = 0 - val healedOutpoints = ArrayList() - val healedSpentOutpoints = ArrayList() - } + /** + * What one heal or classify store step wrote: its [counts], merged into + * the run only once the step commits, and the rows to log then, each + * with its action. [stale] is a classify step that wrote nothing because + * a round committed between the page's read and its apply, or one is + * open at the apply: the page is classified again. + */ + private class StoreStepResult( + val counts: TxoReconcileReport, + val logged: List> = emptyList(), + val stale: Boolean = false, + ) /** * One page of the wallet's unspent store rows for the classify pass: @@ -1804,25 +1776,9 @@ class PlatformWalletPersistenceHandler( val fetched: Int, val lastOutpoint: ByteArray?, val unresolvedAccount: Int, - val skippedForeign: Int, val generation: Long, ) - /** Counts from one classify-apply step. */ - private class FlipCounts { - var flipped = 0 - var flippedDuffs = 0L - var unspent = 0 - var unknown = 0 - var notOwned = 0 - - /** A round committed between the read and the apply, or one is - * open at the apply: nothing was written and the page is - * classified again. */ - var staleGeneration = false - val flippedOutpoints = ArrayList() - } - /** * Reconcile the Room `txos` mirror against the engine — the Kotlin port * of Swift's `runCoreTxoReconcile` (dashpay/platform#4638 Part 2), for @@ -1852,10 +1808,11 @@ class PlatformWalletPersistenceHandler( * * Every decision is the engine's: which coins may be healed, which rows * to flip, how many pages one run reads, where the run's next page - * starts and how often a stale page is classified again. This side - * pages its store, resolves owners, applies the host-side checks (the - * DIP-15 contact exclusion, the address row, the round fences) and - * writes. + * starts and how often a stale page is classified again — and the + * DIP-15 contact exclusion: the engine never offers a contact's coin to + * heal and gives a contact's row no verdict. This side pages its store, + * resolves owners, applies the host-side checks (the address row, the + * round fences) and writes. * * It never deletes a row, never un-marks a spent row, and never acts on * absence alone. It is idempotent (a consistent store yields a report @@ -1913,40 +1870,19 @@ class PlatformWalletPersistenceHandler( ): TxoReconcileReport { val limit = pageSize.coerceAtLeast(1) val walletRef = logReference(walletId) - val tally = ReconcileTally() - tally.resumedFromCheckpoint = start != TxoReconcileStart.NewCycle - - // Watch-only DIP-15 contact (external) accounts, resolved once up - // front because BOTH passes need the exclusion and the account set - // does not move under a run. The engine tracks these accounts' coins - // to show payments TO contacts, but they are the CONTACT's money and - // must never be healed into the store as ours. Before the check - // lived on the insert pass, a fresh restore's post-backfill reconcile - // healed every contact-payment coin into the store (12 rows / - // 0.05692493 tDASH on the large-wallet validation run of 2026-08-25). - val foreignAccountIds = reconcileStoreStep(tally, walletRef, "accounts", isCancelled) { - database.accountDao() - .observeByWalletAndType(walletId, ACCOUNT_TYPE_TAG_DASHPAY_EXTERNAL).first() - .map { it.id } - .toSet() + val tally = ReconcileTally(resumedFromCheckpoint = start != TxoReconcileStart.NewCycle) + val healDone = when (start) { + // The heal pass of this cycle already reached its end. + is TxoReconcileStart.Classify -> true + // A resumed heal pass needs nothing from here: the engine + // serves the page after the last one reported applied. + TxoReconcileStart.NewCycle, TxoReconcileStart.HealResumed -> + healMissingTxos(walletId, limit, isCancelled, engine, tally, walletRef) } - if (foreignAccountIds != null) { - val healDone = when (start) { - // The heal pass of this cycle already reached its end. - is TxoReconcileStart.Classify -> true - // A resumed heal pass needs nothing from here: the engine - // serves the page after the last one reported applied. - TxoReconcileStart.NewCycle, TxoReconcileStart.HealResumed -> - healMissingTxos(walletId, limit, foreignAccountIds, isCancelled, engine, tally, walletRef) - } - if (healDone) { - classifyStoreRows( - walletId, classifyStartCursor(start), limit, foreignAccountIds, isCancelled, engine, tally, - walletRef, - ) - } + if (healDone) { + classifyStoreRows(walletId, classifyStartCursor(start), limit, isCancelled, engine, tally, walletRef) } - val report = tally.toReport() + val report = tally.report logReconcileSummary(report, walletRef) return report } @@ -2002,7 +1938,6 @@ class PlatformWalletPersistenceHandler( private suspend fun healMissingTxos( walletId: ByteArray, limit: Int, - foreignAccountIds: Set, isCancelled: () -> Boolean, engine: TxoReconcileEngine, tally: ReconcileTally, @@ -2029,36 +1964,16 @@ class PlatformWalletPersistenceHandler( return false } if (page.utxos.isNotEmpty()) { - val counts = reconcileStoreStep(tally, walletRef, "heal", isCancelled) { - database.withTransaction { - val c = HealCounts() - // Address -> its saved row (null when absent), for this page only. - val addressRows = HashMap() - for (row in page.utxos) { - healEngineRow(walletId, row, foreignAccountIds, c, addressRows) - } - c - } + val healed = reconcileStoreStep(tally, walletRef, "heal", isCancelled) { + database.withTransaction { healEngineRows(walletId, page.utxos) } } ?: return false - tally.alreadyPresent += counts.alreadyPresent - tally.inserted += counts.inserted - tally.insertedDuffs += counts.insertedDuffs - tally.healedSpent += counts.healedSpent - tally.skippedImmature += counts.skippedImmature - tally.skippedUnresolvedAccount += counts.skippedUnresolvedAccount - tally.skippedMissingAddress += counts.skippedMissingAddress - tally.skippedInvalid += counts.skippedInvalid - tally.skippedForeign += counts.skippedForeign - // Logged once the step committed, so a rolled-back step - // never claims a row it did not write. - for (outpoint in counts.healedOutpoints) logReconcileItem("healed", outpoint, walletRef) - for (outpoint in counts.healedSpentOutpoints) logReconcileItem("healed_spent", outpoint, walletRef) + commitStoreStep(tally, healed, walletRef) } // Counted once the page's step is done, as Swift counts it. - tally.engineRows += page.utxos.size + tally += TxoReconcileReport(engineRows = page.utxos.size) page } finally { - withContext(NonCancellable) { engine.releaseVerdicts() } + engine.releaseVerdicts() } // The page is applied and its guard released: report it, so the // next page, and the wallet's next run if this one stops here, @@ -2071,14 +1986,24 @@ class PlatformWalletPersistenceHandler( tally.stopOnBudget() return false } - if (!page.hasMore || page.utxos.isEmpty()) return true + // The engine reports no more on an empty page. + if (!page.hasMore) return true } } /** - * Apply one engine inventory row to the store (inside the caller's + * Merge a committed store step into the run, then log its rows — only + * now, so a rolled-back step never claims a row it did not write. + */ + private fun commitStoreStep(tally: ReconcileTally, step: StoreStepResult, walletRef: String) { + tally += step.counts + for ((action, outpoint) in step.logged) logReconcileItem(action, outpoint, walletRef) + } + + /** + * Apply one engine inventory page to the store (inside the caller's * exclusion lock and Room transaction) — Swift's - * `reconcileHealMissingTxos`, in its order: + * `reconcileHealMissingTxos`. Each row is judged in its order: * * 1. fields this store cannot decode (txid not 32 bytes of hex, a * malformed script or identity half), or the engine's @@ -2098,112 +2023,150 @@ class PlatformWalletPersistenceHandler( * no build could sign for it. A later run heals it once the address * is persisted under the account. * - * Between 2 and 3 sits the contact exclusion (see [reconcileTxos]). A - * row that passes is inserted through [upsertUtxoRow] — the changeset + * The engine never offers a watch-only DIP-15 contact coin (the + * contact's money, not this wallet's), so there is no contact check + * here. The store is read in bulk, before any row is written: one + * presence query for the page's eligible rows, one account lookup per + * distinct tuple, and one address query for the rows the store lacks + * whose account resolves. Nothing a heal writes changes an account or an + * address row, and a row the page itself inserted counts as present for + * a later row on the same outpoint, so every row gets the answer it + * would get read one at a time. + * + * A row that passes is inserted through [upsertUtxoRow] — the changeset * path's own insert, so both writers honour the same rules (stub parent * transaction, account stamp, address link, pending-input drain) — with * `isConfirmed = true` (an eligible coin is engine-confirmed) and the * engine's other flags as given. A row the drain wrote spent on the spot - * is counted as [HealCounts.healedSpent]: the engine holds the coin and - * the store now says spent, which is not a repair of the divergence the - * engine reported. + * is counted as [TxoReconcileReport.healedSpent]: the engine holds the + * coin and the store now says spent, which is not a repair of the + * divergence the engine reported. */ - private suspend fun healEngineRow( - walletId: ByteArray, - row: EngineUtxoRow, - foreignAccountIds: Set, - counts: HealCounts, - addressRows: MutableMap, - ) { - val txid = row.txid.hexToByteArrayOrNull() - val script = row.scriptHex.hexToByteArrayOrNull() - val userIdentityId = row.userIdentityId?.hexToByteArrayOrNull() - val friendIdentityId = row.friendIdentityId?.hexToByteArrayOrNull() - if (txid == null || txid.size != 32 || script == null || - (row.userIdentityId != null && userIdentityId == null) || - (row.friendIdentityId != null && friendIdentityId == null) - ) { - counts.skippedInvalid++ - return - } - when (row.healVerdict) { - TXO_HEAL_VERDICT_ELIGIBLE -> Unit - TXO_HEAL_VERDICT_IMMATURE -> { - counts.skippedImmature++ - return - } - // TXO_HEAL_VERDICT_INVALID, and any verdict this build does not - // know: nothing the engine vouched for. - else -> { - counts.skippedInvalid++ - return + private suspend fun healEngineRows(walletId: ByteArray, rows: List): StoreStepResult { + var counts = TxoReconcileReport() + // Steps 1 and 2: what the row says and what the engine says of it. + val eligible = ArrayList(rows.size) + for (row in rows) { + val candidate = HealCandidate.decode(row) + when { + candidate == null -> counts += TxoReconcileReport(skippedInvalid = 1) + row.healVerdict == TXO_HEAL_VERDICT_ELIGIBLE -> eligible += candidate + row.healVerdict == TXO_HEAL_VERDICT_IMMATURE -> counts += TxoReconcileReport(skippedImmature = 1) + // TXO_HEAL_VERDICT_INVALID, and any verdict this build does + // not know: nothing the engine vouched for. + else -> counts += TxoReconcileReport(skippedInvalid = 1) } } - // The inventory tags every coin with its owning account tuple. The - // tag is the authoritative foreign check — a watch-only external - // account's coin is the CONTACT's money whether or not its address - // row survived persistence. The address-based check (through - // core_addresses.accountId, the same second path the classify pass - // resolves store rows by) stays as a fallback. The engine's - // inventory never offers a contact's coin; this side does not rely - // on that alone. - if (row.typeTag == ACCOUNT_TYPE_TAG_DASHPAY_EXTERNAL) { - counts.skippedForeign++ - return - } - val addressRow = addressRows.memo(row.address) { - database.coreAddressDao().getByAddress(row.address) + val present = chunkedFlatMap(eligible.map { it.outpoint }) { database.txoDao().getExistingOutpoints(it) } + .mapTo(HashSet()) { it.toHex() } + // The owning account of each row the store lacks, one lookup per + // tuple; absent identity halves are the non-DashPay case. + val accountByTuple = HashMap() + val accounts = eligible.associateWith { c -> + if (c.outpointHex in present) { + null + } else { + accountByTuple.memo(c.accountTuple) { + findReconcileAccount( + walletId, c.row.typeTag, c.row.index, c.row.standardTag, c.row.registrationIndex, + c.row.keyClass, c.userIdentityId ?: ByteArray(0), c.friendIdentityId ?: ByteArray(0), + ) + } + } } - val addressOwner = addressRow?.accountId - if (addressOwner != null && addressOwner in foreignAccountIds) { - counts.skippedForeign++ - return + val addressRows = chunkedFlatMap( + eligible.filter { accounts[it] != null }.map { it.row.address }.distinct(), + ) { database.coreAddressDao().getByAddresses(it) }.associateBy { it.address } + + val healed = ArrayList() + val healedSpent = ArrayList() + for (c in eligible) { + if (c.outpointHex in present) { + counts += TxoReconcileReport(alreadyPresent = 1) + continue + } + val account = accounts[c] + if (account == null) { + counts += TxoReconcileReport(skippedUnresolvedAccount = 1) + continue + } + // A cold restore rebuilds each account's address pool from the + // rows filed under it, so an address with no row, a row with no + // account, or a row under another account leaves the coin with + // no derivation path: it would be restored into the balance, but + // no build could sign for it. Leave it out until the address is + // persisted under this account; a later run heals it. + if (addressRows[c.row.address]?.accountId != account.id) { + counts += TxoReconcileReport(skippedMissingAddress = 1) + continue + } + val row = c.row + val writtenSpent = upsertUtxoRow( + database, walletId, c.txid, row.vout, row.amount, row.address, c.script, row.height, + isCoinbase = row.isCoinbase, + isConfirmed = true, + isInstantLocked = row.isInstantlocked, + isLocked = row.isLocked, + resolvedAccountId = account.id, + ) + present += c.outpointHex + // netAmount is NOT mutated here. The record's net may already be + // correct (a corrective record callback can land while its TXO + // delivery races this run), and adding the healed amount to an + // already-corrected net double-credits. + if (writtenSpent) { + // A pending-input claim or a swept tombstone already covered + // this outpoint, so the drain wrote the row spent on the spot. + counts += TxoReconcileReport(healedSpent = 1) + healedSpent += c.outpoint + } else { + counts += TxoReconcileReport(inserted = 1, insertedDuffs = row.amount) + healed += c.outpoint + } } + return StoreStepResult( + counts, + healed.map { "healed" to it } + healedSpent.map { "healed_spent" to it }, + ) + } + + /** An engine inventory row whose fields this store could decode. */ + private class HealCandidate( + val row: EngineUtxoRow, + val txid: ByteArray, + val script: ByteArray, + val userIdentityId: ByteArray?, + val friendIdentityId: ByteArray?, + ) { val outpoint = makeOutpoint(txid, row.vout) - if (database.txoDao().getByOutpoint(outpoint) != null) { - counts.alreadyPresent++ - return - } - // Absent identity halves are the non-DashPay case (no identity). - val account = findReconcileAccount( - walletId, row.typeTag, row.index, row.standardTag, row.registrationIndex, row.keyClass, - userIdentityId ?: ByteArray(0), friendIdentityId ?: ByteArray(0), + val outpointHex = outpoint.toHex() + val accountTuple = AccountTuple( + row.typeTag, row.standardTag, row.index, row.registrationIndex, row.keyClass, + row.userIdentityId, row.friendIdentityId, ) - if (account == null) { - counts.skippedUnresolvedAccount++ - return - } - // A cold restore rebuilds each account's address pool from the rows - // filed under it, so an address with no row, a row with no account, - // or a row under another account leaves the coin with no derivation - // path: it would be restored into the balance, but no build could - // sign for it. Leave it out until the address is persisted under - // this account; a later run heals it. - if (addressRow?.accountId != account.id) { - counts.skippedMissingAddress++ - return - } - val writtenSpent = upsertUtxoRow( - database, walletId, txid, row.vout, row.amount, row.address, script, row.height, - isCoinbase = row.isCoinbase, - isConfirmed = true, - isInstantLocked = row.isInstantlocked, - isLocked = row.isLocked, - resolvedAccountId = account.id, + + /** The seven fields an engine row names its account by. */ + data class AccountTuple( + val typeTag: Int, + val standardTag: Int, + val index: Int, + val registrationIndex: Int, + val keyClass: Int, + val userIdentityId: String?, + val friendIdentityId: String?, ) - // netAmount is NOT mutated here. The record's net may already be - // correct (a corrective record callback can land while its TXO - // delivery races this run), and adding the healed amount to an - // already-corrected net double-credits. - if (writtenSpent) { - // A pending-input claim or a swept tombstone already covered - // this outpoint, so the drain wrote the row spent on the spot. - counts.healedSpent++ - counts.healedSpentOutpoints += outpoint - } else { - counts.inserted++ - counts.insertedDuffs += row.amount - counts.healedOutpoints += outpoint + + companion object { + /** [row]'s fields as bytes, or null when one does not decode: a + * txid that is not 32 bytes of hex, a malformed script or + * identity half. */ + fun decode(row: EngineUtxoRow): HealCandidate? { + val txid = row.txid.hexToByteArrayOrNull()?.takeIf { it.size == 32 } ?: return null + val script = row.scriptHex.hexToByteArrayOrNull() ?: return null + val user = row.userIdentityId?.let { it.hexToByteArrayOrNull() ?: return null } + val friend = row.friendIdentityId?.let { it.hexToByteArrayOrNull() ?: return null } + return HealCandidate(row, txid, script, user, friend) + } } } @@ -2268,7 +2231,7 @@ class PlatformWalletPersistenceHandler( * shift the rows after it. * * Every store page is classified, one with nothing to ask (an empty - * page, or every row unresolved or a contact's) too: each classification + * page, or every row unresolved) too: each classification * is one page of the run's budget, and it names the page * [TxoReconcileEngine.classifyPageApplied] reports. A page is reported * once its flips committed (or it had none to write) and its guard is @@ -2276,12 +2239,15 @@ class PlatformWalletPersistenceHandler( * engine's store-cursor bound); the store's last page — a short one — is * reported with no cursor, which ends the cycle. A stale page is not * reported. + * + * A page's rows count toward the run once, unless the page is handed + * back unwalked — a budget-spent read, or a stale page classified + * again; every other exit keeps the page counted. */ private suspend fun classifyStoreRows( walletId: ByteArray, from: ByteArray, limit: Int, - foreignAccountIds: Set, isCancelled: () -> Boolean, engine: TxoReconcileEngine, tally: ReconcileTally, @@ -2292,14 +2258,12 @@ class PlatformWalletPersistenceHandler( while (true) { if (isCancelled()) return val page = reconcileStoreStep(tally, walletRef, "unspent_page", isCancelled) { - readUnspentPage(walletId, after, limit, foreignAccountIds) + readUnspentPage(walletId, after, limit) } ?: return - tally.storeRows += page.rows.size - tally.unresolvedAccount += page.unresolvedAccount - tally.skippedForeign += page.skippedForeign // The engine guard is held from the classification to the // `finally`, across the apply only — see above. + var handedBack = false try { val answer = engine.classify(encodeOutpointQueries(page.queries)) if (answer == null) { @@ -2310,9 +2274,7 @@ class PlatformWalletPersistenceHandler( if (answer.budgetSpent) { // Nothing classified, nothing held: this run walked none // of the page, and the wallet's next run reads it. - tally.storeRows -= page.rows.size - tally.unresolvedAccount -= page.unresolvedAccount - tally.skippedForeign -= page.skippedForeign + handedBack = true tally.stopOnBudget() return } @@ -2321,34 +2283,30 @@ class PlatformWalletPersistenceHandler( return } if (page.rows.isNotEmpty()) { - val counts = reconcileStoreStep(tally, walletRef, "flip", isCancelled) { + val applied = reconcileStoreStep(tally, walletRef, "flip", isCancelled) { if (committedRoundGeneration != page.generation || openRound(walletId) != null) { - FlipCounts().apply { staleGeneration = true } + StoreStepResult(TxoReconcileReport(), stale = true) } else { database.withTransaction { applyEngineClasses(page.rows, answer.classes) } } } ?: return - if (counts.staleGeneration) { + if (applied.stale) { staleInARow++ - tally.staleRetries++ + tally += TxoReconcileReport(staleRetries = 1) if (!engine.retryStalePage(staleInARow)) return // Same cursor, not reported: nothing was written, so // the page is read again and classified against the // store as it is now. - tally.storeRows -= page.rows.size - tally.unresolvedAccount -= page.unresolvedAccount - tally.skippedForeign -= page.skippedForeign + handedBack = true continue } - tally.flipped += counts.flipped - tally.flippedDuffs += counts.flippedDuffs - tally.unspent += counts.unspent - tally.unknown += counts.unknown - tally.notOwned += counts.notOwned - for (outpoint in counts.flippedOutpoints) logReconcileItem("flipped_spent", outpoint, walletRef) + commitStoreStep(tally, applied, walletRef) } } finally { - withContext(NonCancellable) { engine.releaseVerdicts() } + if (!handedBack) { + tally += TxoReconcileReport(storeRows = page.rows.size, unresolvedAccount = page.unresolvedAccount) + } + engine.releaseVerdicts() } staleInARow = 0 // The page is applied and its guard released: report it. A short @@ -2357,7 +2315,7 @@ class PlatformWalletPersistenceHandler( val next = if (page.fetched < limit) null else page.lastOutpoint val budgetLeft = reportPageApplied(tally, isCancelled) { engine.classifyPageApplied(next) } ?: return if (next == null) { - tally.completed = true + tally.report = tally.report.copy(completed = true) return } if (!budgetLeft) { @@ -2379,23 +2337,14 @@ class PlatformWalletPersistenceHandler( * the question. A row still mid-insert (no 32-byte txid) is not * classifiable at all. * - * Watch-only DIP-15 contact rows are excluded via [foreignAccountIds]. - * Production changeset writes leave txos.accountId null and route - * ownership through the address projection, so the owner resolves - * BOTH paths ([resolveTxoOwnerAccount]) — an accountId-only check - * silently classifies every contact row. + * A watch-only DIP-15 contact row is asked about like any other: the + * engine gives a contact's coin no verdict, so it is never flipped. */ - private suspend fun readUnspentPage( - walletId: ByteArray, - after: ByteArray, - limit: Int, - foreignAccountIds: Set, - ): UnspentPage { + private suspend fun readUnspentPage(walletId: ByteArray, after: ByteArray, limit: Int): UnspentPage { val fetched = database.txoDao().pageUnspentByWallet(walletId, after, limit) val rows = ArrayList(fetched.size) val queries = ArrayList(fetched.size) var unresolvedAccount = 0 - var skippedForeign = 0 val accountByAddress = HashMap() val accountById = HashMap() for (row in fetched) { @@ -2406,10 +2355,6 @@ class PlatformWalletPersistenceHandler( unresolvedAccount++ continue } - if (owner.id in foreignAccountIds) { - skippedForeign++ - continue - } rows.add(row) queries.add(outpointQueryFor(owner, txid, row.vout, row.scriptPubKey)) } @@ -2419,7 +2364,6 @@ class PlatformWalletPersistenceHandler( fetched = fetched.size, lastOutpoint = fetched.lastOrNull()?.outpoint, unresolvedAccount = unresolvedAccount, - skippedForeign = skippedForeign, generation = committedRoundGeneration, ) } @@ -2459,40 +2403,33 @@ class PlatformWalletPersistenceHandler( * rust-dashcore#992), its `lastUpdated` bumped, and every pending-input * row on the outpoint dropped (Swift's `removePendingInputs`), so a * claim on a settled coin does not linger. `isSpent` is monotonic: a - * row already spent, or gone, by now is stale and left alone. Unspent, - * unknown and not-owned verdicts are counted, never acted on: absence of - * a coin from the engine proves nothing. + * row already spent, or gone, by now is stale and left alone (the + * spent-mark matches only an unspent row). Unspent, unknown and + * not-owned verdicts are counted, never acted on: absence of a coin from + * the engine proves nothing. */ - private suspend fun applyEngineClasses(rows: List, verdicts: ByteArray): FlipCounts { - val counts = FlipCounts() + private suspend fun applyEngineClasses(rows: List, verdicts: ByteArray): StoreStepResult { + var counts = TxoReconcileReport() + val flipped = ArrayList() for ((i, row) in rows.withIndex()) { - when (verdicts[i]) { - OUTPOINT_CLASS_UNSPENT -> counts.unspent++ - OUTPOINT_CLASS_NOT_OWNED -> counts.notOwned++ + counts += when (verdicts[i]) { + OUTPOINT_CLASS_UNSPENT -> TxoReconcileReport(unspent = 1) + OUTPOINT_CLASS_NOT_OWNED -> TxoReconcileReport(notOwned = 1) OUTPOINT_CLASS_KNOWN_UNCREDITED -> { - val current = database.txoDao().getByOutpoint(row.outpoint) // A row already spent, or gone, changed under the walk. - if (current == null || current.isSpent || - database.txoDao().markSpentByOutpoint(row.outpoint, now()) == 0 - ) { - continue - } - val claims = database.documentDao().getPendingInputsByOutpoint(row.outpoint) - if (claims.isNotEmpty()) { - database.documentDao().deletePendingInputsByIds(claims.map { it.id }) - } - counts.flipped++ - counts.flippedDuffs += current.amount - counts.flippedOutpoints += row.outpoint + if (database.txoDao().markSpentByOutpoint(row.outpoint, now()) == 0) continue + flipped += row.outpoint + TxoReconcileReport(flipped = 1, flippedDuffs = row.amount) } - // OUTPOINT_CLASS_UNKNOWN, and any code this build does not - // know: the engine has no opinion. After a restart its - // finalized set is empty, so this is the ordinary answer - // rather than the alarming one. - else -> counts.unknown++ + // 0 (unknown), and any code this build does not know: the + // engine has no opinion. After a restart its finalized set + // is empty, so this is the ordinary answer rather than the + // alarming one. + else -> TxoReconcileReport(unknown = 1) } } - return counts + chunked(flipped) { database.documentDao().deletePendingInputsByOutpoints(it) } + return StoreStepResult(counts, flipped.map { "flipped_spent" to it }) } /** @@ -2513,18 +2450,15 @@ class PlatformWalletPersistenceHandler( isCancelled: () -> Boolean, step: suspend () -> T, ): T? = - try { + txoReconcileOrNull( + onFailure = { message -> + tally += TxoReconcileReport(storeFailures = 1) + Log.e(TAG, message) + }, + what = "store step failed operation=$operation", + walletRef = walletRef, + ) { callbackExclusion.withLock { if (isCancelled()) null else step() } - } catch (cancellation: kotlin.coroutines.cancellation.CancellationException) { - throw cancellation - } catch (t: Throwable) { - tally.storeFailures++ - Log.e( - TAG, - "txos reconcile: store step failed operation=$operation " + - "error=${t.javaClass.simpleName} wallet=$walletRef", - ) - null } /** One line per healed / healed-spent / flipped row: the action and @@ -2553,7 +2487,7 @@ class PlatformWalletPersistenceHandler( "skippedImmature=${report.skippedImmature} " + "skippedUnresolvedAccount=${report.skippedUnresolvedAccount} " + "skippedMissingAddress=${report.skippedMissingAddress} " + - "skippedInvalid=${report.skippedInvalid} skippedForeign=${report.skippedForeign} " + + "skippedInvalid=${report.skippedInvalid} " + "unresolvedAccount=${report.unresolvedAccount} staleRetries=${report.staleRetries} " + "transportFailures=${report.transportFailures} storeFailures=${report.storeFailures} " + "completed=${report.completed} stoppedOnBudget=${report.stoppedOnBudget} " + @@ -5525,10 +5459,6 @@ class PlatformWalletPersistenceHandler( runBlocking(dispatcher) { block() } companion object { - /** `AccountTypeTagFFI::DashpayExternalAccount` — watch-only DIP-15 - * contact accounts the engine's inventories never report. */ - internal const val ACCOUNT_TYPE_TAG_DASHPAY_EXTERNAL = 13 - /** Bytes of a `txos.outpoint` key: 32-byte txid (wire order) plus * the vout as a little-endian `Int`. Also the wire format of the * reconcile's outpoint-classification batch. */ @@ -5544,6 +5474,7 @@ class PlatformWalletPersistenceHandler( internal const val UTXO_CREDIT_VERDICT_DOOMED: Byte = 2 /** Not credited for a reason the bridge cannot name; no context. */ + @VisibleForTesting internal const val UTXO_CREDIT_VERDICT_UNCREDITED: Byte = 3 /** @@ -5561,6 +5492,7 @@ class PlatformWalletPersistenceHandler( * `TXO_HEAL_VERDICT_*` (`platform_wallet::manager::txo_reconcile:: * TxoHealVerdict`). The engine decides them; only * [TXO_HEAL_VERDICT_ELIGIBLE] may be inserted. */ + @VisibleForTesting internal const val TXO_HEAL_VERDICT_INVALID = 0 internal const val TXO_HEAL_VERDICT_IMMATURE = 1 internal const val TXO_HEAL_VERDICT_ELIGIBLE = 2 @@ -5618,8 +5550,7 @@ class PlatformWalletPersistenceHandler( * record that spends the outpoint. Plain absence from the live * UTXO set is NOT evidence — a mempool spend that never confirms * and a conflict sweep both leave a coin absent with its funding - * known — so those arrive as [OUTPOINT_CLASS_UNKNOWN]. */ - internal const val OUTPOINT_CLASS_UNKNOWN: Byte = 0 + * known — so those arrive as unknown (0). */ internal const val OUTPOINT_CLASS_UNSPENT: Byte = 1 internal const val OUTPOINT_CLASS_KNOWN_UNCREDITED: Byte = 2 internal const val OUTPOINT_CLASS_NOT_OWNED: Byte = 3 @@ -5817,17 +5748,38 @@ class NativeStoredTransactionInputs( // ── Free functions (unit-testable, no `this`) ───────────────────────── -/** Lowercase hex of a byte array (used as the changeset-buffer key). */ /** - * Pack DIP-15 accepted-account indices (`u32`s crossing JNI as an - * `IntArray` bit-pattern) into a big-endian 4-bytes-per-entry BLOB for - * the `contactAcceptedAccounts` column. Empty → null (matches the - * absent-optional convention of the other nullable columns). + * Run one step of a TXO-store reconcile run, or of its launch, and turn a + * failure into null: a [kotlin.coroutines.cancellation.CancellationException] + * is rethrown, anything else is handed to [onFailure] as the line + * `txos reconcile: error= wallet=` — the + * exception type only, since a message can echo the input it rejected. */ +internal inline fun txoReconcileOrNull( + onFailure: (message: String) -> Unit, + what: String, + walletRef: String, + block: () -> T, +): T? = + try { + block() + } catch (cancellation: kotlin.coroutines.cancellation.CancellationException) { + throw cancellation + } catch (t: Throwable) { + onFailure("txos reconcile: $what error=${t.javaClass.simpleName} wallet=$walletRef") + null + } + /** [getOrPut] that also remembers a null result, so a miss is looked up once. */ private inline fun MutableMap.memo(key: K, load: () -> V?): V? = if (containsKey(key)) get(key) else load().also { put(key, it) } +/** + * Pack DIP-15 accepted-account indices (`u32`s crossing JNI as an + * `IntArray` bit-pattern) into a big-endian 4-bytes-per-entry BLOB for + * the `contactAcceptedAccounts` column. Empty → null (matches the + * absent-optional convention of the other nullable columns). + */ internal fun encodeAcceptedAccounts(accounts: IntArray): ByteArray? { if (accounts.isEmpty()) return null val buffer = java.nio.ByteBuffer.allocate(accounts.size * 4) // big-endian by default @@ -5842,6 +5794,7 @@ internal fun decodeAcceptedAccounts(blob: ByteArray?): IntArray { return IntArray(blob.size / 4) { buffer.int } } +/** Lowercase hex of a byte array (used as the changeset-buffer key). */ internal fun ByteArray.toHex(): String { val out = CharArray(size * 2) val hex = "0123456789abcdef".toCharArray() diff --git a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/dao/CoreAddressDao.kt b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/dao/CoreAddressDao.kt index 6138a971626..fb987d47693 100644 --- a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/dao/CoreAddressDao.kt +++ b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/dao/CoreAddressDao.kt @@ -21,6 +21,11 @@ interface CoreAddressDao { @Query("SELECT * FROM core_addresses WHERE address = :address") suspend fun getByAddress(address: String): CoreAddressEntity? + /** The saved rows of [addresses] (absent ones are left out). Callers + * chunk the list under the 999-variable ceiling. */ + @Query("SELECT * FROM core_addresses WHERE address IN (:addresses)") + suspend fun getByAddresses(addresses: List): List + @Query("SELECT * FROM core_addresses WHERE accountId = :accountId ORDER BY poolTypeTag, addressIndex") fun observeByAccount(accountId: Long): Flow> diff --git a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/dao/DocumentDao.kt b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/dao/DocumentDao.kt index de86eea2e0b..afb327050d0 100644 --- a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/dao/DocumentDao.kt +++ b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/dao/DocumentDao.kt @@ -228,6 +228,11 @@ interface DocumentDao { @Query("DELETE FROM pending_inputs WHERE id IN (:ids)") suspend fun deletePendingInputsByIds(ids: List) + /** Every pending-input row on any of [outpoints] — the reconcile's + * flipped coins; chunked by the caller. */ + @Query("DELETE FROM pending_inputs WHERE outpoint IN (:outpoints)") + suspend fun deletePendingInputsByOutpoints(outpoints: List) + /** * Delete every wallet's tombstones on [outpoints] — outputs of a * transaction swept in this round, dead coins nobody may hold a claim diff --git a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/dao/TxoDao.kt b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/dao/TxoDao.kt index 5b7ee10fc39..c78679d92e5 100644 --- a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/dao/TxoDao.kt +++ b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/dao/TxoDao.kt @@ -98,6 +98,14 @@ interface TxoDao { @Query("SELECT * FROM txos WHERE outpoint IN (:outpoints)") suspend fun getByOutpoints(outpoints: List): List + /** + * Which of [outpoints] the store holds a row for — the reconcile heal + * pass's presence check for one engine page. Callers chunk the list + * under the 999-variable ceiling. + */ + @Query("SELECT outpoint FROM txos WHERE outpoint IN (:outpoints)") + suspend fun getExistingOutpoints(outpoints: List): List + /** * Every row still linked to one of [spendingTxids] — the sweep pass's * link-keyed fallback for a loser whose stored bytes cannot name its diff --git a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/ManagedPlatformWallet.kt b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/ManagedPlatformWallet.kt index 5a01e25f73d..9fdad53f104 100644 --- a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/ManagedPlatformWallet.kt +++ b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/ManagedPlatformWallet.kt @@ -1295,18 +1295,6 @@ class ManagedPlatformWallet internal constructor( cleanable.clean() } - /** Split a flat `byte[]` of concatenated 32-byte ids into per-id rows. */ - private fun splitIds(flat: ByteArray): List { - if (flat.size < 32) return emptyList() - val out = ArrayList(flat.size / 32) - var offset = 0 - while (offset + 32 <= flat.size) { - out.add(flat.copyOfRange(offset, offset + 32)) - offset += 32 - } - return out - } - /** * Encode [outputs] to the credit-outputs blob the transfer FFI reads: * `u32 rowCount` then per row `u8 addressType, u8[20] hash, u64 credits`. @@ -1409,3 +1397,16 @@ class ManagedPlatformWallet internal constructor( } } } + +/** Split a flat `byte[]` of concatenated 32-byte ids into per-id rows; a + * ragged tail shorter than one id is dropped. */ +internal fun splitIds(flat: ByteArray): List { + if (flat.size < 32) return emptyList() + val out = ArrayList(flat.size / 32) + var offset = 0 + while (offset + 32 <= flat.size) { + out.add(flat.copyOfRange(offset, offset + 32)) + offset += 32 + } + return out +} diff --git a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/PlatformWalletManager.kt b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/PlatformWalletManager.kt index 058ba92ceb1..44adc18183d 100644 --- a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/PlatformWalletManager.kt +++ b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/PlatformWalletManager.kt @@ -36,6 +36,7 @@ import org.dashfoundation.dashsdk.persistence.hexToByteArray import org.dashfoundation.dashsdk.persistence.logReference import org.dashfoundation.dashsdk.persistence.toBase58String import org.dashfoundation.dashsdk.persistence.toHex +import org.dashfoundation.dashsdk.persistence.txoReconcileOrNull import org.json.JSONArray import org.dashfoundation.dashsdk.security.BiometricGate import org.dashfoundation.dashsdk.security.IdentityKeyPrivateKeyDeriver @@ -1488,7 +1489,6 @@ class PlatformWalletManager( return try { teardownGate.withOp { if (isCancelled()) return@withOp stale() - val tipOut = IntArray(1) val startOut = IntArray(1) val storeAfterOut = arrayOfNulls(1) // NonCancellable: a RUN return holds a native claim that only @@ -1499,7 +1499,7 @@ class PlatformWalletManager( val gate = withContext(NonCancellable + Dispatchers.IO) { mapNativeErrors { WalletManagerNative.walletManagerTxoReconcileBegin( - managerHandle, walletId, tipOut, startOut, storeAfterOut, + managerHandle, walletId, startOut, storeAfterOut, ) } } @@ -1529,7 +1529,7 @@ class PlatformWalletManager( withContext(NonCancellable + Dispatchers.IO) { endTxoReconcileRun( walletRef = walletRef, - freeHeldVerdicts = engine::freeHeldVerdicts, + freeHeldVerdicts = engine::releaseVerdicts, finish = { mapNativeErrors { WalletManagerNative.walletManagerTxoReconcileFinish(managerHandle, walletId, ran) @@ -1619,8 +1619,6 @@ class PlatformWalletManager( WalletManagerNative.walletManagerTxoReconcileClassifyPageApplied(managerHandle, walletId, nextAfter) } - override suspend fun releaseVerdicts() = freeHeldVerdicts() - override fun retryStalePage(staleInARow: Int): Boolean = // A failed call stops the run: retrying is the engine's call, // and an engine that cannot answer has not allowed it. @@ -1634,7 +1632,7 @@ class PlatformWalletManager( * read that waits on the engine behind a writer that waits on this * guard. */ - fun freeHeldVerdicts() { + override fun releaseVerdicts() { val token = held.getAndSet(0L) if (token != 0L) WalletManagerNative.txoReconcileHeldVerdictsFree(token) } @@ -1646,17 +1644,11 @@ class PlatformWalletManager( private suspend fun call(what: String, read: () -> T): T? = withContext(Dispatchers.IO) { - try { - mapNativeErrors { read() } - } catch (cancellation: kotlin.coroutines.cancellation.CancellationException) { - throw cancellation - } catch (t: Throwable) { - android.util.Log.w( - "PlatformWalletManager", - "txos reconcile: $what failed error=${t.javaClass.simpleName} wallet=$walletRef", - ) - null - } + txoReconcileOrNull( + onFailure = { android.util.Log.w("PlatformWalletManager", it) }, + what = "$what failed", + walletRef = walletRef, + ) { mapNativeErrors { read() } } } } @@ -2765,21 +2757,15 @@ class PlatformWalletManager( } }.getOrNull() ?: return val loaded = wallets.value - for (offset in 0 until due.size - due.size % 32 step 32) { - val key = due.copyOfRange(offset, offset + 32).toHex() + for (id in splitIds(due)) { + val key = id.toHex() val walletId = loaded[key]?.walletId ?: continue txoReconcileLaunches.launch(scope, key) { - try { - reconcileTxoStore(walletId) - } catch (cancellation: kotlin.coroutines.cancellation.CancellationException) { - throw cancellation - } catch (t: Throwable) { - android.util.Log.w( - "PlatformWalletManager", - "txos reconcile: run failed error=${t.javaClass.simpleName} " + - "wallet=${logReference(walletId)}", - ) - } + txoReconcileOrNull( + onFailure = { android.util.Log.w("PlatformWalletManager", it) }, + what = "run failed", + walletRef = logReference(walletId), + ) { reconcileTxoStore(walletId) } } } } diff --git a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcile.kt b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcile.kt index 476b14300fe..95ae2654a35 100644 --- a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcile.kt +++ b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/wallet/TxoReconcile.kt @@ -1,5 +1,6 @@ package org.dashfoundation.dashsdk.wallet +import androidx.annotation.VisibleForTesting import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.launch import org.dashfoundation.dashsdk.persistence.PlatformWalletPersistenceHandler @@ -113,5 +114,6 @@ internal class TxoReconcileLaunches { } /** Whether a launch of [walletKey] has not ended. */ + @VisibleForTesting operator fun contains(walletKey: String): Boolean = walletKey in launched } diff --git a/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandlerTest.kt b/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandlerTest.kt index a9a7ca5b2ab..58dab13c176 100644 --- a/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandlerTest.kt +++ b/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandlerTest.kt @@ -8,6 +8,7 @@ import androidx.room.Room import androidx.sqlite.db.SupportSQLiteDatabase import androidx.sqlite.db.SupportSQLiteOpenHelper import androidx.sqlite.db.SupportSQLiteQuery +import androidx.sqlite.db.SupportSQLiteStatement import androidx.sqlite.db.framework.FrameworkSQLiteOpenHelperFactory import androidx.test.core.app.ApplicationProvider import kotlinx.coroutines.Dispatchers @@ -207,13 +208,19 @@ class PlatformWalletPersistenceHandlerTest { wallet, 0, 0, 0, 0, 0, ByteArray(0), ByteArray(0), ByteArray(78) { xpubFill }, ) val account = db.accountDao().observeByWallet(wallet).first().single() + saveCoreAddress(account.id, address) + } + + /** Save the BIP44 `core_addresses` row for [address] under [accountId] + * (null: a row with no account), as a real wallet load leaves it. */ + private suspend fun saveCoreAddress(accountId: Long?, address: String) { db.coreAddressDao().upsert( CoreAddressEntity( address = address, poolTypeTag = 0, addressIndex = 0, derivationPath = "m/44'/1'/0'/0/0", - accountId = account.id, + accountId = accountId, ), ) } @@ -7135,17 +7142,8 @@ class PlatformWalletPersistenceHandlerTest { /** Save the `core_addresses` row for [address] under [accountId], as a * real wallet load leaves it. The heal pass refuses a coin whose * address has no saved row. */ - private suspend fun saveReconcileAddress(accountId: Long, address: String = reconcileAddress) { - db.coreAddressDao().upsert( - CoreAddressEntity( - address = address, - poolTypeTag = 0, - addressIndex = 0, - derivationPath = "m/44'/1'/0'/0/0", - accountId = accountId, - ), - ) - } + private suspend fun saveReconcileAddress(accountId: Long, address: String = reconcileAddress) = + saveCoreAddress(accountId, address) /** An unspent, confirmed row of [wallet] for `txid:vout`, owned through * the address projection exactly as the changeset path writes it. */ @@ -7579,6 +7577,10 @@ class PlatformWalletPersistenceHandlerTest { assertTrue(summary, summary.contains("completed=true")) } + /** `AccountTypeTagFFI::DashpayExternalAccount`: a contact's watch-only + * DIP-15 chain. */ + private val ACCOUNT_TYPE_TAG_DASHPAY_EXTERNAL = 13 + /** A watch-only DIP-15 contact (DashPay external) account of [walletId], * owning [address] through the core_addresses projection. */ private suspend fun seedContactAccount(index: Int, address: String): Long { @@ -7586,7 +7588,7 @@ class PlatformWalletPersistenceHandlerTest { val accountId = db.accountDao().insert( org.dashfoundation.dashsdk.persistence.entities.AccountEntity( walletId = walletId, - accountType = PlatformWalletPersistenceHandler.ACCOUNT_TYPE_TAG_DASHPAY_EXTERNAL, + accountType = ACCOUNT_TYPE_TAG_DASHPAY_EXTERNAL, accountIndex = index, accountTypeName = "DashpayExternalAccount", ), @@ -7606,14 +7608,15 @@ class PlatformWalletPersistenceHandlerTest { } @Test - fun watchOnlyContactRowsAreExcludedFromTheClassifyPassHoweverTheyResolve() = runTest { - // Watch-only DIP-15 contact rows are never in the engine's - // inventories; asking about them would be a false positive on every - // wallet with contact payments. Vout 6 is filed under the contact - // account directly. Vout 8 is the production shape: changeset - // writes leave txos.accountId null and route ownership through - // coreAddressId -> core_addresses.accountId, and the exclusion must - // resolve that path too, or every contact row gets classified. + fun watchOnlyContactRowsAreAskedAboutAndLeftUnspentHoweverTheyResolve() = runTest { + // A watch-only DIP-15 contact row is the contact's money: the engine + // gives it no verdict (`classify_outpoints`), so it is asked about + // like any other row and never flipped — even when the inventory + // names a spender for it. Vout 6 is filed under the contact account + // directly. Vout 8 is the production shape: changeset writes leave + // txos.accountId null and route ownership through coreAddressId -> + // core_addresses.accountId, so the query must name the contact + // account that way too. val contactAccountId = seedContactAccount(index = 0, address = "yContactRouted") for ((vout, address) in listOf(6 to "yContactAddr", 8 to "yContactRouted")) { handler.onWalletChangesetUtxoAdded( @@ -7627,11 +7630,19 @@ class PlatformWalletPersistenceHandlerTest { db.txoDao().getByOutpoint(makeOutpoint(changeTxid, 8))!!.accountId, ) - val report = handler.reconcileInventory(inventoryJson()) + val engine = FakeEngine(inventoryJson(spent = listOf(changeTxid to 6, changeTxid to 8))) + attributeUnownedTxos(walletId) + val report = handler.reconcileWith(engine) - assertEquals(2, report.skippedForeign) - assertEquals(0, report.storeRows) - assertEquals(0, report.unknown) + assertTrue(report.completed) + assertEquals(2, report.storeRows) + assertEquals(2, report.unknown) + assertEquals(0, report.flipped) + assertEquals( + "both rows are asked about under the contact account", + listOf(ACCOUNT_TYPE_TAG_DASHPAY_EXTERNAL, ACCOUNT_TYPE_TAG_DASHPAY_EXTERNAL), + engine.queries.map { it["typeTag"]!!.jsonPrimitive.int }, + ) assertFalse(storedIsSpent(changeTxid, 6)) assertFalse(storedIsSpent(changeTxid, 8)) } @@ -7759,31 +7770,34 @@ class PlatformWalletPersistenceHandlerTest { @Test fun theHealPassNeverHealsContactAccountCoins() = runTest { - // The engine's inventory carries the watch-only DIP-15 external - // accounts' coins — it tracks them to show payments TO contacts, but - // they are the CONTACT's money (12 rows / 5,692,493 duffs healed into - // the store on the 2026-08-25 large-wallet validation run before - // this exclusion). Such a coin is skipped and counted as foreign, - // not healed. The inventory tag is the authoritative check: vout 13 - // is skipped even though its address row never survived - // persistence. The address is the fallback: vout 9's resolves to an - // external account. + // The engine tracks the watch-only DIP-15 external accounts' coins + // to show payments TO contacts, but they are the CONTACT's money (12 + // rows / 5,692,493 duffs healed into the store on the 2026-08-25 + // large-wallet validation run before the exclusion). The engine's + // heal page never offers one (`wallet_utxos_page`): vout 13 is not + // in it at all. A funds-account coin whose address the store files + // under a contact account (vout 9) has no address row under its own + // account, so it is not healed either. seedContactAccount(index = 2, address = "yContactPaid") - val report = handler.reconcileInventory( + val engine = FakeEngine( inventoryJson( rows = listOf( engineRowJson(changeTxid, vout = 9, amount = 10_000L, address = "yContactPaid", height = 1_400_000), engineRowJson( changeTxid, vout = 13, amount = 10_000L, address = "yContactNoRow", height = 1_400_000, - typeTag = 13, identities = "11".repeat(32) to "22".repeat(32), + typeTag = ACCOUNT_TYPE_TAG_DASHPAY_EXTERNAL, identities = "11".repeat(32) to "22".repeat(32), ), ), ), ) + attributeUnownedTxos(walletId) + val report = handler.reconcileWith(engine) + assertTrue(report.completed) + assertEquals("the contact's coin is never offered", 1, report.engineRows) + assertEquals(1, report.skippedMissingAddress) assertEquals(0, report.inserted) assertEquals(0L, report.insertedDuffs) - assertEquals(2, report.skippedForeign) for (vout in listOf(9, 13)) { assertNull( "the contact's coin must not enter the store", @@ -7879,15 +7893,7 @@ class PlatformWalletPersistenceHandlerTest { inventoryJson(rows = listOf(engineRowJson(changeTxid, vout = 13, amount = 70_000L))), ) - db.coreAddressDao().upsert( - CoreAddressEntity( - address = reconcileAddress, - poolTypeTag = 0, - addressIndex = 0, - derivationPath = "m/44'/1'/0'/0/0", - accountId = null, - ), - ) + saveCoreAddress(accountId = null, address = reconcileAddress) val noAccount = handler.reconcileWith(engine) assertTrue(noAccount.completed) assertEquals("a row with no account", 1, noAccount.skippedMissingAddress) @@ -7899,7 +7905,6 @@ class PlatformWalletPersistenceHandlerTest { val otherAccount = handler.reconcileWith(engine) assertTrue(otherAccount.completed) assertEquals("a row under another account", 1, otherAccount.skippedMissingAddress) - assertEquals(0, otherAccount.skippedForeign) assertEquals(0, otherAccount.inserted) assertNull(db.txoDao().getByOutpoint(makeOutpoint(changeTxid, 13))) @@ -8414,11 +8419,11 @@ class PlatformWalletPersistenceHandlerTest { @Test fun theRunStopsWhenTheHealCannotReadTheStore() = runTest { - // The heal asks the store whether it already holds each engine - // coin. A read that fails there must not read as "absent" (that - // would insert a row the store may well hold): the step fails, it - // rolls back, and the run stops. - val faults = useFaultedDatabase("SELECT * FROM txos WHERE outpoint") + // The heal asks the store, once per page, which engine coins it + // already holds. A read that fails there must not read as "absent" + // (that would insert a row the store may well hold): the step fails, + // it rolls back, and the run stops. + val faults = useFaultedDatabase("SELECT outpoint FROM txos WHERE outpoint IN") seedReconcileWallet() val txid = ByteArray(32) { 0x84.toByte() } faults.armed = true @@ -8439,11 +8444,11 @@ class PlatformWalletPersistenceHandlerTest { } @Test - fun theRunStopsWhenTheFlipLookupCannotRead() = runTest { - // The classify pass re-reads each known-uncredited row before it - // flips it. A read that fails there must not read as "stale": the - // step fails, nothing staged is kept, and the row stays as it was. - val faults = useFaultedDatabase("SELECT * FROM txos WHERE outpoint") + fun theRunStopsWhenTheFlipCannotWrite() = runTest { + // The classify pass marks each known-uncredited row spent. A write + // that fails there must not read as "stale": the step fails, nothing + // staged is kept, and the row stays as it was. + val faults = useFaultedDatabase("UPDATE txos SET isSpent = 1") seedReconcileWallet() val txid = ByteArray(32) { 0x85.toByte() } seedUnspentTxo(txid) @@ -8456,7 +8461,7 @@ class PlatformWalletPersistenceHandlerTest { assertEquals(1, report.storeFailures) assertEquals("the page itself was read", 1, report.storeRows) assertEquals(0, report.flipped) - assertEquals("a failed read is not a stale page either", 0, report.staleRetries) + assertEquals("a failed write is not a stale page either", 0, report.staleRetries) assertFalse(db.txoDao().getByOutpoint(makeOutpoint(txid, 0))!!.isSpent) val failure = org.robolectric.shadows.ShadowLog.getLogs() .map { it.msg } @@ -8488,12 +8493,11 @@ class PlatformWalletPersistenceHandlerTest { val report = handler.reconcileWith( engine, - // The run checks once for the account read, then before each - // engine read, before each store step and before each applied - // report: the first four admit the account read, page one, its - // heal and its report; the fifth (before page two) reports the - // epoch bumped. - isCancelled = { ++checks >= 5 }, + // The run checks before each engine read, before each store step + // and before each applied report: the first three admit page + // one, its heal and its report; the fourth (before page two) + // reports the epoch bumped. + isCancelled = { ++checks >= 4 }, ) assertFalse(report.completed) @@ -8966,8 +8970,13 @@ class PlatformWalletPersistenceHandlerTest { .parseToJsonElement(inventoryJson).jsonObject fun keys(list: String) = root[list]?.jsonArray.orEmpty().map { it.jsonObject } .map { key(it["txid"]!!.jsonPrimitive.content, it["vout"]!!.jsonPrimitive.int) }.toSet() - utxos = root["utxos"]?.jsonArray?.map { it.jsonObject } ?: emptyList() - unspentKeys = keys("utxos") + val inventory = root["utxos"]?.jsonArray?.map { it.jsonObject } ?: emptyList() + // A contact's watch-only coin never enters a heal page + // (`wallet_utxos_page`). + utxos = inventory.filter { !isContact(it) } + // Nor does a coin the engine holds only in a contact's chain + // answer `unspent` for this wallet's row. + unspentKeys = utxos.map { key(it["txid"]!!.jsonPrimitive.content, it["vout"]!!.jsonPrimitive.int) }.toSet() spentKeys = keys("spent") notOwnedKeys = keys("notOwned") } @@ -9058,6 +9067,9 @@ class PlatformWalletPersistenceHandlerTest { requireNotNull(q["scriptHex"]) { "classification query carries no script" } val k = key(q["txid"]!!.jsonPrimitive.content, q["vout"]!!.jsonPrimitive.int) verdicts[i] = when { + // A contact's watch-only chain gets no verdict at all + // (`classify_outpoints`). + isContact(q) -> 0 k in notOwnedKeys -> 3 k in unspentKeys -> 1 k in spentKeys -> 2 @@ -9083,7 +9095,7 @@ class PlatformWalletPersistenceHandlerTest { return pagesLeft > 0 } - override suspend fun releaseVerdicts() { + override fun releaseVerdicts() { if (held) events += "release" held = false } @@ -9095,6 +9107,10 @@ class PlatformWalletPersistenceHandlerTest { private fun key(txidHex: String, vout: Int) = "$txidHex:$vout" + /** A row or query of a DashPay external (contact) account. */ + private fun isContact(row: kotlinx.serialization.json.JsonObject) = + row["typeTag"]?.jsonPrimitive?.int == 13 + companion object { /** The rs-unified-sdk-jni `OutpointQuery` keys with no serde * default (its flattened `UtxoAccountTuple` plus the outpoint): @@ -9382,7 +9398,9 @@ class PlatformWalletPersistenceHandlerTest { /** * Room open-helper factory that fails exactly one prepared statement — * the one whose SQL starts with [failingSqlPrefix] — once [armed], and - * delegates every other read and write to real SQLite. + * delegates every other read and write to real SQLite. A write fails when + * it is compiled, so it must not have been compiled (and cached by Room) + * before the injector is armed. * * Faults ONE query rather than dropping its table: `asset_locks` is read * by three independent restore builders, so a table-level fault proves @@ -9425,5 +9443,9 @@ private class SingleStatementFaultInjector( ): Cursor = if (shouldFail(query)) throw SQLiteException("injected read failure: ${query.sql}") else delegate.query(query, cancellationSignal) + + override fun compileStatement(sql: String): SupportSQLiteStatement = + if (armed && sql.startsWith(failingSqlPrefix)) throw SQLiteException("injected write failure: $sql") + else delegate.compileStatement(sql) } } diff --git a/packages/rs-unified-sdk-jni/src/persistence.rs b/packages/rs-unified-sdk-jni/src/persistence.rs index b095e1e4f4c..6fb2354e45d 100644 --- a/packages/rs-unified-sdk-jni/src/persistence.rs +++ b/packages/rs-unified-sdk-jni/src/persistence.rs @@ -4925,7 +4925,7 @@ unsafe extern "C" fn tramp_get_core_tx_record_free( /// /// # Safety /// `ptr` must point to `count` valid `T`s (or be null with `count == 0`). -unsafe fn slice_or_empty<'a, T>(ptr: *const T, count: usize) -> &'a [T] { +pub(crate) unsafe fn slice_or_empty<'a, T>(ptr: *const T, count: usize) -> &'a [T] { if ptr.is_null() || count == 0 { &[] } else { diff --git a/packages/rs-unified-sdk-jni/src/wallet_manager.rs b/packages/rs-unified-sdk-jni/src/wallet_manager.rs index 263c0481009..1ebe367f7a3 100644 --- a/packages/rs-unified-sdk-jni/src/wallet_manager.rs +++ b/packages/rs-unified-sdk-jni/src/wallet_manager.rs @@ -42,7 +42,7 @@ use crate::events::{build_event_extension, build_event_vtable, KotlinEventCtx}; use crate::persistence::{ - build_extension, build_vtable, restrict_account_writers, KotlinPersistenceCtx, + build_extension, build_vtable, restrict_account_writers, slice_or_empty, KotlinPersistenceCtx, }; use crate::support::{guard, take_pwffi_error, throw_pwffi_result, throw_sdk_exception, JVM}; use jni::objects::{GlobalRef, JByteArray, JClass, JObject, JObjectArray, JString, JValue}; @@ -3669,39 +3669,6 @@ impl UtxoAccountTuple { friend_identity_id: identity(&e.friend_identity_id), } } - - /// The two 32-byte identity halves, or `None` when a hex string the - /// host handed back is malformed — a tuple this layer did not emit. - fn identity_ids(&self) -> Option<([u8; 32], [u8; 32])> { - fn id32(hex_id: &Option) -> Option<[u8; 32]> { - match hex_id { - None => Some([0u8; 32]), - Some(h) => hex32(h), - } - } - Some(( - id32(&self.user_identity_id)?, - id32(&self.friend_identity_id)?, - )) - } - - /// The two identity halves and the FFI outpoint of a row keyed by this - /// tuple and `txid`/`vout`, or the error naming the malformed field. - fn keyed_outpoint( - &self, - txid: &str, - vout: u32, - ) -> Result<([u8; 32], [u8; 32], platform_wallet_ffi::OutPointFFI), &'static str> { - let (user_identity_id, friend_identity_id) = self - .identity_ids() - .ok_or("outpoint query carries a malformed identity id")?; - let txid = hex32(txid).ok_or("outpoint query carries a malformed txid")?; - Ok(( - user_identity_id, - friend_identity_id, - platform_wallet_ffi::OutPointFFI { txid, vout }, - )) - } } /// 32 bytes from their hex form, or `None` when it is not exactly that. @@ -3746,11 +3713,7 @@ impl UtxoPageRow { /// bytes — the heal page's contract for every row it returns. unsafe fn from_heal_row(r: &platform_wallet_ffi::TxoHealRowFFI) -> Self { let e = &r.utxo; - let script: &[u8] = if e.script_pubkey.is_null() || e.script_pubkey_len == 0 { - &[] - } else { - std::slice::from_raw_parts(e.script_pubkey, e.script_pubkey_len) - }; + let script = slice_or_empty(e.script_pubkey, e.script_pubkey_len); // Never null per the accessor's contract; treated as empty if it // ever is (the engine's verdict on such a row is `invalid`). let address = cstr_opt(e.address).unwrap_or_default(); @@ -3784,12 +3747,12 @@ struct UtxoPage { } impl UtxoPage { - /// The page for `utxos`. A page with no rows reports no more, whatever - /// the accessor said: there is nothing after it to read. + /// The page for `utxos`. `has_more` is the engine's, which is already + /// false on an empty page. fn new(utxos: Vec, has_more: bool) -> Self { UtxoPage { - has_more: has_more && !utxos.is_empty(), utxos, + has_more, budget_spent: false, } } @@ -3828,8 +3791,18 @@ impl OutpointQuery { fn to_ffi( &self, ) -> Result<(platform_wallet_ffi::OutpointOwnershipQueryFFI, Vec), &'static str> { - let (user_identity_id, friend_identity_id, outpoint) = - self.account.keyed_outpoint(&self.txid, self.vout)?; + // An absent identity half is all-zero; a present one this layer did + // not emit (malformed hex) fails the batch. + let identity = |hex_id: &Option| match hex_id { + None => Some([0u8; 32]), + Some(h) => hex32(h), + }; + const MALFORMED_IDENTITY: &str = "outpoint query carries a malformed identity id"; + let user_identity_id = + identity(&self.account.user_identity_id).ok_or(MALFORMED_IDENTITY)?; + let friend_identity_id = + identity(&self.account.friend_identity_id).ok_or(MALFORMED_IDENTITY)?; + let txid = hex32(&self.txid).ok_or("outpoint query carries a malformed txid")?; let script = hex::decode(&self.script_hex) .map_err(|_| "outpoint query carries a malformed script")?; let query = platform_wallet_ffi::OutpointOwnershipQueryFFI { @@ -3840,7 +3813,10 @@ impl OutpointQuery { key_class: self.account.key_class, user_identity_id, friend_identity_id, - outpoint, + outpoint: platform_wallet_ffi::OutPointFFI { + txid, + vout: self.vout, + }, script_pubkey: ptr::null(), script_pubkey_len: 0, }; @@ -3881,9 +3857,9 @@ pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_w /// Claim one wallet for a TXO-store reconcile run and check that it may /// start. Returns `TXO_RECONCILE_GATE_RUN` (0) or a `TXO_RECONCILE_SKIP_*` -/// reason; on 0, `outTipHeight[0]` is the scan tip the run is measured -/// against (the claim keeps it; the host only reports it), and the claim -/// holds until `walletManagerTxoReconcileFinish`. Bridges +/// reason; on 0 the claim holds until `walletManagerTxoReconcileFinish`. +/// The claim keeps the scan tip the run is measured against; the host +/// never sees it. Bridges /// `platform_wallet_manager_txo_reconcile_begin`, where every gate lives. /// /// On 0 `outStart[0]` is where the run starts, a `TXO_RECONCILE_START_*` @@ -3900,7 +3876,6 @@ pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_w _class: JClass, manager_handle: jlong, wallet_id: JByteArray, - out_tip_height: jni::sys::jintArray, out_start: jni::sys::jintArray, out_store_after: jni::sys::jobjectArray, ) -> jint { @@ -3908,27 +3883,23 @@ pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_w let Some(wid) = read_id32(env, &wallet_id) else { return -1; }; - const OUTS: &str = - "outTipHeight and outStart must be IntArray(1), outStoreAfter an Array(1)"; - if out_tip_height.is_null() || out_start.is_null() || out_store_after.is_null() { + const OUTS: &str = "outStart must be an IntArray(1), outStoreAfter an Array(1)"; + if out_start.is_null() || out_store_after.is_null() { throw_sdk_exception(env, 1, OUTS); return -1; } // SAFETY: non-null arrays from the Kotlin caller. - let out_tip = unsafe { jni::objects::JIntArray::from_raw(out_tip_height) }; let out_start = unsafe { jni::objects::JIntArray::from_raw(out_start) }; let out_store_after = unsafe { JObjectArray::from_raw(out_store_after) }; // Checked before the claim is taken: a wrong-sized array would // otherwise fail a write below with the wallet already claimed. - if !has_out_slot(env.get_array_length(&out_tip)) - || !has_out_slot(env.get_array_length(&out_start)) + if !has_out_slot(env.get_array_length(&out_start)) || !has_out_slot(env.get_array_length(&out_store_after)) { throw_sdk_exception(env, 1, OUTS); return -1; } let mut gate = 0u8; - let mut tip_height = 0u32; let mut start = platform_wallet_ffi::TXO_RECONCILE_START_HEAL; let mut store_after = [0u8; platform_wallet_ffi::TXO_RECONCILE_STORE_CURSOR_MAX]; let mut store_after_len = 0usize; @@ -3937,7 +3908,6 @@ pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_w manager_handle as Handle, wid.as_ptr(), &mut gate as *mut u8, - &mut tip_height as *mut u32, &mut start as *mut u8, store_after.as_mut_ptr(), &mut store_after_len as *mut usize, @@ -3963,12 +3933,9 @@ pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_w false, ); }); - // A tip past i32::MAX cannot cross as a jint; no Dash height is near it. - let tip = jint::try_from(tip_height).unwrap_or(jint::MAX); - if env.set_int_array_region(&out_tip, 0, &[tip]).is_err() - || env - .set_int_array_region(&out_start, 0, &[jint::from(start)]) - .is_err() + if env + .set_int_array_region(&out_start, 0, &[jint::from(start)]) + .is_err() { return -1; } @@ -4169,28 +4136,24 @@ pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_w }; // Every exit before the hand-over frees the guard — see // `walletManagerTxoReconcileClassifyOutpoints`. - let held = UnsentHeldVerdicts::new( - held, - platform_wallet_ffi::platform_wallet_txo_reconcile_held_verdicts_free, - ); + let unsent = until_held_handed_over(held); if take_pwffi_error(env, result) { return ptr::null_mut(); } - let mut rows: Vec = Vec::with_capacity(count); - if !entries.is_null() && count > 0 { - let items = unsafe { std::slice::from_raw_parts(entries, count) }; - for r in items { - // SAFETY: every row the heal page returns satisfies the - // pointer contract `from_heal_row` documents. - rows.push(unsafe { UtxoPageRow::from_heal_row(r) }); - } - unsafe { - platform_wallet_ffi::platform_wallet_txo_reconcile_heal_page_free( - entries as *mut platform_wallet_ffi::TxoHealRowFFI, - count, - ) - }; - } + // SAFETY: the heal page returned `count` rows at `entries` (or + // null/0), each satisfying the pointer contract `from_heal_row` + // documents; they are read before the page is freed. + let rows: Vec = unsafe { slice_or_empty(entries, count) } + .iter() + .map(|r| unsafe { UtxoPageRow::from_heal_row(r) }) + .collect(); + // A null or empty page is a no-op to free. + unsafe { + platform_wallet_ffi::platform_wallet_txo_reconcile_heal_page_free( + entries as *mut platform_wallet_ffi::TxoHealRowFFI, + count, + ) + }; let page = if budget_spent { UtxoPage::budget_spent() } else { @@ -4206,7 +4169,7 @@ pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_w let Ok(page) = env.new_string(json).map(|s| s.into_raw()) else { return ptr::null_mut(); }; - if !held.send(env, &out) { + if !send_held(env, &out, held, unsent) { return ptr::null_mut(); } page @@ -4338,10 +4301,7 @@ pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_w }; // From here every exit — an error, a failed write, a panic that // `guard` catches — frees the guard, until Kotlin owns it. - let held = UnsentHeldVerdicts::new( - held, - platform_wallet_ffi::platform_wallet_txo_reconcile_held_verdicts_free, - ); + let unsent = until_held_handed_over(held); // `scripts` must outlive the call above; this keeps that explicit // against a future reorder. drop(scripts); @@ -4358,7 +4318,7 @@ pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_w let Ok(answers) = env.byte_array_from_slice(&classes).map(|a| a.into_raw()) else { return ptr::null_mut(); }; - if !held.send(env, &out) { + if !send_held(env, &out, held, unsent) { return ptr::null_mut(); } answers @@ -4389,10 +4349,7 @@ pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_w &mut may_continue, ) }; - if take_pwffi_error(env, result) || !may_continue { - return JNI_FALSE; - } - JNI_TRUE + page_applied_answer(env, result, may_continue) }) } @@ -4434,13 +4391,24 @@ pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_w &mut may_continue, ) }; - if take_pwffi_error(env, result) || !may_continue { - return JNI_FALSE; - } - JNI_TRUE + page_applied_answer(env, result, may_continue) }) } +/// The answer a `…PageApplied` export returns: whether the run has budget +/// left for another page, false (with the SDK exception pending) when the +/// report failed. +fn page_applied_answer( + env: &mut JNIEnv, + result: PlatformWalletFFIResult, + may_continue: bool, +) -> jboolean { + if take_pwffi_error(env, result) || !may_continue { + return JNI_FALSE; + } + JNI_TRUE +} + /// Validate `outHeld` as a `LongArray(1)` and write "nothing held" to it, /// before any other check of a held read can return, so a reused array /// never carries a token from an earlier call into this one, whichever way @@ -4465,59 +4433,39 @@ fn clear_out_held<'local>( Some(out) } -/// A held-verdicts guard `walletManagerTxoHealPageJson` or +/// The guard for a held-verdicts token `walletManagerTxoHealPageJson` or /// `walletManagerTxoReconcileClassifyOutpoints` has not handed to Kotlin -/// yet. Dropped before [`Self::hand_over`] — an early return, or a panic -/// unwinding into `guard` — it frees the guard at once. Left to -/// `walletManagerTxoReconcileFinish` instead, it would be released only by -/// a call that resolves the manager handle in the native registry while -/// the guard is still held, the registry/engine lock cycle the token exists -/// to avoid. -struct UnsentHeldVerdicts { - held: *mut T, - free: unsafe extern "C" fn(*mut T), -} - -impl UnsentHeldVerdicts { - fn new(held: *mut T, free: unsafe extern "C" fn(*mut T)) -> Self { - Self { held, free } - } - - fn as_ptr(&self) -> *mut T { - self.held - } - - /// Kotlin owns the guard now and frees it with - /// `txoReconcileHeldVerdictsFree`. - fn hand_over(mut self) { - self.held = ptr::null_mut(); - } -} - -impl UnsentHeldVerdicts { - /// Write the guard to `outHeld[0]`, last, once the call's answer is - /// ready to return, and hand it to Kotlin. `false` — the guard freed - /// here — when the write failed. - fn send(self, env: &mut JNIEnv, out: &jni::objects::JLongArray) -> bool { - if env - .set_long_array_region(out, 0, &[self.as_ptr() as jlong]) - .is_err() - { - return false; - } - self.hand_over(); - true - } +/// yet: dropped before the hand-over — an early return, or a panic +/// unwinding into `guard` — it frees the token at once (null frees +/// nothing). Left to `walletManagerTxoReconcileFinish` instead, it would be +/// released only by a call that resolves the manager handle in the native +/// registry while the guard is still held, the registry/engine lock cycle +/// the token exists to avoid. +fn until_held_handed_over( + held: *mut platform_wallet_ffi::TxoReconcileHeldVerdictsFFI, +) -> UntilHandedOver { + // SAFETY: `held` came from the native held read and is freed here only + // if it was never handed over, so nothing else frees it. + UntilHandedOver::new(move || unsafe { + platform_wallet_ffi::platform_wallet_txo_reconcile_held_verdicts_free(held) + }) } -impl Drop for UnsentHeldVerdicts { - fn drop(&mut self) { - if !self.held.is_null() { - // SAFETY: `held` came from the native held read and was never - // handed over, so nothing else frees it. - unsafe { (self.free)(self.held) }; - } +/// Write `held` to `outHeld[0]`, last, once the call's answer is ready to +/// return, and hand it to Kotlin, which frees it with +/// `txoReconcileHeldVerdictsFree`. `false` — the token freed here, by +/// `unsent` — when the write failed. +fn send_held( + env: &mut JNIEnv, + out: &jni::objects::JLongArray, + held: *mut platform_wallet_ffi::TxoReconcileHeldVerdictsFFI, + unsent: UntilHandedOver, +) -> bool { + if env.set_long_array_region(out, 0, &[held as jlong]).is_err() { + return false; } + unsent.hand_over(); + true } /// Read a 32-byte id from a Java `byte[]`; throws + returns None on the @@ -5543,16 +5491,12 @@ mod tests { } #[test] - fn a_last_or_empty_page_reports_no_more_and_carries_no_cursor() { + fn a_last_page_reports_no_more_and_carries_no_cursor() { let last = UtxoPage::new(rows(&golden_coins()), false); let value = serde_json::to_value(&last).expect("page serializes"); assert_eq!(value["hasMore"], serde_json::Value::Bool(false)); assert_eq!(value["budgetSpent"], serde_json::Value::Bool(false)); assert!(value.get("cursor").is_none(), "the run keeps its own place"); - - // An empty page has nothing after it, whatever the accessor said. - let empty = UtxoPage::new(Vec::new(), true); - assert!(!empty.has_more); } #[test] @@ -5713,8 +5657,10 @@ mod tests { use std::cell::Cell; use std::sync::atomic::{AtomicUsize, Ordering}; - use super::super::{has_out_slot, UntilHandedOver}; + use super::super::{has_out_slot, until_held_handed_over, UntilHandedOver}; + /// A begin claim or a held-verdicts token (`until_held_handed_over`) + /// not handed to Kotlin is released once. #[test] fn an_unreturned_claim_is_released_once_on_drop() { let released = Cell::new(0); @@ -5722,6 +5668,13 @@ mod tests { assert_eq!(released.get(), 1); } + #[test] + fn an_unsent_null_held_token_frees_nothing() { + // The native free is a no-op on null, which is what an errored + // held read leaves behind. + drop(until_held_handed_over(std::ptr::null_mut())); + } + #[test] fn a_claim_handed_to_kotlin_is_not_released() { let released = Cell::new(0); @@ -5736,7 +5689,7 @@ mod tests { let _claim = UntilHandedOver::new(|| { released.fetch_add(1, Ordering::SeqCst); }); - panic!("writing outTipHeight failed"); + panic!("writing outStart failed"); }); assert!(unwound.is_err()); assert_eq!( @@ -5757,62 +5710,4 @@ mod tests { assert!(!has_out_slot(Err(jni::errors::Error::NullPtr("array")))); } } - - mod unsent_held_verdicts { - use std::ptr::NonNull; - use std::sync::atomic::{AtomicUsize, Ordering}; - - use super::super::UnsentHeldVerdicts; - - /// A stand-in guard: the frees below only count, never dereference. - fn token() -> *mut u8 { - NonNull::::dangling().as_ptr() - } - - #[test] - fn an_unsent_guard_is_freed_once_on_drop() { - static FREED: AtomicUsize = AtomicUsize::new(0); - unsafe extern "C" fn free(_: *mut u8) { - FREED.fetch_add(1, Ordering::SeqCst); - } - drop(UnsentHeldVerdicts::new(token(), free)); - assert_eq!(FREED.load(Ordering::SeqCst), 1); - } - - #[test] - fn a_guard_handed_to_kotlin_is_not_freed() { - static FREED: AtomicUsize = AtomicUsize::new(0); - unsafe extern "C" fn free(_: *mut u8) { - FREED.fetch_add(1, Ordering::SeqCst); - } - let held = UnsentHeldVerdicts::new(token(), free); - assert_eq!(held.as_ptr(), token()); - held.hand_over(); - assert_eq!(FREED.load(Ordering::SeqCst), 0); - } - - #[test] - fn a_panic_before_the_hand_over_frees_the_guard() { - static FREED: AtomicUsize = AtomicUsize::new(0); - unsafe extern "C" fn free(_: *mut u8) { - FREED.fetch_add(1, Ordering::SeqCst); - } - let unwound = std::panic::catch_unwind(|| { - let _held = UnsentHeldVerdicts::new(token(), free); - panic!("writing outHeld failed"); - }); - assert!(unwound.is_err()); - assert_eq!(FREED.load(Ordering::SeqCst), 1, "freed while unwinding"); - } - - #[test] - fn nothing_held_is_nothing_freed() { - static FREED: AtomicUsize = AtomicUsize::new(0); - unsafe extern "C" fn free(_: *mut u8) { - FREED.fetch_add(1, Ordering::SeqCst); - } - drop(UnsentHeldVerdicts::new(std::ptr::null_mut(), free)); - assert_eq!(FREED.load(Ordering::SeqCst), 0); - } - } } From 7dc01996f87333ea24ee8d92c24d3f36c7080d75 Mon Sep 17 00:00:00 2001 From: HashEngineering Date: Thu, 5 Nov 2026 20:22:12 -0800 Subject: [PATCH 4/6] test(kotlin-sdk): deliver credit verdicts through the real JNI and check the restore set Add a test-only JNI entry, nativeTestDeliverUtxoCreditVerdictRound, that builds a round's CoreChangeSet (records plus utxo_credit_verdicts) in Rust and stores it through an FFIPersister wired to the Kotlin bridge the way nativeCreate wires the manager's. The production store path then runs unchanged: the verdict slot, the JNI verdict and changeset trampolines, and the Kotlin handler writing Room. Only the engine that produces the verdicts is skipped; the Rust producer tests cover it. WalletManagerRoundTripTest delivers four coins of which only one is credited (observed-spent, doomed, and an uncredited re-delivery of a held coin) and checks the Room rows, the restore set onLoadWalletList builds, and the balance of a freshly loaded manager. A dropped, mispacked or shifted verdict changes that result; two control tests show the dropped and shifted cases. The entry is ungated, like nativeVerifyPersistenceBridgeDescriptors, and is reachable only through the internal WalletManagerNative object. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 2 + .../wallet/WalletManagerRoundTripTest.kt | 217 ++++++++++ .../dashsdk/ffi/WalletManagerNative.kt | 25 ++ packages/rs-unified-sdk-jni/Cargo.toml | 7 +- packages/rs-unified-sdk-jni/src/lib.rs | 1 + .../src/verdict_round_test_hook.rs | 386 ++++++++++++++++++ 6 files changed, 637 insertions(+), 1 deletion(-) create mode 100644 packages/rs-unified-sdk-jni/src/verdict_round_test_hook.rs diff --git a/Cargo.lock b/Cargo.lock index 7fa50b4e1ba..21d761f429f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6691,7 +6691,9 @@ dependencies = [ "dpp", "hex", "jni 0.21.1", + "key-wallet", "log", + "platform-wallet", "platform-wallet-ffi", "rs-sdk-ffi", "serde", diff --git a/packages/kotlin-sdk/sdk/src/androidTest/kotlin/org/dashfoundation/dashsdk/wallet/WalletManagerRoundTripTest.kt b/packages/kotlin-sdk/sdk/src/androidTest/kotlin/org/dashfoundation/dashsdk/wallet/WalletManagerRoundTripTest.kt index 77154af19e8..864ea66eda1 100644 --- a/packages/kotlin-sdk/sdk/src/androidTest/kotlin/org/dashfoundation/dashsdk/wallet/WalletManagerRoundTripTest.kt +++ b/packages/kotlin-sdk/sdk/src/androidTest/kotlin/org/dashfoundation/dashsdk/wallet/WalletManagerRoundTripTest.kt @@ -7,7 +7,9 @@ import kotlinx.coroutines.runBlocking import org.dashfoundation.dashsdk.Network import org.dashfoundation.dashsdk.Sdk import org.dashfoundation.dashsdk.config.SdkConfig +import org.dashfoundation.dashsdk.ffi.WalletManagerNative import org.dashfoundation.dashsdk.persistence.DashDatabase +import org.dashfoundation.dashsdk.persistence.PlatformWalletPersistenceHandler import org.dashfoundation.dashsdk.security.WalletStorage import org.junit.After import org.junit.Assert.assertEquals @@ -259,4 +261,219 @@ class WalletManagerRoundTripTest { assertEquals(1, state.getInt("ignoredSenders")) } } + + /** + * Credit verdicts through REAL JNI (dashpay/platform#4638), then reopen. + * + * The JVM `PlatformWalletPersistenceHandlerTest` restart tests hand the + * handler its verdicts directly, so they stay green if the native side + * drops or mis-packs one. This round goes through + * `nativeTestDeliverUtxoCreditVerdictRound`: Rust builds the verdicts + * out of a `CoreChangeSet`, `FFIPersister::store` fires the verdict slot + * before the changeset, the JNI trampolines pack the 36-byte outpoints, + * codes and heights and the round's `utxos_added`, and the Kotlin + * handler stages and commits the rows. A fresh manager then restores + * from Room, and the engine's balance must count the credited coin + * only. + * + * Fixture, all paying the wallet's own BIP44 receive addresses: + * - tx 1 vout 0, [CREDITED]: no verdict, so it is restored. + * - tx 1 vout 1, [BORN_SPENT]: observed spent, so its row is born spent. + * - tx 1 vout 2, [DOOMED]: doomed, so its row is born spent. + * - tx 2 vout 0, [HELD]: stored earlier and then marked spent with no + * spender link (a sweep hold). Re-delivered with an uncredited + * verdict, it must stay spent. Without the verdict, the re-delivery + * would clear the row and the coin would come back. + * + * What a native fault does here: a dropped verdict (any of the three) + * restores that coin. A mis-packed outpoint (txid bytes or vout + * encoding) matches no row, so all three come back. Verdicts shifted + * by one output put a verdict on the credited coin. A code moved onto + * the wrong entry puts the context-free uncredited code on a fresh + * row, which is then stored unspent. A count or length mismatch fails + * the round, and the hook throws. Each case changes the asserted + * restore set; [droppedVerdictsRestoreThePhantomCoins] and + * [verdictsShiftedByOneOutputChangeTheRestoreSet] check that it does. + * Not covered: swapping the two born-spent codes, or the observed-spent + * height, which the Kotlin handler only checks for length. + */ + @Test + fun creditVerdictRoundThroughJniRestoresOnlyCreditedCoins() = runBlocking { + val outcome = deliverVerdictScenario( + verdicts = byteArrayOf( + CODE_CREDITED, + PlatformWalletPersistenceHandler.UTXO_CREDIT_VERDICT_OBSERVED_SPENT, + PlatformWalletPersistenceHandler.UTXO_CREDIT_VERDICT_DOOMED, + PlatformWalletPersistenceHandler.UTXO_CREDIT_VERDICT_UNCREDITED, + ), + ) + + // Room, as the handler committed it from the JNI-delivered round. + assertEquals( + "every fixture output has a row", + setOf(CREDITED, BORN_SPENT, DOOMED, HELD), + outcome.spentByAmount.keys, + ) + assertEquals(false, outcome.spentByAmount[CREDITED]) + assertEquals("observed-spent verdict → born spent", true, outcome.spentByAmount[BORN_SPENT]) + assertEquals("doomed verdict → born spent", true, outcome.spentByAmount[DOOMED]) + assertEquals("uncredited verdict vetoes the clear", true, outcome.spentByAmount[HELD]) + assertTrue("a verdict never mints a spender link", outcome.unlinked) + + // The restore set the load callback hands Rust, and what the engine + // actually holds after a fresh manager loaded it. + assertEquals(listOf(CREDITED), outcome.restoredAmounts) + assertEquals(CREDITED, outcome.reloadedEngineBalance) + } + + /** + * Control for [creditVerdictRoundThroughJniRestoresOnlyCreditedCoins]: + * the same round with its verdicts dropped restores every coin. So that + * test's assertions do catch a lost verdict, and its fixture is not + * passing for an unrelated reason. + */ + @Test + fun droppedVerdictsRestoreThePhantomCoins() = runBlocking { + val dropped = deliverVerdictScenario( + verdicts = byteArrayOf(CODE_CREDITED, CODE_CREDITED, CODE_CREDITED, CODE_CREDITED), + ) + assertEquals( + listOf(CREDITED, BORN_SPENT, DOOMED, HELD), + dropped.restoredAmounts, + ) + assertEquals(CREDITED + BORN_SPENT + DOOMED + HELD, dropped.reloadedEngineBalance) + } + + /** + * Control for [creditVerdictRoundThroughJniRestoresOnlyCreditedCoins]: + * verdicts shifted by one output (a vout mis-pack) give a different + * restore set. + */ + @Test + fun verdictsShiftedByOneOutputChangeTheRestoreSet() = runBlocking { + val shifted = deliverVerdictScenario( + verdicts = byteArrayOf( + PlatformWalletPersistenceHandler.UTXO_CREDIT_VERDICT_OBSERVED_SPENT, + PlatformWalletPersistenceHandler.UTXO_CREDIT_VERDICT_DOOMED, + PlatformWalletPersistenceHandler.UTXO_CREDIT_VERDICT_UNCREDITED, + CODE_CREDITED, + ), + ) + // The credited coin is born spent, the uncredited code leaves a + // fresh row unspent, and the held coin is cleared and comes back. + assertEquals(listOf(DOOMED, HELD), shifted.restoredAmounts) + assertEquals(DOOMED + HELD, shifted.reloadedEngineBalance) + } + + private class VerdictOutcome( + val spentByAmount: Map, + val unlinked: Boolean, + val restoredAmounts: List, + val reloadedEngineBalance: Long, + ) + + /** + * Create the wallet, store [HELD] in a verdict-free round and mark it + * spent with no link, deliver the four-output round with [verdicts] + * (one per output, in the fixture order of the test KDoc) through the + * JNI hook, then read the result back from Room, from the restore + * callback and from a freshly loaded manager. + */ + private suspend fun deliverVerdictScenario(verdicts: ByteArray): VerdictOutcome { + val walletId: ByteArray + val receive: List + PlatformWalletManager(sdk, Network.TESTNET, db, walletStorage).use { manager -> + walletId = manager.createWallet( + mnemonic = testMnemonic, + name = "verdict-round-trip", + createDefaultAccounts = true, + ).walletId + val bip44 = db.accountDao().observeByWallet(walletId).first().single { + it.accountType == 0 && it.standardTag == 0 && it.accountIndex == 0 + } + receive = db.coreAddressDao().observeByAccount(bip44.id).first() + .filter { it.poolTypeTag == 0 } + .sortedBy { it.addressIndex } + .map { it.address } + .take(4) + assertEquals("four BIP44 receive addresses persisted", 4, receive.size) + } + + PlatformWalletPersistenceHandler(db, network = Network.TESTNET).use { handler -> + // Round 1: tx 2 alone, credited, so the HELD row exists. + WalletManagerNative.nativeTestDeliverUtxoCreditVerdictRound( + handler, + walletId, + txTags = intArrayOf(2), + addresses = arrayOf(receive[3]), + amounts = longArrayOf(HELD), + verdicts = byteArrayOf(CODE_CREDITED), + spentAtHeights = intArrayOf(0), + blockHeight = BLOCK_HEIGHT, + ) + val held = db.txoDao().observeByWallet(walletId).first().single { it.amount == HELD } + assertEquals(false, held.isSpent) + // A sweep hold: spent, no spender link. + assertEquals(1, db.txoDao().markSpentByOutpoint(held.outpoint, java.util.Date())) + + // Round 2: tx 1 (three outputs) plus tx 2 again, with the verdicts. + WalletManagerNative.nativeTestDeliverUtxoCreditVerdictRound( + handler, + walletId, + txTags = intArrayOf(1, 1, 1, 2), + addresses = arrayOf(receive[0], receive[1], receive[2], receive[3]), + amounts = longArrayOf(CREDITED, BORN_SPENT, DOOMED, HELD), + verdicts = verdicts, + spentAtHeights = IntArray(verdicts.size) { i -> + if (verdicts[i] == + PlatformWalletPersistenceHandler.UTXO_CREDIT_VERDICT_OBSERVED_SPENT + ) { + BLOCK_HEIGHT - 1 + } else { + 0 + } + }, + blockHeight = BLOCK_HEIGHT, + ) + } + + val rows = db.txoDao().observeByWallet(walletId).first() + val restored = PlatformWalletPersistenceHandler(db, network = Network.TESTNET).use { handler -> + handler.onLoadWalletList() + .single { it.walletId.contentEquals(walletId) } + .utxos + .map { it.valueDuffs } + .sorted() + } + val balance = PlatformWalletManager(sdk, Network.TESTNET, db, walletStorage).use { reloaded -> + reloaded.loadPersistedWallets() + assertNotNull(reloaded.wallet(forWalletId = walletId)) + val balances = org.json.JSONArray(requireNotNull(reloaded.accountBalances(walletId))) + (0 until balances.length()).sumOf { i -> + val account = balances.getJSONObject(i) + account.getLong("confirmed") + account.getLong("unconfirmed") + + account.getLong("immature") + account.getLong("locked") + } + } + return VerdictOutcome( + spentByAmount = rows.associate { it.amount to it.isSpent }, + unlinked = rows.all { it.spendingTxid == null }, + restoredAmounts = restored, + reloadedEngineBalance = balance, + ) + } + + private companion object { + /** No verdict: the engine credited the output. */ + const val CODE_CREDITED: Byte = 0 + + // Distinct amounts identify each fixture output in every read-back, + // and any subset of them sums to a distinct balance. + const val CREDITED = 100_000L + const val BORN_SPENT = 200_000L + const val DOOMED = 400_000L + const val HELD = 800_000L + + const val BLOCK_HEIGHT = 1_000 + } } diff --git a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/ffi/WalletManagerNative.kt b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/ffi/WalletManagerNative.kt index bde6af67013..d0641aa0e89 100644 --- a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/ffi/WalletManagerNative.kt +++ b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/ffi/WalletManagerNative.kt @@ -73,6 +73,31 @@ internal object WalletManagerNative { */ external fun nativeEffectivePersistenceCapabilityBits(bridge: NativePersistenceBridge): Long + /** + * TEST ONLY (`rs-unified-sdk-jni/src/verdict_round_test_hook.rs`): + * deliver one credit-verdict round to [bridge] through the real native + * persistence path — `FFIPersister::store`, the verdict packing and the + * JNI trampolines — as the SPV engine would after processing a + * transaction it did not credit, which otherwise needs a synced chain. + * + * Output `i` pays `amounts[i]` to `addresses[i]` in the fixture + * transaction `txTags[i]` (its vout is its position among the outputs + * sharing that tag; the same tag and outputs always give the same txid), + * confirmed at [blockHeight]. `verdicts[i]` is `0` for a credited output, + * else a `UTXO_CREDIT_VERDICT_*` code with `spentAtHeights[i]`. Throws + * [DashSDKException] when the fixture is malformed or the round fails. + */ + external fun nativeTestDeliverUtxoCreditVerdictRound( + bridge: NativePersistenceBridge, + walletId: ByteArray, + txTags: IntArray, + addresses: Array, + amounts: LongArray, + verdicts: ByteArray, + spentAtHeights: IntArray, + blockHeight: Int, + ) + // ── Wallet creation / restore ───────────────────────────────────── /** diff --git a/packages/rs-unified-sdk-jni/Cargo.toml b/packages/rs-unified-sdk-jni/Cargo.toml index 9b396969e5f..c11afb665ff 100644 --- a/packages/rs-unified-sdk-jni/Cargo.toml +++ b/packages/rs-unified-sdk-jni/Cargo.toml @@ -25,12 +25,17 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" hex = "0.4" zeroize = "1" +# The test-only credit-verdict round hook (`verdict_round_test_hook.rs`) +# builds a `CoreChangeSet` and drives it through `FFIPersister::store`. +# All three are already in the graph through platform-wallet-ffi. +dashcore = { workspace = true } +key-wallet = { workspace = true } +platform-wallet = { path = "../rs-platform-wallet" } [target.'cfg(target_os = "android")'.dependencies] android_logger = "0.14" [dev-dependencies] -dashcore = { workspace = true } # Anchors the cross-language golden-fixture test to the canonical DashPay # contract id, so the mirrored Kotlin constant can't drift undetected. dashpay-contract = { path = "../dashpay-contract" } diff --git a/packages/rs-unified-sdk-jni/src/lib.rs b/packages/rs-unified-sdk-jni/src/lib.rs index aaa43632342..4d091c26cb2 100644 --- a/packages/rs-unified-sdk-jni/src/lib.rs +++ b/packages/rs-unified-sdk-jni/src/lib.rs @@ -33,6 +33,7 @@ mod support; mod tokens; mod transactions; mod tx_decode; +mod verdict_round_test_hook; mod wallet_manager; use jni::sys::{jint, JNI_VERSION_1_6}; diff --git a/packages/rs-unified-sdk-jni/src/verdict_round_test_hook.rs b/packages/rs-unified-sdk-jni/src/verdict_round_test_hook.rs new file mode 100644 index 00000000000..ea0ae2d1f44 --- /dev/null +++ b/packages/rs-unified-sdk-jni/src/verdict_round_test_hook.rs @@ -0,0 +1,386 @@ +//! Test-only entry point: deliver one credit-verdict round to a Kotlin +//! `NativePersistenceBridge` through the REAL native persistence path. +//! +//! A verdict round is otherwise produced only by the SPV engine processing +//! a transaction it does not credit, which needs a synced chain. The +//! instrumented `WalletManagerRoundTripTest` instead builds the round's +//! `CoreChangeSet` here and hands it to an `FFIPersister` wired to the +//! bridge exactly as `nativeCreate` wires the manager's +//! ([`build_vtable`] + [`restrict_account_writers`] + [`build_extension`]), +//! so everything downstream of the engine runs for real: +//! `FFIPersister::store` (begin → verdict slot → changeset → end), +//! `build_utxo_credit_verdicts_for_callback`, the JNI trampolines that pack +//! the verdicts and the `utxos_added` entries, and the Kotlin handler that +//! stages and commits the rows. JVM/Robolectric tests reach none of that. +//! +//! Like `nativeVerifyPersistenceBridgeDescriptors`, it ships in the library +//! (the instrumented suite runs against the normal build) and is reachable +//! only through the `internal` Kotlin `WalletManagerNative` object. It +//! writes only into the bridge the caller passes, which already owns its +//! database, so it grants no access the caller lacks. + +use crate::persistence::{ + build_extension, build_vtable, restrict_account_writers, KotlinPersistenceCtx, +}; +use crate::support::{guard, throw_sdk_exception}; +use dashcore::{Address, BlockHash, OutPoint, ScriptBuf, Transaction, TxIn, TxOut, Txid, Witness}; +use jni::objects::{JByteArray, JClass, JIntArray, JLongArray, JObject, JObjectArray, JString}; +use jni::sys::jint; +use jni::JNIEnv; +use key_wallet::account::{AccountType, StandardAccountType}; +use key_wallet::managed_account::transaction_record::{ + OutputDetail, OutputRole, TransactionDirection, TransactionRecord, +}; +use key_wallet::transaction_checking::transaction_router::TransactionType; +use key_wallet::transaction_checking::{BlockInfo, TransactionContext}; +use platform_wallet::changeset::changeset::UtxoCreditVerdict; +use platform_wallet::changeset::{ + CoreChangeSet, PersistenceCapabilities, PlatformWalletChangeSet, PlatformWalletPersistence, +}; +use platform_wallet_ffi::{FFIPersister, PersistenceExtensionCallbacks}; +use std::ffi::c_void; +use std::str::FromStr; + +/// No verdict: the engine credited the output. +const CREDITED: i8 = 0; + +/// One fixture output: the transaction it belongs to (`tx_tag`), where it +/// pays, how much, and the engine's verdict on it (`CREDITED` or a +/// `UTXO_CREDIT_VERDICT_*` code). +#[derive(Debug, Clone)] +struct FixtureOutput { + tx_tag: u32, + address: Address, + amount: u64, + verdict: i8, + spent_at_height: u32, +} + +/// Build the round: one BIP44-account-0 `Received` record per distinct +/// `tx_tag` (outputs in input order, so an output's vout is its position +/// within its tag), confirmed at `block_height`, plus the verdict map keyed +/// by the real outpoints. A tag always yields the same txid for the same +/// outputs, so a later round can re-deliver a transaction an earlier round +/// stored. +fn build_round(outputs: &[FixtureOutput], block_height: u32) -> Result { + let mut tags: Vec = Vec::new(); + for output in outputs { + if !tags.contains(&output.tx_tag) { + tags.push(output.tx_tag); + } + } + let account = AccountType::Standard { + index: 0, + standard_account_type: StandardAccountType::BIP44Account, + }; + let context = TransactionContext::InBlock(BlockInfo::new( + block_height, + BlockHash::from_byte_array([0x42; 32]), + 1_700_000_000, + )); + let mut core = CoreChangeSet::default(); + for tag in tags { + let group: Vec<&FixtureOutput> = outputs.iter().filter(|o| o.tx_tag == tag).collect(); + let tx = Transaction { + version: 2, + lock_time: 0, + input: vec![TxIn { + // A foreign funding coin, distinct per tag. + previous_output: OutPoint { + txid: Txid::from_byte_array([(tag & 0xFF) as u8 ^ 0xA5; 32]), + vout: tag, + }, + script_sig: ScriptBuf::new(), + sequence: 0xffff_ffff, + witness: Witness::new(), + }], + output: group + .iter() + .map(|o| TxOut { + value: o.amount, + script_pubkey: o.address.script_pubkey(), + }) + .collect(), + special_transaction_payload: None, + }; + let txid = tx.txid(); + let mut details = Vec::with_capacity(group.len()); + let mut net: i64 = 0; + for (vout, o) in group.iter().enumerate() { + let vout = vout as u32; + details.push(OutputDetail { + index: vout, + role: OutputRole::Received, + address: Some(o.address.clone()), + value: o.amount, + }); + net = net.saturating_add(o.amount as i64); + let verdict = match o.verdict { + CREDITED => None, + 1 => Some(UtxoCreditVerdict::ObservedSpent { + height: o.spent_at_height, + }), + 2 => Some(UtxoCreditVerdict::Doomed), + 3 => Some(UtxoCreditVerdict::Uncredited), + other => return Err(format!("unknown verdict code {other}")), + }; + if let Some(verdict) = verdict { + core.utxo_credit_verdicts + .insert(OutPoint { txid, vout }, verdict); + } + } + let record = TransactionRecord::new( + tx, + account, + context.clone(), + TransactionType::Standard, + TransactionDirection::Incoming, + Vec::new(), + details, + net, + ); + core.records.push(record.clone()); + core.account_records.push(record); + } + Ok(core) +} + +/// Copy the Java arrays into fixture outputs; every array must be N long. +#[allow(clippy::too_many_arguments)] +fn read_outputs( + env: &mut JNIEnv, + tx_tags: &JIntArray, + addresses: &JObjectArray, + amounts: &JLongArray, + verdicts: &JByteArray, + spent_at_heights: &JIntArray, +) -> Result, String> { + let jni_err = |e: jni::errors::Error| e.to_string(); + let n = env.get_array_length(addresses).map_err(jni_err)? as usize; + let lengths = [ + env.get_array_length(tx_tags).map_err(jni_err)? as usize, + env.get_array_length(amounts).map_err(jni_err)? as usize, + env.get_array_length(verdicts).map_err(jni_err)? as usize, + env.get_array_length(spent_at_heights).map_err(jni_err)? as usize, + ]; + if lengths.iter().any(|len| *len != n) { + return Err(format!( + "fixture arrays disagree on length: {n} vs {lengths:?}" + )); + } + let mut tags = vec![0i32; n]; + env.get_int_array_region(tx_tags, 0, &mut tags) + .map_err(jni_err)?; + let mut values = vec![0i64; n]; + env.get_long_array_region(amounts, 0, &mut values) + .map_err(jni_err)?; + let mut codes = vec![0i8; n]; + env.get_byte_array_region(verdicts, 0, &mut codes) + .map_err(jni_err)?; + let mut heights = vec![0i32; n]; + env.get_int_array_region(spent_at_heights, 0, &mut heights) + .map_err(jni_err)?; + let mut out = Vec::with_capacity(n); + for i in 0..n { + let element = env + .get_object_array_element(addresses, i as i32) + .map_err(jni_err)?; + let text: String = env + .get_string(&JString::from(element)) + .map_err(jni_err)? + .into(); + let address = Address::from_str(&text) + .map_err(|e| format!("bad fixture address {text}: {e}"))? + .assume_checked(); + out.push(FixtureOutput { + tx_tag: u32::try_from(tags[i]).map_err(|_| "negative tx tag".to_string())?, + address, + amount: u64::try_from(values[i]).map_err(|_| "negative amount".to_string())?, + verdict: codes[i], + spent_at_height: u32::try_from(heights[i]) + .map_err(|_| "negative spent-at height".to_string())?, + }); + } + Ok(out) +} + +/// TEST ONLY — see the module doc. Delivers one round carrying the fixture +/// outputs (`txTags[i]` groups output `i` into a transaction; its vout is +/// its position in that group) and their credit verdicts (`0` = credited, +/// else a `UTXO_CREDIT_VERDICT_*` code) to `bridge` for `walletId`, through +/// a persister wired like the manager's. Throws `DashSDKException` when +/// the fixture is malformed or the round fails. +#[no_mangle] +#[allow(clippy::too_many_arguments)] +pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_nativeTestDeliverUtxoCreditVerdictRound( + mut env: JNIEnv, + _class: JClass, + bridge: JObject, + wallet_id: JByteArray, + tx_tags: JIntArray, + addresses: JObjectArray, + amounts: JLongArray, + verdicts: JByteArray, + spent_at_heights: JIntArray, + block_height: jint, +) { + guard(&mut env, (), |env| { + let fixture = (|| -> Result<(PlatformWalletChangeSet, [u8; 32]), String> { + let id = env + .convert_byte_array(&wallet_id) + .map_err(|e| e.to_string())?; + let id: [u8; 32] = id + .try_into() + .map_err(|_| "walletId must be 32 bytes".to_string())?; + let outputs = read_outputs( + env, + &tx_tags, + &addresses, + &amounts, + &verdicts, + &spent_at_heights, + )?; + let height = + u32::try_from(block_height).map_err(|_| "negative block height".to_string())?; + let core = build_round(&outputs, height)?; + Ok(( + PlatformWalletChangeSet { + core: Some(core), + ..Default::default() + }, + id, + )) + })(); + let (changeset, id) = match fixture { + Ok(value) => value, + Err(message) => { + throw_sdk_exception(env, 99, &format!("verdict-round fixture: {message}")); + return; + } + }; + + let global = match env.new_global_ref(&bridge) { + Ok(global) => global, + Err(_) => { + throw_sdk_exception(env, 99, "NewGlobalRef(persistence bridge) failed"); + return; + } + }; + // Wired exactly as `nativeCreate` wires the manager's persister. + // The persister owns the context box and frees it (release_fn) on + // drop, at the end of this call. + let ctx = Box::into_raw(Box::new(KotlinPersistenceCtx::new(global))); + let mut vtable = build_vtable(ctx as *mut c_void); + restrict_account_writers(env, &bridge, &mut vtable); + let extension = build_extension(env, &bridge); + let persister = FFIPersister::new_with_persistence_capabilities_and_extensions( + vtable, + PersistenceCapabilities::NONE, + PersistenceExtensionCallbacks { + dpns_name_states: extension.on_persist_dpns_name_states_fn, + wallet_changeset_sweeps: extension.on_persist_wallet_changeset_sweeps_fn, + wallet_changeset_chain_lock_height: extension + .on_persist_wallet_changeset_chain_lock_height_fn, + wallet_changeset_utxo_verdicts: extension + .on_persist_wallet_changeset_utxo_verdicts_fn, + wallet_dashpay_backfill: extension.on_persist_wallet_dashpay_backfill_fn, + ..Default::default() + }, + ); + let result = persister.store(id, changeset); + drop(persister); + if let Err(e) = result { + throw_sdk_exception(env, 99, &format!("verdict round failed: {e}")); + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn address(index: u32) -> Address { + let hash = dashcore::PubkeyHash::from_byte_array([index as u8; 20]); + Address::new( + dashcore::Network::Testnet, + dashcore::address::Payload::PubkeyHash(hash), + ) + } + + fn output(tx_tag: u32, index: u32, amount: u64, verdict: i8) -> FixtureOutput { + FixtureOutput { + tx_tag, + address: address(index), + amount, + verdict, + spent_at_height: if verdict == 1 { 900 } else { 0 }, + } + } + + /// Vouts follow position within a tag, a tag is one record in both the + /// folded and per-account lists, and the verdict map is keyed by the + /// real outpoints — credited outputs carry none. + #[test] + fn build_round_groups_by_tag_and_keys_verdicts_by_real_outpoint() { + let core = build_round( + &[ + output(1, 1, 100, CREDITED), + output(2, 4, 800, 3), + output(1, 2, 200, 1), + output(1, 3, 400, 2), + ], + 1_000, + ) + .expect("fixture builds"); + assert_eq!(core.records.len(), 2); + assert_eq!(core.account_records.len(), 2); + let first = &core.records[0]; + let second = &core.records[1]; + assert_eq!( + first + .transaction + .output + .iter() + .map(|o| o.value) + .collect::>(), + vec![100, 200, 400] + ); + assert_eq!( + core.utxo_credit_verdicts + .get(&OutPoint { + txid: first.txid, + vout: 1 + }) + .copied(), + Some(UtxoCreditVerdict::ObservedSpent { height: 900 }) + ); + assert_eq!( + core.utxo_credit_verdicts + .get(&OutPoint { + txid: first.txid, + vout: 2 + }) + .copied(), + Some(UtxoCreditVerdict::Doomed) + ); + assert_eq!( + core.utxo_credit_verdicts + .get(&OutPoint { + txid: second.txid, + vout: 0 + }) + .copied(), + Some(UtxoCreditVerdict::Uncredited) + ); + assert_eq!(core.utxo_credit_verdicts.len(), 3); + + // Same tag + outputs → same txid, so a later round re-delivers it. + let again = build_round(&[output(2, 4, 800, CREDITED)], 1_000).expect("fixture builds"); + assert_eq!(again.records[0].txid, second.txid); + } + + #[test] + fn build_round_rejects_an_unknown_verdict_code() { + assert!(build_round(&[output(1, 1, 100, 9)], 1_000).is_err()); + } +} From 1a83643a87a4994f84a2156b3b3f64179c5ba61f Mon Sep 17 00:00:00 2001 From: HashEngineering Date: Thu, 5 Nov 2026 21:29:25 -0800 Subject: [PATCH 5/6] fix(kotlin-sdk): clear a re-delivered coin unless its spender is settled upsertUtxoRow kept a spent row spent on a credited re-delivery whenever it had any spender link. A coin linked to a mempool spender that an observed-spent verdict then marked spent stayed spent after a reorg handed it back credited, and the restore set left it out, so its balance was gone after a restart. Follow Swift's upsertUtxo: a credited re-delivery with no verdict clears the spent flag and the sweep stamp unless the linked spender's transaction is at InstantSend-locked or above. The link is kept; the input index is dropped only for an unlinked row. Verdicts are unchanged: observed-spent and doomed still write spent, and uncredited only vetoes the clear. This also clears a spent row linked to a mempool spender with no verdict involved, which the old rule kept spent, as Swift does. Co-Authored-By: Claude Opus 5.5 --- .../PlatformWalletPersistenceHandler.kt | 57 +++++-- .../PlatformWalletPersistenceHandlerTest.kt | 141 +++++++++++++++++- 2 files changed, 181 insertions(+), 17 deletions(-) diff --git a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandler.kt b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandler.kt index 39d8d86bb26..a142d5769c3 100644 --- a/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandler.kt +++ b/packages/kotlin-sdk/sdk/src/main/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandler.kt @@ -1315,13 +1315,18 @@ class PlatformWalletPersistenceHandler( // re-discovers the spend; refusing the re-delivery would lock a // real coin out forever after a reorg of the winner, and on this // side of the FFI a row at `isSpent = true` is never restored to - // Rust again. So an UNLINKED row — a sweep hold with its stamp, or - // a legacy flag with nothing behind it — is cleared, stamp - // included. A LINKED row keeps its flag and stamp: the link is - // this store's recorded spend attribution, the pending drain - // below and the sweep pass own that transition, and a spender - // that reached a block is confirmed evidence a re-delivery never - // displaces. + // Rust again. So a spent row is cleared, stamp included — a sweep + // hold, a legacy flag with nothing behind it, or a row linked to a + // mempool spender alike. The link itself is kept: it is this + // store's spend attribution, and the pending drain below, the + // spend channels and the sweep pass own it. The one exception, as + // in Swift's `upsertUtxo`, is a row linked to a spender whose + // stored row is at or above InstantSend-locked + // ([hasSettledSpender]): confirmed evidence on record is never + // displaced by a re-delivery. A mempool link is not that evidence + // — a mempool spender can be dropped or reorged away, and holding + // the flag behind it would hide a coin the wallet credits from + // every later restore. // // Any credit verdict vetoes that clear (dashpay/platform#4638): the // wallet is not handing this coin back as unspent — its record @@ -1330,12 +1335,14 @@ class PlatformWalletPersistenceHandler( // spendable (a block spent it before the output was recognised — // the spender may never have been recorded, rust-dashcore#992 — or // the record can never confirm), so the row is written spent with - // no spender link. `isSpent` is monotonic: a row already spent is - // left as it is. An uncredited verdict carries no context and only - // keeps the clear from resurrecting the row. - val linked = existing?.spendingTxid != null - val keepSpentState = linked || (existing != null && creditVerdict != null) - val alreadySpent = keepSpentState && existing!!.isSpent + // no new spender link. `isSpent` is monotonic: a row already spent + // is left as it is. An uncredited verdict carries no context and + // only keeps the clear from resurrecting the row. A verdict does + // not outlive its round: a later credited re-delivery of the same + // coin (a reorg handed it back) clears the row by the rule above. + val clearRecovered = existing != null && existing.isSpent && creditVerdict == null && + !hasSettledSpender(db, existing) + val alreadySpent = existing?.isSpent == true && !clearRecovered val bornSpent = creditVerdict == UTXO_CREDIT_VERDICT_OBSERVED_SPENT || creditVerdict == UTXO_CREDIT_VERDICT_DOOMED if (creditVerdict != null && creditTally != null) { @@ -1361,12 +1368,17 @@ class PlatformWalletPersistenceHandler( walletId = walletId, txid = txid, spendingTxid = existing?.spendingTxid, - spendingInputIndex = existing?.spendingInputIndex, + // A cleared row with no link has no spend to index. + spendingInputIndex = if (clearRecovered && existing?.spendingTxid == null) { + null + } else { + existing?.spendingInputIndex + }, accountId = existing?.accountId ?: resolvedAccountId, coreAddressId = existing?.coreAddressId ?: coreAddressIdIfPresent(db, coreAddressId), createdAt = existing?.createdAt ?: java.util.Date(), lastUpdated = now(), - supersededByTxid = if (keepSpentState) existing!!.supersededByTxid else null, + supersededByTxid = if (clearRecovered) null else existing?.supersededByTxid, ) db.txoDao().upsert(row) var written = row @@ -2609,6 +2621,21 @@ class PlatformWalletPersistenceHandler( return !chainlockOverIsLock } + /** + * Whether [txo] is linked to a spender whose stored row is + * network-final — IS-locked, in-block or chainlocked + * (context >= [CONTEXT_INSTANT_SEND]). The recovery clear in + * [upsertUtxoRow] refuses only for such a spender, the check Swift's + * `upsertUtxo` makes on `spendingTransaction.context`. A link whose + * spender row is missing, or still at mempool context, is not + * settled. + */ + private suspend fun hasSettledSpender(db: DashDatabase, txo: TxoEntity): Boolean { + val link = txo.spendingTxid ?: return false + val spender = db.transactionDao().getByTxid(link) ?: return false + return spender.context >= CONTEXT_INSTANT_SEND + } + // ── Sweeps ──────────────────────────────────────────────────────── /** One sweep batch, unpacked from the JNI trampoline's flat arrays at the callback. */ diff --git a/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandlerTest.kt b/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandlerTest.kt index 58dab13c176..9d6e4dbce40 100644 --- a/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandlerTest.kt +++ b/packages/kotlin-sdk/sdk/src/test/kotlin/org/dashfoundation/dashsdk/persistence/PlatformWalletPersistenceHandlerTest.kt @@ -2486,8 +2486,8 @@ class PlatformWalletPersistenceHandlerTest { // good. Same answer as the SQLite store's upsert valve, which now // holds only never-materialised placeholders: a stamped, UNLINKED // row the wallet hands back as a UTXO is cleared, stamp included. - // A row still LINKED to a spender keeps its flag — the link is the - // store's recorded spend attribution and the sweep pass owns it. + // Only a row LINKED to a settled spender (IS-locked or better) + // keeps its flag — confirmed evidence a re-delivery never displaces. seedWalletWithAddress(walletId, "yUtxoAddr") val fundingTxid = ByteArray(32) { 48 } val coin = makeOutpoint(fundingTxid, 0) @@ -9232,6 +9232,143 @@ class PlatformWalletPersistenceHandlerTest { assertTrue(winner.contentEquals(row.supersededByTxid)) } + /** A credited re-delivery of the [seedFundedCoin] coin at `fundingTxid:0`, inside whatever bracket the caller opened. */ + private fun redeliverFundedCoin(h: PlatformWalletPersistenceHandler, fundingTxid: ByteArray): Int = + h.onWalletChangesetUtxoAdded( + walletId, fundingTxid, 0, 100_000, "yUtxoAddr", ByteArray(25) { 6 }, + 100, false, true, false, false, + ) + + /** + * Seed a funded coin linked to a spender recorded at [spenderContext], + * then deliver it with an observed-spent verdict (a block spent it). The + * coin is spent and keeps its link. Returns the coin's outpoint. + */ + private suspend fun seedLinkedCoinMarkedSpentByAVerdict( + fundingTxid: ByteArray, + spender: ByteArray, + spenderContext: Int, + ): ByteArray { + val coin = seedFundedCoin(fundingTxid) + recordMempoolSpend(spender, coin, context = spenderContext) + val linked = db.txoDao().getByOutpoint(coin)!! + assertTrue("sanity: linked to the spender", spender.contentEquals(linked.spendingTxid)) + verdictRound { + stageVerdicts(handler, CreditVerdict(fundingTxid, 0, observedSpent, 2_402_896)) + redeliverFundedCoin(handler, fundingTxid) + } + val marked = db.txoDao().getByOutpoint(coin)!! + assertTrue("the observed-spent verdict marks the coin spent", marked.isSpent) + assertTrue("and keeps the existing link", spender.contentEquals(marked.spendingTxid)) + assertTrue(handler.onLoadWalletList().single().utxos.none { it.prevTxid.contentEquals(fundingTxid) }) + return coin + } + + @Test + fun aMempoolSpenderLinkDoesNotMakeAnObservedSpentVerdictPermanent() = runTest { + // The reviewer's sequence: a coin linked to a mempool (context 0) + // spender, still unspent; an observed-spent verdict marks it spent + // with the link kept; a reorg then hands the coin back — the engine + // credits it again and re-delivers it with no verdict. A mempool link + // is not settled evidence (Swift's `upsertUtxo` checks the spender's + // context), so the re-delivery clears the flag and the coin is + // restored again, now and after a restart. + val fundingTxid = ByteArray(32) { 0x71 } + val spender = ByteArray(32) { 0x72 } + val coin = seedFundedCoin(fundingTxid) + recordMempoolSpend(spender, coin, context = 0) + assertFalse("sanity: a mempool link leaves the coin unspent", db.txoDao().getByOutpoint(coin)!!.isSpent) + verdictRound { + stageVerdicts(handler, CreditVerdict(fundingTxid, 0, observedSpent, 2_402_896)) + redeliverFundedCoin(handler, fundingTxid) + } + val marked = db.txoDao().getByOutpoint(coin)!! + assertTrue("the observed-spent verdict marks the coin spent", marked.isSpent) + assertTrue("and keeps the mempool link", spender.contentEquals(marked.spendingTxid)) + assertTrue(handler.onLoadWalletList().single().utxos.isEmpty()) + + verdictRound { redeliverFundedCoin(handler, fundingTxid) } + + val recovered = db.txoDao().getByOutpoint(coin)!! + assertFalse("the credited re-delivery clears the verdict's flag", recovered.isSpent) + assertTrue("the spend attribution is kept", spender.contentEquals(recovered.spendingTxid)) + assertEquals("a linked row keeps its vin index", 0, recovered.spendingInputIndex) + assertNull(recovered.supersededByTxid) + assertTrue( + "the restore set includes the coin again", + handler.onLoadWalletList().single().utxos.any { it.prevTxid.contentEquals(fundingTxid) && it.vout == 0 }, + ) + assertTrue( + "and so does a relaunch's", + newHandler().onLoadWalletList().single().utxos.any { it.prevTxid.contentEquals(fundingTxid) && it.vout == 0 }, + ) + } + + // The control: the same sequence with the linked spender IS-locked + // (context 1) or in a block (context 2). Confirmed evidence on record + // is never displaced by a re-delivery, so the coin stays spent and out + // of the restore set, now and after a restart. + private suspend fun assertASettledSpenderKeepsTheCoinSpent(spenderContext: Int) { + val fundingTxid = ByteArray(32) { 0x73 } + val spender = ByteArray(32) { 0x74 } + val coin = seedLinkedCoinMarkedSpentByAVerdict(fundingTxid, spender, spenderContext) + + verdictRound { redeliverFundedCoin(handler, fundingTxid) } + + val kept = db.txoDao().getByOutpoint(coin)!! + assertTrue("context $spenderContext: the settled spender keeps the coin spent", kept.isSpent) + assertTrue(spender.contentEquals(kept.spendingTxid)) + assertTrue( + "context $spenderContext: and out of the restore set", + newHandler().onLoadWalletList().single().utxos.none { it.prevTxid.contentEquals(fundingTxid) }, + ) + } + + @Test + fun anInstantSendLockedSpenderLinkKeepsAnObservedSpentVerdictAcrossACreditedRedelivery() = runTest { + assertASettledSpenderKeepsTheCoinSpent(spenderContext = 1) + } + + @Test + fun anInBlockSpenderLinkKeepsAnObservedSpentVerdictAcrossACreditedRedelivery() = runTest { + assertASettledSpenderKeepsTheCoinSpent(spenderContext = 2) + } + + @Test + fun aSpentRowLinkedToAMempoolSpenderIsClearedByACreditedRedelivery() = runTest { + // No verdict involved: a spent row whose link names a mempool + // spender (here a legacy flag and stamp) is cleared when the wallet + // hands the coin back as unspent, its stamp with it; the link + // stays. Before this rule ANY link kept the flag. An unlinked spent + // row clears too, and drops its stale vin index, as in Swift. + val fundingTxid = ByteArray(32) { 0x78 } + val spender = ByteArray(32) { 0x79 } + val stamp = ByteArray(32) { 0x7a } + val coin = seedFundedCoin(fundingTxid) + recordMempoolSpend(spender, coin) + db.txoDao().upsert( + db.txoDao().getByOutpoint(coin)!!.copy(isSpent = true, supersededByTxid = stamp), + ) + + verdictRound { redeliverFundedCoin(handler, fundingTxid) } + + val cleared = db.txoDao().getByOutpoint(coin)!! + assertFalse(cleared.isSpent) + assertNull("the stamp clears with the flag", cleared.supersededByTxid) + assertTrue("the link is kept", spender.contentEquals(cleared.spendingTxid)) + assertEquals(0, cleared.spendingInputIndex) + assertEquals(1, handler.onLoadWalletList().single().utxos.size) + + // Unlinked, with a stale index: cleared, index dropped. + db.txoDao().upsert( + db.txoDao().getByOutpoint(coin)!!.copy(isSpent = true, spendingTxid = null, spendingInputIndex = 3), + ) + verdictRound { redeliverFundedCoin(handler, fundingTxid) } + val unlinked = db.txoDao().getByOutpoint(coin)!! + assertFalse(unlinked.isSpent) + assertNull(unlinked.spendingInputIndex) + } + @Test fun verdictsDoNotOutliveTheirRound() = runTest { // A verdict for an outpoint the round never delivers is dropped with From d581b733e617a8fb77e1e0f927cf66148d866c4b Mon Sep 17 00:00:00 2001 From: HashEngineering Date: Tue, 10 Nov 2026 09:59:36 -0800 Subject: [PATCH 6/6] fix(kotlin-sdk): pass the reconcile finish outcome to platform-wallet #5796 changed platform_wallet_manager_txo_reconcile_finish to take an outcome code instead of a bool. The JNI bridge maps Kotlin's `ran` to TXO_RECONCILE_FINISH_RAN or TXO_RECONCILE_FINISH_NOT_RUN, and the failed-begin backstop passes NOT_RUN. Kotlin never defers a run, so it keeps its boolean. Co-Authored-By: Claude Opus 5.5 --- packages/rs-unified-sdk-jni/src/wallet_manager.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/rs-unified-sdk-jni/src/wallet_manager.rs b/packages/rs-unified-sdk-jni/src/wallet_manager.rs index 1ebe367f7a3..72dbc4587cd 100644 --- a/packages/rs-unified-sdk-jni/src/wallet_manager.rs +++ b/packages/rs-unified-sdk-jni/src/wallet_manager.rs @@ -3930,7 +3930,7 @@ pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_w let _ = platform_wallet_ffi::platform_wallet_manager_txo_reconcile_finish( manager_handle as Handle, wid.as_ptr(), - false, + platform_wallet_ffi::TXO_RECONCILE_FINISH_NOT_RUN, ); }); if env @@ -3995,7 +3995,9 @@ impl Drop for UntilHandedOver { /// Release the claim a 0 from `walletManagerTxoReconcileBegin` granted, /// and, as a backstop, any engine guard the run still holds; `ran` is -/// whether the run went ahead, which restarts the wallet's cadence. Bridges +/// whether the run went ahead, which restarts the wallet's cadence, and is +/// passed on as `TXO_RECONCILE_FINISH_RAN` or `TXO_RECONCILE_FINISH_NOT_RUN` +/// (Kotlin never defers a run). Bridges /// `platform_wallet_manager_txo_reconcile_finish`. Resolves the manager /// handle, so Kotlin frees the last page's guard first /// (`txoReconcileHeldVerdictsFree`). @@ -4015,7 +4017,11 @@ pub extern "system" fn Java_org_dashfoundation_dashsdk_ffi_WalletManagerNative_w platform_wallet_ffi::platform_wallet_manager_txo_reconcile_finish( manager_handle as Handle, wid.as_ptr(), - ran != JNI_FALSE, + if ran != JNI_FALSE { + platform_wallet_ffi::TXO_RECONCILE_FINISH_RAN + } else { + platform_wallet_ffi::TXO_RECONCILE_FINISH_NOT_RUN + }, ) }; let _ = take_pwffi_error(env, result);