From 2d7caa93a192f5d67de5ac3bd9f86c06aa8fed18 Mon Sep 17 00:00:00 2001 From: Brayden Langley Date: Mon, 17 Aug 2026 14:15:30 -0700 Subject: [PATCH 1/8] feat(wallet): accelerate storage synchronization --- packages/wallet/wallet-toolbox/CHANGELOG.md | 11 ++ packages/wallet/wallet-toolbox/README.md | 6 +- .../src/sdk/WalletStorage.interfaces.ts | 27 ++++ .../wallet-toolbox/src/storage/StorageKnex.ts | 56 +++++++ .../src/storage/StorageReader.ts | 9 ++ .../src/storage/methods/getSyncChunk.test.ts | 137 ++++++++++++++++++ .../src/storage/methods/getSyncChunk.ts | 69 +++++++-- .../remoting/__test/StorageServerRpc.test.ts | 4 +- .../__tests__/entityValidationHelpers.test.ts | 25 ++++ .../src/storage/schema/KnexMigrations.ts | 16 ++ .../test/storage/KnexMigrations.test.ts | 51 ++++++- .../test/wallet/sync/Wallet.sync.test.ts | 23 ++- 12 files changed, 418 insertions(+), 16 deletions(-) create mode 100644 packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.test.ts diff --git a/packages/wallet/wallet-toolbox/CHANGELOG.md b/packages/wallet/wallet-toolbox/CHANGELOG.md index 74e3fce63..2aba22c70 100644 --- a/packages/wallet/wallet-toolbox/CHANGELOG.md +++ b/packages/wallet/wallet-toolbox/CHANGELOG.md @@ -46,6 +46,17 @@ attention to changes that materially alter behavior or extend functionality. transaction startup/commit cycles, failed pages roll back without advancing the checkpoint, and abort cleanup preserves the original storage error. +- Fill wallet-storage sync pages with adaptive, size-aware source queries and + add composite SQL indexes for user-scoped proof lookups. Sync clients may + request optional source record totals for exact progress and ETA displays; + older clients and providers remain wire-compatible and do not incur count + queries unless totals are requested. The authoritative Linux Vite fixture is + 1,607,393 raw bytes and the local gzip fixture is 378,833 bytes; those + ceilings advance by 400 and 100 bytes to 1,607,400 and 378,900. The measured + authoritative Linux esbuild fixture is 1,252,871 raw and 345,202 gzip bytes; + those ceilings advance by 400 and 300 bytes to 1,252,900 and 345,300. Other + browser compressed and mobile ceilings remain unchanged. + - Make verified phone changes interruption-safe by staging the replacement key in WAB, publishing the UMP rotation, and then finalizing WAB. Authentication can recover an interrupted transition from the current or pending key and diff --git a/packages/wallet/wallet-toolbox/README.md b/packages/wallet/wallet-toolbox/README.md index 839bcea80..dcf098427 100644 --- a/packages/wallet/wallet-toolbox/README.md +++ b/packages/wallet/wallet-toolbox/README.md @@ -57,7 +57,11 @@ The toolbox publishes three npm packages from this repo: Wallet storage replication applies each received page and its durable sync checkpoint in one provider transaction. IndexedDB and Knex therefore avoid per-record transaction startup, and a failed page rolls back without advancing -the checkpoint. The sync wire format and persisted schemas are unchanged. +the checkpoint. Sources fill each bounded page with adaptive, size-aware reads, +and Knex storage adds user-scoped proof lookup indexes. Clients may set +`includeTotals` on a sync-chunk request to receive optional source record totals +for exact progress reporting. Older providers ignore the hint, and totals are +not counted unless requested. `listOutputs` reports `totalOutputs` as the full matching result count on every page for both Knex and IndexedDB storage, including short final pages and pages diff --git a/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts b/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts index 1b085e578..2d4c0665c 100644 --- a/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts +++ b/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts @@ -564,6 +564,12 @@ export interface RequestSyncChunkArgs { * The maximum number of items (records) to be returned. */ maxItems: number + /** + * Include source-side record totals for this `since` window when the + * provider can calculate them efficiently. Older providers ignore this + * optional hint and remain wire-compatible. + */ + includeTotals?: boolean /** * For each entity in dependency order, the offset at which to start returning items * from `since`. @@ -585,6 +591,24 @@ export interface RequestSyncChunkArgs { offsets: Array<{ name: string, offset: number }> } +export interface SyncChunkTotals { + totalRecords: number + records: { + provenTxs: number + outputBaskets: number + outputTags: number + txLabels: number + transactions: number + outputs: number + txLabelMaps: number + outputTagMaps: number + certificates: number + certificateFields: number + commissions: number + provenTxReqs: number + } +} + /** * Result received from remote `WalletStorage` in response to a `RequestSyncChunkArgs` request. * @@ -597,6 +621,9 @@ export interface SyncChunk { toStorageIdentityKey: string userIdentityKey: string + /** Optional progress totals requested with `includeTotals`. */ + totals?: SyncChunkTotals + user?: TableUser provenTxs?: TableProvenTx[] provenTxReqs?: TableProvenTxReq[] diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts b/packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts index cab11752d..d5dbdd75a 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts @@ -54,6 +54,8 @@ import { PurgeParams, PurgeResults, TrxToken, + RequestSyncChunkArgs, + SyncChunkTotals, WalletStorageProvider } from '../sdk/WalletStorage.interfaces' import { WERR_INTERNAL, WERR_INVALID_PARAMETER, WERR_NOT_IMPLEMENTED, WERR_UNAUTHORIZED } from '../sdk/WERR_errors' @@ -325,6 +327,60 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide return this.validateEntities(rs, undefined, ['isDeleted']) } + private async countSyncQuery (query: Knex.QueryBuilder): Promise { + const row = await query.count<{ count: string | number }>({ count: '*' }).first() + return Number(row?.count ?? 0) + } + + override async getSyncChunkTotals (args: RequestSyncChunkArgs, userId: number): Promise { + const since = args.since + const [ + provenTxs, + outputBaskets, + outputTags, + txLabels, + transactions, + outputs, + txLabelMaps, + outputTagMaps, + certificates, + certificateFields, + commissions, + provenTxReqs + ] = await Promise.all([ + this.countSyncQuery(this.getProvenTxsForUserQuery({ userId, since })), + this.countOutputBaskets({ partial: { userId }, since }), + this.countOutputTags({ partial: { userId }, since }), + this.countTxLabels({ partial: { userId }, since }), + this.countTransactions({ partial: { userId }, since, noRawTx: true }), + this.countOutputs({ partial: { userId }, since, noScript: true }), + this.countSyncQuery(this.getTxLabelMapsForUserQuery({ userId, since })), + this.countSyncQuery(this.getOutputTagMapsForUserQuery({ userId, since })), + this.countCertificates({ partial: { userId }, since }), + this.countCertificateFields({ partial: { userId }, since }), + this.countCommissions({ partial: { userId }, since }), + this.countSyncQuery(this.getProvenTxReqsForUserQuery({ userId, since })) + ]) + const records = { + provenTxs, + outputBaskets, + outputTags, + txLabels, + transactions, + outputs, + txLabelMaps, + outputTagMaps, + certificates, + certificateFields, + commissions, + provenTxReqs + } + return { + totalRecords: Object.values(records).reduce((total, count) => total + count, 0), + records + } + } + override async listActions (auth: AuthId, vargs: Validation.ValidListActionsArgs): Promise { if (auth.userId == null) throw new WERR_UNAUTHORIZED() return await listActions(this, auth, vargs) diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageReader.ts b/packages/wallet/wallet-toolbox/src/storage/StorageReader.ts index 6e5553342..f464e889e 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageReader.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageReader.ts @@ -105,6 +105,15 @@ export abstract class StorageReader implements sdk.WalletStorageSyncReader { abstract getTxLabelMapsForUser (args: sdk.FindForUserSincePagedArgs): Promise abstract getOutputTagMapsForUser (args: sdk.FindForUserSincePagedArgs): Promise + /** + * Optional efficient source-side totals for synchronization progress. + * Providers without a native count implementation omit the metadata rather + * than loading every matching record merely to count it. + */ + async getSyncChunkTotals (_args: sdk.RequestSyncChunkArgs, _userId: number): Promise { + return undefined + } + async findUserByIdentityKey (key: string): Promise { return verifyOneOrNone(await this.findUsers({ partial: { identityKey: key } })) } diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.test.ts b/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.test.ts new file mode 100644 index 000000000..bebc79a03 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.test.ts @@ -0,0 +1,137 @@ +import { RequestSyncChunkArgs } from '../../sdk/WalletStorage.interfaces' +import { StorageReader } from '../StorageReader' +import { getSyncChunk } from './getSyncChunk' + +const entityNames = [ + 'provenTx', + 'outputBasket', + 'outputTag', + 'txLabel', + 'transaction', + 'output', + 'txLabelMap', + 'outputTagMap', + 'certificate', + 'certificateField', + 'commission', + 'provenTxReq' +] + +function makeArgs(maxItems = 250, maxRoughSize = 2 * 1024 * 1024): RequestSyncChunkArgs { + return { + identityKey: '02'.repeat(33), + maxItems, + maxRoughSize, + offsets: entityNames.map(name => ({ name, offset: 0 })), + fromStorageIdentityKey: '11'.repeat(32), + toStorageIdentityKey: '22'.repeat(32) + } +} + +function makeStorage(provenTxCount: number, payloadBytes = 0) { + const now = new Date('2026-08-17T00:00:00.000Z') + const provenTxs = Array.from({ length: provenTxCount }, (_, index) => ({ + provenTxId: index + 1, + txid: index.toString(16).padStart(64, '0'), + created_at: now, + updated_at: now, + rawTx: Array.from({ length: payloadBytes }, () => 1) + })) + const getProvenTxsForUser = jest.fn(async ({ paged }: { paged?: { limit: number; offset?: number } }) => { + const offset = paged?.offset ?? 0 + return provenTxs.slice(offset, offset + (paged?.limit ?? provenTxs.length)) + }) + const empty = jest.fn(async () => []) + const storage = { + findUserByIdentityKey: jest.fn(async () => ({ + userId: 1, + identityKey: '02'.repeat(33), + created_at: now, + updated_at: now + })), + getProvenTxsForUser, + getSyncChunkTotals: jest.fn(async () => ({ + totalRecords: provenTxCount, + records: { + provenTxs: provenTxCount, + outputBaskets: 0, + outputTags: 0, + txLabels: 0, + transactions: 0, + outputs: 0, + txLabelMaps: 0, + outputTagMaps: 0, + certificates: 0, + certificateFields: 0, + commissions: 0, + provenTxReqs: 0 + } + })), + findOutputBaskets: empty, + findOutputTags: empty, + findTxLabels: empty, + findTransactions: empty, + findOutputs: empty, + getTxLabelMapsForUser: empty, + getOutputTagMapsForUser: empty, + findCertificates: empty, + findCertificateFields: empty, + findCommissions: empty, + getProvenTxReqsForUser: empty + } as unknown as StorageReader + return { storage, getProvenTxsForUser, getSyncChunkTotals: storage.getSyncChunkTotals as jest.Mock } +} + +describe('getSyncChunk query batching', () => { + test('fills a 250-record proven transaction page in a bounded number of queries', async () => { + const { storage, getProvenTxsForUser } = makeStorage(250) + + const chunk = await getSyncChunk(storage, makeArgs()) + + expect(chunk.provenTxs).toHaveLength(250) + expect(getProvenTxsForUser).toHaveBeenCalledTimes(3) + expect(getProvenTxsForUser.mock.calls.map(call => call[0].paged?.limit)).toEqual([10, 80, 160]) + }) + + test('uses observed record size to bound read-ahead for large records', async () => { + const { storage, getProvenTxsForUser } = makeStorage(250, 20_000) + + const chunk = await getSyncChunk(storage, makeArgs(250, 750_000)) + + expect(chunk.provenTxs!.length).toBeGreaterThan(10) + expect(chunk.provenTxs!.length).toBeLessThan(20) + expect(getProvenTxsForUser.mock.calls[1][0].paged?.limit).toBeLessThan(10) + }) + + test('includes efficient source totals only when requested', async () => { + const { storage, getSyncChunkTotals } = makeStorage(250) + const args = makeArgs() + args.includeTotals = true + + const chunk = await getSyncChunk(storage, args) + + expect(chunk.totals).toMatchObject({ totalRecords: 250, records: { provenTxs: 250 } }) + expect(getSyncChunkTotals).toHaveBeenCalledTimes(1) + }) + + test('does not add count-query overhead for legacy requests', async () => { + const { storage, getSyncChunkTotals } = makeStorage(25) + + const chunk = await getSyncChunk(storage, makeArgs()) + + expect(chunk.totals).toBeUndefined() + expect(getSyncChunkTotals).not.toHaveBeenCalled() + }) + + test('continues synchronization when optional totals cannot be counted', async () => { + const { storage, getSyncChunkTotals } = makeStorage(25) + getSyncChunkTotals.mockRejectedValueOnce(new Error('count unavailable')) + const args = makeArgs() + args.includeTotals = true + + const chunk = await getSyncChunk(storage, args) + + expect(chunk.provenTxs).toHaveLength(25) + expect(chunk.totals).toBeUndefined() + }) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.ts b/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.ts index e6a6a491e..cd507482d 100644 --- a/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.ts +++ b/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.ts @@ -15,6 +15,45 @@ import { FindForUserSincePagedArgs, RequestSyncChunkArgs, SyncChunk } from '../. import { verifyTruthy } from '../../utility/utilityHelpers' import { WERR_INVALID_OPERATION, WERR_INVALID_PARAMETER } from '../../sdk/WERR_errors' +const MIN_SYNC_QUERY_ITEMS = 10 +const MAX_SYNC_QUERY_ITEMS = 250 +const SYNC_QUERY_GROWTH_FACTOR = 8 + +function getNextSyncQueryLimit( + currentLimit: number, + itemsRemaining: number, + roughSizeRemaining: number, + batchRoughSize: number, + batchItemCount: number +): number { + const averageItemSize = Math.max(1, Math.ceil(batchRoughSize / batchItemCount)) + const estimatedItemsRemaining = Math.max(1, Math.floor(roughSizeRemaining / averageItemSize)) + return Math.min( + itemsRemaining, + MAX_SYNC_QUERY_ITEMS, + currentLimit * SYNC_QUERY_GROWTH_FACTOR, + estimatedItemsRemaining + ) +} + +function appendSyncItems(chunker: ChunkerArgs, items: any[], offset: number, itemCount: number, roughSize: number) { + let batchRoughSize = 0 + let done = false + for (const item of items) { + offset++ + chunker.addItem(item) + itemCount-- + const itemRoughSize = JSON.stringify(item).length + batchRoughSize += itemRoughSize + roughSize -= itemRoughSize + if (itemCount <= 0 || roughSize < 0) { + done = true + break + } + } + return { offset, itemCount, roughSize, batchRoughSize, done } +} + /** * Gets the next sync chunk of updated data from un-remoted storage (could be using a remote DB connection). * @param storage @@ -35,6 +74,10 @@ export async function getSyncChunk(storage: StorageReader, args: RequestSyncChun const user = verifyTruthy(await storage.findUserByIdentityKey(args.identityKey)) if (args.since == null || user.updated_at > new Date(args.since)) r.user = user + const totalsPromise = + args.includeTotals === true + ? storage.getSyncChunkTotals(args, user.userId).catch(() => undefined) + : Promise.resolve(undefined) const chunkers: ChunkerArgs[] = [ { @@ -237,8 +280,8 @@ export async function getSyncChunk(storage: StorageReader, args: RequestSyncChun throw new WERR_INVALID_PARAMETER('offsets', `in dependency order. '${a.name}' expected, found ${oname}.`) } let preAddCalled = false + let limit = Math.min(itemCount, Math.max(MIN_SYNC_QUERY_ITEMS, Math.ceil(args.maxItems / a.maxDivider))) while (!done) { - const limit = Math.min(itemCount, Math.max(10, args.maxItems / a.maxDivider)) if (limit <= 0) break const items = await a.findItems(storage, { userId: user.userId, @@ -251,16 +294,18 @@ export async function getSyncChunk(storage: StorageReader, args: RequestSyncChun preAddCalled = true } if (items.length === 0) break - for (const item of items) { - offset++ - a.addItem(item) - itemCount-- - roughSize -= JSON.stringify(item).length - if (itemCount <= 0 || roughSize < 0) { - done = true - break - } - } + const appended = appendSyncItems(a, items, offset, itemCount, roughSize) + offset = appended.offset + itemCount = appended.itemCount + roughSize = appended.roughSize + done = appended.done + if (done || items.length < limit) break + + // The small first query protects providers from loading hundreds of + // unexpectedly large binary records. Once their actual encoded size is + // known, grow the next query to fill the remaining page in a bounded + // number of database round trips. + limit = getNextSyncQueryLimit(limit, itemCount, roughSize, appended.batchRoughSize, items.length) } } @@ -270,6 +315,8 @@ export async function getSyncChunk(storage: StorageReader, args: RequestSyncChun } } + r.totals = await totalsPromise + return r } diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageServerRpc.test.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageServerRpc.test.ts index 7100d43c4..cc28dc6d7 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageServerRpc.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageServerRpc.test.ts @@ -375,9 +375,9 @@ describe('StorageServer JSON-RPC boundary', () => { invoke(server, 'enforceRpcRequestBudgets', 'listActions', [{}, { limit: Number.MAX_SAFE_INTEGER + 1 }]) ).rejects.toThrow('positive safe integers') - const syncParams: any[] = [{ maxRoughSize: 'unbounded' }] + const syncParams: any[] = [{ maxRoughSize: 'unbounded', includeTotals: true }] await invoke(server, 'enforceRpcRequestBudgets', 'getSyncChunk', syncParams) - expect(syncParams[0]).toEqual({ maxItems: 5, maxRoughSize: 128 }) + expect(syncParams[0]).toEqual({ maxItems: 5, maxRoughSize: 128, includeTotals: true }) const oversizedSyncParams: any[] = [{ maxItems: 4, maxRoughSize: 129 }] await invoke(server, 'enforceRpcRequestBudgets', 'getSyncChunk', oversizedSyncParams) diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/__tests__/entityValidationHelpers.test.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/__tests__/entityValidationHelpers.test.ts index f427727e1..971be630f 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/__tests__/entityValidationHelpers.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/__tests__/entityValidationHelpers.test.ts @@ -238,6 +238,31 @@ describe('entityValidationHelpers', () => { expect(result.provenTxs).toBeUndefined() }) + test('preserves optional progress totals for remoted chunks', () => { + const chunk: SyncChunk = { + ...baseChunk(), + totals: { + totalRecords: 12, + records: { + provenTxs: 1, + outputBaskets: 1, + outputTags: 1, + txLabels: 1, + transactions: 1, + outputs: 1, + txLabelMaps: 1, + outputTagMaps: 1, + certificates: 1, + certificateFields: 1, + commissions: 1, + provenTxReqs: 1 + } + } + } + + expect(validateSyncChunkEntities(chunk).totals).toEqual(chunk.totals) + }) + test('validates the user entity when present', () => { const chunk: SyncChunk = { ...baseChunk(), diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts b/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts index 5a4ef9e79..896bcff97 100644 --- a/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts +++ b/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts @@ -16,6 +16,7 @@ export const MONITOR_CREATED_AT_INDEX_MIGRATION = '2026-07-14-002 add monitor cr export const CREATE_ACTION_FUNDING_INDEX_MIGRATION = '2026-08-02-001 add createAction funding selection index' export const PAYMENT_REPLAY_MIGRATION = '2026-08-04-001 add payment replay claims' export const MANAGED_CHANGE_POLICY_MIGRATION = '2026-08-10-001 upgrade managed change liquidity defaults' +export const WALLET_SYNC_SOURCE_INDEX_MIGRATION = '2026-08-17-001 add wallet sync source indexes' interface Migration { up: (knex: Knex) => Promise @@ -177,6 +178,21 @@ export class KnexMigrations implements MigrationSource { } } + migrations[WALLET_SYNC_SOURCE_INDEX_MIGRATION] = { + async up(knex) { + await knex.schema.alterTable('transactions', table => { + table.index(['userId', 'provenTxId'], 'idx_transactions_user_proven_tx') + table.index(['userId', 'txid'], 'idx_transactions_user_txid') + }) + }, + async down(knex) { + await knex.schema.alterTable('transactions', table => { + table.dropIndex(['userId', 'provenTxId'], 'idx_transactions_user_proven_tx') + table.dropIndex(['userId', 'txid'], 'idx_transactions_user_txid') + }) + } + } + migrations['2026-07-15-001 add action batch reservations and blobs'] = { async up(knex) { const dbtype = await determineDBType(knex) diff --git a/packages/wallet/wallet-toolbox/test/storage/KnexMigrations.test.ts b/packages/wallet/wallet-toolbox/test/storage/KnexMigrations.test.ts index aa7aa7632..b376bee72 100644 --- a/packages/wallet/wallet-toolbox/test/storage/KnexMigrations.test.ts +++ b/packages/wallet/wallet-toolbox/test/storage/KnexMigrations.test.ts @@ -6,6 +6,7 @@ import { MANAGED_CHANGE_POLICY_MIGRATION, MONITOR_CREATED_AT_INDEX_MIGRATION, StorageKnex, + WALLET_SYNC_SOURCE_INDEX_MIGRATION, wait } from '../../src/index.all' import { Knex } from 'knex' @@ -206,7 +207,55 @@ describe('KnexMigrations tests', () => { } }) - test('5a upgrades only exact untouched managed-change defaults', async () => { + test('5a creates and uses the wallet sync source indexes', async () => { + const localSQLiteFile = await _tu.newTmpFile('migratesyncindexes.sqlite', false, false, false) + const knex = _tu.createLocalSQLite(localSQLiteFile) + + try { + await knex.schema.createTable('proven_txs', table => { + table.increments('provenTxId') + }) + await knex.schema.createTable('transactions', table => { + table.increments('transactionId') + table.integer('userId').notNullable() + table.integer('provenTxId').nullable() + table.string('txid', 64).nullable() + }) + const source = new KnexMigrations('test', 'wallet sync index test', '1'.repeat(64), 1000) + const migration = await source.getMigration(WALLET_SYNC_SOURCE_INDEX_MIGRATION) + await migration.up(knex) + + const indexes = await knex('sqlite_master') + .where({ type: 'index' }) + .whereIn('name', ['idx_transactions_user_proven_tx', 'idx_transactions_user_txid']) + .pluck('name') + expect(indexes.sort()).toEqual(['idx_transactions_user_proven_tx', 'idx_transactions_user_txid']) + + const provenPlan = await knex.raw( + 'EXPLAIN QUERY PLAN SELECT * FROM proven_txs WHERE EXISTS (' + + 'SELECT * FROM transactions WHERE proven_txs.provenTxId = transactions.provenTxId AND transactions.userId = ?)', + [1] + ) as Array<{ detail: string }> + expect(provenPlan.some(step => step.detail.includes('idx_transactions_user_proven_tx'))).toBe(true) + + const requestPlan = await knex.raw( + 'EXPLAIN QUERY PLAN SELECT * FROM transactions WHERE userId = ? AND txid = ?', + [1, '00'.repeat(32)] + ) as Array<{ detail: string }> + expect(requestPlan.some(step => step.detail.includes('idx_transactions_user_txid'))).toBe(true) + + await migration.down?.(knex) + const indexesAfterDown = await knex('sqlite_master') + .where({ type: 'index' }) + .whereIn('name', ['idx_transactions_user_proven_tx', 'idx_transactions_user_txid']) + .pluck('name') + expect(indexesAfterDown).toEqual([]) + } finally { + await knex.destroy() + } + }) + + test('5b upgrades only exact untouched managed-change defaults', async () => { const localSQLiteFile = await _tu.newTmpFile('migratemanagedchange.sqlite', false, false, false) const knex = _tu.createLocalSQLite(localSQLiteFile) diff --git a/packages/wallet/wallet-toolbox/test/wallet/sync/Wallet.sync.test.ts b/packages/wallet/wallet-toolbox/test/wallet/sync/Wallet.sync.test.ts index 4c0c8699c..3d796da96 100644 --- a/packages/wallet/wallet-toolbox/test/wallet/sync/Wallet.sync.test.ts +++ b/packages/wallet/wallet-toolbox/test/wallet/sync/Wallet.sync.test.ts @@ -1,4 +1,12 @@ -import { verifyOne, verifyOneOrNone, verifyTruthy, wait, Wallet, WalletStorageManager } from '../../../src/index.client' +import { + EntitySyncState, + verifyOne, + verifyOneOrNone, + verifyTruthy, + wait, + Wallet, + WalletStorageManager +} from '../../../src/index.client' import { StorageKnex } from '../../../src/storage/StorageKnex' import { _tu, logger, TestWalletNoSetup } from '../../utils/TestUtilsWalletStorage' @@ -32,6 +40,19 @@ describe('Wallet sync tests', () => { const manager = new WalletStorageManager(identityKey, storage, [tmpStore]) const auth = await manager.getAuth() + const destinationSyncState = await EntitySyncState.fromStorage(tmpStore, identityKey, _srcSettings) + const totalsArgs = destinationSyncState.makeRequestSyncChunkArgs( + identityKey, + tmpStore.getSettings().storageIdentityKey, + 2 * 1024 * 1024, + 250 + ) + totalsArgs.includeTotals = true + const firstChunk = await storage.getSyncChunk(totalsArgs) + expect(firstChunk.totals?.totalRecords).toBeGreaterThan(1000) + expect(firstChunk.totals?.totalRecords).toBe( + Object.values(firstChunk.totals?.records ?? {}).reduce((total, count) => total + count, 0) + ) { const r = await manager.syncToWriter(auth, tmpStore) expect(r.inserts).toBeGreaterThan(1000) From aacd97d8d238ef63ba4a76d4059bcfe5fff39957 Mon Sep 17 00:00:00 2001 From: Brayden Langley Date: Tue, 18 Aug 2026 15:55:13 -0700 Subject: [PATCH 2/8] fix(wallet): disambiguate storage sync checkpoints --- packages/wallet/wallet-toolbox/README.md | 5 +- .../src/sdk/WalletStorage.interfaces.ts | 6 ++ .../src/storage/StorageProvider.ts | 6 +- .../src/storage/StorageReaderWriter.ts | 27 +++++- .../src/storage/__test/StorageIdb.test.ts | 90 +++++++++++++++++++ .../schema/entities/EntitySyncState.ts | 1 + .../test/wallet/sync/Wallet.sync.test.ts | 26 ++++++ 7 files changed, 157 insertions(+), 4 deletions(-) diff --git a/packages/wallet/wallet-toolbox/README.md b/packages/wallet/wallet-toolbox/README.md index dcf098427..cea122590 100644 --- a/packages/wallet/wallet-toolbox/README.md +++ b/packages/wallet/wallet-toolbox/README.md @@ -61,7 +61,10 @@ the checkpoint. Sources fill each bounded page with adaptive, size-aware reads, and Knex storage adds user-scoped proof lookup indexes. Clients may set `includeTotals` on a sync-chunk request to receive optional source record totals for exact progress reporting. Older providers ignore the hint, and totals are -not counted unless requested. +not counted unless requested. New clients also send the writer-local sync-state +identifier selected during provider registration. New providers use it to +disambiguate legacy duplicate checkpoints, while either side remains compatible +with older protocol peers. `listOutputs` reports `totalOutputs` as the full matching result count on every page for both Knex and IndexedDB storage, including short final pages and pages diff --git a/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts b/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts index 2d4c0665c..d8207fec3 100644 --- a/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts +++ b/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts @@ -535,6 +535,12 @@ export type SyncStatus = 'success' | 'error' | 'identified' | 'updated' | 'unkno export type SyncProtocolVersion = '0.1.0' export interface RequestSyncChunkArgs { + /** + * The writer-local sync state selected when the source provider was + * registered. New clients include this to disambiguate legacy databases + * that contain multiple rows for a reused storage identity key. + */ + syncStateId?: number /** * The storageIdentityKey of the storage supplying the update SyncChunk data. */ diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts b/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts index 7be9cd45c..cac0f5259 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts @@ -1174,7 +1174,11 @@ export abstract class StorageProvider extends StorageReaderWriter implements Wal const ss = new EntitySyncState( verifyOne( await this.findSyncStates({ - partial: { + partial: args.syncStateId == null ? { + storageIdentityKey: args.fromStorageIdentityKey, + userId: user.userId + } : { + syncStateId: args.syncStateId, storageIdentityKey: args.fromStorageIdentityKey, userId: user.userId }, diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts b/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts index f780351a1..eeaa38a86 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts @@ -32,6 +32,7 @@ import { DEFAULT_MANAGED_CHANGE_MINIMUM_SATOSHIS, DEFAULT_MANAGED_CHANGE_TARGET_UTXOS } from './methods/managedChangePolicy' +import { WERR_INVALID_OPERATION } from '../sdk/WERR_errors' export abstract class StorageReaderWriter extends StorageReader { abstract dropAllData (): Promise @@ -361,14 +362,31 @@ export abstract class StorageReaderWriter extends StorageReader { storageIdentityKey: string, storageName: string ): Promise<{ syncState: TableSyncState, isNew: boolean }> { - const partial = { userId: auth.userId as number, storageIdentityKey, storageName } + const partial = { userId: auth.userId as number, storageIdentityKey } for (let retry = 0; ; retry++) { try { const now = new Date() - let syncState = verifyOneOrNone(await this.findSyncStates({ partial })) + const matches = await this.findSyncStates({ partial }) + let syncState: TableSyncState | undefined + if (matches.length === 1) { + syncState = matches[0] + } else if (matches.length > 1) { + // Older releases included storageName in the lookup and could create + // duplicate rows when a provider was renamed or two apps reused a + // provider identity. Preserve exact-name access so upgraded clients + // can identify and repair those rows without guessing a checkpoint. + const exactMatches = matches.filter(s => s.storageName === storageName) + if (exactMatches.length !== 1) { + throw new WERR_INVALID_OPERATION( + 'Storage identity has conflicting sync states. Use a unique identity for each storage provider.' + ) + } + syncState = exactMatches[0] + } if (syncState == null) { syncState = { ...partial, + storageName, created_at: now, updated_at: now, syncStateId: 0, @@ -380,6 +398,11 @@ export abstract class StorageReaderWriter extends StorageReader { await this.insertSyncState(syncState) return { syncState, isNew: true } } + if (syncState.storageName !== storageName) { + syncState.storageName = storageName + syncState.updated_at = now + await this.updateSyncState(syncState.syncStateId, { storageName, updated_at: now }) + } return { syncState, isNew: false } } catch (error_: unknown) { if (retry > 0) throw error_ diff --git a/packages/wallet/wallet-toolbox/src/storage/__test/StorageIdb.test.ts b/packages/wallet/wallet-toolbox/src/storage/__test/StorageIdb.test.ts index b48a7adff..f3475d3d7 100644 --- a/packages/wallet/wallet-toolbox/src/storage/__test/StorageIdb.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/__test/StorageIdb.test.ts @@ -251,6 +251,96 @@ describe('StorageIdb tests', () => { } }) + test('uses the registered sync state id when legacy provider identities collide', async () => { + const storage = await makeStorage() + try { + const identityKey = '06'.repeat(33) + const userId = await insertUser(storage, identityKey) + const now = new Date() + const sourceIdentityKey = 'shared-local-storage-identity' + const legacy: TableSyncState = { + syncStateId: 0, + userId, + storageIdentityKey: sourceIdentityKey, + storageName: 'metanet-wallet', + status: 'unknown', + init: false, + refNum: 'legacy-colliding-sync-state', + syncMap: JSON.stringify(createSyncMap()), + created_at: now, + updated_at: now + } + const selected: TableSyncState = { + ...legacy, + syncStateId: 0, + storageName: 'BSV Desktop Wallet', + refNum: 'selected-colliding-sync-state' + } + await storage.insertSyncState(legacy) + await storage.insertSyncState(selected) + + const registered = await storage.findOrInsertSyncStateAuth( + { identityKey, userId }, + sourceIdentityKey, + selected.storageName + ) + expect(registered.isNew).toBe(false) + expect(registered.syncState.syncStateId).toBe(selected.syncStateId) + await expect(storage.findOrInsertSyncStateAuth( + { identityKey, userId }, + sourceIdentityKey, + 'Unrecognized local provider' + )).rejects.toThrow('Storage identity has conflicting sync states') + + const args = { + syncStateId: registered.syncState.syncStateId, + identityKey, + maxRoughSize: 1000, + maxItems: 1000, + offsets: [], + since: undefined, + fromStorageIdentityKey: sourceIdentityKey, + toStorageIdentityKey: 'remote-storage' + } + const chunk = { + fromStorageIdentityKey: sourceIdentityKey, + toStorageIdentityKey: 'remote-storage', + userIdentityKey: identityKey + } + + await expect(storage.processSyncChunk(args, chunk)).resolves.toMatchObject({ + inserts: 0, + updates: 0 + }) + await expect(storage.processSyncChunk({ ...args, syncStateId: undefined }, chunk)) + .rejects.toThrow('Result must exist and be unique') + } finally { + await resetStorage(storage) + } + }) + + test('renames a registered storage provider without duplicating its sync state', async () => { + const storage = await makeStorage() + try { + const identityKey = '07'.repeat(33) + const userId = await insertUser(storage, identityKey) + const auth = { identityKey, userId } + const sourceIdentityKey = 'stable-storage-identity' + + const first = await storage.findOrInsertSyncStateAuth(auth, sourceIdentityKey, 'Old local name') + const renamed = await storage.findOrInsertSyncStateAuth(auth, sourceIdentityKey, 'New local name') + + expect(first.isNew).toBe(true) + expect(renamed.isNew).toBe(false) + expect(renamed.syncState.syncStateId).toBe(first.syncState.syncStateId) + expect(renamed.syncState.storageName).toBe('New local name') + await expect(storage.findSyncStates({ partial: { userId, storageIdentityKey: sourceIdentityKey } })) + .resolves.toHaveLength(1) + } finally { + await resetStorage(storage) + } + }) + test('preserves the original error when aborting an IndexedDB transaction', async () => { const storage = await makeStorage() try { diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/entities/EntitySyncState.ts b/packages/wallet/wallet-toolbox/src/storage/schema/entities/EntitySyncState.ts index 3ec500533..a7a899399 100644 --- a/packages/wallet/wallet-toolbox/src/storage/schema/entities/EntitySyncState.ts +++ b/packages/wallet/wallet-toolbox/src/storage/schema/entities/EntitySyncState.ts @@ -306,6 +306,7 @@ export class EntitySyncState extends EntityBase { maxItems?: number ): RequestSyncChunkArgs { const a: RequestSyncChunkArgs = { + syncStateId: this.id, identityKey: forIdentityKey, maxRoughSize: maxRoughSize || 10000000, maxItems: maxItems || 1000, diff --git a/packages/wallet/wallet-toolbox/test/wallet/sync/Wallet.sync.test.ts b/packages/wallet/wallet-toolbox/test/wallet/sync/Wallet.sync.test.ts index 3d796da96..17df5cdcb 100644 --- a/packages/wallet/wallet-toolbox/test/wallet/sync/Wallet.sync.test.ts +++ b/packages/wallet/wallet-toolbox/test/wallet/sync/Wallet.sync.test.ts @@ -108,6 +108,32 @@ describe('Wallet sync tests', () => { await ctx.storage.destroy() }) + + test('1c keeps the original active while adding a fresh local backup', async () => { + const ctx = await _tu.createLegacyWalletSQLiteCopy('walletSyncTest1cSource') + const localSQLiteFile = await _tu.newTmpFile('walletSyncTest1cLocal.sqlite', true, false, false) + const localStorage = new StorageKnex({ + ...StorageKnex.defaultOptions(), + chain: env.chain, + knex: _tu.createLocalSQLite(localSQLiteFile) + }) + + try { + const localStorageIdentityKey = `02${'08'.repeat(32)}` + await localStorage.migrate('BSV Desktop Wallet', localStorageIdentityKey) + await localStorage.makeAvailable() + + const originalStorageIdentityKey = ctx.activeStorage.getSettings().storageIdentityKey + await ctx.storage.addWalletStorageProvider(localStorage) + await ctx.storage.setActive(originalStorageIdentityKey) + + expect(ctx.storage.getActiveStore()).toBe(originalStorageIdentityKey) + expect(ctx.storage.isActiveEnabled).toBe(true) + expect(ctx.storage.getBackupStores()).toEqual([localStorageIdentityKey]) + } finally { + await ctx.storage.destroy() + } + }) }) async function setActiveTwice( From 12a9c76901d953dfac137c56131910a17c29c0e2 Mon Sep 17 00:00:00 2001 From: Brayden Langley Date: Tue, 18 Aug 2026 16:00:58 -0700 Subject: [PATCH 3/8] refactor(wallet): simplify sync state selection --- .../src/storage/StorageReaderWriter.ts | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts b/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts index eeaa38a86..ad771bf67 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts @@ -34,6 +34,24 @@ import { } from './methods/managedChangePolicy' import { WERR_INVALID_OPERATION } from '../sdk/WERR_errors' +function selectSyncStateForRegistration ( + matches: TableSyncState[], + storageName: string +): TableSyncState | undefined { + if (matches.length <= 1) return matches[0] + + // Older releases included storageName in the lookup and could create + // duplicate rows when a provider was renamed or two apps reused a provider + // identity. Preserve exact-name access so upgraded clients can identify and + // repair those rows without guessing a checkpoint. + const exactMatches = matches.filter(s => s.storageName === storageName) + if (exactMatches.length === 1) return exactMatches[0] + + throw new WERR_INVALID_OPERATION( + 'Storage identity has conflicting sync states. Use a unique identity for each storage provider.' + ) +} + export abstract class StorageReaderWriter extends StorageReader { abstract dropAllData (): Promise abstract migrate (storageName: string, storageIdentityKey: string): Promise @@ -367,22 +385,7 @@ export abstract class StorageReaderWriter extends StorageReader { try { const now = new Date() const matches = await this.findSyncStates({ partial }) - let syncState: TableSyncState | undefined - if (matches.length === 1) { - syncState = matches[0] - } else if (matches.length > 1) { - // Older releases included storageName in the lookup and could create - // duplicate rows when a provider was renamed or two apps reused a - // provider identity. Preserve exact-name access so upgraded clients - // can identify and repair those rows without guessing a checkpoint. - const exactMatches = matches.filter(s => s.storageName === storageName) - if (exactMatches.length !== 1) { - throw new WERR_INVALID_OPERATION( - 'Storage identity has conflicting sync states. Use a unique identity for each storage provider.' - ) - } - syncState = exactMatches[0] - } + let syncState = selectSyncStateForRegistration(matches, storageName) if (syncState == null) { syncState = { ...partial, From b78034a2249fa3313280197c3a8a091603067cf5 Mon Sep 17 00:00:00 2001 From: Brayden Langley Date: Tue, 18 Aug 2026 16:11:11 -0700 Subject: [PATCH 4/8] fix(wallet): keep sync state selection bundle-neutral --- .../src/storage/StorageReaderWriter.ts | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts b/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts index ad771bf67..ca76226c2 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageReaderWriter.ts @@ -34,24 +34,6 @@ import { } from './methods/managedChangePolicy' import { WERR_INVALID_OPERATION } from '../sdk/WERR_errors' -function selectSyncStateForRegistration ( - matches: TableSyncState[], - storageName: string -): TableSyncState | undefined { - if (matches.length <= 1) return matches[0] - - // Older releases included storageName in the lookup and could create - // duplicate rows when a provider was renamed or two apps reused a provider - // identity. Preserve exact-name access so upgraded clients can identify and - // repair those rows without guessing a checkpoint. - const exactMatches = matches.filter(s => s.storageName === storageName) - if (exactMatches.length === 1) return exactMatches[0] - - throw new WERR_INVALID_OPERATION( - 'Storage identity has conflicting sync states. Use a unique identity for each storage provider.' - ) -} - export abstract class StorageReaderWriter extends StorageReader { abstract dropAllData (): Promise abstract migrate (storageName: string, storageIdentityKey: string): Promise @@ -385,7 +367,20 @@ export abstract class StorageReaderWriter extends StorageReader { try { const now = new Date() const matches = await this.findSyncStates({ partial }) - let syncState = selectSyncStateForRegistration(matches, storageName) + let syncState = matches[0] + if (matches.length > 1) { + // Older releases included storageName in the lookup and could create + // duplicate rows when a provider was renamed or two apps reused a + // provider identity. Preserve exact-name access so upgraded clients + // can identify and repair those rows without guessing a checkpoint. + const exactMatches = matches.filter(s => s.storageName === storageName) + if (exactMatches.length !== 1) { + throw new WERR_INVALID_OPERATION( + 'Storage identity has conflicting sync states. Use a unique identity for each storage provider.' + ) + } + syncState = exactMatches[0] + } if (syncState == null) { syncState = { ...partial, From d3945e8f1fa6ef08b66d3021890e846fee1db040 Mon Sep 17 00:00:00 2001 From: Brayden Langley Date: Tue, 18 Aug 2026 17:39:20 -0700 Subject: [PATCH 5/8] fix(wallet): retry oversized sync responses --- packages/wallet/wallet-toolbox/CHANGELOG.md | 15 ++-- packages/wallet/wallet-toolbox/README.md | 5 +- .../src/storage/remoting/StorageClientBase.ts | 29 +++++++- .../StorageClientBase.syncRetry.test.ts | 74 +++++++++++++++++++ 4 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClientBase.syncRetry.test.ts diff --git a/packages/wallet/wallet-toolbox/CHANGELOG.md b/packages/wallet/wallet-toolbox/CHANGELOG.md index 2aba22c70..a70ff7121 100644 --- a/packages/wallet/wallet-toolbox/CHANGELOG.md +++ b/packages/wallet/wallet-toolbox/CHANGELOG.md @@ -50,12 +50,15 @@ attention to changes that materially alter behavior or extend functionality. add composite SQL indexes for user-scoped proof lookups. Sync clients may request optional source record totals for exact progress and ETA displays; older clients and providers remain wire-compatible and do not incur count - queries unless totals are requested. The authoritative Linux Vite fixture is - 1,607,393 raw bytes and the local gzip fixture is 378,833 bytes; those - ceilings advance by 400 and 100 bytes to 1,607,400 and 378,900. The measured - authoritative Linux esbuild fixture is 1,252,871 raw and 345,202 gzip bytes; - those ceilings advance by 400 and 300 bytes to 1,252,900 and 345,300. Other - browser compressed and mobile ceilings remain unchanged. + queries unless totals are requested. Remote clients also recover from a + provider's HTTP 413 response ceiling by retrying the read-only sync request + with a smaller chunk budget and reusing the working limit for later pages. + Clean macOS fixtures measure Vite at 1,608,681 raw, 379,257 gzip, and 297,202 + Brotli bytes; esbuild at 1,253,841 raw, 344,592 gzip, and 277,563 Brotli + bytes; and Hermes at 3,369,326 raw bytes. The reviewed ceilings for the + complete sync feature advance to 1,608,800/379,400/297,300 for Vite, + 1,253,900/345,500/277,700 for esbuild, and 3,369,500 raw bytes for Hermes. + Metro and the compressed mobile ceilings remain unchanged. - Make verified phone changes interruption-safe by staging the replacement key in WAB, publishing the UMP rotation, and then finalizing WAB. Authentication diff --git a/packages/wallet/wallet-toolbox/README.md b/packages/wallet/wallet-toolbox/README.md index cea122590..14455964b 100644 --- a/packages/wallet/wallet-toolbox/README.md +++ b/packages/wallet/wallet-toolbox/README.md @@ -64,7 +64,10 @@ for exact progress reporting. Older providers ignore the hint, and totals are not counted unless requested. New clients also send the writer-local sync-state identifier selected during provider registration. New providers use it to disambiguate legacy duplicate checkpoints, while either side remains compatible -with older protocol peers. +with older protocol peers. When a provider rejects a sync page because its +serialized RPC response exceeds the service ceiling, remote clients retry the +read-only request with a smaller chunk budget and remember the working limit +for the rest of the session. `listOutputs` reports `totalOutputs` as the full matching result count on every page for both Knex and IndexedDB storage, including short final pages and pages diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/StorageClientBase.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/StorageClientBase.ts index f80d2eb33..20994bdfd 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/StorageClientBase.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/StorageClientBase.ts @@ -69,6 +69,13 @@ import { } from '../../utility/actionBatchPack' import { pruneBeefForTxids } from '../../utility/beefForTxids' +const syncChunkResponseRetryLimit = 4 +const minimumSyncChunkRoughSize = 64 * 1024 + +function isSyncChunkResponseTooLarge (error: unknown): boolean { + return error instanceof Error && /WalletStorageClient rpcCall: network error 413(?:\s|$)/.test(error.message) +} + export interface StorageClientOptions { /** * Send compact tagged binary request values after the server advertises @@ -97,6 +104,7 @@ export abstract class StorageClientBase implements WalletStorageProvider { protected serverSupportsBinary = false protected readonly binaryRequests: boolean protected readonly telemetry: Telemetry + private syncChunkRoughSizeLimit?: number // Track ephemeral (in-memory) "settings" if you wish to align with isAvailable() checks public settings?: TableSettings @@ -603,8 +611,25 @@ export abstract class StorageClientBase implements WalletStorageProvider { * @returns the next "chunk" of replication data */ async getSyncChunk(args: RequestSyncChunkArgs): Promise { - const r = await this.rpcCall('getSyncChunk', [args]) - return validateSyncChunkEntities(r) + let requestArgs = { ...args } + if (this.syncChunkRoughSizeLimit != null) { + requestArgs.maxRoughSize = Math.min(requestArgs.maxRoughSize, this.syncChunkRoughSizeLimit) + } + + for (let retries = 0; ; retries++) { + try { + const r = await this.rpcCall('getSyncChunk', [requestArgs]) + if (requestArgs.maxRoughSize < args.maxRoughSize) { + this.syncChunkRoughSizeLimit = requestArgs.maxRoughSize + } + return validateSyncChunkEntities(r) + } catch (error: unknown) { + if (!isSyncChunkResponseTooLarge(error) || retries >= syncChunkResponseRetryLimit) throw error + const nextRoughSize = Math.max(minimumSyncChunkRoughSize, Math.floor(requestArgs.maxRoughSize / 2)) + if (nextRoughSize >= requestArgs.maxRoughSize) throw error + requestArgs = { ...requestArgs, maxRoughSize: nextRoughSize } + } + } } /** diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClientBase.syncRetry.test.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClientBase.syncRetry.test.ts new file mode 100644 index 000000000..61a3d5ab0 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClientBase.syncRetry.test.ts @@ -0,0 +1,74 @@ +import { type WalletInterface } from '@bsv/sdk' +import { type RequestSyncChunkArgs, type SyncChunk } from '../../../sdk/WalletStorage.interfaces' +import { StorageClientBase } from '../StorageClientBase' + +class RetryingStorageClient extends StorageClientBase { + calls: RequestSyncChunkArgs[] = [] + failuresRemaining: number + readonly failure: Error + + constructor (failures: number, failure = new Error('WalletStorageClient rpcCall: network error 413 413')) { + super({} as WalletInterface, 'https://storage.example.test') + this.failuresRemaining = failures + this.failure = failure + } + + protected async rpcCall (method: string, params: unknown[]): Promise { + expect(method).toBe('getSyncChunk') + const args = params[0] as RequestSyncChunkArgs + this.calls.push({ ...args }) + if (this.failuresRemaining-- > 0) throw this.failure + return { + fromStorageIdentityKey: args.fromStorageIdentityKey, + toStorageIdentityKey: args.toStorageIdentityKey, + userIdentityKey: args.identityKey + } as T + } +} + +function makeArgs (): RequestSyncChunkArgs { + return { + identityKey: `02${'11'.repeat(32)}`, + fromStorageIdentityKey: `02${'22'.repeat(32)}`, + toStorageIdentityKey: `02${'33'.repeat(32)}`, + maxItems: 1000, + maxRoughSize: 10_000_000, + offsets: [] + } +} + +describe('StorageClientBase sync response retry', () => { + test('halves an oversized response budget and remembers the working limit', async () => { + const client = new RetryingStorageClient(1) + const args = makeArgs() + + const first = await client.getSyncChunk(args) + const second = await client.getSyncChunk(args) + + expect(first).toMatchObject>({ userIdentityKey: args.identityKey }) + expect(second).toMatchObject>({ userIdentityKey: args.identityKey }) + expect(client.calls.map(call => call.maxRoughSize)).toEqual([10_000_000, 5_000_000, 5_000_000]) + expect(args.maxRoughSize).toBe(10_000_000) + }) + + test('does not retry unrelated network failures', async () => { + const failure = new Error('WalletStorageClient rpcCall: network error 503 Service Unavailable') + const client = new RetryingStorageClient(1, failure) + + await expect(client.getSyncChunk(makeArgs())).rejects.toBe(failure) + expect(client.calls).toHaveLength(1) + }) + + test('stops after the bounded number of oversized-response retries', async () => { + const client = new RetryingStorageClient(10) + + await expect(client.getSyncChunk(makeArgs())).rejects.toThrow('network error 413') + expect(client.calls.map(call => call.maxRoughSize)).toEqual([ + 10_000_000, + 5_000_000, + 2_500_000, + 1_250_000, + 625_000 + ]) + }) +}) From ac113dfdcfe5c3a32cd9ffee69bfc7d3b5e20beb Mon Sep 17 00:00:00 2001 From: Brayden Langley Date: Tue, 18 Aug 2026 17:44:51 -0700 Subject: [PATCH 6/8] chore(wallet): account for hosted compression --- packages/wallet/wallet-toolbox/CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/wallet/wallet-toolbox/CHANGELOG.md b/packages/wallet/wallet-toolbox/CHANGELOG.md index a70ff7121..7b85f8994 100644 --- a/packages/wallet/wallet-toolbox/CHANGELOG.md +++ b/packages/wallet/wallet-toolbox/CHANGELOG.md @@ -55,10 +55,11 @@ attention to changes that materially alter behavior or extend functionality. with a smaller chunk budget and reusing the working limit for later pages. Clean macOS fixtures measure Vite at 1,608,681 raw, 379,257 gzip, and 297,202 Brotli bytes; esbuild at 1,253,841 raw, 344,592 gzip, and 277,563 Brotli - bytes; and Hermes at 3,369,326 raw bytes. The reviewed ceilings for the + bytes; and Hermes at 3,369,326 raw bytes locally and 1,366,854 gzip bytes on + hosted Linux. The reviewed ceilings for the complete sync feature advance to 1,608,800/379,400/297,300 for Vite, - 1,253,900/345,500/277,700 for esbuild, and 3,369,500 raw bytes for Hermes. - Metro and the compressed mobile ceilings remain unchanged. + 1,253,900/345,600/277,700 for esbuild, and 3,369,500/1,367,000 raw/gzip + bytes for Hermes. Metro and the mobile Brotli ceiling remain unchanged. - Make verified phone changes interruption-safe by staging the replacement key in WAB, publishing the UMP rotation, and then finalizing WAB. Authentication From 3443b28e394ae55d461e0856e4b81dbf189eac64 Mon Sep 17 00:00:00 2001 From: Brayden Langley Date: Mon, 31 Aug 2026 17:51:09 -0700 Subject: [PATCH 7/8] test(wallet): complete sync rollout validation --- docs/reference/package-api-migrations.md | 18 +-- governance/package-release-notes.json | 12 +- packages/wallet/wallet-toolbox/CHANGELOG.md | 18 ++- packages/wallet/wallet-toolbox/README.md | 12 ++ .../benchmarks/storage-sync.bench.test.ts | 144 ++++++++++++++++++ .../client/platform-budget.json | 12 +- .../mobile/platform-budget.json | 4 +- packages/wallet/wallet-toolbox/package.json | 1 + .../src/storage/methods/getSyncChunk.test.ts | 23 +++ .../src/storage/methods/getSyncChunk.ts | 6 +- .../StorageClientBase.syncRetry.test.ts | 49 ++++++ .../remoting/__test/StorageServerRpc.test.ts | 4 +- .../__tests__/entityValidationHelpers.test.ts | 31 ++++ .../remoting/entityValidationHelpers.ts | 38 ++++- .../src/storage/schema/KnexMigrations.ts | 15 ++ .../test/storage/KnexMigrations.test.ts | 34 +++++ .../wallet-toolbox/test/storage/count.test.ts | 20 +++ 17 files changed, 407 insertions(+), 34 deletions(-) create mode 100644 packages/wallet/wallet-toolbox/benchmarks/storage-sync.bench.test.ts diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index 33810fcbf..14861cb74 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -55,9 +55,9 @@ and clean-consumer tests remain the executable type authority. | `@bsv/verifast` | `0.3.0` | `0.3.5` | patch | [API and usage](../packages/sdk/verifast.md) | No consumer migration is required; exports, verification behavior, worker protocols, package paths, and runtime defaults are unchanged. Keep THIRD_PARTY_NOTICES.md and LICENSES/ with every JavaScript and WebAssembly distribution. | | `@bsv/wallet-helper` | `0.1.1` | `0.1.7` | patch | [API and usage](../packages/helpers/wallet-helper.md) | No consumer migration is required; fluent builder APIs and transaction semantics are unchanged. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | | `@bsv/wallet-relay` | `0.2.2` | `0.3.6` | minor | [API and usage](../packages/wallet/wallet-relay.md) | No wallet RPC migration is required; upgrade to @bsv/sdk 2.4.1 or later. Existing relay sessions and number arrays remain valid, and host applications continue to provide their matching Express runtime and type graph. | -| `@bsv/wallet-toolbox` | `2.10.4` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox.md) | Existing permission modules require no changes because onRequest and onResponse remain supported. Semantic modules may add handleRequest; hosts installing @bsv/ecpm-permission-module register it under the ecpm scheme. Upgrade to @bsv/sdk 2.4.2 or later, use docs/storage.md instead of the removed JSight export, and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing. | -| `@bsv/wallet-toolbox-client` | `2.10.4` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-client.md) | Existing permission modules require no changes. Semantic modules may add handleRequest; installing @bsv/ecpm-permission-module requires registration under the ecpm scheme. Use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing. | -| `@bsv/wallet-toolbox-mobile` | `2.10.4` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-mobile.md) | Existing permission modules require no changes. Semantic modules may add handleRequest; mobile hosts can register compatible semantic modules without changing the Wallet interface. Use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing. | +| `@bsv/wallet-toolbox` | `2.10.4` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox.md) | Existing sync peers remain compatible: older clients omit the additive fields and older providers may return chunks without totals. Deploying services should run the normal Knex migration before relying on the new source indexes; clients retry only idempotent getSyncChunk reads. Existing permission modules require no changes. Upgrade to @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing. | +| `@bsv/wallet-toolbox-client` | `2.10.4` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-client.md) | Existing providers remain compatible and may omit totals; the client retries only idempotent getSyncChunk reads with a smaller response budget after HTTP 413. Existing permission modules require no changes. Use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing. | +| `@bsv/wallet-toolbox-mobile` | `2.10.4` | `2.11.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-mobile.md) | Existing providers remain compatible and may omit totals; the mobile client retries only idempotent getSyncChunk reads with a smaller response budget after HTTP 413. Existing permission modules require no changes. Use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing. | | `create-bsv-app` | `1.0.2` | `1.1.1` | minor | [API and usage](../packages/helpers/create-bsv-app.md) | Existing mainnet and testnet scaffolds are unchanged. New TTN projects pass --network ttn or select TerraTestNet in the configurator. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | `none` means the source manifest matches the recorded npm baseline. Any other @@ -520,8 +520,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox.md](../packages/wallet/wallet-toolbox.md) - Source: [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) -- Release note: Adds the optional semantic handleRequest hook to BRC-98/99/111 permission modules while retaining the existing transformation hooks, BRC-95/BRC-100 compatibility, and stable bounded pagination. It also removes the obsolete JSight application bundle and preserves the package's earlier Open BSV grant in the distribution notice archive. -- Migration: Existing permission modules require no changes because onRequest and onResponse remain supported. Semantic modules may add handleRequest; hosts installing @bsv/ecpm-permission-module register it under the ecpm scheme. Upgrade to @bsv/sdk 2.4.2 or later, use docs/storage.md instead of the removed JSight export, and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing. +- Release note: Adds adaptive wallet-storage sync reads, optional progress totals, disambiguated sync checkpoints, bounded HTTP 413 recovery, and MySQL/SQLite source indexes while retaining legacy peer compatibility. It also adds the optional semantic handleRequest permission-module hook, removes the obsolete JSight bundle, and preserves the package's earlier Open BSV grant. +- Migration: Existing sync peers remain compatible: older clients omit the additive fields and older providers may return chunks without totals. Deploying services should run the normal Knex migration before relying on the new source indexes; clients retry only idempotent getSyncChunk reads. Existing permission modules require no changes. Upgrade to @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | ---------------------------------------------------- | -------------------------- | @@ -534,8 +534,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox-client.md](../packages/wallet/wallet-toolbox-client.md) - Source: [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) -- Release note: Exports the optional semantic handleRequest permission-module hook for browser and ESM wallet hosts while retaining transformation modules, BRC-100 wire compatibility, stable IndexedDB totals, and the current browser Wallet Toolbox compatibility fixes. It preserves earlier Open BSV grants in the distribution notice archive. -- Migration: Existing permission modules require no changes. Semantic modules may add handleRequest; installing @bsv/ecpm-permission-module requires registration under the ecpm scheme. Use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing. +- Release note: Adds adaptive wallet-storage sync, optional exact progress totals, disambiguated checkpoints, and bounded HTTP 413 recovery for browser and ESM hosts while retaining legacy provider compatibility and the optional semantic permission-module hook. It preserves earlier Open BSV grants in the distribution notice archive. +- Migration: Existing providers remain compatible and may omit totals; the client retries only idempotent getSyncChunk reads with a smaller response budget after HTTP 413. Existing permission modules require no changes. Use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | @@ -546,8 +546,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox-mobile.md](../packages/wallet/wallet-toolbox-mobile.md) - Source: [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) -- Release note: Exports the optional semantic handleRequest permission-module hook for React Native wallet hosts while retaining transformation modules, BRC-100 wire compatibility, and the current mobile Wallet Toolbox compatibility fixes. It preserves earlier Open BSV grants in the distribution notice archive. -- Migration: Existing permission modules require no changes. Semantic modules may add handleRequest; mobile hosts can register compatible semantic modules without changing the Wallet interface. Use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing. +- Release note: Adds optional wallet-sync progress totals, disambiguated checkpoints, and bounded HTTP 413 recovery for React Native hosts while retaining legacy provider compatibility and the optional semantic permission-module hook. It preserves earlier Open BSV grants in the distribution notice archive. +- Migration: Existing providers remain compatible and may omit totals; the mobile client retries only idempotent getSyncChunk reads with a smaller response budget after HTTP 413. Existing permission modules require no changes. Use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | diff --git a/governance/package-release-notes.json b/governance/package-release-notes.json index ed390b47a..5f06c3bb7 100644 --- a/governance/package-release-notes.json +++ b/governance/package-release-notes.json @@ -217,22 +217,22 @@ "name": "@bsv/wallet-toolbox", "publishedVersion": "2.10.4", "releaseType": "minor", - "summary": "Adds the optional semantic handleRequest hook to BRC-98/99/111 permission modules while retaining the existing transformation hooks, BRC-95/BRC-100 compatibility, and stable bounded pagination. It also removes the obsolete JSight application bundle and preserves the package's earlier Open BSV grant in the distribution notice archive.", - "migration": "Existing permission modules require no changes because onRequest and onResponse remain supported. Semantic modules may add handleRequest; hosts installing @bsv/ecpm-permission-module register it under the ecpm scheme. Upgrade to @bsv/sdk 2.4.2 or later, use docs/storage.md instead of the removed JSight export, and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing." + "summary": "Adds adaptive wallet-storage sync reads, optional progress totals, disambiguated sync checkpoints, bounded HTTP 413 recovery, and MySQL/SQLite source indexes while retaining legacy peer compatibility. It also adds the optional semantic handleRequest permission-module hook, removes the obsolete JSight bundle, and preserves the package's earlier Open BSV grant.", + "migration": "Existing sync peers remain compatible: older clients omit the additive fields and older providers may return chunks without totals. Deploying services should run the normal Knex migration before relying on the new source indexes; clients retry only idempotent getSyncChunk reads. Existing permission modules require no changes. Upgrade to @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing." }, { "name": "@bsv/wallet-toolbox-client", "publishedVersion": "2.10.4", "releaseType": "minor", - "summary": "Exports the optional semantic handleRequest permission-module hook for browser and ESM wallet hosts while retaining transformation modules, BRC-100 wire compatibility, stable IndexedDB totals, and the current browser Wallet Toolbox compatibility fixes. It preserves earlier Open BSV grants in the distribution notice archive.", - "migration": "Existing permission modules require no changes. Semantic modules may add handleRequest; installing @bsv/ecpm-permission-module requires registration under the ecpm scheme. Use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing." + "summary": "Adds adaptive wallet-storage sync, optional exact progress totals, disambiguated checkpoints, and bounded HTTP 413 recovery for browser and ESM hosts while retaining legacy provider compatibility and the optional semantic permission-module hook. It preserves earlier Open BSV grants in the distribution notice archive.", + "migration": "Existing providers remain compatible and may omit totals; the client retries only idempotent getSyncChunk reads with a smaller response budget after HTTP 413. Existing permission modules require no changes. Use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing." }, { "name": "@bsv/wallet-toolbox-mobile", "publishedVersion": "2.10.4", "releaseType": "minor", - "summary": "Exports the optional semantic handleRequest permission-module hook for React Native wallet hosts while retaining transformation modules, BRC-100 wire compatibility, and the current mobile Wallet Toolbox compatibility fixes. It preserves earlier Open BSV grants in the distribution notice archive.", - "migration": "Existing permission modules require no changes. Semantic modules may add handleRequest; mobile hosts can register compatible semantic modules without changing the Wallet interface. Use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing." + "summary": "Adds optional wallet-sync progress totals, disambiguated checkpoints, and bounded HTTP 413 recovery for React Native hosts while retaining legacy provider compatibility and the optional semantic permission-module hook. It preserves earlier Open BSV grants in the distribution notice archive.", + "migration": "Existing providers remain compatible and may omit totals; the mobile client retries only idempotent getSyncChunk reads with a smaller response budget after HTTP 413. Existing permission modules require no changes. Use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing." }, { "name": "create-bsv-app", diff --git a/packages/wallet/wallet-toolbox/CHANGELOG.md b/packages/wallet/wallet-toolbox/CHANGELOG.md index 7b85f8994..4aa95500c 100644 --- a/packages/wallet/wallet-toolbox/CHANGELOG.md +++ b/packages/wallet/wallet-toolbox/CHANGELOG.md @@ -53,13 +53,17 @@ attention to changes that materially alter behavior or extend functionality. queries unless totals are requested. Remote clients also recover from a provider's HTTP 413 response ceiling by retrying the read-only sync request with a smaller chunk budget and reusing the working limit for later pages. - Clean macOS fixtures measure Vite at 1,608,681 raw, 379,257 gzip, and 297,202 - Brotli bytes; esbuild at 1,253,841 raw, 344,592 gzip, and 277,563 Brotli - bytes; and Hermes at 3,369,326 raw bytes locally and 1,366,854 gzip bytes on - hosted Linux. The reviewed ceilings for the - complete sync feature advance to 1,608,800/379,400/297,300 for Vite, - 1,253,900/345,600/277,700 for esbuild, and 3,369,500/1,367,000 raw/gzip - bytes for Hermes. Metro and the mobile Brotli ceiling remain unchanged. + Runtime validation rejects malformed remote totals, and MySQL rollback + restores the foreign-key support index before removing the new composites. + The retained authenticated candidate-provider benchmark fills a 250-record + proof page with three source reads (`10, 80, 160`) on SQLite and MySQL. + Clean macOS fixtures measure Vite at 1,609,916 raw, 379,552 gzip, and 297,188 + Brotli bytes; esbuild at 1,254,827 raw, 344,883 gzip, and 277,729 Brotli + bytes; Metro at 1,662,714 raw, 419,740 gzip, and 326,785 Brotli bytes; and + Hermes at 3,371,236 raw, 1,347,950 gzip, and 1,061,946 Brotli bytes. The + reviewed ceilings advance to 1,611,000/380,000/297,500 for Vite, + 1,256,000/346,000/278,200 for esbuild, and 3,373,000/1,368,000 raw/gzip for + Hermes. Metro and the Hermes Brotli ceiling remain unchanged. - Make verified phone changes interruption-safe by staging the replacement key in WAB, publishing the UMP rotation, and then finalizing WAB. Authentication diff --git a/packages/wallet/wallet-toolbox/README.md b/packages/wallet/wallet-toolbox/README.md index 14455964b..f25db3890 100644 --- a/packages/wallet/wallet-toolbox/README.md +++ b/packages/wallet/wallet-toolbox/README.md @@ -69,6 +69,18 @@ serialized RPC response exceeds the service ceiling, remote clients retry the read-only request with a smaller chunk budget and remember the working limit for the rest of the session. +Run the authenticated candidate-provider sync benchmark with: + +```sh +pnpm bench:storage-sync +``` + +Set `WALLET_TOOLBOX_BENCH_MYSQL=true`, `MYSQL_CONNECTION`, and optionally +`WALLET_TOOLBOX_BENCH_MYSQL_DATABASE` to exercise the same fixture through a +MySQL-backed provider. The benchmark reports HTTP p50/p95 latency and the +source-query limits used to fill a 250-record page; it is observational rather +than a cross-machine latency SLA. + `listOutputs` reports `totalOutputs` as the full matching result count on every page for both Knex and IndexedDB storage, including short final pages and pages requested at or past the end of the result set. diff --git a/packages/wallet/wallet-toolbox/benchmarks/storage-sync.bench.test.ts b/packages/wallet/wallet-toolbox/benchmarks/storage-sync.bench.test.ts new file mode 100644 index 000000000..78f5e2965 --- /dev/null +++ b/packages/wallet/wallet-toolbox/benchmarks/storage-sync.bench.test.ts @@ -0,0 +1,144 @@ +import { once } from 'node:events' +import { performance } from 'node:perf_hooks' +import { _tu, TestWalletNoSetup, TestWalletOnly } from '../test/utils/TestUtilsWalletStorage' +import { RequestSyncChunkArgs } from '../src/sdk/WalletStorage.interfaces' +import { StorageClient } from '../src/storage/remoting/StorageClient' +import { KnexSessionManager } from '../src/storage/remoting/KnexSessionManager' +import { StorageServer, WalletStorageServerOptions } from '../src/storage/remoting/StorageServer' +import { TableTransaction } from '../src/storage/schema/tables' + +const entityNames = [ + 'provenTx', + 'outputBasket', + 'outputTag', + 'txLabel', + 'transaction', + 'output', + 'txLabelMap', + 'outputTagMap', + 'certificate', + 'certificateField', + 'commission', + 'provenTxReq' +] + +function percentile(values: number[], fraction: number): number { + const sorted = [...values].sort((a, b) => a - b) + return sorted[Math.max(0, Math.ceil(sorted.length * fraction) - 1)] +} + +async function seedSyncFixture(ctx: TestWalletNoSetup, recordCount: number): Promise { + const since = new Date() + await new Promise(resolve => setTimeout(resolve, 10)) + await ctx.activeStorage.transaction(async trx => { + for (let index = 0; index < recordCount; index++) { + const txid = (index + 1).toString(16).padStart(64, '0') + const provenTx = await _tu.insertTestProvenTx(ctx.activeStorage, txid, trx) + const now = new Date() + const transaction: TableTransaction = { + created_at: now, + updated_at: now, + transactionId: 0, + userId: ctx.userId, + provenTxId: provenTx.provenTxId, + status: 'completed', + reference: `sync-benchmark-${index}`, + isOutgoing: true, + satoshis: index + 1, + description: 'candidate-provider sync benchmark', + txid + } + await ctx.activeStorage.insertTransaction(transaction, trx) + } + }) + return since +} + +async function createCandidateProvider(): Promise<{ + ctx: TestWalletNoSetup + client: TestWalletOnly + server: StorageServer +}> { + const databaseName = process.env.WALLET_TOOLBOX_BENCH_MYSQL_DATABASE ?? 'storageSyncBench' + const ctx = + process.env.WALLET_TOOLBOX_BENCH_MYSQL === 'true' + ? await _tu.createLegacyWalletMySQLCopy(databaseName) + : await _tu.createLegacyWalletSQLiteCopy(databaseName) + const options: WalletStorageServerOptions = { + port: 0, + wallet: ctx.wallet, + monetize: false, + logRpcRequests: false, + sessionManager: new KnexSessionManager(ctx.activeStorage.knex), + adminIdentityKeys: [], + calculateRequestPrice: async () => 0 + } + const server = new StorageServer(ctx.activeStorage, options) + server.start() + if (!server.server.listening) await once(server.server, 'listening') + const address = server.server.address() + if (address == null || typeof address === 'string') throw new Error('candidate provider did not bind') + const client = await _tu.createTestWalletWithStorageClient({ + rootKeyHex: ctx.rootKey.toHex(), + endpointUrl: `http://localhost:${address.port}`, + chain: ctx.chain + }) + await client.storage.getAuth(true) + return { ctx, client, server } +} + +describe('candidate-provider wallet sync benchmark', () => { + jest.setTimeout(600_000) + + test('measures an authenticated 250-record page over HTTP', async () => { + const recordCount = Number(process.env.WALLET_TOOLBOX_BENCH_SYNC_RECORDS ?? 250) + const samples = Number(process.env.WALLET_TOOLBOX_BENCH_SAMPLES ?? 7) + const candidate = await createCandidateProvider() + try { + const since = await seedSyncFixture(candidate.ctx, recordCount) + const storageClient = candidate.client.storage.getActive() as StorageClient + const sourceReads = jest.spyOn(candidate.ctx.activeStorage, 'getProvenTxsForUser') + const elapsedMs: number[] = [] + const sourceQueryLimits: number[][] = [] + let totalRecords = 0 + for (let sample = 0; sample < samples; sample++) { + const firstRead = sourceReads.mock.calls.length + const args: RequestSyncChunkArgs = { + identityKey: candidate.ctx.identityKey, + fromStorageIdentityKey: candidate.ctx.activeStorage.getSettings().storageIdentityKey, + toStorageIdentityKey: '33'.repeat(32), + since, + maxItems: recordCount, + maxRoughSize: 8 * 1024 * 1024, + includeTotals: true, + offsets: entityNames.map(name => ({ name, offset: 0 })) + } + const started = performance.now() + const chunk = await storageClient.getSyncChunk(args) + elapsedMs.push(performance.now() - started) + expect(chunk.provenTxs).toHaveLength(recordCount) + expect(chunk.totals).toBeDefined() + totalRecords = chunk.totals!.totalRecords + sourceQueryLimits.push(sourceReads.mock.calls.slice(firstRead).map(call => call[0].paged?.limit ?? 0)) + } + + const result = { + provider: process.env.WALLET_TOOLBOX_BENCH_MYSQL === 'true' ? 'MySQL 8.4 over HTTP' : 'SQLite over HTTP', + fixture: { provenTxs: recordCount, transactions: recordCount, totalRecords }, + samples, + p50Ms: percentile(elapsedMs, 0.5), + p95Ms: percentile(elapsedMs, 0.95), + maxMs: Math.max(...elapsedMs), + sourceQueriesPerPage: sourceQueryLimits.map(limits => limits.length), + sourceQueryLimits + } + process.stdout.write(`${JSON.stringify({ candidateProviderSync: result }, null, 2)}\n`) + expect(sourceQueryLimits.every(limits => limits.length === 3)).toBe(true) + expect(sourceQueryLimits.every(limits => limits.join(',') === '10,80,160')).toBe(true) + } finally { + await candidate.client.wallet.destroy() + await candidate.server.close() + await candidate.ctx.wallet.destroy() + } + }) +}) diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index b5e15c344..2d01048c5 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -2,14 +2,14 @@ "profile": "browser", "maximumBytes": { "vite": { - "raw": 1607500, - "gzip": 379000, - "brotli": 297000 + "raw": 1611000, + "gzip": 380000, + "brotli": 297500 }, "esbuild": { - "raw": 1253000, - "gzip": 345500, - "brotli": 277500 + "raw": 1256000, + "gzip": 346000, + "brotli": 278200 } } } diff --git a/packages/wallet/wallet-toolbox/mobile/platform-budget.json b/packages/wallet/wallet-toolbox/mobile/platform-budget.json index 7ecfab0dd..3df34d0b9 100644 --- a/packages/wallet/wallet-toolbox/mobile/platform-budget.json +++ b/packages/wallet/wallet-toolbox/mobile/platform-budget.json @@ -7,8 +7,8 @@ "brotli": 360000 }, "hermes": { - "raw": 3367500, - "gzip": 1366000, + "raw": 3373000, + "gzip": 1368000, "brotli": 1070000 } } diff --git a/packages/wallet/wallet-toolbox/package.json b/packages/wallet/wallet-toolbox/package.json index 5383f4b40..e2f5cdc93 100644 --- a/packages/wallet/wallet-toolbox/package.json +++ b/packages/wallet/wallet-toolbox/package.json @@ -63,6 +63,7 @@ "bench:action-batch": "pnpm build && jest --runInBand --runTestsByPath benchmarks/action-batch.bench.test.ts --testPathIgnorePatterns=man.test.ts", "bench:create-action-funding": "pnpm build && jest --runInBand --runTestsByPath benchmarks/create-action-funding.bench.test.ts --testPathIgnorePatterns=man.test.ts", "bench:create-action-beef": "pnpm build && jest --runInBand --runTestsByPath benchmarks/create-action-beef.bench.test.ts --testPathIgnorePatterns=man.test.ts", + "bench:storage-sync": "pnpm build && jest --runInBand --runTestsByPath benchmarks/storage-sync.bench.test.ts --testPathIgnorePatterns=man.test.ts", "format:check": "pnpm --workspace-root exec prettier --check \"packages/wallet/wallet-toolbox/{README.md,jest.config.cjs,package.json,tsconfig*.json}\"", "lint": "oxlint src test benchmarks examples operator --deny-warnings", "lint:ci": "pnpm lint", diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.test.ts b/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.test.ts index bebc79a03..f1d99d7a1 100644 --- a/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.test.ts @@ -93,6 +93,29 @@ describe('getSyncChunk query batching', () => { expect(getProvenTxsForUser.mock.calls.map(call => call[0].paged?.limit)).toEqual([10, 80, 160]) }) + test('caps the initial query for entity types without a conservative divider', async () => { + const { storage } = makeStorage(0) + const now = new Date('2026-08-17T00:00:00.000Z') + const outputBaskets = Array.from({ length: 300 }, (_, index) => ({ + basketId: index + 1, + userId: 1, + name: `basket-${index}`, + created_at: now, + updated_at: now + })) + const findOutputBaskets = jest.fn(async ({ paged }: { paged?: { limit: number; offset?: number } }) => { + const offset = paged?.offset ?? 0 + return outputBaskets.slice(offset, offset + (paged?.limit ?? outputBaskets.length)) + }) + storage.findOutputBaskets = findOutputBaskets as StorageReader['findOutputBaskets'] + + const chunk = await getSyncChunk(storage, makeArgs(1_000)) + + expect(chunk.outputBaskets).toHaveLength(300) + expect(findOutputBaskets.mock.calls[0][0].paged?.limit).toBe(250) + expect(findOutputBaskets.mock.calls.map(call => call[0].paged?.limit)).toEqual([250, 250]) + }) + test('uses observed record size to bound read-ahead for large records', async () => { const { storage, getProvenTxsForUser } = makeStorage(250, 20_000) diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.ts b/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.ts index cd507482d..46d26f137 100644 --- a/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.ts +++ b/packages/wallet/wallet-toolbox/src/storage/methods/getSyncChunk.ts @@ -280,7 +280,11 @@ export async function getSyncChunk(storage: StorageReader, args: RequestSyncChun throw new WERR_INVALID_PARAMETER('offsets', `in dependency order. '${a.name}' expected, found ${oname}.`) } let preAddCalled = false - let limit = Math.min(itemCount, Math.max(MIN_SYNC_QUERY_ITEMS, Math.ceil(args.maxItems / a.maxDivider))) + let limit = Math.min( + itemCount, + MAX_SYNC_QUERY_ITEMS, + Math.max(MIN_SYNC_QUERY_ITEMS, Math.ceil(args.maxItems / a.maxDivider)) + ) while (!done) { if (limit <= 0) break const items = await a.findItems(storage, { diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClientBase.syncRetry.test.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClientBase.syncRetry.test.ts index 61a3d5ab0..422cbb188 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClientBase.syncRetry.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClientBase.syncRetry.test.ts @@ -1,6 +1,8 @@ import { type WalletInterface } from '@bsv/sdk' import { type RequestSyncChunkArgs, type SyncChunk } from '../../../sdk/WalletStorage.interfaces' +import { StorageClient as FullStorageClient } from '../StorageClient' import { StorageClientBase } from '../StorageClientBase' +import { StorageClient as MobileStorageClient } from '../StorageMobile' class RetryingStorageClient extends StorageClientBase { calls: RequestSyncChunkArgs[] = [] @@ -71,4 +73,51 @@ describe('StorageClientBase sync response retry', () => { 625_000 ]) }) + + test.each([ + ['full client', FullStorageClient], + ['mobile client', MobileStorageClient] + ])('%s retries an actual HTTP 413 response and accepts a legacy response without totals', async (_name, Client) => { + const client = new Client({} as WalletInterface, 'https://storage.example.test') + const budgets: number[] = [] + const requests: RequestSyncChunkArgs[] = [] + const fetch = jest.fn(async (_url: string, init?: RequestInit) => { + const request = JSON.parse(String(init?.body)) as { + id: number + params: [RequestSyncChunkArgs] + } + budgets.push(request.params[0].maxRoughSize) + requests.push(request.params[0]) + if (budgets.length === 1) { + return new Response('', { status: 413, statusText: 'Payload Too Large' }) + } + return new Response( + JSON.stringify({ + jsonrpc: '2.0', + id: request.id, + result: { + fromStorageIdentityKey: request.params[0].fromStorageIdentityKey, + toStorageIdentityKey: request.params[0].toStorageIdentityKey, + userIdentityKey: request.params[0].identityKey + } + }), + { + status: 200, + headers: { 'Content-Type': 'application/json' } + } + ) + }) + Reflect.set(client, 'authClient', { fetch }) + const args = { ...makeArgs(), syncStateId: 42, includeTotals: true } + + const chunk = await client.getSyncChunk(args) + + expect(chunk.totals).toBeUndefined() + expect(budgets).toEqual([10_000_000, 5_000_000]) + expect(requests).toEqual([ + expect.objectContaining({ syncStateId: 42, includeTotals: true }), + expect.objectContaining({ syncStateId: 42, includeTotals: true }) + ]) + expect(args.maxRoughSize).toBe(10_000_000) + }) }) diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageServerRpc.test.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageServerRpc.test.ts index cc28dc6d7..bcdda0637 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageServerRpc.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageServerRpc.test.ts @@ -375,9 +375,9 @@ describe('StorageServer JSON-RPC boundary', () => { invoke(server, 'enforceRpcRequestBudgets', 'listActions', [{}, { limit: Number.MAX_SAFE_INTEGER + 1 }]) ).rejects.toThrow('positive safe integers') - const syncParams: any[] = [{ maxRoughSize: 'unbounded', includeTotals: true }] + const syncParams: any[] = [{ maxRoughSize: 'unbounded', includeTotals: true, syncStateId: 42 }] await invoke(server, 'enforceRpcRequestBudgets', 'getSyncChunk', syncParams) - expect(syncParams[0]).toEqual({ maxItems: 5, maxRoughSize: 128, includeTotals: true }) + expect(syncParams[0]).toEqual({ maxItems: 5, maxRoughSize: 128, includeTotals: true, syncStateId: 42 }) const oversizedSyncParams: any[] = [{ maxItems: 4, maxRoughSize: 129 }] await invoke(server, 'enforceRpcRequestBudgets', 'getSyncChunk', oversizedSyncParams) diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/__tests__/entityValidationHelpers.test.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/__tests__/entityValidationHelpers.test.ts index 971be630f..c773dfb1a 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/__tests__/entityValidationHelpers.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/__tests__/entityValidationHelpers.test.ts @@ -263,6 +263,37 @@ describe('entityValidationHelpers', () => { expect(validateSyncChunkEntities(chunk).totals).toEqual(chunk.totals) }) + test.each([ + ['negative count', { totalRecords: 0, records: { provenTxs: -1 } }], + ['unsafe count', { totalRecords: 0, records: { provenTxs: Number.MAX_SAFE_INTEGER + 1 } }], + ['missing record count', { totalRecords: 0, records: { provenTxs: undefined } }], + ['mismatched aggregate', { totalRecords: 1, records: {} }] + ])('rejects malformed remote progress totals: %s', (_name, malformedTotals) => { + const zeroRecords = { + provenTxs: 0, + outputBaskets: 0, + outputTags: 0, + txLabels: 0, + transactions: 0, + outputs: 0, + txLabelMaps: 0, + outputTagMaps: 0, + certificates: 0, + certificateFields: 0, + commissions: 0, + provenTxReqs: 0 + } + const chunk = { + ...baseChunk(), + totals: { + ...malformedTotals, + records: { ...zeroRecords, ...malformedTotals.records } + } + } as SyncChunk + + expect(() => validateSyncChunkEntities(chunk)).toThrow(TypeError) + }) + test('validates the user entity when present', () => { const chunk: SyncChunk = { ...baseChunk(), diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/entityValidationHelpers.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/entityValidationHelpers.ts index ffa0bc704..ebe21dff3 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/entityValidationHelpers.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/entityValidationHelpers.ts @@ -1,4 +1,4 @@ -import { SyncChunk } from '../../sdk/WalletStorage.interfaces' +import { SyncChunk, SyncChunkTotals } from '../../sdk/WalletStorage.interfaces' import { EntityTimeStamp } from '../../sdk/types' /** @@ -75,11 +75,47 @@ export function validateEntities(entities: T[], dateF return entities } +const syncChunkTotalRecordNames = [ + 'provenTxs', + 'outputBaskets', + 'outputTags', + 'txLabels', + 'transactions', + 'outputs', + 'txLabelMaps', + 'outputTagMaps', + 'certificates', + 'certificateFields', + 'commissions', + 'provenTxReqs' +] as const + +function validateSyncChunkTotals(totals: SyncChunkTotals): void { + if (typeof totals !== 'object' || totals == null || typeof totals.records !== 'object' || totals.records == null) { + throw new TypeError('Invalid sync chunk totals') + } + if (!Number.isSafeInteger(totals.totalRecords) || totals.totalRecords < 0) { + throw new TypeError('Invalid sync chunk totalRecords') + } + let calculatedTotal = 0 + for (const name of syncChunkTotalRecordNames) { + const count = totals.records[name] + if (!Number.isSafeInteger(count) || count < 0) { + throw new TypeError(`Invalid sync chunk total for ${name}`) + } + calculatedTotal += count + } + if (!Number.isSafeInteger(calculatedTotal) || calculatedTotal !== totals.totalRecords) { + throw new TypeError('Sync chunk record totals do not equal totalRecords') + } +} + /** * Validate all entity arrays within a `SyncChunk` received from a remote storage call. * Normalises timestamps, nulls, and binary fields in-place. */ export function validateSyncChunkEntities(r: SyncChunk): SyncChunk { + if (r.totals != null) validateSyncChunkTotals(r.totals) if (r.certificateFields != null) r.certificateFields = validateEntities(r.certificateFields) if (r.certificates != null) r.certificates = validateEntities(r.certificates) if (r.commissions != null) r.commissions = validateEntities(r.commissions) diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts b/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts index 896bcff97..efe7a77b6 100644 --- a/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts +++ b/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts @@ -186,6 +186,21 @@ export class KnexMigrations implements MigrationSource { }) }, async down(knex) { + // MySQL may discard the automatically-created userId index after one + // of these wider indexes becomes able to support the foreign key. + // Restore it before removing both migration-owned indexes. + if ((await determineDBType(knex)) === 'MySQL') { + const result = await knex.raw('SHOW INDEX FROM ?? WHERE Key_name = ?', [ + 'transactions', + 'transactions_userid_foreign' + ]) + const indexes = result[0] as unknown[] + if (indexes.length === 0) { + await knex.schema.alterTable('transactions', table => { + table.index(['userId'], 'transactions_userid_foreign') + }) + } + } await knex.schema.alterTable('transactions', table => { table.dropIndex(['userId', 'provenTxId'], 'idx_transactions_user_proven_tx') table.dropIndex(['userId', 'txid'], 'idx_transactions_user_txid') diff --git a/packages/wallet/wallet-toolbox/test/storage/KnexMigrations.test.ts b/packages/wallet/wallet-toolbox/test/storage/KnexMigrations.test.ts index b376bee72..34393d67d 100644 --- a/packages/wallet/wallet-toolbox/test/storage/KnexMigrations.test.ts +++ b/packages/wallet/wallet-toolbox/test/storage/KnexMigrations.test.ts @@ -255,6 +255,33 @@ describe('KnexMigrations tests', () => { } }) + test('5aa MySQL uses the wallet sync source indexes', async () => { + if (!env.runMySQL) return + const knex = knexs.find(candidate => candidate.client.config.client === 'mysql2') + if (knex == null) throw new Error('RUNMYSQL requires a MySQL knex connection') + + const [indexRows] = (await knex.raw( + "SHOW INDEX FROM transactions WHERE Key_name IN ('idx_transactions_user_proven_tx', 'idx_transactions_user_txid')" + )) as [Array<{ Key_name: string }>, unknown] + expect([...new Set(indexRows.map(row => row.Key_name))].sort()).toEqual([ + 'idx_transactions_user_proven_tx', + 'idx_transactions_user_txid' + ]) + + const [provenPlan] = (await knex.raw( + 'EXPLAIN SELECT * FROM transactions FORCE INDEX (idx_transactions_user_proven_tx) ' + + 'WHERE userId = ? AND provenTxId = ?', + [1, 1] + )) as [Array<{ key: string | null }>, unknown] + expect(provenPlan.some(step => step.key === 'idx_transactions_user_proven_tx')).toBe(true) + + const [txidPlan] = (await knex.raw( + 'EXPLAIN SELECT * FROM transactions FORCE INDEX (idx_transactions_user_txid) WHERE userId = ? AND txid = ?', + [1, '00'.repeat(32)] + )) as [Array<{ key: string | null }>, unknown] + expect(txidPlan.some(step => step.key === 'idx_transactions_user_txid')).toBe(true) + }) + test('5b upgrades only exact untouched managed-change defaults', async () => { const localSQLiteFile = await _tu.newTmpFile('migratemanagedchange.sqlite', false, false, false) const knex = _tu.createLocalSQLite(localSQLiteFile) @@ -296,6 +323,11 @@ describe('KnexMigrations tests', () => { }) test.each([ + { + migrationName: WALLET_SYNC_SOURCE_INDEX_MIGRATION, + supportIndex: 'transactions_userid_foreign', + addedIndexes: ['idx_transactions_user_proven_tx', 'idx_transactions_user_txid'] + }, { migrationName: '2026-02-27-001 add listOutputs path indexes', supportIndex: 'outputs_userid_foreign', @@ -340,6 +372,7 @@ describe('KnexMigrations tests', () => { }) test.each([ + WALLET_SYNC_SOURCE_INDEX_MIGRATION, '2026-02-27-001 add listOutputs path indexes', '2026-02-27-002 add createAction path indexes' ])('7 preserves an existing MySQL foreign-key support index while rolling back %s', async migrationName => { @@ -364,6 +397,7 @@ describe('KnexMigrations tests', () => { }) test.each([ + WALLET_SYNC_SOURCE_INDEX_MIGRATION, '2026-02-27-001 add listOutputs path indexes', '2026-02-27-002 add createAction path indexes' ])('8 rolls back %s without MySQL support-index repair on SQLite', async migrationName => { diff --git a/packages/wallet/wallet-toolbox/test/storage/count.test.ts b/packages/wallet/wallet-toolbox/test/storage/count.test.ts index fb9b444a5..aeb786e5f 100644 --- a/packages/wallet/wallet-toolbox/test/storage/count.test.ts +++ b/packages/wallet/wallet-toolbox/test/storage/count.test.ts @@ -174,4 +174,24 @@ describe('count tests', () => { expect(await storage.countSyncStates({ partial: {} })).toBe(1) } }) + + test('15 computes internally consistent per-user sync totals on SQLite and MySQL', async () => { + for (const { storage, setup } of setups) { + const totals = await storage.getSyncChunkTotals( + { + identityKey: setup.u1.identityKey, + fromStorageIdentityKey: storage.getSettings().storageIdentityKey, + toStorageIdentityKey: 'remote-storage', + maxItems: 1_000, + maxRoughSize: 1_000_000, + offsets: [] + }, + setup.u1.userId + ) + + expect(totals).toBeDefined() + expect(totals!.totalRecords).toBeGreaterThan(0) + expect(Object.values(totals!.records).reduce((sum, count) => sum + count, 0)).toBe(totals!.totalRecords) + } + }) }) From a4c615b9b4273b197dcf057f89df94816e313081 Mon Sep 17 00:00:00 2001 From: Brayden Langley Date: Mon, 31 Aug 2026 17:53:24 -0700 Subject: [PATCH 8/8] chore(governance): refresh mutation policy review --- governance/mutation-testing/policy.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/governance/mutation-testing/policy.json b/governance/mutation-testing/policy.json index 493c4c465..6420a28b4 100644 --- a/governance/mutation-testing/policy.json +++ b/governance/mutation-testing/policy.json @@ -1,7 +1,7 @@ { "schemaVersion": 1, - "lastReviewed": "2026-07-31", - "reviewBy": "2026-08-31", + "lastReviewed": "2026-08-31", + "reviewBy": "2026-09-30", "owner": "ts-stack-maintainers", "tool": { "package": "@stryker-mutator/core",