bsv-blockchain · BraydenLangley · Aug 17, 2026 · Aug 18, 2026 · Aug 18, 2026 · Aug 18, 2026
@@ -1,7 +1,7 @@
 {
   "schemaVersion": 1,
-  "lastReviewed": "2026-07-31",
-  "reviewBy": "2026-08-31",
+  "lastReviewed": "2026-08-31",
+  "reviewBy": "2026-09-30",
   "owner": "ts-stack-maintainers",
   "tool": {
     "package": "@stryker-mutator/core",

@@ -217,22 +217,22 @@
       "name": "@bsv/wallet-toolbox",
       "publishedVersion": "2.10.4",
       "releaseType": "minor",
-      "summary": "Adds the optional semantic handleRequest hook to BRC-98/99/111 permission modules while retaining the existing transformation hooks, BRC-95/BRC-100 compatibility, and stable bounded pagination. It also removes the obsolete JSight application bundle and preserves the package's earlier Open BSV grant in the distribution notice archive.",
-      "migration": "Existing permission modules require no changes because onRequest and onResponse remain supported. Semantic modules may add handleRequest; hosts installing @bsv/ecpm-permission-module register it under the ecpm scheme. Upgrade to @bsv/sdk 2.4.2 or later, use docs/storage.md instead of the removed JSight export, and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing."
+      "summary": "Adds adaptive wallet-storage sync reads, optional progress totals, disambiguated sync checkpoints, bounded HTTP 413 recovery, and MySQL/SQLite source indexes while retaining legacy peer compatibility. It also adds the optional semantic handleRequest permission-module hook, removes the obsolete JSight bundle, and preserves the package's earlier Open BSV grant.",
+      "migration": "Existing sync peers remain compatible: older clients omit the additive fields and older providers may return chunks without totals. Deploying services should run the normal Knex migration before relying on the new source indexes; clients retry only idempotent getSyncChunk reads. Existing permission modules require no changes. Upgrade to @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing."
     },
     {
       "name": "@bsv/wallet-toolbox-client",
       "publishedVersion": "2.10.4",
       "releaseType": "minor",
-      "summary": "Exports the optional semantic handleRequest permission-module hook for browser and ESM wallet hosts while retaining transformation modules, BRC-100 wire compatibility, stable IndexedDB totals, and the current browser Wallet Toolbox compatibility fixes. It preserves earlier Open BSV grants in the distribution notice archive.",
-      "migration": "Existing permission modules require no changes. Semantic modules may add handleRequest; installing @bsv/ecpm-permission-module requires registration under the ecpm scheme. Use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing."
+      "summary": "Adds adaptive wallet-storage sync, optional exact progress totals, disambiguated checkpoints, and bounded HTTP 413 recovery for browser and ESM hosts while retaining legacy provider compatibility and the optional semantic permission-module hook. It preserves earlier Open BSV grants in the distribution notice archive.",
+      "migration": "Existing providers remain compatible and may omit totals; the client retries only idempotent getSyncChunk reads with a smaller response budget after HTTP 413. Existing permission modules require no changes. Use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing."
     },
     {
       "name": "@bsv/wallet-toolbox-mobile",
       "publishedVersion": "2.10.4",
       "releaseType": "minor",
-      "summary": "Exports the optional semantic handleRequest permission-module hook for React Native wallet hosts while retaining transformation modules, BRC-100 wire compatibility, and the current mobile Wallet Toolbox compatibility fixes. It preserves earlier Open BSV grants in the distribution notice archive.",
-      "migration": "Existing permission modules require no changes. Semantic modules may add handleRequest; mobile hosts can register compatible semantic modules without changing the Wallet interface. Use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing."
+      "summary": "Adds optional wallet-sync progress totals, disambiguated checkpoints, and bounded HTTP 413 recovery for React Native hosts while retaining legacy provider compatibility and the optional semantic permission-module hook. It preserves earlier Open BSV grants in the distribution notice archive.",
+      "migration": "Existing providers remain compatible and may omit totals; the mobile client retries only idempotent getSyncChunk reads with a smaller response budget after HTTP 413. Existing permission modules require no changes. Use @bsv/sdk 2.4.2 or later and retain THIRD_PARTY_NOTICES.md and LICENSES/ when redistributing."
     },
     {
       "name": "create-bsv-app",

@@ -46,6 +46,25 @@ attention to changes that materially alter behavior or extend functionality.
   transaction startup/commit cycles, failed pages roll back without advancing
   the checkpoint, and abort cleanup preserves the original storage error.
 
+- Fill wallet-storage sync pages with adaptive, size-aware source queries and
+  add composite SQL indexes for user-scoped proof lookups. Sync clients may
+  request optional source record totals for exact progress and ETA displays;
+  older clients and providers remain wire-compatible and do not incur count
+  queries unless totals are requested. Remote clients also recover from a
+  provider's HTTP 413 response ceiling by retrying the read-only sync request
+  with a smaller chunk budget and reusing the working limit for later pages.
+  Runtime validation rejects malformed remote totals, and MySQL rollback
+  restores the foreign-key support index before removing the new composites.
+  The retained authenticated candidate-provider benchmark fills a 250-record
+  proof page with three source reads (`10, 80, 160`) on SQLite and MySQL.
+  Clean macOS fixtures measure Vite at 1,609,916 raw, 379,552 gzip, and 297,188
+  Brotli bytes; esbuild at 1,254,827 raw, 344,883 gzip, and 277,729 Brotli
+  bytes; Metro at 1,662,714 raw, 419,740 gzip, and 326,785 Brotli bytes; and
+  Hermes at 3,371,236 raw, 1,347,950 gzip, and 1,061,946 Brotli bytes. The
+  reviewed ceilings advance to 1,611,000/380,000/297,500 for Vite,
+  1,256,000/346,000/278,200 for esbuild, and 3,373,000/1,368,000 raw/gzip for
+  Hermes. Metro and the Hermes Brotli ceiling remain unchanged.
+
 - Make verified phone changes interruption-safe by staging the replacement key
   in WAB, publishing the UMP rotation, and then finalizing WAB. Authentication
   can recover an interrupted transition from the current or pending key and

@@ -57,7 +57,29 @@ The toolbox publishes three npm packages from this repo:
 Wallet storage replication applies each received page and its durable sync
 checkpoint in one provider transaction. IndexedDB and Knex therefore avoid
 per-record transaction startup, and a failed page rolls back without advancing
-the checkpoint. The sync wire format and persisted schemas are unchanged.
+the checkpoint. Sources fill each bounded page with adaptive, size-aware reads,
+and Knex storage adds user-scoped proof lookup indexes. Clients may set
+`includeTotals` on a sync-chunk request to receive optional source record totals
+for exact progress reporting. Older providers ignore the hint, and totals are
+not counted unless requested. New clients also send the writer-local sync-state
+identifier selected during provider registration. New providers use it to
+disambiguate legacy duplicate checkpoints, while either side remains compatible
+with older protocol peers. When a provider rejects a sync page because its
+serialized RPC response exceeds the service ceiling, remote clients retry the
+read-only request with a smaller chunk budget and remember the working limit
+for the rest of the session.
+
+Run the authenticated candidate-provider sync benchmark with:
+
+```sh
+pnpm bench:storage-sync
+```
+
+Set `WALLET_TOOLBOX_BENCH_MYSQL=true`, `MYSQL_CONNECTION`, and optionally
+`WALLET_TOOLBOX_BENCH_MYSQL_DATABASE` to exercise the same fixture through a
+MySQL-backed provider. The benchmark reports HTTP p50/p95 latency and the
+source-query limits used to fill a 250-record page; it is observational rather
+than a cross-machine latency SLA.
 
 `listOutputs` reports `totalOutputs` as the full matching result count on every
 page for both Knex and IndexedDB storage, including short final pages and pages

@@ -0,0 +1,144 @@
+import { once } from 'node:events'
+import { performance } from 'node:perf_hooks'
+import { _tu, TestWalletNoSetup, TestWalletOnly } from '../test/utils/TestUtilsWalletStorage'
+import { RequestSyncChunkArgs } from '../src/sdk/WalletStorage.interfaces'
+import { StorageClient } from '../src/storage/remoting/StorageClient'
+import { KnexSessionManager } from '../src/storage/remoting/KnexSessionManager'
+import { StorageServer, WalletStorageServerOptions } from '../src/storage/remoting/StorageServer'
+import { TableTransaction } from '../src/storage/schema/tables'
+
+const entityNames = [
+  'provenTx',
+  'outputBasket',
+  'outputTag',
+  'txLabel',
+  'transaction',
+  'output',
+  'txLabelMap',
+  'outputTagMap',
+  'certificate',
+  'certificateField',
+  'commission',
+  'provenTxReq'
+]
+
+function percentile(values: number[], fraction: number): number {
+  const sorted = [...values].sort((a, b) => a - b)
+  return sorted[Math.max(0, Math.ceil(sorted.length * fraction) - 1)]
+}
+
+async function seedSyncFixture(ctx: TestWalletNoSetup, recordCount: number): Promise<Date> {
+  const since = new Date()
+  await new Promise(resolve => setTimeout(resolve, 10))
+  await ctx.activeStorage.transaction(async trx => {
+    for (let index = 0; index < recordCount; index++) {
+      const txid = (index + 1).toString(16).padStart(64, '0')
+      const provenTx = await _tu.insertTestProvenTx(ctx.activeStorage, txid, trx)
+      const now = new Date()
+      const transaction: TableTransaction = {
+        created_at: now,
+        updated_at: now,
+        transactionId: 0,
+        userId: ctx.userId,
+        provenTxId: provenTx.provenTxId,
+        status: 'completed',
+        reference: `sync-benchmark-${index}`,
+        isOutgoing: true,
+        satoshis: index + 1,
+        description: 'candidate-provider sync benchmark',
+        txid
+      }
+      await ctx.activeStorage.insertTransaction(transaction, trx)
+    }
+  })
+  return since
+}
+
+async function createCandidateProvider(): Promise<{
+  ctx: TestWalletNoSetup
+  client: TestWalletOnly
+  server: StorageServer
+}> {
+  const databaseName = process.env.WALLET_TOOLBOX_BENCH_MYSQL_DATABASE ?? 'storageSyncBench'
+  const ctx =
+    process.env.WALLET_TOOLBOX_BENCH_MYSQL === 'true'
+      ? await _tu.createLegacyWalletMySQLCopy(databaseName)
+      : await _tu.createLegacyWalletSQLiteCopy(databaseName)
+  const options: WalletStorageServerOptions = {
+    port: 0,
+    wallet: ctx.wallet,
+    monetize: false,
+    logRpcRequests: false,
+    sessionManager: new KnexSessionManager(ctx.activeStorage.knex),
+    adminIdentityKeys: [],
+    calculateRequestPrice: async () => 0
+  }
+  const server = new StorageServer(ctx.activeStorage, options)
+  server.start()
+  if (!server.server.listening) await once(server.server, 'listening')
+  const address = server.server.address()
+  if (address == null || typeof address === 'string') throw new Error('candidate provider did not bind')
+  const client = await _tu.createTestWalletWithStorageClient({
+    rootKeyHex: ctx.rootKey.toHex(),
+    endpointUrl: `http://localhost:${address.port}`,
+    chain: ctx.chain
+  })
+  await client.storage.getAuth(true)
+  return { ctx, client, server }
+}
+
+describe('candidate-provider wallet sync benchmark', () => {
+  jest.setTimeout(600_000)
+
+  test('measures an authenticated 250-record page over HTTP', async () => {
+    const recordCount = Number(process.env.WALLET_TOOLBOX_BENCH_SYNC_RECORDS ?? 250)
+    const samples = Number(process.env.WALLET_TOOLBOX_BENCH_SAMPLES ?? 7)
+    const candidate = await createCandidateProvider()
+    try {
+      const since = await seedSyncFixture(candidate.ctx, recordCount)
+      const storageClient = candidate.client.storage.getActive() as StorageClient
+      const sourceReads = jest.spyOn(candidate.ctx.activeStorage, 'getProvenTxsForUser')
+      const elapsedMs: number[] = []
+      const sourceQueryLimits: number[][] = []
+      let totalRecords = 0
+      for (let sample = 0; sample < samples; sample++) {
+        const firstRead = sourceReads.mock.calls.length
+        const args: RequestSyncChunkArgs = {
+          identityKey: candidate.ctx.identityKey,
+          fromStorageIdentityKey: candidate.ctx.activeStorage.getSettings().storageIdentityKey,
+          toStorageIdentityKey: '33'.repeat(32),
+          since,
+          maxItems: recordCount,
+          maxRoughSize: 8 * 1024 * 1024,
+          includeTotals: true,
+          offsets: entityNames.map(name => ({ name, offset: 0 }))
+        }
+        const started = performance.now()
+        const chunk = await storageClient.getSyncChunk(args)
+        elapsedMs.push(performance.now() - started)
+        expect(chunk.provenTxs).toHaveLength(recordCount)
+        expect(chunk.totals).toBeDefined()
+        totalRecords = chunk.totals!.totalRecords
+        sourceQueryLimits.push(sourceReads.mock.calls.slice(firstRead).map(call => call[0].paged?.limit ?? 0))
+      }
+
+      const result = {
+        provider: process.env.WALLET_TOOLBOX_BENCH_MYSQL === 'true' ? 'MySQL 8.4 over HTTP' : 'SQLite over HTTP',
+        fixture: { provenTxs: recordCount, transactions: recordCount, totalRecords },
+        samples,
+        p50Ms: percentile(elapsedMs, 0.5),
+        p95Ms: percentile(elapsedMs, 0.95),
+        maxMs: Math.max(...elapsedMs),
+        sourceQueriesPerPage: sourceQueryLimits.map(limits => limits.length),
+        sourceQueryLimits
+      }
+      process.stdout.write(`${JSON.stringify({ candidateProviderSync: result }, null, 2)}\n`)
+      expect(sourceQueryLimits.every(limits => limits.length === 3)).toBe(true)
+      expect(sourceQueryLimits.every(limits => limits.join(',') === '10,80,160')).toBe(true)
+    } finally {
+      await candidate.client.wallet.destroy()
+      await candidate.server.close()
+      await candidate.ctx.wallet.destroy()
+    }
+  })
+})
@@ -2,14 +2,14 @@
   "profile": "browser",
   "maximumBytes": {
     "vite": {
-      "raw": 1607500,
-      "gzip": 379000,
-      "brotli": 297000
+      "raw": 1611000,
+      "gzip": 380000,
+      "brotli": 297500
     },
     "esbuild": {
-      "raw": 1253000,
-      "gzip": 345500,
-      "brotli": 277500
+      "raw": 1256000,
+      "gzip": 346000,
+      "brotli": 278200
     }
   }
 }
@@ -7,8 +7,8 @@
       "brotli": 360000
     },
     "hermes": {
-      "raw": 3367500,
-      "gzip": 1366000,
+      "raw": 3373000,
+      "gzip": 1368000,
       "brotli": 1070000
     }
   }

@@ -63,6 +63,7 @@
     "bench:action-batch": "pnpm build && jest --runInBand --runTestsByPath benchmarks/action-batch.bench.test.ts --testPathIgnorePatterns=man.test.ts",
     "bench:create-action-funding": "pnpm build && jest --runInBand --runTestsByPath benchmarks/create-action-funding.bench.test.ts --testPathIgnorePatterns=man.test.ts",
     "bench:create-action-beef": "pnpm build && jest --runInBand --runTestsByPath benchmarks/create-action-beef.bench.test.ts --testPathIgnorePatterns=man.test.ts",
+    "bench:storage-sync": "pnpm build && jest --runInBand --runTestsByPath benchmarks/storage-sync.bench.test.ts --testPathIgnorePatterns=man.test.ts",
     "format:check": "pnpm --workspace-root exec prettier --check \"packages/wallet/wallet-toolbox/{README.md,jest.config.cjs,package.json,tsconfig*.json}\"",
     "lint": "oxlint src test benchmarks examples operator --deny-warnings",
     "lint:ci": "pnpm lint",

@@ -535,6 +535,12 @@ export type SyncStatus = 'success' | 'error' | 'identified' | 'updated' | 'unkno
 export type SyncProtocolVersion = '0.1.0'
 
 export interface RequestSyncChunkArgs {
+  /**
+   * The writer-local sync state selected when the source provider was
+   * registered. New clients include this to disambiguate legacy databases
+   * that contain multiple rows for a reused storage identity key.
+   */
+  syncStateId?: number
   /**
    * The storageIdentityKey of the storage supplying the update SyncChunk data.
    */
@@ -564,6 +570,12 @@ export interface RequestSyncChunkArgs {
    * The maximum number of items (records) to be returned.
    */
   maxItems: number
+  /**
+   * Include source-side record totals for this `since` window when the
+   * provider can calculate them efficiently. Older providers ignore this
+   * optional hint and remain wire-compatible.
+   */
+  includeTotals?: boolean
   /**
    * For each entity in dependency order, the offset at which to start returning items
    * from `since`.
@@ -585,6 +597,24 @@ export interface RequestSyncChunkArgs {
   offsets: Array<{ name: string, offset: number }>
 }
 
+export interface SyncChunkTotals {
+  totalRecords: number
+  records: {
+    provenTxs: number
+    outputBaskets: number
+    outputTags: number
+    txLabels: number
+    transactions: number
+    outputs: number
+    txLabelMaps: number
+    outputTagMaps: number
+    certificates: number
+    certificateFields: number
+    commissions: number
+    provenTxReqs: number
+  }
+}
+
 /**
  * Result received from remote `WalletStorage` in response to a `RequestSyncChunkArgs` request.
  *
@@ -597,6 +627,9 @@ export interface SyncChunk {
   toStorageIdentityKey: string
   userIdentityKey: string
 
+  /** Optional progress totals requested with `includeTotals`. */
+  totals?: SyncChunkTotals
+
   user?: TableUser
   provenTxs?: TableProvenTx[]
   provenTxReqs?: TableProvenTxReq[]