From a3d7ed71465b69c2fc48a63c54bd8b85b491674c Mon Sep 17 00:00:00 2001 From: bean1352 Date: Tue, 23 Jun 2026 16:12:13 +0200 Subject: [PATCH 01/40] Add bucket storage report types and builder --- .../src/storage/SyncRulesBucketStorage.ts | 12 ++ .../service-core/src/storage/bucket-report.ts | 117 ++++++++++++++++++ .../service-core/src/storage/storage-index.ts | 1 + .../test/src/bucket-report.test.ts | 93 ++++++++++++++ 4 files changed, 223 insertions(+) create mode 100644 packages/service-core/src/storage/bucket-report.ts create mode 100644 packages/service-core/test/src/bucket-report.test.ts diff --git a/packages/service-core/src/storage/SyncRulesBucketStorage.ts b/packages/service-core/src/storage/SyncRulesBucketStorage.ts index a5a28027d..d20c2fc8b 100644 --- a/packages/service-core/src/storage/SyncRulesBucketStorage.ts +++ b/packages/service-core/src/storage/SyncRulesBucketStorage.ts @@ -8,6 +8,7 @@ import { import * as bson from 'bson'; import { PerformanceTracer } from '../tracing/PerformanceTracer.js'; import * as util from '../util/util-index.js'; +import { BucketReport, GetBucketReportOptions } from './bucket-report.js'; import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js'; import { BucketStorageFactory } from './BucketStorageFactory.js'; import { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js'; @@ -153,6 +154,17 @@ export interface SyncRulesBucketStorage * inside this replication stream. */ cleanupStoppedSyncConfigs?(options: CleanupStoppedSyncConfigsOptions): Promise; + + /** + * Per-bucket report of total operations vs total live rows in storage. + * + * Intended for an on-demand admin/diagnostics view (e.g. answering "why is my Data Synced so high"), + * not as a live gauge. Operation counts are read from pre-aggregated bucket state; live-row counts are + * derived from current stored rows. May be expensive on large instances. + * + * Optional: storage providers that don't implement it are reported as unsupported by the API. + */ + getBucketReport?(options?: GetBucketReportOptions): Promise; } export interface SyncRulesBucketStorageListener { diff --git a/packages/service-core/src/storage/bucket-report.ts b/packages/service-core/src/storage/bucket-report.ts new file mode 100644 index 000000000..5c3ae47be --- /dev/null +++ b/packages/service-core/src/storage/bucket-report.ts @@ -0,0 +1,117 @@ +/** + * Per-bucket storage report for an active sync config. + * + * Surfaces the same "total rows vs total operations" signal as the diagnostics app + * (https://github.com/powersync-ja/powersync-js/tree/main/tools/diagnostics-app), but + * measured server-side, per bucket, across the whole instance instead of per-client. + * + * - An **operation** is any entry in a bucket's append-only history (`PUT`, `REMOVE`, `MOVE`, `CLEAR`). + * - A **row** is a distinct live object currently in the bucket. + * + * A new client downloads every operation, not just live rows, so `operations / rows` is effectively a + * fragmentation / compaction-efficiency score: a fully compacted bucket trends towards ~1, while a high + * ratio is the usual cause of an unexpectedly high "Data Synced" metric and is reclaimable via compact/defragment. + */ + +export interface BucketOperationStat { + /** Total operations in the bucket's history (PUT/REMOVE/MOVE/CLEAR). */ + operations: number; + /** Approximate size of the operation history in bytes. */ + operationBytes: number; +} + +export interface BucketStorageStats { + /** Full bucket name, e.g. `by_user["u1"]`. */ + bucket: string; + /** Total operations in the bucket's history. */ + operations: number; + /** Distinct live rows currently in the bucket. */ + rows: number; + /** Approximate size of the operation history in bytes. */ + operationBytes: number; + /** + * `operations / max(rows, 1)`. ~1 is healthy (fully compacted); higher means more operation-history + * overhead that a compact/defragment can reclaim. + */ + fragmentation: number; +} + +export interface BucketReportTotals { + /** Total number of buckets in the active sync config (before any `limit`). */ + bucketCount: number; + /** Sum of operations across all buckets. */ + operations: number; + /** + * Sum of per-bucket live rows. Note this double-counts rows that belong to multiple buckets, + * so it is a sum of per-bucket counts rather than a distinct instance-wide row total. + */ + rows: number; + /** Sum of operation-history bytes across all buckets. */ + operationBytes: number; +} + +export interface BucketReport { + /** Per-bucket stats, ranked worst-first (most operations, then most fragmented). */ + buckets: BucketStorageStats[]; + /** Instance-wide totals, computed across all buckets even when `buckets` is truncated by `limit`. */ + totals: BucketReportTotals; + /** True if `buckets` was truncated by `limit`. `totals` still reflects all buckets. */ + truncated: boolean; +} + +export interface GetBucketReportOptions { + /** + * Maximum number of buckets to return, ranked by operation count descending (worst offenders first). + * Totals are still computed across all buckets. Defaults to no limit. + */ + limit?: number; +} + +/** + * Combine per-bucket operation stats and live-row counts (each keyed by full bucket name) into a + * ranked {@link BucketReport}. Backend storage adapters collect the two maps however is cheapest for + * them; this builder owns the shared merge/rank/total logic so the calculation never drifts between + * backends. + */ +export function buildBucketReport( + operationStats: Map, + rowCounts: Map, + options?: GetBucketReportOptions +): BucketReport { + const bucketNames = new Set([...operationStats.keys(), ...rowCounts.keys()]); + + const buckets: BucketStorageStats[] = []; + const totals: BucketReportTotals = { bucketCount: 0, operations: 0, rows: 0, operationBytes: 0 }; + + for (const bucket of bucketNames) { + const opStat = operationStats.get(bucket); + const operations = opStat?.operations ?? 0; + const operationBytes = opStat?.operationBytes ?? 0; + const rows = rowCounts.get(bucket) ?? 0; + + buckets.push({ + bucket, + operations, + rows, + operationBytes, + fragmentation: operations / Math.max(rows, 1) + }); + + totals.bucketCount += 1; + totals.operations += operations; + totals.rows += rows; + totals.operationBytes += operationBytes; + } + + // Worst-first: most operations, then most fragmented. + buckets.sort((a, b) => b.operations - a.operations || b.fragmentation - a.fragmentation); + + let truncated = false; + let reported = buckets; + if (options?.limit != null && buckets.length > options.limit) { + reported = buckets.slice(0, options.limit); + truncated = true; + } + + return { buckets: reported, totals, truncated }; +} diff --git a/packages/service-core/src/storage/storage-index.ts b/packages/service-core/src/storage/storage-index.ts index dc52b6988..9188bb226 100644 --- a/packages/service-core/src/storage/storage-index.ts +++ b/packages/service-core/src/storage/storage-index.ts @@ -1,4 +1,5 @@ export * from './bson.js'; +export * from './bucket-report.js'; export * from './BucketStorage.js'; export * from './BucketStorageBatch.js'; export * from './BucketStorageFactory.js'; diff --git a/packages/service-core/test/src/bucket-report.test.ts b/packages/service-core/test/src/bucket-report.test.ts new file mode 100644 index 000000000..45618f71a --- /dev/null +++ b/packages/service-core/test/src/bucket-report.test.ts @@ -0,0 +1,93 @@ +import { buildBucketReport, type BucketOperationStat } from '@/storage/bucket-report.js'; +import { describe, expect, it } from 'vitest'; + +describe('buildBucketReport', () => { + const ops = (operations: number, operationBytes = 0): BucketOperationStat => ({ operations, operationBytes }); + + it('merges operation stats and row counts per bucket and derives fragmentation', () => { + const report = buildBucketReport( + new Map([ + ['global[]', ops(100, 1024)], + ['by_user["u1"]', ops(10, 256)] + ]), + new Map([ + ['global[]', 10], + ['by_user["u1"]', 10] + ]) + ); + + const global = report.buckets.find((b) => b.bucket === 'global[]')!; + expect(global).toMatchObject({ + operations: 100, + rows: 10, + operationBytes: 1024, + fragmentation: 10 + }); + + const byUser = report.buckets.find((b) => b.bucket === 'by_user["u1"]')!; + expect(byUser.fragmentation).toBe(1); + }); + + it('ranks buckets worst-first by operations', () => { + const report = buildBucketReport( + new Map([ + ['a[]', ops(5)], + ['b[]', ops(50)], + ['c[]', ops(20)] + ]), + new Map() + ); + + expect(report.buckets.map((b) => b.bucket)).toEqual(['b[]', 'c[]', 'a[]']); + }); + + it('treats a bucket with operations but no live rows as fully fragmented (rows floored at 1)', () => { + const report = buildBucketReport(new Map([['gone[]', ops(42)]]), new Map()); + + expect(report.buckets[0]).toMatchObject({ operations: 42, rows: 0, fragmentation: 42 }); + }); + + it('includes buckets that have rows but no recorded operations', () => { + const report = buildBucketReport(new Map(), new Map([['fresh[]', 7]])); + + expect(report.buckets[0]).toMatchObject({ bucket: 'fresh[]', operations: 0, rows: 7, fragmentation: 0 }); + }); + + it('computes instance-wide totals across all buckets', () => { + const report = buildBucketReport( + new Map([ + ['a[]', ops(100, 10)], + ['b[]', ops(20, 5)] + ]), + new Map([ + ['a[]', 4], + ['b[]', 2] + ]) + ); + + expect(report.totals).toEqual({ bucketCount: 2, operations: 120, rows: 6, operationBytes: 15 }); + }); + + it('truncates the bucket list by limit but keeps totals across all buckets', () => { + const report = buildBucketReport( + new Map([ + ['a[]', ops(100)], + ['b[]', ops(50)], + ['c[]', ops(10)] + ]), + new Map(), + { limit: 2 } + ); + + expect(report.truncated).toBe(true); + expect(report.buckets.map((b) => b.bucket)).toEqual(['a[]', 'b[]']); + expect(report.totals).toMatchObject({ bucketCount: 3, operations: 160 }); + }); + + it('is not truncated when the limit exceeds the bucket count', () => { + const report = buildBucketReport(new Map([['a[]', ops(1)]]), new Map(), { limit: 10 }); + + expect(report.truncated).toBe(false); + expect(report.buckets).toHaveLength(1); + }); +}); From 8b3a0a3b98f6974c2b40a705aace3242521d7e51 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Tue, 23 Jun 2026 16:12:27 +0200 Subject: [PATCH 02/40] Implement bucket report for MongoDB storage --- .../implementation/MongoSyncBucketStorage.ts | 90 ++++++++++++++++++- .../v1/MongoSyncBucketStorageV1.ts | 9 ++ .../v3/MongoSyncBucketStorageV3.ts | 10 +++ 3 files changed, 108 insertions(+), 1 deletion(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 2538eb878..af819875c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -29,7 +29,7 @@ import * as timers from 'timers/promises'; import { retryOnMongoMaxTimeMSExpired } from '../../utils/util.js'; import { MongoBucketStorage } from '../MongoBucketStorage.js'; import type { VersionedPowerSyncMongo } from './db.js'; -import { StorageConfig } from './models.js'; +import { BucketStateDocumentBase, StorageConfig } from './models.js'; import { MongoBucketBatchOptions } from './MongoBucketBatch.js'; import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js'; import { MongoCompactOptions, MongoCompactor } from './MongoCompactor.js'; @@ -361,6 +361,94 @@ export abstract class MongoSyncBucketStorage } } + async getBucketReport(options?: storage.GetBucketReportOptions): Promise { + const [operationStats, rowCounts] = await Promise.all([ + this.collectBucketOperationStats(), + this.collectBucketLiveRowCounts() + ]); + return storage.buildBucketReport(operationStats, rowCounts, options); + } + + /** + * Operation count and operation-history bytes per bucket, read from the pre-aggregated bucket state + * (compacted_state + estimate_since_compact). Cheap: one document per bucket, no scan of bucket data. + * + * Implementations supply their version-specific bucket state collection(s) to {@link aggregateBucketOperationStats}. + */ + protected abstract collectBucketOperationStats(): Promise>; + + /** + * Distinct live rows per bucket, derived from the current stored rows and their bucket memberships. + * + * Implementations supply their version-specific current-row collection(s) to {@link aggregateBucketLiveRowCounts}. + */ + protected abstract collectBucketLiveRowCounts(): Promise>; + + /** + * Aggregate operation count and operation-history bytes per bucket from a bucket state collection. + * + * Operations are pre-aggregated in bucket state, so this reads a single document per bucket rather than + * scanning bucket data. Shared by the storage versions, which differ only in which collection (and filter) + * holds their bucket state. + */ + protected async aggregateBucketOperationStats( + collection: mongo.Collection, + match?: mongo.Filter + ): Promise> { + const pipeline: mongo.Document[] = []; + if (match != null) { + pipeline.push({ $match: match }); + } + pipeline.push({ + $project: { + _id: 1, + operations: { + $add: [{ $ifNull: ['$compacted_state.count', 0] }, { $ifNull: ['$estimate_since_compact.count', 0] }] + }, + operationBytes: { + $add: [ + { $toDouble: { $ifNull: ['$compacted_state.bytes', 0] } }, + { $toDouble: { $ifNull: ['$estimate_since_compact.bytes', 0] } } + ] + } + } + }); + + const result = new Map(); + const cursor = collection.aggregate<{ _id: { b: string }; operations: number; operationBytes: number }>(pipeline); + for await (const doc of cursor.stream()) { + result.set(doc._id.b, { operations: doc.operations, operationBytes: doc.operationBytes }); + } + return result; + } + + /** + * Aggregate distinct live rows per bucket across one or more current-row collections. + * + * Each stored row records its bucket memberships, so unwinding those memberships and grouping by bucket gives + * the distinct live row count. Counts are summed across collections, since a bucket may contain rows from + * multiple source tables (each in its own collection). + */ + protected async aggregateBucketLiveRowCounts( + collections: mongo.Collection[], + match?: mongo.Filter + ): Promise> { + const pipeline: mongo.Document[] = []; + if (match != null) { + pipeline.push({ $match: match }); + } + pipeline.push({ $unwind: '$buckets' }, { $group: { _id: '$buckets.bucket', count: { $sum: 1 } } }); + + const result = new Map(); + for (const collection of collections) { + const cursor = collection.aggregate<{ _id: string; count: number }>(pipeline, { allowDiskUse: true }); + for await (const doc of cursor.stream()) { + result.set(doc._id, (result.get(doc._id) ?? 0) + doc.count); + } + } + return result; + } + /** * The highest op id persisted for this stream, whether or not covered by a checkpoint. * diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index f4292930d..c9cabf53f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -180,6 +180,15 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage { return new MongoCompactorV1(this, this.db, options); } + // For storage v1/v2, bucket state and current rows are shared collections scoped by group (replication stream). + protected collectBucketOperationStats(): Promise> { + return this.aggregateBucketOperationStats(this.db.bucketStateV1, { '_id.g': this.replicationStreamId }); + } + + protected collectBucketLiveRowCounts(): Promise> { + return this.aggregateBucketLiveRowCounts([this.db.sourceRecordsV1], { '_id.g': this.replicationStreamId }); + } + protected createMongoParameterCompactor( checkpoint: InternalOpId, options: storage.CompactOptions diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index e38973258..dd5269c88 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -180,6 +180,16 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { return new MongoCompactorV3(this, this.db, options); } + // For storage v3, bucket state is a per-stream collection and current rows are split into per-table collections. + protected collectBucketOperationStats(): Promise> { + return this.aggregateBucketOperationStats(this.db.bucketState(this.replicationStreamId)); + } + + protected async collectBucketLiveRowCounts(): Promise> { + const collections = await this.db.listSourceRecordCollections(this.replicationStreamId); + return this.aggregateBucketLiveRowCounts(collections); + } + protected createMongoParameterCompactor( checkpoint: InternalOpId, options: storage.CompactOptions From 0c894948881f3edf7672939f355b5fc956682fe2 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Tue, 23 Jun 2026 16:12:41 +0200 Subject: [PATCH 03/40] Implement bucket report for Postgres storage --- .../src/storage/PostgresSyncRulesStorage.ts | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index 60a355fbd..44201060a 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -151,6 +151,55 @@ export class PostgresSyncRulesStorage }).compact(); } + async getBucketReport(options?: storage.GetBucketReportOptions): Promise { + // Operations + operation-history bytes per bucket. + const operationRows = await this.db.sql` + SELECT + bucket_name, + COUNT(*)::BIGINT AS operations, + COALESCE(SUM(OCTET_LENGTH(data)), 0)::BIGINT AS operation_bytes + FROM + bucket_data + WHERE + group_id = ${{ type: 'int4', value: this.replicationStreamId }} + GROUP BY + bucket_name; + `.rows<{ bucket_name: string; operations: bigint; operation_bytes: bigint }>(); + + const operationStats = new Map( + operationRows.map((row) => [ + row.bucket_name, + { operations: Number(row.operations), operationBytes: Number(row.operation_bytes) } + ]) + ); + + // Distinct live rows per bucket, from each row's bucket memberships. The current-data table is + // version-specific (current_data for v1/v2, v3_current_data for v3), so the table name is interpolated + // from the resolved store rather than parameterised. + const rowCounts = new Map(); + for await (const batch of this.db.streamRows<{ bucket: string; rows: bigint }>({ + statement: ` + SELECT + elem ->> 'bucket' AS bucket, + COUNT(*)::BIGINT AS rows + FROM + ${this.currentDataStore.table} cd, + jsonb_array_elements(cd.buckets) AS elem + WHERE + cd.group_id = $1 + GROUP BY + elem ->> 'bucket' + `, + params: [{ type: 'int4', value: this.replicationStreamId }] + })) { + for (const row of batch) { + rowCounts.set(row.bucket, Number(row.rows)); + } + } + + return storage.buildBucketReport(operationStats, rowCounts, options); + } + async populatePersistentChecksumCache(_options: PopulateChecksumCacheOptions): Promise { // no-op - checksum cache is not implemented for Postgres yet return { buckets: 0 }; From 4e8298c9cc4226ab2e50c8f288e631549c107234 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Tue, 23 Jun 2026 16:13:02 +0200 Subject: [PATCH 04/40] Add bucket-report admin endpoint --- .../src/routes/endpoints/admin.ts | 60 ++++++++++++++++++- packages/types/src/routes.ts | 42 +++++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) diff --git a/packages/service-core/src/routes/endpoints/admin.ts b/packages/service-core/src/routes/endpoints/admin.ts index 596e33fda..56a64e6fe 100644 --- a/packages/service-core/src/routes/endpoints/admin.ts +++ b/packages/service-core/src/routes/endpoints/admin.ts @@ -267,4 +267,62 @@ export const validate = routeDefinition({ } }); -export const ADMIN_ROUTES = [executeSql, diagnostics, getSchema, reprocess, validate]; +/** + * Per-bucket report of total operations vs total live rows in storage, for the active sync config. + * + * Answers the recurring "why is my Data Synced so high" question instance-wide (not per-user like the + * diagnostics client): a high `operations / rows` ratio indicates fragmented buckets that a compact or + * defragment can reclaim. + */ +export const bucketReport = routeDefinition({ + path: '/api/admin/v1/bucket-report', + method: router.HTTPMethod.POST, + authorize: authApi, + validator: schema.createTsCodecValidator(internal_routes.BucketReportRequest, { allowAdditional: true }), + handler: async (payload) => { + const { + context: { service_context } + } = payload; + const { + storageEngine: { activeBucketStorage } + } = service_context; + + const active = await activeBucketStorage.getActiveSyncConfig(); + if (active == null) { + throw new errors.ServiceError({ + status: 422, + code: ErrorCode.PSYNC_S4104, + description: 'No active sync config' + }); + } + + if (active.storage.getBucketReport == null) { + throw new errors.ServiceError({ + status: 422, + code: ErrorCode.PSYNC_S2001, + description: 'The configured storage provider does not support bucket reporting' + }); + } + + const report = await active.storage.getBucketReport({ limit: payload.params.limit }); + + return internal_routes.BucketReportResponse.encode({ + buckets: report.buckets.map((bucket) => ({ + bucket: bucket.bucket, + operations: bucket.operations, + rows: bucket.rows, + operation_bytes: bucket.operationBytes, + fragmentation: bucket.fragmentation + })), + totals: { + bucket_count: report.totals.bucketCount, + operations: report.totals.operations, + rows: report.totals.rows, + operation_bytes: report.totals.operationBytes + }, + truncated: report.truncated + }); + } +}); + +export const ADMIN_ROUTES = [executeSql, diagnostics, getSchema, reprocess, validate, bucketReport]; diff --git a/packages/types/src/routes.ts b/packages/types/src/routes.ts index 5177e3c77..e9ad58145 100644 --- a/packages/types/src/routes.ts +++ b/packages/types/src/routes.ts @@ -77,3 +77,45 @@ export type ValidateRequest = t.Encoded; export const ValidateResponse = SyncRulesStatus; export type ValidateResponse = t.Encoded; + +export const BucketReportRequest = t.object({ + /** + * Maximum number of buckets to return, ranked by operation count descending (worst offenders first). + * Totals are still computed across all buckets. Omit for no limit. + */ + limit: t.number.optional() +}); +export type BucketReportRequest = t.Encoded; + +export const BucketStorageStats = t.object({ + /** Full bucket name, e.g. `by_user["u1"]`. */ + bucket: t.string, + /** Total operations in the bucket's history (PUT/REMOVE/MOVE/CLEAR). */ + operations: t.number, + /** Distinct live rows currently in the bucket. */ + rows: t.number, + /** Approximate size of the operation history in bytes. */ + operation_bytes: t.number, + /** + * `operations / max(rows, 1)`. ~1 is healthy (fully compacted); higher means more operation-history + * overhead that a compact/defragment can reclaim. + */ + fragmentation: t.number +}); +export type BucketStorageStats = t.Encoded; + +export const BucketReportResponse = t.object({ + /** Per-bucket stats, ranked worst-first (most operations, then most fragmented). */ + buckets: t.array(BucketStorageStats), + totals: t.object({ + /** Total number of buckets in the active sync config (before any `limit`). */ + bucket_count: t.number, + operations: t.number, + /** Sum of per-bucket live rows. Rows in multiple buckets are counted per bucket. */ + rows: t.number, + operation_bytes: t.number + }), + /** True if `buckets` was truncated by `limit`. `totals` still reflects all buckets. */ + truncated: t.boolean +}); +export type BucketReportResponse = t.Encoded; From 73f5ed6732c74243f0b74eb5b65db56f5ee99c88 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Tue, 23 Jun 2026 16:13:18 +0200 Subject: [PATCH 05/40] Add bucket report tests and changeset --- .changeset/bucket-storage-report.md | 9 + .../test/src/storage.test.ts | 7 + .../test/src/storage.test.ts | 7 + .../src/tests/register-bucket-report-tests.ts | 202 ++++++++++++++++++ .../src/tests/tests-index.ts | 1 + 5 files changed, 226 insertions(+) create mode 100644 .changeset/bucket-storage-report.md create mode 100644 packages/service-core-tests/src/tests/register-bucket-report-tests.ts diff --git a/.changeset/bucket-storage-report.md b/.changeset/bucket-storage-report.md new file mode 100644 index 000000000..66a076d6f --- /dev/null +++ b/.changeset/bucket-storage-report.md @@ -0,0 +1,9 @@ +--- +'@powersync/service-core': minor +'@powersync/service-types': minor +'@powersync/service-module-mongodb-storage': minor +'@powersync/service-module-postgres-storage': minor +'@powersync/service-core-tests': minor +--- + +Add a `POST /api/admin/v1/bucket-report` admin endpoint reporting operations vs rows per bucket. diff --git a/modules/module-mongodb-storage/test/src/storage.test.ts b/modules/module-mongodb-storage/test/src/storage.test.ts index 21f0b3be5..7d94836a8 100644 --- a/modules/module-mongodb-storage/test/src/storage.test.ts +++ b/modules/module-mongodb-storage/test/src/storage.test.ts @@ -17,6 +17,13 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { describe(`Mongo Sync Bucket Storage - Checkpoints - v${storageVersion}`, () => register.registerDataStorageCheckpointTests({ ...INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion })); + + describe(`Mongo Sync Bucket Storage - Bucket report - v${storageVersion}`, () => + register.registerBucketReportTests({ + ...INITIALIZED_MONGO_STORAGE_FACTORY, + storageVersion, + compressedBucketStorage: storageVersion >= 3 + })); } describe('Sync Bucket Validation', register.registerBucketValidationTests); diff --git a/modules/module-postgres-storage/test/src/storage.test.ts b/modules/module-postgres-storage/test/src/storage.test.ts index 098810576..31596a83b 100644 --- a/modules/module-postgres-storage/test/src/storage.test.ts +++ b/modules/module-postgres-storage/test/src/storage.test.ts @@ -19,6 +19,13 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { describe(`Postgres Sync Bucket Storage - Checkpoints - v${storageVersion}`, () => register.registerDataStorageCheckpointTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion })); + describe(`Postgres Sync Bucket Storage - Bucket report - v${storageVersion}`, () => + register.registerBucketReportTests({ + ...POSTGRES_STORAGE_FACTORY, + storageVersion, + compressedBucketStorage: false + })); + describe(`Postgres Sync Bucket Storage - pg-specific - v${storageVersion}`, () => { /** * The split of returned results can vary depending on storage drivers. diff --git a/packages/service-core-tests/src/tests/register-bucket-report-tests.ts b/packages/service-core-tests/src/tests/register-bucket-report-tests.ts new file mode 100644 index 000000000..d96c83633 --- /dev/null +++ b/packages/service-core-tests/src/tests/register-bucket-report-tests.ts @@ -0,0 +1,202 @@ +import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; +import { expect, test } from 'vitest'; +import * as test_utils from '../test-utils/test-utils-index.js'; + +/** + * Tests for {@link storage.SyncRulesBucketStorage.getBucketReport}: per-bucket operations vs live rows. + * + * Asserts on stable counts (operations, rows, fragmentation, totals) rather than op_ids or checksums, + * which differ between storage backends and versions. + */ +export function registerBucketReportTests(config: storage.TestStorageConfig) { + const generateStorageFactory = config.factory; + const storageVersion = config.storageVersion ?? storage.CURRENT_STORAGE_VERSION; + + const GLOBAL_SYNC_RULES = ` +bucket_definitions: + global: + data: [select * from test] +`; + + // A constant parameter query keeps op_ids stable across backends (no bucket_parameter records); the data + // query routes each row into a bucket keyed by its own `b` value, so rows land in grouped["b1"]/grouped["b2"]. + const GROUPED_SYNC_RULES = ` bucket_definitions: + grouped: + parameters: select 'b' as b + data: + - select * from test where b = bucket.b`; + + const getReport = (bucketStorage: storage.SyncRulesBucketStorage, options?: storage.GetBucketReportOptions) => { + if (bucketStorage.getBucketReport == null) { + throw new Error('Storage backend does not implement getBucketReport'); + } + return bucketStorage.getBucketReport(options); + }; + + test('reports operations and live rows for a single bucket', async () => { + await using factory = await generateStorageFactory(); + const { stream, content } = await test_utils.deploySyncRules( + factory, + updateSyncRulesFromYaml(GLOBAL_SYNC_RULES, { storageVersion }) + ); + const bucketStorage = factory.getInstance(stream); + + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + for (const id of ['t1', 't2', 't3']) { + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { id }, + afterReplicaId: test_utils.rid(id) + }); + } + await writer.commit('1/1'); + await writer.flush(); + + const bucket = test_utils.bucketRequest(content, 'global[]').bucket; + const report = await getReport(bucketStorage); + + expect(report.totals.bucketCount).toEqual(1); + expect(report.truncated).toEqual(false); + + const stats = report.buckets.find((b) => b.bucket === bucket)!; + // Three inserts of distinct ids: three operations, three live rows, fully compacted (ratio 1). + expect(stats).toMatchObject({ operations: 3, rows: 3, fragmentation: 1 }); + expect(stats.operationBytes).toBeGreaterThan(0); + expect(report.totals).toMatchObject({ operations: 3, rows: 3 }); + }); + + test('operations exceed live rows after updates, and compaction reduces fragmentation', async () => { + await using factory = await generateStorageFactory(); + const { stream, content } = await test_utils.deploySyncRules( + factory, + updateSyncRulesFromYaml(GLOBAL_SYNC_RULES, { storageVersion }) + ); + const bucketStorage = factory.getInstance(stream); + + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + // Two rows, each inserted then updated twice: six operations over two live rows. + for (const id of ['t1', 't2']) { + for (const value of ['a', 'b', 'c']) { + await writer.save({ + sourceTable: testTable, + tag: value === 'a' ? storage.SaveOperationTag.INSERT : storage.SaveOperationTag.UPDATE, + after: { id, value }, + afterReplicaId: test_utils.rid(id) + }); + } + } + await writer.commit('1/1'); + await writer.flush(); + + const bucket = test_utils.bucketRequest(content, 'global[]').bucket; + + const before = await getReport(bucketStorage); + const beforeStats = before.buckets.find((b) => b.bucket === bucket)!; + expect(beforeStats).toMatchObject({ operations: 6, rows: 2, fragmentation: 3 }); + + await bucketStorage.compact({ + clearBatchLimit: 10, + moveBatchLimit: 10, + moveBatchQueryLimit: 10, + minBucketChanges: 1, + minChangeRatio: 0 + }); + + const after = await getReport(bucketStorage); + const afterStats = after.buckets.find((b) => b.bucket === bucket)!; + // Live rows are unchanged; the operation history shrinks toward the live row count. + expect(afterStats.rows).toEqual(2); + expect(afterStats.operations).toBeLessThan(beforeStats.operations); + expect(afterStats.operations).toBeGreaterThanOrEqual(afterStats.rows); + expect(afterStats.fragmentation).toBeLessThan(beforeStats.fragmentation); + }); + + test('reports every bucket, ranks worst-first, and totals across all buckets', async () => { + await using factory = await generateStorageFactory(); + const { stream, content } = await test_utils.deploySyncRules( + factory, + updateSyncRulesFromYaml(GROUPED_SYNC_RULES, { storageVersion }) + ); + const bucketStorage = factory.getInstance(stream); + + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + // grouped["b1"]: one row, three operations (insert + two updates). + for (const value of ['a', 'b', 'c']) { + await writer.save({ + sourceTable: testTable, + tag: value === 'a' ? storage.SaveOperationTag.INSERT : storage.SaveOperationTag.UPDATE, + after: { id: 't1', b: 'b1', value }, + afterReplicaId: test_utils.rid('t1') + }); + } + // grouped["b2"]: two rows, two operations. + for (const id of ['t2', 't3']) { + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { id, b: 'b2' }, + afterReplicaId: test_utils.rid(id) + }); + } + await writer.commit('1/1'); + await writer.flush(); + + const b1 = test_utils.bucketRequest(content, 'grouped["b1"]').bucket; + const b2 = test_utils.bucketRequest(content, 'grouped["b2"]').bucket; + + const report = await getReport(bucketStorage); + expect(report.totals.bucketCount).toEqual(2); + expect(report.totals).toMatchObject({ operations: 5, rows: 3 }); + + // Ranked worst-first by operation count: b1 (3) before b2 (2). + expect(report.buckets.map((b) => b.bucket)).toEqual([b1, b2]); + expect(report.buckets.find((b) => b.bucket === b1)).toMatchObject({ operations: 3, rows: 1 }); + expect(report.buckets.find((b) => b.bucket === b2)).toMatchObject({ operations: 2, rows: 2 }); + }); + + test('limit truncates the bucket list but totals still span all buckets', async () => { + await using factory = await generateStorageFactory(); + const { stream, content } = await test_utils.deploySyncRules( + factory, + updateSyncRulesFromYaml(GROUPED_SYNC_RULES, { storageVersion }) + ); + const bucketStorage = factory.getInstance(stream); + + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + // grouped["b1"]: two operations; grouped["b2"]: one operation. + for (const value of ['a', 'b']) { + await writer.save({ + sourceTable: testTable, + tag: value === 'a' ? storage.SaveOperationTag.INSERT : storage.SaveOperationTag.UPDATE, + after: { id: 't1', b: 'b1', value }, + afterReplicaId: test_utils.rid('t1') + }); + } + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { id: 't2', b: 'b2' }, + afterReplicaId: test_utils.rid('t2') + }); + await writer.commit('1/1'); + await writer.flush(); + + const b1 = test_utils.bucketRequest(content, 'grouped["b1"]').bucket; + + const report = await getReport(bucketStorage, { limit: 1 }); + expect(report.truncated).toEqual(true); + expect(report.buckets.map((b) => b.bucket)).toEqual([b1]); + // Totals still cover every bucket, not just the truncated list. + expect(report.totals.bucketCount).toEqual(2); + expect(report.totals).toMatchObject({ operations: 3, rows: 2 }); + }); +} diff --git a/packages/service-core-tests/src/tests/tests-index.ts b/packages/service-core-tests/src/tests/tests-index.ts index a40468a32..5be5e48f1 100644 --- a/packages/service-core-tests/src/tests/tests-index.ts +++ b/packages/service-core-tests/src/tests/tests-index.ts @@ -1,3 +1,4 @@ +export * from './register-bucket-report-tests.js'; export * from './register-bucket-validation-tests.js'; export * from './register-compacting-tests.js'; export * from './register-data-storage-checkpoint-tests.js'; From 386447017a8ff7a831d7ba59aedee7032420abda Mon Sep 17 00:00:00 2001 From: bean1352 Date: Wed, 24 Jun 2026 13:50:32 +0200 Subject: [PATCH 06/40] Add bucket report query timeout constant --- packages/service-core/src/storage/bucket-report.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/service-core/src/storage/bucket-report.ts b/packages/service-core/src/storage/bucket-report.ts index 5c3ae47be..f4d4b910e 100644 --- a/packages/service-core/src/storage/bucket-report.ts +++ b/packages/service-core/src/storage/bucket-report.ts @@ -13,6 +13,13 @@ * ratio is the usual cause of an unexpectedly high "Data Synced" metric and is reclaimable via compact/defragment. */ +/** + * Time budget for the per-bucket report's storage aggregations (Mongo `maxTimeMS`, Postgres + * `statement_timeout`). The report scans current rows (and, on Postgres, all bucket data), which can be + * expensive on large instances, so the queries are bounded rather than allowed to run unbounded. + */ +export const BUCKET_REPORT_TIMEOUT_MS: number = 60_000; + export interface BucketOperationStat { /** Total operations in the bucket's history (PUT/REMOVE/MOVE/CLEAR). */ operations: number; From eb9b614ac79e845f5d4ed9e8c1159e80e58b07b8 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Wed, 24 Jun 2026 13:50:47 +0200 Subject: [PATCH 07/40] Scope v3 bucket report to active config and bound Mongo queries --- .../implementation/MongoSyncBucketStorage.ts | 31 +++-- .../v1/MongoSyncBucketStorageV1.ts | 5 +- .../v3/MongoSyncBucketStorageV3.ts | 11 +- .../test/src/bucket-report-scoping.test.ts | 113 ++++++++++++++++++ 4 files changed, 150 insertions(+), 10 deletions(-) create mode 100644 modules/module-mongodb-storage/test/src/bucket-report-scoping.test.ts diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index af819875c..bc4440f55 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -22,7 +22,12 @@ import { utils, WatchWriteCheckpointOptions } from '@powersync/service-core'; -import { HydratedSyncConfig, ParameterLookupRows, ScopedParameterLookup } from '@powersync/service-sync-rules'; +import { + BucketDefinitionId, + HydratedSyncConfig, + ParameterLookupRows, + ScopedParameterLookup +} from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { LRUCache } from 'lru-cache'; import * as timers from 'timers/promises'; @@ -415,7 +420,9 @@ export abstract class MongoSyncBucketStorage }); const result = new Map(); - const cursor = collection.aggregate<{ _id: { b: string }; operations: number; operationBytes: number }>(pipeline); + const cursor = collection.aggregate<{ _id: { b: string }; operations: number; operationBytes: number }>(pipeline, { + maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS + }); for await (const doc of cursor.stream()) { result.set(doc._id.b, { operations: doc.operations, operationBytes: doc.operationBytes }); } @@ -428,20 +435,30 @@ export abstract class MongoSyncBucketStorage * Each stored row records its bucket memberships, so unwinding those memberships and grouping by bucket gives * the distinct live row count. Counts are summed across collections, since a bucket may contain rows from * multiple source tables (each in its own collection). + * + * `options.bucketDefinitionIds` restricts to memberships of those definitions - used by V3 to exclude rows + * still tagged with stopped/old definitions that share the stream but are not part of the active config. */ protected async aggregateBucketLiveRowCounts( collections: mongo.Collection[], - match?: mongo.Filter + options?: { match?: mongo.Filter; bucketDefinitionIds?: BucketDefinitionId[] } ): Promise> { const pipeline: mongo.Document[] = []; - if (match != null) { - pipeline.push({ $match: match }); + if (options?.match != null) { + pipeline.push({ $match: options.match }); + } + pipeline.push({ $unwind: '$buckets' }); + if (options?.bucketDefinitionIds != null) { + pipeline.push({ $match: { 'buckets.def': { $in: options.bucketDefinitionIds } } }); } - pipeline.push({ $unwind: '$buckets' }, { $group: { _id: '$buckets.bucket', count: { $sum: 1 } } }); + pipeline.push({ $group: { _id: '$buckets.bucket', count: { $sum: 1 } } }); const result = new Map(); for (const collection of collections) { - const cursor = collection.aggregate<{ _id: string; count: number }>(pipeline, { allowDiskUse: true }); + const cursor = collection.aggregate<{ _id: string; count: number }>(pipeline, { + allowDiskUse: true, + maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS + }); for await (const doc of cursor.stream()) { result.set(doc._id, (result.get(doc._id) ?? 0) + doc.count); } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index c9cabf53f..2873b1232 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -186,7 +186,10 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage { } protected collectBucketLiveRowCounts(): Promise> { - return this.aggregateBucketLiveRowCounts([this.db.sourceRecordsV1], { '_id.g': this.replicationStreamId }); + // V1/V2 have a single sync config per replication stream, so scoping by group is sufficient. + return this.aggregateBucketLiveRowCounts([this.db.sourceRecordsV1], { + match: { '_id.g': this.replicationStreamId } + }); } protected createMongoParameterCompactor( diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index dd5269c88..0f3e4e954 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -181,13 +181,20 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { } // For storage v3, bucket state is a per-stream collection and current rows are split into per-table collections. + // A replication stream can host multiple sync configs (active + processing + stopped, until cleanup runs), all + // sharing these collections. Scope to the active config's definition ids so the report excludes stale buckets + // from old/stopped definitions. `this.storageIds` is derived from the active config only (see getActiveSyncConfig). protected collectBucketOperationStats(): Promise> { - return this.aggregateBucketOperationStats(this.db.bucketState(this.replicationStreamId)); + return this.aggregateBucketOperationStats(this.db.bucketState(this.replicationStreamId), { + '_id.d': { $in: this.storageIds.bucketDefinitionIds } + }); } protected async collectBucketLiveRowCounts(): Promise> { const collections = await this.db.listSourceRecordCollections(this.replicationStreamId); - return this.aggregateBucketLiveRowCounts(collections); + return this.aggregateBucketLiveRowCounts(collections, { + bucketDefinitionIds: this.storageIds.bucketDefinitionIds + }); } protected createMongoParameterCompactor( diff --git a/modules/module-mongodb-storage/test/src/bucket-report-scoping.test.ts b/modules/module-mongodb-storage/test/src/bucket-report-scoping.test.ts new file mode 100644 index 000000000..1dcd749d9 --- /dev/null +++ b/modules/module-mongodb-storage/test/src/bucket-report-scoping.test.ts @@ -0,0 +1,113 @@ +import { MongoSyncBucketStorageV3 } from '@module/storage/implementation/v3/MongoSyncBucketStorageV3.js'; +import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; +import { test_utils } from '@powersync/service-core-tests'; +import * as bson from 'bson'; +import { describe, expect, test } from 'vitest'; +import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; + +function sourceDescriptor(name: string, objectId: string): storage.SourceEntityDescriptor { + return { + connectionTag: storage.SourceTable.DEFAULT_TAG, + objectId, + schema: 'public', + name, + replicaIdColumns: [{ name: 'id', type: 'VARCHAR', typeId: 25 }] + }; +} + +function objectIdGenerator(id: string) { + let used = false; + return () => { + if (used) { + throw new Error(`Can only generate a single id using ${id}`); + } + used = true; + return new bson.ObjectId(id); + }; +} + +/** + * In V3 a replication stream can host multiple sync configs (active + stopped, until cleanup runs), all sharing + * the per-stream bucket_state and source_records collections. The report must only include the active config's + * bucket definitions, not stale ones from a previous (now stopped) config. + */ +describe('bucket report scoping - mongodb v3', () => { + test('excludes buckets from stopped/old sync configs sharing the replication stream', async () => { + await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); + + // Config 1: replicate todos. Writing a row creates a data bucket for this config. + const first = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +config: + edition: 3 + +streams: + by_owner: + query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id') +`, + { storageVersion: 3 } + ) + ); + const firstStorage = factory.getInstance(first) as MongoSyncBucketStorageV3; + await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); + const todosTable = ( + await firstWriter.resolveTables({ + connection_id: 1, + source: sourceDescriptor('todos', 'todos-relation'), + idGenerator: objectIdGenerator('6544e3899293153fa7b38360') + }) + ).tables[0]; + await firstWriter.save({ + sourceTable: todosTable, + tag: storage.SaveOperationTag.INSERT, + after: { id: 'todo-1', owner_id: 'user-1' }, + afterReplicaId: test_utils.rid('todo-1') + }); + await firstWriter.markAllSnapshotDone('1/1'); + await firstWriter.commit('1/1'); + await firstWriter.flush(); + + // While config 1 is active, its bucket(s) show up in the report. + const firstReport = await firstStorage.getBucketReport(); + expect(firstReport.totals.bucketCount).toBeGreaterThan(0); + + // Config 2: a different stream over a different table. Config 1 transitions to STOP, but its bucket_state + // and source_records rows remain in the shared collections until cleanup runs (which we deliberately skip). + const second = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +config: + edition: 3 + +streams: + by_project: + query: SELECT * FROM scenes WHERE project_id = subscription.parameter('project_id') +`, + { storageVersion: 3 } + ) + ); + expect(second.replicationStreamId).toBe(first.replicationStreamId); + + // Drive config 2 to snapshot-done so it becomes ACTIVE and config 1 transitions to STOP (config 1 keeps + // serving until the new config finishes processing). Config 1's stale rows remain until cleanup, which we skip. + const replicatingStreams = await factory.getReplicatingReplicationStreams(); + expect(replicatingStreams).toHaveLength(1); + const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorageV3; + await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS); + await secondWriter.markAllSnapshotDone('2/1'); + await secondWriter.commit('2/1'); + await secondWriter.flush(); + + const activeConfig = await factory.getActiveSyncConfig(); + expect(activeConfig).not.toBeNull(); + const activeStorage = activeConfig!.storage as MongoSyncBucketStorageV3; + const secondReport = await activeStorage.getBucketReport(); + + // Config 2 has no replicated data, and config 1's stale buckets must be excluded. Without scoping to the + // active config's definition ids, config 1's bucket would leak in here. + expect(secondReport.totals.bucketCount).toEqual(0); + const firstBucketNames = new Set(firstReport.buckets.map((b) => b.bucket)); + expect(secondReport.buckets.some((b) => firstBucketNames.has(b.bucket))).toBe(false); + }); +}); From 35aeb984ce9f2d8f4dddd2ee27c2a014a2da0419 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Wed, 24 Jun 2026 13:51:03 +0200 Subject: [PATCH 08/40] Bound Postgres bucket report with a statement timeout --- .../src/storage/PostgresSyncRulesStorage.ts | 84 ++++++++++--------- 1 file changed, 46 insertions(+), 38 deletions(-) diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index 44201060a..b159834b1 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -152,50 +152,58 @@ export class PostgresSyncRulesStorage } async getBucketReport(options?: storage.GetBucketReportOptions): Promise { - // Operations + operation-history bytes per bucket. - const operationRows = await this.db.sql` - SELECT - bucket_name, - COUNT(*)::BIGINT AS operations, - COALESCE(SUM(OCTET_LENGTH(data)), 0)::BIGINT AS operation_bytes - FROM - bucket_data - WHERE - group_id = ${{ type: 'int4', value: this.replicationStreamId }} - GROUP BY - bucket_name; - `.rows<{ bucket_name: string; operations: bigint; operation_bytes: bigint }>(); - - const operationStats = new Map( - operationRows.map((row) => [ - row.bucket_name, - { operations: Number(row.operations), operationBytes: Number(row.operation_bytes) } - ]) - ); + // Both queries scan storage (Postgres has no pre-aggregated bucket state), so they run in a transaction + // with a statement timeout rather than letting an admin request run unbounded on a large instance. + const { operationStats, rowCounts } = await this.db.transaction(async (db) => { + await db.query(`SET LOCAL statement_timeout = ${storage.BUCKET_REPORT_TIMEOUT_MS}`); - // Distinct live rows per bucket, from each row's bucket memberships. The current-data table is - // version-specific (current_data for v1/v2, v3_current_data for v3), so the table name is interpolated - // from the resolved store rather than parameterised. - const rowCounts = new Map(); - for await (const batch of this.db.streamRows<{ bucket: string; rows: bigint }>({ - statement: ` + // Operations + operation-history bytes per bucket. + const operationRows = await db.sql` SELECT - elem ->> 'bucket' AS bucket, - COUNT(*)::BIGINT AS rows + bucket_name, + COUNT(*)::BIGINT AS operations, + COALESCE(SUM(OCTET_LENGTH(data)), 0)::BIGINT AS operation_bytes FROM - ${this.currentDataStore.table} cd, - jsonb_array_elements(cd.buckets) AS elem + bucket_data WHERE - cd.group_id = $1 + group_id = ${{ type: 'int4', value: this.replicationStreamId }} GROUP BY - elem ->> 'bucket' - `, - params: [{ type: 'int4', value: this.replicationStreamId }] - })) { - for (const row of batch) { - rowCounts.set(row.bucket, Number(row.rows)); + bucket_name; + `.rows<{ bucket_name: string; operations: bigint; operation_bytes: bigint }>(); + + const operationStats = new Map( + operationRows.map((row) => [ + row.bucket_name, + { operations: Number(row.operations), operationBytes: Number(row.operation_bytes) } + ]) + ); + + // Distinct live rows per bucket, from each row's bucket memberships. The current-data table is + // version-specific (current_data for v1/v2, v3_current_data for v3), so the table name is interpolated + // from the resolved store rather than parameterised. + const rowCounts = new Map(); + for await (const batch of db.streamRows<{ bucket: string; rows: bigint }>({ + statement: ` + SELECT + elem ->> 'bucket' AS bucket, + COUNT(*)::BIGINT AS rows + FROM + ${this.currentDataStore.table} cd, + jsonb_array_elements(cd.buckets) AS elem + WHERE + cd.group_id = $1 + GROUP BY + elem ->> 'bucket' + `, + params: [{ type: 'int4', value: this.replicationStreamId }] + })) { + for (const row of batch) { + rowCounts.set(row.bucket, Number(row.rows)); + } } - } + + return { operationStats, rowCounts }; + }); return storage.buildBucketReport(operationStats, rowCounts, options); } From 9575b7edb734e0517f0959334329f2a97e2f7e08 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Wed, 24 Jun 2026 14:02:00 +0200 Subject: [PATCH 09/40] Return a friendly timeout error from the bucket report --- .../implementation/MongoSyncBucketStorage.ts | 15 +++++++++----- .../src/storage/PostgresSyncRulesStorage.ts | 20 +++++++++++++++++++ packages/service-errors/src/codes.ts | 8 +++++++- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index bc4440f55..bb3769b8a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -367,11 +367,16 @@ export abstract class MongoSyncBucketStorage } async getBucketReport(options?: storage.GetBucketReportOptions): Promise { - const [operationStats, rowCounts] = await Promise.all([ - this.collectBucketOperationStats(), - this.collectBucketLiveRowCounts() - ]); - return storage.buildBucketReport(operationStats, rowCounts, options); + try { + const [operationStats, rowCounts] = await Promise.all([ + this.collectBucketOperationStats(), + this.collectBucketLiveRowCounts() + ]); + return storage.buildBucketReport(operationStats, rowCounts, options); + } catch (e) { + // Translate a maxTimeMS expiry into a friendly "query timed out" error instead of a raw 500. + throw lib_mongo.mapQueryError(e, 'while building the bucket report'); + } } /** diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index b159834b1..30b05184a 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -38,6 +38,9 @@ import { PostgresCurrentDataStore } from './current-data-store.js'; import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js'; import { PostgresCompactor } from './PostgresCompactor.js'; +/** Postgres SQLSTATE raised when a statement is cancelled, e.g. by statement_timeout. */ +const POSTGRES_QUERY_CANCELED = '57014'; + export type PostgresSyncRulesStorageOptions = { factory: PostgresBucketStorageFactory; db: lib_postgres.DatabaseClient; @@ -152,6 +155,23 @@ export class PostgresSyncRulesStorage } async getBucketReport(options?: storage.GetBucketReportOptions): Promise { + try { + return await this.collectBucketReport(options); + } catch (e) { + // statement_timeout cancels the query with SQLSTATE 57014 (query_canceled). Translate it into a + // friendly "query timed out" error instead of a raw 500. + if (e?.cause?.code === POSTGRES_QUERY_CANCELED) { + throw new framework.DatabaseQueryError( + framework.ErrorCode.PSYNC_S2501, + 'Query timed out while building the bucket report', + e + ); + } + throw e; + } + } + + private async collectBucketReport(options?: storage.GetBucketReportOptions): Promise { // Both queries scan storage (Postgres has no pre-aggregated bucket state), so they run in a transaction // with a statement timeout rather than letting an admin request run unbounded on a large instance. const { operationStats, rowCounts } = await this.db.transaction(async (db) => { diff --git a/packages/service-errors/src/codes.ts b/packages/service-errors/src/codes.ts index e423dd25f..3306b89c3 100644 --- a/packages/service-errors/src/codes.ts +++ b/packages/service-errors/src/codes.ts @@ -492,7 +492,13 @@ export enum ErrorCode { */ PSYNC_S2404 = 'PSYNC_S2404', - // ## PSYNC_S23xx: Sync API errors - Postgres Storage + // ## PSYNC_S25xx: Sync API errors - Postgres Storage + + /** + * Query timed out. Could be due to a large query or a temporary load issue on the storage database. + * Retry the request. + */ + PSYNC_S2501 = 'PSYNC_S2501', // ## PSYNC_S3xxx: Service configuration issues From 550fa20fe752732c2fb924afba739d060a7c94f3 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Thu, 25 Jun 2026 11:01:34 +0200 Subject: [PATCH 10/40] Add instance-wide fragmentation to bucket report totals --- .../src/tests/register-bucket-report-tests.ts | 3 ++- packages/service-core/src/routes/endpoints/admin.ts | 3 ++- packages/service-core/src/storage/bucket-report.ts | 10 +++++++++- packages/service-core/test/src/bucket-report.test.ts | 3 ++- packages/types/src/routes.ts | 4 +++- 5 files changed, 18 insertions(+), 5 deletions(-) diff --git a/packages/service-core-tests/src/tests/register-bucket-report-tests.ts b/packages/service-core-tests/src/tests/register-bucket-report-tests.ts index d96c83633..9f9b6925c 100644 --- a/packages/service-core-tests/src/tests/register-bucket-report-tests.ts +++ b/packages/service-core-tests/src/tests/register-bucket-report-tests.ts @@ -153,7 +153,8 @@ bucket_definitions: const report = await getReport(bucketStorage); expect(report.totals.bucketCount).toEqual(2); - expect(report.totals).toMatchObject({ operations: 5, rows: 3 }); + // Instance-wide fragmentation is the row-weighted ratio 5/3, not the mean of the per-bucket ratios (3 and 1). + expect(report.totals).toMatchObject({ operations: 5, rows: 3, fragmentation: 5 / 3 }); // Ranked worst-first by operation count: b1 (3) before b2 (2). expect(report.buckets.map((b) => b.bucket)).toEqual([b1, b2]); diff --git a/packages/service-core/src/routes/endpoints/admin.ts b/packages/service-core/src/routes/endpoints/admin.ts index 8dec6bac7..5db39a64c 100644 --- a/packages/service-core/src/routes/endpoints/admin.ts +++ b/packages/service-core/src/routes/endpoints/admin.ts @@ -319,7 +319,8 @@ export const bucketReport = routeDefinition({ bucket_count: report.totals.bucketCount, operations: report.totals.operations, rows: report.totals.rows, - operation_bytes: report.totals.operationBytes + operation_bytes: report.totals.operationBytes, + fragmentation: report.totals.fragmentation }, truncated: report.truncated }); diff --git a/packages/service-core/src/storage/bucket-report.ts b/packages/service-core/src/storage/bucket-report.ts index f4d4b910e..a336142fc 100644 --- a/packages/service-core/src/storage/bucket-report.ts +++ b/packages/service-core/src/storage/bucket-report.ts @@ -55,6 +55,12 @@ export interface BucketReportTotals { rows: number; /** Sum of operation-history bytes across all buckets. */ operationBytes: number; + /** + * Instance-wide fragmentation: `operations / max(rows, 1)`, i.e. the row-weighted average of the + * per-bucket ratios. ~1 is healthy; a higher value means a new client downloads that many operations + * per live row across the whole instance, which is the headline cause of a high "Data Synced" metric. + */ + fragmentation: number; } export interface BucketReport { @@ -88,7 +94,7 @@ export function buildBucketReport( const bucketNames = new Set([...operationStats.keys(), ...rowCounts.keys()]); const buckets: BucketStorageStats[] = []; - const totals: BucketReportTotals = { bucketCount: 0, operations: 0, rows: 0, operationBytes: 0 }; + const totals: BucketReportTotals = { bucketCount: 0, operations: 0, rows: 0, operationBytes: 0, fragmentation: 0 }; for (const bucket of bucketNames) { const opStat = operationStats.get(bucket); @@ -110,6 +116,8 @@ export function buildBucketReport( totals.operationBytes += operationBytes; } + totals.fragmentation = totals.operations / Math.max(totals.rows, 1); + // Worst-first: most operations, then most fragmented. buckets.sort((a, b) => b.operations - a.operations || b.fragmentation - a.fragmentation); diff --git a/packages/service-core/test/src/bucket-report.test.ts b/packages/service-core/test/src/bucket-report.test.ts index 45618f71a..e2c7a0968 100644 --- a/packages/service-core/test/src/bucket-report.test.ts +++ b/packages/service-core/test/src/bucket-report.test.ts @@ -65,7 +65,8 @@ describe('buildBucketReport', () => { ]) ); - expect(report.totals).toEqual({ bucketCount: 2, operations: 120, rows: 6, operationBytes: 15 }); + // fragmentation is the row-weighted ratio 120/6 = 20, not the mean of the per-bucket ratios (25 and 10). + expect(report.totals).toEqual({ bucketCount: 2, operations: 120, rows: 6, operationBytes: 15, fragmentation: 20 }); }); it('truncates the bucket list by limit but keeps totals across all buckets', () => { diff --git a/packages/types/src/routes.ts b/packages/types/src/routes.ts index e9ad58145..29485a384 100644 --- a/packages/types/src/routes.ts +++ b/packages/types/src/routes.ts @@ -113,7 +113,9 @@ export const BucketReportResponse = t.object({ operations: t.number, /** Sum of per-bucket live rows. Rows in multiple buckets are counted per bucket. */ rows: t.number, - operation_bytes: t.number + operation_bytes: t.number, + /** Instance-wide `operations / max(rows, 1)`: operations a new client downloads per live row. */ + fragmentation: t.number }), /** True if `buckets` was truncated by `limit`. `totals` still reflects all buckets. */ truncated: t.boolean From 299a80a37271e418916301841c0f42eab872f354 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Thu, 25 Jun 2026 11:54:00 +0200 Subject: [PATCH 11/40] Clamp bucket report limit and narrow the Mongo timeout catch --- .../implementation/MongoSyncBucketStorage.ts | 21 ++++++++++----- .../src/storage/PostgresSyncRulesStorage.ts | 8 +++--- .../service-core/src/storage/bucket-report.ts | 26 ++++++++++--------- 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index bb3769b8a..e672b73ee 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -367,28 +367,35 @@ export abstract class MongoSyncBucketStorage } async getBucketReport(options?: storage.GetBucketReportOptions): Promise { + let operationStats: Map; + let rowCounts: Map; try { - const [operationStats, rowCounts] = await Promise.all([ + [operationStats, rowCounts] = await Promise.all([ this.collectBucketOperationStats(), this.collectBucketLiveRowCounts() ]); - return storage.buildBucketReport(operationStats, rowCounts, options); } catch (e) { - // Translate a maxTimeMS expiry into a friendly "query timed out" error instead of a raw 500. + // Translate a storage query timeout (maxTimeMS) into a specific, retryable error code rather than a + // generic internal error. Only the storage reads are wrapped; the pure-JS merge below cannot time out. throw lib_mongo.mapQueryError(e, 'while building the bucket report'); } + return storage.buildBucketReport(operationStats, rowCounts, options); } /** * Operation count and operation-history bytes per bucket, read from the pre-aggregated bucket state * (compacted_state + estimate_since_compact). Cheap: one document per bucket, no scan of bucket data. * + * Note: for v1/v2 storage, bucket_state is not backfilled (see models.ts: "only populated by new updates"). + * Buckets whose data predates bucket_state tracking, and which have not been updated or compacted since, + * have no document here and so report zero operations. v3 always has bucket_state. + * * Implementations supply their version-specific bucket state collection(s) to {@link aggregateBucketOperationStats}. */ protected abstract collectBucketOperationStats(): Promise>; /** - * Distinct live rows per bucket, derived from the current stored rows and their bucket memberships. + * Live rows per bucket, derived from the current stored rows and their bucket memberships. * * Implementations supply their version-specific current-row collection(s) to {@link aggregateBucketLiveRowCounts}. */ @@ -437,9 +444,9 @@ export abstract class MongoSyncBucketStorage /** * Aggregate distinct live rows per bucket across one or more current-row collections. * - * Each stored row records its bucket memberships, so unwinding those memberships and grouping by bucket gives - * the distinct live row count. Counts are summed across collections, since a bucket may contain rows from - * multiple source tables (each in its own collection). + * Each stored row records its bucket memberships, so unwinding those memberships and grouping by bucket counts + * the live rows in each bucket (one membership per stored row). Counts are summed across collections, since a + * bucket may contain rows from multiple source tables (each in its own collection). * * `options.bucketDefinitionIds` restricts to memberships of those definitions - used by V3 to exclude rows * still tagged with stopped/old definitions that share the stream but are not part of the active config. diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index 30b05184a..df85b729a 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -159,7 +159,7 @@ export class PostgresSyncRulesStorage return await this.collectBucketReport(options); } catch (e) { // statement_timeout cancels the query with SQLSTATE 57014 (query_canceled). Translate it into a - // friendly "query timed out" error instead of a raw 500. + // specific, retryable timeout code rather than a generic internal error. if (e?.cause?.code === POSTGRES_QUERY_CANCELED) { throw new framework.DatabaseQueryError( framework.ErrorCode.PSYNC_S2501, @@ -198,9 +198,9 @@ export class PostgresSyncRulesStorage ]) ); - // Distinct live rows per bucket, from each row's bucket memberships. The current-data table is - // version-specific (current_data for v1/v2, v3_current_data for v3), so the table name is interpolated - // from the resolved store rather than parameterised. + // Live rows per bucket (one membership per stored row), from each row's bucket memberships. The + // current-data table is version-specific (current_data for v1/v2, v3_current_data for v3), so the table + // name is interpolated from the resolved store rather than parameterised. const rowCounts = new Map(); for await (const batch of db.streamRows<{ bucket: string; rows: bigint }>({ statement: ` diff --git a/packages/service-core/src/storage/bucket-report.ts b/packages/service-core/src/storage/bucket-report.ts index a336142fc..62ec99080 100644 --- a/packages/service-core/src/storage/bucket-report.ts +++ b/packages/service-core/src/storage/bucket-report.ts @@ -1,10 +1,6 @@ /** * Per-bucket storage report for an active sync config. * - * Surfaces the same "total rows vs total operations" signal as the diagnostics app - * (https://github.com/powersync-ja/powersync-js/tree/main/tools/diagnostics-app), but - * measured server-side, per bucket, across the whole instance instead of per-client. - * * - An **operation** is any entry in a bucket's append-only history (`PUT`, `REMOVE`, `MOVE`, `CLEAR`). * - A **row** is a distinct live object currently in the bucket. * @@ -32,7 +28,7 @@ export interface BucketStorageStats { bucket: string; /** Total operations in the bucket's history. */ operations: number; - /** Distinct live rows currently in the bucket. */ + /** Live rows currently in the bucket. */ rows: number; /** Approximate size of the operation history in bytes. */ operationBytes: number; @@ -44,7 +40,7 @@ export interface BucketStorageStats { } export interface BucketReportTotals { - /** Total number of buckets in the active sync config (before any `limit`). */ + /** Number of buckets with stored operations or rows (before any `limit`). */ bucketCount: number; /** Sum of operations across all buckets. */ operations: number; @@ -75,7 +71,9 @@ export interface BucketReport { export interface GetBucketReportOptions { /** * Maximum number of buckets to return, ranked by operation count descending (worst offenders first). - * Totals are still computed across all buckets. Defaults to no limit. + * This caps the response size only: every backend still aggregates all buckets (and `totals` covers + * them all), so it is not a query-cost bound. Non-integer or negative values are floored and clamped + * to 0. Defaults to no limit. */ limit?: number; } @@ -83,8 +81,9 @@ export interface GetBucketReportOptions { /** * Combine per-bucket operation stats and live-row counts (each keyed by full bucket name) into a * ranked {@link BucketReport}. Backend storage adapters collect the two maps however is cheapest for - * them; this builder owns the shared merge/rank/total logic so the calculation never drifts between - * backends. + * them; this builder owns the shared merge/rank/total logic so that part stays identical across + * backends. The inputs are not identical: operation counts are exact on Postgres (a `COUNT(*)`) but a + * pre-aggregated estimate on MongoDB, so the same data can yield slightly different counts per backend. */ export function buildBucketReport( operationStats: Map, @@ -123,9 +122,12 @@ export function buildBucketReport( let truncated = false; let reported = buckets; - if (options?.limit != null && buckets.length > options.limit) { - reported = buckets.slice(0, options.limit); - truncated = true; + if (options?.limit != null) { + const limit = Math.max(0, Math.floor(options.limit)); + if (buckets.length > limit) { + reported = buckets.slice(0, limit); + truncated = true; + } } return { buckets: reported, totals, truncated }; From bda1d06130938b7e37a115cde41ef32a20810b72 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Thu, 25 Jun 2026 11:54:15 +0200 Subject: [PATCH 12/40] Clarify bucket report API docs and add service-errors changeset --- .changeset/bucket-storage-report.md | 1 + packages/service-core/src/routes/endpoints/admin.ts | 4 ++-- .../service-core/src/storage/SyncRulesBucketStorage.ts | 5 +++-- packages/service-errors/src/codes.ts | 4 ++-- packages/types/src/routes.ts | 7 ++++--- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.changeset/bucket-storage-report.md b/.changeset/bucket-storage-report.md index 66a076d6f..e6b79249d 100644 --- a/.changeset/bucket-storage-report.md +++ b/.changeset/bucket-storage-report.md @@ -4,6 +4,7 @@ '@powersync/service-module-mongodb-storage': minor '@powersync/service-module-postgres-storage': minor '@powersync/service-core-tests': minor +'@powersync/service-errors': patch --- Add a `POST /api/admin/v1/bucket-report` admin endpoint reporting operations vs rows per bucket. diff --git a/packages/service-core/src/routes/endpoints/admin.ts b/packages/service-core/src/routes/endpoints/admin.ts index 5db39a64c..b3b8ab18a 100644 --- a/packages/service-core/src/routes/endpoints/admin.ts +++ b/packages/service-core/src/routes/endpoints/admin.ts @@ -271,8 +271,8 @@ export const validate = routeDefinition({ /** * Per-bucket report of total operations vs total live rows in storage, for the active sync config. * - * Answers the recurring "why is my Data Synced so high" question instance-wide (not per-user like the - * diagnostics client): a high `operations / rows` ratio indicates fragmented buckets that a compact or + * Answers the recurring "why is my Data Synced so high" question instance-wide + * a high `operations / rows` ratio indicates fragmented buckets that a compact or * defragment can reclaim. */ export const bucketReport = routeDefinition({ diff --git a/packages/service-core/src/storage/SyncRulesBucketStorage.ts b/packages/service-core/src/storage/SyncRulesBucketStorage.ts index d20c2fc8b..b2d8110d8 100644 --- a/packages/service-core/src/storage/SyncRulesBucketStorage.ts +++ b/packages/service-core/src/storage/SyncRulesBucketStorage.ts @@ -159,8 +159,9 @@ export interface SyncRulesBucketStorage * Per-bucket report of total operations vs total live rows in storage. * * Intended for an on-demand admin/diagnostics view (e.g. answering "why is my Data Synced so high"), - * not as a live gauge. Operation counts are read from pre-aggregated bucket state; live-row counts are - * derived from current stored rows. May be expensive on large instances. + * not as a live gauge. Operation and live-row counts are aggregated from storage; the exact source is + * backend-specific (MongoDB reads pre-aggregated bucket state, Postgres scans bucket data). May be + * expensive on large instances. * * Optional: storage providers that don't implement it are reported as unsupported by the API. */ diff --git a/packages/service-errors/src/codes.ts b/packages/service-errors/src/codes.ts index 3306b89c3..a530a3ff9 100644 --- a/packages/service-errors/src/codes.ts +++ b/packages/service-errors/src/codes.ts @@ -495,8 +495,8 @@ export enum ErrorCode { // ## PSYNC_S25xx: Sync API errors - Postgres Storage /** - * Query timed out. Could be due to a large query or a temporary load issue on the storage database. - * Retry the request. + * Postgres storage query timed out. Could be due to a large query or a temporary load issue on the + * storage database. Retry the request. */ PSYNC_S2501 = 'PSYNC_S2501', diff --git a/packages/types/src/routes.ts b/packages/types/src/routes.ts index 29485a384..733063a43 100644 --- a/packages/types/src/routes.ts +++ b/packages/types/src/routes.ts @@ -81,7 +81,8 @@ export type ValidateResponse = t.Encoded; export const BucketReportRequest = t.object({ /** * Maximum number of buckets to return, ranked by operation count descending (worst offenders first). - * Totals are still computed across all buckets. Omit for no limit. + * Caps the response only, not the query cost: totals are still computed across all buckets. Omit for + * no limit. */ limit: t.number.optional() }); @@ -92,7 +93,7 @@ export const BucketStorageStats = t.object({ bucket: t.string, /** Total operations in the bucket's history (PUT/REMOVE/MOVE/CLEAR). */ operations: t.number, - /** Distinct live rows currently in the bucket. */ + /** Live rows currently in the bucket. */ rows: t.number, /** Approximate size of the operation history in bytes. */ operation_bytes: t.number, @@ -108,7 +109,7 @@ export const BucketReportResponse = t.object({ /** Per-bucket stats, ranked worst-first (most operations, then most fragmented). */ buckets: t.array(BucketStorageStats), totals: t.object({ - /** Total number of buckets in the active sync config (before any `limit`). */ + /** Number of buckets with stored operations or rows (before any `limit`). */ bucket_count: t.number, operations: t.number, /** Sum of per-bucket live rows. Rows in multiple buckets are counted per bucket. */ From bc9384d1e6a4f7abd0705de3512787b199800d63 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Thu, 25 Jun 2026 11:54:37 +0200 Subject: [PATCH 13/40] Add bucket report route test and strengthen storage tests --- .../test/src/bucket-report-scoping.test.ts | 21 +++++- .../test/src/storage.test.ts | 6 +- .../test/src/storage.test.ts | 6 +- .../src/tests/register-bucket-report-tests.ts | 9 +++ .../test/src/routes/admin.test.ts | 69 ++++++++++++++++++- 5 files changed, 97 insertions(+), 14 deletions(-) diff --git a/modules/module-mongodb-storage/test/src/bucket-report-scoping.test.ts b/modules/module-mongodb-storage/test/src/bucket-report-scoping.test.ts index 1dcd749d9..040e90cd9 100644 --- a/modules/module-mongodb-storage/test/src/bucket-report-scoping.test.ts +++ b/modules/module-mongodb-storage/test/src/bucket-report-scoping.test.ts @@ -95,6 +95,20 @@ streams: expect(replicatingStreams).toHaveLength(1); const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorageV3; await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS); + // Give config 2 its own replicated row, so the report has an active-config bucket to include. + const scenesTable = ( + await secondWriter.resolveTables({ + connection_id: 1, + source: sourceDescriptor('scenes', 'scenes-relation'), + idGenerator: objectIdGenerator('6544e3899293153fa7b38361') + }) + ).tables[0]; + await secondWriter.save({ + sourceTable: scenesTable, + tag: storage.SaveOperationTag.INSERT, + after: { id: 'scene-1', project_id: 'project-1' }, + afterReplicaId: test_utils.rid('scene-1') + }); await secondWriter.markAllSnapshotDone('2/1'); await secondWriter.commit('2/1'); await secondWriter.flush(); @@ -104,9 +118,10 @@ streams: const activeStorage = activeConfig!.storage as MongoSyncBucketStorageV3; const secondReport = await activeStorage.getBucketReport(); - // Config 2 has no replicated data, and config 1's stale buckets must be excluded. Without scoping to the - // active config's definition ids, config 1's bucket would leak in here. - expect(secondReport.totals.bucketCount).toEqual(0); + // The active config's own bucket is included (include-active), while config 1's stale buckets, which still + // exist in the shared collections, are excluded (exclude-stale). Without scoping to the active config's + // definition ids, config 1's bucket would leak in here. + expect(secondReport.totals.bucketCount).toBeGreaterThan(0); const firstBucketNames = new Set(firstReport.buckets.map((b) => b.bucket)); expect(secondReport.buckets.some((b) => firstBucketNames.has(b.bucket))).toBe(false); }); diff --git a/modules/module-mongodb-storage/test/src/storage.test.ts b/modules/module-mongodb-storage/test/src/storage.test.ts index 7d94836a8..6cfa12b84 100644 --- a/modules/module-mongodb-storage/test/src/storage.test.ts +++ b/modules/module-mongodb-storage/test/src/storage.test.ts @@ -19,11 +19,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { register.registerDataStorageCheckpointTests({ ...INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion })); describe(`Mongo Sync Bucket Storage - Bucket report - v${storageVersion}`, () => - register.registerBucketReportTests({ - ...INITIALIZED_MONGO_STORAGE_FACTORY, - storageVersion, - compressedBucketStorage: storageVersion >= 3 - })); + register.registerBucketReportTests({ ...INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion })); } describe('Sync Bucket Validation', register.registerBucketValidationTests); diff --git a/modules/module-postgres-storage/test/src/storage.test.ts b/modules/module-postgres-storage/test/src/storage.test.ts index 31596a83b..18c074f96 100644 --- a/modules/module-postgres-storage/test/src/storage.test.ts +++ b/modules/module-postgres-storage/test/src/storage.test.ts @@ -20,11 +20,7 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { register.registerDataStorageCheckpointTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion })); describe(`Postgres Sync Bucket Storage - Bucket report - v${storageVersion}`, () => - register.registerBucketReportTests({ - ...POSTGRES_STORAGE_FACTORY, - storageVersion, - compressedBucketStorage: false - })); + register.registerBucketReportTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion })); describe(`Postgres Sync Bucket Storage - pg-specific - v${storageVersion}`, () => { /** diff --git a/packages/service-core-tests/src/tests/register-bucket-report-tests.ts b/packages/service-core-tests/src/tests/register-bucket-report-tests.ts index 9f9b6925c..82cae7aca 100644 --- a/packages/service-core-tests/src/tests/register-bucket-report-tests.ts +++ b/packages/service-core-tests/src/tests/register-bucket-report-tests.ts @@ -160,6 +160,15 @@ bucket_definitions: expect(report.buckets.map((b) => b.bucket)).toEqual([b1, b2]); expect(report.buckets.find((b) => b.bucket === b1)).toMatchObject({ operations: 3, rows: 1 }); expect(report.buckets.find((b) => b.bucket === b2)).toMatchObject({ operations: 2, rows: 2 }); + + // operationBytes is aggregated differently per backend ($toDouble sum on Mongo, OCTET_LENGTH sum on + // Postgres); assert every bucket is non-zero and that the per-bucket bytes add up to the instance total. + expect(report.totals.operationBytes).toBeGreaterThan(0); + for (const bucket of report.buckets) { + expect(bucket.operationBytes).toBeGreaterThan(0); + } + const summedBytes = report.buckets.reduce((total, bucket) => total + bucket.operationBytes, 0); + expect(summedBytes).toEqual(report.totals.operationBytes); }); test('limit truncates the bucket list but totals still span all buckets', async () => { diff --git a/packages/service-core/test/src/routes/admin.test.ts b/packages/service-core/test/src/routes/admin.test.ts index 8f0fa6e24..fa2bdce93 100644 --- a/packages/service-core/test/src/routes/admin.test.ts +++ b/packages/service-core/test/src/routes/admin.test.ts @@ -2,7 +2,7 @@ import { BasicRouterRequest, Context, JwtPayload, ParsedSyncConfigSet, storage } import { logger } from '@powersync/lib-services-framework'; import { SqlSyncRules } from '@powersync/service-sync-rules'; import { describe, expect, it, vi } from 'vitest'; -import { diagnostics, reprocess, validate } from '../../../src/routes/endpoints/admin.js'; +import { bucketReport, diagnostics, reprocess, validate } from '../../../src/routes/endpoints/admin.js'; import { mockServiceContext } from './mocks.js'; describe('admin routes', () => { @@ -209,4 +209,71 @@ bucket_definitions: expect(activeBucketStorage.updateSyncRules).not.toHaveBeenCalled(); }); }); + + describe('bucket-report', () => { + const report = { + buckets: [ + { bucket: '1#by_user["u1"]', operations: 4750, rows: 95, operationBytes: 1216000, fragmentation: 50 }, + { bucket: '1#global[]', operations: 1000, rows: 1000, operationBytes: 3145728, fragmentation: 1 } + ], + totals: { bucketCount: 2, operations: 5750, rows: 1095, operationBytes: 4361728, fragmentation: 5750 / 1095 }, + truncated: false + }; + + it('returns the report, forwards the limit, and maps fields to snake_case', async () => { + const getBucketReport = vi.fn(async () => report); + const activeBucketStorage = { + getActiveSyncConfig: vi.fn(async () => ({ + content: makeSyncConfigContent({}), + replicationStream: {}, + storage: { getBucketReport } + })) + }; + + const response = await bucketReport.handler({ + context: makeContext(activeBucketStorage), + params: { limit: 20 }, + request + }); + + expect(getBucketReport).toHaveBeenCalledWith({ limit: 20 }); + expect(response.buckets[0]).toEqual({ + bucket: '1#by_user["u1"]', + operations: 4750, + rows: 95, + operation_bytes: 1216000, + fragmentation: 50 + }); + expect(response.totals).toEqual({ + bucket_count: 2, + operations: 5750, + rows: 1095, + operation_bytes: 4361728, + fragmentation: 5750 / 1095 + }); + expect(response.truncated).toBe(false); + }); + + it('rejects when there is no active sync config', async () => { + const activeBucketStorage = { getActiveSyncConfig: vi.fn(async () => null) }; + + await expect( + bucketReport.handler({ context: makeContext(activeBucketStorage), params: {}, request }) + ).rejects.toMatchObject({ errorData: { status: 422, code: 'PSYNC_S4104' } }); + }); + + it('rejects when the storage provider does not support bucket reporting', async () => { + const activeBucketStorage = { + getActiveSyncConfig: vi.fn(async () => ({ + content: makeSyncConfigContent({}), + replicationStream: {}, + storage: {} + })) + }; + + await expect( + bucketReport.handler({ context: makeContext(activeBucketStorage), params: {}, request }) + ).rejects.toMatchObject({ errorData: { status: 422, code: 'PSYNC_S2001' } }); + }); + }); }); From b02ec7d583d6ed76c02afc13b3b8df8234b9916c Mon Sep 17 00:00:00 2001 From: bean1352 Date: Mon, 29 Jun 2026 13:07:56 +0200 Subject: [PATCH 14/40] Rework bucket report contract for top-N sampling --- .../src/routes/endpoints/admin.ts | 6 +- .../service-core/src/storage/bucket-report.ts | 134 ++++++++---------- packages/types/src/routes.ts | 20 +-- 3 files changed, 74 insertions(+), 86 deletions(-) diff --git a/packages/service-core/src/routes/endpoints/admin.ts b/packages/service-core/src/routes/endpoints/admin.ts index b3b8ab18a..7e3d4216b 100644 --- a/packages/service-core/src/routes/endpoints/admin.ts +++ b/packages/service-core/src/routes/endpoints/admin.ts @@ -313,14 +313,14 @@ export const bucketReport = routeDefinition({ operations: bucket.operations, rows: bucket.rows, operation_bytes: bucket.operationBytes, - fragmentation: bucket.fragmentation + fragmentation: bucket.fragmentation, + rows_estimated: bucket.rowsEstimated })), totals: { bucket_count: report.totals.bucketCount, operations: report.totals.operations, - rows: report.totals.rows, operation_bytes: report.totals.operationBytes, - fragmentation: report.totals.fragmentation + estimated: report.totals.estimated }, truncated: report.truncated }); diff --git a/packages/service-core/src/storage/bucket-report.ts b/packages/service-core/src/storage/bucket-report.ts index 62ec99080..3823034b6 100644 --- a/packages/service-core/src/storage/bucket-report.ts +++ b/packages/service-core/src/storage/bucket-report.ts @@ -7,28 +7,30 @@ * A new client downloads every operation, not just live rows, so `operations / rows` is effectively a * fragmentation / compaction-efficiency score: a fully compacted bucket trends towards ~1, while a high * ratio is the usual cause of an unexpectedly high "Data Synced" metric and is reclaimable via compact/defragment. + * + * Scaling note: the report does NOT scan all storage. It ranks buckets by their pre-aggregated operation + * count and returns the worst offenders (top-N). Row counts (and therefore fragmentation) for those buckets + * are derived by sampling the operation history, so on large buckets they are estimates, flagged per bucket. */ /** - * Time budget for the per-bucket report's storage aggregations (Mongo `maxTimeMS`, Postgres - * `statement_timeout`). The report scans current rows (and, on Postgres, all bucket data), which can be - * expensive on large instances, so the queries are bounded rather than allowed to run unbounded. + * Time budget for the per-bucket report's bucket-selection aggregation (`maxTimeMS`). Bounded so an admin + * request on a large instance fails fast instead of running unbounded. */ export const BUCKET_REPORT_TIMEOUT_MS: number = 60_000; -export interface BucketOperationStat { - /** Total operations in the bucket's history (PUT/REMOVE/MOVE/CLEAR). */ - operations: number; - /** Approximate size of the operation history in bytes. */ - operationBytes: number; -} +/** + * Number of worst-offender buckets returned when the request omits a `limit`. Row counts are sampled per + * returned bucket, so this also bounds how much sampling work the report does. + */ +export const DEFAULT_BUCKET_REPORT_LIMIT: number = 50; export interface BucketStorageStats { /** Full bucket name, e.g. `by_user["u1"]`. */ bucket: string; /** Total operations in the bucket's history. */ operations: number; - /** Live rows currently in the bucket. */ + /** Live rows in the bucket. Exact for small buckets, otherwise a sampled estimate (see `rowsEstimated`). */ rows: number; /** Approximate size of the operation history in bytes. */ operationBytes: number; @@ -37,98 +39,82 @@ export interface BucketStorageStats { * overhead that a compact/defragment can reclaim. */ fragmentation: number; + /** True if `rows` (and therefore `fragmentation`) is a sampled estimate rather than an exact count. */ + rowsEstimated: boolean; } export interface BucketReportTotals { - /** Number of buckets with stored operations or rows (before any `limit`). */ + /** Number of buckets with stored operations. Estimated when the bucket set was sampled (see `estimated`). */ bucketCount: number; - /** Sum of operations across all buckets. */ + /** Sum of operations across all buckets. Estimated when the bucket set was sampled. */ operations: number; - /** - * Sum of per-bucket live rows. Note this double-counts rows that belong to multiple buckets, - * so it is a sum of per-bucket counts rather than a distinct instance-wide row total. - */ - rows: number; - /** Sum of operation-history bytes across all buckets. */ + /** Sum of operation-history bytes across all buckets. Estimated when the bucket set was sampled. */ operationBytes: number; /** - * Instance-wide fragmentation: `operations / max(rows, 1)`, i.e. the row-weighted average of the - * per-bucket ratios. ~1 is healthy; a higher value means a new client downloads that many operations - * per live row across the whole instance, which is the headline cause of a high "Data Synced" metric. + * True if the totals are estimated because the bucket set was too large to scan in full and was sampled. + * Row counts are never totalled here (they are sampled per returned bucket, not across the whole instance). */ - fragmentation: number; + estimated: boolean; } export interface BucketReport { - /** Per-bucket stats, ranked worst-first (most operations, then most fragmented). */ + /** Worst-offender buckets, ranked by operation count then fragmentation. */ buckets: BucketStorageStats[]; - /** Instance-wide totals, computed across all buckets even when `buckets` is truncated by `limit`. */ + /** Instance-wide operation totals. Does not include row counts (those are per-bucket estimates only). */ totals: BucketReportTotals; - /** True if `buckets` was truncated by `limit`. `totals` still reflects all buckets. */ + /** True if there are more buckets than returned (more than `limit`). */ truncated: boolean; } export interface GetBucketReportOptions { /** * Maximum number of buckets to return, ranked by operation count descending (worst offenders first). - * This caps the response size only: every backend still aggregates all buckets (and `totals` covers - * them all), so it is not a query-cost bound. Non-integer or negative values are floored and clamped - * to 0. Defaults to no limit. + * Row counts are sampled per returned bucket, so this also bounds the report's cost. Non-integer or + * negative values are floored and clamped to 1. Defaults to {@link DEFAULT_BUCKET_REPORT_LIMIT}. */ limit?: number; } +/** A bucket's exact operation stats plus its (possibly sampled) row count, before ranking. */ +export interface RankedBucketInput { + bucket: string; + operations: number; + operationBytes: number; + rows: number; + rowsEstimated: boolean; +} + /** - * Combine per-bucket operation stats and live-row counts (each keyed by full bucket name) into a - * ranked {@link BucketReport}. Backend storage adapters collect the two maps however is cheapest for - * them; this builder owns the shared merge/rank/total logic so that part stays identical across - * backends. The inputs are not identical: operation counts are exact on Postgres (a `COUNT(*)`) but a - * pre-aggregated estimate on MongoDB, so the same data can yield slightly different counts per backend. + * Normalize a requested limit to a positive integer, falling back to {@link DEFAULT_BUCKET_REPORT_LIMIT}. */ -export function buildBucketReport( - operationStats: Map, - rowCounts: Map, - options?: GetBucketReportOptions -): BucketReport { - const bucketNames = new Set([...operationStats.keys(), ...rowCounts.keys()]); - - const buckets: BucketStorageStats[] = []; - const totals: BucketReportTotals = { bucketCount: 0, operations: 0, rows: 0, operationBytes: 0, fragmentation: 0 }; - - for (const bucket of bucketNames) { - const opStat = operationStats.get(bucket); - const operations = opStat?.operations ?? 0; - const operationBytes = opStat?.operationBytes ?? 0; - const rows = rowCounts.get(bucket) ?? 0; - - buckets.push({ - bucket, - operations, - rows, - operationBytes, - fragmentation: operations / Math.max(rows, 1) - }); - - totals.bucketCount += 1; - totals.operations += operations; - totals.rows += rows; - totals.operationBytes += operationBytes; +export function resolveBucketReportLimit(limit?: number): number { + if (limit == null) { + return DEFAULT_BUCKET_REPORT_LIMIT; } + return Math.max(1, Math.floor(limit)); +} - totals.fragmentation = totals.operations / Math.max(totals.rows, 1); +/** + * Assemble the final {@link BucketReport} from per-bucket stats and instance-wide totals. Storage adapters + * select and sample the buckets however is cheapest for them; this owns the shared fragmentation / ranking / + * truncation logic so it cannot drift. Pure (no I/O) so it is unit-testable. + */ +export function assembleBucketReport(buckets: RankedBucketInput[], totals: BucketReportTotals): BucketReport { + const stats: BucketStorageStats[] = buckets.map((b) => ({ + bucket: b.bucket, + operations: b.operations, + rows: b.rows, + operationBytes: b.operationBytes, + fragmentation: b.operations / Math.max(b.rows, 1), + rowsEstimated: b.rowsEstimated + })); // Worst-first: most operations, then most fragmented. - buckets.sort((a, b) => b.operations - a.operations || b.fragmentation - a.fragmentation); - - let truncated = false; - let reported = buckets; - if (options?.limit != null) { - const limit = Math.max(0, Math.floor(options.limit)); - if (buckets.length > limit) { - reported = buckets.slice(0, limit); - truncated = true; - } - } + stats.sort((a, b) => b.operations - a.operations || b.fragmentation - a.fragmentation); - return { buckets: reported, totals, truncated }; + return { + buckets: stats, + totals, + truncated: totals.bucketCount > stats.length + }; } diff --git a/packages/types/src/routes.ts b/packages/types/src/routes.ts index 733063a43..179dc6c3f 100644 --- a/packages/types/src/routes.ts +++ b/packages/types/src/routes.ts @@ -93,7 +93,7 @@ export const BucketStorageStats = t.object({ bucket: t.string, /** Total operations in the bucket's history (PUT/REMOVE/MOVE/CLEAR). */ operations: t.number, - /** Live rows currently in the bucket. */ + /** Live rows in the bucket. Exact for small buckets, otherwise a sampled estimate (see `rows_estimated`). */ rows: t.number, /** Approximate size of the operation history in bytes. */ operation_bytes: t.number, @@ -101,24 +101,26 @@ export const BucketStorageStats = t.object({ * `operations / max(rows, 1)`. ~1 is healthy (fully compacted); higher means more operation-history * overhead that a compact/defragment can reclaim. */ - fragmentation: t.number + fragmentation: t.number, + /** True if `rows` (and therefore `fragmentation`) is a sampled estimate rather than an exact count. */ + rows_estimated: t.boolean }); export type BucketStorageStats = t.Encoded; export const BucketReportResponse = t.object({ - /** Per-bucket stats, ranked worst-first (most operations, then most fragmented). */ + /** Worst-offender buckets, ranked by operation count then fragmentation. */ buckets: t.array(BucketStorageStats), totals: t.object({ - /** Number of buckets with stored operations or rows (before any `limit`). */ + /** Number of buckets with stored operations. Estimated when the bucket set was sampled. */ bucket_count: t.number, + /** Sum of operations across all buckets. Estimated when the bucket set was sampled. */ operations: t.number, - /** Sum of per-bucket live rows. Rows in multiple buckets are counted per bucket. */ - rows: t.number, + /** Sum of operation-history bytes across all buckets. Estimated when the bucket set was sampled. */ operation_bytes: t.number, - /** Instance-wide `operations / max(rows, 1)`: operations a new client downloads per live row. */ - fragmentation: t.number + /** True if the totals are estimated because the bucket set was sampled rather than fully scanned. */ + estimated: t.boolean }), - /** True if `buckets` was truncated by `limit`. `totals` still reflects all buckets. */ + /** True if there are more buckets than returned (more than `limit`). */ truncated: t.boolean }); export type BucketReportResponse = t.Encoded; From 65ea0c6c4a685117b4b5a7acd2f9cd22fca38fda Mon Sep 17 00:00:00 2001 From: bean1352 Date: Mon, 29 Jun 2026 13:08:14 +0200 Subject: [PATCH 15/40] Sample MongoDB bucket report instead of scanning all storage --- .../implementation/MongoSyncBucketStorage.ts | 269 +++++++++++++----- .../v1/MongoSyncBucketStorageV1.ts | 35 ++- .../v3/MongoSyncBucketStorageV3.ts | 45 ++- 3 files changed, 250 insertions(+), 99 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index e672b73ee..b9df68e43 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -64,6 +64,41 @@ interface InternalCheckpointChanges extends CheckpointChanges { */ const CHECKPOINT_TIMEOUT_MS = 60_000; +/** + * Above this many buckets, the report ranks a bounded `$sample` of bucket_state rather than every bucket, so + * the request cannot exhaust memory or run unbounded. Below it, the ranking is exact. + */ +const BUCKET_SELECTION_SAMPLE_THRESHOLD = 50_000; + +/** Number of buckets to sample when over {@link BUCKET_SELECTION_SAMPLE_THRESHOLD}. */ +const BUCKET_SELECTION_SAMPLE_SIZE = 10_000; + +/** + * Target number of operations to sample per bucket when estimating its row count. Buckets with fewer + * operations than this are read in full (exact); larger buckets are sampled down to roughly this many. + */ +const BUCKET_ROW_SAMPLE_TARGET = 1_000; + +/** A worst-offender bucket selected from bucket_state, with the version-specific context needed to sample it. */ +export interface TopBucketCandidate { + bucket: string; + operations: number; + operationBytes: number; + /** v3 only: the bucket definition id, used to locate its per-definition bucket_data collection. */ + defId?: BucketDefinitionId; +} + +export interface TopBucketSelection { + buckets: TopBucketCandidate[]; + totals: storage.BucketReportTotals; +} + +export interface BucketRowEstimate { + rows: number; + /** True if `rows` is a sampled estimate rather than an exact count. */ + estimated: boolean; +} + export abstract class MongoSyncBucketStorage extends BaseObserver implements storage.SyncRulesBucketStorage @@ -367,115 +402,191 @@ export abstract class MongoSyncBucketStorage } async getBucketReport(options?: storage.GetBucketReportOptions): Promise { - let operationStats: Map; - let rowCounts: Map; + const limit = storage.resolveBucketReportLimit(options?.limit); try { - [operationStats, rowCounts] = await Promise.all([ - this.collectBucketOperationStats(), - this.collectBucketLiveRowCounts() - ]); + // Rank the worst-offender buckets and total operations from the pre-aggregated bucket state (bounded, + // in the database), then estimate each returned bucket's row count by sampling its operation history. + const { buckets, totals } = await this.collectTopBuckets(limit); + const ranked: storage.RankedBucketInput[] = []; + for (const candidate of buckets) { + const estimate = await this.estimateBucketRows(candidate); + ranked.push({ + bucket: candidate.bucket, + operations: candidate.operations, + operationBytes: candidate.operationBytes, + rows: estimate.rows, + rowsEstimated: estimate.estimated + }); + } + return storage.assembleBucketReport(ranked, totals); } catch (e) { // Translate a storage query timeout (maxTimeMS) into a specific, retryable error code rather than a - // generic internal error. Only the storage reads are wrapped; the pure-JS merge below cannot time out. + // generic internal error. throw lib_mongo.mapQueryError(e, 'while building the bucket report'); } - return storage.buildBucketReport(operationStats, rowCounts, options); } /** - * Operation count and operation-history bytes per bucket, read from the pre-aggregated bucket state - * (compacted_state + estimate_since_compact). Cheap: one document per bucket, no scan of bucket data. - * - * Note: for v1/v2 storage, bucket_state is not backfilled (see models.ts: "only populated by new updates"). - * Buckets whose data predates bucket_state tracking, and which have not been updated or compacted since, - * have no document here and so report zero operations. v3 always has bucket_state. - * - * Implementations supply their version-specific bucket state collection(s) to {@link aggregateBucketOperationStats}. + * Select the worst-offender buckets (by operation count) plus instance-wide operation totals from the + * pre-aggregated bucket state. Ranking and limiting happen in the database, so memory stays bounded. + * Implementations supply their version-specific bucket state collection and active-config filter. */ - protected abstract collectBucketOperationStats(): Promise>; + protected abstract collectTopBuckets(limit: number): Promise; /** - * Live rows per bucket, derived from the current stored rows and their bucket memberships. - * - * Implementations supply their version-specific current-row collection(s) to {@link aggregateBucketLiveRowCounts}. + * Estimate a single bucket's live row count by sampling its operation history. Implementations differ + * because v1/v2 store one document per operation while v3 batches operations per document. */ - protected abstract collectBucketLiveRowCounts(): Promise>; + protected abstract estimateBucketRows(candidate: TopBucketCandidate): Promise; /** - * Aggregate operation count and operation-history bytes per bucket from a bucket state collection. + * Rank buckets by operation count in the database and compute instance-wide operation totals, reading the + * pre-aggregated bucket state (compacted_state + estimate_since_compact). One document per bucket, no scan + * of bucket data. + * + * For very large bucket sets the candidates are drawn from a bounded `$sample` rather than the whole + * collection (so the request cannot run unbounded or exhaust memory), and the totals are scaled from the + * sample and flagged estimated. `allowDiskUse: false` makes an over-threshold exact attempt fail fast + * rather than spill to disk and degrade the live instance. * - * Operations are pre-aggregated in bucket state, so this reads a single document per bucket rather than - * scanning bucket data. Shared by the storage versions, which differ only in which collection (and filter) - * holds their bucket state. + * Note: for v1/v2 storage, bucket_state is not backfilled (see models.ts: "only populated by new updates"), + * so buckets that predate bucket_state tracking and have not been updated or compacted since are missing + * here and under-counted. v3 always has bucket_state. */ - protected async aggregateBucketOperationStats( + protected async aggregateTopBuckets( collection: mongo.Collection, - match?: mongo.Filter - ): Promise> { - const pipeline: mongo.Document[] = []; - if (match != null) { - pipeline.push({ $match: match }); + match: mongo.Filter, + limit: number + ): Promise<{ + buckets: { id: T['_id']; operations: number; operationBytes: number }[]; + totals: storage.BucketReportTotals; + }> { + const operations = { + $add: [{ $ifNull: ['$compacted_state.count', 0] }, { $ifNull: ['$estimate_since_compact.count', 0] }] + }; + const operationBytes = { + $add: [ + { $toDouble: { $ifNull: ['$compacted_state.bytes', 0] } }, + { $toDouble: { $ifNull: ['$estimate_since_compact.bytes', 0] } } + ] + }; + + // estimatedDocumentCount ignores the match filter, so this is an upper bound on the active bucket count. + // That is fine: it only decides whether to sample, and over-estimating just switches to sampling sooner. + const totalBuckets = await collection.estimatedDocumentCount(); + const sampled = totalBuckets > BUCKET_SELECTION_SAMPLE_THRESHOLD; + + const pipeline: mongo.Document[] = [{ $match: match }]; + if (sampled) { + pipeline.push({ $sample: { size: BUCKET_SELECTION_SAMPLE_SIZE } }); } pipeline.push({ - $project: { - _id: 1, - operations: { - $add: [{ $ifNull: ['$compacted_state.count', 0] }, { $ifNull: ['$estimate_since_compact.count', 0] }] - }, - operationBytes: { - $add: [ - { $toDouble: { $ifNull: ['$compacted_state.bytes', 0] } }, - { $toDouble: { $ifNull: ['$estimate_since_compact.bytes', 0] } } - ] - } + $facet: { + totals: [ + { + $group: { + _id: null, + operations: { $sum: operations }, + operationBytes: { $sum: operationBytes }, + bucketCount: { $sum: 1 } + } + } + ], + top: [{ $project: { _id: 1, operations, operationBytes } }, { $sort: { operations: -1 } }, { $limit: limit }] } }); - const result = new Map(); - const cursor = collection.aggregate<{ _id: { b: string }; operations: number; operationBytes: number }>(pipeline, { - maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS - }); - for await (const doc of cursor.stream()) { - result.set(doc._id.b, { operations: doc.operations, operationBytes: doc.operationBytes }); + type FacetResult = { + totals: { operations: number; operationBytes: number; bucketCount: number }[]; + top: { _id: T['_id']; operations: number; operationBytes: number }[]; + }; + const [result] = await collection + .aggregate(pipeline, { allowDiskUse: false, maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS }) + .toArray(); + + const rawTotals = result?.totals[0] ?? { operations: 0, operationBytes: 0, bucketCount: 0 }; + const buckets = (result?.top ?? []).map((doc) => ({ + id: doc._id, + operations: doc.operations, + operationBytes: doc.operationBytes + })); + + if (!sampled) { + return { + buckets, + totals: { + bucketCount: rawTotals.bucketCount, + operations: rawTotals.operations, + operationBytes: rawTotals.operationBytes, + estimated: false + } + }; } - return result; + + // Scale the sampled totals up to the full collection. + const scale = totalBuckets / Math.max(rawTotals.bucketCount, 1); + return { + buckets, + totals: { + bucketCount: totalBuckets, + operations: Math.round(rawTotals.operations * scale), + operationBytes: Math.round(rawTotals.operationBytes * scale), + estimated: true + } + }; } /** - * Aggregate distinct live rows per bucket across one or more current-row collections. + * Estimate a bucket's live rows from a sample of its operations. * - * Each stored row records its bucket memberships, so unwinding those memberships and grouping by bucket counts - * the live rows in each bucket (one membership per stored row). Counts are summed across collections, since a - * bucket may contain rows from multiple source tables (each in its own collection). - * - * `options.bucketDefinitionIds` restricts to memberships of those definitions - used by V3 to exclude rows - * still tagged with stopped/old definitions that share the stream but are not part of the active config. + * `pipelinePrefix` must select the bucket's operations (and, when `sampled`, randomly down-sample them) and + * yield documents with top-level `op`, `table` and `row_id` fields. Fragmentation is then + * `sampledOps / distinctRows` and the row count is `operations / fragmentation`. Exact (not sampled) when + * the whole bucket fits within the sample target. */ - protected async aggregateBucketLiveRowCounts( - collections: mongo.Collection[], - options?: { match?: mongo.Filter; bucketDefinitionIds?: BucketDefinitionId[] } - ): Promise> { - const pipeline: mongo.Document[] = []; - if (options?.match != null) { - pipeline.push({ $match: options.match }); - } - pipeline.push({ $unwind: '$buckets' }); - if (options?.bucketDefinitionIds != null) { - pipeline.push({ $match: { 'buckets.def': { $in: options.bucketDefinitionIds } } }); - } - pipeline.push({ $group: { _id: '$buckets.bucket', count: { $sum: 1 } } }); - - const result = new Map(); - for (const collection of collections) { - const cursor = collection.aggregate<{ _id: string; count: number }>(pipeline, { - allowDiskUse: true, - maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS - }); - for await (const doc of cursor.stream()) { - result.set(doc._id, (result.get(doc._id) ?? 0) + doc.count); + protected async estimateRowsFromOperationSample( + collection: mongo.Collection, + pipelinePrefix: mongo.Document[], + operations: number, + sampled: boolean + ): Promise { + const pipeline: mongo.Document[] = [ + ...pipelinePrefix, + { + $facet: { + sampledOps: [{ $count: 'count' }], + distinctRows: [ + { $match: { op: { $in: ['PUT', 'REMOVE'] } } }, + { $group: { _id: { table: '$table', row_id: '$row_id' } } }, + { $count: 'count' } + ] + } } + ]; + + type FacetResult = { sampledOps: { count: number }[]; distinctRows: { count: number }[] }; + const [result] = await collection + .aggregate(pipeline, { allowDiskUse: false, maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS }) + .toArray(); + + const sampledOps = result?.sampledOps[0]?.count ?? 0; + const distinctRows = result?.distinctRows[0]?.count ?? 0; + if (sampledOps == 0 || distinctRows == 0) { + // Nothing row-bearing was sampled (e.g. a bucket of only MOVE/CLEAR ops): treat as fully fragmented. + return { rows: 0, estimated: sampled }; } - return result; + // fragmentation = sampledOps / distinctRows; rows = operations / fragmentation = operations * distinctRows / sampledOps. + return { rows: Math.round((operations * distinctRows) / sampledOps), estimated: sampled }; + } + + /** Whether a bucket with this many operations should be sampled rather than read in full. */ + protected shouldSampleBucketRows(operations: number): boolean { + return operations > BUCKET_ROW_SAMPLE_TARGET; + } + + /** `$sampleRate` for sampling roughly {@link BUCKET_ROW_SAMPLE_TARGET} of a bucket's operations. */ + protected bucketRowSampleRate(operations: number): number { + return BUCKET_ROW_SAMPLE_TARGET / operations; } /** diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index 2873b1232..ab9db340d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -24,7 +24,13 @@ import { MongoChecksums } from '../MongoChecksums.js'; import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; import { MongoPersistedReplicationStream } from '../MongoPersistedReplicationStream.js'; -import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js'; +import { + BucketRowEstimate, + MongoSyncBucketStorage, + MongoSyncBucketStorageOptions, + TopBucketCandidate, + TopBucketSelection +} from '../MongoSyncBucketStorage.js'; import { BucketDataDocumentV1, BucketDataKeyV1, @@ -180,16 +186,27 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage { return new MongoCompactorV1(this, this.db, options); } - // For storage v1/v2, bucket state and current rows are shared collections scoped by group (replication stream). - protected collectBucketOperationStats(): Promise> { - return this.aggregateBucketOperationStats(this.db.bucketStateV1, { '_id.g': this.replicationStreamId }); + // For storage v1/v2, bucket state and bucket data are shared collections scoped by group (replication stream). + protected async collectTopBuckets(limit: number): Promise { + const { buckets, totals } = await this.aggregateTopBuckets( + this.db.bucketStateV1, + { '_id.g': this.replicationStreamId }, + limit + ); + return { + buckets: buckets.map((b) => ({ bucket: b.id.b, operations: b.operations, operationBytes: b.operationBytes })), + totals + }; } - protected collectBucketLiveRowCounts(): Promise> { - // V1/V2 have a single sync config per replication stream, so scoping by group is sufficient. - return this.aggregateBucketLiveRowCounts([this.db.sourceRecordsV1], { - match: { '_id.g': this.replicationStreamId } - }); + protected estimateBucketRows(candidate: TopBucketCandidate): Promise { + // v1/v2 store one document per operation, so a bucket's ops are an id-prefix range that can be sampled directly. + const sampled = this.shouldSampleBucketRows(candidate.operations); + const prefix: mongo.Document[] = [{ $match: { '_id.g': this.replicationStreamId, '_id.b': candidate.bucket } }]; + if (sampled) { + prefix.push({ $match: { $sampleRate: this.bucketRowSampleRate(candidate.operations) } }); + } + return this.estimateRowsFromOperationSample(this.db.bucketDataV1, prefix, candidate.operations, sampled); } protected createMongoParameterCompactor( diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index 0f3e4e954..37203b13e 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -23,7 +23,13 @@ import { MongoChecksums } from '../MongoChecksums.js'; import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; import { MongoPersistedReplicationStream } from '../MongoPersistedReplicationStream.js'; -import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js'; +import { + BucketRowEstimate, + MongoSyncBucketStorage, + MongoSyncBucketStorageOptions, + TopBucketCandidate, + TopBucketSelection +} from '../MongoSyncBucketStorage.js'; import { loadBucketDataDocument } from './bucket-format.js'; import { BucketDataDocumentV3, @@ -180,21 +186,38 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { return new MongoCompactorV3(this, this.db, options); } - // For storage v3, bucket state is a per-stream collection and current rows are split into per-table collections. + // For storage v3, bucket state is a per-stream collection and bucket data is split into per-definition collections. // A replication stream can host multiple sync configs (active + processing + stopped, until cleanup runs), all // sharing these collections. Scope to the active config's definition ids so the report excludes stale buckets // from old/stopped definitions. `this.storageIds` is derived from the active config only (see getActiveSyncConfig). - protected collectBucketOperationStats(): Promise> { - return this.aggregateBucketOperationStats(this.db.bucketState(this.replicationStreamId), { - '_id.d': { $in: this.storageIds.bucketDefinitionIds } - }); + protected async collectTopBuckets(limit: number): Promise { + const { buckets, totals } = await this.aggregateTopBuckets( + this.db.bucketState(this.replicationStreamId), + { '_id.d': { $in: this.storageIds.bucketDefinitionIds } }, + limit + ); + return { + buckets: buckets.map((b) => ({ + bucket: b.id.b, + operations: b.operations, + operationBytes: b.operationBytes, + defId: b.id.d + })), + totals + }; } - protected async collectBucketLiveRowCounts(): Promise> { - const collections = await this.db.listSourceRecordCollections(this.replicationStreamId); - return this.aggregateBucketLiveRowCounts(collections, { - bucketDefinitionIds: this.storageIds.bucketDefinitionIds - }); + protected estimateBucketRows(candidate: TopBucketCandidate): Promise { + // v3 batches operations into documents (one doc holds an `ops` array), in a per-definition collection. + // Sample whole batch documents, then unwind to operation level so the shared estimator sees one doc per op. + const sampled = this.shouldSampleBucketRows(candidate.operations); + const collection = this.db.bucketData(this.replicationStreamId, candidate.defId!); + const prefix: mongo.Document[] = [{ $match: { '_id.b': candidate.bucket } }]; + if (sampled) { + prefix.push({ $match: { $sampleRate: this.bucketRowSampleRate(candidate.operations) } }); + } + prefix.push({ $unwind: '$ops' }, { $replaceRoot: { newRoot: '$ops' } }); + return this.estimateRowsFromOperationSample(collection, prefix, candidate.operations, sampled); } protected createMongoParameterCompactor( From 98297de044d5c687b55f6ce7db67ad5979c39952 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Mon, 29 Jun 2026 13:08:28 +0200 Subject: [PATCH 16/40] Limit bucket report to MongoDB storage --- .changeset/bucket-storage-report.md | 4 +- .../src/storage/PostgresSyncRulesStorage.ts | 77 ------------------- .../test/src/storage.test.ts | 3 - packages/service-errors/src/codes.ts | 8 +- 4 files changed, 2 insertions(+), 90 deletions(-) diff --git a/.changeset/bucket-storage-report.md b/.changeset/bucket-storage-report.md index e6b79249d..b37a614d3 100644 --- a/.changeset/bucket-storage-report.md +++ b/.changeset/bucket-storage-report.md @@ -2,9 +2,7 @@ '@powersync/service-core': minor '@powersync/service-types': minor '@powersync/service-module-mongodb-storage': minor -'@powersync/service-module-postgres-storage': minor '@powersync/service-core-tests': minor -'@powersync/service-errors': patch --- -Add a `POST /api/admin/v1/bucket-report` admin endpoint reporting operations vs rows per bucket. +Add a `POST /api/admin/v1/bucket-report` admin endpoint reporting operations vs rows per bucket (MongoDB storage). diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index df85b729a..60a355fbd 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -38,9 +38,6 @@ import { PostgresCurrentDataStore } from './current-data-store.js'; import { PostgresBucketStorageFactory } from './PostgresBucketStorageFactory.js'; import { PostgresCompactor } from './PostgresCompactor.js'; -/** Postgres SQLSTATE raised when a statement is cancelled, e.g. by statement_timeout. */ -const POSTGRES_QUERY_CANCELED = '57014'; - export type PostgresSyncRulesStorageOptions = { factory: PostgresBucketStorageFactory; db: lib_postgres.DatabaseClient; @@ -154,80 +151,6 @@ export class PostgresSyncRulesStorage }).compact(); } - async getBucketReport(options?: storage.GetBucketReportOptions): Promise { - try { - return await this.collectBucketReport(options); - } catch (e) { - // statement_timeout cancels the query with SQLSTATE 57014 (query_canceled). Translate it into a - // specific, retryable timeout code rather than a generic internal error. - if (e?.cause?.code === POSTGRES_QUERY_CANCELED) { - throw new framework.DatabaseQueryError( - framework.ErrorCode.PSYNC_S2501, - 'Query timed out while building the bucket report', - e - ); - } - throw e; - } - } - - private async collectBucketReport(options?: storage.GetBucketReportOptions): Promise { - // Both queries scan storage (Postgres has no pre-aggregated bucket state), so they run in a transaction - // with a statement timeout rather than letting an admin request run unbounded on a large instance. - const { operationStats, rowCounts } = await this.db.transaction(async (db) => { - await db.query(`SET LOCAL statement_timeout = ${storage.BUCKET_REPORT_TIMEOUT_MS}`); - - // Operations + operation-history bytes per bucket. - const operationRows = await db.sql` - SELECT - bucket_name, - COUNT(*)::BIGINT AS operations, - COALESCE(SUM(OCTET_LENGTH(data)), 0)::BIGINT AS operation_bytes - FROM - bucket_data - WHERE - group_id = ${{ type: 'int4', value: this.replicationStreamId }} - GROUP BY - bucket_name; - `.rows<{ bucket_name: string; operations: bigint; operation_bytes: bigint }>(); - - const operationStats = new Map( - operationRows.map((row) => [ - row.bucket_name, - { operations: Number(row.operations), operationBytes: Number(row.operation_bytes) } - ]) - ); - - // Live rows per bucket (one membership per stored row), from each row's bucket memberships. The - // current-data table is version-specific (current_data for v1/v2, v3_current_data for v3), so the table - // name is interpolated from the resolved store rather than parameterised. - const rowCounts = new Map(); - for await (const batch of db.streamRows<{ bucket: string; rows: bigint }>({ - statement: ` - SELECT - elem ->> 'bucket' AS bucket, - COUNT(*)::BIGINT AS rows - FROM - ${this.currentDataStore.table} cd, - jsonb_array_elements(cd.buckets) AS elem - WHERE - cd.group_id = $1 - GROUP BY - elem ->> 'bucket' - `, - params: [{ type: 'int4', value: this.replicationStreamId }] - })) { - for (const row of batch) { - rowCounts.set(row.bucket, Number(row.rows)); - } - } - - return { operationStats, rowCounts }; - }); - - return storage.buildBucketReport(operationStats, rowCounts, options); - } - async populatePersistentChecksumCache(_options: PopulateChecksumCacheOptions): Promise { // no-op - checksum cache is not implemented for Postgres yet return { buckets: 0 }; diff --git a/modules/module-postgres-storage/test/src/storage.test.ts b/modules/module-postgres-storage/test/src/storage.test.ts index 18c074f96..098810576 100644 --- a/modules/module-postgres-storage/test/src/storage.test.ts +++ b/modules/module-postgres-storage/test/src/storage.test.ts @@ -19,9 +19,6 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { describe(`Postgres Sync Bucket Storage - Checkpoints - v${storageVersion}`, () => register.registerDataStorageCheckpointTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion })); - describe(`Postgres Sync Bucket Storage - Bucket report - v${storageVersion}`, () => - register.registerBucketReportTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion })); - describe(`Postgres Sync Bucket Storage - pg-specific - v${storageVersion}`, () => { /** * The split of returned results can vary depending on storage drivers. diff --git a/packages/service-errors/src/codes.ts b/packages/service-errors/src/codes.ts index a530a3ff9..e423dd25f 100644 --- a/packages/service-errors/src/codes.ts +++ b/packages/service-errors/src/codes.ts @@ -492,13 +492,7 @@ export enum ErrorCode { */ PSYNC_S2404 = 'PSYNC_S2404', - // ## PSYNC_S25xx: Sync API errors - Postgres Storage - - /** - * Postgres storage query timed out. Could be due to a large query or a temporary load issue on the - * storage database. Retry the request. - */ - PSYNC_S2501 = 'PSYNC_S2501', + // ## PSYNC_S23xx: Sync API errors - Postgres Storage // ## PSYNC_S3xxx: Service configuration issues From cf1b79050e5dda42e4ce5aa359b44505de6aa85b Mon Sep 17 00:00:00 2001 From: bean1352 Date: Mon, 29 Jun 2026 13:08:43 +0200 Subject: [PATCH 17/40] Update bucket report tests for the sampling contract --- .../src/tests/register-bucket-report-tests.ts | 15 ++- .../test/src/bucket-report.test.ts | 126 +++++++++--------- .../test/src/routes/admin.test.ts | 26 +++- 3 files changed, 89 insertions(+), 78 deletions(-) diff --git a/packages/service-core-tests/src/tests/register-bucket-report-tests.ts b/packages/service-core-tests/src/tests/register-bucket-report-tests.ts index 82cae7aca..024cdfed5 100644 --- a/packages/service-core-tests/src/tests/register-bucket-report-tests.ts +++ b/packages/service-core-tests/src/tests/register-bucket-report-tests.ts @@ -5,8 +5,10 @@ import * as test_utils from '../test-utils/test-utils-index.js'; /** * Tests for {@link storage.SyncRulesBucketStorage.getBucketReport}: per-bucket operations vs live rows. * - * Asserts on stable counts (operations, rows, fragmentation, totals) rather than op_ids or checksums, - * which differ between storage backends and versions. + * Asserts on stable counts (operations, rows, fragmentation, operation totals) rather than op_ids or + * checksums, which differ between storage backends and versions. The buckets here are tiny (well under the + * row-sample target), so row counts are exact (`rowsEstimated: false`); the sampling path is exercised in + * the higher-volume manual tests. */ export function registerBucketReportTests(config: storage.TestStorageConfig) { const generateStorageFactory = config.factory; @@ -63,9 +65,9 @@ bucket_definitions: const stats = report.buckets.find((b) => b.bucket === bucket)!; // Three inserts of distinct ids: three operations, three live rows, fully compacted (ratio 1). - expect(stats).toMatchObject({ operations: 3, rows: 3, fragmentation: 1 }); + expect(stats).toMatchObject({ operations: 3, rows: 3, fragmentation: 1, rowsEstimated: false }); expect(stats.operationBytes).toBeGreaterThan(0); - expect(report.totals).toMatchObject({ operations: 3, rows: 3 }); + expect(report.totals).toMatchObject({ operations: 3, estimated: false }); }); test('operations exceed live rows after updates, and compaction reduces fragmentation', async () => { @@ -153,8 +155,7 @@ bucket_definitions: const report = await getReport(bucketStorage); expect(report.totals.bucketCount).toEqual(2); - // Instance-wide fragmentation is the row-weighted ratio 5/3, not the mean of the per-bucket ratios (3 and 1). - expect(report.totals).toMatchObject({ operations: 5, rows: 3, fragmentation: 5 / 3 }); + expect(report.totals).toMatchObject({ operations: 5, estimated: false }); // Ranked worst-first by operation count: b1 (3) before b2 (2). expect(report.buckets.map((b) => b.bucket)).toEqual([b1, b2]); @@ -207,6 +208,6 @@ bucket_definitions: expect(report.buckets.map((b) => b.bucket)).toEqual([b1]); // Totals still cover every bucket, not just the truncated list. expect(report.totals.bucketCount).toEqual(2); - expect(report.totals).toMatchObject({ operations: 3, rows: 2 }); + expect(report.totals).toMatchObject({ operations: 3, estimated: false }); }); } diff --git a/packages/service-core/test/src/bucket-report.test.ts b/packages/service-core/test/src/bucket-report.test.ts index e2c7a0968..2a1e330e7 100644 --- a/packages/service-core/test/src/bucket-report.test.ts +++ b/packages/service-core/test/src/bucket-report.test.ts @@ -1,94 +1,90 @@ -import { buildBucketReport, type BucketOperationStat } from '@/storage/bucket-report.js'; +import { + assembleBucketReport, + BucketReportTotals, + DEFAULT_BUCKET_REPORT_LIMIT, + RankedBucketInput, + resolveBucketReportLimit +} from '@/storage/bucket-report.js'; import { describe, expect, it } from 'vitest'; -describe('buildBucketReport', () => { - const ops = (operations: number, operationBytes = 0): BucketOperationStat => ({ operations, operationBytes }); +describe('assembleBucketReport', () => { + const bucket = ( + name: string, + operations: number, + rows: number, + extra?: Partial + ): RankedBucketInput => ({ + bucket: name, + operations, + rows, + operationBytes: extra?.operationBytes ?? 0, + rowsEstimated: extra?.rowsEstimated ?? false + }); + + const totals = (bucketCount: number, extra?: Partial): BucketReportTotals => ({ + bucketCount, + operations: extra?.operations ?? 0, + operationBytes: extra?.operationBytes ?? 0, + estimated: extra?.estimated ?? false + }); - it('merges operation stats and row counts per bucket and derives fragmentation', () => { - const report = buildBucketReport( - new Map([ - ['global[]', ops(100, 1024)], - ['by_user["u1"]', ops(10, 256)] - ]), - new Map([ - ['global[]', 10], - ['by_user["u1"]', 10] - ]) + it('derives fragmentation and passes through rowsEstimated', () => { + const report = assembleBucketReport( + [bucket('global[]', 100, 10, { operationBytes: 1024 }), bucket('by_user["u1"]', 30, 30, { rowsEstimated: true })], + totals(2) ); - const global = report.buckets.find((b) => b.bucket === 'global[]')!; - expect(global).toMatchObject({ + expect(report.buckets.find((b) => b.bucket === 'global[]')).toMatchObject({ operations: 100, rows: 10, operationBytes: 1024, - fragmentation: 10 + fragmentation: 10, + rowsEstimated: false + }); + expect(report.buckets.find((b) => b.bucket === 'by_user["u1"]')).toMatchObject({ + fragmentation: 1, + rowsEstimated: true }); - - const byUser = report.buckets.find((b) => b.bucket === 'by_user["u1"]')!; - expect(byUser.fragmentation).toBe(1); }); - it('ranks buckets worst-first by operations', () => { - const report = buildBucketReport( - new Map([ - ['a[]', ops(5)], - ['b[]', ops(50)], - ['c[]', ops(20)] - ]), - new Map() - ); + it('ranks buckets worst-first by operations then fragmentation', () => { + const report = assembleBucketReport([bucket('a[]', 5, 5), bucket('b[]', 50, 5), bucket('c[]', 50, 50)], totals(3)); + // b and c both have 50 ops; b is more fragmented (10 vs 1) so it ranks first. expect(report.buckets.map((b) => b.bucket)).toEqual(['b[]', 'c[]', 'a[]']); }); - it('treats a bucket with operations but no live rows as fully fragmented (rows floored at 1)', () => { - const report = buildBucketReport(new Map([['gone[]', ops(42)]]), new Map()); + it('floors rows at 1 so a bucket with operations but no rows is fully fragmented', () => { + const report = assembleBucketReport([bucket('gone[]', 42, 0)], totals(1)); expect(report.buckets[0]).toMatchObject({ operations: 42, rows: 0, fragmentation: 42 }); }); - it('includes buckets that have rows but no recorded operations', () => { - const report = buildBucketReport(new Map(), new Map([['fresh[]', 7]])); + it('marks truncated when there are more buckets than returned', () => { + const truncated = assembleBucketReport([bucket('a[]', 10, 1), bucket('b[]', 5, 1)], totals(5)); + expect(truncated.truncated).toBe(true); - expect(report.buckets[0]).toMatchObject({ bucket: 'fresh[]', operations: 0, rows: 7, fragmentation: 0 }); + const complete = assembleBucketReport([bucket('a[]', 10, 1), bucket('b[]', 5, 1)], totals(2)); + expect(complete.truncated).toBe(false); }); - it('computes instance-wide totals across all buckets', () => { - const report = buildBucketReport( - new Map([ - ['a[]', ops(100, 10)], - ['b[]', ops(20, 5)] - ]), - new Map([ - ['a[]', 4], - ['b[]', 2] - ]) - ); + it('carries the totals through unchanged', () => { + const t = totals(2, { operations: 120, operationBytes: 15, estimated: true }); + const report = assembleBucketReport([bucket('a[]', 100, 4), bucket('b[]', 20, 2)], t); - // fragmentation is the row-weighted ratio 120/6 = 20, not the mean of the per-bucket ratios (25 and 10). - expect(report.totals).toEqual({ bucketCount: 2, operations: 120, rows: 6, operationBytes: 15, fragmentation: 20 }); + expect(report.totals).toEqual({ bucketCount: 2, operations: 120, operationBytes: 15, estimated: true }); }); +}); - it('truncates the bucket list by limit but keeps totals across all buckets', () => { - const report = buildBucketReport( - new Map([ - ['a[]', ops(100)], - ['b[]', ops(50)], - ['c[]', ops(10)] - ]), - new Map(), - { limit: 2 } - ); - - expect(report.truncated).toBe(true); - expect(report.buckets.map((b) => b.bucket)).toEqual(['a[]', 'b[]']); - expect(report.totals).toMatchObject({ bucketCount: 3, operations: 160 }); +describe('resolveBucketReportLimit', () => { + it('defaults when no limit is given', () => { + expect(resolveBucketReportLimit(undefined)).toBe(DEFAULT_BUCKET_REPORT_LIMIT); }); - it('is not truncated when the limit exceeds the bucket count', () => { - const report = buildBucketReport(new Map([['a[]', ops(1)]]), new Map(), { limit: 10 }); - - expect(report.truncated).toBe(false); - expect(report.buckets).toHaveLength(1); + it('floors and clamps to a positive integer', () => { + expect(resolveBucketReportLimit(2.7)).toBe(2); + expect(resolveBucketReportLimit(-5)).toBe(1); + expect(resolveBucketReportLimit(0)).toBe(1); + expect(resolveBucketReportLimit(20)).toBe(20); }); }); diff --git a/packages/service-core/test/src/routes/admin.test.ts b/packages/service-core/test/src/routes/admin.test.ts index fa2bdce93..be49e840b 100644 --- a/packages/service-core/test/src/routes/admin.test.ts +++ b/packages/service-core/test/src/routes/admin.test.ts @@ -213,10 +213,24 @@ bucket_definitions: describe('bucket-report', () => { const report = { buckets: [ - { bucket: '1#by_user["u1"]', operations: 4750, rows: 95, operationBytes: 1216000, fragmentation: 50 }, - { bucket: '1#global[]', operations: 1000, rows: 1000, operationBytes: 3145728, fragmentation: 1 } + { + bucket: '1#by_user["u1"]', + operations: 4750, + rows: 95, + operationBytes: 1216000, + fragmentation: 50, + rowsEstimated: true + }, + { + bucket: '1#global[]', + operations: 1000, + rows: 1000, + operationBytes: 3145728, + fragmentation: 1, + rowsEstimated: false + } ], - totals: { bucketCount: 2, operations: 5750, rows: 1095, operationBytes: 4361728, fragmentation: 5750 / 1095 }, + totals: { bucketCount: 2, operations: 5750, operationBytes: 4361728, estimated: false }, truncated: false }; @@ -242,14 +256,14 @@ bucket_definitions: operations: 4750, rows: 95, operation_bytes: 1216000, - fragmentation: 50 + fragmentation: 50, + rows_estimated: true }); expect(response.totals).toEqual({ bucket_count: 2, operations: 5750, - rows: 1095, operation_bytes: 4361728, - fragmentation: 5750 / 1095 + estimated: false }); expect(response.truncated).toBe(false); }); From 75825bc417b9dc096ab2b5bfdb8e5667629a7acf Mon Sep 17 00:00:00 2001 From: bean1352 Date: Mon, 29 Jun 2026 13:27:02 +0200 Subject: [PATCH 18/40] Improve bucket report row estimate and sample buckets concurrently --- .../implementation/MongoSyncBucketStorage.ts | 82 +++++++++++++++---- 1 file changed, 66 insertions(+), 16 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index b9df68e43..a9aee2f8a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -79,6 +79,9 @@ const BUCKET_SELECTION_SAMPLE_SIZE = 10_000; */ const BUCKET_ROW_SAMPLE_TARGET = 1_000; +/** Maximum number of per-bucket row-estimate queries to run concurrently while building a report. */ +const BUCKET_ROW_SAMPLE_CONCURRENCY = 10; + /** A worst-offender bucket selected from bucket_state, with the version-specific context needed to sample it. */ export interface TopBucketCandidate { bucket: string; @@ -407,17 +410,29 @@ export abstract class MongoSyncBucketStorage // Rank the worst-offender buckets and total operations from the pre-aggregated bucket state (bounded, // in the database), then estimate each returned bucket's row count by sampling its operation history. const { buckets, totals } = await this.collectTopBuckets(limit); - const ranked: storage.RankedBucketInput[] = []; - for (const candidate of buckets) { - const estimate = await this.estimateBucketRows(candidate); - ranked.push({ - bucket: candidate.bucket, - operations: candidate.operations, - operationBytes: candidate.operationBytes, - rows: estimate.rows, - rowsEstimated: estimate.estimated - }); - } + // Each bucket's row estimate is an independent query; run a bounded number concurrently so the report + // cost scales with the limit without firing one query per bucket serially. + const ranked: storage.RankedBucketInput[] = new Array(buckets.length); + let cursor = 0; + const runWorker = async () => { + while (true) { + const index = cursor++; + if (index >= buckets.length) { + return; + } + const candidate = buckets[index]; + const estimate = await this.estimateBucketRows(candidate); + ranked[index] = { + bucket: candidate.bucket, + operations: candidate.operations, + operationBytes: candidate.operationBytes, + rows: estimate.rows, + rowsEstimated: estimate.estimated + }; + } + }; + const workers = Math.min(BUCKET_ROW_SAMPLE_CONCURRENCY, buckets.length); + await Promise.all(Array.from({ length: workers }, () => runWorker())); return storage.assembleBucketReport(ranked, totals); } catch (e) { // Translate a storage query timeout (maxTimeMS) into a specific, retryable error code rather than a @@ -540,9 +555,9 @@ export abstract class MongoSyncBucketStorage * Estimate a bucket's live rows from a sample of its operations. * * `pipelinePrefix` must select the bucket's operations (and, when `sampled`, randomly down-sample them) and - * yield documents with top-level `op`, `table` and `row_id` fields. Fragmentation is then - * `sampledOps / distinctRows` and the row count is `operations / fragmentation`. Exact (not sampled) when - * the whole bucket fits within the sample target. + * yield documents with top-level `op`, `table` and `row_id` fields. Returns the distinct row count (exact + * when the whole bucket was read, otherwise estimated via {@link estimateDistinctRows}); fragmentation is + * then `operations / rows`. */ protected async estimateRowsFromOperationSample( collection: mongo.Collection, @@ -575,8 +590,43 @@ export abstract class MongoSyncBucketStorage // Nothing row-bearing was sampled (e.g. a bucket of only MOVE/CLEAR ops): treat as fully fragmented. return { rows: 0, estimated: sampled }; } - // fragmentation = sampledOps / distinctRows; rows = operations / fragmentation = operations * distinctRows / sampledOps. - return { rows: Math.round((operations * distinctRows) / sampledOps), estimated: sampled }; + if (!sampled) { + // Read in full: the distinct row count is exact. + return { rows: distinctRows, estimated: false }; + } + return { rows: this.estimateDistinctRows(operations, sampledOps, distinctRows), estimated: true }; + } + + /** + * Estimate the true distinct row count of a bucket from a sample of its operations. + * + * Each operation is included in the sample with probability `r = sampledOps / operations`, so a row with + * `k` operations is seen with probability `1 - (1 - r)^k`. Assuming operations are spread roughly evenly + * across rows (so each of `R` rows has about `operations / R` of them), the expected number of distinct + * rows in the sample is `R * (1 - (1 - r)^(operations / R))`. This is monotonic in `R`, so we binary-search + * for the `R` that matches the observed distinct count. + * + * The naive `distinctRows / r` over-counts rows (and so under-states fragmentation) whenever the sample + * already covered most rows - exactly the highly-fragmented buckets the report exists to surface. + */ + protected estimateDistinctRows(operations: number, sampledOps: number, distinctRows: number): number { + const r = Math.min(1, sampledOps / operations); + if (r >= 1) { + return distinctRows; + } + const expectedDistinct = (rows: number) => rows * (1 - Math.pow(1 - r, operations / rows)); + // True distinct count is between the observed distinct (a lower bound) and one row per operation. + let lo = distinctRows; + let hi = operations; + for (let i = 0; i < 60; i++) { + const mid = (lo + hi) / 2; + if (expectedDistinct(mid) < distinctRows) { + lo = mid; + } else { + hi = mid; + } + } + return Math.round((lo + hi) / 2); } /** Whether a bucket with this many operations should be sampled rather than read in full. */ From 368502253d2e714f7e6787dc55e4e5ea0aaa284f Mon Sep 17 00:00:00 2001 From: bean1352 Date: Wed, 1 Jul 2026 13:23:03 +0200 Subject: [PATCH 19/40] Improve bucket report row and total estimates --- .../implementation/MongoSyncBucketStorage.ts | 81 +++++++++---------- .../service-core/src/storage/bucket-report.ts | 35 ++++++++ .../test/src/bucket-report.test.ts | 32 ++++++++ 3 files changed, 106 insertions(+), 42 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 69664309c..67c2e8e7f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -74,10 +74,16 @@ const BUCKET_SELECTION_SAMPLE_THRESHOLD = 50_000; const BUCKET_SELECTION_SAMPLE_SIZE = 10_000; /** - * Target number of operations to sample per bucket when estimating its row count. Buckets with fewer - * operations than this are read in full (exact); larger buckets are sampled down to roughly this many. + * Fewest operations sampled per bucket when estimating its row count. Buckets with fewer operations than + * this are read in full (exact). */ -const BUCKET_ROW_SAMPLE_TARGET = 1_000; +const BUCKET_ROW_SAMPLE_MIN = 1_000; + +/** + * Most operations sampled per bucket, capping the per-bucket cost on very large buckets at the price of a + * weaker estimate for buckets that are both extremely wide and barely fragmented (see {@link bucketRowSampleTarget}). + */ +const BUCKET_ROW_SAMPLE_MAX = 25_000; /** Maximum number of per-bucket row-estimate queries to run concurrently while building a report. */ const BUCKET_ROW_SAMPLE_CONCURRENCY = 10; @@ -486,10 +492,12 @@ export abstract class MongoSyncBucketStorage ] }; - // estimatedDocumentCount ignores the match filter, so this is an upper bound on the active bucket count. - // That is fine: it only decides whether to sample, and over-estimating just switches to sampling sooner. - const totalBuckets = await collection.estimatedDocumentCount(); - const sampled = totalBuckets > BUCKET_SELECTION_SAMPLE_THRESHOLD; + // estimatedDocumentCount is O(1) but ignores the match filter, so this is an upper bound on the active + // bucket count. That is fine for the sampling decision: over-estimating only switches to sampling sooner. + // It must NOT be used to scale the sampled totals though - the collection can hold buckets outside the + // match (other replication groups for v1/v2, inactive definitions for v3), which would over-scale. + const estimatedTotalBuckets = await collection.estimatedDocumentCount(); + const sampled = estimatedTotalBuckets > BUCKET_SELECTION_SAMPLE_THRESHOLD; const pipeline: mongo.Document[] = [{ $match: match }]; if (sampled) { @@ -538,12 +546,16 @@ export abstract class MongoSyncBucketStorage }; } - // Scale the sampled totals up to the full collection. - const scale = totalBuckets / Math.max(rawTotals.bucketCount, 1); + // Scale the sampled totals up to the full *matched* set. countDocuments respects the match filter (so it + // excludes other groups / inactive definitions) and uses the _id index; it only runs on the already-large + // sampled path, and is bounded by maxTimeMS like the rest of the report. When the matched set fits within + // the sample, rawTotals is already exact and the scale collapses to 1. + const matchedBuckets = await collection.countDocuments(match, { maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS }); + const scale = matchedBuckets / Math.max(rawTotals.bucketCount, 1); return { buckets, totals: { - bucketCount: totalBuckets, + bucketCount: matchedBuckets, operations: Math.round(rawTotals.operations * scale), operationBytes: Math.round(rawTotals.operationBytes * scale), estimated: true @@ -556,7 +568,7 @@ export abstract class MongoSyncBucketStorage * * `pipelinePrefix` must select the bucket's operations (and, when `sampled`, randomly down-sample them) and * yield documents with top-level `op`, `table` and `row_id` fields. Returns the distinct row count (exact - * when the whole bucket was read, otherwise estimated via {@link estimateDistinctRows}); fragmentation is + * when the whole bucket was read, otherwise estimated via {@link storage.estimateDistinctRows}); fragmentation is * then `operations / rows`. */ protected async estimateRowsFromOperationSample( @@ -594,49 +606,34 @@ export abstract class MongoSyncBucketStorage // Read in full: the distinct row count is exact. return { rows: distinctRows, estimated: false }; } - return { rows: this.estimateDistinctRows(operations, sampledOps, distinctRows), estimated: true }; + return { rows: storage.estimateDistinctRows(operations, sampledOps, distinctRows), estimated: true }; } /** - * Estimate the true distinct row count of a bucket from a sample of its operations. - * - * Each operation is included in the sample with probability `r = sampledOps / operations`, so a row with - * `k` operations is seen with probability `1 - (1 - r)^k`. Assuming operations are spread roughly evenly - * across rows (so each of `R` rows has about `operations / R` of them), the expected number of distinct - * rows in the sample is `R * (1 - (1 - r)^(operations / R))`. This is monotonic in `R`, so we binary-search - * for the `R` that matches the observed distinct count. + * How many operations to sample when estimating a bucket's row count. * - * The naive `distinctRows / r` over-counts rows (and so under-states fragmentation) whenever the sample - * already covered most rows - exactly the highly-fragmented buckets the report exists to surface. + * {@link storage.estimateDistinctRows} recovers the true row count from how often the sample lands on the + * same row twice ("collisions"). A bucket with `R` rows produces collisions only once the sample size + * approaches `sqrt(R)`, and needs roughly `sqrt(100 * R)` before they carry a usable signal. `R` is unknown + * up front but is bounded by the operation count, so sampling `sqrt(200 * operations)` operations yields on + * the order of 100 expected collisions even in the worst case of one row per operation - enough to keep the + * estimate stable rather than swinging with sampling noise. Clamped to [MIN, MAX] to bound per-bucket cost; + * above the MAX-implied width the estimate degrades gracefully (only for buckets both very wide and barely + * fragmented, which are not the fragmented offenders the report exists to surface). */ - protected estimateDistinctRows(operations: number, sampledOps: number, distinctRows: number): number { - const r = Math.min(1, sampledOps / operations); - if (r >= 1) { - return distinctRows; - } - const expectedDistinct = (rows: number) => rows * (1 - Math.pow(1 - r, operations / rows)); - // True distinct count is between the observed distinct (a lower bound) and one row per operation. - let lo = distinctRows; - let hi = operations; - for (let i = 0; i < 60; i++) { - const mid = (lo + hi) / 2; - if (expectedDistinct(mid) < distinctRows) { - lo = mid; - } else { - hi = mid; - } - } - return Math.round((lo + hi) / 2); + protected bucketRowSampleTarget(operations: number): number { + const target = Math.ceil(Math.sqrt(200 * operations)); + return Math.min(BUCKET_ROW_SAMPLE_MAX, Math.max(BUCKET_ROW_SAMPLE_MIN, target)); } /** Whether a bucket with this many operations should be sampled rather than read in full. */ protected shouldSampleBucketRows(operations: number): boolean { - return operations > BUCKET_ROW_SAMPLE_TARGET; + return operations > this.bucketRowSampleTarget(operations); } - /** `$sampleRate` for sampling roughly {@link BUCKET_ROW_SAMPLE_TARGET} of a bucket's operations. */ + /** `$sampleRate` for sampling roughly {@link bucketRowSampleTarget} operations from a bucket. */ protected bucketRowSampleRate(operations: number): number { - return BUCKET_ROW_SAMPLE_TARGET / operations; + return this.bucketRowSampleTarget(operations) / operations; } /** diff --git a/packages/service-core/src/storage/bucket-report.ts b/packages/service-core/src/storage/bucket-report.ts index 3823034b6..c0db63134 100644 --- a/packages/service-core/src/storage/bucket-report.ts +++ b/packages/service-core/src/storage/bucket-report.ts @@ -94,6 +94,41 @@ export function resolveBucketReportLimit(limit?: number): number { return Math.max(1, Math.floor(limit)); } +/** + * Estimate the true distinct row count of a bucket from a sample of its operations. + * + * Each operation is included in the sample with probability `r = sampledOps / operations`, so a row with + * `k` operations is seen with probability `1 - (1 - r)^k`. Assuming operations are spread roughly evenly + * across rows (so each of `R` rows has about `operations / R` of them), the expected number of distinct + * rows in the sample is `R * (1 - (1 - r)^(operations / R))`. This is monotonic in `R`, so we binary-search + * for the `R` that matches the observed distinct count. + * + * The naive `distinctRows / r` over-counts rows (and so under-states fragmentation) whenever the sample + * already covered most rows - exactly the highly-fragmented buckets the report exists to surface. + * + * Pure (no I/O) so it is unit-testable; storage adapters supply the sampled counts. + */ +export function estimateDistinctRows(operations: number, sampledOps: number, distinctRows: number): number { + const r = Math.min(1, sampledOps / operations); + if (r >= 1) { + return distinctRows; + } + const expectedDistinct = (rows: number) => rows * (1 - Math.pow(1 - r, operations / rows)); + // The true row count is between the observed distinct count (a lower bound) and one row per operation. + // Binary-search that range until it is narrower than a single row, at which point rounding is exact. + let lo = distinctRows; + let hi = operations; + while (hi - lo > 0.5) { + const mid = (lo + hi) / 2; + if (expectedDistinct(mid) < distinctRows) { + lo = mid; + } else { + hi = mid; + } + } + return Math.round((lo + hi) / 2); +} + /** * Assemble the final {@link BucketReport} from per-bucket stats and instance-wide totals. Storage adapters * select and sample the buckets however is cheapest for them; this owns the shared fragmentation / ranking / diff --git a/packages/service-core/test/src/bucket-report.test.ts b/packages/service-core/test/src/bucket-report.test.ts index 2a1e330e7..6b506340a 100644 --- a/packages/service-core/test/src/bucket-report.test.ts +++ b/packages/service-core/test/src/bucket-report.test.ts @@ -2,6 +2,7 @@ import { assembleBucketReport, BucketReportTotals, DEFAULT_BUCKET_REPORT_LIMIT, + estimateDistinctRows, RankedBucketInput, resolveBucketReportLimit } from '@/storage/bucket-report.js'; @@ -76,6 +77,37 @@ describe('assembleBucketReport', () => { }); }); +describe('estimateDistinctRows', () => { + it('returns the observed distinct count when the whole bucket was sampled', () => { + // r >= 1: nothing was left out, so the observed distinct count is already exact. + expect(estimateDistinctRows(100, 100, 40)).toBe(40); + expect(estimateDistinctRows(100, 150, 40)).toBe(40); + }); + + it('recovers a heavily fragmented bucket the naive estimate would inflate', () => { + // 10 rows x 1000 ops each; a 10% sample sees ~1000 ops but still only the same 10 distinct rows. + // Naive distinct/rate would report 10 / 0.1 = 100 rows (10x too many, so 10x too little fragmentation). + const rows = estimateDistinctRows(10_000, 1_000, 10); + expect(rows).toBeGreaterThanOrEqual(9); + expect(rows).toBeLessThanOrEqual(12); + }); + + it('recovers a moderately fragmented bucket', () => { + // 500 rows x 2 ops each, 50% sample. Ground truth: 500*(1-0.5^2) = 375 distinct sampled rows. + // Naive distinct/rate would report 375 / 0.5 = 750 rows; the estimator should recover ~500. + const rows = estimateDistinctRows(1_000, 500, 375); + expect(rows).toBeGreaterThan(480); + expect(rows).toBeLessThan(520); + }); + + it('matches the naive estimate when there are no sampling collisions', () => { + // 2000 rows, 1 op each, 50% sample: no row is seen twice, so distinct/rate is already correct (~2000). + const rows = estimateDistinctRows(2_000, 1_000, 1_000); + expect(rows).toBeGreaterThan(1_900); + expect(rows).toBeLessThan(2_100); + }); +}); + describe('resolveBucketReportLimit', () => { it('defaults when no limit is given', () => { expect(resolveBucketReportLimit(undefined)).toBe(DEFAULT_BUCKET_REPORT_LIMIT); From 6a7d10f06812866a5a50dce06f1600a2165dbccc Mon Sep 17 00:00:00 2001 From: bean1352 Date: Wed, 1 Jul 2026 13:23:19 +0200 Subject: [PATCH 20/40] Use the _id index for bucket report row sampling --- .../implementation/v1/MongoSyncBucketStorageV1.ts | 13 ++++++++++++- .../implementation/v3/MongoSyncBucketStorageV3.ts | 8 ++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index ab9db340d..dfe345e68 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -202,7 +202,18 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage { protected estimateBucketRows(candidate: TopBucketCandidate): Promise { // v1/v2 store one document per operation, so a bucket's ops are an id-prefix range that can be sampled directly. const sampled = this.shouldSampleBucketRows(candidate.operations); - const prefix: mongo.Document[] = [{ $match: { '_id.g': this.replicationStreamId, '_id.b': candidate.bucket } }]; + // Range-match on the whole `_id` (g, b, o) so the {_id} index is used; a dotted `{'_id.g','_id.b'}` match + // cannot use the compound-object index and would scan the whole collection per bucket. + const prefix: mongo.Document[] = [ + { + $match: { + _id: idPrefixFilter<{ g: number; b: string; o: unknown }>( + { g: this.replicationStreamId, b: candidate.bucket }, + ['o'] + ) + } + } + ]; if (sampled) { prefix.push({ $match: { $sampleRate: this.bucketRowSampleRate(candidate.operations) } }); } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index 37203b13e..d6063f0a5 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -15,7 +15,7 @@ import { import { JSONBig } from '@powersync/service-jsonbig'; import { ParameterLookupRows, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules'; import * as bson from 'bson'; -import { mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js'; +import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js'; import { MongoBucketStorage } from '../../MongoBucketStorage.js'; import { BucketDataDoc } from '../common/BucketDataDoc.js'; import { MongoSyncBucketStorageCheckpoint } from '../common/MongoSyncBucketStorageCheckpoint.js'; @@ -212,7 +212,11 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { // Sample whole batch documents, then unwind to operation level so the shared estimator sees one doc per op. const sampled = this.shouldSampleBucketRows(candidate.operations); const collection = this.db.bucketData(this.replicationStreamId, candidate.defId!); - const prefix: mongo.Document[] = [{ $match: { '_id.b': candidate.bucket } }]; + // Range-match on the whole `_id` (b, o) so the {_id} index is used; a dotted `{'_id.b': ...}` match + // cannot use the compound-object index and would scan the whole collection per bucket. + const prefix: mongo.Document[] = [ + { $match: { _id: idPrefixFilter<{ b: string; o: unknown }>({ b: candidate.bucket }, ['o']) } } + ]; if (sampled) { prefix.push({ $match: { $sampleRate: this.bucketRowSampleRate(candidate.operations) } }); } From d09548ae7eb848d392b85dd0857d96aba6fbdf9f Mon Sep 17 00:00:00 2001 From: bean1352 Date: Wed, 1 Jul 2026 13:23:33 +0200 Subject: [PATCH 21/40] Clarify bucket report limit docs --- .../src/tests/register-bucket-report-tests.ts | 4 ++-- packages/types/src/routes.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/service-core-tests/src/tests/register-bucket-report-tests.ts b/packages/service-core-tests/src/tests/register-bucket-report-tests.ts index 024cdfed5..6cbbbb181 100644 --- a/packages/service-core-tests/src/tests/register-bucket-report-tests.ts +++ b/packages/service-core-tests/src/tests/register-bucket-report-tests.ts @@ -162,8 +162,8 @@ bucket_definitions: expect(report.buckets.find((b) => b.bucket === b1)).toMatchObject({ operations: 3, rows: 1 }); expect(report.buckets.find((b) => b.bucket === b2)).toMatchObject({ operations: 2, rows: 2 }); - // operationBytes is aggregated differently per backend ($toDouble sum on Mongo, OCTET_LENGTH sum on - // Postgres); assert every bucket is non-zero and that the per-bucket bytes add up to the instance total. + // operationBytes is an aggregated ($toDouble) sum; assert every bucket is non-zero and that the + // per-bucket bytes add up to the instance total. expect(report.totals.operationBytes).toBeGreaterThan(0); for (const bucket of report.buckets) { expect(bucket.operationBytes).toBeGreaterThan(0); diff --git a/packages/types/src/routes.ts b/packages/types/src/routes.ts index 179dc6c3f..ed742541c 100644 --- a/packages/types/src/routes.ts +++ b/packages/types/src/routes.ts @@ -81,8 +81,8 @@ export type ValidateResponse = t.Encoded; export const BucketReportRequest = t.object({ /** * Maximum number of buckets to return, ranked by operation count descending (worst offenders first). - * Caps the response only, not the query cost: totals are still computed across all buckets. Omit for - * no limit. + * Row counts are sampled per returned bucket, so this also bounds the report's cost. Defaults to 50 when + * omitted; non-integer or negative values are floored and clamped to 1. */ limit: t.number.optional() }); From 00ed2de0e17dfa08eeaf303cef9cc152f55348ea Mon Sep 17 00:00:00 2001 From: bean1352 Date: Wed, 1 Jul 2026 14:03:12 +0200 Subject: [PATCH 22/40] Fix bucket report comments and collection typing --- .../src/storage/implementation/MongoSyncBucketStorage.ts | 4 ++-- packages/service-core/src/routes/endpoints/admin.ts | 5 ++--- packages/service-core/src/storage/SyncRulesBucketStorage.ts | 3 +-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index c9dc7cdbf..e170d406c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -581,8 +581,8 @@ export abstract class MongoSyncBucketStorage * when the whole bucket was read, otherwise estimated via {@link storage.estimateDistinctRows}); fragmentation is * then `operations / rows`. */ - protected async estimateRowsFromOperationSample( - collection: mongo.Collection, + protected async estimateRowsFromOperationSample( + collection: mongo.Collection, pipelinePrefix: mongo.Document[], operations: number, sampled: boolean diff --git a/packages/service-core/src/routes/endpoints/admin.ts b/packages/service-core/src/routes/endpoints/admin.ts index 7e3d4216b..9676c8c23 100644 --- a/packages/service-core/src/routes/endpoints/admin.ts +++ b/packages/service-core/src/routes/endpoints/admin.ts @@ -271,9 +271,8 @@ export const validate = routeDefinition({ /** * Per-bucket report of total operations vs total live rows in storage, for the active sync config. * - * Answers the recurring "why is my Data Synced so high" question instance-wide - * a high `operations / rows` ratio indicates fragmented buckets that a compact or - * defragment can reclaim. + * Answers the recurring "why is my Data Synced so high" question. A high `operations / rows` ratio + * indicates fragmented buckets that a compact or defragment can reclaim. */ export const bucketReport = routeDefinition({ path: '/api/admin/v1/bucket-report', diff --git a/packages/service-core/src/storage/SyncRulesBucketStorage.ts b/packages/service-core/src/storage/SyncRulesBucketStorage.ts index 24a0109b5..7fef0b0db 100644 --- a/packages/service-core/src/storage/SyncRulesBucketStorage.ts +++ b/packages/service-core/src/storage/SyncRulesBucketStorage.ts @@ -173,8 +173,7 @@ export interface SyncRulesBucketStorage * Per-bucket report of total operations vs total live rows in storage. * * Intended for an on-demand admin/diagnostics view (e.g. answering "why is my Data Synced so high"), - * not as a live gauge. Operation and live-row counts are aggregated from storage; the exact source is - * backend-specific (MongoDB reads pre-aggregated bucket state, Postgres scans bucket data). May be + * not as a live gauge. How the counts are derived is backend-specific, and the report may be relatively * expensive on large instances. * * Optional: storage providers that don't implement it are reported as unsupported by the API. From 62016e9721b36daff0329553f0206c80324162bd Mon Sep 17 00:00:00 2001 From: bean1352 Date: Wed, 1 Jul 2026 15:20:19 +0200 Subject: [PATCH 23/40] Test bucket report row sampling and handle empty samples --- .../implementation/MongoSyncBucketStorage.ts | 64 +++++++++++-------- .../v1/MongoSyncBucketStorageV1.ts | 31 +++++---- .../v3/MongoSyncBucketStorageV3.ts | 23 ++++--- .../src/tests/register-bucket-report-tests.ts | 56 ++++++++++++++++ 4 files changed, 124 insertions(+), 50 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index e170d406c..6c75f2e5d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -576,40 +576,52 @@ export abstract class MongoSyncBucketStorage /** * Estimate a bucket's live rows from a sample of its operations. * - * `pipelinePrefix` must select the bucket's operations (and, when `sampled`, randomly down-sample them) and - * yield documents with top-level `op`, `table` and `row_id` fields. Returns the distinct row count (exact - * when the whole bucket was read, otherwise estimated via {@link storage.estimateDistinctRows}); fragmentation is - * then `operations / rows`. + * `buildPrefix(applySample)` returns a pipeline prefix that selects the bucket's operations (down-sampled + * when `applySample` is true) and yields documents with top-level `op`, `table` and `row_id` fields. + * Returns the distinct row count (exact when the whole bucket was read, otherwise estimated via + * {@link storage.estimateDistinctRows}); fragmentation is then `operations / rows`. */ protected async estimateRowsFromOperationSample( collection: mongo.Collection, - pipelinePrefix: mongo.Document[], + buildPrefix: (applySample: boolean) => mongo.Document[], operations: number, sampled: boolean ): Promise { - const pipeline: mongo.Document[] = [ - ...pipelinePrefix, - { - $facet: { - sampledOps: [{ $count: 'count' }], - distinctRows: [ - { $match: { op: { $in: ['PUT', 'REMOVE'] } } }, - { $group: { _id: { table: '$table', row_id: '$row_id' } } }, - { $count: 'count' } - ] + const runCounts = async (applySample: boolean) => { + const pipeline: mongo.Document[] = [ + ...buildPrefix(applySample), + { + $facet: { + sampledOps: [{ $count: 'count' }], + distinctRows: [ + { $match: { op: { $in: ['PUT', 'REMOVE'] } } }, + { $group: { _id: { table: '$table', row_id: '$row_id' } } }, + { $count: 'count' } + ] + } } - } - ]; - - type FacetResult = { sampledOps: { count: number }[]; distinctRows: { count: number }[] }; - const [result] = await collection - .aggregate(pipeline, { allowDiskUse: false, maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS }) - .toArray(); + ]; + type FacetResult = { sampledOps: { count: number }[]; distinctRows: { count: number }[] }; + const [result] = await collection + .aggregate(pipeline, { allowDiskUse: false, maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS }) + .toArray(); + return { + sampledOps: result?.sampledOps[0]?.count ?? 0, + distinctRows: result?.distinctRows[0]?.count ?? 0 + }; + }; - const sampledOps = result?.sampledOps[0]?.count ?? 0; - const distinctRows = result?.distinctRows[0]?.count ?? 0; - if (sampledOps == 0 || distinctRows == 0) { - // Nothing row-bearing was sampled (e.g. a bucket of only MOVE/CLEAR ops): treat as fully fragmented. + let { sampledOps, distinctRows } = await runCounts(sampled); + if (sampled && sampledOps == 0) { + // A document-level `$sampleRate` can select nothing when a bucket spans very few storage documents + // (v3 batches operations into a document). Fall back to an exact read so the bucket is not reported as + // zero rows. This reads the whole bucket only in the rare empty-sample case, which cannot happen for a + // bucket large enough to span many documents. + distinctRows = (await runCounts(false)).distinctRows; + return { rows: distinctRows, estimated: false }; + } + if (distinctRows == 0) { + // Nothing row-bearing was found (e.g. a bucket of only MOVE/CLEAR ops): treat as fully fragmented. return { rows: 0, estimated: sampled }; } if (!sampled) { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index 9dae4bd92..2ee0dfa39 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -204,22 +204,25 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage { protected estimateBucketRows(candidate: TopBucketCandidate): Promise { // v1/v2 store one document per operation, so a bucket's ops are an id-prefix range that can be sampled directly. const sampled = this.shouldSampleBucketRows(candidate.operations); - // Range-match on the whole `_id` (g, b, o) so the {_id} index is used; a dotted `{'_id.g','_id.b'}` match - // cannot use the compound-object index and would scan the whole collection per bucket. - const prefix: mongo.Document[] = [ - { - $match: { - _id: idPrefixFilter<{ g: number; b: string; o: unknown }>( - { g: this.replicationStreamId, b: candidate.bucket }, - ['o'] - ) + const buildPrefix = (applySample: boolean): mongo.Document[] => { + // Range-match on the whole `_id` (g, b, o) so the {_id} index is used; a dotted `{'_id.g','_id.b'}` match + // cannot use the compound-object index and would scan the whole collection per bucket. + const prefix: mongo.Document[] = [ + { + $match: { + _id: idPrefixFilter<{ g: number; b: string; o: unknown }>( + { g: this.replicationStreamId, b: candidate.bucket }, + ['o'] + ) + } } + ]; + if (applySample) { + prefix.push({ $match: { $sampleRate: this.bucketRowSampleRate(candidate.operations) } }); } - ]; - if (sampled) { - prefix.push({ $match: { $sampleRate: this.bucketRowSampleRate(candidate.operations) } }); - } - return this.estimateRowsFromOperationSample(this.db.bucketDataV1, prefix, candidate.operations, sampled); + return prefix; + }; + return this.estimateRowsFromOperationSample(this.db.bucketDataV1, buildPrefix, candidate.operations, sampled); } protected createMongoParameterCompactor( diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index 9bb6b4945..fe2d0328f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -214,16 +214,19 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { // Sample whole batch documents, then unwind to operation level so the shared estimator sees one doc per op. const sampled = this.shouldSampleBucketRows(candidate.operations); const collection = this.db.bucketData(this.replicationStreamId, candidate.defId!); - // Range-match on the whole `_id` (b, o) so the {_id} index is used; a dotted `{'_id.b': ...}` match - // cannot use the compound-object index and would scan the whole collection per bucket. - const prefix: mongo.Document[] = [ - { $match: { _id: idPrefixFilter<{ b: string; o: unknown }>({ b: candidate.bucket }, ['o']) } } - ]; - if (sampled) { - prefix.push({ $match: { $sampleRate: this.bucketRowSampleRate(candidate.operations) } }); - } - prefix.push({ $unwind: '$ops' }, { $replaceRoot: { newRoot: '$ops' } }); - return this.estimateRowsFromOperationSample(collection, prefix, candidate.operations, sampled); + const buildPrefix = (applySample: boolean): mongo.Document[] => { + // Range-match on the whole `_id` (b, o) so the {_id} index is used; a dotted `{'_id.b': ...}` match + // cannot use the compound-object index and would scan the whole collection per bucket. + const prefix: mongo.Document[] = [ + { $match: { _id: idPrefixFilter<{ b: string; o: unknown }>({ b: candidate.bucket }, ['o']) } } + ]; + if (applySample) { + prefix.push({ $match: { $sampleRate: this.bucketRowSampleRate(candidate.operations) } }); + } + prefix.push({ $unwind: '$ops' }, { $replaceRoot: { newRoot: '$ops' } }); + return prefix; + }; + return this.estimateRowsFromOperationSample(collection, buildPrefix, candidate.operations, sampled); } protected createMongoParameterCompactor( diff --git a/packages/service-core-tests/src/tests/register-bucket-report-tests.ts b/packages/service-core-tests/src/tests/register-bucket-report-tests.ts index 6cbbbb181..5a3b16078 100644 --- a/packages/service-core-tests/src/tests/register-bucket-report-tests.ts +++ b/packages/service-core-tests/src/tests/register-bucket-report-tests.ts @@ -210,4 +210,60 @@ bucket_definitions: expect(report.totals.bucketCount).toEqual(2); expect(report.totals).toMatchObject({ operations: 3, estimated: false }); }); + + test('samples the row count for a bucket above the sampling threshold', async () => { + await using factory = await generateStorageFactory(); + const { stream, content } = await test_utils.deploySyncRules( + factory, + updateSyncRulesFromYaml(GLOBAL_SYNC_RULES, { storageVersion }) + ); + const bucketStorage = factory.getInstance(stream); + + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + + // 50 rows, each updated 25 times, is 1,300 operations against 50 live rows. That is past the 1,000 + // operation threshold, so the report samples the operation history rather than reading it in full and + // the row count comes back as an estimate. The value per update varies so no two writes are identical. + // Each round is flushed separately so the operations span many storage documents, as they would in real + // replication (some backends batch operations per document, and a sample must see more than one). + const rowCount = 50; + const updatesPerRow = 25; + for (let row = 0; row < rowCount; row++) { + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { id: `r${row}` }, + afterReplicaId: test_utils.rid(`r${row}`) + }); + } + await writer.commit('1/1'); + await writer.flush(); + for (let update = 0; update < updatesPerRow; update++) { + for (let row = 0; row < rowCount; row++) { + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.UPDATE, + after: { id: `r${row}`, value: `v${update}` }, + afterReplicaId: test_utils.rid(`r${row}`) + }); + } + await writer.commit('1/1'); + await writer.flush(); + } + + const bucket = test_utils.bucketRequest(content, 'global[]').bucket; + const report = await getReport(bucketStorage); + const stats = report.buckets.find((b) => b.bucket === bucket)!; + + // The operation count is exact (read from bucket_state); the row count is a sampled estimate. + expect(stats.operations).toEqual(rowCount + rowCount * updatesPerRow); + expect(stats.rowsEstimated).toEqual(true); + // The sample covers enough of a bucket this fragmented to recover the 50 live rows within a small margin. + expect(stats.rows).toBeGreaterThanOrEqual(45); + expect(stats.rows).toBeLessThanOrEqual(55); + // Fragmentation is operations / rows, so a heavily updated bucket reads well above 1. + expect(stats.fragmentation).toBeGreaterThan(10); + }); } From 674bd85cc3b91a41076deac59429daad4f16920d Mon Sep 17 00:00:00 2001 From: bean1352 Date: Thu, 2 Jul 2026 11:26:19 +0200 Subject: [PATCH 24/40] Clean up comments for readability --- .../implementation/MongoSyncBucketStorage.ts | 14 ++++++-------- .../service-core/src/storage/bucket-report.ts | 17 +++++++++-------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 6c75f2e5d..8c7a67644 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -634,14 +634,12 @@ export abstract class MongoSyncBucketStorage /** * How many operations to sample when estimating a bucket's row count. * - * {@link storage.estimateDistinctRows} recovers the true row count from how often the sample lands on the - * same row twice ("collisions"). A bucket with `R` rows produces collisions only once the sample size - * approaches `sqrt(R)`, and needs roughly `sqrt(100 * R)` before they carry a usable signal. `R` is unknown - * up front but is bounded by the operation count, so sampling `sqrt(200 * operations)` operations yields on - * the order of 100 expected collisions even in the worst case of one row per operation - enough to keep the - * estimate stable rather than swinging with sampling noise. Clamped to [MIN, MAX] to bound per-bucket cost; - * above the MAX-implied width the estimate degrades gracefully (only for buckets both very wide and barely - * fragmented, which are not the fragmented offenders the report exists to surface). + * {@link storage.estimateDistinctRows} infers the row count from how often the sample lands on the same + * row twice, so the sample must be large enough to contain such repeats. Sampling `sqrt(200 * operations)` + * operations yields on the order of 100 expected repeats even in the worst case of one row per operation, + * which keeps the estimate stable instead of swinging with sampling noise. The clamp bounds per-bucket + * cost; past the cap only very wide, barely fragmented buckets lose accuracy, and those are not the + * offenders the report exists to surface. */ protected bucketRowSampleTarget(operations: number): number { const target = Math.ceil(Math.sqrt(200 * operations)); diff --git a/packages/service-core/src/storage/bucket-report.ts b/packages/service-core/src/storage/bucket-report.ts index c0db63134..c502e4fa4 100644 --- a/packages/service-core/src/storage/bucket-report.ts +++ b/packages/service-core/src/storage/bucket-report.ts @@ -95,16 +95,17 @@ export function resolveBucketReportLimit(limit?: number): number { } /** - * Estimate the true distinct row count of a bucket from a sample of its operations. + * Estimate the true distinct row count of a bucket from a random sample of its operations. * - * Each operation is included in the sample with probability `r = sampledOps / operations`, so a row with - * `k` operations is seen with probability `1 - (1 - r)^k`. Assuming operations are spread roughly evenly - * across rows (so each of `R` rows has about `operations / R` of them), the expected number of distinct - * rows in the sample is `R * (1 - (1 - r)^(operations / R))`. This is monotonic in `R`, so we binary-search - * for the `R` that matches the observed distinct count. + * The signal is repetition: a sample that keeps landing on the same rows means few rows, while a sample + * where every operation lands on a new row means many. Formally, each operation is included in the sample + * with probability `r = sampledOps / operations`, so a row with `k` operations appears with probability + * `1 - (1 - r)^k`. Assuming operations are spread roughly evenly across `R` rows (`k = operations / R`), + * the expected number of distinct rows in the sample is `R * (1 - (1 - r)^(operations / R))`. That grows + * with `R`, so a binary search finds the `R` matching the observed distinct count. * - * The naive `distinctRows / r` over-counts rows (and so under-states fragmentation) whenever the sample - * already covered most rows - exactly the highly-fragmented buckets the report exists to surface. + * The naive `distinctRows / r` ignores repetition and over-counts rows (under-stating fragmentation) on + * exactly the highly fragmented buckets the report exists to surface. * * Pure (no I/O) so it is unit-testable; storage adapters supply the sampled counts. */ From d4eb5cc611d872df02d6cfe7c16708d775c826cc Mon Sep 17 00:00:00 2001 From: bean1352 Date: Thu, 2 Jul 2026 11:52:29 +0200 Subject: [PATCH 25/40] Exclude compaction MOVE ops from bucket report row estimates --- .../implementation/MongoSyncBucketStorage.ts | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 8c7a67644..089112c43 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -593,6 +593,7 @@ export abstract class MongoSyncBucketStorage { $facet: { sampledOps: [{ $count: 'count' }], + rowOps: [{ $match: { op: { $in: ['PUT', 'REMOVE'] } } }, { $count: 'count' }], distinctRows: [ { $match: { op: { $in: ['PUT', 'REMOVE'] } } }, { $group: { _id: { table: '$table', row_id: '$row_id' } } }, @@ -601,34 +602,46 @@ export abstract class MongoSyncBucketStorage } } ]; - type FacetResult = { sampledOps: { count: number }[]; distinctRows: { count: number }[] }; + type FacetResult = { + sampledOps: { count: number }[]; + rowOps: { count: number }[]; + distinctRows: { count: number }[]; + }; const [result] = await collection .aggregate(pipeline, { allowDiskUse: false, maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS }) .toArray(); return { sampledOps: result?.sampledOps[0]?.count ?? 0, + rowOps: result?.rowOps[0]?.count ?? 0, distinctRows: result?.distinctRows[0]?.count ?? 0 }; }; - let { sampledOps, distinctRows } = await runCounts(sampled); - if (sampled && sampledOps == 0) { + let counts = await runCounts(sampled); + if (sampled && counts.sampledOps == 0) { // A document-level `$sampleRate` can select nothing when a bucket spans very few storage documents // (v3 batches operations into a document). Fall back to an exact read so the bucket is not reported as // zero rows. This reads the whole bucket only in the rare empty-sample case, which cannot happen for a // bucket large enough to span many documents. - distinctRows = (await runCounts(false)).distinctRows; - return { rows: distinctRows, estimated: false }; + return { rows: (await runCounts(false)).distinctRows, estimated: false }; } - if (distinctRows == 0) { + if (counts.distinctRows == 0) { // Nothing row-bearing was found (e.g. a bucket of only MOVE/CLEAR ops): treat as fully fragmented. return { rows: 0, estimated: sampled }; } if (!sampled) { // Read in full: the distinct row count is exact. - return { rows: distinctRows, estimated: false }; + return { rows: counts.distinctRows, estimated: false }; } - return { rows: storage.estimateDistinctRows(operations, sampledOps, distinctRows), estimated: true }; + // Only PUT/REMOVE operations carry a row identity; MOVE/CLEAR (produced by compaction) do not. Run the + // estimator over the row-bearing operations only, scaling the bucket's operation count by the row-bearing + // share observed in the sample. Including identity-less operations in the model under-counts rows on + // compacted buckets. For uncompacted buckets rowOps equals sampledOps and this changes nothing. + const rowBearingOperations = Math.round(operations * (counts.rowOps / counts.sampledOps)); + return { + rows: storage.estimateDistinctRows(rowBearingOperations, counts.rowOps, counts.distinctRows), + estimated: true + }; } /** From 7d8fa995f593a8f10a46d56fd8066ece95c15afc Mon Sep 17 00:00:00 2001 From: bean1352 Date: Thu, 2 Jul 2026 13:01:43 +0200 Subject: [PATCH 26/40] Add definition rollup, action suggestions, and tables to the bucket report --- .../implementation/MongoSyncBucketStorage.ts | 195 ++++++++++++++---- .../v1/MongoSyncBucketStorageV1.ts | 37 +++- .../v3/MongoSyncBucketStorageV3.ts | 31 ++- .../src/tests/register-bucket-report-tests.ts | 84 +++++++- .../src/routes/endpoints/admin.ts | 18 +- .../service-core/src/storage/bucket-report.ts | 172 ++++++++++++++- .../test/src/bucket-report.test.ts | 173 +++++++++++++--- .../test/src/routes/admin.test.ts | 44 +++- packages/types/src/routes.ts | 51 ++++- 9 files changed, 714 insertions(+), 91 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 089112c43..9a8183d4c 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -90,6 +90,9 @@ const BUCKET_ROW_SAMPLE_MAX = 25_000; /** Maximum number of per-bucket row-estimate queries to run concurrently while building a report. */ const BUCKET_ROW_SAMPLE_CONCURRENCY = 10; +/** Maximum number of tables listed per bucket or definition in the report. */ +const BUCKET_REPORT_TABLE_LIMIT = 10; + /** A worst-offender bucket selected from bucket_state, with the version-specific context needed to sample it. */ export interface TopBucketCandidate { bucket: string; @@ -99,15 +102,33 @@ export interface TopBucketCandidate { defId?: BucketDefinitionId; } +/** A bucket definition aggregated from bucket_state, with the context needed to sample its rows. */ +export interface TopDefinitionCandidate { + /** Definition name as it prefixes bucket names, e.g. `1#by_user`. */ + definition: string; + bucketCount: number; + operations: number; + operationBytes: number; + /** v3 only: the bucket definition id, used to locate its per-definition bucket_data collection. */ + defId?: BucketDefinitionId; +} + export interface TopBucketSelection { buckets: TopBucketCandidate[]; + definitions: TopDefinitionCandidate[]; + /** True if more definitions exist than `definitions` holds ({@link storage.BUCKET_REPORT_DEFINITION_LIMIT}). */ + definitionsTruncated: boolean; totals: storage.BucketReportTotals; } export interface BucketRowEstimate { rows: number; - /** True if `rows` is a sampled estimate rather than an exact count. */ + /** Operations carrying a row identity (PUT/REMOVE), i.e. excluding MOVE/CLEAR compaction residue. */ + rowOperations: number; + /** True if `rows` and `rowOperations` are sampled estimates rather than exact counts. */ estimated: boolean; + /** Tables in the (sampled) row-bearing history, ordered by their share of it, largest first. */ + tables: string[]; } export abstract class MongoSyncBucketStorage @@ -423,33 +444,71 @@ export abstract class MongoSyncBucketStorage async getBucketReport(options?: storage.GetBucketReportOptions): Promise { const limit = storage.resolveBucketReportLimit(options?.limit); try { - // Rank the worst-offender buckets and total operations from the pre-aggregated bucket state (bounded, - // in the database), then estimate each returned bucket's row count by sampling its operation history. - const { buckets, totals } = await this.collectTopBuckets(limit); - // Each bucket's row estimate is an independent query; run a bounded number concurrently so the report - // cost scales with the limit without firing one query per bucket serially. - const ranked: storage.RankedBucketInput[] = new Array(buckets.length); + // Rank the worst-offender buckets, the per-definition rollup, and total operations from the + // pre-aggregated bucket state (bounded, in the database), then estimate each returned bucket's and + // definition's row count by sampling its operation history. + const { buckets, definitions, definitionsTruncated, totals } = await this.collectTopBuckets(limit); + // Each row estimate is an independent query; run a bounded number concurrently so the report cost + // scales with the limit without firing one query per bucket serially. Definitions sample their whole + // history and are the slowest jobs, so dispatch them first to overlap with the per-bucket estimates. + const rankedBuckets: storage.RankedBucketInput[] = new Array(buckets.length); + const rankedDefinitions: storage.RankedDefinitionInput[] = new Array(definitions.length); + const jobs = buckets.length + definitions.length; let cursor = 0; const runWorker = async () => { while (true) { const index = cursor++; - if (index >= buckets.length) { + if (index >= jobs) { return; } - const candidate = buckets[index]; - const estimate = await this.estimateBucketRows(candidate); - ranked[index] = { - bucket: candidate.bucket, - operations: candidate.operations, - operationBytes: candidate.operationBytes, - rows: estimate.rows, - rowsEstimated: estimate.estimated - }; + if (index < definitions.length) { + const candidate = definitions[index]; + // A definition's row sample reads its whole (sampled) history, which on a very large instance + // can exceed the time budget even when the per-bucket estimates are fine. The rollup is + // supplementary: omit the definition rather than failing the whole report. + try { + const estimate = await this.estimateDefinitionRows(candidate); + rankedDefinitions[index] = { + definition: candidate.definition, + bucketCount: candidate.bucketCount, + operations: candidate.operations, + operationBytes: candidate.operationBytes, + rows: estimate.rows, + rowOperations: estimate.rowOperations, + rowsEstimated: estimate.estimated, + tables: estimate.tables + }; + } catch (e) { + this.logger.warn( + `Skipping bucket report rollup for definition ${candidate.definition}: row sampling failed`, + e + ); + } + } else { + const candidate = buckets[index - definitions.length]; + const estimate = await this.estimateBucketRows(candidate); + rankedBuckets[index - definitions.length] = { + bucket: candidate.bucket, + operations: candidate.operations, + operationBytes: candidate.operationBytes, + rows: estimate.rows, + rowOperations: estimate.rowOperations, + rowsEstimated: estimate.estimated, + tables: estimate.tables + }; + } } }; - const workers = Math.min(BUCKET_ROW_SAMPLE_CONCURRENCY, buckets.length); + const workers = Math.min(BUCKET_ROW_SAMPLE_CONCURRENCY, jobs); await Promise.all(Array.from({ length: workers }, () => runWorker())); - return storage.assembleBucketReport(ranked, totals); + const sampledDefinitions = rankedDefinitions.filter((d) => d != null); + return storage.assembleBucketReport( + rankedBuckets, + sampledDefinitions, + totals, + // The rollup is also incomplete if a definition was dropped because sampling it failed. + definitionsTruncated || sampledDefinitions.length < definitions.length + ); } catch (e) { // Translate a storage query timeout (maxTimeMS) into a specific, retryable error code rather than a // generic internal error. @@ -458,9 +517,10 @@ export abstract class MongoSyncBucketStorage } /** - * Select the worst-offender buckets (by operation count) plus instance-wide operation totals from the - * pre-aggregated bucket state. Ranking and limiting happen in the database, so memory stays bounded. - * Implementations supply their version-specific bucket state collection and active-config filter. + * Select the worst-offender buckets (by operation count), the per-definition rollup, and instance-wide + * operation totals from the pre-aggregated bucket state. Ranking and limiting happen in the database, so + * memory stays bounded. Implementations supply their version-specific bucket state collection and + * active-config filter. */ protected abstract collectTopBuckets(limit: number): Promise; @@ -470,6 +530,12 @@ export abstract class MongoSyncBucketStorage */ protected abstract estimateBucketRows(candidate: TopBucketCandidate): Promise; + /** + * Estimate a whole definition's row count (a row counted once per bucket containing it) by sampling the + * definition's operation history, exactly like {@link estimateBucketRows} but at definition grain. + */ + protected abstract estimateDefinitionRows(candidate: TopDefinitionCandidate): Promise; + /** * Rank buckets by operation count in the database and compute instance-wide operation totals, reading the * pre-aggregated bucket state (compacted_state + estimate_since_compact). One document per bucket, no scan @@ -490,6 +556,8 @@ export abstract class MongoSyncBucketStorage limit: number ): Promise<{ buckets: { id: T['_id']; operations: number; operationBytes: number }[]; + definitions: TopDefinitionCandidate[]; + definitionsTruncated: boolean; totals: storage.BucketReportTotals; }> { const operations = { @@ -501,6 +569,10 @@ export abstract class MongoSyncBucketStorage { $toDouble: { $ifNull: ['$estimate_since_compact.bytes', 0] } } ] }; + // Bucket names are `[]`, so everything before the first `[` groups a + // bucket into its definition. v3 additionally carries the definition id in `_id.d`; `$first` is exact + // because all buckets sharing a name prefix share the definition (undefined for v1/v2). + const definitionKey = { $arrayElemAt: [{ $split: ['$_id.b', '['] }, 0] }; // estimatedDocumentCount is O(1) but ignores the match filter, so this is an upper bound on the active // bucket count. That is fine for the sampling decision: over-estimating only switches to sampling sooner. @@ -525,13 +597,34 @@ export abstract class MongoSyncBucketStorage } } ], - top: [{ $project: { _id: 1, operations, operationBytes } }, { $sort: { operations: -1 } }, { $limit: limit }] + top: [{ $project: { _id: 1, operations, operationBytes } }, { $sort: { operations: -1 } }, { $limit: limit }], + definitions: [ + { + $group: { + _id: definitionKey, + operations: { $sum: operations }, + operationBytes: { $sum: operationBytes }, + bucketCount: { $sum: 1 }, + defId: { $first: '$_id.d' } + } + }, + { $sort: { operations: -1 } }, + // One past the cap: an extra result only signals that the rollup was truncated. + { $limit: storage.BUCKET_REPORT_DEFINITION_LIMIT + 1 } + ] } }); type FacetResult = { totals: { operations: number; operationBytes: number; bucketCount: number }[]; top: { _id: T['_id']; operations: number; operationBytes: number }[]; + definitions: { + _id: string; + operations: number; + operationBytes: number; + bucketCount: number; + defId?: BucketDefinitionId; + }[]; }; const [result] = await collection .aggregate(pipeline, { allowDiskUse: false, maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS }) @@ -543,10 +636,22 @@ export abstract class MongoSyncBucketStorage operations: doc.operations, operationBytes: doc.operationBytes })); + const rawDefinitions = result?.definitions ?? []; + const definitionsTruncated = rawDefinitions.length > storage.BUCKET_REPORT_DEFINITION_LIMIT; + const mapDefinitions = (scale: number): TopDefinitionCandidate[] => + rawDefinitions.slice(0, storage.BUCKET_REPORT_DEFINITION_LIMIT).map((d) => ({ + definition: d._id, + bucketCount: Math.round(d.bucketCount * scale), + operations: Math.round(d.operations * scale), + operationBytes: Math.round(d.operationBytes * scale), + defId: d.defId + })); if (!sampled) { return { buckets, + definitions: mapDefinitions(1), + definitionsTruncated, totals: { bucketCount: rawTotals.bucketCount, operations: rawTotals.operations, @@ -559,11 +664,15 @@ export abstract class MongoSyncBucketStorage // Scale the sampled totals up to the full *matched* set. countDocuments respects the match filter (so it // excludes other groups / inactive definitions) and uses the _id index; it only runs on the already-large // sampled path, and is bounded by maxTimeMS like the rest of the report. When the matched set fits within - // the sample, rawTotals is already exact and the scale collapses to 1. + // the sample, rawTotals is already exact and the scale collapses to 1. The sample is uniform across + // buckets, so the per-definition sums scale by the same factor; a definition small enough to be missed + // by the sample entirely is absent. const matchedBuckets = await collection.countDocuments(match, { maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS }); const scale = matchedBuckets / Math.max(rawTotals.bucketCount, 1); return { buckets, + definitions: mapDefinitions(scale), + definitionsTruncated, totals: { bucketCount: matchedBuckets, operations: Math.round(rawTotals.operations * scale), @@ -574,18 +683,23 @@ export abstract class MongoSyncBucketStorage } /** - * Estimate a bucket's live rows from a sample of its operations. + * Estimate a bucket's (or definition's) live rows from a sample of its operations. * - * `buildPrefix(applySample)` returns a pipeline prefix that selects the bucket's operations (down-sampled - * when `applySample` is true) and yields documents with top-level `op`, `table` and `row_id` fields. - * Returns the distinct row count (exact when the whole bucket was read, otherwise estimated via + * `buildPrefix(applySample)` returns a pipeline prefix that selects the operations (down-sampled when + * `applySample` is true) and yields documents with top-level `op`, `table` and `row_id` fields. Returns + * the distinct row count (exact when the whole history was read, otherwise estimated via * {@link storage.estimateDistinctRows}); fragmentation is then `operations / rows`. + * + * `rowKey` is the `$group` key that identifies a row. Per-bucket estimates use the default (the bucket is + * fixed by the prefix); definition-level estimates must include the bucket name so a row is counted once + * per bucket containing it. */ protected async estimateRowsFromOperationSample( collection: mongo.Collection, buildPrefix: (applySample: boolean) => mongo.Document[], operations: number, - sampled: boolean + sampled: boolean, + rowKey: mongo.Document = { table: '$table', row_id: '$row_id' } ): Promise { const runCounts = async (applySample: boolean) => { const pipeline: mongo.Document[] = [ @@ -596,8 +710,14 @@ export abstract class MongoSyncBucketStorage rowOps: [{ $match: { op: { $in: ['PUT', 'REMOVE'] } } }, { $count: 'count' }], distinctRows: [ { $match: { op: { $in: ['PUT', 'REMOVE'] } } }, - { $group: { _id: { table: '$table', row_id: '$row_id' } } }, + { $group: { _id: rowKey } }, { $count: 'count' } + ], + tables: [ + { $match: { op: { $in: ['PUT', 'REMOVE'] } } }, + { $group: { _id: '$table', operations: { $sum: 1 } } }, + { $sort: { operations: -1 } }, + { $limit: BUCKET_REPORT_TABLE_LIMIT } ] } } @@ -606,6 +726,7 @@ export abstract class MongoSyncBucketStorage sampledOps: { count: number }[]; rowOps: { count: number }[]; distinctRows: { count: number }[]; + tables: { _id: string }[]; }; const [result] = await collection .aggregate(pipeline, { allowDiskUse: false, maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS }) @@ -613,7 +734,8 @@ export abstract class MongoSyncBucketStorage return { sampledOps: result?.sampledOps[0]?.count ?? 0, rowOps: result?.rowOps[0]?.count ?? 0, - distinctRows: result?.distinctRows[0]?.count ?? 0 + distinctRows: result?.distinctRows[0]?.count ?? 0, + tables: (result?.tables ?? []).map((t) => t._id) }; }; @@ -623,15 +745,16 @@ export abstract class MongoSyncBucketStorage // (v3 batches operations into a document). Fall back to an exact read so the bucket is not reported as // zero rows. This reads the whole bucket only in the rare empty-sample case, which cannot happen for a // bucket large enough to span many documents. - return { rows: (await runCounts(false)).distinctRows, estimated: false }; + const exact = await runCounts(false); + return { rows: exact.distinctRows, rowOperations: exact.rowOps, estimated: false, tables: exact.tables }; } if (counts.distinctRows == 0) { // Nothing row-bearing was found (e.g. a bucket of only MOVE/CLEAR ops): treat as fully fragmented. - return { rows: 0, estimated: sampled }; + return { rows: 0, rowOperations: 0, estimated: sampled, tables: [] }; } if (!sampled) { // Read in full: the distinct row count is exact. - return { rows: counts.distinctRows, estimated: false }; + return { rows: counts.distinctRows, rowOperations: counts.rowOps, estimated: false, tables: counts.tables }; } // Only PUT/REMOVE operations carry a row identity; MOVE/CLEAR (produced by compaction) do not. Run the // estimator over the row-bearing operations only, scaling the bucket's operation count by the row-bearing @@ -640,7 +763,9 @@ export abstract class MongoSyncBucketStorage const rowBearingOperations = Math.round(operations * (counts.rowOps / counts.sampledOps)); return { rows: storage.estimateDistinctRows(rowBearingOperations, counts.rowOps, counts.distinctRows), - estimated: true + rowOperations: rowBearingOperations, + estimated: true, + tables: counts.tables }; } diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index 2ee0dfa39..89605aacd 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -29,7 +29,8 @@ import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions, TopBucketCandidate, - TopBucketSelection + TopBucketSelection, + TopDefinitionCandidate } from '../MongoSyncBucketStorage.js'; import { BucketDataDocumentV1, @@ -190,13 +191,15 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage { // For storage v1/v2, bucket state and bucket data are shared collections scoped by group (replication stream). protected async collectTopBuckets(limit: number): Promise { - const { buckets, totals } = await this.aggregateTopBuckets( + const { buckets, definitions, definitionsTruncated, totals } = await this.aggregateTopBuckets( this.db.bucketStateV1, { '_id.g': this.replicationStreamId }, limit ); return { buckets: buckets.map((b) => ({ bucket: b.id.b, operations: b.operations, operationBytes: b.operationBytes })), + definitions, + definitionsTruncated, totals }; } @@ -225,6 +228,36 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage { return this.estimateRowsFromOperationSample(this.db.bucketDataV1, buildPrefix, candidate.operations, sampled); } + protected estimateDefinitionRows(candidate: TopDefinitionCandidate): Promise { + const sampled = this.shouldSampleBucketRows(candidate.operations); + const buildPrefix = (applySample: boolean): mongo.Document[] => { + // All of a definition's bucket names start with `[`, so an `_id` range on that string + // prefix selects exactly the definition's operations via the index. `\\` (0x5C) is the character + // after `[` (0x5B), so [`[`, `\\`) cannot include any other definition: + // a longer definition name would have to differ at or before the `[`. + const prefix: mongo.Document[] = [ + { + $match: { + _id: { + $gte: { g: this.replicationStreamId, b: `${candidate.definition}[`, o: new bson.MinKey() }, + $lt: { g: this.replicationStreamId, b: `${candidate.definition}\\`, o: new bson.MinKey() } + } + } + } + ]; + if (applySample) { + prefix.push({ $match: { $sampleRate: this.bucketRowSampleRate(candidate.operations) } }); + } + return prefix; + }; + // Include the bucket name in the row key: at definition grain a row counts once per bucket holding it. + return this.estimateRowsFromOperationSample(this.db.bucketDataV1, buildPrefix, candidate.operations, sampled, { + b: '$_id.b', + table: '$table', + row_id: '$row_id' + }); + } + protected createMongoParameterCompactor( checkpoint: InternalOpId, options: storage.CompactOptions diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index fe2d0328f..64bd78d91 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -28,7 +28,8 @@ import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions, TopBucketCandidate, - TopBucketSelection + TopBucketSelection, + TopDefinitionCandidate } from '../MongoSyncBucketStorage.js'; import { loadBucketDataDocument } from './bucket-format.js'; import { @@ -193,7 +194,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { // sharing these collections. Scope to the active config's definition ids so the report excludes stale buckets // from old/stopped definitions. `this.storageIds` is derived from the active config only (see getActiveSyncConfig). protected async collectTopBuckets(limit: number): Promise { - const { buckets, totals } = await this.aggregateTopBuckets( + const { buckets, definitions, definitionsTruncated, totals } = await this.aggregateTopBuckets( this.db.bucketState(this.replicationStreamId), { '_id.d': { $in: this.storageIds.bucketDefinitionIds } }, limit @@ -205,6 +206,8 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { operationBytes: b.operationBytes, defId: b.id.d })), + definitions, + definitionsTruncated, totals }; } @@ -229,6 +232,30 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { return this.estimateRowsFromOperationSample(collection, buildPrefix, candidate.operations, sampled); } + protected estimateDefinitionRows(candidate: TopDefinitionCandidate): Promise { + // A definition's operations are exactly its per-definition bucket_data collection, so no match stage is + // needed. Keep the bucket name alongside each unwound operation: at definition grain a row counts once + // per bucket holding it. + const sampled = this.shouldSampleBucketRows(candidate.operations); + const collection = this.db.bucketData(this.replicationStreamId, candidate.defId!); + const buildPrefix = (applySample: boolean): mongo.Document[] => { + const prefix: mongo.Document[] = []; + if (applySample) { + prefix.push({ $match: { $sampleRate: this.bucketRowSampleRate(candidate.operations) } }); + } + prefix.push( + { $unwind: '$ops' }, + { $project: { b: '$_id.b', op: '$ops.op', table: '$ops.table', row_id: '$ops.row_id' } } + ); + return prefix; + }; + return this.estimateRowsFromOperationSample(collection, buildPrefix, candidate.operations, sampled, { + b: '$b', + table: '$table', + row_id: '$row_id' + }); + } + protected createMongoParameterCompactor( checkpoint: InternalOpId, options: storage.CompactOptions diff --git a/packages/service-core-tests/src/tests/register-bucket-report-tests.ts b/packages/service-core-tests/src/tests/register-bucket-report-tests.ts index 5a3b16078..3c3d8e58b 100644 --- a/packages/service-core-tests/src/tests/register-bucket-report-tests.ts +++ b/packages/service-core-tests/src/tests/register-bucket-report-tests.ts @@ -61,13 +61,33 @@ bucket_definitions: const report = await getReport(bucketStorage); expect(report.totals.bucketCount).toEqual(1); - expect(report.truncated).toEqual(false); + expect(report.bucketsTruncated).toEqual(false); + expect(report.definitionsTruncated).toEqual(false); const stats = report.buckets.find((b) => b.bucket === bucket)!; // Three inserts of distinct ids: three operations, three live rows, fully compacted (ratio 1). - expect(stats).toMatchObject({ operations: 3, rows: 3, fragmentation: 1, rowsEstimated: false }); + expect(stats).toMatchObject({ + operations: 3, + rows: 3, + fragmentation: 1, + rowsEstimated: false, + suggestedAction: 'none', + tables: ['test'] + }); expect(stats.operationBytes).toBeGreaterThan(0); expect(report.totals).toMatchObject({ operations: 3, estimated: false }); + + // The definition rollup aggregates the single bucket. The definition name is the bucket-name prefix. + expect(report.definitions).toHaveLength(1); + expect(report.definitions[0]).toMatchObject({ + definition: bucket.split('[')[0], + bucketCount: 1, + operations: 3, + rows: 3, + fragmentation: 1, + suggestedAction: 'none', + tables: ['test'] + }); }); test('operations exceed live rows after updates, and compaction reduces fragmentation', async () => { @@ -162,6 +182,15 @@ bucket_definitions: expect(report.buckets.find((b) => b.bucket === b1)).toMatchObject({ operations: 3, rows: 1 }); expect(report.buckets.find((b) => b.bucket === b2)).toMatchObject({ operations: 2, rows: 2 }); + // Both buckets belong to one definition; the rollup sums them, counting each bucket's rows separately. + expect(report.definitions).toHaveLength(1); + expect(report.definitions[0]).toMatchObject({ + definition: b1.split('[')[0], + bucketCount: 2, + operations: 5, + rows: 3 + }); + // operationBytes is an aggregated ($toDouble) sum; assert every bucket is non-zero and that the // per-bucket bytes add up to the instance total. expect(report.totals.operationBytes).toBeGreaterThan(0); @@ -204,13 +233,46 @@ bucket_definitions: const b1 = test_utils.bucketRequest(content, 'grouped["b1"]').bucket; const report = await getReport(bucketStorage, { limit: 1 }); - expect(report.truncated).toEqual(true); + expect(report.bucketsTruncated).toEqual(true); expect(report.buckets.map((b) => b.bucket)).toEqual([b1]); // Totals still cover every bucket, not just the truncated list. expect(report.totals.bucketCount).toEqual(2); expect(report.totals).toMatchObject({ operations: 3, estimated: false }); }); + test('caps the definition rollup and flags the truncation', async () => { + // Two definitions past the rollup cap; a single row lands in every definition's global bucket. + const definitionCount = storage.BUCKET_REPORT_DEFINITION_LIMIT + 2; + const manyDefinitions = + 'bucket_definitions:\n' + + Array.from({ length: definitionCount }, (_, i) => ` def${i}:\n data: [select * from test]\n`).join(''); + + await using factory = await generateStorageFactory(); + const { stream } = await test_utils.deploySyncRules( + factory, + updateSyncRulesFromYaml(manyDefinitions, { storageVersion }) + ); + const bucketStorage = factory.getInstance(stream); + + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { id: 't1' }, + afterReplicaId: test_utils.rid('t1') + }); + await writer.commit('1/1'); + await writer.flush(); + + const report = await getReport(bucketStorage); + expect(report.totals.bucketCount).toEqual(definitionCount); + expect(report.bucketsTruncated).toEqual(false); + expect(report.definitions).toHaveLength(storage.BUCKET_REPORT_DEFINITION_LIMIT); + expect(report.definitionsTruncated).toEqual(true); + }); + test('samples the row count for a bucket above the sampling threshold', async () => { await using factory = await generateStorageFactory(); const { stream, content } = await test_utils.deploySyncRules( @@ -265,5 +327,21 @@ bucket_definitions: expect(stats.rows).toBeLessThanOrEqual(55); // Fragmentation is operations / rows, so a heavily updated bucket reads well above 1. expect(stats.fragmentation).toBeGreaterThan(10); + // The history is un-compacted superseded churn, which a compact reclaims. + expect(stats.suggestedAction).toEqual('compact'); + // The sampled history names the tables a defragment would touch. + expect(stats.tables).toEqual(['test']); + + // The definition rollup samples the same history at definition grain. + expect(report.definitions).toHaveLength(1); + const defStats = report.definitions[0]; + expect(defStats).toMatchObject({ + bucketCount: 1, + operations: stats.operations, + suggestedAction: 'compact', + tables: ['test'] + }); + expect(defStats.rows).toBeGreaterThanOrEqual(45); + expect(defStats.rows).toBeLessThanOrEqual(55); }); } diff --git a/packages/service-core/src/routes/endpoints/admin.ts b/packages/service-core/src/routes/endpoints/admin.ts index 9676c8c23..f52aa8bc3 100644 --- a/packages/service-core/src/routes/endpoints/admin.ts +++ b/packages/service-core/src/routes/endpoints/admin.ts @@ -313,7 +313,20 @@ export const bucketReport = routeDefinition({ rows: bucket.rows, operation_bytes: bucket.operationBytes, fragmentation: bucket.fragmentation, - rows_estimated: bucket.rowsEstimated + rows_estimated: bucket.rowsEstimated, + suggested_action: bucket.suggestedAction, + tables: bucket.tables + })), + definitions: report.definitions.map((definition) => ({ + definition: definition.definition, + bucket_count: definition.bucketCount, + operations: definition.operations, + operation_bytes: definition.operationBytes, + rows: definition.rows, + fragmentation: definition.fragmentation, + rows_estimated: definition.rowsEstimated, + suggested_action: definition.suggestedAction, + tables: definition.tables })), totals: { bucket_count: report.totals.bucketCount, @@ -321,7 +334,8 @@ export const bucketReport = routeDefinition({ operation_bytes: report.totals.operationBytes, estimated: report.totals.estimated }, - truncated: report.truncated + buckets_truncated: report.bucketsTruncated, + definitions_truncated: report.definitionsTruncated }); } }); diff --git a/packages/service-core/src/storage/bucket-report.ts b/packages/service-core/src/storage/bucket-report.ts index c502e4fa4..d5920c20a 100644 --- a/packages/service-core/src/storage/bucket-report.ts +++ b/packages/service-core/src/storage/bucket-report.ts @@ -25,6 +25,31 @@ export const BUCKET_REPORT_TIMEOUT_MS: number = 60_000; */ export const DEFAULT_BUCKET_REPORT_LIMIT: number = 50; +/** + * Maximum number of bucket definitions in the report's definition rollup. Rows are sampled per returned + * definition (like per-bucket rows), so this bounds that sampling work. Configs rarely approach this many + * definitions. + */ +export const BUCKET_REPORT_DEFINITION_LIMIT: number = 20; + +/** Fragmentation below this is considered healthy: no maintenance action is suggested. */ +export const BUCKET_ACTION_MIN_FRAGMENTATION: number = 2; + +/** + * When at least this share of a bucket's operations is compaction residue (MOVE/CLEAR, no row identity), + * compaction has already done its work and only a defragment reduces what new clients download. + */ +export const BUCKET_ACTION_RESIDUE_SHARE: number = 0.5; + +/** + * When at least this share of a bucket's row-bearing operations (PUT/REMOVE) is superseded history (more + * operations than rows), a compact reclaims it (as MOVE conversions and a CLEAR prefix). + */ +export const BUCKET_ACTION_SUPERSEDED_SHARE: number = 0.5; + +/** Suggested maintenance action for a bucket or definition. See {@link suggestBucketAction}. */ +export type BucketAction = 'none' | 'compact' | 'defragment' | 'both'; + export interface BucketStorageStats { /** Full bucket name, e.g. `by_user["u1"]`. */ bucket: string; @@ -41,6 +66,41 @@ export interface BucketStorageStats { fragmentation: number; /** True if `rows` (and therefore `fragmentation`) is a sampled estimate rather than an exact count. */ rowsEstimated: boolean; + /** Suggested maintenance action derived from the operation mix. See {@link suggestBucketAction}. */ + suggestedAction: BucketAction; + /** + * Tables making up the (sampled) operation history, ordered by their share of it, largest first. These + * are the tables whose rows a defragment should touch. + */ + tables: string[]; +} + +/** Aggregated stats for one bucket definition (one `bucket_definitions` entry in the sync config). */ +export interface BucketDefinitionStats { + /** Definition name as it prefixes bucket names, e.g. `1#by_user` (versioned in storage v2 and later). */ + definition: string; + /** Number of buckets in this definition with stored operations. */ + bucketCount: number; + /** Total operations across the definition's buckets. */ + operations: number; + /** Approximate size of the definition's operation history in bytes. */ + operationBytes: number; + /** + * Live rows across the definition's buckets, counting a row once per bucket that contains it (the + * download-relevant meaning). Sampled estimate for all but tiny definitions (see `rowsEstimated`). + */ + rows: number; + /** `operations / max(rows, 1)` across the whole definition. */ + fragmentation: number; + /** True if `rows` (and therefore `fragmentation`) is a sampled estimate rather than an exact count. */ + rowsEstimated: boolean; + /** Suggested maintenance action derived from the operation mix. See {@link suggestBucketAction}. */ + suggestedAction: BucketAction; + /** + * Tables making up the (sampled) operation history, ordered by their share of it, largest first. These + * are the tables whose rows a defragment should touch. + */ + tables: string[]; } export interface BucketReportTotals { @@ -60,10 +120,20 @@ export interface BucketReportTotals { export interface BucketReport { /** Worst-offender buckets, ranked by operation count then fragmentation. */ buckets: BucketStorageStats[]; + /** + * Per-definition rollup, ranked by operation count. Answers "which sync-rules definition should I look + * at" where `buckets` answers "which exact buckets". Capped at {@link BUCKET_REPORT_DEFINITION_LIMIT}. + */ + definitions: BucketDefinitionStats[]; /** Instance-wide operation totals. Does not include row counts (those are per-bucket estimates only). */ totals: BucketReportTotals; /** True if there are more buckets than returned (more than `limit`). */ - truncated: boolean; + bucketsTruncated: boolean; + /** + * True if the definition rollup is incomplete: more definitions exist than + * {@link BUCKET_REPORT_DEFINITION_LIMIT}, or a definition was dropped because sampling it failed. + */ + definitionsTruncated: boolean; } export interface GetBucketReportOptions { @@ -81,7 +151,28 @@ export interface RankedBucketInput { operations: number; operationBytes: number; rows: number; + /** + * Operations that carry a row identity (PUT/REMOVE), i.e. everything except compaction residue + * (MOVE/CLEAR). Estimated alongside `rows` for sampled buckets. + */ + rowOperations: number; + rowsEstimated: boolean; + /** Tables in the (sampled) operation history, ordered by their share of it, largest first. */ + tables: string[]; +} + +/** A definition's aggregated operation stats plus its (possibly sampled) row count, before ranking. */ +export interface RankedDefinitionInput { + definition: string; + bucketCount: number; + operations: number; + operationBytes: number; + rows: number; + /** As in {@link RankedBucketInput.rowOperations}, across the whole definition. */ + rowOperations: number; rowsEstimated: boolean; + /** As in {@link RankedBucketInput.tables}, across the whole definition. */ + tables: string[]; } /** @@ -131,26 +222,91 @@ export function estimateDistinctRows(operations: number, sampledOps: number, dis } /** - * Assemble the final {@link BucketReport} from per-bucket stats and instance-wide totals. Storage adapters - * select and sample the buckets however is cheapest for them; this owns the shared fragmentation / ranking / - * truncation logic so it cannot drift. Pure (no I/O) so it is unit-testable. + * Suggest the maintenance action that reduces what new clients download from a bucket, based on its + * operation mix. Grounded in the compaction semantics (see `docs/storage/compacting-operations.md`): + * + * - A **compact** converts superseded PUT/REMOVE operations into MOVE operations (reclaiming their bytes) + * and collapses a leading run of REMOVE/MOVE operations into one CLEAR. It helps when a bucket carries + * un-compacted superseded history: `rowOperations` well above `rows`. + * - A **defragment** (touch every row, then compact) is what collapses the operation count once the history + * is mostly MOVE/CLEAR residue that a compact alone preserves: `operations` well above `rowOperations`. + * - When both kinds of overhead are present, or the mix is inconclusive, suggest both. + * + * The thresholds are heuristics; the report is intended to be re-run after acting on it. Inputs may be + * sampled estimates, which is fine at these margins. */ -export function assembleBucketReport(buckets: RankedBucketInput[], totals: BucketReportTotals): BucketReport { +export function suggestBucketAction(operations: number, rowOperations: number, rows: number): BucketAction { + const fragmentation = operations / Math.max(rows, 1); + if (fragmentation < BUCKET_ACTION_MIN_FRAGMENTATION) { + return 'none'; + } + const residueShare = (operations - rowOperations) / Math.max(operations, 1); + const supersededShare = (rowOperations - rows) / Math.max(rowOperations, 1); + const defragmentNeeded = residueShare >= BUCKET_ACTION_RESIDUE_SHARE; + const compactUseful = supersededShare >= BUCKET_ACTION_SUPERSEDED_SHARE; + if (defragmentNeeded && compactUseful) { + return 'both'; + } + if (defragmentNeeded) { + return 'defragment'; + } + if (compactUseful) { + return 'compact'; + } + // Fragmented, but neither share dominates: a mixed history where a compact reclaims part and the rest + // needs a defragment. + return 'both'; +} + +/** + * Assemble the final {@link BucketReport} from per-bucket stats, per-definition stats, and instance-wide + * totals. Storage adapters select and sample the buckets however is cheapest for them; this owns the shared + * fragmentation / ranking / truncation / action logic so it cannot drift. Pure (no I/O) so it is + * unit-testable. + * + * Bucket truncation is derived from the totals; only the adapter knows whether the definition list was cut, + * so it passes `definitionsTruncated` in. + */ +export function assembleBucketReport( + buckets: RankedBucketInput[], + definitions: RankedDefinitionInput[], + totals: BucketReportTotals, + definitionsTruncated = false +): BucketReport { const stats: BucketStorageStats[] = buckets.map((b) => ({ bucket: b.bucket, operations: b.operations, rows: b.rows, operationBytes: b.operationBytes, fragmentation: b.operations / Math.max(b.rows, 1), - rowsEstimated: b.rowsEstimated + rowsEstimated: b.rowsEstimated, + suggestedAction: suggestBucketAction(b.operations, b.rowOperations, b.rows), + tables: b.tables + })); + + const definitionStats: BucketDefinitionStats[] = definitions.map((d) => ({ + definition: d.definition, + bucketCount: d.bucketCount, + operations: d.operations, + operationBytes: d.operationBytes, + rows: d.rows, + fragmentation: d.operations / Math.max(d.rows, 1), + rowsEstimated: d.rowsEstimated, + suggestedAction: suggestBucketAction(d.operations, d.rowOperations, d.rows), + tables: d.tables })); // Worst-first: most operations, then most fragmented. - stats.sort((a, b) => b.operations - a.operations || b.fragmentation - a.fragmentation); + const worstFirst = (a: { operations: number; fragmentation: number }, b: typeof a) => + b.operations - a.operations || b.fragmentation - a.fragmentation; + stats.sort(worstFirst); + definitionStats.sort(worstFirst); return { buckets: stats, + definitions: definitionStats, totals, - truncated: totals.bucketCount > stats.length + bucketsTruncated: totals.bucketCount > stats.length, + definitionsTruncated }; } diff --git a/packages/service-core/test/src/bucket-report.test.ts b/packages/service-core/test/src/bucket-report.test.ts index 6b506340a..d31f770be 100644 --- a/packages/service-core/test/src/bucket-report.test.ts +++ b/packages/service-core/test/src/bucket-report.test.ts @@ -4,34 +4,59 @@ import { DEFAULT_BUCKET_REPORT_LIMIT, estimateDistinctRows, RankedBucketInput, - resolveBucketReportLimit + RankedDefinitionInput, + resolveBucketReportLimit, + suggestBucketAction } from '@/storage/bucket-report.js'; import { describe, expect, it } from 'vitest'; +// Row-bearing operations default to all operations (no compaction residue) unless overridden. +const bucket = ( + name: string, + operations: number, + rows: number, + extra?: Partial +): RankedBucketInput => ({ + bucket: name, + operations, + rows, + operationBytes: extra?.operationBytes ?? 0, + rowOperations: extra?.rowOperations ?? operations, + rowsEstimated: extra?.rowsEstimated ?? false, + tables: extra?.tables ?? [] +}); + +const definition = ( + name: string, + operations: number, + rows: number, + extra?: Partial +): RankedDefinitionInput => ({ + definition: name, + bucketCount: extra?.bucketCount ?? 1, + operations, + rows, + operationBytes: extra?.operationBytes ?? 0, + rowOperations: extra?.rowOperations ?? operations, + rowsEstimated: extra?.rowsEstimated ?? false, + tables: extra?.tables ?? [] +}); + +const totals = (bucketCount: number, extra?: Partial): BucketReportTotals => ({ + bucketCount, + operations: extra?.operations ?? 0, + operationBytes: extra?.operationBytes ?? 0, + estimated: extra?.estimated ?? false +}); + describe('assembleBucketReport', () => { - const bucket = ( - name: string, - operations: number, - rows: number, - extra?: Partial - ): RankedBucketInput => ({ - bucket: name, - operations, - rows, - operationBytes: extra?.operationBytes ?? 0, - rowsEstimated: extra?.rowsEstimated ?? false - }); - - const totals = (bucketCount: number, extra?: Partial): BucketReportTotals => ({ - bucketCount, - operations: extra?.operations ?? 0, - operationBytes: extra?.operationBytes ?? 0, - estimated: extra?.estimated ?? false - }); - - it('derives fragmentation and passes through rowsEstimated', () => { + it('derives fragmentation and passes through rowsEstimated and tables', () => { const report = assembleBucketReport( - [bucket('global[]', 100, 10, { operationBytes: 1024 }), bucket('by_user["u1"]', 30, 30, { rowsEstimated: true })], + [ + bucket('global[]', 100, 10, { operationBytes: 1024, tables: ['todos', 'lists'] }), + bucket('by_user["u1"]', 30, 30, { rowsEstimated: true }) + ], + [], totals(2) ); @@ -40,7 +65,8 @@ describe('assembleBucketReport', () => { rows: 10, operationBytes: 1024, fragmentation: 10, - rowsEstimated: false + rowsEstimated: false, + tables: ['todos', 'lists'] }); expect(report.buckets.find((b) => b.bucket === 'by_user["u1"]')).toMatchObject({ fragmentation: 1, @@ -49,32 +75,115 @@ describe('assembleBucketReport', () => { }); it('ranks buckets worst-first by operations then fragmentation', () => { - const report = assembleBucketReport([bucket('a[]', 5, 5), bucket('b[]', 50, 5), bucket('c[]', 50, 50)], totals(3)); + const report = assembleBucketReport( + [bucket('a[]', 5, 5), bucket('b[]', 50, 5), bucket('c[]', 50, 50)], + [], + totals(3) + ); // b and c both have 50 ops; b is more fragmented (10 vs 1) so it ranks first. expect(report.buckets.map((b) => b.bucket)).toEqual(['b[]', 'c[]', 'a[]']); }); it('floors rows at 1 so a bucket with operations but no rows is fully fragmented', () => { - const report = assembleBucketReport([bucket('gone[]', 42, 0)], totals(1)); + const report = assembleBucketReport([bucket('gone[]', 42, 0)], [], totals(1)); expect(report.buckets[0]).toMatchObject({ operations: 42, rows: 0, fragmentation: 42 }); }); - it('marks truncated when there are more buckets than returned', () => { - const truncated = assembleBucketReport([bucket('a[]', 10, 1), bucket('b[]', 5, 1)], totals(5)); - expect(truncated.truncated).toBe(true); + it('marks the bucket list truncated when there are more buckets than returned', () => { + const truncated = assembleBucketReport([bucket('a[]', 10, 1), bucket('b[]', 5, 1)], [], totals(5)); + expect(truncated.bucketsTruncated).toBe(true); - const complete = assembleBucketReport([bucket('a[]', 10, 1), bucket('b[]', 5, 1)], totals(2)); - expect(complete.truncated).toBe(false); + const complete = assembleBucketReport([bucket('a[]', 10, 1), bucket('b[]', 5, 1)], [], totals(2)); + expect(complete.bucketsTruncated).toBe(false); + }); + + it('passes the definition truncation flag through, defaulting to complete', () => { + expect(assembleBucketReport([], [definition('a', 1, 1)], totals(1)).definitionsTruncated).toBe(false); + expect(assembleBucketReport([], [definition('a', 1, 1)], totals(1), true).definitionsTruncated).toBe(true); }); it('carries the totals through unchanged', () => { const t = totals(2, { operations: 120, operationBytes: 15, estimated: true }); - const report = assembleBucketReport([bucket('a[]', 100, 4), bucket('b[]', 20, 2)], t); + const report = assembleBucketReport([bucket('a[]', 100, 4), bucket('b[]', 20, 2)], [], t); expect(report.totals).toEqual({ bucketCount: 2, operations: 120, operationBytes: 15, estimated: true }); }); + + it('assembles and ranks the definition rollup with derived fragmentation and action', () => { + const report = assembleBucketReport( + [], + [ + definition('1#by_user', 100, 100, { bucketCount: 10 }), + definition('1#by_org', 500, 100, { bucketCount: 5, operationBytes: 2048 }) + ], + totals(15) + ); + + // Ranked by operations: by_org (500) before by_user (100). + expect(report.definitions.map((d) => d.definition)).toEqual(['1#by_org', '1#by_user']); + expect(report.definitions[0]).toMatchObject({ + definition: '1#by_org', + bucketCount: 5, + operations: 500, + operationBytes: 2048, + rows: 100, + fragmentation: 5, + suggestedAction: 'compact' + }); + expect(report.definitions[1]).toMatchObject({ fragmentation: 1, suggestedAction: 'none' }); + }); + + it('derives per-bucket suggested actions from the operation mix', () => { + const report = assembleBucketReport( + [ + // Healthy: one op per row. + bucket('healthy[]', 100, 100), + // Un-compacted churn: every op carries a row identity, far more ops than rows. + bucket('churned[]', 1000, 100), + // Compacted residue: mostly MOVE/CLEAR ops left behind by a compact. + bucket('compacted[]', 1000, 100, { rowOperations: 150 }) + ], + [], + totals(3) + ); + + const action = (name: string) => report.buckets.find((b) => b.bucket === name)?.suggestedAction; + expect(action('healthy[]')).toEqual('none'); + expect(action('churned[]')).toEqual('compact'); + expect(action('compacted[]')).toEqual('defragment'); + }); +}); + +describe('suggestBucketAction', () => { + it('suggests nothing for healthy buckets', () => { + expect(suggestBucketAction(100, 100, 100)).toEqual('none'); + expect(suggestBucketAction(150, 150, 100)).toEqual('none'); + expect(suggestBucketAction(0, 0, 0)).toEqual('none'); + }); + + it('suggests compact for un-compacted superseded history', () => { + // All operations carry row identity, but there are 10x more of them than rows. + expect(suggestBucketAction(1000, 1000, 100)).toEqual('compact'); + }); + + it('suggests defragment when compaction residue dominates', () => { + // 850 of 1000 ops are MOVE/CLEAR: a compact already ran and cannot reclaim more. + expect(suggestBucketAction(1000, 150, 100)).toEqual('defragment'); + // A bucket of only MOVE/CLEAR ops (rows 0) is pure residue. + expect(suggestBucketAction(500, 0, 0)).toEqual('defragment'); + }); + + it('suggests both when residue and fresh superseded history are both present', () => { + // 600 residue ops plus 400 row-bearing ops over 100 rows: defragment for the residue, compact for the churn. + expect(suggestBucketAction(1000, 400, 100)).toEqual('both'); + }); + + it('suggests both for a fragmented but inconclusive mix', () => { + // Fragmented (frag 2.5), yet neither residue (40%) nor superseded share (33%) dominates. + expect(suggestBucketAction(1000, 600, 400)).toEqual('both'); + }); }); describe('estimateDistinctRows', () => { diff --git a/packages/service-core/test/src/routes/admin.test.ts b/packages/service-core/test/src/routes/admin.test.ts index be49e840b..f7eea90c9 100644 --- a/packages/service-core/test/src/routes/admin.test.ts +++ b/packages/service-core/test/src/routes/admin.test.ts @@ -219,7 +219,9 @@ bucket_definitions: rows: 95, operationBytes: 1216000, fragmentation: 50, - rowsEstimated: true + rowsEstimated: true, + suggestedAction: 'compact', + tables: ['todos'] }, { bucket: '1#global[]', @@ -227,11 +229,27 @@ bucket_definitions: rows: 1000, operationBytes: 3145728, fragmentation: 1, - rowsEstimated: false + rowsEstimated: false, + suggestedAction: 'none', + tables: ['lists'] + } + ], + definitions: [ + { + definition: '1#by_user', + bucketCount: 1, + operations: 4750, + operationBytes: 1216000, + rows: 95, + fragmentation: 50, + rowsEstimated: true, + suggestedAction: 'compact', + tables: ['todos'] } ], totals: { bucketCount: 2, operations: 5750, operationBytes: 4361728, estimated: false }, - truncated: false + bucketsTruncated: false, + definitionsTruncated: true }; it('returns the report, forwards the limit, and maps fields to snake_case', async () => { @@ -257,15 +275,31 @@ bucket_definitions: rows: 95, operation_bytes: 1216000, fragmentation: 50, - rows_estimated: true + rows_estimated: true, + suggested_action: 'compact', + tables: ['todos'] }); + expect(response.definitions).toEqual([ + { + definition: '1#by_user', + bucket_count: 1, + operations: 4750, + operation_bytes: 1216000, + rows: 95, + fragmentation: 50, + rows_estimated: true, + suggested_action: 'compact', + tables: ['todos'] + } + ]); expect(response.totals).toEqual({ bucket_count: 2, operations: 5750, operation_bytes: 4361728, estimated: false }); - expect(response.truncated).toBe(false); + expect(response.buckets_truncated).toBe(false); + expect(response.definitions_truncated).toBe(true); }); it('rejects when there is no active sync config', async () => { diff --git a/packages/types/src/routes.ts b/packages/types/src/routes.ts index ed742541c..6036d536b 100644 --- a/packages/types/src/routes.ts +++ b/packages/types/src/routes.ts @@ -88,6 +88,13 @@ export const BucketReportRequest = t.object({ }); export type BucketReportRequest = t.Encoded; +export const SuggestedBucketAction = t + .literal('none') + .or(t.literal('compact')) + .or(t.literal('defragment')) + .or(t.literal('both')); +export type SuggestedBucketAction = t.Encoded; + export const BucketStorageStats = t.object({ /** Full bucket name, e.g. `by_user["u1"]`. */ bucket: t.string, @@ -103,13 +110,51 @@ export const BucketStorageStats = t.object({ */ fragmentation: t.number, /** True if `rows` (and therefore `fragmentation`) is a sampled estimate rather than an exact count. */ - rows_estimated: t.boolean + rows_estimated: t.boolean, + /** + * Suggested maintenance action, derived from the bucket's operation mix: `none` (healthy), `compact` + * (un-compacted superseded history to reclaim), `defragment` (mostly compaction residue that only a + * defragment collapses), or `both`. + */ + suggested_action: SuggestedBucketAction, + /** + * Tables making up the (sampled) operation history, ordered by their share of it, largest first. These + * are the tables whose rows a defragment should touch. + */ + tables: t.array(t.string) }); export type BucketStorageStats = t.Encoded; +export const BucketDefinitionStats = t.object({ + /** Definition name as it prefixes bucket names, e.g. `1#by_user` (versioned in storage v2 and later). */ + definition: t.string, + /** Number of buckets in this definition with stored operations. */ + bucket_count: t.number, + /** Total operations across the definition's buckets. */ + operations: t.number, + /** Approximate size of the definition's operation history in bytes. */ + operation_bytes: t.number, + /** + * Live rows across the definition's buckets, counting a row once per bucket that contains it. A sampled + * estimate for all but tiny definitions (see `rows_estimated`). + */ + rows: t.number, + /** `operations / max(rows, 1)` across the whole definition. */ + fragmentation: t.number, + /** True if `rows` (and therefore `fragmentation`) is a sampled estimate rather than an exact count. */ + rows_estimated: t.boolean, + /** Suggested maintenance action for the definition; same values as `buckets[].suggested_action`. */ + suggested_action: SuggestedBucketAction, + /** Tables in the definition's (sampled) operation history, ordered by their share of it, largest first. */ + tables: t.array(t.string) +}); +export type BucketDefinitionStats = t.Encoded; + export const BucketReportResponse = t.object({ /** Worst-offender buckets, ranked by operation count then fragmentation. */ buckets: t.array(BucketStorageStats), + /** Per-definition rollup, ranked by operation count then fragmentation. */ + definitions: t.array(BucketDefinitionStats), totals: t.object({ /** Number of buckets with stored operations. Estimated when the bucket set was sampled. */ bucket_count: t.number, @@ -121,6 +166,8 @@ export const BucketReportResponse = t.object({ estimated: t.boolean }), /** True if there are more buckets than returned (more than `limit`). */ - truncated: t.boolean + buckets_truncated: t.boolean, + /** True if the definition rollup is incomplete: more definitions exist than the report caps at. */ + definitions_truncated: t.boolean }); export type BucketReportResponse = t.Encoded; From c1c39995d1c497efebcf0f7ffd3798617847533e Mon Sep 17 00:00:00 2001 From: bean1352 Date: Thu, 2 Jul 2026 14:27:38 +0200 Subject: [PATCH 27/40] Raise the bucket report action threshold to 3x fragmentation --- packages/service-core/src/storage/bucket-report.ts | 7 +++++-- packages/service-core/test/src/bucket-report.test.ts | 10 ++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/service-core/src/storage/bucket-report.ts b/packages/service-core/src/storage/bucket-report.ts index d5920c20a..59f901ff1 100644 --- a/packages/service-core/src/storage/bucket-report.ts +++ b/packages/service-core/src/storage/bucket-report.ts @@ -32,8 +32,11 @@ export const DEFAULT_BUCKET_REPORT_LIMIT: number = 50; */ export const BUCKET_REPORT_DEFINITION_LIMIT: number = 20; -/** Fragmentation below this is considered healthy: no maintenance action is suggested. */ -export const BUCKET_ACTION_MIN_FRAGMENTATION: number = 2; +/** + * Fragmentation below this is considered healthy: no maintenance action is suggested. Matches the 3x rule + * of thumb used by the PowerSync diagnostics app. + */ +export const BUCKET_ACTION_MIN_FRAGMENTATION: number = 3; /** * When at least this share of a bucket's operations is compaction residue (MOVE/CLEAR, no row identity), diff --git a/packages/service-core/test/src/bucket-report.test.ts b/packages/service-core/test/src/bucket-report.test.ts index d31f770be..1878c7c5b 100644 --- a/packages/service-core/test/src/bucket-report.test.ts +++ b/packages/service-core/test/src/bucket-report.test.ts @@ -157,15 +157,17 @@ describe('assembleBucketReport', () => { }); describe('suggestBucketAction', () => { - it('suggests nothing for healthy buckets', () => { + it('suggests nothing for buckets under 3x fragmentation', () => { expect(suggestBucketAction(100, 100, 100)).toEqual('none'); - expect(suggestBucketAction(150, 150, 100)).toEqual('none'); + expect(suggestBucketAction(250, 250, 100)).toEqual('none'); expect(suggestBucketAction(0, 0, 0)).toEqual('none'); }); it('suggests compact for un-compacted superseded history', () => { // All operations carry row identity, but there are 10x more of them than rows. expect(suggestBucketAction(1000, 1000, 100)).toEqual('compact'); + // 3x fragmentation is the threshold where actions start. + expect(suggestBucketAction(300, 300, 100)).toEqual('compact'); }); it('suggests defragment when compaction residue dominates', () => { @@ -181,8 +183,8 @@ describe('suggestBucketAction', () => { }); it('suggests both for a fragmented but inconclusive mix', () => { - // Fragmented (frag 2.5), yet neither residue (40%) nor superseded share (33%) dominates. - expect(suggestBucketAction(1000, 600, 400)).toEqual('both'); + // Fragmented (frag 3), yet neither residue (40%) nor superseded share (44%) dominates. + expect(suggestBucketAction(1200, 720, 400)).toEqual('both'); }); }); From 784ab818bb08d62b7e8cbda45ffa4e5be1c233de Mon Sep 17 00:00:00 2001 From: bean1352 Date: Thu, 2 Jul 2026 14:29:27 +0200 Subject: [PATCH 28/40] Trim the fragmentation threshold comment --- packages/service-core/src/storage/bucket-report.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/service-core/src/storage/bucket-report.ts b/packages/service-core/src/storage/bucket-report.ts index 59f901ff1..00455baf2 100644 --- a/packages/service-core/src/storage/bucket-report.ts +++ b/packages/service-core/src/storage/bucket-report.ts @@ -32,10 +32,7 @@ export const DEFAULT_BUCKET_REPORT_LIMIT: number = 50; */ export const BUCKET_REPORT_DEFINITION_LIMIT: number = 20; -/** - * Fragmentation below this is considered healthy: no maintenance action is suggested. Matches the 3x rule - * of thumb used by the PowerSync diagnostics app. - */ +/** Fragmentation below this is considered healthy: no maintenance action is suggested. */ export const BUCKET_ACTION_MIN_FRAGMENTATION: number = 3; /** From f4ea6e5cd38540750c7dd1a7331d1d613c2af3f2 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Mon, 6 Jul 2026 09:37:20 +0200 Subject: [PATCH 29/40] Reject invalid bucket report limits instead of clamping --- .../service-core/src/storage/bucket-report.ts | 25 ++++++++++++++++--- .../test/src/bucket-report.test.ts | 20 ++++++++++++--- packages/types/src/routes.ts | 2 +- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/packages/service-core/src/storage/bucket-report.ts b/packages/service-core/src/storage/bucket-report.ts index 00455baf2..2fb4e49a5 100644 --- a/packages/service-core/src/storage/bucket-report.ts +++ b/packages/service-core/src/storage/bucket-report.ts @@ -12,6 +12,7 @@ * count and returns the worst offenders (top-N). Row counts (and therefore fragmentation) for those buckets * are derived by sampling the operation history, so on large buckets they are estimates, flagged per bucket. */ +import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; /** * Time budget for the per-bucket report's bucket-selection aggregation (`maxTimeMS`). Bounded so an admin @@ -25,6 +26,12 @@ export const BUCKET_REPORT_TIMEOUT_MS: number = 60_000; */ export const DEFAULT_BUCKET_REPORT_LIMIT: number = 50; +/** + * Highest `limit` a request may ask for. Every returned bucket costs a row-sampling query, so an unbounded + * limit would let a single request overload the storage database. + */ +export const MAX_BUCKET_REPORT_LIMIT: number = 1_000; + /** * Maximum number of bucket definitions in the report's definition rollup. Rows are sampled per returned * definition (like per-bucket rows), so this bounds that sampling work. Configs rarely approach this many @@ -139,8 +146,9 @@ export interface BucketReport { export interface GetBucketReportOptions { /** * Maximum number of buckets to return, ranked by operation count descending (worst offenders first). - * Row counts are sampled per returned bucket, so this also bounds the report's cost. Non-integer or - * negative values are floored and clamped to 1. Defaults to {@link DEFAULT_BUCKET_REPORT_LIMIT}. + * Row counts are sampled per returned bucket, so this also bounds the report's cost. Must be an integer + * between 1 and {@link MAX_BUCKET_REPORT_LIMIT}; anything else is rejected with a validation error. + * Defaults to {@link DEFAULT_BUCKET_REPORT_LIMIT}. */ limit?: number; } @@ -176,13 +184,22 @@ export interface RankedDefinitionInput { } /** - * Normalize a requested limit to a positive integer, falling back to {@link DEFAULT_BUCKET_REPORT_LIMIT}. + * Resolve the requested limit, falling back to {@link DEFAULT_BUCKET_REPORT_LIMIT}. Invalid values are + * rejected rather than clamped, so a caller asking for more than the maximum fails loudly instead of + * silently getting fewer buckets than requested. */ export function resolveBucketReportLimit(limit?: number): number { if (limit == null) { return DEFAULT_BUCKET_REPORT_LIMIT; } - return Math.max(1, Math.floor(limit)); + if (!Number.isInteger(limit) || limit < 1 || limit > MAX_BUCKET_REPORT_LIMIT) { + throw new ServiceError({ + status: 400, + code: ErrorCode.PSYNC_S2001, + description: `limit must be an integer between 1 and ${MAX_BUCKET_REPORT_LIMIT}` + }); + } + return limit; } /** diff --git a/packages/service-core/test/src/bucket-report.test.ts b/packages/service-core/test/src/bucket-report.test.ts index 1878c7c5b..61679e91f 100644 --- a/packages/service-core/test/src/bucket-report.test.ts +++ b/packages/service-core/test/src/bucket-report.test.ts @@ -3,6 +3,7 @@ import { BucketReportTotals, DEFAULT_BUCKET_REPORT_LIMIT, estimateDistinctRows, + MAX_BUCKET_REPORT_LIMIT, RankedBucketInput, RankedDefinitionInput, resolveBucketReportLimit, @@ -224,10 +225,21 @@ describe('resolveBucketReportLimit', () => { expect(resolveBucketReportLimit(undefined)).toBe(DEFAULT_BUCKET_REPORT_LIMIT); }); - it('floors and clamps to a positive integer', () => { - expect(resolveBucketReportLimit(2.7)).toBe(2); - expect(resolveBucketReportLimit(-5)).toBe(1); - expect(resolveBucketReportLimit(0)).toBe(1); + it('accepts integers up to the maximum', () => { + expect(resolveBucketReportLimit(1)).toBe(1); expect(resolveBucketReportLimit(20)).toBe(20); + expect(resolveBucketReportLimit(MAX_BUCKET_REPORT_LIMIT)).toBe(MAX_BUCKET_REPORT_LIMIT); + }); + + it('rejects invalid limits instead of clamping them', () => { + for (const invalid of [0, -5, 2.7, MAX_BUCKET_REPORT_LIMIT + 1]) { + let error: any; + try { + resolveBucketReportLimit(invalid); + } catch (e) { + error = e; + } + expect(error?.errorData, `limit ${invalid}`).toMatchObject({ status: 400 }); + } }); }); diff --git a/packages/types/src/routes.ts b/packages/types/src/routes.ts index 6036d536b..ef4f5e710 100644 --- a/packages/types/src/routes.ts +++ b/packages/types/src/routes.ts @@ -82,7 +82,7 @@ export const BucketReportRequest = t.object({ /** * Maximum number of buckets to return, ranked by operation count descending (worst offenders first). * Row counts are sampled per returned bucket, so this also bounds the report's cost. Defaults to 50 when - * omitted; non-integer or negative values are floored and clamped to 1. + * omitted. Must be an integer between 1 and 1000; anything else is rejected with a validation error. */ limit: t.number.optional() }); From e0ad5384c7fb7781d35a0a89efb6dddbead23838 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Mon, 6 Jul 2026 09:38:23 +0200 Subject: [PATCH 30/40] Sample the bucket ranking on the index without fetching unsampled buckets --- .../implementation/MongoSyncBucketStorage.ts | 98 ++++++++++++++----- .../v1/MongoSyncBucketStorageV1.ts | 4 +- .../v3/MongoSyncBucketStorageV3.ts | 20 +++- .../service-core/src/storage/bucket-report.ts | 2 +- packages/types/src/routes.ts | 2 +- 5 files changed, 97 insertions(+), 29 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 9a8183d4c..a72e08bd2 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -3,9 +3,11 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { BaseObserver, DO_NOT_LOG, + ErrorCode, Logger, ReplicationAbortedError, - ServiceAssertionError + ServiceAssertionError, + ServiceError } from '@powersync/lib-services-framework'; import { BroadcastIterable, @@ -67,14 +69,25 @@ interface InternalCheckpointChanges extends CheckpointChanges { const CHECKPOINT_TIMEOUT_MS = 60_000; /** - * Above this many buckets, the report ranks a bounded `$sample` of bucket_state rather than every bucket, so - * the request cannot exhaust memory or run unbounded. Below it, the ranking is exact. + * Above this many buckets (a collection-wide estimate), the report ranks a bounded sample of bucket_state + * rather than every bucket, so the request cannot exhaust memory or run unbounded. Below it, the ranking is + * exact. */ const BUCKET_SELECTION_SAMPLE_THRESHOLD = 50_000; -/** Number of buckets to sample when over {@link BUCKET_SELECTION_SAMPLE_THRESHOLD}. */ +/** + * Approximate number of buckets sampled when over {@link BUCKET_SELECTION_SAMPLE_THRESHOLD}. The sample is + * drawn with `$sampleRate`, so the achieved count varies slightly around this. + */ const BUCKET_SELECTION_SAMPLE_SIZE = 10_000; +/** + * Most bucket_state index entries one report query may scan. Even when sampling fetches few documents, the + * covered index scan and the matched-bucket count still touch every matched index entry once, so past this + * the report fails fast instead of scaling without bound. + */ +const BUCKET_SELECTION_SCAN_MAX = 1_000_000; + /** * Fewest operations sampled per bucket when estimating its row count. Buckets with fewer operations than * this are read in full (exact). @@ -541,10 +554,10 @@ export abstract class MongoSyncBucketStorage * pre-aggregated bucket state (compacted_state + estimate_since_compact). One document per bucket, no scan * of bucket data. * - * For very large bucket sets the candidates are drawn from a bounded `$sample` rather than the whole - * collection (so the request cannot run unbounded or exhaust memory), and the totals are scaled from the - * sample and flagged estimated. `allowDiskUse: false` makes an over-threshold exact attempt fail fast - * rather than spill to disk and degrade the live instance. + * For very large bucket sets the candidates are drawn from a bounded sample of the matched `_id` index + * range rather than the whole collection (so the request cannot run unbounded or exhaust memory), and the + * totals are scaled from the sample and flagged estimated. `allowDiskUse: false` makes an over-threshold + * exact attempt fail fast rather than spill to disk and degrade the live instance. * * Note: for v1/v2 storage, bucket_state is not backfilled (see models.ts: "only populated by new updates"), * so buckets that predate bucket_state tracking and have not been updated or compacted since are missing @@ -574,16 +587,49 @@ export abstract class MongoSyncBucketStorage // because all buckets sharing a name prefix share the definition (undefined for v1/v2). const definitionKey = { $arrayElemAt: [{ $split: ['$_id.b', '['] }, 0] }; + // Reports are bulk reads: run them with the configured bulk read preference (secondaries where + // configured) so they do not load the primary. + const readPreference = this.readPreference; + // estimatedDocumentCount is O(1) but ignores the match filter, so this is an upper bound on the active // bucket count. That is fine for the sampling decision: over-estimating only switches to sampling sooner. - // It must NOT be used to scale the sampled totals though - the collection can hold buckets outside the - // match (other replication groups for v1/v2, inactive definitions for v3), which would over-scale. - const estimatedTotalBuckets = await collection.estimatedDocumentCount(); - const sampled = estimatedTotalBuckets > BUCKET_SELECTION_SAMPLE_THRESHOLD; + const estimatedTotalBuckets = await collection.estimatedDocumentCount({ readPreference }); + + let matchedBuckets: number | null = null; + if (estimatedTotalBuckets > BUCKET_SELECTION_SAMPLE_THRESHOLD) { + // The exact matched-bucket count. `match` is an `_id` range, so this is an index-only scan; it sets + // the sample rate, scales the sampled sums back up, and doubles as the exact totals.bucketCount. + // `limit` caps how many index entries the count may touch: hitting the cap means the instance is past + // what this report is designed to scan, so fail fast rather than read the index without bound. + matchedBuckets = await collection.countDocuments(match, { + maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS, + readPreference, + limit: BUCKET_SELECTION_SCAN_MAX + 1 + }); + if (matchedBuckets > BUCKET_SELECTION_SCAN_MAX) { + throw new ServiceError({ + status: 422, + code: ErrorCode.PSYNC_S2001, + description: `Bucket report is not supported on this instance: more than ${BUCKET_SELECTION_SCAN_MAX} buckets match the active sync configuration` + }); + } + } + const sampleRate = matchedBuckets == null ? 1 : BUCKET_SELECTION_SAMPLE_SIZE / Math.max(matchedBuckets, 1); + const sampled = sampleRate < 1; const pipeline: mongo.Document[] = [{ $match: match }]; if (sampled) { - pipeline.push({ $sample: { size: BUCKET_SELECTION_SAMPLE_SIZE } }); + // Sample on the index alone, then fetch only the sampled documents: the range $match plus the _id + // projection is a covered index scan (explain shows docsExamined: 0), $sampleRate keeps roughly + // SAMPLE_SIZE ids, and the self-$lookup fetches just those. Sampling after a plain $match would fetch + // every matched document only to discard most of them. + pipeline.push( + { $project: { _id: 1 } }, + { $match: { $sampleRate: sampleRate } }, + { $lookup: { from: collection.collectionName, localField: '_id', foreignField: '_id', as: 'doc' } }, + { $unwind: '$doc' }, + { $replaceRoot: { newRoot: '$doc' } } + ); } pipeline.push({ $facet: { @@ -627,7 +673,11 @@ export abstract class MongoSyncBucketStorage }[]; }; const [result] = await collection - .aggregate(pipeline, { allowDiskUse: false, maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS }) + .aggregate(pipeline, { + allowDiskUse: false, + maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS, + readPreference + }) .toArray(); const rawTotals = result?.totals[0] ?? { operations: 0, operationBytes: 0, bucketCount: 0 }; @@ -661,20 +711,16 @@ export abstract class MongoSyncBucketStorage }; } - // Scale the sampled totals up to the full *matched* set. countDocuments respects the match filter (so it - // excludes other groups / inactive definitions) and uses the _id index; it only runs on the already-large - // sampled path, and is bounded by maxTimeMS like the rest of the report. When the matched set fits within - // the sample, rawTotals is already exact and the scale collapses to 1. The sample is uniform across - // buckets, so the per-definition sums scale by the same factor; a definition small enough to be missed - // by the sample entirely is absent. - const matchedBuckets = await collection.countDocuments(match, { maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS }); - const scale = matchedBuckets / Math.max(rawTotals.bucketCount, 1); + // Scale the sampled sums up to the full matched set, using the exact matched count from above. The + // sample is uniform across buckets, so the per-definition sums scale by the same factor; a definition + // small enough to be missed by the sample entirely is absent. bucketCount itself is exact. + const scale = matchedBuckets! / Math.max(rawTotals.bucketCount, 1); return { buckets, definitions: mapDefinitions(scale), definitionsTruncated, totals: { - bucketCount: matchedBuckets, + bucketCount: matchedBuckets!, operations: Math.round(rawTotals.operations * scale), operationBytes: Math.round(rawTotals.operationBytes * scale), estimated: true @@ -729,7 +775,11 @@ export abstract class MongoSyncBucketStorage tables: { _id: string }[]; }; const [result] = await collection - .aggregate(pipeline, { allowDiskUse: false, maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS }) + .aggregate(pipeline, { + allowDiskUse: false, + maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS, + readPreference: this.readPreference + }) .toArray(); return { sampledOps: result?.sampledOps[0]?.count ?? 0, diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index 89605aacd..d77863556 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -191,9 +191,11 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage { // For storage v1/v2, bucket state and bucket data are shared collections scoped by group (replication stream). protected async collectTopBuckets(limit: number): Promise { + // Range-match on the whole `_id` (g, b) so the {_id} index bounds the scan; a dotted `{'_id.g': ...}` + // match cannot use the compound-object index and would scan the whole collection. const { buckets, definitions, definitionsTruncated, totals } = await this.aggregateTopBuckets( this.db.bucketStateV1, - { '_id.g': this.replicationStreamId }, + { _id: idPrefixFilter<{ g: number; b: string }>({ g: this.replicationStreamId }, ['b']) }, limit ); return { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index 64bd78d91..b6563efdd 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -13,7 +13,12 @@ import { utils } from '@powersync/service-core'; import { JSONBig } from '@powersync/service-jsonbig'; -import { ParameterLookupRows, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules'; +import { + BucketDefinitionId, + ParameterLookupRows, + ScopedParameterLookup, + SqliteJsonRow +} from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js'; import { MongoBucketStorage } from '../../MongoBucketStorage.js'; @@ -194,9 +199,20 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { // sharing these collections. Scope to the active config's definition ids so the report excludes stale buckets // from old/stopped definitions. `this.storageIds` is derived from the active config only (see getActiveSyncConfig). protected async collectTopBuckets(limit: number): Promise { + const definitionIds = this.storageIds.bucketDefinitionIds; + if (definitionIds.length == 0) { + return { + buckets: [], + definitions: [], + definitionsTruncated: false, + totals: { bucketCount: 0, operations: 0, operationBytes: 0, estimated: false } + }; + } + // One `_id` range per active definition, so the {_id} index bounds the scan per definition; a dotted + // `{'_id.d': ...}` match cannot use the compound-object index and would scan the whole collection. const { buckets, definitions, definitionsTruncated, totals } = await this.aggregateTopBuckets( this.db.bucketState(this.replicationStreamId), - { '_id.d': { $in: this.storageIds.bucketDefinitionIds } }, + { $or: definitionIds.map((d) => ({ _id: idPrefixFilter<{ d: BucketDefinitionId; b: string }>({ d }, ['b']) })) }, limit ); return { diff --git a/packages/service-core/src/storage/bucket-report.ts b/packages/service-core/src/storage/bucket-report.ts index 2fb4e49a5..6943ea53f 100644 --- a/packages/service-core/src/storage/bucket-report.ts +++ b/packages/service-core/src/storage/bucket-report.ts @@ -111,7 +111,7 @@ export interface BucketDefinitionStats { } export interface BucketReportTotals { - /** Number of buckets with stored operations. Estimated when the bucket set was sampled (see `estimated`). */ + /** Number of buckets with stored operations. Exact, even when the rest of the totals are estimates. */ bucketCount: number; /** Sum of operations across all buckets. Estimated when the bucket set was sampled. */ operations: number; diff --git a/packages/types/src/routes.ts b/packages/types/src/routes.ts index ef4f5e710..737f84130 100644 --- a/packages/types/src/routes.ts +++ b/packages/types/src/routes.ts @@ -156,7 +156,7 @@ export const BucketReportResponse = t.object({ /** Per-definition rollup, ranked by operation count then fragmentation. */ definitions: t.array(BucketDefinitionStats), totals: t.object({ - /** Number of buckets with stored operations. Estimated when the bucket set was sampled. */ + /** Number of buckets with stored operations. Exact, even when the other totals are estimates. */ bucket_count: t.number, /** Sum of operations across all buckets. Estimated when the bucket set was sampled. */ operations: t.number, From e706504202a8842a19ee5a78b7c9a8abe769bf40 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Wed, 26 Aug 2026 09:27:23 +0200 Subject: [PATCH 31/40] Derive bucket report rows from full compact statistics --- .../service-core/src/storage/bucket-report.ts | 249 +++++++++--------- 1 file changed, 128 insertions(+), 121 deletions(-) diff --git a/packages/service-core/src/storage/bucket-report.ts b/packages/service-core/src/storage/bucket-report.ts index 6943ea53f..a5d79dbcb 100644 --- a/packages/service-core/src/storage/bucket-report.ts +++ b/packages/service-core/src/storage/bucket-report.ts @@ -8,9 +8,11 @@ * fragmentation / compaction-efficiency score: a fully compacted bucket trends towards ~1, while a high * ratio is the usual cause of an unexpectedly high "Data Synced" metric and is reclaimable via compact/defragment. * - * Scaling note: the report does NOT scan all storage. It ranks buckets by their pre-aggregated operation - * count and returns the worst offenders (top-N). Row counts (and therefore fragmentation) for those buckets - * are derived by sampling the operation history, so on large buckets they are estimates, flagged per bucket. + * Scaling note: the report reads only the pre-aggregated per-bucket state (`bucket_state`), never the + * operation history itself. Operation counts are exact; row counts come from the statistics captured by the + * last full compact of each bucket, so they are a snapshot as of that compact rather than a live counter. + * Storage versions that do not capture compact statistics (v1/v2) report operations only, with row-derived + * fields null and the suggested action `unknown`. */ import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; @@ -20,21 +22,14 @@ import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; */ export const BUCKET_REPORT_TIMEOUT_MS: number = 60_000; -/** - * Number of worst-offender buckets returned when the request omits a `limit`. Row counts are sampled per - * returned bucket, so this also bounds how much sampling work the report does. - */ +/** Number of worst-offender buckets returned when the request omits a `limit`. */ export const DEFAULT_BUCKET_REPORT_LIMIT: number = 50; -/** - * Highest `limit` a request may ask for. Every returned bucket costs a row-sampling query, so an unbounded - * limit would let a single request overload the storage database. - */ +/** Highest `limit` a request may ask for, bounding the response size. */ export const MAX_BUCKET_REPORT_LIMIT: number = 1_000; /** - * Maximum number of bucket definitions in the report's definition rollup. Rows are sampled per returned - * definition (like per-bucket rows), so this bounds that sampling work. Configs rarely approach this many + * Maximum number of bucket definitions in the report's definition rollup. Configs rarely approach this many * definitions. */ export const BUCKET_REPORT_DEFINITION_LIMIT: number = 20; @@ -54,32 +49,45 @@ export const BUCKET_ACTION_RESIDUE_SHARE: number = 0.5; */ export const BUCKET_ACTION_SUPERSEDED_SHARE: number = 0.5; -/** Suggested maintenance action for a bucket or definition. See {@link suggestBucketAction}. */ -export type BucketAction = 'none' | 'compact' | 'defragment' | 'both'; +/** + * Suggested maintenance action for a bucket or definition. `unknown` means the storage has no compact + * statistics to derive one from (v1/v2 storage, or a bucket that has never been fully compacted). + * See {@link suggestBucketAction}. + */ +export type BucketAction = 'none' | 'compact' | 'defragment' | 'both' | 'unknown'; export interface BucketStorageStats { /** Full bucket name, e.g. `by_user["u1"]`. */ bucket: string; - /** Total operations in the bucket's history. */ + /** Total operations in the bucket's history. Exact and current. */ operations: number; - /** Live rows in the bucket. Exact for small buckets, otherwise a sampled estimate (see `rowsEstimated`). */ - rows: number; /** Approximate size of the operation history in bytes. */ operationBytes: number; + /** + * Operations written after the last full compact — the staleness indicator for `rows` and + * `fragmentation`, which are snapshots from that compact. Equals `operations` when the bucket has + * never been fully compacted (the whole history is uncompacted). + */ + uncompactedOperations: number; + /** + * Live rows in the bucket as of the last full compact, or null if the bucket has never been fully + * compacted (or the storage version does not capture compact statistics). + */ + rows: number | null; /** * `operations / max(rows, 1)`. ~1 is healthy (fully compacted); higher means more operation-history - * overhead that a compact/defragment can reclaim. + * overhead that a compact/defragment can reclaim. Null whenever `rows` is null. */ - fragmentation: number; - /** True if `rows` (and therefore `fragmentation`) is a sampled estimate rather than an exact count. */ - rowsEstimated: boolean; - /** Suggested maintenance action derived from the operation mix. See {@link suggestBucketAction}. */ - suggestedAction: BucketAction; + fragmentation: number | null; + /** When the bucket was last fully compacted, which is when `rows` was captured. */ + lastFullCompactAt: Date | null; /** - * Tables making up the (sampled) operation history, ordered by their share of it, largest first. These - * are the tables whose rows a defragment should touch. + * When the scheduled compactor will next consider this bucket, if scheduling data exists. A suggested + * compact with a future `nextCompactAt` means the compact is already planned but throttled until then. */ - tables: string[]; + nextCompactAt: Date | null; + /** Suggested maintenance action derived from the compact statistics. See {@link suggestBucketAction}. */ + suggestedAction: BucketAction; } /** Aggregated stats for one bucket definition (one `bucket_definitions` entry in the sync config). */ @@ -88,26 +96,25 @@ export interface BucketDefinitionStats { definition: string; /** Number of buckets in this definition with stored operations. */ bucketCount: number; - /** Total operations across the definition's buckets. */ + /** Total operations across the definition's buckets. Exact and current. */ operations: number; /** Approximate size of the definition's operation history in bytes. */ operationBytes: number; /** - * Live rows across the definition's buckets, counting a row once per bucket that contains it (the - * download-relevant meaning). Sampled estimate for all but tiny definitions (see `rowsEstimated`). + * Operations not covered by any bucket's last full compact, i.e. written since (or in buckets never + * fully compacted). The staleness indicator for `rows` and `fragmentation`. */ - rows: number; - /** `operations / max(rows, 1)` across the whole definition. */ - fragmentation: number; - /** True if `rows` (and therefore `fragmentation`) is a sampled estimate rather than an exact count. */ - rowsEstimated: boolean; - /** Suggested maintenance action derived from the operation mix. See {@link suggestBucketAction}. */ - suggestedAction: BucketAction; + uncompactedOperations: number; /** - * Tables making up the (sampled) operation history, ordered by their share of it, largest first. These - * are the tables whose rows a defragment should touch. + * Live rows across the definition's buckets, counting a row once per bucket that contains it (the + * download-relevant meaning). Derived from each bucket's last full compact; when only some buckets have + * been fully compacted the count is extrapolated from those, and it is null when none have. */ - tables: string[]; + rows: number | null; + /** `operations / max(rows, 1)` across the whole definition. Null whenever `rows` is null. */ + fragmentation: number | null; + /** Suggested maintenance action derived from the compact statistics. See {@link suggestBucketAction}. */ + suggestedAction: BucketAction; } export interface BucketReportTotals { @@ -119,7 +126,7 @@ export interface BucketReportTotals { operationBytes: number; /** * True if the totals are estimated because the bucket set was too large to scan in full and was sampled. - * Row counts are never totalled here (they are sampled per returned bucket, not across the whole instance). + * Row counts are never totalled here. */ estimated: boolean; } @@ -132,55 +139,57 @@ export interface BucketReport { * at" where `buckets` answers "which exact buckets". Capped at {@link BUCKET_REPORT_DEFINITION_LIMIT}. */ definitions: BucketDefinitionStats[]; - /** Instance-wide operation totals. Does not include row counts (those are per-bucket estimates only). */ + /** Instance-wide operation totals. Does not include row counts. */ totals: BucketReportTotals; /** True if there are more buckets than returned (more than `limit`). */ bucketsTruncated: boolean; - /** - * True if the definition rollup is incomplete: more definitions exist than - * {@link BUCKET_REPORT_DEFINITION_LIMIT}, or a definition was dropped because sampling it failed. - */ + /** True if the definition rollup is incomplete (more definitions exist than the rollup cap). */ definitionsTruncated: boolean; } export interface GetBucketReportOptions { /** * Maximum number of buckets to return, ranked by operation count descending (worst offenders first). - * Row counts are sampled per returned bucket, so this also bounds the report's cost. Must be an integer - * between 1 and {@link MAX_BUCKET_REPORT_LIMIT}; anything else is rejected with a validation error. - * Defaults to {@link DEFAULT_BUCKET_REPORT_LIMIT}. + * Must be an integer between 1 and {@link MAX_BUCKET_REPORT_LIMIT}; anything else is rejected with a + * validation error. Defaults to {@link DEFAULT_BUCKET_REPORT_LIMIT}. */ limit?: number; } -/** A bucket's exact operation stats plus its (possibly sampled) row count, before ranking. */ +/** + * A bucket's exact operation stats plus the statistics captured by its last full compact, before ranking. + * The compact fields are null/absent when the bucket has never been fully compacted or the storage version + * does not capture them (v1/v2). + */ export interface RankedBucketInput { bucket: string; operations: number; operationBytes: number; - rows: number; + /** Operations in the prefix covered by the last full compact. */ + compactedOperations?: number | null; /** - * Operations that carry a row identity (PUT/REMOVE), i.e. everything except compaction residue - * (MOVE/CLEAR). Estimated alongside `rows` for sampled buckets. + * PUT operations in that compacted prefix. After a full compact each PUT is generally a unique row, so + * this doubles as the bucket's live row count as of the compact. */ - rowOperations: number; - rowsEstimated: boolean; - /** Tables in the (sampled) operation history, ordered by their share of it, largest first. */ - tables: string[]; + compactedPuts?: number | null; + /** When the last full compact ran. */ + lastFullCompactAt?: Date | null; + /** When the scheduled compactor will next consider this bucket. */ + nextCompactAt?: Date | null; } -/** A definition's aggregated operation stats plus its (possibly sampled) row count, before ranking. */ +/** A definition's aggregated operation stats plus its buckets' summed compact statistics, before ranking. */ export interface RankedDefinitionInput { definition: string; bucketCount: number; operations: number; operationBytes: number; - rows: number; - /** As in {@link RankedBucketInput.rowOperations}, across the whole definition. */ - rowOperations: number; - rowsEstimated: boolean; - /** As in {@link RankedBucketInput.tables}, across the whole definition. */ - tables: string[]; + /** Number of the definition's buckets that have full-compact statistics. */ + compactedBucketCount?: number; + /** Sum of {@link RankedBucketInput.compactedOperations} across those buckets. */ + compactedOperations?: number; + /** Sum of {@link RankedBucketInput.compactedPuts} across those buckets. */ + compactedPuts?: number; } /** @@ -202,42 +211,6 @@ export function resolveBucketReportLimit(limit?: number): number { return limit; } -/** - * Estimate the true distinct row count of a bucket from a random sample of its operations. - * - * The signal is repetition: a sample that keeps landing on the same rows means few rows, while a sample - * where every operation lands on a new row means many. Formally, each operation is included in the sample - * with probability `r = sampledOps / operations`, so a row with `k` operations appears with probability - * `1 - (1 - r)^k`. Assuming operations are spread roughly evenly across `R` rows (`k = operations / R`), - * the expected number of distinct rows in the sample is `R * (1 - (1 - r)^(operations / R))`. That grows - * with `R`, so a binary search finds the `R` matching the observed distinct count. - * - * The naive `distinctRows / r` ignores repetition and over-counts rows (under-stating fragmentation) on - * exactly the highly fragmented buckets the report exists to surface. - * - * Pure (no I/O) so it is unit-testable; storage adapters supply the sampled counts. - */ -export function estimateDistinctRows(operations: number, sampledOps: number, distinctRows: number): number { - const r = Math.min(1, sampledOps / operations); - if (r >= 1) { - return distinctRows; - } - const expectedDistinct = (rows: number) => rows * (1 - Math.pow(1 - r, operations / rows)); - // The true row count is between the observed distinct count (a lower bound) and one row per operation. - // Binary-search that range until it is narrower than a single row, at which point rounding is exact. - let lo = distinctRows; - let hi = operations; - while (hi - lo > 0.5) { - const mid = (lo + hi) / 2; - if (expectedDistinct(mid) < distinctRows) { - lo = mid; - } else { - hi = mid; - } - } - return Math.round((lo + hi) / 2); -} - /** * Suggest the maintenance action that reduces what new clients download from a bucket, based on its * operation mix. Grounded in the compaction semantics (see `docs/storage/compacting-operations.md`): @@ -249,8 +222,8 @@ export function estimateDistinctRows(operations: number, sampledOps: number, dis * is mostly MOVE/CLEAR residue that a compact alone preserves: `operations` well above `rowOperations`. * - When both kinds of overhead are present, or the mix is inconclusive, suggest both. * - * The thresholds are heuristics; the report is intended to be re-run after acting on it. Inputs may be - * sampled estimates, which is fine at these margins. + * The thresholds are heuristics; the report is intended to be re-run after acting on it. Inputs derive from + * the last full compact's statistics, which is fine at these margins. */ export function suggestBucketAction(operations: number, rowOperations: number, rows: number): BucketAction { const fragmentation = operations / Math.max(rows, 1); @@ -275,9 +248,35 @@ export function suggestBucketAction(operations: number, rowOperations: number, r return 'both'; } +/** + * Derive rows / fragmentation / suggested action from full-compact statistics. + * + * The compacted prefix holds `compactedPuts` row-bearing operations (one per live row) and + * `compactedOperations - compactedPuts` residue operations (MOVE/CLEAR) that only a defragment reclaims. + * Operations written after the compact are raw PUT/REMOVE history, so the total row-bearing count is + * `operations - residue`. Without compact statistics nothing row-related can be derived. + */ +function deriveRowStats( + operations: number, + compactedOperations: number | null | undefined, + compactedPuts: number | null | undefined +): Pick { + if (compactedOperations == null || compactedPuts == null) { + return { rows: null, fragmentation: null, suggestedAction: 'unknown' }; + } + const rows = compactedPuts; + const residue = Math.max(0, compactedOperations - compactedPuts); + const rowOperations = Math.max(0, operations - residue); + return { + rows, + fragmentation: operations / Math.max(rows, 1), + suggestedAction: suggestBucketAction(operations, rowOperations, rows) + }; +} + /** * Assemble the final {@link BucketReport} from per-bucket stats, per-definition stats, and instance-wide - * totals. Storage adapters select and sample the buckets however is cheapest for them; this owns the shared + * totals. Storage adapters select the buckets however is cheapest for them; this owns the shared * fragmentation / ranking / truncation / action logic so it cannot drift. Pure (no I/O) so it is * unit-testable. * @@ -293,29 +292,37 @@ export function assembleBucketReport( const stats: BucketStorageStats[] = buckets.map((b) => ({ bucket: b.bucket, operations: b.operations, - rows: b.rows, operationBytes: b.operationBytes, - fragmentation: b.operations / Math.max(b.rows, 1), - rowsEstimated: b.rowsEstimated, - suggestedAction: suggestBucketAction(b.operations, b.rowOperations, b.rows), - tables: b.tables + uncompactedOperations: Math.max(0, b.operations - (b.compactedOperations ?? 0)), + lastFullCompactAt: b.lastFullCompactAt ?? null, + nextCompactAt: b.nextCompactAt ?? null, + ...deriveRowStats(b.operations, b.compactedOperations, b.compactedPuts) })); - const definitionStats: BucketDefinitionStats[] = definitions.map((d) => ({ - definition: d.definition, - bucketCount: d.bucketCount, - operations: d.operations, - operationBytes: d.operationBytes, - rows: d.rows, - fragmentation: d.operations / Math.max(d.rows, 1), - rowsEstimated: d.rowsEstimated, - suggestedAction: suggestBucketAction(d.operations, d.rowOperations, d.rows), - tables: d.tables - })); + const definitionStats: BucketDefinitionStats[] = definitions.map((d) => { + // When only some of the definition's buckets have full-compact statistics, extrapolate from those + // buckets to the whole definition, assuming the compacted subset is representative. + const compactedBucketCount = d.compactedBucketCount ?? 0; + const scale = compactedBucketCount > 0 ? d.bucketCount / compactedBucketCount : 0; + const derived = + compactedBucketCount > 0 && d.compactedOperations != null && d.compactedPuts != null + ? deriveRowStats(d.operations, Math.round(d.compactedOperations * scale), Math.round(d.compactedPuts * scale)) + : deriveRowStats(d.operations, null, null); + return { + definition: d.definition, + bucketCount: d.bucketCount, + operations: d.operations, + operationBytes: d.operationBytes, + // Unlike the row derivation above, this uses the plain (non-extrapolated) compacted sum: it counts + // the operations no full compact has covered, which is exact rather than an estimate. + uncompactedOperations: Math.max(0, d.operations - (d.compactedOperations ?? 0)), + ...derived + }; + }); // Worst-first: most operations, then most fragmented. - const worstFirst = (a: { operations: number; fragmentation: number }, b: typeof a) => - b.operations - a.operations || b.fragmentation - a.fragmentation; + const worstFirst = (a: { operations: number; fragmentation: number | null }, b: typeof a) => + b.operations - a.operations || (b.fragmentation ?? 0) - (a.fragmentation ?? 0); stats.sort(worstFirst); definitionStats.sort(worstFirst); From a20cf9b35625768f522a8a9276c0c9778ea94dd9 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Wed, 26 Aug 2026 09:27:48 +0200 Subject: [PATCH 32/40] Compute bucket reports from bucket_state only --- .../implementation/MongoSyncBucketStorage.ts | 358 ++++-------------- .../v1/MongoSyncBucketStorageV1.ts | 80 +--- .../v3/MongoSyncBucketStorageV3.ts | 85 +---- 3 files changed, 112 insertions(+), 411 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 868b2af7b..0720ba74d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -25,12 +25,7 @@ import { utils, WatchWriteCheckpointOptions } from '@powersync/service-core'; -import { - BucketDefinitionId, - HydratedSyncConfig, - ParameterLookupRows, - ScopedParameterLookup -} from '@powersync/service-sync-rules'; +import { HydratedSyncConfig, ParameterLookupRows, ScopedParameterLookup } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { LRUCache } from 'lru-cache'; import * as timers from 'timers/promises'; @@ -42,7 +37,7 @@ import { } from './common/MongoSyncBucketStorageCheckpoint.js'; import { DEFAULT_INLINE_THRESHOLD_BYTES } from './common/PersistedBatch.js'; import type { VersionedPowerSyncMongo } from './db.js'; -import { BucketStateDocumentBase, StorageConfig } from './models.js'; +import { StorageConfig } from './models.js'; import { MongoBucketBatchOptions } from './MongoBucketBatch.js'; import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js'; import { MongoCompactOptions, MongoCompactor } from './MongoCompactor.js'; @@ -108,60 +103,37 @@ const BUCKET_SELECTION_SAMPLE_SIZE = 10_000; */ const BUCKET_SELECTION_SCAN_MAX = 1_000_000; -/** - * Fewest operations sampled per bucket when estimating its row count. Buckets with fewer operations than - * this are read in full (exact). - */ -const BUCKET_ROW_SAMPLE_MIN = 1_000; - -/** - * Most operations sampled per bucket, capping the per-bucket cost on very large buckets at the price of a - * weaker estimate for buckets that are both extremely wide and barely fragmented (see {@link bucketRowSampleTarget}). - */ -const BUCKET_ROW_SAMPLE_MAX = 25_000; - -/** Maximum number of per-bucket row-estimate queries to run concurrently while building a report. */ -const BUCKET_ROW_SAMPLE_CONCURRENCY = 10; - -/** Maximum number of tables listed per bucket or definition in the report. */ -const BUCKET_REPORT_TABLE_LIMIT = 10; - -/** A worst-offender bucket selected from bucket_state, with the version-specific context needed to sample it. */ -export interface TopBucketCandidate { - bucket: string; - operations: number; - operationBytes: number; - /** v3 only: the bucket definition id, used to locate its per-definition bucket_data collection. */ - defId?: BucketDefinitionId; -} - -/** A bucket definition aggregated from bucket_state, with the context needed to sample its rows. */ -export interface TopDefinitionCandidate { - /** Definition name as it prefixes bucket names, e.g. `1#by_user`. */ - definition: string; - bucketCount: number; - operations: number; - operationBytes: number; - /** v3 only: the bucket definition id, used to locate its per-definition bucket_data collection. */ - defId?: BucketDefinitionId; -} - export interface TopBucketSelection { - buckets: TopBucketCandidate[]; - definitions: TopDefinitionCandidate[]; + buckets: storage.RankedBucketInput[]; + definitions: storage.RankedDefinitionInput[]; /** True if more definitions exist than `definitions` holds ({@link storage.BUCKET_REPORT_DEFINITION_LIMIT}). */ definitionsTruncated: boolean; totals: storage.BucketReportTotals; } -export interface BucketRowEstimate { - rows: number; - /** Operations carrying a row identity (PUT/REMOVE), i.e. excluding MOVE/CLEAR compaction residue. */ - rowOperations: number; - /** True if `rows` and `rowOperations` are sampled estimates rather than exact counts. */ - estimated: boolean; - /** Tables in the (sampled) row-bearing history, ordered by their share of it, largest first. */ - tables: string[]; +/** + * Version-specific aggregation expressions over a bucket_state document, feeding + * {@link MongoSyncBucketStorage.aggregateTopBuckets}. + */ +export interface BucketStateReportExpressions { + /** The bucket's current total operation count. */ + operations: mongo.Document; + /** The bucket's current operation-history bytes, as a numeric expression. */ + operationBytes: mongo.Document; + /** + * Statistics captured by the bucket's last full compact. Omitted for storage versions that do not record + * them (v1/v2), which limits the report to operation counts. + */ + fullCompact?: { + /** Operation count of the compacted prefix, e.g. `'$last_full_compact.count'`. */ + operations: unknown; + /** PUT count of the compacted prefix (the row count as of the compact). */ + puts: unknown; + /** When the full compact ran. */ + at: unknown; + /** When the scheduled compactor next considers the bucket. */ + nextCompactAt: unknown; + }; } export abstract class MongoSyncBucketStorage @@ -495,71 +467,11 @@ export abstract class MongoSyncBucketStorage async getBucketReport(options?: storage.GetBucketReportOptions): Promise { const limit = storage.resolveBucketReportLimit(options?.limit); try { - // Rank the worst-offender buckets, the per-definition rollup, and total operations from the - // pre-aggregated bucket state (bounded, in the database), then estimate each returned bucket's and - // definition's row count by sampling its operation history. + // Everything comes from the pre-aggregated bucket state (one document per bucket, ranked and limited + // in the database): exact operation counts plus the last full compact's statistics, from which the + // row-level fields are derived. The operation history itself is never read. const { buckets, definitions, definitionsTruncated, totals } = await this.collectTopBuckets(limit); - // Each row estimate is an independent query; run a bounded number concurrently so the report cost - // scales with the limit without firing one query per bucket serially. Definitions sample their whole - // history and are the slowest jobs, so dispatch them first to overlap with the per-bucket estimates. - const rankedBuckets: storage.RankedBucketInput[] = new Array(buckets.length); - const rankedDefinitions: storage.RankedDefinitionInput[] = new Array(definitions.length); - const jobs = buckets.length + definitions.length; - let cursor = 0; - const runWorker = async () => { - while (true) { - const index = cursor++; - if (index >= jobs) { - return; - } - if (index < definitions.length) { - const candidate = definitions[index]; - // A definition's row sample reads its whole (sampled) history, which on a very large instance - // can exceed the time budget even when the per-bucket estimates are fine. The rollup is - // supplementary: omit the definition rather than failing the whole report. - try { - const estimate = await this.estimateDefinitionRows(candidate); - rankedDefinitions[index] = { - definition: candidate.definition, - bucketCount: candidate.bucketCount, - operations: candidate.operations, - operationBytes: candidate.operationBytes, - rows: estimate.rows, - rowOperations: estimate.rowOperations, - rowsEstimated: estimate.estimated, - tables: estimate.tables - }; - } catch (e) { - this.logger.warn( - `Skipping bucket report rollup for definition ${candidate.definition}: row sampling failed`, - e - ); - } - } else { - const candidate = buckets[index - definitions.length]; - const estimate = await this.estimateBucketRows(candidate); - rankedBuckets[index - definitions.length] = { - bucket: candidate.bucket, - operations: candidate.operations, - operationBytes: candidate.operationBytes, - rows: estimate.rows, - rowOperations: estimate.rowOperations, - rowsEstimated: estimate.estimated, - tables: estimate.tables - }; - } - } - }; - const workers = Math.min(BUCKET_ROW_SAMPLE_CONCURRENCY, jobs); - await Promise.all(Array.from({ length: workers }, () => runWorker())); - const sampledDefinitions = rankedDefinitions.filter((d) => d != null); - return storage.assembleBucketReport( - rankedBuckets, - sampledDefinitions, - totals, - // The rollup is also incomplete if a definition was dropped because sampling it failed. - definitionsTruncated || sampledDefinitions.length < definitions.length - ); + return storage.assembleBucketReport(buckets, definitions, totals, definitionsTruncated); } catch (e) { // Translate a storage query timeout (maxTimeMS) into a specific, retryable error code rather than a // generic internal error. @@ -570,27 +482,14 @@ export abstract class MongoSyncBucketStorage /** * Select the worst-offender buckets (by operation count), the per-definition rollup, and instance-wide * operation totals from the pre-aggregated bucket state. Ranking and limiting happen in the database, so - * memory stays bounded. Implementations supply their version-specific bucket state collection and - * active-config filter. + * memory stays bounded. Implementations supply their version-specific bucket state collection, + * active-config filter, and stat expressions. */ protected abstract collectTopBuckets(limit: number): Promise; - /** - * Estimate a single bucket's live row count by sampling its operation history. Implementations differ - * because v1/v2 store one document per operation while v3 batches operations per document. - */ - protected abstract estimateBucketRows(candidate: TopBucketCandidate): Promise; - - /** - * Estimate a whole definition's row count (a row counted once per bucket containing it) by sampling the - * definition's operation history, exactly like {@link estimateBucketRows} but at definition grain. - */ - protected abstract estimateDefinitionRows(candidate: TopDefinitionCandidate): Promise; - /** * Rank buckets by operation count in the database and compute instance-wide operation totals, reading the - * pre-aggregated bucket state (compacted_state + estimate_since_compact). One document per bucket, no scan - * of bucket data. + * pre-aggregated bucket state. One document per bucket, no scan of bucket data. * * For very large bucket sets the candidates are drawn from a bounded sample of the matched `_id` index * range rather than the whole collection (so the request cannot run unbounded or exhaust memory), and the @@ -601,28 +500,15 @@ export abstract class MongoSyncBucketStorage * so buckets that predate bucket_state tracking and have not been updated or compacted since are missing * here and under-counted. v3 always has bucket_state. */ - protected async aggregateTopBuckets( + protected async aggregateTopBuckets( collection: mongo.Collection, match: mongo.Filter, - limit: number - ): Promise<{ - buckets: { id: T['_id']; operations: number; operationBytes: number }[]; - definitions: TopDefinitionCandidate[]; - definitionsTruncated: boolean; - totals: storage.BucketReportTotals; - }> { - const operations = { - $add: [{ $ifNull: ['$compacted_state.count', 0] }, { $ifNull: ['$estimate_since_compact.count', 0] }] - }; - const operationBytes = { - $add: [ - { $toDouble: { $ifNull: ['$compacted_state.bytes', 0] } }, - { $toDouble: { $ifNull: ['$estimate_since_compact.bytes', 0] } } - ] - }; + limit: number, + exprs: BucketStateReportExpressions + ): Promise { + const { operations, operationBytes, fullCompact } = exprs; // Bucket names are `[]`, so everything before the first `[` groups a - // bucket into its definition. v3 additionally carries the definition id in `_id.d`; `$first` is exact - // because all buckets sharing a name prefix share the definition (undefined for v1/v2). + // bucket into its definition. const definitionKey = { $arrayElemAt: [{ $split: ['$_id.b', '['] }, 0] }; // Reports are bulk reads: run them with the configured bulk read preference (secondaries where @@ -669,6 +555,9 @@ export abstract class MongoSyncBucketStorage { $replaceRoot: { newRoot: '$doc' } } ); } + // BSON comparison order places every concrete value above null/missing, so this is true exactly when + // the bucket has full-compact statistics. + const hasFullCompact = fullCompact == null ? false : { $gt: [fullCompact.operations, null] }; pipeline.push({ $facet: { totals: [ @@ -681,7 +570,24 @@ export abstract class MongoSyncBucketStorage } } ], - top: [{ $project: { _id: 1, operations, operationBytes } }, { $sort: { operations: -1 } }, { $limit: limit }], + top: [ + { + $project: { + _id: 0, + bucket: '$_id.b', + operations, + operationBytes, + ...(fullCompact && { + compactedOperations: { $ifNull: [fullCompact.operations, null] }, + compactedPuts: { $ifNull: [fullCompact.puts, null] }, + lastFullCompactAt: { $ifNull: [fullCompact.at, null] }, + nextCompactAt: { $ifNull: [fullCompact.nextCompactAt, null] } + }) + } + }, + { $sort: { operations: -1 } }, + { $limit: limit } + ], definitions: [ { $group: { @@ -689,7 +595,11 @@ export abstract class MongoSyncBucketStorage operations: { $sum: operations }, operationBytes: { $sum: operationBytes }, bucketCount: { $sum: 1 }, - defId: { $first: '$_id.d' } + ...(fullCompact && { + compactedBucketCount: { $sum: { $cond: [hasFullCompact, 1, 0] } }, + compactedOperations: { $sum: { $ifNull: [fullCompact.operations, 0] } }, + compactedPuts: { $sum: { $ifNull: [fullCompact.puts, 0] } } + }) } }, { $sort: { operations: -1 } }, @@ -701,13 +611,15 @@ export abstract class MongoSyncBucketStorage type FacetResult = { totals: { operations: number; operationBytes: number; bucketCount: number }[]; - top: { _id: T['_id']; operations: number; operationBytes: number }[]; + top: storage.RankedBucketInput[]; definitions: { _id: string; operations: number; operationBytes: number; bucketCount: number; - defId?: BucketDefinitionId; + compactedBucketCount?: number; + compactedOperations?: number; + compactedPuts?: number; }[]; }; const [result] = await collection @@ -719,20 +631,20 @@ export abstract class MongoSyncBucketStorage .toArray(); const rawTotals = result?.totals[0] ?? { operations: 0, operationBytes: 0, bucketCount: 0 }; - const buckets = (result?.top ?? []).map((doc) => ({ - id: doc._id, - operations: doc.operations, - operationBytes: doc.operationBytes - })); + const buckets = result?.top ?? []; const rawDefinitions = result?.definitions ?? []; const definitionsTruncated = rawDefinitions.length > storage.BUCKET_REPORT_DEFINITION_LIMIT; - const mapDefinitions = (scale: number): TopDefinitionCandidate[] => + const mapDefinitions = (scale: number): storage.RankedDefinitionInput[] => rawDefinitions.slice(0, storage.BUCKET_REPORT_DEFINITION_LIMIT).map((d) => ({ definition: d._id, bucketCount: Math.round(d.bucketCount * scale), operations: Math.round(d.operations * scale), operationBytes: Math.round(d.operationBytes * scale), - defId: d.defId + ...(fullCompact && { + compactedBucketCount: Math.round((d.compactedBucketCount ?? 0) * scale), + compactedOperations: Math.round((d.compactedOperations ?? 0) * scale), + compactedPuts: Math.round((d.compactedPuts ?? 0) * scale) + }) })); if (!sampled) { @@ -766,122 +678,6 @@ export abstract class MongoSyncBucketStorage }; } - /** - * Estimate a bucket's (or definition's) live rows from a sample of its operations. - * - * `buildPrefix(applySample)` returns a pipeline prefix that selects the operations (down-sampled when - * `applySample` is true) and yields documents with top-level `op`, `table` and `row_id` fields. Returns - * the distinct row count (exact when the whole history was read, otherwise estimated via - * {@link storage.estimateDistinctRows}); fragmentation is then `operations / rows`. - * - * `rowKey` is the `$group` key that identifies a row. Per-bucket estimates use the default (the bucket is - * fixed by the prefix); definition-level estimates must include the bucket name so a row is counted once - * per bucket containing it. - */ - protected async estimateRowsFromOperationSample( - collection: mongo.Collection, - buildPrefix: (applySample: boolean) => mongo.Document[], - operations: number, - sampled: boolean, - rowKey: mongo.Document = { table: '$table', row_id: '$row_id' } - ): Promise { - const runCounts = async (applySample: boolean) => { - const pipeline: mongo.Document[] = [ - ...buildPrefix(applySample), - { - $facet: { - sampledOps: [{ $count: 'count' }], - rowOps: [{ $match: { op: { $in: ['PUT', 'REMOVE'] } } }, { $count: 'count' }], - distinctRows: [ - { $match: { op: { $in: ['PUT', 'REMOVE'] } } }, - { $group: { _id: rowKey } }, - { $count: 'count' } - ], - tables: [ - { $match: { op: { $in: ['PUT', 'REMOVE'] } } }, - { $group: { _id: '$table', operations: { $sum: 1 } } }, - { $sort: { operations: -1 } }, - { $limit: BUCKET_REPORT_TABLE_LIMIT } - ] - } - } - ]; - type FacetResult = { - sampledOps: { count: number }[]; - rowOps: { count: number }[]; - distinctRows: { count: number }[]; - tables: { _id: string }[]; - }; - const [result] = await collection - .aggregate(pipeline, { - allowDiskUse: false, - maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS, - readPreference: this.readPreference - }) - .toArray(); - return { - sampledOps: result?.sampledOps[0]?.count ?? 0, - rowOps: result?.rowOps[0]?.count ?? 0, - distinctRows: result?.distinctRows[0]?.count ?? 0, - tables: (result?.tables ?? []).map((t) => t._id) - }; - }; - - let counts = await runCounts(sampled); - if (sampled && counts.sampledOps == 0) { - // A document-level `$sampleRate` can select nothing when a bucket spans very few storage documents - // (v3 batches operations into a document). Fall back to an exact read so the bucket is not reported as - // zero rows. This reads the whole bucket only in the rare empty-sample case, which cannot happen for a - // bucket large enough to span many documents. - const exact = await runCounts(false); - return { rows: exact.distinctRows, rowOperations: exact.rowOps, estimated: false, tables: exact.tables }; - } - if (counts.distinctRows == 0) { - // Nothing row-bearing was found (e.g. a bucket of only MOVE/CLEAR ops): treat as fully fragmented. - return { rows: 0, rowOperations: 0, estimated: sampled, tables: [] }; - } - if (!sampled) { - // Read in full: the distinct row count is exact. - return { rows: counts.distinctRows, rowOperations: counts.rowOps, estimated: false, tables: counts.tables }; - } - // Only PUT/REMOVE operations carry a row identity; MOVE/CLEAR (produced by compaction) do not. Run the - // estimator over the row-bearing operations only, scaling the bucket's operation count by the row-bearing - // share observed in the sample. Including identity-less operations in the model under-counts rows on - // compacted buckets. For uncompacted buckets rowOps equals sampledOps and this changes nothing. - const rowBearingOperations = Math.round(operations * (counts.rowOps / counts.sampledOps)); - return { - rows: storage.estimateDistinctRows(rowBearingOperations, counts.rowOps, counts.distinctRows), - rowOperations: rowBearingOperations, - estimated: true, - tables: counts.tables - }; - } - - /** - * How many operations to sample when estimating a bucket's row count. - * - * {@link storage.estimateDistinctRows} infers the row count from how often the sample lands on the same - * row twice, so the sample must be large enough to contain such repeats. Sampling `sqrt(200 * operations)` - * operations yields on the order of 100 expected repeats even in the worst case of one row per operation, - * which keeps the estimate stable instead of swinging with sampling noise. The clamp bounds per-bucket - * cost; past the cap only very wide, barely fragmented buckets lose accuracy, and those are not the - * offenders the report exists to surface. - */ - protected bucketRowSampleTarget(operations: number): number { - const target = Math.ceil(Math.sqrt(200 * operations)); - return Math.min(BUCKET_ROW_SAMPLE_MAX, Math.max(BUCKET_ROW_SAMPLE_MIN, target)); - } - - /** Whether a bucket with this many operations should be sampled rather than read in full. */ - protected shouldSampleBucketRows(operations: number): boolean { - return operations > this.bucketRowSampleTarget(operations); - } - - /** `$sampleRate` for sampling roughly {@link bucketRowSampleTarget} operations from a bucket. */ - protected bucketRowSampleRate(operations: number): number { - return this.bucketRowSampleTarget(operations) / operations; - } - /** * The highest op id persisted for this stream, whether or not covered by a checkpoint. * diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index ec2893d77..d3d8aa68d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -38,13 +38,10 @@ import { MongoCompactOptions } from '../MongoCompactor.js'; import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; import { MongoPersistedReplicationStream } from '../MongoPersistedReplicationStream.js'; import { - BucketRowEstimate, MongoCheckpointState, MongoSyncBucketStorage, MongoSyncBucketStorageOptions, - TopBucketCandidate, - TopBucketSelection, - TopDefinitionCandidate + TopBucketSelection } from '../MongoSyncBucketStorage.js'; import { BucketDataDocumentV1, @@ -234,74 +231,27 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage { } // For storage v1/v2, bucket state and bucket data are shared collections scoped by group (replication stream). + // v1/v2 bucket_state does not capture full-compact statistics, so the report is limited to operation + // counts: rows, fragmentation and the suggested action are not available. protected async collectTopBuckets(limit: number): Promise { // Range-match on the whole `_id` (g, b) so the {_id} index bounds the scan; a dotted `{'_id.g': ...}` // match cannot use the compound-object index and would scan the whole collection. - const { buckets, definitions, definitionsTruncated, totals } = await this.aggregateTopBuckets( + return await this.aggregateTopBuckets( this.db.bucketStateV1, { _id: idPrefixFilter<{ g: number; b: string }>({ g: this.replicationStreamId }, ['b']) }, - limit - ); - return { - buckets: buckets.map((b) => ({ bucket: b.id.b, operations: b.operations, operationBytes: b.operationBytes })), - definitions, - definitionsTruncated, - totals - }; - } - - protected estimateBucketRows(candidate: TopBucketCandidate): Promise { - // v1/v2 store one document per operation, so a bucket's ops are an id-prefix range that can be sampled directly. - const sampled = this.shouldSampleBucketRows(candidate.operations); - const buildPrefix = (applySample: boolean): mongo.Document[] => { - // Range-match on the whole `_id` (g, b, o) so the {_id} index is used; a dotted `{'_id.g','_id.b'}` match - // cannot use the compound-object index and would scan the whole collection per bucket. - const prefix: mongo.Document[] = [ - { - $match: { - _id: idPrefixFilter<{ g: number; b: string; o: unknown }>( - { g: this.replicationStreamId, b: candidate.bucket }, - ['o'] - ) - } - } - ]; - if (applySample) { - prefix.push({ $match: { $sampleRate: this.bucketRowSampleRate(candidate.operations) } }); - } - return prefix; - }; - return this.estimateRowsFromOperationSample(this.db.bucketDataV1, buildPrefix, candidate.operations, sampled); - } - - protected estimateDefinitionRows(candidate: TopDefinitionCandidate): Promise { - const sampled = this.shouldSampleBucketRows(candidate.operations); - const buildPrefix = (applySample: boolean): mongo.Document[] => { - // All of a definition's bucket names start with `[`, so an `_id` range on that string - // prefix selects exactly the definition's operations via the index. `\\` (0x5C) is the character - // after `[` (0x5B), so [`[`, `\\`) cannot include any other definition: - // a longer definition name would have to differ at or before the `[`. - const prefix: mongo.Document[] = [ - { - $match: { - _id: { - $gte: { g: this.replicationStreamId, b: `${candidate.definition}[`, o: new bson.MinKey() }, - $lt: { g: this.replicationStreamId, b: `${candidate.definition}\\`, o: new bson.MinKey() } - } - } + limit, + { + operations: { + $add: [{ $ifNull: ['$compacted_state.count', 0] }, { $ifNull: ['$estimate_since_compact.count', 0] }] + }, + operationBytes: { + $add: [ + { $toDouble: { $ifNull: ['$compacted_state.bytes', 0] } }, + { $toDouble: { $ifNull: ['$estimate_since_compact.bytes', 0] } } + ] } - ]; - if (applySample) { - prefix.push({ $match: { $sampleRate: this.bucketRowSampleRate(candidate.operations) } }); } - return prefix; - }; - // Include the bucket name in the row key: at definition grain a row counts once per bucket holding it. - return this.estimateRowsFromOperationSample(this.db.bucketDataV1, buildPrefix, candidate.operations, sampled, { - b: '$_id.b', - table: '$table', - row_id: '$row_id' - }); + ); } protected createMongoParameterCompactor( diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index 7678c8eb6..5a6696bba 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -34,13 +34,10 @@ import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js'; import { MongoParameterCompactor } from '../MongoParameterCompactor.js'; import { MongoPersistedReplicationStream } from '../MongoPersistedReplicationStream.js'; import { - BucketRowEstimate, MongoCheckpointState, MongoSyncBucketStorage, MongoSyncBucketStorageOptions, - TopBucketCandidate, - TopBucketSelection, - TopDefinitionCandidate + TopBucketSelection } from '../MongoSyncBucketStorage.js'; import { loadBucketDataDocument, maxOpId } from './bucket-format.js'; import { @@ -227,10 +224,10 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { return { buckets: compactedBuckets }; } - // For storage v3, bucket state is a per-stream collection and bucket data is split into per-definition collections. - // A replication stream can host multiple sync configs (active + processing + stopped, until cleanup runs), all - // sharing these collections. Scope to the active config's definition ids so the report excludes stale buckets - // from old/stopped definitions. `this.storageIds` is derived from the active config only (see getActiveSyncConfig). + // For storage v3, bucket state is a per-stream collection shared by every sync config the replication + // stream hosts (active + processing + stopped, until cleanup runs). Scope to the active config's definition + // ids so the report excludes stale buckets from old/stopped definitions. `this.storageIds` is derived from + // the active config only (see getActiveSyncConfig). protected async collectTopBuckets(limit: number): Promise { const definitionIds = this.storageIds.bucketDefinitionIds; if (definitionIds.length == 0) { @@ -243,66 +240,24 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { } // One `_id` range per active definition, so the {_id} index bounds the scan per definition; a dotted // `{'_id.d': ...}` match cannot use the compound-object index and would scan the whole collection. - const { buckets, definitions, definitionsTruncated, totals } = await this.aggregateTopBuckets( + return await this.aggregateTopBuckets( this.db.bucketState(this.replicationStreamId), { $or: definitionIds.map((d) => ({ _id: idPrefixFilter<{ d: BucketDefinitionId; b: string }>({ d }, ['b']) })) }, - limit - ); - return { - buckets: buckets.map((b) => ({ - bucket: b.id.b, - operations: b.operations, - operationBytes: b.operationBytes, - defId: b.id.d - })), - definitions, - definitionsTruncated, - totals - }; - } - - protected estimateBucketRows(candidate: TopBucketCandidate): Promise { - // v3 batches operations into documents (one doc holds an `ops` array), in a per-definition collection. - // Sample whole batch documents, then unwind to operation level so the shared estimator sees one doc per op. - const sampled = this.shouldSampleBucketRows(candidate.operations); - const collection = this.db.bucketData(this.replicationStreamId, candidate.defId!); - const buildPrefix = (applySample: boolean): mongo.Document[] => { - // Range-match on the whole `_id` (b, o) so the {_id} index is used; a dotted `{'_id.b': ...}` match - // cannot use the compound-object index and would scan the whole collection per bucket. - const prefix: mongo.Document[] = [ - { $match: { _id: idPrefixFilter<{ b: string; o: unknown }>({ b: candidate.bucket }, ['o']) } } - ]; - if (applySample) { - prefix.push({ $match: { $sampleRate: this.bucketRowSampleRate(candidate.operations) } }); - } - prefix.push({ $unwind: '$ops' }, { $replaceRoot: { newRoot: '$ops' } }); - return prefix; - }; - return this.estimateRowsFromOperationSample(collection, buildPrefix, candidate.operations, sampled); - } - - protected estimateDefinitionRows(candidate: TopDefinitionCandidate): Promise { - // A definition's operations are exactly its per-definition bucket_data collection, so no match stage is - // needed. Keep the bucket name alongside each unwound operation: at definition grain a row counts once - // per bucket holding it. - const sampled = this.shouldSampleBucketRows(candidate.operations); - const collection = this.db.bucketData(this.replicationStreamId, candidate.defId!); - const buildPrefix = (applySample: boolean): mongo.Document[] => { - const prefix: mongo.Document[] = []; - if (applySample) { - prefix.push({ $match: { $sampleRate: this.bucketRowSampleRate(candidate.operations) } }); + limit, + { + // bucket_stats is maintained by writers and compactors, so these are current, exact counts. + operations: { $ifNull: ['$bucket_stats.count', 0] }, + operationBytes: { $toDouble: { $ifNull: ['$bucket_stats.bytes', 0] } }, + // The last full compact's statistics: its `puts` count doubles as the bucket's live row count as of + // that compact, from which the report derives rows, fragmentation and the suggested action. + fullCompact: { + operations: '$last_full_compact.count', + puts: '$last_full_compact.puts', + at: '$last_full_compact.at', + nextCompactAt: '$next_compact_check' + } } - prefix.push( - { $unwind: '$ops' }, - { $project: { b: '$_id.b', op: '$ops.op', table: '$ops.table', row_id: '$ops.row_id' } } - ); - return prefix; - }; - return this.estimateRowsFromOperationSample(collection, buildPrefix, candidate.operations, sampled, { - b: '$b', - table: '$table', - row_id: '$row_id' - }); + ); } protected createMongoParameterCompactor( From b44e722ac609b249de92e79680d0812c8d71a8b3 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Wed, 26 Aug 2026 09:28:30 +0200 Subject: [PATCH 33/40] Add uncompacted operations and compact times to the bucket report API --- .changeset/bucket-storage-report.md | 3 +- packages/service-client/src/client.ts | 35 ++++++++++ .../src/routes/endpoints/admin.ts | 20 +++--- packages/types/src/routes.ts | 68 +++++++++++-------- 4 files changed, 88 insertions(+), 38 deletions(-) diff --git a/.changeset/bucket-storage-report.md b/.changeset/bucket-storage-report.md index b37a614d3..0a7b4af28 100644 --- a/.changeset/bucket-storage-report.md +++ b/.changeset/bucket-storage-report.md @@ -3,6 +3,7 @@ '@powersync/service-types': minor '@powersync/service-module-mongodb-storage': minor '@powersync/service-core-tests': minor +'@powersync/service-client': minor --- -Add a `POST /api/admin/v1/bucket-report` admin endpoint reporting operations vs rows per bucket (MongoDB storage). +Add a `POST /api/admin/v1/bucket-report` admin endpoint reporting per-bucket operation counts, with rows and fragmentation derived from each bucket's last full compact (MongoDB storage; storage v1/v2 report operation counts only). diff --git a/packages/service-client/src/client.ts b/packages/service-client/src/client.ts index f047b7c6b..56eb77d1f 100644 --- a/packages/service-client/src/client.ts +++ b/packages/service-client/src/client.ts @@ -174,4 +174,39 @@ export class InstanceClient ext reprocess = this.createEndpoint({ path: '/api/admin/v1/reprocess' }); + + /** + * Per-bucket storage report for the active sync config: exact operation counts, with rows and + * fragmentation derived from each bucket's last full compact (null until a bucket has been fully + * compacted, and on storage versions without compact statistics). + * + * Example: + * ```typescript + * const report = await client.bucketReport({ limit: 50 }); + * // { + * // buckets: [ + * // { + * // bucket: 'by_user["u1"]', + * // operations: 502, + * // operation_bytes: 54149, + * // uncompacted_operations: 0, + * // rows: 3, + * // fragmentation: 167.3, + * // last_full_compact_at: '2026-08-21T11:26:19.689Z', + * // next_compact_at: null, + * // suggested_action: 'defragment' + * // } + * // ], + * // definitions: [ + * // { definition: 'by_user', bucket_count: 4, operations: 72, ... } + * // ], + * // totals: { bucket_count: 10, operations: 50, operation_bytes: 15031, estimated: false }, + * // buckets_truncated: false, + * // definitions_truncated: false + * // } + * ``` + */ + bucketReport = this.createEndpoint({ + path: '/api/admin/v1/bucket-report' + }); } diff --git a/packages/service-core/src/routes/endpoints/admin.ts b/packages/service-core/src/routes/endpoints/admin.ts index 466430a26..24c692ba2 100644 --- a/packages/service-core/src/routes/endpoints/admin.ts +++ b/packages/service-core/src/routes/endpoints/admin.ts @@ -270,10 +270,12 @@ export const validate = routeDefinition({ }); /** - * Per-bucket report of total operations vs total live rows in storage, for the active sync config. + * Per-bucket report of total operations vs live rows in storage, for the active sync config. * * Answers the recurring "why is my Data Synced so high" question. A high `operations / rows` ratio - * indicates fragmented buckets that a compact or defragment can reclaim. + * indicates fragmented buckets that a compact or defragment can reclaim. Row counts derive from each + * bucket's last full compact (bucket_state only, no operation-history scan), so they are null for buckets + * that have never been fully compacted and on storage versions without compact statistics (v1/v2). */ export const bucketReport = routeDefinition({ path: '/api/admin/v1/bucket-report', @@ -311,23 +313,23 @@ export const bucketReport = routeDefinition({ buckets: report.buckets.map((bucket) => ({ bucket: bucket.bucket, operations: bucket.operations, - rows: bucket.rows, operation_bytes: bucket.operationBytes, + uncompacted_operations: bucket.uncompactedOperations, + rows: bucket.rows, fragmentation: bucket.fragmentation, - rows_estimated: bucket.rowsEstimated, - suggested_action: bucket.suggestedAction, - tables: bucket.tables + last_full_compact_at: bucket.lastFullCompactAt?.toISOString() ?? null, + next_compact_at: bucket.nextCompactAt?.toISOString() ?? null, + suggested_action: bucket.suggestedAction })), definitions: report.definitions.map((definition) => ({ definition: definition.definition, bucket_count: definition.bucketCount, operations: definition.operations, operation_bytes: definition.operationBytes, + uncompacted_operations: definition.uncompactedOperations, rows: definition.rows, fragmentation: definition.fragmentation, - rows_estimated: definition.rowsEstimated, - suggested_action: definition.suggestedAction, - tables: definition.tables + suggested_action: definition.suggestedAction })), totals: { bucket_count: report.totals.bucketCount, diff --git a/packages/types/src/routes.ts b/packages/types/src/routes.ts index 737f84130..ea6e60831 100644 --- a/packages/types/src/routes.ts +++ b/packages/types/src/routes.ts @@ -81,8 +81,8 @@ export type ValidateResponse = t.Encoded; export const BucketReportRequest = t.object({ /** * Maximum number of buckets to return, ranked by operation count descending (worst offenders first). - * Row counts are sampled per returned bucket, so this also bounds the report's cost. Defaults to 50 when - * omitted. Must be an integer between 1 and 1000; anything else is rejected with a validation error. + * Defaults to 50 when omitted. Must be an integer between 1 and 1000; anything else is rejected with a + * validation error. */ limit: t.number.optional() }); @@ -92,36 +92,46 @@ export const SuggestedBucketAction = t .literal('none') .or(t.literal('compact')) .or(t.literal('defragment')) - .or(t.literal('both')); + .or(t.literal('both')) + .or(t.literal('unknown')); export type SuggestedBucketAction = t.Encoded; export const BucketStorageStats = t.object({ /** Full bucket name, e.g. `by_user["u1"]`. */ bucket: t.string, - /** Total operations in the bucket's history (PUT/REMOVE/MOVE/CLEAR). */ + /** Total operations in the bucket's history (PUT/REMOVE/MOVE/CLEAR). Exact and current. */ operations: t.number, - /** Live rows in the bucket. Exact for small buckets, otherwise a sampled estimate (see `rows_estimated`). */ - rows: t.number, /** Approximate size of the operation history in bytes. */ operation_bytes: t.number, + /** + * Operations written after the last full compact — the staleness indicator for `rows` and + * `fragmentation`, which are snapshots from that compact. Equals `operations` when the bucket has never + * been fully compacted. + */ + uncompacted_operations: t.number, + /** + * Live rows in the bucket as of its last full compact, or null if the bucket has never been fully + * compacted (or the storage version does not capture compact statistics). + */ + rows: t.number.or(t.Null), /** * `operations / max(rows, 1)`. ~1 is healthy (fully compacted); higher means more operation-history - * overhead that a compact/defragment can reclaim. + * overhead that a compact/defragment can reclaim. Null whenever `rows` is null. */ - fragmentation: t.number, - /** True if `rows` (and therefore `fragmentation`) is a sampled estimate rather than an exact count. */ - rows_estimated: t.boolean, + fragmentation: t.number.or(t.Null), + /** ISO timestamp of the bucket's last full compact, which is when `rows` was captured. */ + last_full_compact_at: t.string.or(t.Null), /** - * Suggested maintenance action, derived from the bucket's operation mix: `none` (healthy), `compact` - * (un-compacted superseded history to reclaim), `defragment` (mostly compaction residue that only a - * defragment collapses), or `both`. + * ISO timestamp of when the scheduled compactor will next consider this bucket. A suggested compact with + * a future `next_compact_at` means the compact is already planned but throttled until then. */ - suggested_action: SuggestedBucketAction, + next_compact_at: t.string.or(t.Null), /** - * Tables making up the (sampled) operation history, ordered by their share of it, largest first. These - * are the tables whose rows a defragment should touch. + * Suggested maintenance action, derived from the bucket's compact statistics: `none` (healthy), `compact` + * (un-compacted superseded history to reclaim), `defragment` (mostly compaction residue that only a + * defragment collapses), `both`, or `unknown` (no compact statistics to derive one from). */ - tables: t.array(t.string) + suggested_action: SuggestedBucketAction }); export type BucketStorageStats = t.Encoded; @@ -130,23 +140,25 @@ export const BucketDefinitionStats = t.object({ definition: t.string, /** Number of buckets in this definition with stored operations. */ bucket_count: t.number, - /** Total operations across the definition's buckets. */ + /** Total operations across the definition's buckets. Exact and current. */ operations: t.number, /** Approximate size of the definition's operation history in bytes. */ operation_bytes: t.number, /** - * Live rows across the definition's buckets, counting a row once per bucket that contains it. A sampled - * estimate for all but tiny definitions (see `rows_estimated`). + * Operations not covered by any bucket's last full compact — the staleness indicator for `rows` and + * `fragmentation`. + */ + uncompacted_operations: t.number, + /** + * Live rows across the definition's buckets, counting a row once per bucket that contains it. Derived + * from each bucket's last full compact (extrapolated when only some buckets have been compacted); null + * when none have. */ - rows: t.number, - /** `operations / max(rows, 1)` across the whole definition. */ - fragmentation: t.number, - /** True if `rows` (and therefore `fragmentation`) is a sampled estimate rather than an exact count. */ - rows_estimated: t.boolean, + rows: t.number.or(t.Null), + /** `operations / max(rows, 1)` across the whole definition. Null whenever `rows` is null. */ + fragmentation: t.number.or(t.Null), /** Suggested maintenance action for the definition; same values as `buckets[].suggested_action`. */ - suggested_action: SuggestedBucketAction, - /** Tables in the definition's (sampled) operation history, ordered by their share of it, largest first. */ - tables: t.array(t.string) + suggested_action: SuggestedBucketAction }); export type BucketDefinitionStats = t.Encoded; From 90f43b4fe434bf6327152635e5f106308f8e3645 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Wed, 26 Aug 2026 09:29:12 +0200 Subject: [PATCH 34/40] Update bucket report tests for compact derived stats --- .../src/tests/register-bucket-report-tests.ts | 179 +++++++--------- .../test/src/bucket-report.test.ts | 199 +++++++++++------- .../test/src/routes/admin.test.ts | 47 +++-- 3 files changed, 224 insertions(+), 201 deletions(-) diff --git a/packages/service-core-tests/src/tests/register-bucket-report-tests.ts b/packages/service-core-tests/src/tests/register-bucket-report-tests.ts index 3c3d8e58b..98792c480 100644 --- a/packages/service-core-tests/src/tests/register-bucket-report-tests.ts +++ b/packages/service-core-tests/src/tests/register-bucket-report-tests.ts @@ -1,18 +1,22 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { expect, test } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; +import { compactActive } from './util.js'; /** - * Tests for {@link storage.SyncRulesBucketStorage.getBucketReport}: per-bucket operations vs live rows. + * Tests for {@link storage.SyncRulesBucketStorage.getBucketReport}: per-bucket operations, with row counts + * and fragmentation derived from each bucket's last full compact. * - * Asserts on stable counts (operations, rows, fragmentation, operation totals) rather than op_ids or - * checksums, which differ between storage backends and versions. The buckets here are tiny (well under the - * row-sample target), so row counts are exact (`rowsEstimated: false`); the sampling path is exercised in - * the higher-volume manual tests. + * The report reads only bucket_state, never the operation history. Operation counts are exact for every + * storage version; row-derived fields exist only on storage versions that capture full-compact statistics + * (v3), and only after a bucket's first full compact. On v1/v2 they are always null with a suggested action + * of `unknown`. */ export function registerBucketReportTests(config: storage.TestStorageConfig) { const generateStorageFactory = config.factory; const storageVersion = config.storageVersion ?? storage.CURRENT_STORAGE_VERSION; + // v3 bucket_state captures full-compact statistics (rows, fragmentation, compact scheduling). + const capturesCompactStats = storageVersion >= 3; const GLOBAL_SYNC_RULES = ` bucket_definitions: @@ -35,7 +39,20 @@ bucket_definitions: return bucketStorage.getBucketReport(options); }; - test('reports operations and live rows for a single bucket', async () => { + // An explicit per-bucket compact always runs a full compact of that bucket, regardless of scheduling. + // Compact through the active sync config: the instance used by the writer retains its original + // PROCESSING stream snapshot, which some storage versions refuse to compact. + const compactBucket = (factory: storage.BucketStorageFactory, bucket: string) => + compactActive(factory, { + compactBuckets: [bucket], + clearBatchLimit: 10, + moveBatchLimit: 10, + moveBatchQueryLimit: 10, + minBucketChanges: 1, + minChangeRatio: 0 + }); + + test('reports operation counts for a single bucket', async () => { await using factory = await generateStorageFactory(); const { stream, content } = await test_utils.deploySyncRules( factory, @@ -65,14 +82,16 @@ bucket_definitions: expect(report.definitionsTruncated).toEqual(false); const stats = report.buckets.find((b) => b.bucket === bucket)!; - // Three inserts of distinct ids: three operations, three live rows, fully compacted (ratio 1). + // Three inserts of distinct ids: three operations. The bucket has never been fully compacted, so no + // row-derived fields exist yet. expect(stats).toMatchObject({ operations: 3, - rows: 3, - fragmentation: 1, - rowsEstimated: false, - suggestedAction: 'none', - tables: ['test'] + // Never compacted: the whole history counts as uncompacted. + uncompactedOperations: 3, + rows: null, + fragmentation: null, + lastFullCompactAt: null, + suggestedAction: 'unknown' }); expect(stats.operationBytes).toBeGreaterThan(0); expect(report.totals).toMatchObject({ operations: 3, estimated: false }); @@ -83,14 +102,12 @@ bucket_definitions: definition: bucket.split('[')[0], bucketCount: 1, operations: 3, - rows: 3, - fragmentation: 1, - suggestedAction: 'none', - tables: ['test'] + rows: null, + suggestedAction: 'unknown' }); }); - test('operations exceed live rows after updates, and compaction reduces fragmentation', async () => { + test('derives rows and fragmentation from the last full compact', async () => { await using factory = await generateStorageFactory(); const { stream, content } = await test_utils.deploySyncRules( factory, @@ -119,23 +136,44 @@ bucket_definitions: const before = await getReport(bucketStorage); const beforeStats = before.buckets.find((b) => b.bucket === bucket)!; - expect(beforeStats).toMatchObject({ operations: 6, rows: 2, fragmentation: 3 }); - - await bucketStorage.compact({ - clearBatchLimit: 10, - moveBatchLimit: 10, - moveBatchQueryLimit: 10, - minBucketChanges: 1, - minChangeRatio: 0 + // Operation counts are exact even before any compact; rows are unknown until one runs. + expect(beforeStats).toMatchObject({ + operations: 6, + uncompactedOperations: 6, + rows: null, + fragmentation: null, + suggestedAction: 'unknown' }); + if (capturesCompactStats) { + // Writers schedule the bucket for compaction, which the report surfaces. + expect(beforeStats.nextCompactAt).toBeInstanceOf(Date); + } + + await compactBucket(factory, bucket); const after = await getReport(bucketStorage); const afterStats = after.buckets.find((b) => b.bucket === bucket)!; - // Live rows are unchanged; the operation history shrinks toward the live row count. - expect(afterStats.rows).toEqual(2); - expect(afterStats.operations).toBeLessThan(beforeStats.operations); - expect(afterStats.operations).toBeGreaterThanOrEqual(afterStats.rows); - expect(afterStats.fragmentation).toBeLessThan(beforeStats.fragmentation); + if (capturesCompactStats) { + // The full compact counted two live rows and shrank the history toward them. + expect(afterStats.rows).toEqual(2); + expect(afterStats.operations).toBeLessThan(beforeStats.operations); + expect(afterStats.operations).toBeGreaterThanOrEqual(2); + expect(afterStats.fragmentation).toEqual(afterStats.operations / 2); + expect(afterStats.lastFullCompactAt).toBeInstanceOf(Date); + // The compact covered the whole history, so the row stats are fully fresh. + expect(afterStats.uncompactedOperations).toEqual(0); + + // The definition rollup derives its rows from the same compact statistics. + expect(after.definitions).toHaveLength(1); + expect(after.definitions[0]).toMatchObject({ bucketCount: 1, rows: 2 }); + } else { + // v1/v2 storage does not capture compact statistics: the report stays limited to operation counts. + expect(afterStats.rows).toBeNull(); + expect(afterStats.fragmentation).toBeNull(); + expect(afterStats.suggestedAction).toEqual('unknown'); + // The compact itself still shrinks the operation history. + expect(afterStats.operations).toBeLessThan(beforeStats.operations); + } }); test('reports every bucket, ranks worst-first, and totals across all buckets', async () => { @@ -179,16 +217,15 @@ bucket_definitions: // Ranked worst-first by operation count: b1 (3) before b2 (2). expect(report.buckets.map((b) => b.bucket)).toEqual([b1, b2]); - expect(report.buckets.find((b) => b.bucket === b1)).toMatchObject({ operations: 3, rows: 1 }); - expect(report.buckets.find((b) => b.bucket === b2)).toMatchObject({ operations: 2, rows: 2 }); + expect(report.buckets.find((b) => b.bucket === b1)).toMatchObject({ operations: 3 }); + expect(report.buckets.find((b) => b.bucket === b2)).toMatchObject({ operations: 2 }); - // Both buckets belong to one definition; the rollup sums them, counting each bucket's rows separately. + // Both buckets belong to one definition; the rollup sums them. expect(report.definitions).toHaveLength(1); expect(report.definitions[0]).toMatchObject({ definition: b1.split('[')[0], bucketCount: 2, - operations: 5, - rows: 3 + operations: 5 }); // operationBytes is an aggregated ($toDouble) sum; assert every bucket is non-zero and that the @@ -272,76 +309,4 @@ bucket_definitions: expect(report.definitions).toHaveLength(storage.BUCKET_REPORT_DEFINITION_LIMIT); expect(report.definitionsTruncated).toEqual(true); }); - - test('samples the row count for a bucket above the sampling threshold', async () => { - await using factory = await generateStorageFactory(); - const { stream, content } = await test_utils.deploySyncRules( - factory, - updateSyncRulesFromYaml(GLOBAL_SYNC_RULES, { storageVersion }) - ); - const bucketStorage = factory.getInstance(stream); - - await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); - const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); - await writer.markAllSnapshotDone('1/1'); - - // 50 rows, each updated 25 times, is 1,300 operations against 50 live rows. That is past the 1,000 - // operation threshold, so the report samples the operation history rather than reading it in full and - // the row count comes back as an estimate. The value per update varies so no two writes are identical. - // Each round is flushed separately so the operations span many storage documents, as they would in real - // replication (some backends batch operations per document, and a sample must see more than one). - const rowCount = 50; - const updatesPerRow = 25; - for (let row = 0; row < rowCount; row++) { - await writer.save({ - sourceTable: testTable, - tag: storage.SaveOperationTag.INSERT, - after: { id: `r${row}` }, - afterReplicaId: test_utils.rid(`r${row}`) - }); - } - await writer.commit('1/1'); - await writer.flush(); - for (let update = 0; update < updatesPerRow; update++) { - for (let row = 0; row < rowCount; row++) { - await writer.save({ - sourceTable: testTable, - tag: storage.SaveOperationTag.UPDATE, - after: { id: `r${row}`, value: `v${update}` }, - afterReplicaId: test_utils.rid(`r${row}`) - }); - } - await writer.commit('1/1'); - await writer.flush(); - } - - const bucket = test_utils.bucketRequest(content, 'global[]').bucket; - const report = await getReport(bucketStorage); - const stats = report.buckets.find((b) => b.bucket === bucket)!; - - // The operation count is exact (read from bucket_state); the row count is a sampled estimate. - expect(stats.operations).toEqual(rowCount + rowCount * updatesPerRow); - expect(stats.rowsEstimated).toEqual(true); - // The sample covers enough of a bucket this fragmented to recover the 50 live rows within a small margin. - expect(stats.rows).toBeGreaterThanOrEqual(45); - expect(stats.rows).toBeLessThanOrEqual(55); - // Fragmentation is operations / rows, so a heavily updated bucket reads well above 1. - expect(stats.fragmentation).toBeGreaterThan(10); - // The history is un-compacted superseded churn, which a compact reclaims. - expect(stats.suggestedAction).toEqual('compact'); - // The sampled history names the tables a defragment would touch. - expect(stats.tables).toEqual(['test']); - - // The definition rollup samples the same history at definition grain. - expect(report.definitions).toHaveLength(1); - const defStats = report.definitions[0]; - expect(defStats).toMatchObject({ - bucketCount: 1, - operations: stats.operations, - suggestedAction: 'compact', - tables: ['test'] - }); - expect(defStats.rows).toBeGreaterThanOrEqual(45); - expect(defStats.rows).toBeLessThanOrEqual(55); - }); } diff --git a/packages/service-core/test/src/bucket-report.test.ts b/packages/service-core/test/src/bucket-report.test.ts index 61679e91f..fe3d7444c 100644 --- a/packages/service-core/test/src/bucket-report.test.ts +++ b/packages/service-core/test/src/bucket-report.test.ts @@ -2,7 +2,6 @@ import { assembleBucketReport, BucketReportTotals, DEFAULT_BUCKET_REPORT_LIMIT, - estimateDistinctRows, MAX_BUCKET_REPORT_LIMIT, RankedBucketInput, RankedDefinitionInput, @@ -11,36 +10,29 @@ import { } from '@/storage/bucket-report.js'; import { describe, expect, it } from 'vitest'; -// Row-bearing operations default to all operations (no compaction residue) unless overridden. -const bucket = ( - name: string, - operations: number, - rows: number, - extra?: Partial -): RankedBucketInput => ({ +const bucket = (name: string, operations: number, extra?: Partial): RankedBucketInput => ({ bucket: name, operations, - rows, - operationBytes: extra?.operationBytes ?? 0, - rowOperations: extra?.rowOperations ?? operations, - rowsEstimated: extra?.rowsEstimated ?? false, - tables: extra?.tables ?? [] + operationBytes: 0, + ...extra +}); + +/** Full-compact statistics where the compacted prefix is the whole history (no writes since). */ +const compacted = (operations: number, puts: number): Partial => ({ + compactedOperations: operations, + compactedPuts: puts }); const definition = ( name: string, operations: number, - rows: number, extra?: Partial ): RankedDefinitionInput => ({ definition: name, - bucketCount: extra?.bucketCount ?? 1, + bucketCount: 1, operations, - rows, - operationBytes: extra?.operationBytes ?? 0, - rowOperations: extra?.rowOperations ?? operations, - rowsEstimated: extra?.rowsEstimated ?? false, - tables: extra?.tables ?? [] + operationBytes: 0, + ...extra }); const totals = (bucketCount: number, extra?: Partial): BucketReportTotals => ({ @@ -51,11 +43,18 @@ const totals = (bucketCount: number, extra?: Partial): Bucke }); describe('assembleBucketReport', () => { - it('derives fragmentation and passes through rowsEstimated and tables', () => { + it('derives rows and fragmentation from the full-compact statistics', () => { + const compactedAt = new Date('2026-01-01T00:00:00Z'); + const nextCompact = new Date('2026-01-02T00:00:00Z'); const report = assembleBucketReport( [ - bucket('global[]', 100, 10, { operationBytes: 1024, tables: ['todos', 'lists'] }), - bucket('by_user["u1"]', 30, 30, { rowsEstimated: true }) + bucket('global[]', 100, { + operationBytes: 1024, + ...compacted(100, 10), + lastFullCompactAt: compactedAt, + nextCompactAt: nextCompact + }), + bucket('by_user["u1"]', 30, compacted(30, 30)) ], [], totals(2) @@ -63,21 +62,63 @@ describe('assembleBucketReport', () => { expect(report.buckets.find((b) => b.bucket === 'global[]')).toMatchObject({ operations: 100, - rows: 10, operationBytes: 1024, + rows: 10, fragmentation: 10, - rowsEstimated: false, - tables: ['todos', 'lists'] + lastFullCompactAt: compactedAt, + nextCompactAt: nextCompact }); expect(report.buckets.find((b) => b.bucket === 'by_user["u1"]')).toMatchObject({ + rows: 30, fragmentation: 1, - rowsEstimated: true + lastFullCompactAt: null, + nextCompactAt: null }); }); + it('reports null rows and an unknown action without full-compact statistics', () => { + const report = assembleBucketReport([bucket('global[]', 100)], [], totals(1)); + + expect(report.buckets[0]).toMatchObject({ + operations: 100, + rows: null, + fragmentation: null, + suggestedAction: 'unknown', + // Never compacted: the whole history is uncompacted. + uncompactedOperations: 100 + }); + }); + + it('reports the operations written since the last full compact', () => { + const report = assembleBucketReport( + [ + // Fully covered by the compact: nothing uncompacted. + bucket('fresh[]', 100, compacted(100, 100)), + // 900 operations landed after the compact: rows/fragmentation are that much out of date. + bucket('stale[]', 1000, compacted(100, 100)) + ], + [ + // Definition grain uses the plain compacted sum (not the extrapolated one used for rows): + // 1000 total ops, 50 covered by compacts of half the buckets, so 950 are uncompacted. + definition('1#partial', 1000, { + bucketCount: 10, + compactedBucketCount: 5, + compactedOperations: 50, + compactedPuts: 50 + }) + ], + totals(12) + ); + + const by = (name: string) => report.buckets.find((b) => b.bucket === name)!; + expect(by('fresh[]').uncompactedOperations).toEqual(0); + expect(by('stale[]').uncompactedOperations).toEqual(900); + expect(report.definitions[0].uncompactedOperations).toEqual(950); + }); + it('ranks buckets worst-first by operations then fragmentation', () => { const report = assembleBucketReport( - [bucket('a[]', 5, 5), bucket('b[]', 50, 5), bucket('c[]', 50, 50)], + [bucket('a[]', 5, compacted(5, 5)), bucket('b[]', 50, compacted(50, 5)), bucket('c[]', 50, compacted(50, 50))], [], totals(3) ); @@ -87,27 +128,27 @@ describe('assembleBucketReport', () => { }); it('floors rows at 1 so a bucket with operations but no rows is fully fragmented', () => { - const report = assembleBucketReport([bucket('gone[]', 42, 0)], [], totals(1)); + const report = assembleBucketReport([bucket('gone[]', 42, compacted(42, 0))], [], totals(1)); expect(report.buckets[0]).toMatchObject({ operations: 42, rows: 0, fragmentation: 42 }); }); it('marks the bucket list truncated when there are more buckets than returned', () => { - const truncated = assembleBucketReport([bucket('a[]', 10, 1), bucket('b[]', 5, 1)], [], totals(5)); + const truncated = assembleBucketReport([bucket('a[]', 10), bucket('b[]', 5)], [], totals(5)); expect(truncated.bucketsTruncated).toBe(true); - const complete = assembleBucketReport([bucket('a[]', 10, 1), bucket('b[]', 5, 1)], [], totals(2)); + const complete = assembleBucketReport([bucket('a[]', 10), bucket('b[]', 5)], [], totals(2)); expect(complete.bucketsTruncated).toBe(false); }); it('passes the definition truncation flag through, defaulting to complete', () => { - expect(assembleBucketReport([], [definition('a', 1, 1)], totals(1)).definitionsTruncated).toBe(false); - expect(assembleBucketReport([], [definition('a', 1, 1)], totals(1), true).definitionsTruncated).toBe(true); + expect(assembleBucketReport([], [definition('a', 1)], totals(1)).definitionsTruncated).toBe(false); + expect(assembleBucketReport([], [definition('a', 1)], totals(1), true).definitionsTruncated).toBe(true); }); it('carries the totals through unchanged', () => { const t = totals(2, { operations: 120, operationBytes: 15, estimated: true }); - const report = assembleBucketReport([bucket('a[]', 100, 4), bucket('b[]', 20, 2)], [], t); + const report = assembleBucketReport([bucket('a[]', 100), bucket('b[]', 20)], [], t); expect(report.totals).toEqual({ bucketCount: 2, operations: 120, operationBytes: 15, estimated: true }); }); @@ -116,8 +157,19 @@ describe('assembleBucketReport', () => { const report = assembleBucketReport( [], [ - definition('1#by_user', 100, 100, { bucketCount: 10 }), - definition('1#by_org', 500, 100, { bucketCount: 5, operationBytes: 2048 }) + definition('1#by_user', 100, { + bucketCount: 10, + compactedBucketCount: 10, + compactedOperations: 100, + compactedPuts: 100 + }), + definition('1#by_org', 500, { + bucketCount: 5, + operationBytes: 2048, + compactedBucketCount: 5, + compactedOperations: 500, + compactedPuts: 100 + }) ], totals(15) ); @@ -131,20 +183,48 @@ describe('assembleBucketReport', () => { operationBytes: 2048, rows: 100, fragmentation: 5, - suggestedAction: 'compact' + suggestedAction: 'defragment' }); expect(report.definitions[1]).toMatchObject({ fragmentation: 1, suggestedAction: 'none' }); }); - it('derives per-bucket suggested actions from the operation mix', () => { + it('extrapolates definition rows when only some buckets have been fully compacted', () => { + const report = assembleBucketReport( + [], + [ + // 5 of 10 buckets compacted, holding 50 rows between them: assume the other half looks the same. + definition('1#partial', 1000, { + bucketCount: 10, + compactedBucketCount: 5, + compactedOperations: 50, + compactedPuts: 50 + }), + // No compacted buckets: nothing row-related can be derived. + definition('1#uncompacted', 1000, { bucketCount: 10, compactedBucketCount: 0 }) + ], + totals(20) + ); + + expect(report.definitions.find((d) => d.definition === '1#partial')).toMatchObject({ + rows: 100, + fragmentation: 10 + }); + expect(report.definitions.find((d) => d.definition === '1#uncompacted')).toMatchObject({ + rows: null, + fragmentation: null, + suggestedAction: 'unknown' + }); + }); + + it('derives per-bucket suggested actions from the compact statistics', () => { const report = assembleBucketReport( [ // Healthy: one op per row. - bucket('healthy[]', 100, 100), - // Un-compacted churn: every op carries a row identity, far more ops than rows. - bucket('churned[]', 1000, 100), - // Compacted residue: mostly MOVE/CLEAR ops left behind by a compact. - bucket('compacted[]', 1000, 100, { rowOperations: 150 }) + bucket('healthy[]', 100, compacted(100, 100)), + // Un-compacted churn since the last full compact: 900 raw ops on top of a clean 100-row prefix. + bucket('churned[]', 1000, compacted(100, 100)), + // Compacted residue: the compact kept 150 rows but left 850 MOVE/CLEAR ops behind. + bucket('residue[]', 1000, compacted(1000, 150)) ], [], totals(3) @@ -153,7 +233,7 @@ describe('assembleBucketReport', () => { const action = (name: string) => report.buckets.find((b) => b.bucket === name)?.suggestedAction; expect(action('healthy[]')).toEqual('none'); expect(action('churned[]')).toEqual('compact'); - expect(action('compacted[]')).toEqual('defragment'); + expect(action('residue[]')).toEqual('defragment'); }); }); @@ -189,37 +269,6 @@ describe('suggestBucketAction', () => { }); }); -describe('estimateDistinctRows', () => { - it('returns the observed distinct count when the whole bucket was sampled', () => { - // r >= 1: nothing was left out, so the observed distinct count is already exact. - expect(estimateDistinctRows(100, 100, 40)).toBe(40); - expect(estimateDistinctRows(100, 150, 40)).toBe(40); - }); - - it('recovers a heavily fragmented bucket the naive estimate would inflate', () => { - // 10 rows x 1000 ops each; a 10% sample sees ~1000 ops but still only the same 10 distinct rows. - // Naive distinct/rate would report 10 / 0.1 = 100 rows (10x too many, so 10x too little fragmentation). - const rows = estimateDistinctRows(10_000, 1_000, 10); - expect(rows).toBeGreaterThanOrEqual(9); - expect(rows).toBeLessThanOrEqual(12); - }); - - it('recovers a moderately fragmented bucket', () => { - // 500 rows x 2 ops each, 50% sample. Ground truth: 500*(1-0.5^2) = 375 distinct sampled rows. - // Naive distinct/rate would report 375 / 0.5 = 750 rows; the estimator should recover ~500. - const rows = estimateDistinctRows(1_000, 500, 375); - expect(rows).toBeGreaterThan(480); - expect(rows).toBeLessThan(520); - }); - - it('matches the naive estimate when there are no sampling collisions', () => { - // 2000 rows, 1 op each, 50% sample: no row is seen twice, so distinct/rate is already correct (~2000). - const rows = estimateDistinctRows(2_000, 1_000, 1_000); - expect(rows).toBeGreaterThan(1_900); - expect(rows).toBeLessThan(2_100); - }); -}); - describe('resolveBucketReportLimit', () => { it('defaults when no limit is given', () => { expect(resolveBucketReportLimit(undefined)).toBe(DEFAULT_BUCKET_REPORT_LIMIT); diff --git a/packages/service-core/test/src/routes/admin.test.ts b/packages/service-core/test/src/routes/admin.test.ts index 2c7f9af79..65889ea56 100644 --- a/packages/service-core/test/src/routes/admin.test.ts +++ b/packages/service-core/test/src/routes/admin.test.ts @@ -217,22 +217,24 @@ bucket_definitions: { bucket: '1#by_user["u1"]', operations: 4750, - rows: 95, operationBytes: 1216000, + uncompactedOperations: 250, + rows: 95, fragmentation: 50, - rowsEstimated: true, - suggestedAction: 'compact', - tables: ['todos'] + lastFullCompactAt: new Date('2026-08-01T00:00:00.000Z'), + nextCompactAt: new Date('2026-08-21T12:00:00.000Z'), + suggestedAction: 'compact' }, { bucket: '1#global[]', operations: 1000, - rows: 1000, operationBytes: 3145728, - fragmentation: 1, - rowsEstimated: false, - suggestedAction: 'none', - tables: ['lists'] + uncompactedOperations: 1000, + rows: null, + fragmentation: null, + lastFullCompactAt: null, + nextCompactAt: null, + suggestedAction: 'unknown' } ], definitions: [ @@ -241,11 +243,10 @@ bucket_definitions: bucketCount: 1, operations: 4750, operationBytes: 1216000, + uncompactedOperations: 250, rows: 95, fragmentation: 50, - rowsEstimated: true, - suggestedAction: 'compact', - tables: ['todos'] + suggestedAction: 'compact' } ], totals: { bucketCount: 2, operations: 5750, operationBytes: 4361728, estimated: false }, @@ -273,12 +274,21 @@ bucket_definitions: expect(response.buckets[0]).toEqual({ bucket: '1#by_user["u1"]', operations: 4750, - rows: 95, operation_bytes: 1216000, + uncompacted_operations: 250, + rows: 95, fragmentation: 50, - rows_estimated: true, - suggested_action: 'compact', - tables: ['todos'] + last_full_compact_at: '2026-08-01T00:00:00.000Z', + next_compact_at: '2026-08-21T12:00:00.000Z', + suggested_action: 'compact' + }); + // Dates and row stats are null for buckets without full-compact statistics. + expect(response.buckets[1]).toMatchObject({ + rows: null, + fragmentation: null, + last_full_compact_at: null, + next_compact_at: null, + suggested_action: 'unknown' }); expect(response.definitions).toEqual([ { @@ -286,11 +296,10 @@ bucket_definitions: bucket_count: 1, operations: 4750, operation_bytes: 1216000, + uncompacted_operations: 250, rows: 95, fragmentation: 50, - rows_estimated: true, - suggested_action: 'compact', - tables: ['todos'] + suggested_action: 'compact' } ]); expect(response.totals).toEqual({ From 40230b1a33a45ca7e536b1bf040392f020c40355 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Wed, 26 Aug 2026 09:45:13 +0200 Subject: [PATCH 35/40] Export BucketReportTotals type --- packages/types/src/routes.ts | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/packages/types/src/routes.ts b/packages/types/src/routes.ts index ea6e60831..93f2d13d8 100644 --- a/packages/types/src/routes.ts +++ b/packages/types/src/routes.ts @@ -162,21 +162,24 @@ export const BucketDefinitionStats = t.object({ }); export type BucketDefinitionStats = t.Encoded; +export const BucketReportTotals = t.object({ + /** Number of buckets with stored operations. Exact, even when the other totals are estimates. */ + bucket_count: t.number, + /** Sum of operations across all buckets. Estimated when the bucket set was sampled. */ + operations: t.number, + /** Sum of operation-history bytes across all buckets. Estimated when the bucket set was sampled. */ + operation_bytes: t.number, + /** True if the totals are estimated because the bucket set was sampled rather than fully scanned. */ + estimated: t.boolean +}); +export type BucketReportTotals = t.Encoded; + export const BucketReportResponse = t.object({ /** Worst-offender buckets, ranked by operation count then fragmentation. */ buckets: t.array(BucketStorageStats), /** Per-definition rollup, ranked by operation count then fragmentation. */ definitions: t.array(BucketDefinitionStats), - totals: t.object({ - /** Number of buckets with stored operations. Exact, even when the other totals are estimates. */ - bucket_count: t.number, - /** Sum of operations across all buckets. Estimated when the bucket set was sampled. */ - operations: t.number, - /** Sum of operation-history bytes across all buckets. Estimated when the bucket set was sampled. */ - operation_bytes: t.number, - /** True if the totals are estimated because the bucket set was sampled rather than fully scanned. */ - estimated: t.boolean - }), + totals: BucketReportTotals, /** True if there are more buckets than returned (more than `limit`). */ buckets_truncated: t.boolean, /** True if the definition rollup is incomplete: more definitions exist than the report caps at. */ From eb66b399880aee0d0c1699e57698f428eeadae4d Mon Sep 17 00:00:00 2001 From: bean1352 Date: Wed, 26 Aug 2026 16:09:17 +0200 Subject: [PATCH 36/40] Default bucket report reads to secondaryPreferred --- .../storage/implementation/MongoSyncBucketStorage.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 0720ba74d..5875e2976 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -511,9 +511,14 @@ export abstract class MongoSyncBucketStorage // bucket into its definition. const definitionKey = { $arrayElemAt: [{ $split: ['$_id.b', '['] }, 0] }; - // Reports are bulk reads: run them with the configured bulk read preference (secondaries where - // configured) so they do not load the primary. - const readPreference = this.readPreference; + // Reports are bulk reads: keep them off the primary by using the configured bulk read preference, + // falling back to secondaryPreferred. Staleness does not matter for a report. + const readPreference = + this.readPreference ?? + new mongo.ReadPreference('secondaryPreferred', undefined, { + // 90 is the minimum value. + maxStalenessSeconds: 90 + }); // estimatedDocumentCount is O(1) but ignores the match filter, so this is an upper bound on the active // bucket count. That is fine for the sampling decision: over-estimating only switches to sampling sooner. From f00e343323c1f2a69ff69dcaed5c3b22e3155ba1 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Wed, 26 Aug 2026 16:24:55 +0200 Subject: [PATCH 37/40] Reuse mongo timeout constant and test helpers --- .../implementation/MongoSyncBucketStorage.ts | 4 +- .../test/src/bucket-report-scoping.test.ts | 40 ++----------------- .../service-core/src/storage/bucket-report.ts | 6 --- 3 files changed, 6 insertions(+), 44 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 5875e2976..a3c07af8a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -531,7 +531,7 @@ export abstract class MongoSyncBucketStorage // `limit` caps how many index entries the count may touch: hitting the cap means the instance is past // what this report is designed to scan, so fail fast rather than read the index without bound. matchedBuckets = await collection.countDocuments(match, { - maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS, + maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS, readPreference, limit: BUCKET_SELECTION_SCAN_MAX + 1 }); @@ -630,7 +630,7 @@ export abstract class MongoSyncBucketStorage const [result] = await collection .aggregate(pipeline, { allowDiskUse: false, - maxTimeMS: storage.BUCKET_REPORT_TIMEOUT_MS, + maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS, readPreference }) .toArray(); diff --git a/modules/module-mongodb-storage/test/src/bucket-report-scoping.test.ts b/modules/module-mongodb-storage/test/src/bucket-report-scoping.test.ts index 040e90cd9..74e3e5ab3 100644 --- a/modules/module-mongodb-storage/test/src/bucket-report-scoping.test.ts +++ b/modules/module-mongodb-storage/test/src/bucket-report-scoping.test.ts @@ -1,31 +1,9 @@ import { MongoSyncBucketStorageV3 } from '@module/storage/implementation/v3/MongoSyncBucketStorageV3.js'; import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { test_utils } from '@powersync/service-core-tests'; -import * as bson from 'bson'; import { describe, expect, test } from 'vitest'; import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; -function sourceDescriptor(name: string, objectId: string): storage.SourceEntityDescriptor { - return { - connectionTag: storage.SourceTable.DEFAULT_TAG, - objectId, - schema: 'public', - name, - replicaIdColumns: [{ name: 'id', type: 'VARCHAR', typeId: 25 }] - }; -} - -function objectIdGenerator(id: string) { - let used = false; - return () => { - if (used) { - throw new Error(`Can only generate a single id using ${id}`); - } - used = true; - return new bson.ObjectId(id); - }; -} - /** * In V3 a replication stream can host multiple sync configs (active + stopped, until cleanup runs), all sharing * the per-stream bucket_state and source_records collections. The report must only include the active config's @@ -51,13 +29,9 @@ streams: ); const firstStorage = factory.getInstance(first) as MongoSyncBucketStorageV3; await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); - const todosTable = ( - await firstWriter.resolveTables({ - connection_id: 1, - source: sourceDescriptor('todos', 'todos-relation'), - idGenerator: objectIdGenerator('6544e3899293153fa7b38360') - }) - ).tables[0]; + // Distinct idIndex per table: source records are shared per replication stream, so the two configs' + // tables must not collide on the semi-hardcoded test id. + const todosTable = await test_utils.resolveTestTable(firstWriter, 'todos', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY, 1); await firstWriter.save({ sourceTable: todosTable, tag: storage.SaveOperationTag.INSERT, @@ -96,13 +70,7 @@ streams: const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorageV3; await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS); // Give config 2 its own replicated row, so the report has an active-config bucket to include. - const scenesTable = ( - await secondWriter.resolveTables({ - connection_id: 1, - source: sourceDescriptor('scenes', 'scenes-relation'), - idGenerator: objectIdGenerator('6544e3899293153fa7b38361') - }) - ).tables[0]; + const scenesTable = await test_utils.resolveTestTable(secondWriter, 'scenes', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY, 2); await secondWriter.save({ sourceTable: scenesTable, tag: storage.SaveOperationTag.INSERT, diff --git a/packages/service-core/src/storage/bucket-report.ts b/packages/service-core/src/storage/bucket-report.ts index a5d79dbcb..ed9384971 100644 --- a/packages/service-core/src/storage/bucket-report.ts +++ b/packages/service-core/src/storage/bucket-report.ts @@ -16,12 +16,6 @@ */ import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; -/** - * Time budget for the per-bucket report's bucket-selection aggregation (`maxTimeMS`). Bounded so an admin - * request on a large instance fails fast instead of running unbounded. - */ -export const BUCKET_REPORT_TIMEOUT_MS: number = 60_000; - /** Number of worst-offender buckets returned when the request omits a `limit`. */ export const DEFAULT_BUCKET_REPORT_LIMIT: number = 50; From 3b61305fe7d05ad4616791fd11f6522021449f72 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Wed, 26 Aug 2026 16:41:51 +0200 Subject: [PATCH 38/40] Add bucket report sampling test --- .../test/src/bucket-report-sampling.test.ts | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 modules/module-mongodb-storage/test/src/bucket-report-sampling.test.ts diff --git a/modules/module-mongodb-storage/test/src/bucket-report-sampling.test.ts b/modules/module-mongodb-storage/test/src/bucket-report-sampling.test.ts new file mode 100644 index 000000000..92281d716 --- /dev/null +++ b/modules/module-mongodb-storage/test/src/bucket-report-sampling.test.ts @@ -0,0 +1,103 @@ +import { MongoSyncBucketStorageV3 } from '@module/storage/implementation/v3/MongoSyncBucketStorageV3.js'; +import { VersionedPowerSyncMongoV3 } from '@module/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; +import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; +import { test_utils } from '@powersync/service-core-tests'; +import { describe, expect, test } from 'vitest'; +import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; + +/** + * Above the sampling threshold the report ranks a sample of bucket_state instead of scanning it in full, + * and scales the totals back up. Replicating enough buckets to cross the threshold would dominate the + * test's runtime, so this replicates one real bucket and clones its state document past the threshold. + * + * The clones are identical, which makes the scaled estimates exact (the sample-rate factor cancels: + * `sampleCount * count * (matched / sampleCount) == matched * count`), so the assertions can use equality + * instead of tolerances. + */ +describe('bucket report sampling - mongodb v3', () => { + // Seeding 55k bucket_state documents takes longer than the default test timeout. + test('samples and scales above the bucket threshold', { timeout: 60_000 }, async () => { + await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); + + const deployed = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +config: + edition: 3 + +streams: + by_owner: + query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id') +`, + { storageVersion: 3 } + ) + ); + const bucketStorage = factory.getInstance(deployed) as MongoSyncBucketStorageV3; + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const todosTable = await test_utils.resolveTestTable(writer, 'todos', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); + await writer.save({ + sourceTable: todosTable, + tag: storage.SaveOperationTag.INSERT, + after: { id: 'todo-1', owner_id: 'user-1' }, + afterReplicaId: test_utils.rid('todo-1') + }); + await writer.markAllSnapshotDone('1/1'); + await writer.commit('1/1'); + await writer.flush(); + + // Below the threshold the report is an exact scan. + const exactReport = await bucketStorage.getBucketReport(); + expect(exactReport.totals).toEqual({ bucketCount: 1, operations: 1, operationBytes: expect.any(Number), estimated: false }); + + const bucketStateCollection = (bucketStorage.db as VersionedPowerSyncMongoV3).bucketState( + bucketStorage.replicationStreamId + ); + const seed = await bucketStateCollection.findOne({}); + if (seed == null) { + throw new Error('Expected a bucket_state document for the replicated bucket'); + } + const definition = seed._id.b.split('[')[0]; + + const CLONES = 55_000; + const INSERT_BATCH = 10_000; + // last_op has a unique index (bucket_updates), so every clone needs its own value. + const baseOp = BigInt(String(seed.last_op)); + for (let offset = 0; offset < CLONES; offset += INSERT_BATCH) { + const batch = Array.from({ length: Math.min(INSERT_BATCH, CLONES - offset) }, (_, i) => ({ + ...seed, + _id: { d: seed._id.d, b: `${definition}["clone${offset + i}"]` }, + last_op: baseOp + BigInt(offset + i + 1) + })); + await bucketStateCollection.insertMany(batch, { ordered: false }); + } + + const totalBuckets = CLONES + 1; + const operationsPerBucket = seed.bucket_stats.count; + const bytesPerBucket = Number(String(seed.bucket_stats.bytes)); + + const limit = 10; + const report = await bucketStorage.getBucketReport({ limit }); + + expect(report.totals.estimated).toBe(true); + // The matched-bucket count is exact even when sampling. + expect(report.totals.bucketCount).toBe(totalBuckets); + expect(report.totals.operations).toBe(totalBuckets * operationsPerBucket); + expect(report.totals.operationBytes).toBe(totalBuckets * bytesPerBucket); + + // The returned buckets are real sampled documents, so their per-bucket stats are exact, not scaled. + expect(report.buckets).toHaveLength(limit); + for (const bucket of report.buckets) { + expect(bucket.operations).toBe(operationsPerBucket); + expect(bucket.operationBytes).toBe(bytesPerBucket); + } + expect(report.bucketsTruncated).toBe(true); + + expect(report.definitions).toHaveLength(1); + expect(report.definitions[0].definition).toBe(definition); + expect(report.definitions[0].bucketCount).toBe(totalBuckets); + expect(report.definitions[0].operations).toBe(totalBuckets * operationsPerBucket); + // No full compact has run, so no row stats exist to derive from. + expect(report.definitions[0].rows).toBeNull(); + expect(report.definitions[0].suggestedAction).toBe('unknown'); + }); +}); From 074cbf6315921fa3fb8fba42868fd2598a7cd71d Mon Sep 17 00:00:00 2001 From: bean1352 Date: Wed, 26 Aug 2026 16:46:43 +0200 Subject: [PATCH 39/40] Use enumLiteral and orNull codec helpers --- packages/types/src/routes.ts | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/packages/types/src/routes.ts b/packages/types/src/routes.ts index 69aef116d..eaa3da127 100644 --- a/packages/types/src/routes.ts +++ b/packages/types/src/routes.ts @@ -1,5 +1,5 @@ import * as t from 'ts-codec'; -import { anyPrimitive } from './codecs.js'; +import { anyPrimitive, enumLiteral, orNull } from './codecs.js'; import { ConnectionStatus, InstanceSchema, SyncRulesStatus } from './definitions.js'; export const GetSchemaRequest = t.object({}); @@ -89,12 +89,7 @@ export const BucketReportRequest = t.object({ }); export type BucketReportRequest = t.Encoded; -export const SuggestedBucketAction = t - .literal('none') - .or(t.literal('compact')) - .or(t.literal('defragment')) - .or(t.literal('both')) - .or(t.literal('unknown')); +export const SuggestedBucketAction = enumLiteral('none', 'compact', 'defragment', 'both', 'unknown'); export type SuggestedBucketAction = t.Encoded; export const BucketStorageStats = t.object({ @@ -114,19 +109,19 @@ export const BucketStorageStats = t.object({ * Live rows in the bucket as of its last full compact, or null if the bucket has never been fully * compacted (or the storage version does not capture compact statistics). */ - rows: t.number.or(t.Null), + rows: orNull(t.number), /** * `operations / max(rows, 1)`. ~1 is healthy (fully compacted); higher means more operation-history * overhead that a compact/defragment can reclaim. Null whenever `rows` is null. */ - fragmentation: t.number.or(t.Null), + fragmentation: orNull(t.number), /** ISO timestamp of the bucket's last full compact, which is when `rows` was captured. */ - last_full_compact_at: t.string.or(t.Null), + last_full_compact_at: orNull(t.string), /** * ISO timestamp of when the scheduled compactor will next consider this bucket. A suggested compact with * a future `next_compact_at` means the compact is already planned but throttled until then. */ - next_compact_at: t.string.or(t.Null), + next_compact_at: orNull(t.string), /** * Suggested maintenance action, derived from the bucket's compact statistics: `none` (healthy), `compact` * (un-compacted superseded history to reclaim), `defragment` (mostly compaction residue that only a @@ -155,9 +150,9 @@ export const BucketDefinitionStats = t.object({ * from each bucket's last full compact (extrapolated when only some buckets have been compacted); null * when none have. */ - rows: t.number.or(t.Null), + rows: orNull(t.number), /** `operations / max(rows, 1)` across the whole definition. Null whenever `rows` is null. */ - fragmentation: t.number.or(t.Null), + fragmentation: orNull(t.number), /** Suggested maintenance action for the definition; same values as `buckets[].suggested_action`. */ suggested_action: SuggestedBucketAction }); From b197fc8d9f69bacb0268dea5456dfaae1e826573 Mon Sep 17 00:00:00 2001 From: bean1352 Date: Thu, 27 Aug 2026 09:07:10 +0200 Subject: [PATCH 40/40] Format bucket report tests --- .../test/src/bucket-report-sampling.test.ts | 7 ++++++- .../test/src/bucket-report-scoping.test.ts | 16 ++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/modules/module-mongodb-storage/test/src/bucket-report-sampling.test.ts b/modules/module-mongodb-storage/test/src/bucket-report-sampling.test.ts index 92281d716..4276c1540 100644 --- a/modules/module-mongodb-storage/test/src/bucket-report-sampling.test.ts +++ b/modules/module-mongodb-storage/test/src/bucket-report-sampling.test.ts @@ -47,7 +47,12 @@ streams: // Below the threshold the report is an exact scan. const exactReport = await bucketStorage.getBucketReport(); - expect(exactReport.totals).toEqual({ bucketCount: 1, operations: 1, operationBytes: expect.any(Number), estimated: false }); + expect(exactReport.totals).toEqual({ + bucketCount: 1, + operations: 1, + operationBytes: expect.any(Number), + estimated: false + }); const bucketStateCollection = (bucketStorage.db as VersionedPowerSyncMongoV3).bucketState( bucketStorage.replicationStreamId diff --git a/modules/module-mongodb-storage/test/src/bucket-report-scoping.test.ts b/modules/module-mongodb-storage/test/src/bucket-report-scoping.test.ts index 74e3e5ab3..239c348a7 100644 --- a/modules/module-mongodb-storage/test/src/bucket-report-scoping.test.ts +++ b/modules/module-mongodb-storage/test/src/bucket-report-scoping.test.ts @@ -31,7 +31,13 @@ streams: await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); // Distinct idIndex per table: source records are shared per replication stream, so the two configs' // tables must not collide on the semi-hardcoded test id. - const todosTable = await test_utils.resolveTestTable(firstWriter, 'todos', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY, 1); + const todosTable = await test_utils.resolveTestTable( + firstWriter, + 'todos', + ['id'], + INITIALIZED_MONGO_STORAGE_FACTORY, + 1 + ); await firstWriter.save({ sourceTable: todosTable, tag: storage.SaveOperationTag.INSERT, @@ -70,7 +76,13 @@ streams: const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorageV3; await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS); // Give config 2 its own replicated row, so the report has an active-config bucket to include. - const scenesTable = await test_utils.resolveTestTable(secondWriter, 'scenes', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY, 2); + const scenesTable = await test_utils.resolveTestTable( + secondWriter, + 'scenes', + ['id'], + INITIALIZED_MONGO_STORAGE_FACTORY, + 2 + ); await secondWriter.save({ sourceTable: scenesTable, tag: storage.SaveOperationTag.INSERT,