diff --git a/.changeset/bucket-storage-report.md b/.changeset/bucket-storage-report.md new file mode 100644 index 000000000..0a7b4af28 --- /dev/null +++ b/.changeset/bucket-storage-report.md @@ -0,0 +1,9 @@ +--- +'@powersync/service-core': minor +'@powersync/service-types': minor +'@powersync/service-module-mongodb-storage': minor +'@powersync/service-core-tests': minor +'@powersync/service-client': minor +--- + +Add a `POST /api/admin/v1/bucket-report` admin endpoint reporting per-bucket operation counts, with rows and fragmentation derived from each bucket's last full compact (MongoDB storage; storage v1/v2 report operation counts only). diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index d3fd8354f..c84e54657 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -4,9 +4,11 @@ import { BaseObserver, logger as defaultLogger, DO_NOT_LOG, + ErrorCode, Logger, ReplicationAbortedError, - ServiceAssertionError + ServiceAssertionError, + ServiceError } from '@powersync/lib-services-framework'; import { BroadcastIterable, @@ -82,6 +84,59 @@ interface InternalCheckpointChanges extends CheckpointChanges { */ const CHECKPOINT_TIMEOUT_MS = 60_000; +/** + * Above this many buckets (a collection-wide estimate), the report ranks a bounded sample of bucket_state + * rather than every bucket, so the request cannot exhaust memory or run unbounded. Below it, the ranking is + * exact. + */ +const BUCKET_SELECTION_SAMPLE_THRESHOLD = 50_000; + +/** + * Approximate number of buckets sampled when over {@link BUCKET_SELECTION_SAMPLE_THRESHOLD}. The sample is + * drawn with `$sampleRate`, so the achieved count varies slightly around this. + */ +const BUCKET_SELECTION_SAMPLE_SIZE = 10_000; + +/** + * Most bucket_state index entries one report query may scan. Even when sampling fetches few documents, the + * covered index scan and the matched-bucket count still touch every matched index entry once, so past this + * the report fails fast instead of scaling without bound. + */ +const BUCKET_SELECTION_SCAN_MAX = 1_000_000; + +export interface TopBucketSelection { + buckets: storage.RankedBucketInput[]; + definitions: storage.RankedDefinitionInput[]; + /** True if more definitions exist than `definitions` holds ({@link storage.BUCKET_REPORT_DEFINITION_LIMIT}). */ + definitionsTruncated: boolean; + totals: storage.BucketReportTotals; +} + +/** + * Version-specific aggregation expressions over a bucket_state document, feeding + * {@link MongoSyncBucketStorage.aggregateTopBuckets}. + */ +export interface BucketStateReportExpressions { + /** The bucket's current total operation count. */ + operations: mongo.Document; + /** The bucket's current operation-history bytes, as a numeric expression. */ + operationBytes: mongo.Document; + /** + * Statistics captured by the bucket's last full compact. Omitted for storage versions that do not record + * them (v1/v2), which limits the report to operation counts. + */ + fullCompact?: { + /** Operation count of the compacted prefix, e.g. `'$last_full_compact.count'`. */ + operations: unknown; + /** PUT count of the compacted prefix (the row count as of the compact). */ + puts: unknown; + /** When the full compact ran. */ + at: unknown; + /** When the scheduled compactor next considers the bucket. */ + nextCompactAt: unknown; + }; +} + export abstract class MongoSyncBucketStorage extends BaseObserver implements storage.SyncRulesBucketStorage @@ -431,6 +486,225 @@ export abstract class MongoSyncBucketStorage options: CompactInitialReplicationOptions ): Promise; + async getBucketReport(options?: storage.GetBucketReportOptions): Promise { + const limit = storage.resolveBucketReportLimit(options?.limit); + try { + // Everything comes from the pre-aggregated bucket state (one document per bucket, ranked and limited + // in the database): exact operation counts plus the last full compact's statistics, from which the + // row-level fields are derived. The operation history itself is never read. + const { buckets, definitions, definitionsTruncated, totals } = await this.collectTopBuckets(limit); + return storage.assembleBucketReport(buckets, definitions, totals, definitionsTruncated); + } catch (e) { + // Translate a storage query timeout (maxTimeMS) into a specific, retryable error code rather than a + // generic internal error. + throw lib_mongo.mapQueryError(e, 'while building the bucket report'); + } + } + + /** + * Select the worst-offender buckets (by operation count), the per-definition rollup, and instance-wide + * operation totals from the pre-aggregated bucket state. Ranking and limiting happen in the database, so + * memory stays bounded. Implementations supply their version-specific bucket state collection, + * active-config filter, and stat expressions. + */ + protected abstract collectTopBuckets(limit: number): Promise; + + /** + * Rank buckets by operation count in the database and compute instance-wide operation totals, reading the + * pre-aggregated bucket state. One document per bucket, no scan of bucket data. + * + * For very large bucket sets the candidates are drawn from a bounded sample of the matched `_id` index + * range rather than the whole collection (so the request cannot run unbounded or exhaust memory), and the + * totals are scaled from the sample and flagged estimated. `allowDiskUse: false` makes an over-threshold + * exact attempt fail fast rather than spill to disk and degrade the live instance. + * + * Note: for v1/v2 storage, bucket_state is not backfilled (see models.ts: "only populated by new updates"), + * so buckets that predate bucket_state tracking and have not been updated or compacted since are missing + * here and under-counted. v3 always has bucket_state. + */ + protected async aggregateTopBuckets( + collection: mongo.Collection, + match: mongo.Filter, + limit: number, + exprs: BucketStateReportExpressions + ): Promise { + const { operations, operationBytes, fullCompact } = exprs; + // Bucket names are `[]`, so everything before the first `[` groups a + // bucket into its definition. + const definitionKey = { $arrayElemAt: [{ $split: ['$_id.b', '['] }, 0] }; + + // Reports are bulk reads: keep them off the primary by using the configured bulk read preference, + // falling back to secondaryPreferred. Staleness does not matter for a report. + const readPreference = + this.readPreference ?? + new mongo.ReadPreference('secondaryPreferred', undefined, { + // 90 is the minimum value. + maxStalenessSeconds: 90 + }); + + // estimatedDocumentCount is O(1) but ignores the match filter, so this is an upper bound on the active + // bucket count. That is fine for the sampling decision: over-estimating only switches to sampling sooner. + const estimatedTotalBuckets = await collection.estimatedDocumentCount({ readPreference }); + + let matchedBuckets: number | null = null; + if (estimatedTotalBuckets > BUCKET_SELECTION_SAMPLE_THRESHOLD) { + // The exact matched-bucket count. `match` is an `_id` range, so this is an index-only scan; it sets + // the sample rate, scales the sampled sums back up, and doubles as the exact totals.bucketCount. + // `limit` caps how many index entries the count may touch: hitting the cap means the instance is past + // what this report is designed to scan, so fail fast rather than read the index without bound. + matchedBuckets = await collection.countDocuments(match, { + maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS, + readPreference, + limit: BUCKET_SELECTION_SCAN_MAX + 1 + }); + if (matchedBuckets > BUCKET_SELECTION_SCAN_MAX) { + throw new ServiceError({ + status: 422, + code: ErrorCode.PSYNC_S2001, + description: `Bucket report is not supported on this instance: more than ${BUCKET_SELECTION_SCAN_MAX} buckets match the active sync configuration` + }); + } + } + const sampleRate = matchedBuckets == null ? 1 : BUCKET_SELECTION_SAMPLE_SIZE / Math.max(matchedBuckets, 1); + const sampled = sampleRate < 1; + + const pipeline: mongo.Document[] = [{ $match: match }]; + if (sampled) { + // Sample on the index alone, then fetch only the sampled documents: the range $match plus the _id + // projection is a covered index scan (explain shows docsExamined: 0), $sampleRate keeps roughly + // SAMPLE_SIZE ids, and the self-$lookup fetches just those. Sampling after a plain $match would fetch + // every matched document only to discard most of them. + pipeline.push( + { $project: { _id: 1 } }, + { $match: { $sampleRate: sampleRate } }, + { $lookup: { from: collection.collectionName, localField: '_id', foreignField: '_id', as: 'doc' } }, + { $unwind: '$doc' }, + { $replaceRoot: { newRoot: '$doc' } } + ); + } + // BSON comparison order places every concrete value above null/missing, so this is true exactly when + // the bucket has full-compact statistics. + const hasFullCompact = fullCompact == null ? false : { $gt: [fullCompact.operations, null] }; + pipeline.push({ + $facet: { + totals: [ + { + $group: { + _id: null, + operations: { $sum: operations }, + operationBytes: { $sum: operationBytes }, + bucketCount: { $sum: 1 } + } + } + ], + top: [ + { + $project: { + _id: 0, + bucket: '$_id.b', + operations, + operationBytes, + ...(fullCompact && { + compactedOperations: { $ifNull: [fullCompact.operations, null] }, + compactedPuts: { $ifNull: [fullCompact.puts, null] }, + lastFullCompactAt: { $ifNull: [fullCompact.at, null] }, + nextCompactAt: { $ifNull: [fullCompact.nextCompactAt, null] } + }) + } + }, + { $sort: { operations: -1 } }, + { $limit: limit } + ], + definitions: [ + { + $group: { + _id: definitionKey, + operations: { $sum: operations }, + operationBytes: { $sum: operationBytes }, + bucketCount: { $sum: 1 }, + ...(fullCompact && { + compactedBucketCount: { $sum: { $cond: [hasFullCompact, 1, 0] } }, + compactedOperations: { $sum: { $ifNull: [fullCompact.operations, 0] } }, + compactedPuts: { $sum: { $ifNull: [fullCompact.puts, 0] } } + }) + } + }, + { $sort: { operations: -1 } }, + // One past the cap: an extra result only signals that the rollup was truncated. + { $limit: storage.BUCKET_REPORT_DEFINITION_LIMIT + 1 } + ] + } + }); + + type FacetResult = { + totals: { operations: number; operationBytes: number; bucketCount: number }[]; + top: storage.RankedBucketInput[]; + definitions: { + _id: string; + operations: number; + operationBytes: number; + bucketCount: number; + compactedBucketCount?: number; + compactedOperations?: number; + compactedPuts?: number; + }[]; + }; + const [result] = await collection + .aggregate(pipeline, { + allowDiskUse: false, + maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS, + readPreference + }) + .toArray(); + + const rawTotals = result?.totals[0] ?? { operations: 0, operationBytes: 0, bucketCount: 0 }; + const buckets = result?.top ?? []; + const rawDefinitions = result?.definitions ?? []; + const definitionsTruncated = rawDefinitions.length > storage.BUCKET_REPORT_DEFINITION_LIMIT; + const mapDefinitions = (scale: number): storage.RankedDefinitionInput[] => + rawDefinitions.slice(0, storage.BUCKET_REPORT_DEFINITION_LIMIT).map((d) => ({ + definition: d._id, + bucketCount: Math.round(d.bucketCount * scale), + operations: Math.round(d.operations * scale), + operationBytes: Math.round(d.operationBytes * scale), + ...(fullCompact && { + compactedBucketCount: Math.round((d.compactedBucketCount ?? 0) * scale), + compactedOperations: Math.round((d.compactedOperations ?? 0) * scale), + compactedPuts: Math.round((d.compactedPuts ?? 0) * scale) + }) + })); + + if (!sampled) { + return { + buckets, + definitions: mapDefinitions(1), + definitionsTruncated, + totals: { + bucketCount: rawTotals.bucketCount, + operations: rawTotals.operations, + operationBytes: rawTotals.operationBytes, + estimated: false + } + }; + } + + // Scale the sampled sums up to the full matched set, using the exact matched count from above. The + // sample is uniform across buckets, so the per-definition sums scale by the same factor; a definition + // small enough to be missed by the sample entirely is absent. bucketCount itself is exact. + const scale = matchedBuckets! / Math.max(rawTotals.bucketCount, 1); + return { + buckets, + definitions: mapDefinitions(scale), + definitionsTruncated, + totals: { + bucketCount: matchedBuckets!, + operations: Math.round(rawTotals.operations * scale), + operationBytes: Math.round(rawTotals.operationBytes * scale), + estimated: true + } + }; + } + /** * The highest op id persisted for this stream, whether or not covered by a checkpoint. * diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index 343a5fef5..d3d8aa68d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -40,7 +40,8 @@ import { MongoPersistedReplicationStream } from '../MongoPersistedReplicationStr import { MongoCheckpointState, MongoSyncBucketStorage, - MongoSyncBucketStorageOptions + MongoSyncBucketStorageOptions, + TopBucketSelection } from '../MongoSyncBucketStorage.js'; import { BucketDataDocumentV1, @@ -229,6 +230,30 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage { return result; } + // For storage v1/v2, bucket state and bucket data are shared collections scoped by group (replication stream). + // v1/v2 bucket_state does not capture full-compact statistics, so the report is limited to operation + // counts: rows, fragmentation and the suggested action are not available. + protected async collectTopBuckets(limit: number): Promise { + // Range-match on the whole `_id` (g, b) so the {_id} index bounds the scan; a dotted `{'_id.g': ...}` + // match cannot use the compound-object index and would scan the whole collection. + return await this.aggregateTopBuckets( + this.db.bucketStateV1, + { _id: idPrefixFilter<{ g: number; b: string }>({ g: this.replicationStreamId }, ['b']) }, + limit, + { + operations: { + $add: [{ $ifNull: ['$compacted_state.count', 0] }, { $ifNull: ['$estimate_since_compact.count', 0] }] + }, + operationBytes: { + $add: [ + { $toDouble: { $ifNull: ['$compacted_state.bytes', 0] } }, + { $toDouble: { $ifNull: ['$estimate_since_compact.bytes', 0] } } + ] + } + } + ); + } + protected createMongoParameterCompactor( checkpoint: InternalOpId, options: storage.CompactOptions diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts index ffcffd472..83149d77f 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts @@ -15,9 +15,14 @@ import { utils } from '@powersync/service-core'; import { JSONBig } from '@powersync/service-jsonbig'; -import { ParameterLookupRows, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules'; +import { + BucketDefinitionId, + ParameterLookupRows, + ScopedParameterLookup, + SqliteJsonRow +} from '@powersync/service-sync-rules'; import * as bson from 'bson'; -import { mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js'; +import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js'; import { MongoBucketStorage } from '../../MongoBucketStorage.js'; import { BucketDataDoc } from '../common/BucketDataDoc.js'; import { @@ -31,7 +36,8 @@ import { MongoPersistedReplicationStream } from '../MongoPersistedReplicationStr import { MongoCheckpointState, MongoSyncBucketStorage, - MongoSyncBucketStorageOptions + MongoSyncBucketStorageOptions, + TopBucketSelection } from '../MongoSyncBucketStorage.js'; import { MongoCheckpointAPIOptions } from '../MongoWriteCheckpointAPI.js'; import { loadBucketDataDocument, maxOpId } from './bucket-format.js'; @@ -229,6 +235,42 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage { return { buckets: compactedBuckets }; } + // For storage v3, bucket state is a per-stream collection shared by every sync config the replication + // stream hosts (active + processing + stopped, until cleanup runs). Scope to the active config's definition + // ids so the report excludes stale buckets from old/stopped definitions. `this.storageIds` is derived from + // the active config only (see getActiveSyncConfig). + protected async collectTopBuckets(limit: number): Promise { + const definitionIds = this.storageIds.bucketDefinitionIds; + if (definitionIds.length == 0) { + return { + buckets: [], + definitions: [], + definitionsTruncated: false, + totals: { bucketCount: 0, operations: 0, operationBytes: 0, estimated: false } + }; + } + // One `_id` range per active definition, so the {_id} index bounds the scan per definition; a dotted + // `{'_id.d': ...}` match cannot use the compound-object index and would scan the whole collection. + return await this.aggregateTopBuckets( + this.db.bucketState(this.replicationStreamId), + { $or: definitionIds.map((d) => ({ _id: idPrefixFilter<{ d: BucketDefinitionId; b: string }>({ d }, ['b']) })) }, + limit, + { + // bucket_stats is maintained by writers and compactors, so these are current, exact counts. + operations: { $ifNull: ['$bucket_stats.count', 0] }, + operationBytes: { $toDouble: { $ifNull: ['$bucket_stats.bytes', 0] } }, + // The last full compact's statistics: its `puts` count doubles as the bucket's live row count as of + // that compact, from which the report derives rows, fragmentation and the suggested action. + fullCompact: { + operations: '$last_full_compact.count', + puts: '$last_full_compact.puts', + at: '$last_full_compact.at', + nextCompactAt: '$next_compact_check' + } + } + ); + } + protected createMongoParameterCompactor( checkpoint: InternalOpId, options: storage.CompactOptions diff --git a/modules/module-mongodb-storage/test/src/bucket-report-sampling.test.ts b/modules/module-mongodb-storage/test/src/bucket-report-sampling.test.ts new file mode 100644 index 000000000..4276c1540 --- /dev/null +++ b/modules/module-mongodb-storage/test/src/bucket-report-sampling.test.ts @@ -0,0 +1,108 @@ +import { MongoSyncBucketStorageV3 } from '@module/storage/implementation/v3/MongoSyncBucketStorageV3.js'; +import { VersionedPowerSyncMongoV3 } from '@module/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; +import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; +import { test_utils } from '@powersync/service-core-tests'; +import { describe, expect, test } from 'vitest'; +import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; + +/** + * Above the sampling threshold the report ranks a sample of bucket_state instead of scanning it in full, + * and scales the totals back up. Replicating enough buckets to cross the threshold would dominate the + * test's runtime, so this replicates one real bucket and clones its state document past the threshold. + * + * The clones are identical, which makes the scaled estimates exact (the sample-rate factor cancels: + * `sampleCount * count * (matched / sampleCount) == matched * count`), so the assertions can use equality + * instead of tolerances. + */ +describe('bucket report sampling - mongodb v3', () => { + // Seeding 55k bucket_state documents takes longer than the default test timeout. + test('samples and scales above the bucket threshold', { timeout: 60_000 }, async () => { + await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); + + const deployed = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +config: + edition: 3 + +streams: + by_owner: + query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id') +`, + { storageVersion: 3 } + ) + ); + const bucketStorage = factory.getInstance(deployed) as MongoSyncBucketStorageV3; + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const todosTable = await test_utils.resolveTestTable(writer, 'todos', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); + await writer.save({ + sourceTable: todosTable, + tag: storage.SaveOperationTag.INSERT, + after: { id: 'todo-1', owner_id: 'user-1' }, + afterReplicaId: test_utils.rid('todo-1') + }); + await writer.markAllSnapshotDone('1/1'); + await writer.commit('1/1'); + await writer.flush(); + + // Below the threshold the report is an exact scan. + const exactReport = await bucketStorage.getBucketReport(); + expect(exactReport.totals).toEqual({ + bucketCount: 1, + operations: 1, + operationBytes: expect.any(Number), + estimated: false + }); + + const bucketStateCollection = (bucketStorage.db as VersionedPowerSyncMongoV3).bucketState( + bucketStorage.replicationStreamId + ); + const seed = await bucketStateCollection.findOne({}); + if (seed == null) { + throw new Error('Expected a bucket_state document for the replicated bucket'); + } + const definition = seed._id.b.split('[')[0]; + + const CLONES = 55_000; + const INSERT_BATCH = 10_000; + // last_op has a unique index (bucket_updates), so every clone needs its own value. + const baseOp = BigInt(String(seed.last_op)); + for (let offset = 0; offset < CLONES; offset += INSERT_BATCH) { + const batch = Array.from({ length: Math.min(INSERT_BATCH, CLONES - offset) }, (_, i) => ({ + ...seed, + _id: { d: seed._id.d, b: `${definition}["clone${offset + i}"]` }, + last_op: baseOp + BigInt(offset + i + 1) + })); + await bucketStateCollection.insertMany(batch, { ordered: false }); + } + + const totalBuckets = CLONES + 1; + const operationsPerBucket = seed.bucket_stats.count; + const bytesPerBucket = Number(String(seed.bucket_stats.bytes)); + + const limit = 10; + const report = await bucketStorage.getBucketReport({ limit }); + + expect(report.totals.estimated).toBe(true); + // The matched-bucket count is exact even when sampling. + expect(report.totals.bucketCount).toBe(totalBuckets); + expect(report.totals.operations).toBe(totalBuckets * operationsPerBucket); + expect(report.totals.operationBytes).toBe(totalBuckets * bytesPerBucket); + + // The returned buckets are real sampled documents, so their per-bucket stats are exact, not scaled. + expect(report.buckets).toHaveLength(limit); + for (const bucket of report.buckets) { + expect(bucket.operations).toBe(operationsPerBucket); + expect(bucket.operationBytes).toBe(bytesPerBucket); + } + expect(report.bucketsTruncated).toBe(true); + + expect(report.definitions).toHaveLength(1); + expect(report.definitions[0].definition).toBe(definition); + expect(report.definitions[0].bucketCount).toBe(totalBuckets); + expect(report.definitions[0].operations).toBe(totalBuckets * operationsPerBucket); + // No full compact has run, so no row stats exist to derive from. + expect(report.definitions[0].rows).toBeNull(); + expect(report.definitions[0].suggestedAction).toBe('unknown'); + }); +}); diff --git a/modules/module-mongodb-storage/test/src/bucket-report-scoping.test.ts b/modules/module-mongodb-storage/test/src/bucket-report-scoping.test.ts new file mode 100644 index 000000000..239c348a7 --- /dev/null +++ b/modules/module-mongodb-storage/test/src/bucket-report-scoping.test.ts @@ -0,0 +1,108 @@ +import { MongoSyncBucketStorageV3 } from '@module/storage/implementation/v3/MongoSyncBucketStorageV3.js'; +import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; +import { test_utils } from '@powersync/service-core-tests'; +import { describe, expect, test } from 'vitest'; +import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; + +/** + * In V3 a replication stream can host multiple sync configs (active + stopped, until cleanup runs), all sharing + * the per-stream bucket_state and source_records collections. The report must only include the active config's + * bucket definitions, not stale ones from a previous (now stopped) config. + */ +describe('bucket report scoping - mongodb v3', () => { + test('excludes buckets from stopped/old sync configs sharing the replication stream', async () => { + await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); + + // Config 1: replicate todos. Writing a row creates a data bucket for this config. + const first = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +config: + edition: 3 + +streams: + by_owner: + query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id') +`, + { storageVersion: 3 } + ) + ); + const firstStorage = factory.getInstance(first) as MongoSyncBucketStorageV3; + await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); + // Distinct idIndex per table: source records are shared per replication stream, so the two configs' + // tables must not collide on the semi-hardcoded test id. + const todosTable = await test_utils.resolveTestTable( + firstWriter, + 'todos', + ['id'], + INITIALIZED_MONGO_STORAGE_FACTORY, + 1 + ); + await firstWriter.save({ + sourceTable: todosTable, + tag: storage.SaveOperationTag.INSERT, + after: { id: 'todo-1', owner_id: 'user-1' }, + afterReplicaId: test_utils.rid('todo-1') + }); + await firstWriter.markAllSnapshotDone('1/1'); + await firstWriter.commit('1/1'); + await firstWriter.flush(); + + // While config 1 is active, its bucket(s) show up in the report. + const firstReport = await firstStorage.getBucketReport(); + expect(firstReport.totals.bucketCount).toBeGreaterThan(0); + + // Config 2: a different stream over a different table. Config 1 transitions to STOP, but its bucket_state + // and source_records rows remain in the shared collections until cleanup runs (which we deliberately skip). + const second = await factory.updateSyncRules( + updateSyncRulesFromYaml( + ` +config: + edition: 3 + +streams: + by_project: + query: SELECT * FROM scenes WHERE project_id = subscription.parameter('project_id') +`, + { storageVersion: 3 } + ) + ); + expect(second.replicationStreamId).toBe(first.replicationStreamId); + + // Drive config 2 to snapshot-done so it becomes ACTIVE and config 1 transitions to STOP (config 1 keeps + // serving until the new config finishes processing). Config 1's stale rows remain until cleanup, which we skip. + const replicatingStreams = await factory.getReplicatingReplicationStreams(); + expect(replicatingStreams).toHaveLength(1); + const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorageV3; + await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS); + // Give config 2 its own replicated row, so the report has an active-config bucket to include. + const scenesTable = await test_utils.resolveTestTable( + secondWriter, + 'scenes', + ['id'], + INITIALIZED_MONGO_STORAGE_FACTORY, + 2 + ); + await secondWriter.save({ + sourceTable: scenesTable, + tag: storage.SaveOperationTag.INSERT, + after: { id: 'scene-1', project_id: 'project-1' }, + afterReplicaId: test_utils.rid('scene-1') + }); + await secondWriter.markAllSnapshotDone('2/1'); + await secondWriter.commit('2/1'); + await secondWriter.flush(); + + const activeConfig = await factory.getActiveSyncConfig(); + expect(activeConfig).not.toBeNull(); + const activeStorage = activeConfig!.storage as MongoSyncBucketStorageV3; + const secondReport = await activeStorage.getBucketReport(); + + // The active config's own bucket is included (include-active), while config 1's stale buckets, which still + // exist in the shared collections, are excluded (exclude-stale). Without scoping to the active config's + // definition ids, config 1's bucket would leak in here. + expect(secondReport.totals.bucketCount).toBeGreaterThan(0); + const firstBucketNames = new Set(firstReport.buckets.map((b) => b.bucket)); + expect(secondReport.buckets.some((b) => firstBucketNames.has(b.bucket))).toBe(false); + }); +}); diff --git a/modules/module-mongodb-storage/test/src/storage.test.ts b/modules/module-mongodb-storage/test/src/storage.test.ts index 3159a8de0..78c10f053 100644 --- a/modules/module-mongodb-storage/test/src/storage.test.ts +++ b/modules/module-mongodb-storage/test/src/storage.test.ts @@ -22,6 +22,9 @@ for (let storageVersion of TEST_STORAGE_VERSIONS) { describe(`Mongo Sync Bucket Storage - Checkpoints - v${storageVersion}`, () => register.registerDataStorageCheckpointTests({ ...INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion })); + describe(`Mongo Sync Bucket Storage - Bucket report - v${storageVersion}`, () => + register.registerBucketReportTests({ ...INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion })); + describe(`Mongo Sync Bucket Storage - write checkpoint metadata - v${storageVersion}`, () => { test('uses checkpoint_requested_at as the client-requested checkpoint marker', async () => { await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); diff --git a/packages/service-client/src/client.ts b/packages/service-client/src/client.ts index f047b7c6b..56eb77d1f 100644 --- a/packages/service-client/src/client.ts +++ b/packages/service-client/src/client.ts @@ -174,4 +174,39 @@ export class InstanceClient ext reprocess = this.createEndpoint({ path: '/api/admin/v1/reprocess' }); + + /** + * Per-bucket storage report for the active sync config: exact operation counts, with rows and + * fragmentation derived from each bucket's last full compact (null until a bucket has been fully + * compacted, and on storage versions without compact statistics). + * + * Example: + * ```typescript + * const report = await client.bucketReport({ limit: 50 }); + * // { + * // buckets: [ + * // { + * // bucket: 'by_user["u1"]', + * // operations: 502, + * // operation_bytes: 54149, + * // uncompacted_operations: 0, + * // rows: 3, + * // fragmentation: 167.3, + * // last_full_compact_at: '2026-08-21T11:26:19.689Z', + * // next_compact_at: null, + * // suggested_action: 'defragment' + * // } + * // ], + * // definitions: [ + * // { definition: 'by_user', bucket_count: 4, operations: 72, ... } + * // ], + * // totals: { bucket_count: 10, operations: 50, operation_bytes: 15031, estimated: false }, + * // buckets_truncated: false, + * // definitions_truncated: false + * // } + * ``` + */ + bucketReport = this.createEndpoint({ + path: '/api/admin/v1/bucket-report' + }); } diff --git a/packages/service-core-tests/src/tests/register-bucket-report-tests.ts b/packages/service-core-tests/src/tests/register-bucket-report-tests.ts new file mode 100644 index 000000000..98792c480 --- /dev/null +++ b/packages/service-core-tests/src/tests/register-bucket-report-tests.ts @@ -0,0 +1,312 @@ +import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; +import { expect, test } from 'vitest'; +import * as test_utils from '../test-utils/test-utils-index.js'; +import { compactActive } from './util.js'; + +/** + * Tests for {@link storage.SyncRulesBucketStorage.getBucketReport}: per-bucket operations, with row counts + * and fragmentation derived from each bucket's last full compact. + * + * The report reads only bucket_state, never the operation history. Operation counts are exact for every + * storage version; row-derived fields exist only on storage versions that capture full-compact statistics + * (v3), and only after a bucket's first full compact. On v1/v2 they are always null with a suggested action + * of `unknown`. + */ +export function registerBucketReportTests(config: storage.TestStorageConfig) { + const generateStorageFactory = config.factory; + const storageVersion = config.storageVersion ?? storage.CURRENT_STORAGE_VERSION; + // v3 bucket_state captures full-compact statistics (rows, fragmentation, compact scheduling). + const capturesCompactStats = storageVersion >= 3; + + const GLOBAL_SYNC_RULES = ` +bucket_definitions: + global: + data: [select * from test] +`; + + // A constant parameter query keeps op_ids stable across backends (no bucket_parameter records); the data + // query routes each row into a bucket keyed by its own `b` value, so rows land in grouped["b1"]/grouped["b2"]. + const GROUPED_SYNC_RULES = ` bucket_definitions: + grouped: + parameters: select 'b' as b + data: + - select * from test where b = bucket.b`; + + const getReport = (bucketStorage: storage.SyncRulesBucketStorage, options?: storage.GetBucketReportOptions) => { + if (bucketStorage.getBucketReport == null) { + throw new Error('Storage backend does not implement getBucketReport'); + } + return bucketStorage.getBucketReport(options); + }; + + // An explicit per-bucket compact always runs a full compact of that bucket, regardless of scheduling. + // Compact through the active sync config: the instance used by the writer retains its original + // PROCESSING stream snapshot, which some storage versions refuse to compact. + const compactBucket = (factory: storage.BucketStorageFactory, bucket: string) => + compactActive(factory, { + compactBuckets: [bucket], + clearBatchLimit: 10, + moveBatchLimit: 10, + moveBatchQueryLimit: 10, + minBucketChanges: 1, + minChangeRatio: 0 + }); + + test('reports operation counts for a single bucket', async () => { + await using factory = await generateStorageFactory(); + const { stream, content } = await test_utils.deploySyncRules( + factory, + updateSyncRulesFromYaml(GLOBAL_SYNC_RULES, { storageVersion }) + ); + const bucketStorage = factory.getInstance(stream); + + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + for (const id of ['t1', 't2', 't3']) { + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { id }, + afterReplicaId: test_utils.rid(id) + }); + } + await writer.commit('1/1'); + await writer.flush(); + + const bucket = test_utils.bucketRequest(content, 'global[]').bucket; + const report = await getReport(bucketStorage); + + expect(report.totals.bucketCount).toEqual(1); + expect(report.bucketsTruncated).toEqual(false); + expect(report.definitionsTruncated).toEqual(false); + + const stats = report.buckets.find((b) => b.bucket === bucket)!; + // Three inserts of distinct ids: three operations. The bucket has never been fully compacted, so no + // row-derived fields exist yet. + expect(stats).toMatchObject({ + operations: 3, + // Never compacted: the whole history counts as uncompacted. + uncompactedOperations: 3, + rows: null, + fragmentation: null, + lastFullCompactAt: null, + suggestedAction: 'unknown' + }); + expect(stats.operationBytes).toBeGreaterThan(0); + expect(report.totals).toMatchObject({ operations: 3, estimated: false }); + + // The definition rollup aggregates the single bucket. The definition name is the bucket-name prefix. + expect(report.definitions).toHaveLength(1); + expect(report.definitions[0]).toMatchObject({ + definition: bucket.split('[')[0], + bucketCount: 1, + operations: 3, + rows: null, + suggestedAction: 'unknown' + }); + }); + + test('derives rows and fragmentation from the last full compact', async () => { + await using factory = await generateStorageFactory(); + const { stream, content } = await test_utils.deploySyncRules( + factory, + updateSyncRulesFromYaml(GLOBAL_SYNC_RULES, { storageVersion }) + ); + const bucketStorage = factory.getInstance(stream); + + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + // Two rows, each inserted then updated twice: six operations over two live rows. + for (const id of ['t1', 't2']) { + for (const value of ['a', 'b', 'c']) { + await writer.save({ + sourceTable: testTable, + tag: value === 'a' ? storage.SaveOperationTag.INSERT : storage.SaveOperationTag.UPDATE, + after: { id, value }, + afterReplicaId: test_utils.rid(id) + }); + } + } + await writer.commit('1/1'); + await writer.flush(); + + const bucket = test_utils.bucketRequest(content, 'global[]').bucket; + + const before = await getReport(bucketStorage); + const beforeStats = before.buckets.find((b) => b.bucket === bucket)!; + // Operation counts are exact even before any compact; rows are unknown until one runs. + expect(beforeStats).toMatchObject({ + operations: 6, + uncompactedOperations: 6, + rows: null, + fragmentation: null, + suggestedAction: 'unknown' + }); + if (capturesCompactStats) { + // Writers schedule the bucket for compaction, which the report surfaces. + expect(beforeStats.nextCompactAt).toBeInstanceOf(Date); + } + + await compactBucket(factory, bucket); + + const after = await getReport(bucketStorage); + const afterStats = after.buckets.find((b) => b.bucket === bucket)!; + if (capturesCompactStats) { + // The full compact counted two live rows and shrank the history toward them. + expect(afterStats.rows).toEqual(2); + expect(afterStats.operations).toBeLessThan(beforeStats.operations); + expect(afterStats.operations).toBeGreaterThanOrEqual(2); + expect(afterStats.fragmentation).toEqual(afterStats.operations / 2); + expect(afterStats.lastFullCompactAt).toBeInstanceOf(Date); + // The compact covered the whole history, so the row stats are fully fresh. + expect(afterStats.uncompactedOperations).toEqual(0); + + // The definition rollup derives its rows from the same compact statistics. + expect(after.definitions).toHaveLength(1); + expect(after.definitions[0]).toMatchObject({ bucketCount: 1, rows: 2 }); + } else { + // v1/v2 storage does not capture compact statistics: the report stays limited to operation counts. + expect(afterStats.rows).toBeNull(); + expect(afterStats.fragmentation).toBeNull(); + expect(afterStats.suggestedAction).toEqual('unknown'); + // The compact itself still shrinks the operation history. + expect(afterStats.operations).toBeLessThan(beforeStats.operations); + } + }); + + test('reports every bucket, ranks worst-first, and totals across all buckets', async () => { + await using factory = await generateStorageFactory(); + const { stream, content } = await test_utils.deploySyncRules( + factory, + updateSyncRulesFromYaml(GROUPED_SYNC_RULES, { storageVersion }) + ); + const bucketStorage = factory.getInstance(stream); + + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + // grouped["b1"]: one row, three operations (insert + two updates). + for (const value of ['a', 'b', 'c']) { + await writer.save({ + sourceTable: testTable, + tag: value === 'a' ? storage.SaveOperationTag.INSERT : storage.SaveOperationTag.UPDATE, + after: { id: 't1', b: 'b1', value }, + afterReplicaId: test_utils.rid('t1') + }); + } + // grouped["b2"]: two rows, two operations. + for (const id of ['t2', 't3']) { + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { id, b: 'b2' }, + afterReplicaId: test_utils.rid(id) + }); + } + await writer.commit('1/1'); + await writer.flush(); + + const b1 = test_utils.bucketRequest(content, 'grouped["b1"]').bucket; + const b2 = test_utils.bucketRequest(content, 'grouped["b2"]').bucket; + + const report = await getReport(bucketStorage); + expect(report.totals.bucketCount).toEqual(2); + expect(report.totals).toMatchObject({ operations: 5, estimated: false }); + + // Ranked worst-first by operation count: b1 (3) before b2 (2). + expect(report.buckets.map((b) => b.bucket)).toEqual([b1, b2]); + expect(report.buckets.find((b) => b.bucket === b1)).toMatchObject({ operations: 3 }); + expect(report.buckets.find((b) => b.bucket === b2)).toMatchObject({ operations: 2 }); + + // Both buckets belong to one definition; the rollup sums them. + expect(report.definitions).toHaveLength(1); + expect(report.definitions[0]).toMatchObject({ + definition: b1.split('[')[0], + bucketCount: 2, + operations: 5 + }); + + // operationBytes is an aggregated ($toDouble) sum; assert every bucket is non-zero and that the + // per-bucket bytes add up to the instance total. + expect(report.totals.operationBytes).toBeGreaterThan(0); + for (const bucket of report.buckets) { + expect(bucket.operationBytes).toBeGreaterThan(0); + } + const summedBytes = report.buckets.reduce((total, bucket) => total + bucket.operationBytes, 0); + expect(summedBytes).toEqual(report.totals.operationBytes); + }); + + test('limit truncates the bucket list but totals still span all buckets', async () => { + await using factory = await generateStorageFactory(); + const { stream, content } = await test_utils.deploySyncRules( + factory, + updateSyncRulesFromYaml(GROUPED_SYNC_RULES, { storageVersion }) + ); + const bucketStorage = factory.getInstance(stream); + + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + // grouped["b1"]: two operations; grouped["b2"]: one operation. + for (const value of ['a', 'b']) { + await writer.save({ + sourceTable: testTable, + tag: value === 'a' ? storage.SaveOperationTag.INSERT : storage.SaveOperationTag.UPDATE, + after: { id: 't1', b: 'b1', value }, + afterReplicaId: test_utils.rid('t1') + }); + } + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { id: 't2', b: 'b2' }, + afterReplicaId: test_utils.rid('t2') + }); + await writer.commit('1/1'); + await writer.flush(); + + const b1 = test_utils.bucketRequest(content, 'grouped["b1"]').bucket; + + const report = await getReport(bucketStorage, { limit: 1 }); + expect(report.bucketsTruncated).toEqual(true); + expect(report.buckets.map((b) => b.bucket)).toEqual([b1]); + // Totals still cover every bucket, not just the truncated list. + expect(report.totals.bucketCount).toEqual(2); + expect(report.totals).toMatchObject({ operations: 3, estimated: false }); + }); + + test('caps the definition rollup and flags the truncation', async () => { + // Two definitions past the rollup cap; a single row lands in every definition's global bucket. + const definitionCount = storage.BUCKET_REPORT_DEFINITION_LIMIT + 2; + const manyDefinitions = + 'bucket_definitions:\n' + + Array.from({ length: definitionCount }, (_, i) => ` def${i}:\n data: [select * from test]\n`).join(''); + + await using factory = await generateStorageFactory(); + const { stream } = await test_utils.deploySyncRules( + factory, + updateSyncRulesFromYaml(manyDefinitions, { storageVersion }) + ); + const bucketStorage = factory.getInstance(stream); + + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); + await writer.markAllSnapshotDone('1/1'); + await writer.save({ + sourceTable: testTable, + tag: storage.SaveOperationTag.INSERT, + after: { id: 't1' }, + afterReplicaId: test_utils.rid('t1') + }); + await writer.commit('1/1'); + await writer.flush(); + + const report = await getReport(bucketStorage); + expect(report.totals.bucketCount).toEqual(definitionCount); + expect(report.bucketsTruncated).toEqual(false); + expect(report.definitions).toHaveLength(storage.BUCKET_REPORT_DEFINITION_LIMIT); + expect(report.definitionsTruncated).toEqual(true); + }); +} diff --git a/packages/service-core-tests/src/tests/tests-index.ts b/packages/service-core-tests/src/tests/tests-index.ts index a40468a32..5be5e48f1 100644 --- a/packages/service-core-tests/src/tests/tests-index.ts +++ b/packages/service-core-tests/src/tests/tests-index.ts @@ -1,3 +1,4 @@ +export * from './register-bucket-report-tests.js'; export * from './register-bucket-validation-tests.js'; export * from './register-compacting-tests.js'; export * from './register-data-storage-checkpoint-tests.js'; diff --git a/packages/service-core/src/routes/endpoints/admin.ts b/packages/service-core/src/routes/endpoints/admin.ts index c3403f578..e5d710315 100644 --- a/packages/service-core/src/routes/endpoints/admin.ts +++ b/packages/service-core/src/routes/endpoints/admin.ts @@ -270,4 +270,78 @@ export const validate = routeDefinition({ } }); -export const ADMIN_ROUTES = [executeSql, diagnostics, getSchema, reprocess, validate]; +/** + * Per-bucket report of total operations vs live rows in storage, for the active sync config. + * + * Answers the recurring "why is my Data Synced so high" question. A high `operations / rows` ratio + * indicates fragmented buckets that a compact or defragment can reclaim. Row counts derive from each + * bucket's last full compact (bucket_state only, no operation-history scan), so they are null for buckets + * that have never been fully compacted and on storage versions without compact statistics (v1/v2). + */ +export const bucketReport = routeDefinition({ + path: '/api/admin/v1/bucket-report', + method: router.HTTPMethod.POST, + authorize: authApi, + validator: schema.createTsCodecValidator(internal_routes.BucketReportRequest, { allowAdditional: true }), + handler: async (payload) => { + const { + context: { service_context } + } = payload; + const { + storageEngine: { activeBucketStorage } + } = service_context; + + const active = await activeBucketStorage.getActiveSyncConfig(); + if (active == null) { + throw new errors.ServiceError({ + status: 422, + code: ErrorCode.PSYNC_S4104, + description: 'No active sync config' + }); + } + + if (active.storage.getBucketReport == null) { + throw new errors.ServiceError({ + status: 422, + code: ErrorCode.PSYNC_S2001, + description: 'The configured storage provider does not support bucket reporting' + }); + } + + const report = await active.storage.getBucketReport({ limit: payload.params.limit }); + + return internal_routes.BucketReportResponse.encode({ + buckets: report.buckets.map((bucket) => ({ + bucket: bucket.bucket, + operations: bucket.operations, + operation_bytes: bucket.operationBytes, + uncompacted_operations: bucket.uncompactedOperations, + rows: bucket.rows, + fragmentation: bucket.fragmentation, + last_full_compact_at: bucket.lastFullCompactAt?.toISOString() ?? null, + next_compact_at: bucket.nextCompactAt?.toISOString() ?? null, + suggested_action: bucket.suggestedAction + })), + definitions: report.definitions.map((definition) => ({ + definition: definition.definition, + bucket_count: definition.bucketCount, + operations: definition.operations, + operation_bytes: definition.operationBytes, + uncompacted_operations: definition.uncompactedOperations, + rows: definition.rows, + fragmentation: definition.fragmentation, + suggested_action: definition.suggestedAction + })), + totals: { + bucket_count: report.totals.bucketCount, + operations: report.totals.operations, + operation_bytes: report.totals.operationBytes, + estimated: report.totals.estimated + }, + buckets_truncated: report.bucketsTruncated, + definitions_truncated: report.definitionsTruncated + }); + } +}); + +export const ADMIN_ROUTES = [executeSql, diagnostics, getSchema, reprocess, validate, bucketReport]; diff --git a/packages/service-core/src/storage/SyncRulesBucketStorage.ts b/packages/service-core/src/storage/SyncRulesBucketStorage.ts index fb0ab9ebe..a45689bb1 100644 --- a/packages/service-core/src/storage/SyncRulesBucketStorage.ts +++ b/packages/service-core/src/storage/SyncRulesBucketStorage.ts @@ -8,6 +8,7 @@ import { import * as bson from 'bson'; import { PerformanceTracer } from '../tracing/PerformanceTracer.js'; import * as util from '../util/util-index.js'; +import { BucketReport, GetBucketReportOptions } from './bucket-report.js'; import { BucketStorageBatch, FlushedResult, SaveUpdate } from './BucketStorageBatch.js'; import { BucketStorageFactory } from './BucketStorageFactory.js'; import { ParsedSyncConfigSet } from './ParsedSyncConfigSet.js'; @@ -183,6 +184,17 @@ export interface SyncRulesBucketStorage * inside this replication stream. */ cleanupStoppedSyncConfigs?(options: CleanupStoppedSyncConfigsOptions): Promise; + + /** + * Per-bucket report of total operations vs total live rows in storage. + * + * Intended for an on-demand admin/diagnostics view (e.g. answering "why is my Data Synced so high"), + * not as a live gauge. How the counts are derived is backend-specific, and the report may be relatively + * expensive on large instances. + * + * Optional: storage providers that don't implement it are reported as unsupported by the API. + */ + getBucketReport?(options?: GetBucketReportOptions): Promise; } export interface SyncRulesBucketStorageListener { diff --git a/packages/service-core/src/storage/bucket-report.ts b/packages/service-core/src/storage/bucket-report.ts new file mode 100644 index 000000000..ed9384971 --- /dev/null +++ b/packages/service-core/src/storage/bucket-report.ts @@ -0,0 +1,330 @@ +/** + * Per-bucket storage report for an active sync config. + * + * - An **operation** is any entry in a bucket's append-only history (`PUT`, `REMOVE`, `MOVE`, `CLEAR`). + * - A **row** is a distinct live object currently in the bucket. + * + * A new client downloads every operation, not just live rows, so `operations / rows` is effectively a + * fragmentation / compaction-efficiency score: a fully compacted bucket trends towards ~1, while a high + * ratio is the usual cause of an unexpectedly high "Data Synced" metric and is reclaimable via compact/defragment. + * + * Scaling note: the report reads only the pre-aggregated per-bucket state (`bucket_state`), never the + * operation history itself. Operation counts are exact; row counts come from the statistics captured by the + * last full compact of each bucket, so they are a snapshot as of that compact rather than a live counter. + * Storage versions that do not capture compact statistics (v1/v2) report operations only, with row-derived + * fields null and the suggested action `unknown`. + */ +import { ErrorCode, ServiceError } from '@powersync/lib-services-framework'; + +/** Number of worst-offender buckets returned when the request omits a `limit`. */ +export const DEFAULT_BUCKET_REPORT_LIMIT: number = 50; + +/** Highest `limit` a request may ask for, bounding the response size. */ +export const MAX_BUCKET_REPORT_LIMIT: number = 1_000; + +/** + * Maximum number of bucket definitions in the report's definition rollup. Configs rarely approach this many + * definitions. + */ +export const BUCKET_REPORT_DEFINITION_LIMIT: number = 20; + +/** Fragmentation below this is considered healthy: no maintenance action is suggested. */ +export const BUCKET_ACTION_MIN_FRAGMENTATION: number = 3; + +/** + * When at least this share of a bucket's operations is compaction residue (MOVE/CLEAR, no row identity), + * compaction has already done its work and only a defragment reduces what new clients download. + */ +export const BUCKET_ACTION_RESIDUE_SHARE: number = 0.5; + +/** + * When at least this share of a bucket's row-bearing operations (PUT/REMOVE) is superseded history (more + * operations than rows), a compact reclaims it (as MOVE conversions and a CLEAR prefix). + */ +export const BUCKET_ACTION_SUPERSEDED_SHARE: number = 0.5; + +/** + * Suggested maintenance action for a bucket or definition. `unknown` means the storage has no compact + * statistics to derive one from (v1/v2 storage, or a bucket that has never been fully compacted). + * See {@link suggestBucketAction}. + */ +export type BucketAction = 'none' | 'compact' | 'defragment' | 'both' | 'unknown'; + +export interface BucketStorageStats { + /** Full bucket name, e.g. `by_user["u1"]`. */ + bucket: string; + /** Total operations in the bucket's history. Exact and current. */ + operations: number; + /** Approximate size of the operation history in bytes. */ + operationBytes: number; + /** + * Operations written after the last full compact — the staleness indicator for `rows` and + * `fragmentation`, which are snapshots from that compact. Equals `operations` when the bucket has + * never been fully compacted (the whole history is uncompacted). + */ + uncompactedOperations: number; + /** + * Live rows in the bucket as of the last full compact, or null if the bucket has never been fully + * compacted (or the storage version does not capture compact statistics). + */ + rows: number | null; + /** + * `operations / max(rows, 1)`. ~1 is healthy (fully compacted); higher means more operation-history + * overhead that a compact/defragment can reclaim. Null whenever `rows` is null. + */ + fragmentation: number | null; + /** When the bucket was last fully compacted, which is when `rows` was captured. */ + lastFullCompactAt: Date | null; + /** + * When the scheduled compactor will next consider this bucket, if scheduling data exists. A suggested + * compact with a future `nextCompactAt` means the compact is already planned but throttled until then. + */ + nextCompactAt: Date | null; + /** Suggested maintenance action derived from the compact statistics. See {@link suggestBucketAction}. */ + suggestedAction: BucketAction; +} + +/** Aggregated stats for one bucket definition (one `bucket_definitions` entry in the sync config). */ +export interface BucketDefinitionStats { + /** Definition name as it prefixes bucket names, e.g. `1#by_user` (versioned in storage v2 and later). */ + definition: string; + /** Number of buckets in this definition with stored operations. */ + bucketCount: number; + /** Total operations across the definition's buckets. Exact and current. */ + operations: number; + /** Approximate size of the definition's operation history in bytes. */ + operationBytes: number; + /** + * Operations not covered by any bucket's last full compact, i.e. written since (or in buckets never + * fully compacted). The staleness indicator for `rows` and `fragmentation`. + */ + uncompactedOperations: number; + /** + * Live rows across the definition's buckets, counting a row once per bucket that contains it (the + * download-relevant meaning). Derived from each bucket's last full compact; when only some buckets have + * been fully compacted the count is extrapolated from those, and it is null when none have. + */ + rows: number | null; + /** `operations / max(rows, 1)` across the whole definition. Null whenever `rows` is null. */ + fragmentation: number | null; + /** Suggested maintenance action derived from the compact statistics. See {@link suggestBucketAction}. */ + suggestedAction: BucketAction; +} + +export interface BucketReportTotals { + /** Number of buckets with stored operations. Exact, even when the rest of the totals are estimates. */ + bucketCount: number; + /** Sum of operations across all buckets. Estimated when the bucket set was sampled. */ + operations: number; + /** Sum of operation-history bytes across all buckets. Estimated when the bucket set was sampled. */ + operationBytes: number; + /** + * True if the totals are estimated because the bucket set was too large to scan in full and was sampled. + * Row counts are never totalled here. + */ + estimated: boolean; +} + +export interface BucketReport { + /** Worst-offender buckets, ranked by operation count then fragmentation. */ + buckets: BucketStorageStats[]; + /** + * Per-definition rollup, ranked by operation count. Answers "which sync-rules definition should I look + * at" where `buckets` answers "which exact buckets". Capped at {@link BUCKET_REPORT_DEFINITION_LIMIT}. + */ + definitions: BucketDefinitionStats[]; + /** Instance-wide operation totals. Does not include row counts. */ + totals: BucketReportTotals; + /** True if there are more buckets than returned (more than `limit`). */ + bucketsTruncated: boolean; + /** True if the definition rollup is incomplete (more definitions exist than the rollup cap). */ + definitionsTruncated: boolean; +} + +export interface GetBucketReportOptions { + /** + * Maximum number of buckets to return, ranked by operation count descending (worst offenders first). + * Must be an integer between 1 and {@link MAX_BUCKET_REPORT_LIMIT}; anything else is rejected with a + * validation error. Defaults to {@link DEFAULT_BUCKET_REPORT_LIMIT}. + */ + limit?: number; +} + +/** + * A bucket's exact operation stats plus the statistics captured by its last full compact, before ranking. + * The compact fields are null/absent when the bucket has never been fully compacted or the storage version + * does not capture them (v1/v2). + */ +export interface RankedBucketInput { + bucket: string; + operations: number; + operationBytes: number; + /** Operations in the prefix covered by the last full compact. */ + compactedOperations?: number | null; + /** + * PUT operations in that compacted prefix. After a full compact each PUT is generally a unique row, so + * this doubles as the bucket's live row count as of the compact. + */ + compactedPuts?: number | null; + /** When the last full compact ran. */ + lastFullCompactAt?: Date | null; + /** When the scheduled compactor will next consider this bucket. */ + nextCompactAt?: Date | null; +} + +/** A definition's aggregated operation stats plus its buckets' summed compact statistics, before ranking. */ +export interface RankedDefinitionInput { + definition: string; + bucketCount: number; + operations: number; + operationBytes: number; + /** Number of the definition's buckets that have full-compact statistics. */ + compactedBucketCount?: number; + /** Sum of {@link RankedBucketInput.compactedOperations} across those buckets. */ + compactedOperations?: number; + /** Sum of {@link RankedBucketInput.compactedPuts} across those buckets. */ + compactedPuts?: number; +} + +/** + * Resolve the requested limit, falling back to {@link DEFAULT_BUCKET_REPORT_LIMIT}. Invalid values are + * rejected rather than clamped, so a caller asking for more than the maximum fails loudly instead of + * silently getting fewer buckets than requested. + */ +export function resolveBucketReportLimit(limit?: number): number { + if (limit == null) { + return DEFAULT_BUCKET_REPORT_LIMIT; + } + if (!Number.isInteger(limit) || limit < 1 || limit > MAX_BUCKET_REPORT_LIMIT) { + throw new ServiceError({ + status: 400, + code: ErrorCode.PSYNC_S2001, + description: `limit must be an integer between 1 and ${MAX_BUCKET_REPORT_LIMIT}` + }); + } + return limit; +} + +/** + * Suggest the maintenance action that reduces what new clients download from a bucket, based on its + * operation mix. Grounded in the compaction semantics (see `docs/storage/compacting-operations.md`): + * + * - A **compact** converts superseded PUT/REMOVE operations into MOVE operations (reclaiming their bytes) + * and collapses a leading run of REMOVE/MOVE operations into one CLEAR. It helps when a bucket carries + * un-compacted superseded history: `rowOperations` well above `rows`. + * - A **defragment** (touch every row, then compact) is what collapses the operation count once the history + * is mostly MOVE/CLEAR residue that a compact alone preserves: `operations` well above `rowOperations`. + * - When both kinds of overhead are present, or the mix is inconclusive, suggest both. + * + * The thresholds are heuristics; the report is intended to be re-run after acting on it. Inputs derive from + * the last full compact's statistics, which is fine at these margins. + */ +export function suggestBucketAction(operations: number, rowOperations: number, rows: number): BucketAction { + const fragmentation = operations / Math.max(rows, 1); + if (fragmentation < BUCKET_ACTION_MIN_FRAGMENTATION) { + return 'none'; + } + const residueShare = (operations - rowOperations) / Math.max(operations, 1); + const supersededShare = (rowOperations - rows) / Math.max(rowOperations, 1); + const defragmentNeeded = residueShare >= BUCKET_ACTION_RESIDUE_SHARE; + const compactUseful = supersededShare >= BUCKET_ACTION_SUPERSEDED_SHARE; + if (defragmentNeeded && compactUseful) { + return 'both'; + } + if (defragmentNeeded) { + return 'defragment'; + } + if (compactUseful) { + return 'compact'; + } + // Fragmented, but neither share dominates: a mixed history where a compact reclaims part and the rest + // needs a defragment. + return 'both'; +} + +/** + * Derive rows / fragmentation / suggested action from full-compact statistics. + * + * The compacted prefix holds `compactedPuts` row-bearing operations (one per live row) and + * `compactedOperations - compactedPuts` residue operations (MOVE/CLEAR) that only a defragment reclaims. + * Operations written after the compact are raw PUT/REMOVE history, so the total row-bearing count is + * `operations - residue`. Without compact statistics nothing row-related can be derived. + */ +function deriveRowStats( + operations: number, + compactedOperations: number | null | undefined, + compactedPuts: number | null | undefined +): Pick { + if (compactedOperations == null || compactedPuts == null) { + return { rows: null, fragmentation: null, suggestedAction: 'unknown' }; + } + const rows = compactedPuts; + const residue = Math.max(0, compactedOperations - compactedPuts); + const rowOperations = Math.max(0, operations - residue); + return { + rows, + fragmentation: operations / Math.max(rows, 1), + suggestedAction: suggestBucketAction(operations, rowOperations, rows) + }; +} + +/** + * Assemble the final {@link BucketReport} from per-bucket stats, per-definition stats, and instance-wide + * totals. Storage adapters select the buckets however is cheapest for them; this owns the shared + * fragmentation / ranking / truncation / action logic so it cannot drift. Pure (no I/O) so it is + * unit-testable. + * + * Bucket truncation is derived from the totals; only the adapter knows whether the definition list was cut, + * so it passes `definitionsTruncated` in. + */ +export function assembleBucketReport( + buckets: RankedBucketInput[], + definitions: RankedDefinitionInput[], + totals: BucketReportTotals, + definitionsTruncated = false +): BucketReport { + const stats: BucketStorageStats[] = buckets.map((b) => ({ + bucket: b.bucket, + operations: b.operations, + operationBytes: b.operationBytes, + uncompactedOperations: Math.max(0, b.operations - (b.compactedOperations ?? 0)), + lastFullCompactAt: b.lastFullCompactAt ?? null, + nextCompactAt: b.nextCompactAt ?? null, + ...deriveRowStats(b.operations, b.compactedOperations, b.compactedPuts) + })); + + const definitionStats: BucketDefinitionStats[] = definitions.map((d) => { + // When only some of the definition's buckets have full-compact statistics, extrapolate from those + // buckets to the whole definition, assuming the compacted subset is representative. + const compactedBucketCount = d.compactedBucketCount ?? 0; + const scale = compactedBucketCount > 0 ? d.bucketCount / compactedBucketCount : 0; + const derived = + compactedBucketCount > 0 && d.compactedOperations != null && d.compactedPuts != null + ? deriveRowStats(d.operations, Math.round(d.compactedOperations * scale), Math.round(d.compactedPuts * scale)) + : deriveRowStats(d.operations, null, null); + return { + definition: d.definition, + bucketCount: d.bucketCount, + operations: d.operations, + operationBytes: d.operationBytes, + // Unlike the row derivation above, this uses the plain (non-extrapolated) compacted sum: it counts + // the operations no full compact has covered, which is exact rather than an estimate. + uncompactedOperations: Math.max(0, d.operations - (d.compactedOperations ?? 0)), + ...derived + }; + }); + + // Worst-first: most operations, then most fragmented. + const worstFirst = (a: { operations: number; fragmentation: number | null }, b: typeof a) => + b.operations - a.operations || (b.fragmentation ?? 0) - (a.fragmentation ?? 0); + stats.sort(worstFirst); + definitionStats.sort(worstFirst); + + return { + buckets: stats, + definitions: definitionStats, + totals, + bucketsTruncated: totals.bucketCount > stats.length, + definitionsTruncated + }; +} diff --git a/packages/service-core/src/storage/storage-index.ts b/packages/service-core/src/storage/storage-index.ts index e057ce8e7..0f37fcb3e 100644 --- a/packages/service-core/src/storage/storage-index.ts +++ b/packages/service-core/src/storage/storage-index.ts @@ -1,4 +1,5 @@ export * from './bson.js'; +export * from './bucket-report.js'; export * from './BucketStorage.js'; export * from './BucketStorageBatch.js'; export * from './BucketStorageFactory.js'; diff --git a/packages/service-core/test/src/bucket-report.test.ts b/packages/service-core/test/src/bucket-report.test.ts new file mode 100644 index 000000000..fe3d7444c --- /dev/null +++ b/packages/service-core/test/src/bucket-report.test.ts @@ -0,0 +1,294 @@ +import { + assembleBucketReport, + BucketReportTotals, + DEFAULT_BUCKET_REPORT_LIMIT, + MAX_BUCKET_REPORT_LIMIT, + RankedBucketInput, + RankedDefinitionInput, + resolveBucketReportLimit, + suggestBucketAction +} from '@/storage/bucket-report.js'; +import { describe, expect, it } from 'vitest'; + +const bucket = (name: string, operations: number, extra?: Partial): RankedBucketInput => ({ + bucket: name, + operations, + operationBytes: 0, + ...extra +}); + +/** Full-compact statistics where the compacted prefix is the whole history (no writes since). */ +const compacted = (operations: number, puts: number): Partial => ({ + compactedOperations: operations, + compactedPuts: puts +}); + +const definition = ( + name: string, + operations: number, + extra?: Partial +): RankedDefinitionInput => ({ + definition: name, + bucketCount: 1, + operations, + operationBytes: 0, + ...extra +}); + +const totals = (bucketCount: number, extra?: Partial): BucketReportTotals => ({ + bucketCount, + operations: extra?.operations ?? 0, + operationBytes: extra?.operationBytes ?? 0, + estimated: extra?.estimated ?? false +}); + +describe('assembleBucketReport', () => { + it('derives rows and fragmentation from the full-compact statistics', () => { + const compactedAt = new Date('2026-01-01T00:00:00Z'); + const nextCompact = new Date('2026-01-02T00:00:00Z'); + const report = assembleBucketReport( + [ + bucket('global[]', 100, { + operationBytes: 1024, + ...compacted(100, 10), + lastFullCompactAt: compactedAt, + nextCompactAt: nextCompact + }), + bucket('by_user["u1"]', 30, compacted(30, 30)) + ], + [], + totals(2) + ); + + expect(report.buckets.find((b) => b.bucket === 'global[]')).toMatchObject({ + operations: 100, + operationBytes: 1024, + rows: 10, + fragmentation: 10, + lastFullCompactAt: compactedAt, + nextCompactAt: nextCompact + }); + expect(report.buckets.find((b) => b.bucket === 'by_user["u1"]')).toMatchObject({ + rows: 30, + fragmentation: 1, + lastFullCompactAt: null, + nextCompactAt: null + }); + }); + + it('reports null rows and an unknown action without full-compact statistics', () => { + const report = assembleBucketReport([bucket('global[]', 100)], [], totals(1)); + + expect(report.buckets[0]).toMatchObject({ + operations: 100, + rows: null, + fragmentation: null, + suggestedAction: 'unknown', + // Never compacted: the whole history is uncompacted. + uncompactedOperations: 100 + }); + }); + + it('reports the operations written since the last full compact', () => { + const report = assembleBucketReport( + [ + // Fully covered by the compact: nothing uncompacted. + bucket('fresh[]', 100, compacted(100, 100)), + // 900 operations landed after the compact: rows/fragmentation are that much out of date. + bucket('stale[]', 1000, compacted(100, 100)) + ], + [ + // Definition grain uses the plain compacted sum (not the extrapolated one used for rows): + // 1000 total ops, 50 covered by compacts of half the buckets, so 950 are uncompacted. + definition('1#partial', 1000, { + bucketCount: 10, + compactedBucketCount: 5, + compactedOperations: 50, + compactedPuts: 50 + }) + ], + totals(12) + ); + + const by = (name: string) => report.buckets.find((b) => b.bucket === name)!; + expect(by('fresh[]').uncompactedOperations).toEqual(0); + expect(by('stale[]').uncompactedOperations).toEqual(900); + expect(report.definitions[0].uncompactedOperations).toEqual(950); + }); + + it('ranks buckets worst-first by operations then fragmentation', () => { + const report = assembleBucketReport( + [bucket('a[]', 5, compacted(5, 5)), bucket('b[]', 50, compacted(50, 5)), bucket('c[]', 50, compacted(50, 50))], + [], + totals(3) + ); + + // b and c both have 50 ops; b is more fragmented (10 vs 1) so it ranks first. + expect(report.buckets.map((b) => b.bucket)).toEqual(['b[]', 'c[]', 'a[]']); + }); + + it('floors rows at 1 so a bucket with operations but no rows is fully fragmented', () => { + const report = assembleBucketReport([bucket('gone[]', 42, compacted(42, 0))], [], totals(1)); + + expect(report.buckets[0]).toMatchObject({ operations: 42, rows: 0, fragmentation: 42 }); + }); + + it('marks the bucket list truncated when there are more buckets than returned', () => { + const truncated = assembleBucketReport([bucket('a[]', 10), bucket('b[]', 5)], [], totals(5)); + expect(truncated.bucketsTruncated).toBe(true); + + const complete = assembleBucketReport([bucket('a[]', 10), bucket('b[]', 5)], [], totals(2)); + expect(complete.bucketsTruncated).toBe(false); + }); + + it('passes the definition truncation flag through, defaulting to complete', () => { + expect(assembleBucketReport([], [definition('a', 1)], totals(1)).definitionsTruncated).toBe(false); + expect(assembleBucketReport([], [definition('a', 1)], totals(1), true).definitionsTruncated).toBe(true); + }); + + it('carries the totals through unchanged', () => { + const t = totals(2, { operations: 120, operationBytes: 15, estimated: true }); + const report = assembleBucketReport([bucket('a[]', 100), bucket('b[]', 20)], [], t); + + expect(report.totals).toEqual({ bucketCount: 2, operations: 120, operationBytes: 15, estimated: true }); + }); + + it('assembles and ranks the definition rollup with derived fragmentation and action', () => { + const report = assembleBucketReport( + [], + [ + definition('1#by_user', 100, { + bucketCount: 10, + compactedBucketCount: 10, + compactedOperations: 100, + compactedPuts: 100 + }), + definition('1#by_org', 500, { + bucketCount: 5, + operationBytes: 2048, + compactedBucketCount: 5, + compactedOperations: 500, + compactedPuts: 100 + }) + ], + totals(15) + ); + + // Ranked by operations: by_org (500) before by_user (100). + expect(report.definitions.map((d) => d.definition)).toEqual(['1#by_org', '1#by_user']); + expect(report.definitions[0]).toMatchObject({ + definition: '1#by_org', + bucketCount: 5, + operations: 500, + operationBytes: 2048, + rows: 100, + fragmentation: 5, + suggestedAction: 'defragment' + }); + expect(report.definitions[1]).toMatchObject({ fragmentation: 1, suggestedAction: 'none' }); + }); + + it('extrapolates definition rows when only some buckets have been fully compacted', () => { + const report = assembleBucketReport( + [], + [ + // 5 of 10 buckets compacted, holding 50 rows between them: assume the other half looks the same. + definition('1#partial', 1000, { + bucketCount: 10, + compactedBucketCount: 5, + compactedOperations: 50, + compactedPuts: 50 + }), + // No compacted buckets: nothing row-related can be derived. + definition('1#uncompacted', 1000, { bucketCount: 10, compactedBucketCount: 0 }) + ], + totals(20) + ); + + expect(report.definitions.find((d) => d.definition === '1#partial')).toMatchObject({ + rows: 100, + fragmentation: 10 + }); + expect(report.definitions.find((d) => d.definition === '1#uncompacted')).toMatchObject({ + rows: null, + fragmentation: null, + suggestedAction: 'unknown' + }); + }); + + it('derives per-bucket suggested actions from the compact statistics', () => { + const report = assembleBucketReport( + [ + // Healthy: one op per row. + bucket('healthy[]', 100, compacted(100, 100)), + // Un-compacted churn since the last full compact: 900 raw ops on top of a clean 100-row prefix. + bucket('churned[]', 1000, compacted(100, 100)), + // Compacted residue: the compact kept 150 rows but left 850 MOVE/CLEAR ops behind. + bucket('residue[]', 1000, compacted(1000, 150)) + ], + [], + totals(3) + ); + + const action = (name: string) => report.buckets.find((b) => b.bucket === name)?.suggestedAction; + expect(action('healthy[]')).toEqual('none'); + expect(action('churned[]')).toEqual('compact'); + expect(action('residue[]')).toEqual('defragment'); + }); +}); + +describe('suggestBucketAction', () => { + it('suggests nothing for buckets under 3x fragmentation', () => { + expect(suggestBucketAction(100, 100, 100)).toEqual('none'); + expect(suggestBucketAction(250, 250, 100)).toEqual('none'); + expect(suggestBucketAction(0, 0, 0)).toEqual('none'); + }); + + it('suggests compact for un-compacted superseded history', () => { + // All operations carry row identity, but there are 10x more of them than rows. + expect(suggestBucketAction(1000, 1000, 100)).toEqual('compact'); + // 3x fragmentation is the threshold where actions start. + expect(suggestBucketAction(300, 300, 100)).toEqual('compact'); + }); + + it('suggests defragment when compaction residue dominates', () => { + // 850 of 1000 ops are MOVE/CLEAR: a compact already ran and cannot reclaim more. + expect(suggestBucketAction(1000, 150, 100)).toEqual('defragment'); + // A bucket of only MOVE/CLEAR ops (rows 0) is pure residue. + expect(suggestBucketAction(500, 0, 0)).toEqual('defragment'); + }); + + it('suggests both when residue and fresh superseded history are both present', () => { + // 600 residue ops plus 400 row-bearing ops over 100 rows: defragment for the residue, compact for the churn. + expect(suggestBucketAction(1000, 400, 100)).toEqual('both'); + }); + + it('suggests both for a fragmented but inconclusive mix', () => { + // Fragmented (frag 3), yet neither residue (40%) nor superseded share (44%) dominates. + expect(suggestBucketAction(1200, 720, 400)).toEqual('both'); + }); +}); + +describe('resolveBucketReportLimit', () => { + it('defaults when no limit is given', () => { + expect(resolveBucketReportLimit(undefined)).toBe(DEFAULT_BUCKET_REPORT_LIMIT); + }); + + it('accepts integers up to the maximum', () => { + expect(resolveBucketReportLimit(1)).toBe(1); + expect(resolveBucketReportLimit(20)).toBe(20); + expect(resolveBucketReportLimit(MAX_BUCKET_REPORT_LIMIT)).toBe(MAX_BUCKET_REPORT_LIMIT); + }); + + it('rejects invalid limits instead of clamping them', () => { + for (const invalid of [0, -5, 2.7, MAX_BUCKET_REPORT_LIMIT + 1]) { + let error: any; + try { + resolveBucketReportLimit(invalid); + } catch (e) { + error = e; + } + expect(error?.errorData, `limit ${invalid}`).toMatchObject({ status: 400 }); + } + }); +}); diff --git a/packages/service-core/test/src/routes/admin.test.ts b/packages/service-core/test/src/routes/admin.test.ts index 8703ec380..5ea70fe9e 100644 --- a/packages/service-core/test/src/routes/admin.test.ts +++ b/packages/service-core/test/src/routes/admin.test.ts @@ -2,7 +2,7 @@ import { BasicRouterRequest, Context, JwtPayload, ParsedSyncConfigSet, storage } import { logger } from '@powersync/lib-services-framework'; import { SqlSyncRules } from '@powersync/service-sync-rules'; import { describe, expect, it, vi } from 'vitest'; -import { diagnostics, reprocess, validate } from '../../../src/routes/endpoints/admin.js'; +import { bucketReport, diagnostics, reprocess, validate } from '../../../src/routes/endpoints/admin.js'; import { mockServiceContext } from './mocks.js'; describe('admin routes', () => { @@ -215,4 +215,128 @@ bucket_definitions: expect(activeBucketStorage.updateSyncRules).not.toHaveBeenCalled(); }); }); + + describe('bucket-report', () => { + const report = { + buckets: [ + { + bucket: '1#by_user["u1"]', + operations: 4750, + operationBytes: 1216000, + uncompactedOperations: 250, + rows: 95, + fragmentation: 50, + lastFullCompactAt: new Date('2026-08-01T00:00:00.000Z'), + nextCompactAt: new Date('2026-08-21T12:00:00.000Z'), + suggestedAction: 'compact' + }, + { + bucket: '1#global[]', + operations: 1000, + operationBytes: 3145728, + uncompactedOperations: 1000, + rows: null, + fragmentation: null, + lastFullCompactAt: null, + nextCompactAt: null, + suggestedAction: 'unknown' + } + ], + definitions: [ + { + definition: '1#by_user', + bucketCount: 1, + operations: 4750, + operationBytes: 1216000, + uncompactedOperations: 250, + rows: 95, + fragmentation: 50, + suggestedAction: 'compact' + } + ], + totals: { bucketCount: 2, operations: 5750, operationBytes: 4361728, estimated: false }, + bucketsTruncated: false, + definitionsTruncated: true + }; + + it('returns the report, forwards the limit, and maps fields to snake_case', async () => { + const getBucketReport = vi.fn(async () => report); + const activeBucketStorage = { + getActiveSyncConfig: vi.fn(async () => ({ + content: makeSyncConfigContent({}), + replicationStream: {}, + storage: { getBucketReport } + })) + }; + + const response = await bucketReport.handler({ + context: makeContext(activeBucketStorage), + params: { limit: 20 }, + request + }); + + expect(getBucketReport).toHaveBeenCalledWith({ limit: 20 }); + expect(response.buckets[0]).toEqual({ + bucket: '1#by_user["u1"]', + operations: 4750, + operation_bytes: 1216000, + uncompacted_operations: 250, + rows: 95, + fragmentation: 50, + last_full_compact_at: '2026-08-01T00:00:00.000Z', + next_compact_at: '2026-08-21T12:00:00.000Z', + suggested_action: 'compact' + }); + // Dates and row stats are null for buckets without full-compact statistics. + expect(response.buckets[1]).toMatchObject({ + rows: null, + fragmentation: null, + last_full_compact_at: null, + next_compact_at: null, + suggested_action: 'unknown' + }); + expect(response.definitions).toEqual([ + { + definition: '1#by_user', + bucket_count: 1, + operations: 4750, + operation_bytes: 1216000, + uncompacted_operations: 250, + rows: 95, + fragmentation: 50, + suggested_action: 'compact' + } + ]); + expect(response.totals).toEqual({ + bucket_count: 2, + operations: 5750, + operation_bytes: 4361728, + estimated: false + }); + expect(response.buckets_truncated).toBe(false); + expect(response.definitions_truncated).toBe(true); + }); + + it('rejects when there is no active sync config', async () => { + const activeBucketStorage = { getActiveSyncConfig: vi.fn(async () => null) }; + + await expect( + bucketReport.handler({ context: makeContext(activeBucketStorage), params: {}, request }) + ).rejects.toMatchObject({ errorData: { status: 422, code: 'PSYNC_S4104' } }); + }); + + it('rejects when the storage provider does not support bucket reporting', async () => { + const activeBucketStorage = { + getActiveSyncConfig: vi.fn(async () => ({ + content: makeSyncConfigContent({}), + replicationStream: {}, + storage: {} + })) + }; + + await expect( + bucketReport.handler({ context: makeContext(activeBucketStorage), params: {}, request }) + ).rejects.toMatchObject({ errorData: { status: 422, code: 'PSYNC_S2001' } }); + }); + }); }); diff --git a/packages/types/src/routes.ts b/packages/types/src/routes.ts index 0448bef8b..eaa3da127 100644 --- a/packages/types/src/routes.ts +++ b/packages/types/src/routes.ts @@ -1,5 +1,5 @@ import * as t from 'ts-codec'; -import { anyPrimitive } from './codecs.js'; +import { anyPrimitive, enumLiteral, orNull } from './codecs.js'; import { ConnectionStatus, InstanceSchema, SyncRulesStatus } from './definitions.js'; export const GetSchemaRequest = t.object({}); @@ -78,3 +78,107 @@ export type ValidateRequest = t.Encoded; export const ValidateResponse = SyncRulesStatus; export type ValidateResponse = t.Encoded; + +export const BucketReportRequest = t.object({ + /** + * Maximum number of buckets to return, ranked by operation count descending (worst offenders first). + * Defaults to 50 when omitted. Must be an integer between 1 and 1000; anything else is rejected with a + * validation error. + */ + limit: t.number.optional() +}); +export type BucketReportRequest = t.Encoded; + +export const SuggestedBucketAction = enumLiteral('none', 'compact', 'defragment', 'both', 'unknown'); +export type SuggestedBucketAction = t.Encoded; + +export const BucketStorageStats = t.object({ + /** Full bucket name, e.g. `by_user["u1"]`. */ + bucket: t.string, + /** Total operations in the bucket's history (PUT/REMOVE/MOVE/CLEAR). Exact and current. */ + operations: t.number, + /** Approximate size of the operation history in bytes. */ + operation_bytes: t.number, + /** + * Operations written after the last full compact — the staleness indicator for `rows` and + * `fragmentation`, which are snapshots from that compact. Equals `operations` when the bucket has never + * been fully compacted. + */ + uncompacted_operations: t.number, + /** + * Live rows in the bucket as of its last full compact, or null if the bucket has never been fully + * compacted (or the storage version does not capture compact statistics). + */ + rows: orNull(t.number), + /** + * `operations / max(rows, 1)`. ~1 is healthy (fully compacted); higher means more operation-history + * overhead that a compact/defragment can reclaim. Null whenever `rows` is null. + */ + fragmentation: orNull(t.number), + /** ISO timestamp of the bucket's last full compact, which is when `rows` was captured. */ + last_full_compact_at: orNull(t.string), + /** + * ISO timestamp of when the scheduled compactor will next consider this bucket. A suggested compact with + * a future `next_compact_at` means the compact is already planned but throttled until then. + */ + next_compact_at: orNull(t.string), + /** + * Suggested maintenance action, derived from the bucket's compact statistics: `none` (healthy), `compact` + * (un-compacted superseded history to reclaim), `defragment` (mostly compaction residue that only a + * defragment collapses), `both`, or `unknown` (no compact statistics to derive one from). + */ + suggested_action: SuggestedBucketAction +}); +export type BucketStorageStats = t.Encoded; + +export const BucketDefinitionStats = t.object({ + /** Definition name as it prefixes bucket names, e.g. `1#by_user` (versioned in storage v2 and later). */ + definition: t.string, + /** Number of buckets in this definition with stored operations. */ + bucket_count: t.number, + /** Total operations across the definition's buckets. Exact and current. */ + operations: t.number, + /** Approximate size of the definition's operation history in bytes. */ + operation_bytes: t.number, + /** + * Operations not covered by any bucket's last full compact — the staleness indicator for `rows` and + * `fragmentation`. + */ + uncompacted_operations: t.number, + /** + * Live rows across the definition's buckets, counting a row once per bucket that contains it. Derived + * from each bucket's last full compact (extrapolated when only some buckets have been compacted); null + * when none have. + */ + rows: orNull(t.number), + /** `operations / max(rows, 1)` across the whole definition. Null whenever `rows` is null. */ + fragmentation: orNull(t.number), + /** Suggested maintenance action for the definition; same values as `buckets[].suggested_action`. */ + suggested_action: SuggestedBucketAction +}); +export type BucketDefinitionStats = t.Encoded; + +export const BucketReportTotals = t.object({ + /** Number of buckets with stored operations. Exact, even when the other totals are estimates. */ + bucket_count: t.number, + /** Sum of operations across all buckets. Estimated when the bucket set was sampled. */ + operations: t.number, + /** Sum of operation-history bytes across all buckets. Estimated when the bucket set was sampled. */ + operation_bytes: t.number, + /** True if the totals are estimated because the bucket set was sampled rather than fully scanned. */ + estimated: t.boolean +}); +export type BucketReportTotals = t.Encoded; + +export const BucketReportResponse = t.object({ + /** Worst-offender buckets, ranked by operation count then fragmentation. */ + buckets: t.array(BucketStorageStats), + /** Per-definition rollup, ranked by operation count then fragmentation. */ + definitions: t.array(BucketDefinitionStats), + totals: BucketReportTotals, + /** True if there are more buckets than returned (more than `limit`). */ + buckets_truncated: t.boolean, + /** True if the definition rollup is incomplete: more definitions exist than the report caps at. */ + definitions_truncated: t.boolean +}); +export type BucketReportResponse = t.Encoded;