From 700bfc12e8034c9396c002e021f525d43bdc5ece Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Fri, 17 Jul 2026 17:51:56 +0530 Subject: [PATCH] fix(bulk-operations): chunk CS Assets bulk delete/move to the 100-item API cap (DX-9732) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CS Assets bulk delete/move sent the entire asset array in one POST, so any request over 100 items failed with HTTP 422 "Assets cannot exceed the max limit of 100." — blocking bulk deletes of >100 assets entirely. - CSAssetsAdapter.bulkDeleteAssets/bulkMoveAssets now split the payload into <=100-item batches (CS_ASSETS_BULK_MUTATE_MAX_ITEMS) dispatched with bounded concurrency (makeConcurrentCall) and per-batch retry (429/5xx; 4xx not retried). - Continue-on-partial-failure: failed batches are collected, not rethrown, since each request commits independently. Aggregate carries job_ids, batch counts, and per-batch failures (with the failed uids). - postJson throws a typed CsAssetsPostError carrying HTTP status (no error-string parsing). - Runner reports partial outcomes, writes failed uids to a {"uids":[...]} file in the bulk-operation log folder for re-run via --asset-uids-file, and states that a submitted delete job is submission (not completion) since delete is async. Tests: adapter chunking/partial-failure/uid-capture; runner partial-failure file write. Co-Authored-By: Claude Opus 4.8 (1M context) --- .talismanrc | 100 +------------ .../src/constants/index.ts | 2 + .../src/types/cs-assets-api.ts | 29 +++- .../src/utils/cs-assets-api-adapter.ts | 100 ++++++++++++- .../unit/utils/cs-assets-api-adapter.test.ts | 63 +++++++- .../src/interfaces/index.ts | 6 + .../src/messages/index.ts | 9 ++ .../src/services/am-asset-service.ts | 19 ++- .../src/utils/cs-assets-runner.ts | 139 ++++++++++++++++-- .../commands/bulk-assets-cs-assets.test.ts | 27 ++++ 10 files changed, 370 insertions(+), 124 deletions(-) diff --git a/.talismanrc b/.talismanrc index aa7705a49..97dd8a4cc 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,100 +1,4 @@ fileignoreconfig: -- filename: pnpm-lock.yaml - checksum: 2e1116588e19b058f0d6a5efa28f54c6a3af11b0f0d8c442ed9866a90b77b713 -- filename: packages/contentstack-clone/test/lib/helpers/command-helpers.test.ts - checksum: 32d385a677a76faa54cb3ddfe92fb1f1f53cfab878a78e2792c823a363956bd2 -- filename: packages/contentstack-clone/test/commands/stack-clone.test.ts - checksum: 4547ba010cc3c9954de49e2b2bef45288dd070c4ad000b6ae3e9a8c9bb1ad296 -- filename: packages/contentstack-asset-management/test/unit/import/workspaces.test.ts - checksum: 67c11d312ba6838659c0b2cefd59d706d4b3c1a9807ed551dcb6d94f4d1d60a3 -- filename: packages/contentstack-clone/test/lib/util/clone-handler.branch.test.ts - checksum: db8c090aaada21d3952c5bc382228a0a11cebb63926512a5e2581bdaf5f4c2f6 -- filename: packages/contentstack-asset-management/test/unit/import/spaces.test.ts - checksum: d2363470664a6cdeaf59c305fa6307704e48b73f92b7d471433c17528daa5450 -- filename: packages/contentstack-asset-management/test/unit/utils/detect-asset-management-export.test.ts - checksum: 1917c51ebf4ba7f94cc440c27341d840911b5c315e250a451732c2d37518db64 -- filename: packages/contentstack-bootstrap/test/bootstrap-integration.test.js - checksum: 67579b79fc5c1b4feead8f537e7bdfd14fac58aa93c127f590505e47d26e0eb0 -- filename: packages/contentstack-import/test/unit/import/modules/index.test.ts - checksum: 27dd2c5f60f9f4c70f2ebf4c08cf8bcbe2d5e4dc7fe52a53972af3fdb57e0e63 -- filename: packages/contentstack-bulk-operations/test/unit/base-bulk-command.test.ts - checksum: 522b91a2d4bb6a73ebe5ab92c0111e74d567c4360470beefd6a7ca8d30a6e311 -- filename: packages/contentstack-content-type/tests/core/command.test.ts - checksum: 81da32fc47083a3421f5f1d9c04149ab313cb238d6416d3182f8b6d64dc8827d -- filename: packages/contentstack-content-type/tests/core/contentstack/client.test.ts - checksum: da15b04aa0a7ab0e48f70b847e68c05372943279e51c22f8539b01bb48033183 -- filename: packages/contentstack-import/test/unit/import/modules/marketplace-apps.test.ts - checksum: 17b3e6e0f7ce12ddabd688996c72d04dba5245fd300a23b541cf7bdfbbf7ef1a -- filename: packages/contentstack-asset-management/test/unit/import/asset-types.test.ts - checksum: 76aad963f5977059f7632d69a6fb36004229f3455b1618c372579dfa3cbc2d62 -- filename: packages/contentstack-asset-management/test/unit/import/assets.test.ts - checksum: f5f0494c9263ebcf7eabff806ff41bd7beed0cb6a0f9a631dd70ffa01524a304 -- filename: packages/contentstack-import/test/unit/utils/common-helper.test.ts - checksum: 7a10c4625d666cf705cd9869b74b3bc2dcec94926cbb5f719e4f8dc4f053b2c7 -- filename: packages/contentstack-content-type/tests/utils/index.test.ts - checksum: 8510fedf5eb6078c27d6a0d65d8bff4aa2d41b545246aed220d1a6a643188ad8 -- filename: packages/contentstack-import/test/unit/utils/interactive.test.ts - checksum: dad20e229e02e847b5e939fdaf987b1a6c9ff4e1529b6d96a51130f7293285a9 -- filename: packages/contentstack-asset-management/test/unit/import/base.test.ts - checksum: 53168bda5251300d8dfd0effd470070a5733c37b77abacc846d91e7086c8c022 -- filename: packages/contentstack-asset-management/test/unit/import/fields.test.ts - checksum: 9e97001a6c92d213a9f55a1458bf39c0996952467e0b4aabd954ddd8ad1805b9 -- filename: packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.ts - checksum: cc82f4b0c50783dd42d8d57ba2af1e5c9aa3857399c5abf39bc5dc3fac105658 -- filename: packages/contentstack-bulk-operations/test/unit/utils/bulk-operation-log-handler.test.ts - checksum: 8698f77bbb828df10ddc3a94726076a262fbf47d2468233c7e10e66cac42f97d -- filename: packages/contentstack-import/test/unit/utils/login-handler.test.ts - checksum: a60f46b4d79ccdbeecfed27ffffbc97a44d6753f36ee74b68b725c8b20bbb2eb -- filename: packages/contentstack-export-to-csv/test/unit/utils/error-handler.test.ts - checksum: cef8f515ab6c0441f67748693473bf1499c2d54945aab1b89baaad37be8af2d9 -- filename: packages/contentstack-export-to-csv/test/unit/utils/teams-export.functional.test.ts - checksum: bcf70dbb201a0a1755f0b8f59550aae95a3e1d03700959eee7e8d415f9647746 -- filename: packages/contentstack-export/test/unit/utils/marketplace-app-helper.test.ts - checksum: 5648d90b0f7dc33a5b020324db5218707861ebd7201802f7fef00f1c03c5ce82 -- filename: packages/contentstack-export/test/unit/export/module-exporter.test.ts - checksum: 485bbd175cc95aff6c1c3c6b2904f92c3d4ea1272a056158a901da3eff9692d6 -- filename: packages/contentstack-import-setup/test/unit/common-helper.test.ts - checksum: ccbd8209d342bf651c2c38326cba13bc3e5c2af4200d7110fdd7c3de4824d21d -- filename: packages/contentstack-export/test/unit/utils/common-helper.test.ts - checksum: a4bfba19846e1075821521a202d856203f02a6bc0e5ec4f51b5265801cb4ddd9 -- filename: packages/contentstack-import/test/unit/import/modules/assets.test.ts - checksum: 259d5c325ec7de08d0f45099df9d3eb7c97374691c1c4d281221e112b9097191 -- filename: packages/contentstack-export/test/unit/export/modules/marketplace-apps.test.ts - checksum: 07eef22a3c471e8da12b1e7e51d45b76410725c4c160a0f35432d9aac5916974 -- filename: packages/contentstack-export/test/unit/utils/interactive.test.ts - checksum: 0ea47db305f576c8c9da8cb34ae915f6b01992208ff1c4380cb7ffd93f538e5d -- filename: packages/contentstack-bulk-operations/test/unit/utils/revert-retry-handler.test.ts - checksum: c34a4bdea186f4ce48171d90b8ab0f1c94162e441acb23e3a0e00cb9f9f9255f -- filename: packages/contentstack-bulk-operations/test/unit/utils/config-builder.test.ts - checksum: e5891257999e9e940956ca77f5a7834be35d259edccd41c06f77736194ad7454 -- filename: packages/contentstack-migrate-rte/test/commands/json-migration.test.js - checksum: 0d18641a32f7cf33d013a5a7c447c1067ba77022513b42304a727f873c741e99 -- filename: packages/contentstack-seed/test/commands/cm/stacks/seed.test.ts - checksum: e469a60f09a9a5fd3e2ce80dd6df8bf4f2c82acad1afebde00f458afa26d36c9 -- filename: packages/contentstack-migrate-rte/test/utils/index.js - checksum: f67235e607a446e5b2dd624046966a1ac5af7bce128aa080ec3c420c48b8bddc -- filename: packages/contentstack-query-export/test/unit/common-helper.test.ts - checksum: a0aa399a2ec15760fe25f7e952d64792b55d3c38d2281858a48e64f0f9081344 -- filename: packages/contentstack-seed/test/seed/contentstack/client.test.ts - checksum: 63bd3325e2717e5653f12e4afa362938e58eeaad5fbe07c1d41e2bd15e3451bd -- filename: packages/contentstack-import/test/unit/commands/cm/stacks/import.test.ts - checksum: d241111877a997f00702eaf171c1fde53a075a9c8f7971a8e6b1577c1c83279d -- filename: packages/contentstack-import/test/unit/import/modules/base-class.test.ts - checksum: 40c504d508d40b6416ccedb5a323365dd97f313b1dcc885e4fec3639e807b383 -- filename: packages/contentstack-import/test/unit/import/modules/content-types.test.ts - checksum: 662cb42217c12af2b4dad5d1355858095c5df062e42c89e09b2680fdbee80699 -- filename: packages/contentstack-import/test/unit/import/modules/entries.test.ts - checksum: a7311e3077fee52bfd23c46d51ebe9ce58d202dd735514a44ec230615f4cd917 -- filename: packages/contentstack-import/test/unit/import/modules/global-fields.test.ts - checksum: 2b4e0d5d11f825d21d45569f681acd886c5f84717873b26998b4dab2e91bb439 -- filename: packages/contentstack-seed/test/seed/interactive.test.ts - checksum: 8364557eb69b9a297281f868420c13c5b61b26b79bc2ceb0a9b548cfb0365d6d -- filename: packages/contentstack-variants/test/unit/export/variant-entries.test.ts - checksum: dd7018cd5cc8b5fab73004b2968afed7ce33abec0b20e1b17f15f2be23b117db -- filename: packages/contentstack-import-setup/test/unit/login-handler.test.ts - checksum: 2df43be75a96e1a3a00dd256628e191909aa9d7f192b672b56cb3772e074958c -- filename: packages/contentstack-export/test/unit/utils/export-config-handler.test.ts - checksum: f612661a8b6784b20ea62794b35192b340a34d8424a54a1a2cffec1878ab7528 -- filename: packages/contentstack-import/test/unit/utils/marketplace-app-helper.test.ts - checksum: de694e861560c4c242100eaf17a6d8e230247b99f94e4b428e55ef065084c10c +- filename: packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts + checksum: 86f16cef82b891d0946871a5b48726bb2592dc8c16de4c878baf39e5a0a505f7 version: "" diff --git a/packages/contentstack-asset-management/src/constants/index.ts b/packages/contentstack-asset-management/src/constants/index.ts index ec288b72a..9f8950278 100644 --- a/packages/contentstack-asset-management/src/constants/index.ts +++ b/packages/contentstack-asset-management/src/constants/index.ts @@ -6,6 +6,8 @@ export const FALLBACK_AM_API_CONCURRENCY = 5; export const DEFAULT_AM_API_CONCURRENCY = FALLBACK_AM_API_CONCURRENCY; export const FALLBACK_AM_API_PAGE_SIZE = 100; export const FALLBACK_AM_API_FETCH_CONCURRENCY = 5; +/** Max assets/uids the CS Assets bulk delete/move endpoints accept per request. */ +export const CS_ASSETS_BULK_MUTATE_MAX_ITEMS = 100; /** Fallback strip lists when import options omit `fieldsImportInvalidKeys` / `assetTypesImportInvalidKeys`. */ export const FALLBACK_FIELDS_IMPORT_INVALID_KEYS = [ diff --git a/packages/contentstack-asset-management/src/types/cs-assets-api.ts b/packages/contentstack-asset-management/src/types/cs-assets-api.ts index 56e45bfff..6809d3090 100644 --- a/packages/contentstack-asset-management/src/types/cs-assets-api.ts +++ b/packages/contentstack-asset-management/src/types/cs-assets-api.ts @@ -127,9 +127,31 @@ export type BulkDeleteAssetItem = { uid: string; locale: string }; export type BulkDeleteAssetsPayload = { assets: BulkDeleteAssetItem[] }; +/** One failed batch when a bulk mutate is split across multiple ≤100-item requests. */ +export type BulkMutateFailure = { + batchIndex: number; + count: number; + status?: number; + error: string; + /** Asset uids in the failed batch, so callers can re-run just the failures. */ + uids: string[]; +}; + +/** Raw response of a single bulk-mutate request (one ≤100-item POST) as returned by the API. */ +export type CsAssetsMutateBatchResponse = { notice?: string; job_id?: string }; + +/** Aggregate result of a bulk delete, combining every dispatched ≤100-item batch. */ export type BulkDeleteAssetsResponse = { + /** First notice returned; a human-facing message, safe to surface as the summary line. */ notice?: string; - job_id?: string; + /** One submitted job id, for a short summary line; `job_ids` holds all of them (batches + * run concurrently, so this is whichever completed first — not a stable "first batch"). */ + primaryJobId?: string; + notices?: string[]; + job_ids?: string[]; + failures?: BulkMutateFailure[]; + batchesTotal?: number; + batchesSucceeded?: number; }; export type BulkMoveAssetsPayload = { @@ -137,8 +159,13 @@ export type BulkMoveAssetsPayload = { target_folder_uid: string; }; +/** Aggregate result of a bulk move (sync; no job ids). */ export type BulkMoveAssetsResponse = { notice?: string; + notices?: string[]; + failures?: BulkMutateFailure[]; + batchesTotal?: number; + batchesSucceeded?: number; }; /** diff --git a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts index 28b0e7a11..8e2362bf0 100644 --- a/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts +++ b/packages/contentstack-asset-management/src/utils/cs-assets-api-adapter.ts @@ -4,7 +4,12 @@ import chunk from 'lodash/chunk'; import { HttpClient, log, authenticationHandler, handleAndLogError } from '@contentstack/cli-utilities'; import { withRetry, RetryableHttpError, isRetryableStatus, parseRetryAfterMs } from './retry'; -import { FALLBACK_AM_API_FETCH_CONCURRENCY, FALLBACK_AM_API_PAGE_SIZE } from '../constants/index'; +import { + CS_ASSETS_BULK_MUTATE_MAX_ITEMS, + FALLBACK_AM_API_CONCURRENCY, + FALLBACK_AM_API_FETCH_CONCURRENCY, + FALLBACK_AM_API_PAGE_SIZE, +} from '../constants/index'; import type { CSAssetsAPIConfig, @@ -13,6 +18,8 @@ import type { BulkDeleteAssetsResponse, BulkMoveAssetsPayload, BulkMoveAssetsResponse, + BulkMutateFailure, + CsAssetsMutateBatchResponse, CreateAssetMetadata, CreateAssetTypePayload, CreateFieldPayload, @@ -91,6 +98,21 @@ export type CustomPromiseHandlerInput = { export type CustomPromiseHandler = (input: CustomPromiseHandlerInput) => Promise; +/** + * Error thrown by {@link CSAssetsAdapter.postJson} for a failed POST. Carries the HTTP + * `status` (undefined for network/transport failures) so callers can classify failures + * without parsing the message string. + */ +export class CsAssetsPostError extends Error { + constructor( + message: string, + public readonly status?: number, + ) { + super(message); + this.name = 'CsAssetsPostError'; + } +} + export class CSAssetsAdapter implements ICSAssetsAdapter { private readonly config: CSAssetsAPIConfig; private readonly apiClient: HttpClient; @@ -607,10 +629,11 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { } const text = await response.text().catch(() => ''); const bodySnippet = this.formatResponseBodyForError(text); - throw new Error( + throw new CsAssetsPostError( `CS Assets API POST failed: status ${response.status} path ${path}${ bodySnippet ? `\nResponse: ${bodySnippet}` : '' }`, + response.status, ); } return response.json() as Promise; @@ -627,12 +650,17 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { : await doPost(); } catch (error) { if (error instanceof RetryableHttpError) { - throw new Error(`CS Assets API POST failed: path ${path} (status ${error.status ?? 'network'}) - ${error.message}`); + throw new CsAssetsPostError( + `CS Assets API POST failed: path ${path} (status ${error.status ?? 'network'}) - ${error.message}`, + error.status, + ); } - if (error instanceof Error && error.message.includes('CS Assets API POST failed')) { + if (error instanceof CsAssetsPostError) { throw error; } - throw new Error(`CS Assets API POST failed: path ${path} - ${error instanceof Error ? error.message : String(error)}`); + throw new CsAssetsPostError( + `CS Assets API POST failed: path ${path} - ${error instanceof Error ? error.message : String(error)}`, + ); } } @@ -737,11 +765,22 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { payload: BulkDeleteAssetsPayload, ): Promise { const path = `/api/spaces/${encodeURIComponent(spaceUid)}/assets/bulk/delete?workspace=${encodeURIComponent(workspaceUid)}`; - return this.postJson(path, payload, { space_key: spaceUid }); + const bodies = chunk(payload.assets, CS_ASSETS_BULK_MUTATE_MAX_ITEMS).map((assets) => ({ assets })); + const { notices, jobIds, failures, batchesTotal } = await this.dispatchBulkMutateBatches(spaceUid, path, bodies); + return { + notice: notices[0], + primaryJobId: jobIds[0], + notices, + job_ids: jobIds, + failures, + batchesTotal, + batchesSucceeded: batchesTotal - failures.length, + }; } /** * POST /api/spaces/{spaceUid}/assets/bulk-move — move assets into a folder. + * Split into ≤{@link CS_ASSETS_BULK_MUTATE_MAX_ITEMS}-item requests (same cap as delete). */ async bulkMoveAssets( spaceUid: string, @@ -749,6 +788,53 @@ export class CSAssetsAdapter implements ICSAssetsAdapter { payload: BulkMoveAssetsPayload, ): Promise { const path = `/api/spaces/${encodeURIComponent(spaceUid)}/assets/bulk-move?workspace=${encodeURIComponent(workspaceUid)}`; - return this.postJson(path, payload, { space_key: spaceUid }); + const bodies = chunk(payload.asset_uids, CS_ASSETS_BULK_MUTATE_MAX_ITEMS).map((asset_uids) => ({ + asset_uids, + target_folder_uid: payload.target_folder_uid, + })); + const { notices, failures, batchesTotal } = await this.dispatchBulkMutateBatches(spaceUid, path, bodies); + return { + notice: notices[0], + notices, + failures, + batchesTotal, + batchesSucceeded: batchesTotal - failures.length, + }; + } + + /** + * Dispatch pre-chunked bulk-mutate request bodies (each already ≤100 items) through + * {@link postJson} with bounded concurrency via {@link makeConcurrentCall}. A batch + * failure is collected — never rethrown — because the CS Assets bulk endpoints commit + * each request independently, so earlier batches are already applied server-side and + * callers must be able to report partial outcomes. `postJson` retries transient + * (429/5xx) failures; 4xx like the 422 item-cap are not retried. + */ + private async dispatchBulkMutateBatches( + spaceUid: string, + path: string, + bodies: unknown[], + ): Promise<{ notices: string[]; jobIds: string[]; failures: BulkMutateFailure[]; batchesTotal: number }> { + const notices: string[] = []; + const jobIds: string[] = []; + const failures: BulkMutateFailure[] = []; + const apiBatches = chunk(bodies, FALLBACK_AM_API_CONCURRENCY); + + await this.makeConcurrentCall({ module: `bulk-mutate ${path}`, apiBatches }, async ({ element, batchIndex, index }) => { + const globalIndex = batchIndex * FALLBACK_AM_API_CONCURRENCY + index; + const body = element as { assets?: { uid: string }[]; asset_uids?: string[] }; + const uids = body.assets ? body.assets.map((a) => a.uid) : (body.asset_uids ?? []); + try { + const r = await this.postJson(path, body, { space_key: spaceUid }, { retry: true }); + if (typeof r.notice === 'string') notices.push(r.notice); + if (typeof r.job_id === 'string') jobIds.push(r.job_id); + } catch (e) { + const status = e instanceof CsAssetsPostError ? e.status : undefined; + const message = e instanceof Error ? e.message : String(e); + failures.push({ batchIndex: globalIndex, count: uids.length, status, error: message, uids }); + } + }); + + return { notices, jobIds, failures, batchesTotal: bodies.length }; } } diff --git a/packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts b/packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts index d9b86a60a..8c19f58ec 100644 --- a/packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts +++ b/packages/contentstack-asset-management/test/unit/utils/cs-assets-api-adapter.test.ts @@ -540,39 +540,94 @@ describe('CSAssetsAdapter', () => { }); }); + const deleteItems = (n: number) => + Array.from({ length: n }, (_, i) => ({ uid: `a${i}`, locale: 'en-us' })); + describe('bulkDeleteAssets', () => { it('POSTs to the bulk delete endpoint with workspace query param', async () => { - fetchStub.resolves(okJsonResponse({ deleted: 2 })); + fetchStub.resolves(okJsonResponse({ notice: 'ok', job_id: 'j1' })); const adapter = new CSAssetsAdapter(baseConfig); - await adapter.bulkDeleteAssets('sp-1', 'ws-main', { asset_uids: ['a1', 'a2'] } as any); + await adapter.bulkDeleteAssets('sp-1', 'ws-main', { assets: deleteItems(2) }); const [url, opts] = fetchStub.firstCall.args; expect(url).to.include('/api/spaces/sp-1/assets/bulk/delete'); expect(url).to.include('workspace=ws-main'); expect(opts.headers['space_key']).to.equal('sp-1'); + expect(JSON.parse(opts.body).assets).to.have.length(2); }); it('uses "main" as default workspace uid', async () => { fetchStub.resolves(okJsonResponse({})); const adapter = new CSAssetsAdapter(baseConfig); - await adapter.bulkDeleteAssets('sp-1', undefined as any, {} as any); + await adapter.bulkDeleteAssets('sp-1', undefined as any, { assets: deleteItems(1) }); const [url] = fetchStub.firstCall.args; expect(url).to.include('workspace=main'); }); + + it('splits >100 assets into ≤100-item batches and aggregates job ids', async () => { + fetchStub.callsFake(async () => okJsonResponse({ notice: 'batch ok', job_id: 'job-x' })); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.bulkDeleteAssets('sp-1', 'main', { assets: deleteItems(250) }); + + expect(fetchStub.callCount).to.equal(3); // 100 + 100 + 50 + for (const call of fetchStub.getCalls()) { + expect(JSON.parse(call.args[1].body).assets.length).to.be.at.most(100); + } + expect(result.batchesTotal).to.equal(3); + expect(result.batchesSucceeded).to.equal(3); + expect(result.job_ids).to.have.length(3); + expect(result.failures).to.have.length(0); + }); + + it('keeps succeeded batches and records failures when one batch fails (partial)', async () => { + // 3 batches; the 2nd (second fetch) returns 422, others succeed. + let n = 0; + fetchStub.callsFake(async () => { + n += 1; + return n === 2 ? failResponse(422, 'Assets cannot exceed the max limit of 100.') : okJsonResponse({ job_id: `j${n}` }); + }); + const adapter = new CSAssetsAdapter(baseConfig); + const result = await adapter.bulkDeleteAssets('sp-1', 'main', { assets: deleteItems(250) }); + + expect(result.batchesTotal).to.equal(3); + expect(result.batchesSucceeded).to.equal(2); + expect(result.failures).to.have.length(1); + expect(result.failures![0].status).to.equal(422); + expect(result.failures![0].error).to.include('422'); + // Failed batch carries its uids so callers can re-run just the failures. + expect(result.failures![0].uids).to.have.length(100); + expect(result.failures![0].uids.every((u) => u.startsWith('a'))).to.equal(true); + }); }); describe('bulkMoveAssets', () => { it('POSTs to the bulk-move endpoint with workspace query param', async () => { fetchStub.resolves(okJsonResponse({ moved: 1 })); const adapter = new CSAssetsAdapter(baseConfig); - await adapter.bulkMoveAssets('sp-1', 'ws-main', { asset_uids: ['a1'], folder_uid: 'f1' } as any); + await adapter.bulkMoveAssets('sp-1', 'ws-main', { asset_uids: ['a1'], target_folder_uid: 'f1' }); const [url, opts] = fetchStub.firstCall.args; expect(url).to.include('/api/spaces/sp-1/assets/bulk-move'); expect(url).to.include('workspace=ws-main'); expect(opts.headers['space_key']).to.equal('sp-1'); }); + + it('splits >100 uids into ≤100-item batches, re-attaching target_folder_uid', async () => { + fetchStub.callsFake(async () => okJsonResponse({ notice: 'moved' })); + const adapter = new CSAssetsAdapter(baseConfig); + const uids = Array.from({ length: 150 }, (_, i) => `u${i}`); + const result = await adapter.bulkMoveAssets('sp-1', 'main', { asset_uids: uids, target_folder_uid: 'f1' }); + + expect(fetchStub.callCount).to.equal(2); // 100 + 50 + for (const call of fetchStub.getCalls()) { + const body = JSON.parse(call.args[1].body); + expect(body.asset_uids.length).to.be.at.most(100); + expect(body.target_folder_uid).to.equal('f1'); + } + expect(result.batchesTotal).to.equal(2); + expect(result.batchesSucceeded).to.equal(2); + }); }); describe('postJson error handling', () => { diff --git a/packages/contentstack-bulk-operations/src/interfaces/index.ts b/packages/contentstack-bulk-operations/src/interfaces/index.ts index f7a9c7596..3d8c655bf 100644 --- a/packages/contentstack-bulk-operations/src/interfaces/index.ts +++ b/packages/contentstack-bulk-operations/src/interfaces/index.ts @@ -269,6 +269,12 @@ export interface CsAssetsBulkOperationResult { notice?: string; jobId?: string; error?: string; + /** Aggregate across the ≤100-item batches a single delete/move is split into. */ + jobIds?: string[]; + batchesTotal?: number; + batchesSucceeded?: number; + batchesFailed?: number; + failures?: { batchIndex: number; count: number; error: string; uids: string[] }[]; } /** Typed flags for CS Assets delete/move operations (cm:stacks:bulk-assets). */ diff --git a/packages/contentstack-bulk-operations/src/messages/index.ts b/packages/contentstack-bulk-operations/src/messages/index.ts index 9f82677a8..52481306d 100644 --- a/packages/contentstack-bulk-operations/src/messages/index.ts +++ b/packages/contentstack-bulk-operations/src/messages/index.ts @@ -247,11 +247,20 @@ const csAssetsBulkMsg = { CS_ASSETS_INVALID_OPERATION: 'Invalid operation: {operation}. Must be delete or move', CS_ASSETS_CONFIRM_SUMMARY: 'Proceed with CS Assets {operation} on {count} item(s)?', CS_ASSETS_DELETE_SUCCESS: 'CS Assets bulk delete job submitted successfully!', + CS_ASSETS_DELETE_JOBS_SUBMITTED: + '{count} bulk delete job(s) submitted. Deletion runs asynchronously — this confirms submission, not completion. Verify at the status URL below:', CS_ASSETS_DELETE_JOB_ID: 'Job ID: {jobId}', CS_ASSETS_DELETE_ASYNC_NOTE: 'The job runs asynchronously — check the bulk task queue for status:', CS_ASSETS_MOVE_SUCCESS: 'CS Assets bulk move completed successfully!', CS_ASSETS_MOVE_ASSETS_COUNT: '{count} asset(s) moved to folder: {folderUid}', CS_ASSETS_OPERATION_FAILED: 'CS Assets {operation} failed.', + CS_ASSETS_BATCH_SUMMARY: 'Dispatched in {batchesTotal} batch(es) of up to 100 — {batchesSucceeded} succeeded.', + CS_ASSETS_PARTIAL_FAILURE: 'CS Assets {operation} partially failed: {batchesFailed} of {batchesTotal} batch(es) failed.', + CS_ASSETS_FAILED_BATCH: 'Batch {batchIndex} ({count} item(s)) failed: {error}', + CS_ASSETS_FAILED_UIDS_WRITTEN: + 'Uids whose {operation} request did not confirm success written to: {path} (these requests failed to return success — the server may or may not have applied them).', + CS_ASSETS_RETRY_HINT: + 'Re-run just these with: --operation {operation} --asset-uids-file {path} (plus the same --space-uid/--org-uid, and --locale for delete). Safe to re-run — the operation is idempotent, so assets already applied are a no-op.', // Merged-command flag matrix validation FLAG_NOT_ALLOWED_FOR_OPERATION: '{flag} is not valid for operation "{operation}".{hint}', diff --git a/packages/contentstack-bulk-operations/src/services/am-asset-service.ts b/packages/contentstack-bulk-operations/src/services/am-asset-service.ts index d80e288f4..b6e37e497 100644 --- a/packages/contentstack-bulk-operations/src/services/am-asset-service.ts +++ b/packages/contentstack-bulk-operations/src/services/am-asset-service.ts @@ -24,10 +24,17 @@ export class CsAssetsService { const response = await this.adapter.bulkDeleteAssets(spaceUid, workspaceUid ?? 'main', { assets: items, }); + const failures = response.failures ?? []; return { - success: true, + success: failures.length === 0, notice: typeof response.notice === 'string' ? response.notice : undefined, - jobId: typeof response.job_id === 'string' ? response.job_id : undefined, + jobId: typeof response.primaryJobId === 'string' ? response.primaryJobId : undefined, + jobIds: response.job_ids, + batchesTotal: response.batchesTotal, + batchesSucceeded: response.batchesSucceeded, + batchesFailed: failures.length, + failures: failures.map((f) => ({ batchIndex: f.batchIndex, count: f.count, error: f.error, uids: f.uids })), + error: failures.length > 0 ? failures.map((f) => f.error).join('; ') : undefined, }; } catch (e: unknown) { return { @@ -48,9 +55,15 @@ export class CsAssetsService { asset_uids: assetUids, target_folder_uid: targetFolderUid, }); + const failures = response.failures ?? []; return { - success: true, + success: failures.length === 0, notice: typeof response.notice === 'string' ? response.notice : undefined, + batchesTotal: response.batchesTotal, + batchesSucceeded: response.batchesSucceeded, + batchesFailed: failures.length, + failures: failures.map((f) => ({ batchIndex: f.batchIndex, count: f.count, error: f.error, uids: f.uids })), + error: failures.length > 0 ? failures.map((f) => f.error).join('; ') : undefined, }; } catch (e: unknown) { return { diff --git a/packages/contentstack-bulk-operations/src/utils/cs-assets-runner.ts b/packages/contentstack-bulk-operations/src/utils/cs-assets-runner.ts index 4428fa881..7144ff51f 100644 --- a/packages/contentstack-bulk-operations/src/utils/cs-assets-runner.ts +++ b/packages/contentstack-bulk-operations/src/utils/cs-assets-runner.ts @@ -1,3 +1,5 @@ +import * as fs from 'node:fs'; +import path from 'node:path'; import chalk from 'chalk'; import { log, createLogContext, cliux, handleAndLogError, authenticationHandler } from '@contentstack/cli-utilities'; @@ -5,6 +7,7 @@ import messages, { $t } from '../messages'; import { CsAssetsService } from '../services'; import { loadAssetUidsFromFile, loadBulkDeleteItemsFromFile, LoadAssetUidsError } from './asset-uids-from-file'; import { generateCsAssetsJobStatusUrl } from './bulk-publish-url-generator'; +import { ensureLogFolder } from './bulk-operation-log-handler'; import { CsAssetsFlags, CsAssetsBulkOperationResult, OperationType } from '../interfaces'; /** @@ -30,20 +33,37 @@ interface CsAssetsOperationPlan { successOpts: (result: CsAssetsBulkOperationResult) => Parameters[1]; } +interface CsAssetsSummaryOpts { + jobId?: string; + jobIds?: string[]; + count?: number; + folderUid?: string; + notice?: string; + error?: string; + spaceUid?: string; + batchesTotal?: number; + batchesSucceeded?: number; +} + function printCsAssetsSummary( op: 'delete' | 'move', - opts: { jobId?: string; count?: number; folderUid?: string; notice?: string; error?: string; spaceUid?: string }, + opts: CsAssetsSummaryOpts, loggerContext: { module: string } ): void { if (opts.error) { log.error($t(messages.CS_ASSETS_OPERATION_FAILED, { operation: op }), loggerContext); log.error(opts.error, loggerContext); - } else if (op === 'delete') { + return; + } + + if (op === 'delete') { + const jobIds = opts.jobIds?.length ? opts.jobIds : opts.jobId ? [opts.jobId] : []; log.success($t(messages.CS_ASSETS_DELETE_SUCCESS), loggerContext); - if (opts.jobId) log.info($t(messages.CS_ASSETS_DELETE_JOB_ID, { jobId: opts.jobId }), loggerContext); - log.info($t(messages.CS_ASSETS_DELETE_ASYNC_NOTE), loggerContext); - const statusUrl = generateCsAssetsJobStatusUrl(opts.spaceUid); - if (statusUrl) log.info(statusUrl, loggerContext); + // Delete is async: a submitted job is not a completed deletion. Say so, and point at the status URL. + log.info($t(messages.CS_ASSETS_DELETE_JOBS_SUBMITTED, { count: jobIds.length }), loggerContext); + for (const jobId of jobIds) { + log.info($t(messages.CS_ASSETS_DELETE_JOB_ID, { jobId }), loggerContext); + } } else { log.success($t(messages.CS_ASSETS_MOVE_SUCCESS), loggerContext); if (opts.count !== undefined && opts.folderUid) { @@ -52,12 +72,88 @@ function printCsAssetsSummary( loggerContext ); } - const statusUrl = generateCsAssetsJobStatusUrl(opts.spaceUid); - if (statusUrl) log.info(statusUrl, loggerContext); } + + const batchesTotal = opts.batchesTotal ?? 0; + if (batchesTotal > 1) { + log.info( + $t(messages.CS_ASSETS_BATCH_SUMMARY, { + batchesTotal, + batchesSucceeded: opts.batchesSucceeded ?? batchesTotal, + }), + loggerContext + ); + } + + const statusUrl = generateCsAssetsJobStatusUrl(opts.spaceUid); + if (statusUrl) log.info(statusUrl, loggerContext); if (opts.notice) log.info(opts.notice, loggerContext); } +/** + * Writes the uids from all failed batches to a `{ "uids": [...] }` file (deduped) so the + * user can re-run just the failures via `--asset-uids-file`. Returns the path, or undefined + * if there were no failed uids or the write failed. + */ +function writeFailedUidsFile( + op: 'delete' | 'move', + result: CsAssetsBulkOperationResult, + loggerContext: { module: string } +): string | undefined { + const uids = [...new Set((result.failures ?? []).flatMap((f) => f.uids))]; + if (uids.length === 0) return undefined; + + const fileName = `cs-assets-${op}-failed-${new Date().toISOString().replace(/[:.]/g, '-')}.json`; + try { + // Co-locate with the other bulk-operation logs rather than polluting cwd. + const filePath = path.join(ensureLogFolder(), fileName); + fs.writeFileSync(filePath, JSON.stringify({ uids }), 'utf8'); + return filePath; + } catch (e) { + log.warn(`Could not write failed-uids file: ${e instanceof Error ? e.message : String(e)}`, loggerContext); + return undefined; + } +} + +/** + * True if at least one ≤100-item batch was accepted by the server, so there is a real + * (full or partial) outcome to report. When the operation ran, `batchesSucceeded` reflects it. + * A total transport failure before any batch leaves `batchesSucceeded` undefined and `success` + * false → falls through to false, and the caller reports it as a full failure. + */ +function didAnyBatchCommit(result: CsAssetsBulkOperationResult): boolean { + if (result.batchesSucceeded !== undefined) return result.batchesSucceeded > 0; + return result.success; +} + +/** Warns about failed batches after a partial success (some batches committed, some did not). */ +function printCsAssetsPartialFailure( + op: 'delete' | 'move', + result: CsAssetsBulkOperationResult, + loggerContext: { module: string } +): void { + log.warn( + $t(messages.CS_ASSETS_PARTIAL_FAILURE, { + operation: op, + batchesFailed: result.batchesFailed ?? result.failures?.length ?? 0, + batchesTotal: result.batchesTotal ?? 0, + }), + loggerContext + ); + for (const f of result.failures ?? []) { + log.error( + $t(messages.CS_ASSETS_FAILED_BATCH, { batchIndex: f.batchIndex, count: f.count, error: f.error }), + loggerContext + ); + } + + const failedFile = writeFailedUidsFile(op, result, loggerContext); + if (failedFile) { + log.info($t(messages.CS_ASSETS_FAILED_UIDS_WRITTEN, { operation: op, path: failedFile }), loggerContext); + log.info($t(messages.CS_ASSETS_RETRY_HINT, { operation: op, path: failedFile }), loggerContext); + } +} + function handleAssetUidsFileError(e: LoadAssetUidsError, loggerContext: { module: string }): void { const pathShown = e.filePath; if (e.kind === 'READ') { @@ -195,7 +291,14 @@ export async function runCsAssetsOperation(options: CsAssetsRunnerOptions): Prom log.info($t(messages.CS_ASSETS_DELETING_ASSETS, { count: deleteRows.length, spaceUid }), loggerContext), execute: (service) => service.bulkDelete(spaceUid, workspace, deleteRows), failureFallback: 'CS Assets bulk delete failed', - successOpts: (result) => ({ jobId: result.jobId, notice: result.notice, spaceUid }), + successOpts: (result) => ({ + jobId: result.jobId, + jobIds: result.jobIds, + notice: result.notice, + spaceUid, + batchesTotal: result.batchesTotal, + batchesSucceeded: result.batchesSucceeded, + }), }; } else { if (f.locale) { @@ -229,7 +332,14 @@ export async function runCsAssetsOperation(options: CsAssetsRunnerOptions): Prom ), execute: (service) => service.bulkMove(spaceUid, workspace, uids, moveFolderUid), failureFallback: 'CS Assets bulk move failed', - successOpts: (result) => ({ count: uids.length, folderUid: moveFolderUid, notice: result.notice, spaceUid }), + successOpts: (result) => ({ + count: uids.length, + folderUid: moveFolderUid, + notice: result.notice, + spaceUid, + batchesTotal: result.batchesTotal, + batchesSucceeded: result.batchesSucceeded, + }), }; } @@ -242,12 +352,19 @@ export async function runCsAssetsOperation(options: CsAssetsRunnerOptions): Prom plan.logStart(); const result = await plan.execute(csAssetsService); - if (!result.success) { + + if (!didAnyBatchCommit(result)) { printCsAssetsSummary(op, { error: result.error ?? plan.failureFallback, spaceUid }, loggerContext); process.exitCode = 1; return; } + + // Full or partial success: report what committed, then flag any failed batches. printCsAssetsSummary(op, plan.successOpts(result), loggerContext); + if (!result.success) { + printCsAssetsPartialFailure(op, result, loggerContext); + process.exitCode = 1; + } } catch (error) { handleAndLogError(error as Error); } diff --git a/packages/contentstack-bulk-operations/test/unit/commands/bulk-assets-cs-assets.test.ts b/packages/contentstack-bulk-operations/test/unit/commands/bulk-assets-cs-assets.test.ts index b004ba5de..cca94c59b 100644 --- a/packages/contentstack-bulk-operations/test/unit/commands/bulk-assets-cs-assets.test.ts +++ b/packages/contentstack-bulk-operations/test/unit/commands/bulk-assets-cs-assets.test.ts @@ -171,6 +171,33 @@ describe('BulkAssets command — CS Assets delete/move path', () => { expect(process.exitCode).to.equal(1); }); + + it('on partial failure: sets exitCode=1 and writes the failed uids to a {uids:[...]} file', async () => { + const assetUidsModule = require('../../../src/utils/asset-uids-from-file'); + sandbox.stub(assetUidsModule, 'loadBulkDeleteItemsFromFile').returns([{ uid: 'u1', locale: 'en-us' }]); + + const amServiceModule = require('../../../src/services/am-asset-service'); + // batch 0 committed, batch 1 (uids u2,u3) failed → partial success. + sandbox.stub(amServiceModule.CsAssetsService.prototype, 'bulkDelete').resolves({ + success: false, + jobId: 'job-ok-0', + jobIds: ['job-ok-0'], + batchesTotal: 2, + batchesSucceeded: 1, + batchesFailed: 1, + failures: [{ batchIndex: 1, count: 2, error: 'status 422 ...', uids: ['u2', 'u3'] }], + }); + + const writeStub = sandbox.stub(require('node:fs'), 'writeFileSync'); + + await command.run(); + + expect(process.exitCode).to.equal(1); + expect(writeStub.calledOnce).to.equal(true); + const [filePath, contents] = writeStub.firstCall.args; + expect(String(filePath)).to.match(/cs-assets-delete-failed-.*\.json$/); + expect(JSON.parse(contents as string)).to.deep.equal({ uids: ['u2', 'u3'] }); + }); }); describe('CS Assets path isolation — no publish/unpublish infrastructure', () => {