From 2157699de21a826b5e746a161c8c7692042658a0 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 8 Sep 2026 18:24:58 +0200 Subject: [PATCH 1/5] fix: keep pagination iterators advancing past fully filtered pages --- src/base/api_client.ts | 20 ++-- src/resource_clients/dataset.ts | 11 ++- src/resource_clients/key_value_store.ts | 2 +- src/resource_clients/request_queue.ts | 5 +- src/utils.ts | 20 +++- test/pagination.test.ts | 123 ++++++++++++++++++++++++ 6 files changed, 164 insertions(+), 17 deletions(-) diff --git a/src/base/api_client.ts b/src/base/api_client.ts index 6e5ab88b2..f168b337a 100644 --- a/src/base/api_client.ts +++ b/src/base/api_client.ts @@ -1,7 +1,7 @@ import type { ApifyClient } from '../apify_client.js'; import type { HttpClient } from '../http_client.js'; import type { PaginatedResponse, PaginationOptions } from '../utils.js'; -import { toPath, toPathSegment } from '../utils.js'; +import { SCANNED_COUNT, toPath, toPathSegment } from '../utils.js'; /** @private */ export interface ApiClientOptions { @@ -112,17 +112,24 @@ export abstract class ApiClient { limit: minForLimitParam(options.limit, chunkSize), } as T); + // A page can scan more items than it returns (see `SCANNED_COUNT`). The next offset and the stop condition + // follow the scanned number: advancing by `items.length` would re-scan items the previous page already + // returned, and stopping at an empty page would end the iteration in front of items a filter hid. + const scannedItems = (page: R): number => + Math.max((page as { [SCANNED_COUNT]?: number })[SCANNED_COUNT] ?? 0, page.items.length); + async function* asyncGenerator() { let currentPage = await paginatedListPromise; yield* currentPage.items; const offset = options.offset ?? 0; const limit = Math.min(options.limit || currentPage.total, currentPage.total); - let currentOffset = offset + currentPage.items.length; - let remainingItems = Math.min(currentPage.total - offset, limit) - currentPage.items.length; + let pageScanned = scannedItems(currentPage); + let currentOffset = offset + pageScanned; + let remainingItems = Math.min(currentPage.total - offset, limit) - pageScanned; while ( - currentPage.items.length > 0 && // Continue only if at least some items were returned in the last page. + pageScanned > 0 && // Continue only if the last page scanned some items. remainingItems > 0 ) { const newOptions = { @@ -132,8 +139,9 @@ export abstract class ApiClient { } as T; currentPage = await getPaginatedList(newOptions); yield* currentPage.items; - currentOffset += currentPage.items.length; - remainingItems -= currentPage.items.length; + pageScanned = scannedItems(currentPage); + currentOffset += pageScanned; + remainingItems -= pageScanned; } } diff --git a/src/resource_clients/dataset.ts b/src/resource_clients/dataset.ts index ec5932ae6..fb5cba695 100644 --- a/src/resource_clients/dataset.ts +++ b/src/resource_clients/dataset.ts @@ -24,6 +24,7 @@ import { paginationOptionsShape, parseArgument, parseResponse, + SCANNED_COUNT, } from '../utils.js'; // A predicate, not a `z.object()` arm, which would walk and copy every key of every pushed item. @@ -408,15 +409,21 @@ export class DatasetClient< } private _createPaginationList(response: ApifyResponse, userProvidedDesc: boolean): PaginatedList { - return { + const page: PaginatedList = { items: response.data, total: Number(response.headers['x-apify-pagination-total']), offset: Number(response.headers['x-apify-pagination-offset']), - count: response.data.length, // because x-apify-pagination-count returns invalid values when hidden/empty items are skipped + // `x-apify-pagination-count` reports the items scanned, which a filter can leave above the items returned. + count: response.data.length, limit: Number(response.headers['x-apify-pagination-limit']), // API returns 999999999999 when no limit is used // TODO: Replace this once https://github.com/apify/apify-core/issues/3503 is solved desc: JSON.parse(response.headers['x-apify-pagination-desc'] ?? userProvidedDesc), }; + + // The offset iterator paginates by the scanned number, so it travels with the page outside its public shape. + return Object.defineProperty(page, SCANNED_COUNT, { + value: Number(response.headers['x-apify-pagination-count']) || 0, + }); } } diff --git a/src/resource_clients/key_value_store.ts b/src/resource_clients/key_value_store.ts index 7d8a46354..f63579bcc 100644 --- a/src/resource_clients/key_value_store.ts +++ b/src/resource_clients/key_value_store.ts @@ -210,8 +210,8 @@ export class KeyValueStoreClient extends ResourceClient { let remainingItems = parsed.limit ? parsed.limit - currentPage.items.length : undefined; + // An empty page alone does not end the listing; only a missing next key does. while ( - currentPage.items.length > 0 && // Continue only if at least some items were returned in the last page. // Continue only if there is some next key. The API answers with `null` on the last page and // omits the field entirely for a listing that was not truncated. currentPage.nextExclusiveStartKey != null && diff --git a/src/resource_clients/request_queue.ts b/src/resource_clients/request_queue.ts index 0a6ee352e..48bb87b35 100644 --- a/src/resource_clients/request_queue.ts +++ b/src/resource_clients/request_queue.ts @@ -739,10 +739,9 @@ export class RequestQueueClient extends ResourceClient { let remainingItems = parsed.limit ? parsed.limit - currentPage.items.length : undefined; - // RQ API response does not indicate whether there are more requests left, so we have to try and in case - // of exhausting all requests we get response with empty items which ends the loop. + // A server-side `filter` can leave a page empty while `nextCursor` still points at more requests, so an + // empty page alone does not end the listing. while ( - currentPage.items.length > 0 && // Continue only if at least some items were returned in the last page. currentPage.nextCursor && // Continue only if the API returned a cursor for the next page. (remainingItems === undefined || remainingItems > 0) // Continue only if the limit was not exceeded. ) { diff --git a/src/utils.ts b/src/utils.ts index 07466c8f0..51243f34a 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -325,12 +325,12 @@ export class RequestQueuePaginationIterator { cursor: nextCursor, exclusiveStartId: nextExclusiveStartId, }); - // There are no more pages to iterate - if (page.items.length === 0) return; - yield page; + // A server-side `filter` can leave a page empty while `nextCursor` still points at more requests, so an + // empty page is skipped rather than taken for the end. Only a missing cursor or the reached limit ends + // the iteration. + if (page.items.length > 0) yield page; iterateItemCount += page.items.length; - // Limit reached stopping to iterate - if ((this.limit && iterateItemCount >= this.limit) || !page.nextCursor) return; + if (!page.nextCursor || (this.limit && iterateItemCount >= this.limit)) return; nextCursor = page.nextCursor; nextExclusiveStartId = undefined; // see comment above - delete it for any page after the first one, and paginate with cursor @@ -414,6 +414,16 @@ export interface PaginatedList extends PaginatedResponse { desc: boolean; } +/** + * Key under which a page of dataset items carries the number of items the API scanned to produce it, as a + * non-enumerable property that stays out of the page's public shape. Dataset filters (`clean`, `skipEmpty`, + * `skipHidden`) drop items only after `offset` and `limit` have been applied, so a page can scan up to `limit` items + * yet return fewer, or none at all; the `x-apify-pagination-count` header reports the scanned number. The offset + * iterator advances and terminates by it, and falls back to `items.length` on a page without it. + * @internal + */ +export const SCANNED_COUNT = Symbol('scannedCount'); + /** * Type representing both a Promise of a paginated list and an async iterable. * diff --git a/test/pagination.test.ts b/test/pagination.test.ts index d6521eb26..5b2b89853 100644 --- a/test/pagination.test.ts +++ b/test/pagination.test.ts @@ -302,6 +302,59 @@ describe('DatasetClient.listItems as async iterable', () => { mockedClient.mockRestore(); } } as any); + + // Mimics the API on a dataset that keeps only the rows `isKept` accepts: `offset` and `limit` pick the scanned + // window over all rows first, and the filter then drops rows from what gets returned, while + // `x-apify-pagination-count` still reports the whole window. + const mockFilteredDataset = (totalRows: number, isKept: (id: number) => boolean) => + vi.spyOn(client.httpClient, 'call').mockImplementation((async (request: any) => { + const offset = request.params.offset ?? 0; + const scanned = range(offset, Math.min(offset + (request.params.limit || totalRows), totalRows)); + + return { + data: scanned.filter(({ id }) => isKept(Number(id))), + headers: { + 'x-apify-pagination-total': String(totalRows), + 'x-apify-pagination-offset': String(offset), + 'x-apify-pagination-count': String(scanned.length), + 'x-apify-pagination-limit': String(request.params.limit), + 'x-apify-pagination-desc': 'false', + }, + }; + }) as any); + + test('continues past a page whose items were all filtered out', async () => { + // The first chunk scans 1000 rows and returns none of them; the items behind it must still be yielded. + const mockedClient = mockFilteredDataset(2000, (id) => id >= 1000); + + try { + const items = []; + for await (const item of client.dataset('some-id').listItems({ clean: true, chunkSize: 1000 })) { + items.push(item); + } + expect(items).toEqual(range(1000, 2000)); + expect(mockedClient.mock.calls.map(([request]: any) => request.params.offset)).toEqual([undefined, 1000]); + } finally { + mockedClient.mockRestore(); + } + }); + + test('advances the offset by the items scanned rather than the items returned', async () => { + // Every chunk of two rows returns one. Advancing by `items.length` would make the next page re-scan, and + // re-yield, a row the previous page already returned. + const mockedClient = mockFilteredDataset(4, (id) => id % 2 === 0); + + try { + const items = []; + for await (const item of client.dataset('some-id').listItems({ clean: true, chunkSize: 2 })) { + items.push(item); + } + expect(items).toEqual([...range(0, 1), ...range(2, 3)]); + expect(mockedClient.mock.calls.map(([request]: any) => request.params.offset)).toEqual([undefined, 2]); + } finally { + mockedClient.mockRestore(); + } + }); }); describe('KeyValueStoreClient.listKeys as async iterable', () => { @@ -396,6 +449,36 @@ describe('KeyValueStoreClient.listKeys as async iterable', () => { mockedClient.mockRestore(); } }); + + test('continues past an empty page while nextExclusiveStartKey points at more keys', async () => { + const pages = new Map([ + [undefined, { items: [], count: 0, limit: maxItemsPerPage, isTruncated: true, nextExclusiveStartKey: 'k' }], + [ + 'k', + { + items: range(0, 2), + count: 2, + limit: maxItemsPerPage, + isTruncated: false, + nextExclusiveStartKey: null, + }, + ], + ]); + const mockedClient = vi.spyOn(client.httpClient, 'call').mockImplementation((async (request: any) => ({ + data: { data: pages.get(request.params.exclusiveStartKey) }, + })) as any); + + try { + const items = []; + for await (const item of client.keyValueStore('some-id').listKeys()) { + items.push(item); + } + expect(items).toEqual(range(0, 2)); + expect(mockedClient).toHaveBeenCalledTimes(2); + } finally { + mockedClient.mockRestore(); + } + }); }); describe('RequestQueueClient.listKeys as async iterable', () => { @@ -480,6 +563,46 @@ describe('RequestQueueClient.listKeys as async iterable', () => { mockedClient.mockRestore(); } } as any); + + // A `filter` can leave a whole page empty while `nextCursor` still points at more requests. + const filteredPages = new Map([ + [undefined, { items: [], limit: maxItemsPerPage, nextCursor: 'c' }], + ['c', { items: range(0, 2), limit: maxItemsPerPage }], + ]); + const mockFilteredQueue = () => + vi.spyOn(client.httpClient, 'call').mockImplementation((async (request: any) => ({ + data: { data: filteredPages.get(request.params.cursor) }, + })) as any); + + test('listRequests() continues past an empty page while nextCursor points at more requests', async () => { + const mockedClient = mockFilteredQueue(); + + try { + const items = []; + for await (const item of client.requestQueue('some-id').listRequests({ filter: ['pending'] })) { + items.push(item); + } + expect(items).toEqual(range(0, 2)); + expect(mockedClient).toHaveBeenCalledTimes(2); + } finally { + mockedClient.mockRestore(); + } + }); + + test('paginateRequests() follows nextCursor past an empty page without yielding it', async () => { + const mockedClient = mockFilteredQueue(); + + try { + const pages = []; + for await (const page of client.requestQueue('some-id').paginateRequests({ filter: ['pending'] })) { + pages.push(page.items); + } + expect(pages).toEqual([range(0, 2)]); + expect(mockedClient).toHaveBeenCalledTimes(2); + } finally { + mockedClient.mockRestore(); + } + }); }); test('chunkSize sizes each request without being sent as a query parameter', async () => { From 8ed789e4995e48b51d1d87e3d35a8f5e1fc4b9f6 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 10 Sep 2026 13:25:19 +0200 Subject: [PATCH 2/5] fix: paginate dataset items by the scanned count alone --- src/base/api_client.ts | 17 +++++++++-------- src/resource_clients/dataset.ts | 3 ++- src/utils.ts | 11 ++++++----- test/pagination.test.ts | 33 ++++++++++++++++++++++++++------- 4 files changed, 43 insertions(+), 21 deletions(-) diff --git a/src/base/api_client.ts b/src/base/api_client.ts index f168b337a..190e9435b 100644 --- a/src/base/api_client.ts +++ b/src/base/api_client.ts @@ -112,11 +112,12 @@ export abstract class ApiClient { limit: minForLimitParam(options.limit, chunkSize), } as T); - // A page can scan more items than it returns (see `SCANNED_COUNT`). The next offset and the stop condition - // follow the scanned number: advancing by `items.length` would re-scan items the previous page already - // returned, and stopping at an empty page would end the iteration in front of items a filter hid. - const scannedItems = (page: R): number => - Math.max((page as { [SCANNED_COUNT]?: number })[SCANNED_COUNT] ?? 0, page.items.length); + // A page can return more or fewer items than the rows it scanned (see `SCANNED_COUNT`). The next offset and + // the stop condition follow the scanned number alone: advancing by `items.length` would re-scan rows after a + // filter dropped some and skip rows after `unwind` multiplied them, and stopping at an empty page would end + // the iteration in front of rows a filter hid. + const scannedRows = (page: R): number => + (page as { [SCANNED_COUNT]?: number })[SCANNED_COUNT] ?? page.items.length; async function* asyncGenerator() { let currentPage = await paginatedListPromise; @@ -124,12 +125,12 @@ export abstract class ApiClient { const offset = options.offset ?? 0; const limit = Math.min(options.limit || currentPage.total, currentPage.total); - let pageScanned = scannedItems(currentPage); + let pageScanned = scannedRows(currentPage); let currentOffset = offset + pageScanned; let remainingItems = Math.min(currentPage.total - offset, limit) - pageScanned; while ( - pageScanned > 0 && // Continue only if the last page scanned some items. + pageScanned > 0 && // Continue only if the last page scanned some rows. remainingItems > 0 ) { const newOptions = { @@ -139,7 +140,7 @@ export abstract class ApiClient { } as T; currentPage = await getPaginatedList(newOptions); yield* currentPage.items; - pageScanned = scannedItems(currentPage); + pageScanned = scannedRows(currentPage); currentOffset += pageScanned; remainingItems -= pageScanned; } diff --git a/src/resource_clients/dataset.ts b/src/resource_clients/dataset.ts index fb5cba695..862e858a7 100644 --- a/src/resource_clients/dataset.ts +++ b/src/resource_clients/dataset.ts @@ -413,7 +413,8 @@ export class DatasetClient< items: response.data, total: Number(response.headers['x-apify-pagination-total']), offset: Number(response.headers['x-apify-pagination-offset']), - // `x-apify-pagination-count` reports the items scanned, which a filter can leave above the items returned. + // `x-apify-pagination-count` reports the rows scanned, which filters and `unwind` can leave above or below + // the items returned. count: response.data.length, limit: Number(response.headers['x-apify-pagination-limit']), // API returns 999999999999 when no limit is used // TODO: Replace this once https://github.com/apify/apify-core/issues/3503 is solved diff --git a/src/utils.ts b/src/utils.ts index 51243f34a..1a2d7f564 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -415,11 +415,12 @@ export interface PaginatedList extends PaginatedResponse { } /** - * Key under which a page of dataset items carries the number of items the API scanned to produce it, as a - * non-enumerable property that stays out of the page's public shape. Dataset filters (`clean`, `skipEmpty`, - * `skipHidden`) drop items only after `offset` and `limit` have been applied, so a page can scan up to `limit` items - * yet return fewer, or none at all; the `x-apify-pagination-count` header reports the scanned number. The offset - * iterator advances and terminates by it, and falls back to `items.length` on a page without it. + * Key under which a page of dataset items carries the number of rows the API scanned to produce it, as a + * non-enumerable property that stays out of the page's public shape. The API applies `offset` and `limit` to the + * dataset's rows first and shapes the result afterwards: filters (`clean`, `skipEmpty`, `skipHidden`) drop items and + * `unwind` splits or drops them, so `items.length` can land on either side of the rows scanned. The + * `x-apify-pagination-count` header reports the scanned number, and the offset iterator advances and terminates by it + * alone. A page without the property falls back to `items.length`. * @internal */ export const SCANNED_COUNT = Symbol('scannedCount'); diff --git a/test/pagination.test.ts b/test/pagination.test.ts index 5b2b89853..5a476da6e 100644 --- a/test/pagination.test.ts +++ b/test/pagination.test.ts @@ -303,16 +303,16 @@ describe('DatasetClient.listItems as async iterable', () => { } } as any); - // Mimics the API on a dataset that keeps only the rows `isKept` accepts: `offset` and `limit` pick the scanned - // window over all rows first, and the filter then drops rows from what gets returned, while + // Mimics the API on a dataset whose rows are reshaped before they are returned: `offset` and `limit` pick the + // scanned window over all rows first, `transform` (a filter, an `unwind`) then decides what gets returned, and // `x-apify-pagination-count` still reports the whole window. - const mockFilteredDataset = (totalRows: number, isKept: (id: number) => boolean) => + const mockTransformedDataset = (totalRows: number, transform: (rows: ReturnType) => unknown[]) => vi.spyOn(client.httpClient, 'call').mockImplementation((async (request: any) => { const offset = request.params.offset ?? 0; const scanned = range(offset, Math.min(offset + (request.params.limit || totalRows), totalRows)); return { - data: scanned.filter(({ id }) => isKept(Number(id))), + data: transform(scanned), headers: { 'x-apify-pagination-total': String(totalRows), 'x-apify-pagination-offset': String(offset), @@ -325,7 +325,7 @@ describe('DatasetClient.listItems as async iterable', () => { test('continues past a page whose items were all filtered out', async () => { // The first chunk scans 1000 rows and returns none of them; the items behind it must still be yielded. - const mockedClient = mockFilteredDataset(2000, (id) => id >= 1000); + const mockedClient = mockTransformedDataset(2000, (rows) => rows.filter(({ id }) => Number(id) >= 1000)); try { const items = []; @@ -339,10 +339,10 @@ describe('DatasetClient.listItems as async iterable', () => { } }); - test('advances the offset by the items scanned rather than the items returned', async () => { + test('advances the offset by the rows scanned rather than the items returned', async () => { // Every chunk of two rows returns one. Advancing by `items.length` would make the next page re-scan, and // re-yield, a row the previous page already returned. - const mockedClient = mockFilteredDataset(4, (id) => id % 2 === 0); + const mockedClient = mockTransformedDataset(4, (rows) => rows.filter(({ id }) => Number(id) % 2 === 0)); try { const items = []; @@ -355,6 +355,25 @@ describe('DatasetClient.listItems as async iterable', () => { mockedClient.mockRestore(); } }); + + test('advances the offset by the rows scanned when unwind returns more items than rows', async () => { + // Every row unwinds into three items. Advancing by `items.length` would push the offset past rows the next + // page has not scanned yet, and skip them. + const unwind = (rows: ReturnType) => + rows.flatMap((row) => ['a', 'b', 'c'].map((part) => ({ ...row, part }))); + const mockedClient = mockTransformedDataset(4, unwind); + + try { + const items = []; + for await (const item of client.dataset('some-id').listItems({ unwind: 'parts', chunkSize: 2 })) { + items.push(item); + } + expect(items).toEqual(unwind(range(0, 4))); + expect(mockedClient.mock.calls.map(([request]: any) => request.params.offset)).toEqual([undefined, 2]); + } finally { + mockedClient.mockRestore(); + } + }); }); describe('KeyValueStoreClient.listKeys as async iterable', () => { From ddf0a17bdad6b78f466934028b1e81f19489922b Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 10 Sep 2026 13:25:19 +0200 Subject: [PATCH 3/5] refactor: let the cursor iterators stop on an empty page again --- src/resource_clients/key_value_store.ts | 2 +- src/resource_clients/request_queue.ts | 5 +- src/utils.ts | 10 ++-- test/pagination.test.ts | 70 ------------------------- 4 files changed, 9 insertions(+), 78 deletions(-) diff --git a/src/resource_clients/key_value_store.ts b/src/resource_clients/key_value_store.ts index f63579bcc..7d8a46354 100644 --- a/src/resource_clients/key_value_store.ts +++ b/src/resource_clients/key_value_store.ts @@ -210,8 +210,8 @@ export class KeyValueStoreClient extends ResourceClient { let remainingItems = parsed.limit ? parsed.limit - currentPage.items.length : undefined; - // An empty page alone does not end the listing; only a missing next key does. while ( + currentPage.items.length > 0 && // Continue only if at least some items were returned in the last page. // Continue only if there is some next key. The API answers with `null` on the last page and // omits the field entirely for a listing that was not truncated. currentPage.nextExclusiveStartKey != null && diff --git a/src/resource_clients/request_queue.ts b/src/resource_clients/request_queue.ts index 48bb87b35..0a6ee352e 100644 --- a/src/resource_clients/request_queue.ts +++ b/src/resource_clients/request_queue.ts @@ -739,9 +739,10 @@ export class RequestQueueClient extends ResourceClient { let remainingItems = parsed.limit ? parsed.limit - currentPage.items.length : undefined; - // A server-side `filter` can leave a page empty while `nextCursor` still points at more requests, so an - // empty page alone does not end the listing. + // RQ API response does not indicate whether there are more requests left, so we have to try and in case + // of exhausting all requests we get response with empty items which ends the loop. while ( + currentPage.items.length > 0 && // Continue only if at least some items were returned in the last page. currentPage.nextCursor && // Continue only if the API returned a cursor for the next page. (remainingItems === undefined || remainingItems > 0) // Continue only if the limit was not exceeded. ) { diff --git a/src/utils.ts b/src/utils.ts index 1a2d7f564..60cf0c371 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -325,12 +325,12 @@ export class RequestQueuePaginationIterator { cursor: nextCursor, exclusiveStartId: nextExclusiveStartId, }); - // A server-side `filter` can leave a page empty while `nextCursor` still points at more requests, so an - // empty page is skipped rather than taken for the end. Only a missing cursor or the reached limit ends - // the iteration. - if (page.items.length > 0) yield page; + // There are no more pages to iterate + if (page.items.length === 0) return; + yield page; iterateItemCount += page.items.length; - if (!page.nextCursor || (this.limit && iterateItemCount >= this.limit)) return; + // Limit reached stopping to iterate + if ((this.limit && iterateItemCount >= this.limit) || !page.nextCursor) return; nextCursor = page.nextCursor; nextExclusiveStartId = undefined; // see comment above - delete it for any page after the first one, and paginate with cursor diff --git a/test/pagination.test.ts b/test/pagination.test.ts index 5a476da6e..7843cdbc0 100644 --- a/test/pagination.test.ts +++ b/test/pagination.test.ts @@ -468,36 +468,6 @@ describe('KeyValueStoreClient.listKeys as async iterable', () => { mockedClient.mockRestore(); } }); - - test('continues past an empty page while nextExclusiveStartKey points at more keys', async () => { - const pages = new Map([ - [undefined, { items: [], count: 0, limit: maxItemsPerPage, isTruncated: true, nextExclusiveStartKey: 'k' }], - [ - 'k', - { - items: range(0, 2), - count: 2, - limit: maxItemsPerPage, - isTruncated: false, - nextExclusiveStartKey: null, - }, - ], - ]); - const mockedClient = vi.spyOn(client.httpClient, 'call').mockImplementation((async (request: any) => ({ - data: { data: pages.get(request.params.exclusiveStartKey) }, - })) as any); - - try { - const items = []; - for await (const item of client.keyValueStore('some-id').listKeys()) { - items.push(item); - } - expect(items).toEqual(range(0, 2)); - expect(mockedClient).toHaveBeenCalledTimes(2); - } finally { - mockedClient.mockRestore(); - } - }); }); describe('RequestQueueClient.listKeys as async iterable', () => { @@ -582,46 +552,6 @@ describe('RequestQueueClient.listKeys as async iterable', () => { mockedClient.mockRestore(); } } as any); - - // A `filter` can leave a whole page empty while `nextCursor` still points at more requests. - const filteredPages = new Map([ - [undefined, { items: [], limit: maxItemsPerPage, nextCursor: 'c' }], - ['c', { items: range(0, 2), limit: maxItemsPerPage }], - ]); - const mockFilteredQueue = () => - vi.spyOn(client.httpClient, 'call').mockImplementation((async (request: any) => ({ - data: { data: filteredPages.get(request.params.cursor) }, - })) as any); - - test('listRequests() continues past an empty page while nextCursor points at more requests', async () => { - const mockedClient = mockFilteredQueue(); - - try { - const items = []; - for await (const item of client.requestQueue('some-id').listRequests({ filter: ['pending'] })) { - items.push(item); - } - expect(items).toEqual(range(0, 2)); - expect(mockedClient).toHaveBeenCalledTimes(2); - } finally { - mockedClient.mockRestore(); - } - }); - - test('paginateRequests() follows nextCursor past an empty page without yielding it', async () => { - const mockedClient = mockFilteredQueue(); - - try { - const pages = []; - for await (const page of client.requestQueue('some-id').paginateRequests({ filter: ['pending'] })) { - pages.push(page.items); - } - expect(pages).toEqual([range(0, 2)]); - expect(mockedClient).toHaveBeenCalledTimes(2); - } finally { - mockedClient.mockRestore(); - } - }); }); test('chunkSize sizes each request without being sent as a query parameter', async () => { From 1b2f8204239981d5df4cbb3b406d51666b19072d Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 10 Sep 2026 13:48:24 +0200 Subject: [PATCH 4/5] test: clarify what the pagination count header reports --- test/datasets.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/datasets.test.ts b/test/datasets.test.ts index 8d1631618..f355ed52f 100644 --- a/test/datasets.test.ts +++ b/test/datasets.test.ts @@ -150,7 +150,7 @@ describe('Dataset methods', () => { 'content-type': 'application/json; chartset=utf-8', 'x-apify-pagination-total': '0', 'x-apify-pagination-offset': '0', - 'x-apify-pagination-count': '1', // wrong on purpose to check that it's not used + 'x-apify-pagination-count': '1', // the scanned rows; `count` reports the items returned, so it stays 0 'x-apify-pagination-limit': '100000', // TODO: https://github.com/apify/apify-core/issues/3503 'x-apify-pagination-desc': false, From 305e806177529ac9abf540ef2fc22e9b59285a96 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 11 Sep 2026 08:56:37 +0200 Subject: [PATCH 5/5] docs: cover the scanned-row limit semantics in the v3 upgrading guide --- docs/04_upgrading/upgrading_v3.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/04_upgrading/upgrading_v3.md b/docs/04_upgrading/upgrading_v3.md index f58f58b2f..f4ed5982b 100644 --- a/docs/04_upgrading/upgrading_v3.md +++ b/docs/04_upgrading/upgrading_v3.md @@ -272,6 +272,22 @@ A URL field whose value isn't a valid absolute URL now fails response validation `ActorVersionCollectionClient.list()` and `ActorEnvVarCollectionClient.list()` now take no arguments. Neither endpoint reads `offset`, `limit` or `desc`, and both return every item in one response, so `chunkSize` had nothing to size either. The `ActorVersionCollectionListOptions` and `ActorEnvVarCollectionListOptions` types that declared those four options, deprecated since v2.21.0, are gone from the package. A call that passed an options object no longer compiles. Drop the argument and the call returns the same items as before. +## `limit` counts scanned rows when iterating `listItems()` + +Iterating `DatasetClient.listItems()` with `for await` pages through the dataset by `offset` and `limit`. The API applies both to the dataset's rows before `clean`, `skipEmpty`, `skipHidden` and `unwind` reshape them, so a page can return fewer items than the rows it covered, or more. v2 advanced the offset and counted down `limit` by the items each page returned. A filter made the next page re-read rows the previous one had covered, `unwind` made it skip rows, and a page that came back empty ended the loop even when rows remained behind it. + +v3 advances and stops by the number of rows the API scanned, taken from the `x-apify-pagination-count` header. `limit` therefore caps the rows scanned, the same way it does on a single awaited `listItems()` call and in the API itself. With a filter set, the loop can yield fewer than `limit` items, each of them once. In v2 the same call made up the difference with items it had already yielded, or stopped early. With `unwind`, the loop can yield more than `limit` items. To collect a fixed number of items from a filtered dataset, count them in the loop: + +```js +const items = []; +for await (const item of client.dataset('dataset-id').listItems({ clean: true, chunkSize: 100 })) { + items.push(item); + if (items.length === 100) break; +} +``` + +`PaginatedList.count` keeps reporting the items returned on the page. The cursor-based iterators, `listKeys()`, `listRequests()` and `paginateRequests()`, are unchanged. + ## The last deprecated options are gone Two options that carried a `@deprecated` marker throughout v2 have been removed.