diff --git a/src/api/routes/datasets.ts b/src/api/routes/datasets.ts index 590f040..fa133b0 100644 --- a/src/api/routes/datasets.ts +++ b/src/api/routes/datasets.ts @@ -26,6 +26,12 @@ import type { ApiServerDeps } from '../server.js'; type ResolveDataset = (req: Request) => Promise; +/** + * What the platform reports as `x-apify-pagination-limit` when the caller gives no `limit`: every + * item from `offset` on is served, and the header carries apify-core's "almost infinity" stand-in. + */ +const PLATFORM_UNLIMITED_PAGE = 999_999_999_999; + /** Answers with a page of the dataset's items, honouring the items endpoint's query parameters. */ export async function sendDatasetItems( req: Request, @@ -58,7 +64,7 @@ export async function sendDatasetItems( 'x-apify-pagination-total': String(page.total), 'x-apify-pagination-offset': String(page.offset), 'x-apify-pagination-count': String(items.length), - 'x-apify-pagination-limit': String(page.limit ?? items.length), + 'x-apify-pagination-limit': String(limit ?? PLATFORM_UNLIMITED_PAGE), 'x-apify-pagination-desc': String(desc), }); res.status(status).json(items); diff --git a/test/integration/dataset-items-paging.test.ts b/test/integration/dataset-items-paging.test.ts new file mode 100644 index 0000000..4c0974a --- /dev/null +++ b/test/integration/dataset-items-paging.test.ts @@ -0,0 +1,48 @@ +/** + * Paging of `GET /v2/datasets/:datasetId/items`: a request without `limit` gets every item from + * `offset` on, as on the platform, and the `x-apify-pagination-*` headers describe the page. + */ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import axios from 'axios'; + +import { startTestServer, type TestServerHandle } from './helpers/test-server.js'; + +describe('dataset items paging', () => { + let server: TestServerHandle; + + beforeEach(async () => { + server = await startTestServer(); + }); + + afterEach(async () => { + await server.close(); + }); + + async function items(datasetId: string, query = '') { + return axios.get(`${server.baseUrl}/v2/datasets/${datasetId}/items${query}`, { + headers: { Authorization: `Bearer ${server.token}` }, + validateStatus: () => true, + }); + } + + it('serves every item when no limit is given, and a page by offset and limit', async () => { + const total = 300; + const dataset = await server.client.datasets().getOrCreate('paged'); + await server.client.dataset(dataset.id).pushItems(Array.from({ length: total }, (_, n) => ({ n }))); + + const all = await items(dataset.id); + expect(all.status).toBe(200); + expect(all.data).toHaveLength(total); + expect(all.headers['x-apify-pagination-total']).toBe(String(total)); + expect(all.headers['x-apify-pagination-count']).toBe(String(total)); + expect(all.headers['x-apify-pagination-limit']).toBe('999999999999'); + + const fromOffset = await items(dataset.id, '?offset=250'); + expect(fromOffset.data).toHaveLength(50); + expect(fromOffset.data[0]).toEqual({ n: 250 }); + + const page = await items(dataset.id, '?limit=5&offset=10'); + expect(page.data.map((item: { n: number }) => item.n)).toEqual([10, 11, 12, 13, 14]); + expect(page.headers['x-apify-pagination-limit']).toBe('5'); + }); +});