feat(api): page dataset items by default

src/api/routes/datasets.ts

@@ -26,6 +26,12 @@ import type { ApiServerDeps } from '../server.js';
 
 type ResolveDataset = (req: Request) => Promise<StorageRecord | null>;
 
+/**
+ * What the platform reports as `x-apify-pagination-limit` when the caller gives no `limit`: every
+ * item from `offset` on is served, and the header carries apify-core's "almost infinity" stand-in.
+ */
+const PLATFORM_UNLIMITED_PAGE = 999_999_999_999;
+
 /** Answers with a page of the dataset's items, honouring the items endpoint's query parameters. */
 export async function sendDatasetItems(
 	req: Request,
@@ -58,7 +64,7 @@ export async function sendDatasetItems(
 		'x-apify-pagination-total': String(page.total),
 		'x-apify-pagination-offset': String(page.offset),
 		'x-apify-pagination-count': String(items.length),
-		'x-apify-pagination-limit': String(page.limit ?? items.length),
+		'x-apify-pagination-limit': String(limit ?? PLATFORM_UNLIMITED_PAGE),
 		'x-apify-pagination-desc': String(desc),
 	});
 	res.status(status).json(items);

test/integration/dataset-items-paging.test.ts

@@ -0,0 +1,48 @@
+/**
+ * Paging of `GET /v2/datasets/:datasetId/items`: a request without `limit` gets every item from
+ * `offset` on, as on the platform, and the `x-apify-pagination-*` headers describe the page.
+ */
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import axios from 'axios';
+
+import { startTestServer, type TestServerHandle } from './helpers/test-server.js';
+
+describe('dataset items paging', () => {
+	let server: TestServerHandle;
+
+	beforeEach(async () => {
+		server = await startTestServer();
+	});
+
+	afterEach(async () => {
+		await server.close();
+	});
+
+	async function items(datasetId: string, query = '') {
+		return axios.get(`${server.baseUrl}/v2/datasets/${datasetId}/items${query}`, {
+			headers: { Authorization: `Bearer ${server.token}` },
+			validateStatus: () => true,
+		});
+	}
+
+	it('serves every item when no limit is given, and a page by offset and limit', async () => {
+		const total = 300;
+		const dataset = await server.client.datasets().getOrCreate('paged');
+		await server.client.dataset(dataset.id).pushItems(Array.from({ length: total }, (_, n) => ({ n })));
+
+		const all = await items(dataset.id);
+		expect(all.status).toBe(200);
+		expect(all.data).toHaveLength(total);
+		expect(all.headers['x-apify-pagination-total']).toBe(String(total));
+		expect(all.headers['x-apify-pagination-count']).toBe(String(total));
+		expect(all.headers['x-apify-pagination-limit']).toBe('999999999999');
+
+		const fromOffset = await items(dataset.id, '?offset=250');
+		expect(fromOffset.data).toHaveLength(50);
+		expect(fromOffset.data[0]).toEqual({ n: 250 });
+
+		const page = await items(dataset.id, '?limit=5&offset=10');
+		expect(page.data.map((item: { n: number }) => item.n)).toEqual([10, 11, 12, 13, 14]);
+		expect(page.headers['x-apify-pagination-limit']).toBe('5');
+	});
+});