From 7c06575d0b9095963a34b4484d679e875674d4b2 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 15:39:04 +0000 Subject: [PATCH 1/5] feat(api): page dataset items by default A request to the items endpoint without a limit now returns the first 250 items; offset and limit reach the rest. --- src/api/routes/datasets.ts | 9 +++- test/integration/dataset-items-paging.test.ts | 49 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 test/integration/dataset-items-paging.test.ts diff --git a/src/api/routes/datasets.ts b/src/api/routes/datasets.ts index 590f040..2d380fb 100644 --- a/src/api/routes/datasets.ts +++ b/src/api/routes/datasets.ts @@ -26,6 +26,13 @@ import type { ApiServerDeps } from '../server.js'; type ResolveDataset = (req: Request) => Promise; +/** + * Items served when the caller gives no `limit`. A local dataset can hold millions of items, and a + * bare `get-items` from the CLI or an agent then floods a terminal; a page keeps the first look cheap + * while `offset`/`limit` still reach everything. + */ +export const DEFAULT_ITEMS_PAGE_LIMIT = 250; + /** Answers with a page of the dataset's items, honouring the items endpoint's query parameters. */ export async function sendDatasetItems( req: Request, @@ -37,7 +44,7 @@ export async function sendDatasetItems( const dataset = await openDataset(record.id); const offset = queryNumber(req, 'offset') ?? 0; - const limit = queryNumber(req, 'limit'); + const limit = queryNumber(req, 'limit') ?? DEFAULT_ITEMS_PAGE_LIMIT; const desc = queryBoolean(req, 'desc') ?? false; const page = await dataset.getData({ offset, limit, desc }); diff --git a/test/integration/dataset-items-paging.test.ts b/test/integration/dataset-items-paging.test.ts new file mode 100644 index 0000000..d1bc32d --- /dev/null +++ b/test/integration/dataset-items-paging.test.ts @@ -0,0 +1,49 @@ +/** + * Paging of `GET /v2/datasets/:datasetId/items`: a request without `limit` gets the default page, + * `offset`/`limit` reach the rest, and the `x-apify-pagination-*` headers describe the page. + */ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import axios from 'axios'; + +import { DEFAULT_ITEMS_PAGE_LIMIT } from '../../src/api/routes/datasets.js'; +import { startTestServer, type TestServerHandle } from './helpers/test-server.js'; + +describe('dataset items paging', () => { + let server: TestServerHandle; + + beforeEach(async () => { + server = await startTestServer(); + }); + + afterEach(async () => { + await server.close(); + }); + + async function items(datasetId: string, query = '') { + return axios.get(`${server.baseUrl}/v2/datasets/${datasetId}/items${query}`, { + headers: { Authorization: `Bearer ${server.token}` }, + validateStatus: () => true, + }); + } + + it('serves the default page when no limit is given, and the rest by offset', async () => { + const total = DEFAULT_ITEMS_PAGE_LIMIT + 50; + const dataset = await server.client.datasets().getOrCreate('paged'); + await server.client.dataset(dataset.id).pushItems(Array.from({ length: total }, (_, n) => ({ n }))); + + const first = await items(dataset.id); + expect(first.status).toBe(200); + expect(first.data).toHaveLength(DEFAULT_ITEMS_PAGE_LIMIT); + expect(first.data[0]).toEqual({ n: 0 }); + expect(first.headers['x-apify-pagination-total']).toBe(String(total)); + expect(first.headers['x-apify-pagination-count']).toBe(String(DEFAULT_ITEMS_PAGE_LIMIT)); + expect(first.headers['x-apify-pagination-limit']).toBe(String(DEFAULT_ITEMS_PAGE_LIMIT)); + + const rest = await items(dataset.id, `?offset=${DEFAULT_ITEMS_PAGE_LIMIT}`); + expect(rest.data).toHaveLength(50); + expect(rest.data[0]).toEqual({ n: DEFAULT_ITEMS_PAGE_LIMIT }); + + const explicit = await items(dataset.id, '?limit=5&offset=10'); + expect(explicit.data.map((item: { n: number }) => item.n)).toEqual([10, 11, 12, 13, 14]); + }); +}); From e37fee62023d58f6d61f68cd3a2fe4eff76de5d2 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 15:39:47 +0000 Subject: [PATCH 2/5] feat(api): make the dataset page size configurable ACTOR_RUNTIME_DATASET_PAGE_LIMIT sets how many items the items endpoint serves without a limit; 250 stays the default. --- src/api/routes/datasets.ts | 11 +++-------- src/config.ts | 14 ++++++++++++++ test/integration/dataset-items-paging.test.ts | 13 ++++++++++++- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/api/routes/datasets.ts b/src/api/routes/datasets.ts index 2d380fb..eee734c 100644 --- a/src/api/routes/datasets.ts +++ b/src/api/routes/datasets.ts @@ -26,13 +26,6 @@ import type { ApiServerDeps } from '../server.js'; type ResolveDataset = (req: Request) => Promise; -/** - * Items served when the caller gives no `limit`. A local dataset can hold millions of items, and a - * bare `get-items` from the CLI or an agent then floods a terminal; a page keeps the first look cheap - * while `offset`/`limit` still reach everything. - */ -export const DEFAULT_ITEMS_PAGE_LIMIT = 250; - /** Answers with a page of the dataset's items, honouring the items endpoint's query parameters. */ export async function sendDatasetItems( req: Request, @@ -44,7 +37,9 @@ export async function sendDatasetItems( const dataset = await openDataset(record.id); const offset = queryNumber(req, 'offset') ?? 0; - const limit = queryNumber(req, 'limit') ?? DEFAULT_ITEMS_PAGE_LIMIT; + // A local dataset can hold millions of items, and a bare `get-items` from the CLI or an agent then + // floods a terminal; a page keeps the first look cheap while `offset`/`limit` still reach everything. + const limit = queryNumber(req, 'limit') ?? datasetPageLimit(); const desc = queryBoolean(req, 'desc') ?? false; const page = await dataset.getData({ offset, limit, desc }); diff --git a/src/config.ts b/src/config.ts index ce72006..e9dcd9a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -76,3 +76,17 @@ export function browserViewerVersionFilePath(): string { export function skillFilePath(): string { return process.env.ACTOR_RUNTIME_SKILL_PATH ?? 'skills/actor-runtime/SKILL.md'; } + +/** Items the dataset items endpoint serves when the caller gives no `limit`. */ +export const DEFAULT_DATASET_PAGE_LIMIT = 250; + +/** Read per request (not cached) so tests can set `ACTOR_RUNTIME_DATASET_PAGE_LIMIT` on the fly. */ +export function datasetPageLimit(env: NodeJS.ProcessEnv = process.env): number { + const raw = env.ACTOR_RUNTIME_DATASET_PAGE_LIMIT; + if (raw === undefined || raw === '') return DEFAULT_DATASET_PAGE_LIMIT; + const parsed = Number(raw); + if (!Number.isInteger(parsed) || parsed < 1) { + throw new Error(`ACTOR_RUNTIME_DATASET_PAGE_LIMIT must be a positive integer, got ${JSON.stringify(raw)}.`); + } + return parsed; +} diff --git a/test/integration/dataset-items-paging.test.ts b/test/integration/dataset-items-paging.test.ts index d1bc32d..0485653 100644 --- a/test/integration/dataset-items-paging.test.ts +++ b/test/integration/dataset-items-paging.test.ts @@ -5,7 +5,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import axios from 'axios'; -import { DEFAULT_ITEMS_PAGE_LIMIT } from '../../src/api/routes/datasets.js'; +import { DEFAULT_DATASET_PAGE_LIMIT as DEFAULT_ITEMS_PAGE_LIMIT } from '../../src/config.js'; import { startTestServer, type TestServerHandle } from './helpers/test-server.js'; describe('dataset items paging', () => { @@ -16,6 +16,7 @@ describe('dataset items paging', () => { }); afterEach(async () => { + delete process.env.ACTOR_RUNTIME_DATASET_PAGE_LIMIT; await server.close(); }); @@ -46,4 +47,14 @@ describe('dataset items paging', () => { const explicit = await items(dataset.id, '?limit=5&offset=10'); expect(explicit.data.map((item: { n: number }) => item.n)).toEqual([10, 11, 12, 13, 14]); }); + + it('takes the page size from ACTOR_RUNTIME_DATASET_PAGE_LIMIT', async () => { + const dataset = await server.client.datasets().getOrCreate('configured'); + await server.client.dataset(dataset.id).pushItems(Array.from({ length: 10 }, (_, n) => ({ n }))); + process.env.ACTOR_RUNTIME_DATASET_PAGE_LIMIT = '3'; + + const page = await items(dataset.id); + expect(page.data.map((item: { n: number }) => item.n)).toEqual([0, 1, 2]); + expect(page.headers['x-apify-pagination-limit']).toBe('3'); + }); }); From 510051cd03343f95c927fda2dcc97fdf5083169d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 15:40:06 +0000 Subject: [PATCH 3/5] fix(api): import the dataset page limit --- src/api/routes/datasets.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/api/routes/datasets.ts b/src/api/routes/datasets.ts index eee734c..a53ccb5 100644 --- a/src/api/routes/datasets.ts +++ b/src/api/routes/datasets.ts @@ -23,6 +23,7 @@ import type { DatasetSchema, StorageRecord } from '../../storage/entities.js'; import { findInvalidDatasetItems } from '../../services/dataset-schema.js'; import { recordDefaultDatasetItems } from '../../services/charging.js'; import type { ApiServerDeps } from '../server.js'; +import { datasetPageLimit } from '../../config.js'; type ResolveDataset = (req: Request) => Promise; From a1d9146a42604f9c81cdca09eab2811673e23331 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 15:40:23 +0000 Subject: [PATCH 4/5] docs(skill): mention the default dataset page --- skills/actor-runtime/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/skills/actor-runtime/SKILL.md b/skills/actor-runtime/SKILL.md index f4e0fba..b9fe495 100644 --- a/skills/actor-runtime/SKILL.md +++ b/skills/actor-runtime/SKILL.md @@ -51,7 +51,7 @@ apify push # upload + build, once apify call --input '{"maxPages":3}' apify runs ls --json apify runs log -apify datasets get-items --format json +apify datasets get-items --format json # the first 250 items; --offset/--limit for the rest ``` ### Apple Silicon and other arm64 hosts From 9672f601e48be579851f232986729573796c81f9 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 27 Oct 2026 15:46:04 +0000 Subject: [PATCH 5/5] fix(api): serve every dataset item without a limit, as the platform does No default page: a request without a limit gets every item from the offset on, and x-apify-pagination-limit carries the platform's unlimited value. The page size knob is gone. --- skills/actor-runtime/SKILL.md | 2 +- src/api/routes/datasets.ts | 13 +++--- src/config.ts | 14 ------ test/integration/dataset-items-paging.test.ts | 44 +++++++------------ 4 files changed, 25 insertions(+), 48 deletions(-) diff --git a/skills/actor-runtime/SKILL.md b/skills/actor-runtime/SKILL.md index b9fe495..f4e0fba 100644 --- a/skills/actor-runtime/SKILL.md +++ b/skills/actor-runtime/SKILL.md @@ -51,7 +51,7 @@ apify push # upload + build, once apify call --input '{"maxPages":3}' apify runs ls --json apify runs log -apify datasets get-items --format json # the first 250 items; --offset/--limit for the rest +apify datasets get-items --format json ``` ### Apple Silicon and other arm64 hosts diff --git a/src/api/routes/datasets.ts b/src/api/routes/datasets.ts index a53ccb5..fa133b0 100644 --- a/src/api/routes/datasets.ts +++ b/src/api/routes/datasets.ts @@ -23,10 +23,15 @@ import type { DatasetSchema, StorageRecord } from '../../storage/entities.js'; import { findInvalidDatasetItems } from '../../services/dataset-schema.js'; import { recordDefaultDatasetItems } from '../../services/charging.js'; import type { ApiServerDeps } from '../server.js'; -import { datasetPageLimit } from '../../config.js'; type ResolveDataset = (req: Request) => Promise; +/** + * What the platform reports as `x-apify-pagination-limit` when the caller gives no `limit`: every + * item from `offset` on is served, and the header carries apify-core's "almost infinity" stand-in. + */ +const PLATFORM_UNLIMITED_PAGE = 999_999_999_999; + /** Answers with a page of the dataset's items, honouring the items endpoint's query parameters. */ export async function sendDatasetItems( req: Request, @@ -38,9 +43,7 @@ export async function sendDatasetItems( const dataset = await openDataset(record.id); const offset = queryNumber(req, 'offset') ?? 0; - // A local dataset can hold millions of items, and a bare `get-items` from the CLI or an agent then - // floods a terminal; a page keeps the first look cheap while `offset`/`limit` still reach everything. - const limit = queryNumber(req, 'limit') ?? datasetPageLimit(); + const limit = queryNumber(req, 'limit'); const desc = queryBoolean(req, 'desc') ?? false; const page = await dataset.getData({ offset, limit, desc }); @@ -61,7 +64,7 @@ export async function sendDatasetItems( 'x-apify-pagination-total': String(page.total), 'x-apify-pagination-offset': String(page.offset), 'x-apify-pagination-count': String(items.length), - 'x-apify-pagination-limit': String(page.limit ?? items.length), + 'x-apify-pagination-limit': String(limit ?? PLATFORM_UNLIMITED_PAGE), 'x-apify-pagination-desc': String(desc), }); res.status(status).json(items); diff --git a/src/config.ts b/src/config.ts index e9dcd9a..ce72006 100644 --- a/src/config.ts +++ b/src/config.ts @@ -76,17 +76,3 @@ export function browserViewerVersionFilePath(): string { export function skillFilePath(): string { return process.env.ACTOR_RUNTIME_SKILL_PATH ?? 'skills/actor-runtime/SKILL.md'; } - -/** Items the dataset items endpoint serves when the caller gives no `limit`. */ -export const DEFAULT_DATASET_PAGE_LIMIT = 250; - -/** Read per request (not cached) so tests can set `ACTOR_RUNTIME_DATASET_PAGE_LIMIT` on the fly. */ -export function datasetPageLimit(env: NodeJS.ProcessEnv = process.env): number { - const raw = env.ACTOR_RUNTIME_DATASET_PAGE_LIMIT; - if (raw === undefined || raw === '') return DEFAULT_DATASET_PAGE_LIMIT; - const parsed = Number(raw); - if (!Number.isInteger(parsed) || parsed < 1) { - throw new Error(`ACTOR_RUNTIME_DATASET_PAGE_LIMIT must be a positive integer, got ${JSON.stringify(raw)}.`); - } - return parsed; -} diff --git a/test/integration/dataset-items-paging.test.ts b/test/integration/dataset-items-paging.test.ts index 0485653..4c0974a 100644 --- a/test/integration/dataset-items-paging.test.ts +++ b/test/integration/dataset-items-paging.test.ts @@ -1,11 +1,10 @@ /** - * Paging of `GET /v2/datasets/:datasetId/items`: a request without `limit` gets the default page, - * `offset`/`limit` reach the rest, and the `x-apify-pagination-*` headers describe the page. + * Paging of `GET /v2/datasets/:datasetId/items`: a request without `limit` gets every item from + * `offset` on, as on the platform, and the `x-apify-pagination-*` headers describe the page. */ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import axios from 'axios'; -import { DEFAULT_DATASET_PAGE_LIMIT as DEFAULT_ITEMS_PAGE_LIMIT } from '../../src/config.js'; import { startTestServer, type TestServerHandle } from './helpers/test-server.js'; describe('dataset items paging', () => { @@ -16,7 +15,6 @@ describe('dataset items paging', () => { }); afterEach(async () => { - delete process.env.ACTOR_RUNTIME_DATASET_PAGE_LIMIT; await server.close(); }); @@ -27,34 +25,24 @@ describe('dataset items paging', () => { }); } - it('serves the default page when no limit is given, and the rest by offset', async () => { - const total = DEFAULT_ITEMS_PAGE_LIMIT + 50; + it('serves every item when no limit is given, and a page by offset and limit', async () => { + const total = 300; const dataset = await server.client.datasets().getOrCreate('paged'); await server.client.dataset(dataset.id).pushItems(Array.from({ length: total }, (_, n) => ({ n }))); - const first = await items(dataset.id); - expect(first.status).toBe(200); - expect(first.data).toHaveLength(DEFAULT_ITEMS_PAGE_LIMIT); - expect(first.data[0]).toEqual({ n: 0 }); - expect(first.headers['x-apify-pagination-total']).toBe(String(total)); - expect(first.headers['x-apify-pagination-count']).toBe(String(DEFAULT_ITEMS_PAGE_LIMIT)); - expect(first.headers['x-apify-pagination-limit']).toBe(String(DEFAULT_ITEMS_PAGE_LIMIT)); + const all = await items(dataset.id); + expect(all.status).toBe(200); + expect(all.data).toHaveLength(total); + expect(all.headers['x-apify-pagination-total']).toBe(String(total)); + expect(all.headers['x-apify-pagination-count']).toBe(String(total)); + expect(all.headers['x-apify-pagination-limit']).toBe('999999999999'); - const rest = await items(dataset.id, `?offset=${DEFAULT_ITEMS_PAGE_LIMIT}`); - expect(rest.data).toHaveLength(50); - expect(rest.data[0]).toEqual({ n: DEFAULT_ITEMS_PAGE_LIMIT }); + const fromOffset = await items(dataset.id, '?offset=250'); + expect(fromOffset.data).toHaveLength(50); + expect(fromOffset.data[0]).toEqual({ n: 250 }); - const explicit = await items(dataset.id, '?limit=5&offset=10'); - expect(explicit.data.map((item: { n: number }) => item.n)).toEqual([10, 11, 12, 13, 14]); - }); - - it('takes the page size from ACTOR_RUNTIME_DATASET_PAGE_LIMIT', async () => { - const dataset = await server.client.datasets().getOrCreate('configured'); - await server.client.dataset(dataset.id).pushItems(Array.from({ length: 10 }, (_, n) => ({ n }))); - process.env.ACTOR_RUNTIME_DATASET_PAGE_LIMIT = '3'; - - const page = await items(dataset.id); - expect(page.data.map((item: { n: number }) => item.n)).toEqual([0, 1, 2]); - expect(page.headers['x-apify-pagination-limit']).toBe('3'); + const page = await items(dataset.id, '?limit=5&offset=10'); + expect(page.data.map((item: { n: number }) => item.n)).toEqual([10, 11, 12, 13, 14]); + expect(page.headers['x-apify-pagination-limit']).toBe('5'); }); });