diff --git a/packages/cli/AGENTS.md b/packages/cli/AGENTS.md index 7500a0b74..b4a94d28f 100644 --- a/packages/cli/AGENTS.md +++ b/packages/cli/AGENTS.md @@ -466,14 +466,25 @@ const { data, nextCursor } = await paginate( Pass `perPage` to the request unchanged. Sentry's page-number paginators (`OffsetPaginator`, used by projects, teams, members, repositories, and -dashboard revisions) encode the page size in the cursor, so a cursor is only -valid with the `per_page` that produced it. `paginate()` therefore returns -opaque CLI cursors (`cli1~~~`) that record the -walk's page size and how many rows of the current page were already returned. -A later `-c next` reuses that page size even when `--limit` changes, and -resumes mid-page by refetching the page and skipping returned rows. Always -pass these cursors back through `paginate()`; never send them to an API -request directly. Test pagination changes against the paginator models in +dashboard revisions) encode the page size in the cursor +(`::`), so a cursor is only valid with the +`per_page` that produced it. `paginate()` reads that page size back from the +cursor, so a later `-c next` reuses it even when `--limit` changes. A fresh +walk splits `--limit` evenly over the fewest requests (201 → 3 × 67). + +The cursors `paginate()` returns are Sentry cursors, so every CLI version and +the API accept them, and stored `-c next` history works across upgrades and +downgrades. When a call stops inside a page, the cursor points at that page +and records how many of its rows were already returned in `is_prev`, in +binary with `_0` for 1 and `0` for 0 after a leading `0` (`84:2:0_00_000_00` +skips 82 = `0b1010010` rows). Sentry parses `is_prev` with Python's `int()`, +which reads any zeros joined by single underscores as 0, so it is still a +next-page cursor. `encodeSkip()` in `src/lib/api/infrastructure.ts` documents +the format. +`paginate()` refetches the page and skips them; anything else that sends the +cursor (with the page size it encodes, if any) gets those rows again, never +fewer. Keep returned cursors valid Sentry +cursors. Test pagination changes against the paginator models in `test/lib/paginate.property.test.ts`. Reference template: `trace/list.ts`, `span/list.ts`, `dashboard/revisions.ts` diff --git a/packages/cli/src/lib/api/infrastructure.ts b/packages/cli/src/lib/api/infrastructure.ts index 3d13761f2..1699031c0 100644 --- a/packages/cli/src/lib/api/infrastructure.ts +++ b/packages/cli/src/lib/api/infrastructure.ts @@ -25,12 +25,7 @@ import { isEnvTokenActive, } from "../db/auth.js"; import { getEnv } from "../env.js"; -import { - ApiError, - AuthError, - stringifyUnknown, - ValidationError, -} from "../errors.js"; +import { ApiError, AuthError, stringifyUnknown } from "../errors.js"; import { logger } from "../logger.js"; import { resolveOrgRegion } from "../region.js"; import { @@ -482,8 +477,24 @@ export async function autoPaginate( return { data: allRows.slice(0, limit) }; } -/** Marks cursors produced by {@link paginate}; Sentry cursors never use it. */ -const PAGE_CURSOR_PREFIX = "cli1~"; +/** + * A Sentry next-page cursor, `::`, split before + * `is_prev`, which may carry a skip count ({@link encodeSkip}). + */ +const NEXT_CURSOR_RE = /^(.*:\d+):(0(?:_?0)*)$/; + +/** Resumes at the first row for every paginator the CLI uses. */ +const FIRST_PAGE_CURSOR = "0:0:0"; + +/** `OffsetPaginator` cursors are `::`. */ +const PAGE_NUMBER_CURSOR_RE = /^(\d+):\d+:[01]$/; + +/** + * Prefix of `cli1~~~` cursors, which nightly + * builds returned. Read so their saved history still resumes; never written, + * because older CLIs send cursors to the API unchanged. + */ +const CLI_CURSOR_PREFIX = "cli1~"; /** * Where a {@link paginate} walk resumes: the server cursor that fetches the @@ -496,22 +507,73 @@ type PagePosition = { skip: number; }; -function encodePageCursor({ cursor, perPage, skip }: PagePosition): string { - return `${PAGE_CURSOR_PREFIX}${perPage}~${skip}~${cursor ?? ""}`; +/** + * Write how many rows of a page were already returned into a cursor's + * `is_prev` field, so the cursor stays a valid Sentry cursor. + * + * Sentry parses `is_prev` with Python's `int()` (`Cursor.from_string` in + * `src/sentry/utils/cursors.py`). `int()` ignores leading zeros and allows a + * single `_` between digits (PEP 515), so any string of `0`s joined by + * single `_`s is 0: "next page". The skip count is written in binary in that + * alphabet: a leading `0`, then `_0` for each 1 bit and `0` for each 0 bit, + * most significant first. 82 is `1010010`, so `0` + `_0 0 _0 0 0 _0 0` + * gives `0_00_000_00`. + * + * The API and older CLIs read the cursor as the page's own cursor and return + * the page from its first row; {@link decodeSkip} recovers the count. + * + * @param skip - Rows of the page already returned; must be at least 1 + */ +function encodeSkip(skip: number): string { + return `0${skip.toString(2).replaceAll("1", "_0")}`; +} + +/** Read the skip count {@link encodeSkip} wrote; plain `0` is no skip. */ +function decodeSkip(isPrev: string): number { + return Number.parseInt(isPrev.slice(1).replaceAll("_0", "1"), 2) || 0; } /** - * Decode a cursor from {@link paginate}, or treat anything else as a raw - * server cursor (older stored cursors, or one pasted from an API response). + * Encode where a walk resumes as a Sentry cursor. A server cursor that is not + * a Sentry next-page cursor cannot carry the skip, so it is returned as is and + * resuming returns that page's rows again rather than missing any. + */ +function encodePageCursor(cursor: string | undefined, skip: number): string { + const server = cursor ?? FIRST_PAGE_CURSOR; + const next = skip > 0 ? NEXT_CURSOR_RE.exec(server) : null; + return next ? `${next[1]}:${encodeSkip(skip)}` : server; +} + +/** + * Decode a cursor from {@link paginate}, an older CLI, or an API response. + * + * A damaged `cli1~` cursor is ignored with a warning and the walk starts from + * the first page, so it never fails a command or reaches the API. */ function decodePageCursor( raw: string | undefined, ): Partial & { cursor: string | undefined } { - if (!raw?.startsWith(PAGE_CURSOR_PREFIX)) { - return { cursor: raw }; + if (!raw) { + return { cursor: undefined }; } + if (raw.startsWith(CLI_CURSOR_PREFIX)) { + return decodeCliCursor(raw); + } + const next = NEXT_CURSOR_RE.exec(raw); + const cursor = next ? `${next[1]}:0` : raw; + const position = { + cursor: cursor === FIRST_PAGE_CURSOR ? undefined : cursor, + skip: next?.[2] ? decodeSkip(next[2]) : 0, + }; + const perPage = serverCursorPageSize(cursor); + return perPage === undefined ? position : { ...position, perPage }; +} + +function decodeCliCursor( + raw: string, +): Partial & { cursor: string | undefined } { const [perPage, skip, ...server] = raw - .slice(PAGE_CURSOR_PREFIX.length) + .slice(CLI_CURSOR_PREFIX.length) .split("~"); const position = { perPage: Number(perPage), @@ -519,15 +581,45 @@ function decodePageCursor( cursor: server.join("~") || undefined, }; if ( - !Number.isInteger(position.perPage) || - position.perPage < 1 || - position.perPage > API_MAX_PER_PAGE || - !Number.isInteger(position.skip) || - position.skip < 0 + Number.isInteger(position.perPage) && + position.perPage >= 1 && + position.perPage <= API_MAX_PER_PAGE && + Number.isInteger(position.skip) && + position.skip >= 0 ) { - throw new ValidationError(`Invalid pagination cursor '${raw}'.`, "cursor"); + return position; + } + logger.warn( + "Ignoring an unreadable pagination cursor; starting from the first page.", + ); + return { cursor: undefined }; +} + +/** + * Page size a server cursor must be resumed with, if it carries one. + * + * `OffsetPaginator` cursors are only valid with the page size that produced + * them. Other paginators put an offset of 0 or a sort value there; a sort + * value that happens to be in range only changes the page size, which they + * tolerate. + */ +function serverCursorPageSize(cursor: string): number | undefined { + const value = Number(PAGE_NUMBER_CURSOR_RE.exec(cursor)?.[1]); + return Number.isInteger(value) && value >= 1 && value <= API_MAX_PER_PAGE + ? value + : undefined; +} + +/** + * Page size for a walk that starts fresh: the fewest requests that can hold + * `limit` rows, split as evenly as possible so the last request is not a + * full page for a few rows (201 rows → 3 pages of 67, not 100 + 100 + 100). + */ +function pageSizeForLimit(limit: number): number { + if (limit <= API_MAX_PER_PAGE) { + return Math.max(1, limit); } - return position; + return Math.ceil(limit / Math.ceil(limit / API_MAX_PER_PAGE)); } /** @@ -537,20 +629,23 @@ function decodePageCursor( * `sentry-cli/no-manual-pagination` lint rule rejects hand-written loops. It * owns the decisions those loops kept getting wrong (#1458, #1608): * - * - Every request of a walk, including later `-c next` calls with a different - * `--limit`, uses the `per_page` of its first request. Sentry's page-number - * paginators (`OffsetPaginator`, used by projects, teams, members, - * repositories, and dashboard revisions) put the page size in the cursor - * (`::`), and pairing it with another `per_page` - * reads the wrong window and silently skips rows. + * - A fresh walk splits `limit` evenly over the fewest requests + * ({@link pageSizeForLimit}). Resuming a page-number cursor + * (`OffsetPaginator`, used by projects, teams, members, repositories, and + * dashboard revisions) reuses the page size the cursor encodes + * (`::`), whatever the new `limit`: pairing it with + * another `per_page` reads the wrong window and silently skips rows. * - The returned cursor always resumes at the first row the caller has not * received. When a call stops inside a page (a smaller `--limit`, or a * server that ignored `per_page`), the cursor refetches that page and skips * the rows already returned. * - * Returned cursors are opaque CLI cursors; pass them back through this - * function, never directly to an API request. Raw server cursors are accepted - * as a starting point. + * Returned cursors are Sentry cursors, so older CLI versions and the API + * accept them too; mid-page cursors record the rows to skip in `is_prev` + * ({@link encodeSkip}), which a caller that does not know about them sees + * again rather than misses (given the page size a page-number + * cursor encodes). Pass cursors back through this function + * to resume exactly. Raw server cursors are accepted as a starting point. * * Callers own region resolution and query building inside `fetchPage`, and * must send `perPage` as the request's `per_page` unchanged. @@ -570,10 +665,10 @@ export async function paginate( ): Promise> { const limit = options.limit ?? defaultLimit; const start = decodePageCursor(options.cursor); - const perPage = - start.perPage ?? Math.max(1, Math.min(limit, API_MAX_PER_PAGE)); let cursor = start.cursor; let skip = start.skip ?? 0; + // Cursors without a page size accept any, so fetch the skipped rows too. + const perPage = start.perPage ?? pageSizeForLimit(limit + skip); const rows: T[] = []; for (let page = 0; page < MAX_PAGINATION_PAGES; page += 1) { @@ -585,11 +680,7 @@ export async function paginate( rows.push(...unseen.slice(0, remaining)); return { data: rows, - nextCursor: encodePageCursor({ - cursor, - perPage, - skip: skip + remaining, - }), + nextCursor: encodePageCursor(cursor, skip + remaining), }; } rows.push(...unseen); @@ -597,7 +688,8 @@ export async function paginate( return { data: rows }; } cursor = nextCursor; - skip = 0; + // A resumed walk can fetch pages smaller than the rows it must skip. + skip = Math.max(0, skip - data.length); if (rows.length === limit) { break; } @@ -611,7 +703,7 @@ export async function paginate( } return { data: rows, - nextCursor: encodePageCursor({ cursor, perPage, skip: 0 }), + nextCursor: encodePageCursor(cursor, skip), }; } diff --git a/packages/cli/test/lib/api/infrastructure.test.ts b/packages/cli/test/lib/api/infrastructure.test.ts index 6f84d1a3d..8e187c099 100644 --- a/packages/cli/test/lib/api/infrastructure.test.ts +++ b/packages/cli/test/lib/api/infrastructure.test.ts @@ -13,11 +13,7 @@ import { unwrapResult, } from "../../../src/lib/api/infrastructure.js"; import { setAuthToken } from "../../../src/lib/db/auth.js"; -import { - ApiError, - HostScopeError, - ValidationError, -} from "../../../src/lib/errors.js"; +import { ApiError, HostScopeError } from "../../../src/lib/errors.js"; import { mockFetch, useEnvSandbox, useTestConfigDir } from "../../helpers.js"; describe("SDK error compatibility", () => { @@ -878,6 +874,7 @@ describe("paginate", () => { expect(spy).toHaveBeenCalledWith(25, undefined); }); + // Raw server cursors, e.g. from an older CLI or an API response. test("forwards initial cursor", async () => { const spy = vi.fn().mockResolvedValue({ data: [] }); await paginate({ cursor: "abc", limit: 5 }, spy); @@ -892,27 +889,108 @@ describe("paginate", () => { expect(spy).toHaveBeenCalledWith(5, undefined); }); + test.each([ + { limit: 101, perPage: 51 }, + { limit: 150, perPage: 75 }, + { limit: 200, perPage: 100 }, + { limit: 201, perPage: 67 }, + { limit: 250, perPage: 84 }, + { limit: 1000, perPage: 100 }, + ])( + "splits --limit $limit evenly into pages of $perPage", + async ({ limit, perPage }) => { + const spy = vi.fn().mockResolvedValue({ data: [] }); + await paginate({ limit }, spy); + expect(spy).toHaveBeenCalledWith(perPage, undefined); + }, + ); + // Cursor continuity across servers and limits is covered by - // test/lib/paginate.property.test.ts; this pins the request sequence. - test("keeps the walk's page size and resumes mid-page across -c next", async () => { - const spy = vi.fn((perPage: number, cursor: string | undefined) => { - const offset = cursor ? Number(cursor) : 0; - const data = Array.from({ length: perPage }, (_, i) => offset + i); - return Promise.resolve({ data, nextCursor: String(offset + perPage) }); + // test/lib/paginate.property.test.ts; these pin the request sequence. + describe("resumes mid-page across -c next", () => { + /** Pages `rows` like Sentry, with `::0` cursors. */ + function server(cursorFor: (perPage: number, offset: number) => string) { + return vi.fn((perPage: number, cursor: string | undefined) => { + const [value = 0, page = 0] = (cursor ?? "0:0:0") + .split(":") + .map(Number); + const offset = value ? value * page : page; + const data = Array.from({ length: perPage }, (_, i) => offset + i); + return Promise.resolve({ + data, + nextCursor: cursorFor(perPage, offset), + }); + }); + } + + test("with the page size of a page-number cursor", async () => { + const spy = server( + (perPage, offset) => `${perPage}:${offset / perPage + 1}:0`, + ); + + const first = await paginate({ limit: 250 }, spy); + expect(first.data).toEqual(Array.from({ length: 250 }, (_, i) => i)); + // Page 84:2 was partly returned: 82 (0b1010010) rows to skip, in binary + // in is_prev with `_0` for 1 and `0` for 0. + expect(first.nextCursor).toBe("84:2:0_00_000_00"); + const second = await paginate( + { limit: 10, cursor: first.nextCursor }, + spy, + ); + expect(second.data).toEqual( + Array.from({ length: 10 }, (_, i) => 250 + i), + ); + + expect(spy.mock.calls).toEqual([ + [84, undefined], + [84, "84:1:0"], + [84, "84:2:0"], + [84, "84:2:0"], + [84, "84:3:0"], + ]); }); - const first = await paginate({ limit: 150 }, spy); - expect(first.data).toEqual(Array.from({ length: 150 }, (_, i) => i)); - const second = await paginate({ limit: 10, cursor: first.nextCursor }, spy); - expect(second.data).toEqual(Array.from({ length: 10 }, (_, i) => 150 + i)); - - // The first call stops inside page 2, so the second refetches it with the - // walk's page size rather than one derived from its own --limit. - expect(spy.mock.calls).toEqual([ - [100, undefined], - [100, "100"], - [100, "100"], - ]); + test("fetching the skipped rows in the same request otherwise", async () => { + const spy = server((perPage, offset) => `0:${offset + perPage}:0`); + + const first = await paginate({ limit: 250 }, spy); + expect(first.nextCursor).toBe("0:168:0_00_000_00"); + const second = await paginate( + { limit: 10, cursor: first.nextCursor }, + spy, + ); + expect(second.data).toEqual( + Array.from({ length: 10 }, (_, i) => 250 + i), + ); + expect(second.nextCursor).toBe("0:260:0"); + + expect(spy.mock.calls.slice(3)).toEqual([[92, "0:168:0"]]); + }); + + test("from the first page", async () => { + const spy = server((perPage, offset) => `0:${offset + perPage}:0`); + + const first = await paginate({ limit: 3 }, (perPage, cursor) => + spy(perPage + 2, cursor), + ); + expect(first.nextCursor).toBe("0:0:0_0_0"); + const second = await paginate( + { limit: 2, cursor: first.nextCursor }, + spy, + ); + expect(second.data).toEqual([3, 4]); + expect(spy.mock.calls.at(-1)).toEqual([5, undefined]); + }); + }); + + test("returns page-boundary cursors unchanged", async () => { + const spy = vi + .fn() + .mockResolvedValue({ data: [1, 2], nextCursor: "2:1:0" }); + expect(await paginate({ limit: 2 }, spy)).toEqual({ + data: [1, 2], + nextCursor: "2:1:0", + }); }); test("returns no cursor once the server is exhausted", async () => { @@ -920,14 +998,47 @@ describe("paginate", () => { expect(await paginate({ limit: 5 }, spy)).toEqual({ data: [1, 2] }); }); + // A pasted page-number cursor is only valid with the page size it encodes. + test.each([ + { cursor: "100:2:0", perPage: 100 }, + { cursor: "25:4:0", perPage: 25 }, + { cursor: "0:200:0", perPage: 67 }, + { cursor: "1735689600000:0:0", perPage: 67 }, + { cursor: "abc", perPage: 67 }, + ])( + "resumes the raw cursor $cursor with pages of $perPage", + async ({ cursor, perPage }) => { + const spy = vi.fn().mockResolvedValue({ data: [] }); + await paginate({ limit: 201, cursor }, spy); + expect(spy.mock.calls).toEqual([[perPage, cursor]]); + }, + ); + + // Every CLI version must be able to send a returned cursor to the API. + test.each(["token", "100:3:1"])( + "refetches the page of the cursor %s, which cannot carry a skip", + async (cursor) => { + const spy = vi + .fn() + .mockResolvedValue({ data: [1, 2, 3], nextCursor: "x" }); + const first = await paginate({ limit: 2, cursor }, spy); + expect(first.nextCursor).toBe(cursor); + }, + ); + + // Nightly builds returned cli1~~~ cursors. + test("resumes a nightly cli1~ cursor", async () => { + const spy = vi.fn().mockResolvedValue({ data: [] }); + await paginate({ limit: 5, cursor: "cli1~84~2~84:2:0" }, spy); + expect(spy.mock.calls).toEqual([[84, "84:2:0"]]); + }); + test.each(["cli1~0~0~abc", "cli1~101~0~abc", "cli1~10~-1~abc", "cli1~x~0~"])( - "rejects the malformed CLI cursor %s", + "restarts from the first page on the unreadable cursor %s", async (cursor) => { const spy = vi.fn().mockResolvedValue({ data: [] }); - await expect(paginate({ limit: 5, cursor }, spy)).rejects.toThrow( - ValidationError, - ); - expect(spy).not.toHaveBeenCalled(); + await paginate({ limit: 5, cursor }, spy); + expect(spy.mock.calls).toEqual([[5, undefined]]); }, ); }); diff --git a/packages/cli/test/lib/api/project-dsns.test.ts b/packages/cli/test/lib/api/project-dsns.test.ts index 6ec9e8e37..19ba613f2 100644 --- a/packages/cli/test/lib/api/project-dsns.test.ts +++ b/packages/cli/test/lib/api/project-dsns.test.ts @@ -160,10 +160,13 @@ describe.each([ ); }); - const first = await list({ limit: 150, cursor: "0:25:0" }); + // 149 rows split as 2 x 75, so the first call stops inside page 2. + const first = await list({ limit: 149, cursor: "0:25:0" }); expect(projectIds).toEqual(organization ? ["42"] : []); - expect(first.data).toHaveLength(150); - expect(first.nextCursor).toContain("0:125:0"); + expect(first.data).toHaveLength(149); + // Page 0:100 was partly returned: 74 (0b1001010) rows to skip, in binary + // in is_prev. + expect(first.nextCursor).toBe("0:100:0_000_00_00"); const second = await list({ limit: 50, cursor: first.nextCursor }); expect( @@ -172,10 +175,12 @@ describe.each([ return [query.get("per_page"), query.get("cursor")]; }), ).toEqual([ - ["100", "0:25:0"], - ["100", "0:125:0"], - // -c next refetches the partly returned page with the walk's page size. - ["100", "0:125:0"], + ["75", "0:25:0"], + ["75", "0:100:0"], + // -c next refetches the partly returned page; offset cursors take any + // page size, so it splits the 74 skipped + 50 requested rows evenly. + ["62", "0:100:0"], + ["62", "0:162:0"], ]); expect(second.nextCursor).toBeUndefined(); expect([...first.data, ...second.data].map((key) => key.name)).toEqual( diff --git a/packages/cli/test/lib/org-list.test.ts b/packages/cli/test/lib/org-list.test.ts index d74029db1..163b430fa 100644 --- a/packages/cli/test/lib/org-list.test.ts +++ b/packages/cli/test/lib/org-list.test.ts @@ -489,14 +489,18 @@ describe("handleOrgAll", () => { const listPaginated = vi.fn( (_org: string, opts: { cursor?: string; perPage: number }) => { expect(opts.perPage).toBeLessThanOrEqual(100); - const offset = opts.cursor ? Number(opts.cursor) : 0; + // OffsetPaginator cursors: ::. + const [size = 0, page = 0] = (opts.cursor ?? "0:0:0") + .split(":") + .map(Number); + const offset = size * page; const data = Array.from({ length: opts.perPage }, (_, i) => ({ id: String(offset + i), name: `W${offset + i}`, })); return Promise.resolve({ data, - nextCursor: String(offset + opts.perPage), + nextCursor: `${opts.perPage}:${offset / opts.perPage + 1}:0`, }); }, ); @@ -512,17 +516,18 @@ describe("handleOrgAll", () => { }); expect(result.items).toHaveLength(250); - // Full pages keep requests to a minimum; the cursor carries the page size - // because team and repo lists use page-number cursors. + // The fewest requests, split evenly instead of overfetching on the last + // one; the cursor carries the page size because team and repo lists use + // page-number cursors. expect(listPaginated.mock.calls.map((call) => call[1].perPage)).toEqual([ - 100, 100, 100, + 84, 84, 84, ]); // The mock always returns a nextCursor, so hasMore is true and // the cursor points to the first item beyond the fetched 250. expect(result.hasMore).toBe(true); // `-c next` with another --limit keeps the walk's page size and refetches - // the third page, skipping the 50 rows already returned. + // the third page, skipping the 82 rows already returned. listPaginated.mockClear(); const next = await handleOrgAll({ config, @@ -536,7 +541,8 @@ describe("handleOrgAll", () => { Array.from({ length: 30 }, (_, i) => String(250 + i)), ); expect(listPaginated.mock.calls.map((call) => call[1])).toEqual([ - { cursor: "200", perPage: 100 }, + { cursor: "84:2:0", perPage: 84 }, + { cursor: "84:3:0", perPage: 84 }, ]); }); }); diff --git a/packages/cli/test/lib/paginate.property.test.ts b/packages/cli/test/lib/paginate.property.test.ts index d48253732..b5e74a3c5 100644 --- a/packages/cli/test/lib/paginate.property.test.ts +++ b/packages/cli/test/lib/paginate.property.test.ts @@ -87,9 +87,14 @@ const paginators = [ * Follow `-c next` until the cursor runs out, like repeated CLI calls, using * the next limit from `limits` (cycled) on each call. */ -async function walk(fetchPage: FetchPage, total: number, limits: number[]) { +async function walk( + fetchPage: FetchPage, + total: number, + limits: number[], + start?: string, +) { const seen: number[] = []; - let cursor: string | undefined; + let cursor = start; // Each call returns at least one row while rows remain. for (let call = 0; call <= total; call += 1) { const limit = limits[call % limits.length] ?? 1; @@ -128,6 +133,32 @@ describe.each(paginators)("property: paginate over $name", ({ make }) => { ); }); + // What an older CLI does with a cursor: send it to the API unchanged. Rows + // the cursor marks as returned come back again, but none are skipped. + test("returns cursors the API resumes from without skipping rows", async () => { + await fcAssert( + asyncProperty(totalArb, limitsArb, async (total, [limit = 1]) => { + const fetchPage = make(total); + let perPage = 0; + const result = await paginate({ limit }, (size, cursor) => { + perPage = size; + return fetchPage(size, cursor); + }); + if (!result.nextCursor) { + return; + } + // is_prev must stay a Python int literal equal to 0: zeros joined by + // single underscores (PEP 515), which `Cursor.from_string` accepts. + expect(result.nextCursor).toMatch(/^\d+:\d+:0(?:_?0)*$/); + const { data } = await fetchPage(perPage, result.nextCursor); + const next = result.data.length; + expect(data[0]).toBeLessThanOrEqual(next); + expect(data).toContain(next); + }), + { numRuns: DEFAULT_NUM_RUNS }, + ); + }); + test("returns every row exactly once, in order, across -c next", async () => { await fcAssert( asyncProperty(totalArb, limitsArb, async (total, limits) => { @@ -138,3 +169,50 @@ describe.each(paginators)("property: paginate over $name", ({ make }) => { ); }); }); + +test("property: a pasted OffsetPaginator cursor resumes without skipping", async () => { + await fcAssert( + asyncProperty( + integer({ min: 1, max: API_MAX_PER_PAGE }), + integer({ min: 1, max: 5 }), + nat(API_MAX_PER_PAGE * 3), + limitsArb, + async (pageSize, page, extra, limits) => { + // A cursor as the API returned it, from a walk with another page size. + const start = pageSize * page; + const seen = await walk( + offsetPaginator(start + extra), + extra, + limits, + `${pageSize}:${page}:0`, + ); + expect(seen).toEqual( + Array.from({ length: extra }, (_, i) => start + i), + ); + }, + ), + { numRuns: DEFAULT_NUM_RUNS }, + ); +}); + +test("property: a fresh walk uses the fewest requests without overfetching", async () => { + await fcAssert( + asyncProperty( + integer({ min: 1, max: API_MAX_PER_PAGE * 10 }), + async (limit) => { + const requested: number[] = []; + const fetchPage = genericOffsetPaginator(limit * 2); + await paginate({ limit }, (perPage, cursor) => { + requested.push(perPage); + return fetchPage(perPage, cursor); + }); + // e.g. 201 rows: 3 requests of 67, not 100 + 100 + 100. + const requests = Math.ceil(limit / API_MAX_PER_PAGE); + expect(requested).toHaveLength(requests); + const fetched = requested.reduce((sum, perPage) => sum + perPage, 0); + expect(fetched - limit).toBeLessThan(requests); + }, + ), + { numRuns: DEFAULT_NUM_RUNS }, + ); +});