From f103e293ebfec63961d905455a4fd6dad4642446 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 3 Nov 2026 20:10:36 +0100 Subject: [PATCH 1/3] test: poll for dataset items in scrapy E2E tests --- tests/e2e/test_scrapy/conftest.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/e2e/test_scrapy/conftest.py b/tests/e2e/test_scrapy/conftest.py index 93a09d799..9d2cbff0d 100644 --- a/tests/e2e/test_scrapy/conftest.py +++ b/tests/e2e/test_scrapy/conftest.py @@ -3,6 +3,8 @@ from pathlib import Path from typing import TYPE_CHECKING +from ..._utils import poll_until_condition + if TYPE_CHECKING: from apify_client._models import Run from apify_client._resource_clients import ActorClientAsync @@ -50,7 +52,13 @@ async def verify_spider_results( products = expected_products or _EXPECTED_PRODUCTS - items = await actor.last_run().dataset().list_items() + # Right after a run finishes, the API can already count the pushed rows while still returning an empty page. + items = await poll_until_condition( + actor.last_run().dataset().list_items, + lambda page: len(page.items) == len(products), + timeout=30, + poll_interval=1, + ) assert items.count == 3 items_by_name = {item['name']: item for item in items.items} From e1cfe2a941cee706694d4d5d7c65bee45d3692a9 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 3 Nov 2026 23:08:23 +0100 Subject: [PATCH 2/3] test: stop scrapy E2E dataset polling once enough items are readable --- tests/e2e/test_scrapy/conftest.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/e2e/test_scrapy/conftest.py b/tests/e2e/test_scrapy/conftest.py index 9d2cbff0d..ab3934254 100644 --- a/tests/e2e/test_scrapy/conftest.py +++ b/tests/e2e/test_scrapy/conftest.py @@ -55,11 +55,10 @@ async def verify_spider_results( # Right after a run finishes, the API can already count the pushed rows while still returning an empty page. items = await poll_until_condition( actor.last_run().dataset().list_items, - lambda page: len(page.items) == len(products), + lambda page: len(page.items) >= len(products), timeout=30, - poll_interval=1, ) - assert items.count == 3 + assert items.count == len(products) items_by_name = {item['name']: item for item in items.items} From 0087914f4682fb7dc259334a162d84bfdc6218a9 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 3 Nov 2026 23:08:24 +0100 Subject: [PATCH 3/3] test: poll for dataset items in the remaining scrapy E2E tests --- tests/e2e/test_actor_scrapy.py | 8 +++++++- tests/e2e/test_scrapy/test_graceful_abort_spider.py | 6 +++++- tests/e2e/test_scrapy/test_reboot_spider.py | 7 ++++++- tests/e2e/test_scrapy/test_redirect_spider.py | 7 ++++++- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/tests/e2e/test_actor_scrapy.py b/tests/e2e/test_actor_scrapy.py index 682d91402..96a454c1c 100644 --- a/tests/e2e/test_actor_scrapy.py +++ b/tests/e2e/test_actor_scrapy.py @@ -3,6 +3,8 @@ from pathlib import Path from typing import TYPE_CHECKING +from .._utils import poll_until_condition + if TYPE_CHECKING: from .conftest import MakeActorFunction, RunActorFunction @@ -40,7 +42,11 @@ async def test_actor_scrapy_title_spider( assert run_result.status == 'SUCCEEDED' - items = await actor.last_run().dataset().list_items() + items = await poll_until_condition( + actor.last_run().dataset().list_items, + lambda page: len(page.items) > 1 and len(page.items) >= page.total, + timeout=30, + ) # The start page and the pages it links to (`DEPTH_LIMIT` is 1, `CLOSESPIDER_PAGECOUNT` is 10), each scraped once. urls = [item['url'] for item in items.items] diff --git a/tests/e2e/test_scrapy/test_graceful_abort_spider.py b/tests/e2e/test_scrapy/test_graceful_abort_spider.py index 5f318f73f..2e60bd3b3 100644 --- a/tests/e2e/test_scrapy/test_graceful_abort_spider.py +++ b/tests/e2e/test_scrapy/test_graceful_abort_spider.py @@ -52,7 +52,11 @@ async def test_graceful_abort_spider(make_actor: MakeActorFunction, apify_client requests = await apify_client_async.request_queue(run.default_request_queue_id).list_requests() handled_urls = [request.url for request in requests.items if request.handled_at is not None] pending_urls = [request.url for request in requests.items if request.handled_at is None] - items = await run_client.dataset().list_items() + items = await poll_until_condition( + run_client.dataset().list_items, + lambda page: len(page.items) >= len(handled_urls), + timeout=30, + ) assert handled_urls, f'No request was marked as handled: {requests.items}' assert Counter(item['url'] for item in items.items) == Counter(handled_urls) diff --git a/tests/e2e/test_scrapy/test_reboot_spider.py b/tests/e2e/test_scrapy/test_reboot_spider.py index 6b98b1a9e..8a960308d 100644 --- a/tests/e2e/test_scrapy/test_reboot_spider.py +++ b/tests/e2e/test_scrapy/test_reboot_spider.py @@ -3,6 +3,7 @@ from collections import Counter from typing import TYPE_CHECKING +from ..._utils import poll_until_condition from .actor_source.spider_reboot import CHAIN_LENGTH, REBOOT_LOG_MARKER from .conftest import get_scrapy_source_files @@ -35,7 +36,11 @@ async def test_reboot_spider( # Each page of the chain was crawled once, the one in flight during the reboot included. expected_urls = [f'http://localhost:8080/delayed/{n}' for n in range(1, CHAIN_LENGTH + 1)] - items = await actor.last_run().dataset().list_items() + items = await poll_until_condition( + actor.last_run().dataset().list_items, + lambda page: len(page.items) >= len(expected_urls), + timeout=30, + ) assert sorted(item['url'] for item in items.items) == expected_urls assert run_result.default_request_queue_id is not None diff --git a/tests/e2e/test_scrapy/test_redirect_spider.py b/tests/e2e/test_scrapy/test_redirect_spider.py index f8e0e4621..672d3d9b1 100644 --- a/tests/e2e/test_scrapy/test_redirect_spider.py +++ b/tests/e2e/test_scrapy/test_redirect_spider.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING +from ..._utils import poll_until_condition from .conftest import get_scrapy_source_files if TYPE_CHECKING: @@ -18,5 +19,9 @@ async def test_redirect_spider(make_actor: MakeActorFunction, run_actor: RunActo run_result = await run_actor(actor) assert run_result.status == 'SUCCEEDED' - items = await actor.last_run().dataset().list_items() + items = await poll_until_condition( + actor.last_run().dataset().list_items, + lambda page: len(page.items) >= 1, + timeout=30, + ) assert items.items == [{'url': 'http://localhost:8080/redirect-target', 'title': 'Redirect Target'}]