test: poll for dataset items in scrapy E2E tests

tests/e2e/test_actor_scrapy.py

@@ -3,6 +3,8 @@
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+from .._utils import poll_until_condition
+
 if TYPE_CHECKING:
     from .conftest import MakeActorFunction, RunActorFunction
 
@@ -40,7 +42,11 @@ async def test_actor_scrapy_title_spider(
 
     assert run_result.status == 'SUCCEEDED'
 
-    items = await actor.last_run().dataset().list_items()
+    items = await poll_until_condition(
+        actor.last_run().dataset().list_items,
+        lambda page: len(page.items) > 1 and len(page.items) >= page.total,
+        timeout=30,
+    )
 
     # The start page and the pages it links to (`DEPTH_LIMIT` is 1, `CLOSESPIDER_PAGECOUNT` is 10), each scraped once.
     urls = [item['url'] for item in items.items]

tests/e2e/test_scrapy/conftest.py

@@ -3,6 +3,8 @@
 from pathlib import Path
 from typing import TYPE_CHECKING
 
+from ..._utils import poll_until_condition
+
 if TYPE_CHECKING:
     from apify_client._models import Run
     from apify_client._resource_clients import ActorClientAsync
@@ -50,8 +52,13 @@ async def verify_spider_results(
 
     products = expected_products or _EXPECTED_PRODUCTS
 
-    items = await actor.last_run().dataset().list_items()
-    assert items.count == 3
+    # Right after a run finishes, the API can already count the pushed rows while still returning an empty page.
+    items = await poll_until_condition(
+        actor.last_run().dataset().list_items,
+        lambda page: len(page.items) >= len(products),
+        timeout=30,
+    )
+    assert items.count == len(products)
 
     items_by_name = {item['name']: item for item in items.items}
 

tests/e2e/test_scrapy/test_graceful_abort_spider.py

@@ -52,7 +52,11 @@ async def test_graceful_abort_spider(make_actor: MakeActorFunction, apify_client
     requests = await apify_client_async.request_queue(run.default_request_queue_id).list_requests()
     handled_urls = [request.url for request in requests.items if request.handled_at is not None]
     pending_urls = [request.url for request in requests.items if request.handled_at is None]
-    items = await run_client.dataset().list_items()
+    items = await poll_until_condition(
+        run_client.dataset().list_items,
+        lambda page: len(page.items) >= len(handled_urls),
+        timeout=30,
+    )
 
     assert handled_urls, f'No request was marked as handled: {requests.items}'
     assert Counter(item['url'] for item in items.items) == Counter(handled_urls)

tests/e2e/test_scrapy/test_reboot_spider.py

@@ -3,6 +3,7 @@
 from collections import Counter
 from typing import TYPE_CHECKING
 
+from ..._utils import poll_until_condition
 from .actor_source.spider_reboot import CHAIN_LENGTH, REBOOT_LOG_MARKER
 from .conftest import get_scrapy_source_files
 
@@ -35,7 +36,11 @@ async def test_reboot_spider(
 
     # Each page of the chain was crawled once, the one in flight during the reboot included.
     expected_urls = [f'http://localhost:8080/delayed/{n}' for n in range(1, CHAIN_LENGTH + 1)]
-    items = await actor.last_run().dataset().list_items()
+    items = await poll_until_condition(
+        actor.last_run().dataset().list_items,
+        lambda page: len(page.items) >= len(expected_urls),
+        timeout=30,
+    )
     assert sorted(item['url'] for item in items.items) == expected_urls
 
     assert run_result.default_request_queue_id is not None

tests/e2e/test_scrapy/test_redirect_spider.py

@@ -2,6 +2,7 @@
 
 from typing import TYPE_CHECKING
 
+from ..._utils import poll_until_condition
 from .conftest import get_scrapy_source_files
 
 if TYPE_CHECKING:
@@ -18,5 +19,9 @@ async def test_redirect_spider(make_actor: MakeActorFunction, run_actor: RunActo
     run_result = await run_actor(actor)
     assert run_result.status == 'SUCCEEDED'
 
-    items = await actor.last_run().dataset().list_items()
+    items = await poll_until_condition(
+        actor.last_run().dataset().list_items,
+        lambda page: len(page.items) >= 1,
+        timeout=30,
+    )
     assert items.items == [{'url': 'http://localhost:8080/redirect-target', 'title': 'Redirect Target'}]