From 05112f98ba25fd5ef86598b694e29b318a13d26e Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 08:29:48 +0100 Subject: [PATCH 01/28] test: expect the trailing slash Scrapy adds to the E2E start URL --- tests/e2e/test_actor_scrapy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/test_actor_scrapy.py b/tests/e2e/test_actor_scrapy.py index 96a454c1..2d06910c 100644 --- a/tests/e2e/test_actor_scrapy.py +++ b/tests/e2e/test_actor_scrapy.py @@ -33,7 +33,7 @@ async def test_actor_scrapy_title_spider( run_result = await run_actor( actor, run_input={ - 'startUrls': [{'url': 'https://crawlee.dev'}], + 'startUrls': [{'url': 'https://crawlee.dev/'}], 'allowedDomains': ['crawlee.dev'], 'proxyConfiguration': {'useApifyProxy': True}, }, @@ -50,7 +50,7 @@ async def test_actor_scrapy_title_spider( # The start page and the pages it links to (`DEPTH_LIMIT` is 1, `CLOSESPIDER_PAGECOUNT` is 10), each scraped once. urls = [item['url'] for item in items.items] - assert 'https://crawlee.dev' in urls + assert 'https://crawlee.dev/' in urls assert len(urls) > 1 assert len(urls) == len(set(urls)), urls From bf5150f871bcfa1fddcfc2061c0a737c1f6807e1 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 08:39:03 +0100 Subject: [PATCH 02/28] test: pin the E2E test Actor dependencies to the project lock file --- tests/e2e/actor_source_base/Dockerfile | 2 +- tests/e2e/actor_source_base/requirements.txt | 2 +- tests/e2e/conftest.py | 29 ++++++++++++++++++-- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/tests/e2e/actor_source_base/Dockerfile b/tests/e2e/actor_source_base/Dockerfile index 1e5df612..a5add91d 100644 --- a/tests/e2e/actor_source_base/Dockerfile +++ b/tests/e2e/actor_source_base/Dockerfile @@ -12,7 +12,7 @@ RUN echo "Python version:" \ && echo "Pip version:" \ && pip --version \ && echo "Installing dependencies:" \ - && pip install --force-reinstall -r requirements.txt \ + && pip install --force-reinstall -r requirements.txt -c constraints.txt \ && echo "All installed Python packages:" \ && pip freeze diff --git a/tests/e2e/actor_source_base/requirements.txt b/tests/e2e/actor_source_base/requirements.txt index e70a7515..f8dcde1b 100644 --- a/tests/e2e/actor_source_base/requirements.txt +++ b/tests/e2e/actor_source_base/requirements.txt @@ -1,4 +1,4 @@ # The test fixture will put the Apify SDK wheel path on the next line APIFY_SDK_WHEEL_PLACEHOLDER uvicorn[standard] -crawlee[parsel]>=1.0.0,<2.0.0,!=1.10.1 +crawlee[parsel]>=1.0.0,<2.0.0 diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index a76dc448..bc687979 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -3,6 +3,7 @@ import base64 import inspect import os +import shutil import subprocess import sys import textwrap @@ -164,8 +165,8 @@ def sdk_wheel_path(tmp_path_factory: pytest.TempPathFactory, testrun_uid: str) - def actor_base_source_files(sdk_wheel_path: Path) -> dict[str, str | bytes]: """Create a dictionary of the base source files for a testing Actor. - It takes the files from `tests/e2e/actor_source_base`, builds the Apify SDK wheel from - the current codebase, and adds them all together in a dictionary. + It takes the files from `tests/e2e/actor_source_base`, builds the Apify SDK wheel from the current codebase, exports + the project lock file as pip constraints, and adds them all together in a dictionary. """ source_files: dict[str, str | bytes] = {} @@ -188,6 +189,30 @@ def actor_base_source_files(sdk_wheel_path: Path) -> dict[str, str | bytes]: 'APIFY_SDK_WHEEL_PLACEHOLDER', f'./{sdk_wheel_file_name}' ) + # Pin every dependency the Actor installs to its version in the project lock file, so the E2E tests run with the + # same versions as the rest of the test suite. + uv_path = shutil.which('uv') + if uv_path is None: + raise RuntimeError('The E2E tests need `uv` on PATH to export the project lock file.') + + source_files['constraints.txt'] = subprocess.run( # noqa: S603 + [ + uv_path, + 'export', + '--frozen', + '--no-hashes', + '--no-emit-project', + '--all-extras', + '--all-groups', + '--format', + 'requirements.txt', + ], + cwd=_SDK_ROOT_PATH, + check=True, + capture_output=True, + text=True, + ).stdout + python_version = f'{sys.version_info[0]}.{sys.version_info[1]}' source_files['Dockerfile'] = str(source_files['Dockerfile']).replace( 'BASE_IMAGE_VERSION_PLACEHOLDER', python_version From 10feae6581c13b107abecd26675ccc8f0eb73aef Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 08:39:04 +0100 Subject: [PATCH 03/28] test: install the E2E test Actor dependencies with uv --- tests/e2e/actor_source_base/Dockerfile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/e2e/actor_source_base/Dockerfile b/tests/e2e/actor_source_base/Dockerfile index a5add91d..e6151868 100644 --- a/tests/e2e/actor_source_base/Dockerfile +++ b/tests/e2e/actor_source_base/Dockerfile @@ -1,6 +1,8 @@ # The test fixture will put the right Python version here FROM apify/actor-python:BASE_IMAGE_VERSION_PLACEHOLDER +COPY --from=ghcr.io/astral-sh/uv:0.13 /uv /bin/ + COPY . ./ RUN apt-get update && apt-get install -y \ @@ -9,11 +11,11 @@ RUN apt-get update && apt-get install -y \ RUN echo "Python version:" \ && python --version \ - && echo "Pip version:" \ - && pip --version \ - && echo "Installing dependencies:" \ - && pip install --force-reinstall -r requirements.txt -c constraints.txt \ + && echo "uv version:" \ + && uv --version \ + && echo "Installing dependencies pinned to the project lock file:" \ + && uv pip install --system --reinstall -r requirements.txt -c constraints.txt \ && echo "All installed Python packages:" \ - && pip freeze + && uv pip freeze --system CMD ["sh", "-c", "python server.py & python -m src"] From fd757cb066f39977c18544a7586f7c54605fcc4a Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 08:43:00 +0100 Subject: [PATCH 04/28] test: keep the uv version of the E2E Actor image in a build argument --- tests/e2e/actor_source_base/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/e2e/actor_source_base/Dockerfile b/tests/e2e/actor_source_base/Dockerfile index e6151868..a00f0723 100644 --- a/tests/e2e/actor_source_base/Dockerfile +++ b/tests/e2e/actor_source_base/Dockerfile @@ -1,7 +1,11 @@ +ARG UV_VERSION=0.13 + +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv + # The test fixture will put the right Python version here FROM apify/actor-python:BASE_IMAGE_VERSION_PLACEHOLDER -COPY --from=ghcr.io/astral-sh/uv:0.13 /uv /bin/ +COPY --from=uv /uv /bin/ COPY . ./ From bb683d377e43456d4b5b02d1762f8f8fa9c5e4b0 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 08:43:17 +0100 Subject: [PATCH 05/28] test: drop the echo labels from the E2E Actor image build --- tests/e2e/actor_source_base/Dockerfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/e2e/actor_source_base/Dockerfile b/tests/e2e/actor_source_base/Dockerfile index a00f0723..4319e0fe 100644 --- a/tests/e2e/actor_source_base/Dockerfile +++ b/tests/e2e/actor_source_base/Dockerfile @@ -13,13 +13,9 @@ RUN apt-get update && apt-get install -y \ git \ && rm -rf /var/lib/apt/lists/* -RUN echo "Python version:" \ - && python --version \ - && echo "uv version:" \ +RUN python --version \ && uv --version \ - && echo "Installing dependencies pinned to the project lock file:" \ && uv pip install --system --reinstall -r requirements.txt -c constraints.txt \ - && echo "All installed Python packages:" \ && uv pip freeze --system CMD ["sh", "-c", "python server.py & python -m src"] From 9e94f7c299c8e813ee070728d72c920212080b94 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 08:46:20 +0100 Subject: [PATCH 06/28] test: stop printing the Python version in the E2E Actor image build --- tests/e2e/actor_source_base/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/e2e/actor_source_base/Dockerfile b/tests/e2e/actor_source_base/Dockerfile index 4319e0fe..4490585c 100644 --- a/tests/e2e/actor_source_base/Dockerfile +++ b/tests/e2e/actor_source_base/Dockerfile @@ -13,8 +13,7 @@ RUN apt-get update && apt-get install -y \ git \ && rm -rf /var/lib/apt/lists/* -RUN python --version \ - && uv --version \ +RUN uv --version \ && uv pip install --system --reinstall -r requirements.txt -c constraints.txt \ && uv pip freeze --system From 41935b39f1e8c4c2a7969cf0d4cf0d3e9e00b936 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 08:49:38 +0100 Subject: [PATCH 07/28] test: stop printing the uv version in the E2E Actor image build --- tests/e2e/actor_source_base/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/e2e/actor_source_base/Dockerfile b/tests/e2e/actor_source_base/Dockerfile index 4490585c..9fac7de0 100644 --- a/tests/e2e/actor_source_base/Dockerfile +++ b/tests/e2e/actor_source_base/Dockerfile @@ -13,8 +13,7 @@ RUN apt-get update && apt-get install -y \ git \ && rm -rf /var/lib/apt/lists/* -RUN uv --version \ - && uv pip install --system --reinstall -r requirements.txt -c constraints.txt \ +RUN uv pip install --system --reinstall -r requirements.txt -c constraints.txt \ && uv pip freeze --system CMD ["sh", "-c", "python server.py & python -m src"] From 6ca57f518aaed3d69906e3fcf9530b6f38a81c90 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 08:51:36 +0100 Subject: [PATCH 08/28] test: export the lock file in the E2E Actor image build --- tests/e2e/actor_source_base/Dockerfile | 4 +++- tests/e2e/conftest.py | 32 +++++--------------------- 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/tests/e2e/actor_source_base/Dockerfile b/tests/e2e/actor_source_base/Dockerfile index 9fac7de0..43a213a3 100644 --- a/tests/e2e/actor_source_base/Dockerfile +++ b/tests/e2e/actor_source_base/Dockerfile @@ -13,7 +13,9 @@ RUN apt-get update && apt-get install -y \ git \ && rm -rf /var/lib/apt/lists/* -RUN uv pip install --system --reinstall -r requirements.txt -c constraints.txt \ +RUN uv export --project sdk_lock --frozen --no-hashes --no-emit-project --all-extras --all-groups \ + --format requirements.txt --output-file constraints.txt \ + && uv pip install --system --reinstall -r requirements.txt -c constraints.txt \ && uv pip freeze --system CMD ["sh", "-c", "python server.py & python -m src"] diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index bc687979..e63e1a27 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -3,7 +3,6 @@ import base64 import inspect import os -import shutil import subprocess import sys import textwrap @@ -165,8 +164,8 @@ def sdk_wheel_path(tmp_path_factory: pytest.TempPathFactory, testrun_uid: str) - def actor_base_source_files(sdk_wheel_path: Path) -> dict[str, str | bytes]: """Create a dictionary of the base source files for a testing Actor. - It takes the files from `tests/e2e/actor_source_base`, builds the Apify SDK wheel from the current codebase, exports - the project lock file as pip constraints, and adds them all together in a dictionary. + It takes the files from `tests/e2e/actor_source_base`, builds the Apify SDK wheel from the current codebase, adds + the project lock file, and puts them all together in a dictionary. """ source_files: dict[str, str | bytes] = {} @@ -189,29 +188,10 @@ def actor_base_source_files(sdk_wheel_path: Path) -> dict[str, str | bytes]: 'APIFY_SDK_WHEEL_PLACEHOLDER', f'./{sdk_wheel_file_name}' ) - # Pin every dependency the Actor installs to its version in the project lock file, so the E2E tests run with the - # same versions as the rest of the test suite. - uv_path = shutil.which('uv') - if uv_path is None: - raise RuntimeError('The E2E tests need `uv` on PATH to export the project lock file.') - - source_files['constraints.txt'] = subprocess.run( # noqa: S603 - [ - uv_path, - 'export', - '--frozen', - '--no-hashes', - '--no-emit-project', - '--all-extras', - '--all-groups', - '--format', - 'requirements.txt', - ], - cwd=_SDK_ROOT_PATH, - check=True, - capture_output=True, - text=True, - ).stdout + # The Actor build exports the project lock file as pip constraints, so the E2E tests run with the same dependency + # versions as the rest of the test suite. + for file_name in ('pyproject.toml', 'uv.lock'): + source_files[f'sdk_lock/{file_name}'] = (_SDK_ROOT_PATH / file_name).read_text(encoding='utf-8') python_version = f'{sys.version_info[0]}.{sys.version_info[1]}' source_files['Dockerfile'] = str(source_files['Dockerfile']).replace( From 8a8bbc9e45cb77b24f5dbf6ee70a4d851e50e126 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 08:53:54 +0100 Subject: [PATCH 09/28] test: build the E2E Actor image in a builder and a runner stage --- tests/e2e/actor_source_base/Dockerfile | 39 ++++++++++++++++++++------ 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/tests/e2e/actor_source_base/Dockerfile b/tests/e2e/actor_source_base/Dockerfile index 43a213a3..b14a0c32 100644 --- a/tests/e2e/actor_source_base/Dockerfile +++ b/tests/e2e/actor_source_base/Dockerfile @@ -1,21 +1,42 @@ +# The test fixture puts the Python version of the test run in place of BASE_IMAGE_VERSION_PLACEHOLDER. +# +# Multi-stage build: +# - builder: Installs the dependencies, pinned to the project lock file, into a virtual environment +# - runner: Actor image with the virtual environment and the Actor source code + ARG UV_VERSION=0.13 FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv -# The test fixture will put the right Python version here -FROM apify/actor-python:BASE_IMAGE_VERSION_PLACEHOLDER +# ============================================================================= +# Build stage +# ============================================================================= +FROM apify/actor-python:BASE_IMAGE_VERSION_PLACEHOLDER AS builder COPY --from=uv /uv /bin/ -COPY . ./ - -RUN apt-get update && apt-get install -y \ - git \ - && rm -rf /var/lib/apt/lists/* +# Copy the dependency files: the requirements, the Apify SDK wheel and the project lock file +COPY requirements.txt *.whl ./ +COPY sdk_lock/ ./sdk_lock/ +# Install the dependencies into a virtual environment, pinned to the project lock file RUN uv export --project sdk_lock --frozen --no-hashes --no-emit-project --all-extras --all-groups \ --format requirements.txt --output-file constraints.txt \ - && uv pip install --system --reinstall -r requirements.txt -c constraints.txt \ - && uv pip freeze --system + && uv venv /opt/venv \ + && uv pip install --python /opt/venv/bin/python -r requirements.txt -c constraints.txt \ + && uv pip freeze --python /opt/venv/bin/python + +# ============================================================================= +# Runtime stage +# ============================================================================= +FROM apify/actor-python:BASE_IMAGE_VERSION_PLACEHOLDER AS runner + +# Copy the virtual environment with the installed dependencies +COPY --from=builder /opt/venv /opt/venv + +ENV PATH="/opt/venv/bin:$PATH" + +# Copy the Actor source code +COPY . ./ CMD ["sh", "-c", "python server.py & python -m src"] From a7fd046a37462ad3b68f5d7aa89c40cb993021bc Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 08:57:19 +0100 Subject: [PATCH 10/28] test: build the Apify SDK in the E2E Actor image --- tests/e2e/actor_source_base/Dockerfile | 14 ++-- tests/e2e/actor_source_base/requirements.txt | 2 - tests/e2e/conftest.py | 80 +++++--------------- 3 files changed, 24 insertions(+), 72 deletions(-) diff --git a/tests/e2e/actor_source_base/Dockerfile b/tests/e2e/actor_source_base/Dockerfile index b14a0c32..e6a12fe5 100644 --- a/tests/e2e/actor_source_base/Dockerfile +++ b/tests/e2e/actor_source_base/Dockerfile @@ -1,7 +1,7 @@ # The test fixture puts the Python version of the test run in place of BASE_IMAGE_VERSION_PLACEHOLDER. # # Multi-stage build: -# - builder: Installs the dependencies, pinned to the project lock file, into a virtual environment +# - builder: Installs the Apify SDK and the dependencies, pinned to the project lock file, into a virtual environment # - runner: Actor image with the virtual environment and the Actor source code ARG UV_VERSION=0.13 @@ -15,15 +15,15 @@ FROM apify/actor-python:BASE_IMAGE_VERSION_PLACEHOLDER AS builder COPY --from=uv /uv /bin/ -# Copy the dependency files: the requirements, the Apify SDK wheel and the project lock file -COPY requirements.txt *.whl ./ -COPY sdk_lock/ ./sdk_lock/ +# Copy the requirements and the Apify SDK source code with the project lock file +COPY requirements.txt ./ +COPY sdk/ ./sdk/ -# Install the dependencies into a virtual environment, pinned to the project lock file -RUN uv export --project sdk_lock --frozen --no-hashes --no-emit-project --all-extras --all-groups \ +# Install the Apify SDK and the dependencies into a virtual environment, pinned to the project lock file +RUN uv export --project sdk --frozen --no-hashes --no-emit-project --all-extras --all-groups \ --format requirements.txt --output-file constraints.txt \ && uv venv /opt/venv \ - && uv pip install --python /opt/venv/bin/python -r requirements.txt -c constraints.txt \ + && uv pip install --python /opt/venv/bin/python ./sdk -r requirements.txt -c constraints.txt \ && uv pip freeze --python /opt/venv/bin/python # ============================================================================= diff --git a/tests/e2e/actor_source_base/requirements.txt b/tests/e2e/actor_source_base/requirements.txt index f8dcde1b..bb282edb 100644 --- a/tests/e2e/actor_source_base/requirements.txt +++ b/tests/e2e/actor_source_base/requirements.txt @@ -1,4 +1,2 @@ -# The test fixture will put the Apify SDK wheel path on the next line -APIFY_SDK_WHEEL_PLACEHOLDER uvicorn[standard] crawlee[parsel]>=1.0.0,<2.0.0 diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index e63e1a27..5cedc5c1 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -3,7 +3,6 @@ import base64 import inspect import os -import subprocess import sys import textwrap from datetime import UTC, datetime, timedelta @@ -11,7 +10,6 @@ from typing import TYPE_CHECKING, Any, Protocol import pytest -from filelock import FileLock from apify_client import ApifyClient, ApifyClientAsync from crawlee import service_locator @@ -120,78 +118,34 @@ def _isolate_test_environment(prepare_test_env: Callable[[], None]) -> None: prepare_test_env() -@pytest.fixture(scope='session') -def sdk_wheel_path(tmp_path_factory: pytest.TempPathFactory, testrun_uid: str) -> Path: - """Build the package wheel if it hasn't been built yet, and return the path to the wheel.""" - # `getbasetemp()` is per-xdist-worker, so both the lock and the indicator file live in its shared parent - # directory, which every worker sees. - shared_tmp_path = tmp_path_factory.getbasetemp().parent - - # Make sure the wheel is not being built concurrently across all the pytest-xdist runners, - # through locking the building process with a temp file. - with FileLock(shared_tmp_path / 'sdk_wheel_build.lock'): - # Make sure the wheel is built exactly once across all the pytest-xdist runners, through an indicator - # file saying that the wheel was already built. - was_wheel_built_this_test_run_file = shared_tmp_path / f'wheel_was_built_in_run_{testrun_uid}' - if not was_wheel_built_this_test_run_file.exists(): - subprocess.run( - args=[sys.executable, '-m', 'build'], - cwd=_SDK_ROOT_PATH, - check=True, - capture_output=True, - ) - was_wheel_built_this_test_run_file.touch() - - # Read the current package version, necessary for getting the right wheel filename. - pyproject_toml_file = (_SDK_ROOT_PATH / 'pyproject.toml').read_text(encoding='utf-8') - for line in pyproject_toml_file.splitlines(): - if line.startswith('version = '): - delim = '"' if '"' in line else "'" - sdk_version = line.split(delim)[1] - break - else: - raise RuntimeError('Unable to find version string.') - - wheel_path = _SDK_ROOT_PATH / 'dist' / f'apify-{sdk_version}-py3-none-any.whl' - - # Just to be sure. - assert wheel_path.exists() - - return wheel_path +def _read_source_file(path: Path) -> str | bytes: + """Read a file to upload as an Actor source file, as text when it is valid UTF-8.""" + try: + return path.read_text(encoding='utf-8') + except ValueError: + return path.read_bytes() @pytest.fixture(scope='session') -def actor_base_source_files(sdk_wheel_path: Path) -> dict[str, str | bytes]: +def actor_base_source_files() -> dict[str, str | bytes]: """Create a dictionary of the base source files for a testing Actor. - It takes the files from `tests/e2e/actor_source_base`, builds the Apify SDK wheel from the current codebase, adds - the project lock file, and puts them all together in a dictionary. + It takes the files from `tests/e2e/actor_source_base` and adds the Apify SDK source code under `sdk/`. The Actor + build installs the SDK from it, with every dependency pinned to the project lock file. """ source_files: dict[str, str | bytes] = {} - # First read the actor_source_base files actor_source_base_path = _SDK_ROOT_PATH / 'tests/e2e/actor_source_base' - for path in actor_source_base_path.glob('**/*'): - if not path.is_file(): - continue - relative_path = str(path.relative_to(actor_source_base_path)) - try: - source_files[relative_path] = path.read_text(encoding='utf-8') - except ValueError: - source_files[relative_path] = path.read_bytes() - - sdk_wheel_file_name = sdk_wheel_path.name - source_files[sdk_wheel_file_name] = sdk_wheel_path.read_bytes() - - source_files['requirements.txt'] = str(source_files['requirements.txt']).replace( - 'APIFY_SDK_WHEEL_PLACEHOLDER', f'./{sdk_wheel_file_name}' - ) + if path.is_file(): + source_files[str(path.relative_to(actor_source_base_path))] = _read_source_file(path) + + sdk_paths = [_SDK_ROOT_PATH / file_name for file_name in ('pyproject.toml', 'uv.lock', 'README.md', 'LICENSE')] + src_paths = (_SDK_ROOT_PATH / 'src').glob('**/*') + sdk_paths.extend(path for path in src_paths if path.is_file() and '__pycache__' not in path.parts) - # The Actor build exports the project lock file as pip constraints, so the E2E tests run with the same dependency - # versions as the rest of the test suite. - for file_name in ('pyproject.toml', 'uv.lock'): - source_files[f'sdk_lock/{file_name}'] = (_SDK_ROOT_PATH / file_name).read_text(encoding='utf-8') + for path in sdk_paths: + source_files[f'sdk/{path.relative_to(_SDK_ROOT_PATH)}'] = _read_source_file(path) python_version = f'{sys.version_info[0]}.{sys.version_info[1]}' source_files['Dockerfile'] = str(source_files['Dockerfile']).replace( From 2366addbf87321d914e71ad6f432737829037f63 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 08:57:20 +0100 Subject: [PATCH 11/28] chore(deps): drop the unused `build` dev dependency --- pyproject.toml | 1 - uv.lock | 25 ------------------------- 2 files changed, 26 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d9addafa..b5ba89e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,6 @@ dev = [ # See https://github.com/apify/apify-client-python/pull/582/ for more details. # We explicitly constrain black>=24.3.0 to override the transitive dependency. "black>=24.3.0", - "build<2.0.0", "crawlee[parsel]", "griffe", "poethepoet<1.0.0", diff --git a/uv.lock b/uv.lock index c079a03e..c23d73c9 100644 --- a/uv.lock +++ b/uv.lock @@ -246,7 +246,6 @@ scrapy = [ [package.dev-dependencies] dev = [ { name = "black" }, - { name = "build" }, { name = "crawlee", extra = ["parsel"] }, { name = "griffe" }, { name = "poethepoet" }, @@ -287,7 +286,6 @@ provides-extras = ["scrapy"] [package.metadata.requires-dev] dev = [ { name = "black", specifier = ">=24.3.0" }, - { name = "build", specifier = "<2.0.0" }, { name = "crawlee", extras = ["parsel"] }, { name = "griffe" }, { name = "poethepoet", specifier = "<1.0.0" }, @@ -580,20 +578,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/42/da/f77571a1e13025991a6e1e265997ea4b5d18dd5092e9fb309531cfb2a305/brotlicffi-1.2.1.0-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:52b99120543ffa3b0667e922c6a7dc6b6e01350f7fd38f21aaca511bb9f8b4f2", size = 465300, upload-time = "2026-10-22T16:19:30.988Z" }, ] -[[package]] -name = "build" -version = "1.6.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "os_name == 'nt'" }, - { name = "packaging" }, - { name = "pyproject-hooks" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/bd/67/4898a44ea4f3f8e213b0954ec0aa0a16971d62a6212d6ea3931e97115b99/build-1.6.1.tar.gz", hash = "sha256:51cc11666391ab6f092070437ac747002ff46f3e4113a3622177ee6b488bfc53", size = 113427, upload-time = "2026-09-10T07:56:00.417Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ad/9b/9fb3585dabcd73a1b2a6267f63f62649347c9e6d072c9fde365b105abb2c/build-1.6.1-py3-none-any.whl", hash = "sha256:ecd351a4be9d35a9eaaba244a7687143c9c7d4aea6ac964e7e7ddab20cbcf4e7", size = 31179, upload-time = "2026-09-10T07:55:59.148Z" }, -] - [[package]] name = "cachetools" version = "7.2.8" @@ -2704,15 +2688,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/ad/2cf6d3fa2fae5c79e1ed9960c0d42badd0f94d81dd12b50604cdc839e648/pyopenssl-26.4.0-py3-none-any.whl", hash = "sha256:f0eb0cb2d581d3ad2b9c489468485e7f2ab6727d08401bcf9d824c3caddf3c1c", size = 56026, upload-time = "2026-08-01T19:50:48.94Z" }, ] -[[package]] -name = "pyproject-hooks" -version = "1.3.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6d/5d/f2ddeef4a855a102aaae5e97826a0260007522ab504421b75addfdb1517c/pyproject_hooks-1.3.3.tar.gz", hash = "sha256:defda19b854fa0d3bd4f76ea4ddcba8abd7dcfcdd585a6690ade050744fc5f43", size = 21013, upload-time = "2026-09-16T08:58:03.999Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/85/11/044d1ae1b4ec0d7af88ee5bc91e081be1022533032b906a7bdabdbb60977/pyproject_hooks-1.3.3-py3-none-any.whl", hash = "sha256:5fc53fdac9f7bd63fbcdc868fb5f90b4784d78a53a3d3388cd738b807441a20b", size = 10724, upload-time = "2026-09-16T08:58:02.96Z" }, -] - [[package]] name = "pypydispatcher" version = "2.1.2" From 9d9230d9eaa473bd8aabf74f2667ad5b6430a1c1 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 09:04:12 +0100 Subject: [PATCH 12/28] test: rename the E2E test Actor package to `actor` --- tests/e2e/README.md | 6 +++--- tests/e2e/actor_source_base/Dockerfile | 2 +- .../{src => actor}/__init__.py | 0 .../{src => actor}/__main__.py | 0 .../actor_source_base/{src => actor}/main.py | 0 tests/e2e/conftest.py | 2 +- tests/e2e/test_actor_key_value_store.py | 2 +- tests/e2e/test_actor_scrapy.py | 21 ++++++++++++------- .../actor_source/playwright.Dockerfile | 2 +- .../test_adaptive_playwright_crawler.py | 2 +- tests/e2e/test_crawlee/test_basic_crawler.py | 2 +- .../test_beautifulsoup_crawler.py | 2 +- tests/e2e/test_crawlee/test_http_crawler.py | 2 +- tests/e2e/test_crawlee/test_parsel_crawler.py | 2 +- .../test_crawlee/test_playwright_crawler.py | 2 +- tests/e2e/test_fixtures.py | 4 ++-- .../test_schema_storages.py | 2 +- .../e2e/test_scrapy/actor_source/__main__.py | 2 +- .../actor_source/main_custom_pipeline.py | 2 +- .../e2e/test_scrapy/actor_source/settings.py | 4 ++-- .../actor_source/settings_custom_pipeline.py | 4 ++-- .../actor_source/spider_delayed_chain.py | 2 +- .../actor_source/spider_in_flight.py | 2 +- .../actor_source/spider_itemloader.py | 2 +- .../test_scrapy/actor_source/spider_reboot.py | 2 +- tests/e2e/test_scrapy/conftest.py | 12 +++++------ .../test_custom_pipeline_spider.py | 6 +++--- 27 files changed, 48 insertions(+), 43 deletions(-) rename tests/e2e/actor_source_base/{src => actor}/__init__.py (100%) rename tests/e2e/actor_source_base/{src => actor}/__main__.py (100%) rename tests/e2e/actor_source_base/{src => actor}/main.py (100%) diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 8f9d5ff1..1c36a83c 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -41,7 +41,7 @@ async def test_something( assert run_result.status == 'SUCCEEDED' ``` -The `src/main.py` file will be set to the function definition, prepended with `import asyncio` and `from apify import Actor`. You can add extra imports directly inside the function body. +The `actor/main.py` file will be set to the function definition, prepended with `import asyncio` and `from apify import Actor`. You can add extra imports directly inside the function body. ### Creating an Actor from source files @@ -74,12 +74,12 @@ Or pass `source_files` for multi-file Actors: ```python actor_source_files = { - 'src/utils.py': """ + 'actor/utils.py': """ from datetime import datetime, timezone def get_current_datetime(): return datetime.now(timezone.utc) """, - 'src/main.py': """ + 'actor/main.py': """ import asyncio from apify import Actor from .utils import get_current_datetime diff --git a/tests/e2e/actor_source_base/Dockerfile b/tests/e2e/actor_source_base/Dockerfile index e6a12fe5..62418007 100644 --- a/tests/e2e/actor_source_base/Dockerfile +++ b/tests/e2e/actor_source_base/Dockerfile @@ -39,4 +39,4 @@ ENV PATH="/opt/venv/bin:$PATH" # Copy the Actor source code COPY . ./ -CMD ["sh", "-c", "python server.py & python -m src"] +CMD ["sh", "-c", "python server.py & python -m actor"] diff --git a/tests/e2e/actor_source_base/src/__init__.py b/tests/e2e/actor_source_base/actor/__init__.py similarity index 100% rename from tests/e2e/actor_source_base/src/__init__.py rename to tests/e2e/actor_source_base/actor/__init__.py diff --git a/tests/e2e/actor_source_base/src/__main__.py b/tests/e2e/actor_source_base/actor/__main__.py similarity index 100% rename from tests/e2e/actor_source_base/src/__main__.py rename to tests/e2e/actor_source_base/actor/__main__.py diff --git a/tests/e2e/actor_source_base/src/main.py b/tests/e2e/actor_source_base/actor/main.py similarity index 100% rename from tests/e2e/actor_source_base/src/main.py rename to tests/e2e/actor_source_base/actor/main.py diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 5cedc5c1..a00de6e1 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -241,7 +241,7 @@ async def _make_actor( ) if main_py: - source_files = {'src/main.py': main_py} + source_files = {'actor/main.py': main_py} assert source_files is not None diff --git a/tests/e2e/test_actor_key_value_store.py b/tests/e2e/test_actor_key_value_store.py index 618613e7..47edf688 100644 --- a/tests/e2e/test_actor_key_value_store.py +++ b/tests/e2e/test_actor_key_value_store.py @@ -30,7 +30,7 @@ async def test_actor_get_input_from_run( "required": ["password"] } """, - 'src/main.py': """ + 'actor/main.py': """ import asyncio from apify import Actor diff --git a/tests/e2e/test_actor_scrapy.py b/tests/e2e/test_actor_scrapy.py index 2d06910c..7c288fd1 100644 --- a/tests/e2e/test_actor_scrapy.py +++ b/tests/e2e/test_actor_scrapy.py @@ -13,16 +13,21 @@ async def test_actor_scrapy_title_spider( make_actor: MakeActorFunction, run_actor: RunActorFunction, ) -> None: - base_path = Path('docs/03_guides/code/scrapy_project') + docs_package_path = Path('docs/03_guides/code/scrapy_project/src') + file_names = ( + '__init__.py', + '__main__.py', + 'items.py', + 'main.py', + 'settings.py', + 'spiders/__init__.py', + 'spiders/title.py', + ) + # The guide's project is the `src` package of the Apify Scrapy template, while test Actors run the `actor` package. actor_source_files = { - 'src/__init__.py': (base_path / 'src/__init__.py').read_text(), - 'src/__main__.py': (base_path / 'src/__main__.py').read_text(), - 'src/items.py': (base_path / 'src/items.py').read_text(), - 'src/main.py': (base_path / 'src/main.py').read_text(), - 'src/settings.py': (base_path / 'src/settings.py').read_text(), - 'src/spiders/__init__.py': (base_path / 'src/spiders/__init__.py').read_text(), - 'src/spiders/title.py': (base_path / 'src/spiders/title.py').read_text(), + f'actor/{file_name}': (docs_package_path / file_name).read_text().replace("'src.", "'actor.") + for file_name in file_names } actor = await make_actor( diff --git a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile index 90f7c246..89f4a80e 100644 --- a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile +++ b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile @@ -10,4 +10,4 @@ RUN pip install --force-reinstall -r requirements.txt # (the base image's pre-installed browser can lag behind a newer Playwright pulled via crawlee[playwright]). RUN playwright install chromium -CMD ["sh", "-c", "python server.py & python -m src"] +CMD ["sh", "-c", "python server.py & python -m actor"] diff --git a/tests/e2e/test_crawlee/test_adaptive_playwright_crawler.py b/tests/e2e/test_crawlee/test_adaptive_playwright_crawler.py index bc84a50a..a4d5a992 100644 --- a/tests/e2e/test_crawlee/test_adaptive_playwright_crawler.py +++ b/tests/e2e/test_crawlee/test_adaptive_playwright_crawler.py @@ -12,7 +12,7 @@ async def test_adaptive_playwright_crawler(make_actor: MakeActorFunction, run_ac actor = await make_actor( label='crawl-adaptive', source_files={ - 'src/main.py': read_actor_source('main_adaptive_playwright_crawler.py'), + 'actor/main.py': read_actor_source('main_adaptive_playwright_crawler.py'), 'Dockerfile': get_playwright_dockerfile(), }, additional_requirements=['crawlee[all]'], diff --git a/tests/e2e/test_crawlee/test_basic_crawler.py b/tests/e2e/test_crawlee/test_basic_crawler.py index 8667f393..5c14ffe2 100644 --- a/tests/e2e/test_crawlee/test_basic_crawler.py +++ b/tests/e2e/test_crawlee/test_basic_crawler.py @@ -12,7 +12,7 @@ async def test_basic_crawler(make_actor: MakeActorFunction, run_actor: RunActorF actor = await make_actor( label='crawl-basic', source_files={ - 'src/main.py': read_actor_source('main_basic_crawler.py'), + 'actor/main.py': read_actor_source('main_basic_crawler.py'), }, ) run_result = await run_actor(actor) diff --git a/tests/e2e/test_crawlee/test_beautifulsoup_crawler.py b/tests/e2e/test_crawlee/test_beautifulsoup_crawler.py index 56d0a497..841ba809 100644 --- a/tests/e2e/test_crawlee/test_beautifulsoup_crawler.py +++ b/tests/e2e/test_crawlee/test_beautifulsoup_crawler.py @@ -12,7 +12,7 @@ async def test_beautifulsoup_crawler(make_actor: MakeActorFunction, run_actor: R actor = await make_actor( label='crawl-bsoup', source_files={ - 'src/main.py': read_actor_source('main_beautifulsoup_crawler.py'), + 'actor/main.py': read_actor_source('main_beautifulsoup_crawler.py'), }, additional_requirements=['crawlee[beautifulsoup]'], ) diff --git a/tests/e2e/test_crawlee/test_http_crawler.py b/tests/e2e/test_crawlee/test_http_crawler.py index 86dc0e37..f29959d1 100644 --- a/tests/e2e/test_crawlee/test_http_crawler.py +++ b/tests/e2e/test_crawlee/test_http_crawler.py @@ -12,7 +12,7 @@ async def test_http_crawler(make_actor: MakeActorFunction, run_actor: RunActorFu actor = await make_actor( label='crawl-http', source_files={ - 'src/main.py': read_actor_source('main_http_crawler.py'), + 'actor/main.py': read_actor_source('main_http_crawler.py'), }, ) run_result = await run_actor(actor) diff --git a/tests/e2e/test_crawlee/test_parsel_crawler.py b/tests/e2e/test_crawlee/test_parsel_crawler.py index 0815c9f2..aafccc55 100644 --- a/tests/e2e/test_crawlee/test_parsel_crawler.py +++ b/tests/e2e/test_crawlee/test_parsel_crawler.py @@ -12,7 +12,7 @@ async def test_parsel_crawler(make_actor: MakeActorFunction, run_actor: RunActor actor = await make_actor( label='crawl-parsel', source_files={ - 'src/main.py': read_actor_source('main_parsel_crawler.py'), + 'actor/main.py': read_actor_source('main_parsel_crawler.py'), }, ) run_result = await run_actor(actor) diff --git a/tests/e2e/test_crawlee/test_playwright_crawler.py b/tests/e2e/test_crawlee/test_playwright_crawler.py index 5d7d8244..4f586300 100644 --- a/tests/e2e/test_crawlee/test_playwright_crawler.py +++ b/tests/e2e/test_crawlee/test_playwright_crawler.py @@ -12,7 +12,7 @@ async def test_playwright_crawler(make_actor: MakeActorFunction, run_actor: RunA actor = await make_actor( label='crawl-playwright', source_files={ - 'src/main.py': read_actor_source('main_playwright_crawler.py'), + 'actor/main.py': read_actor_source('main_playwright_crawler.py'), 'Dockerfile': get_playwright_dockerfile(), }, additional_requirements=['crawlee[playwright]'], diff --git a/tests/e2e/test_fixtures.py b/tests/e2e/test_fixtures.py index ec2a02bf..e4c7db6a 100644 --- a/tests/e2e/test_fixtures.py +++ b/tests/e2e/test_fixtures.py @@ -66,13 +66,13 @@ async def test_actor_from_source_files( ) -> None: test_started_at = datetime.now(UTC) actor_source_files = { - 'src/utils.py': """ + 'actor/utils.py': """ from datetime import datetime, timezone def get_current_datetime(): return datetime.now(timezone.utc) """, - 'src/main.py': """ + 'actor/main.py': """ import asyncio from apify import Actor from .utils import get_current_datetime diff --git a/tests/e2e/test_schema_storages/test_schema_storages.py b/tests/e2e/test_schema_storages/test_schema_storages.py index 8ad98b7a..8aecefae 100644 --- a/tests/e2e/test_schema_storages/test_schema_storages.py +++ b/tests/e2e/test_schema_storages/test_schema_storages.py @@ -17,7 +17,7 @@ async def test_configuration_storages(make_actor: MakeActorFunction, run_actor: actor = await make_actor( label='schema_storages', source_files={ - 'src/main.py': read_actor_source('main.py'), + 'actor/main.py': read_actor_source('main.py'), '.actor/actor.json': read_actor_source('actor.json'), }, ) diff --git a/tests/e2e/test_scrapy/actor_source/__main__.py b/tests/e2e/test_scrapy/actor_source/__main__.py index 8443f09b..537d8e03 100644 --- a/tests/e2e/test_scrapy/actor_source/__main__.py +++ b/tests/e2e/test_scrapy/actor_source/__main__.py @@ -5,7 +5,7 @@ from .main import main from apify.scrapy import initialize_logging, run_scrapy_actor -os.environ['SCRAPY_SETTINGS_MODULE'] = 'src.settings' +os.environ['SCRAPY_SETTINGS_MODULE'] = 'actor.settings' if __name__ == '__main__': initialize_logging() diff --git a/tests/e2e/test_scrapy/actor_source/main_custom_pipeline.py b/tests/e2e/test_scrapy/actor_source/main_custom_pipeline.py index e29ae914..a8cbcef0 100644 --- a/tests/e2e/test_scrapy/actor_source/main_custom_pipeline.py +++ b/tests/e2e/test_scrapy/actor_source/main_custom_pipeline.py @@ -11,7 +11,7 @@ async def main() -> None: async with Actor: - os.environ['SCRAPY_SETTINGS_MODULE'] = 'src.settings_custom_pipeline' + os.environ['SCRAPY_SETTINGS_MODULE'] = 'actor.settings_custom_pipeline' settings = apply_apify_settings() runner = AsyncCrawlerRunner(settings) await runner.crawl(Spider, start_urls=['http://localhost:8080/']) diff --git a/tests/e2e/test_scrapy/actor_source/settings.py b/tests/e2e/test_scrapy/actor_source/settings.py index 860f9432..37ddc107 100644 --- a/tests/e2e/test_scrapy/actor_source/settings.py +++ b/tests/e2e/test_scrapy/actor_source/settings.py @@ -1,8 +1,8 @@ BOT_NAME = 'testbot' LOG_LEVEL = 'INFO' -NEWSPIDER_MODULE = 'src.spiders' +NEWSPIDER_MODULE = 'actor.spiders' ROBOTSTXT_OBEY = False -SPIDER_MODULES = ['src.spiders'] +SPIDER_MODULES = ['actor.spiders'] TELNETCONSOLE_ENABLED = False TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor' HTTPCACHE_ENABLED = False diff --git a/tests/e2e/test_scrapy/actor_source/settings_custom_pipeline.py b/tests/e2e/test_scrapy/actor_source/settings_custom_pipeline.py index 7dfa245d..69a32679 100644 --- a/tests/e2e/test_scrapy/actor_source/settings_custom_pipeline.py +++ b/tests/e2e/test_scrapy/actor_source/settings_custom_pipeline.py @@ -1,5 +1,5 @@ -from src.settings import * # noqa: F403 # ty: ignore[unresolved-import] +from actor.settings import * # noqa: F403 # ty: ignore[unresolved-import] ITEM_PIPELINES = { - 'src.pipelines.PriceCleanerPipeline': 100, + 'actor.pipelines.PriceCleanerPipeline': 100, } diff --git a/tests/e2e/test_scrapy/actor_source/spider_delayed_chain.py b/tests/e2e/test_scrapy/actor_source/spider_delayed_chain.py index 074b784b..89ae4582 100644 --- a/tests/e2e/test_scrapy/actor_source/spider_delayed_chain.py +++ b/tests/e2e/test_scrapy/actor_source/spider_delayed_chain.py @@ -30,7 +30,7 @@ class DelayedChainSpider(Spider): name = 'delayed_chain_spider' custom_settings = { # noqa: RUF012 - 'DOWNLOADER_MIDDLEWARES': {'src.spiders.spider.MarkInFlightMiddleware': 543}, + 'DOWNLOADER_MIDDLEWARES': {'actor.spiders.spider.MarkInFlightMiddleware': 543}, # One request at a time, so the test knows what is in flight when it interrupts the run. 'CONCURRENT_REQUESTS': 1, } diff --git a/tests/e2e/test_scrapy/actor_source/spider_in_flight.py b/tests/e2e/test_scrapy/actor_source/spider_in_flight.py index d1b1b3e4..f6503425 100644 --- a/tests/e2e/test_scrapy/actor_source/spider_in_flight.py +++ b/tests/e2e/test_scrapy/actor_source/spider_in_flight.py @@ -30,7 +30,7 @@ class InFlightSpider(Spider): name = 'in_flight_spider' custom_settings = { # noqa: RUF012 - 'DOWNLOADER_MIDDLEWARES': {'src.spiders.spider.MarkInFlightMiddleware': 543}, + 'DOWNLOADER_MIDDLEWARES': {'actor.spiders.spider.MarkInFlightMiddleware': 543}, # The page answers after 10 minutes; the download must not time out before the test aborts the run. 'DOWNLOAD_TIMEOUT': 600, } diff --git a/tests/e2e/test_scrapy/actor_source/spider_itemloader.py b/tests/e2e/test_scrapy/actor_source/spider_itemloader.py index fc79c588..bb5d5dcc 100644 --- a/tests/e2e/test_scrapy/actor_source/spider_itemloader.py +++ b/tests/e2e/test_scrapy/actor_source/spider_itemloader.py @@ -6,7 +6,7 @@ from scrapy import Request, Spider from scrapy.loader import ItemLoader -from src.items import ProductItem # ty: ignore[unresolved-import] +from actor.items import ProductItem # ty: ignore[unresolved-import] if TYPE_CHECKING: from collections.abc import AsyncIterator, Generator diff --git a/tests/e2e/test_scrapy/actor_source/spider_reboot.py b/tests/e2e/test_scrapy/actor_source/spider_reboot.py index f3ca35d0..f0ffa86b 100644 --- a/tests/e2e/test_scrapy/actor_source/spider_reboot.py +++ b/tests/e2e/test_scrapy/actor_source/spider_reboot.py @@ -54,7 +54,7 @@ class RebootSpider(Spider): name = 'reboot_spider' custom_settings = { # noqa: RUF012 - 'DOWNLOADER_MIDDLEWARES': {'src.spiders.spider.RebootOnceMiddleware': 543}, + 'DOWNLOADER_MIDDLEWARES': {'actor.spiders.spider.RebootOnceMiddleware': 543}, 'CONCURRENT_REQUESTS': 1, } diff --git a/tests/e2e/test_scrapy/conftest.py b/tests/e2e/test_scrapy/conftest.py index ab393425..048ae5d7 100644 --- a/tests/e2e/test_scrapy/conftest.py +++ b/tests/e2e/test_scrapy/conftest.py @@ -23,12 +23,12 @@ def get_scrapy_source_files( extra_source_files: dict[str, str] | None = None, ) -> dict[str, str]: source_files: dict[str, str] = { - 'src/__main__.py': read_actor_source('__main__.py'), - 'src/main.py': read_actor_source('main.py'), - 'src/settings.py': read_actor_source('settings.py'), - 'src/items.py': read_actor_source('items.py'), - 'src/spiders/__init__.py': f'from .spider import {spider_class_name} as Spider\n', - 'src/spiders/spider.py': read_actor_source(spider_file), + 'actor/__main__.py': read_actor_source('__main__.py'), + 'actor/main.py': read_actor_source('main.py'), + 'actor/settings.py': read_actor_source('settings.py'), + 'actor/items.py': read_actor_source('items.py'), + 'actor/spiders/__init__.py': f'from .spider import {spider_class_name} as Spider\n', + 'actor/spiders/spider.py': read_actor_source(spider_file), } if extra_source_files: source_files.update(extra_source_files) diff --git a/tests/e2e/test_scrapy/test_custom_pipeline_spider.py b/tests/e2e/test_scrapy/test_custom_pipeline_spider.py index 7c0e8307..2ccf1cd7 100644 --- a/tests/e2e/test_scrapy/test_custom_pipeline_spider.py +++ b/tests/e2e/test_scrapy/test_custom_pipeline_spider.py @@ -21,9 +21,9 @@ async def test_custom_pipeline_spider(make_actor: MakeActorFunction, run_actor: 'spider_custom_pipeline.py', 'CustomPipelineSpider', extra_source_files={ - 'src/main.py': read_actor_source('main_custom_pipeline.py'), - 'src/settings_custom_pipeline.py': read_actor_source('settings_custom_pipeline.py'), - 'src/pipelines.py': read_actor_source('pipelines.py'), + 'actor/main.py': read_actor_source('main_custom_pipeline.py'), + 'actor/settings_custom_pipeline.py': read_actor_source('settings_custom_pipeline.py'), + 'actor/pipelines.py': read_actor_source('pipelines.py'), }, ), additional_requirements=['scrapy>=2.14.0'], From 982121c4030620599242cb79cdcf8e119aca37a3 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 09:04:23 +0100 Subject: [PATCH 13/28] test: pass the Python version to the E2E Actor builds as a build argument --- tests/e2e/actor_source_base/Dockerfile | 8 ++++---- tests/e2e/conftest.py | 9 ++++----- .../e2e/test_crawlee/actor_source/playwright.Dockerfile | 5 ++++- tests/e2e/test_crawlee/conftest.py | 8 +------- 4 files changed, 13 insertions(+), 17 deletions(-) diff --git a/tests/e2e/actor_source_base/Dockerfile b/tests/e2e/actor_source_base/Dockerfile index 62418007..a3204835 100644 --- a/tests/e2e/actor_source_base/Dockerfile +++ b/tests/e2e/actor_source_base/Dockerfile @@ -1,9 +1,9 @@ -# The test fixture puts the Python version of the test run in place of BASE_IMAGE_VERSION_PLACEHOLDER. -# # Multi-stage build: # - builder: Installs the Apify SDK and the dependencies, pinned to the project lock file, into a virtual environment # - runner: Actor image with the virtual environment and the Actor source code +# The test fixture passes the Python version of the test run as a build argument +ARG PYTHON_VERSION=3.14 ARG UV_VERSION=0.13 FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv @@ -11,7 +11,7 @@ FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv # ============================================================================= # Build stage # ============================================================================= -FROM apify/actor-python:BASE_IMAGE_VERSION_PLACEHOLDER AS builder +FROM apify/actor-python:${PYTHON_VERSION} AS builder COPY --from=uv /uv /bin/ @@ -29,7 +29,7 @@ RUN uv export --project sdk --frozen --no-hashes --no-emit-project --all-extras # ============================================================================= # Runtime stage # ============================================================================= -FROM apify/actor-python:BASE_IMAGE_VERSION_PLACEHOLDER AS runner +FROM apify/actor-python:${PYTHON_VERSION} AS runner # Copy the virtual environment with the installed dependencies COPY --from=builder /opt/venv /opt/venv diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index a00de6e1..c4c6f6be 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -30,6 +30,7 @@ _TOKEN_ENV_VAR = 'APIFY_TEST_USER_API_TOKEN' _API_URL_ENV_VAR = 'APIFY_INTEGRATION_TESTS_API_URL' _SDK_ROOT_PATH = Path(__file__).parent.parent.parent.resolve() +_PYTHON_VERSION = f'{sys.version_info[0]}.{sys.version_info[1]}' _MAX_BUILD_ATTEMPTS = 2 # The Actors in these suites install browsers or Scrapy into their images, so their platform builds take several @@ -147,11 +148,6 @@ def actor_base_source_files() -> dict[str, str | bytes]: for path in sdk_paths: source_files[f'sdk/{path.relative_to(_SDK_ROOT_PATH)}'] = _read_source_file(path) - python_version = f'{sys.version_info[0]}.{sys.version_info[1]}' - source_files['Dockerfile'] = str(source_files['Dockerfile']).replace( - 'BASE_IMAGE_VERSION_PLACEHOLDER', python_version - ) - return source_files @@ -291,6 +287,9 @@ async def _make_actor( 'buildTag': 'latest', 'sourceType': 'SOURCE_FILES', 'sourceFiles': source_files_for_api, + # The Dockerfile reads the Python version of the test run as a build argument. + 'envVars': [{'name': 'PYTHON_VERSION', 'value': _PYTHON_VERSION}], + 'applyEnvVarsToBuild': True, } ], ) diff --git a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile index 89f4a80e..149f6dd6 100644 --- a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile +++ b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile @@ -1,4 +1,7 @@ -FROM apify/actor-python-playwright:PYTHON_VERSION_PLACEHOLDER +# The test fixture passes the Python version of the test run as a build argument +ARG PYTHON_VERSION=3.14 + +FROM apify/actor-python-playwright:${PYTHON_VERSION} COPY . ./ diff --git a/tests/e2e/test_crawlee/conftest.py b/tests/e2e/test_crawlee/conftest.py index e5bbbbb3..12413843 100644 --- a/tests/e2e/test_crawlee/conftest.py +++ b/tests/e2e/test_crawlee/conftest.py @@ -1,6 +1,5 @@ from __future__ import annotations -import sys from pathlib import Path from typing import TYPE_CHECKING @@ -10,8 +9,6 @@ from apify_client._models import Run from apify_client._resource_clients import ActorClientAsync -_PYTHON_VERSION = f'{sys.version_info[0]}.{sys.version_info[1]}' - _ACTOR_SOURCE_DIR = Path(__file__).parent / 'actor_source' @@ -20,10 +17,7 @@ def read_actor_source(filename: str) -> str: def get_playwright_dockerfile() -> str: - return read_actor_source('playwright.Dockerfile').replace( - 'PYTHON_VERSION_PLACEHOLDER', - _PYTHON_VERSION, - ) + return read_actor_source('playwright.Dockerfile') _EXPECTED_PRODUCTS = { From ee1d29f790579a465ec9d4071c2786a7d13b7542 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 09:04:24 +0100 Subject: [PATCH 14/28] test: install the Playwright E2E test Actors from the project lock file --- .../actor_source/playwright.Dockerfile | 41 ++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile index 149f6dd6..0f3667e3 100644 --- a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile +++ b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile @@ -1,16 +1,45 @@ +# Multi-stage build: +# - builder: Installs the Apify SDK and the dependencies, pinned to the project lock file, into a virtual environment +# - runner: Actor image with Playwright browsers, the virtual environment and the Actor source code + # The test fixture passes the Python version of the test run as a build argument ARG PYTHON_VERSION=3.14 +ARG UV_VERSION=0.13 -FROM apify/actor-python-playwright:${PYTHON_VERSION} +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv -COPY . ./ +# ============================================================================= +# Build stage +# ============================================================================= +FROM apify/actor-python-playwright:${PYTHON_VERSION} AS builder + +COPY --from=uv /uv /bin/ + +# Copy the requirements and the Apify SDK source code with the project lock file +COPY requirements.txt ./ +COPY sdk/ ./sdk/ -RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* +# Install the Apify SDK and the dependencies into a virtual environment, pinned to the project lock file +RUN uv export --project sdk --frozen --no-hashes --no-emit-project --all-extras --all-groups \ + --format requirements.txt --output-file constraints.txt \ + && uv venv /opt/venv \ + && uv pip install --python /opt/venv/bin/python ./sdk -r requirements.txt -c constraints.txt \ + && uv pip freeze --python /opt/venv/bin/python -RUN pip install --force-reinstall -r requirements.txt +# ============================================================================= +# Runtime stage +# ============================================================================= +FROM apify/actor-python-playwright:${PYTHON_VERSION} AS runner -# Reinstall the Chromium binary so it matches the just-installed Playwright version -# (the base image's pre-installed browser can lag behind a newer Playwright pulled via crawlee[playwright]). +# Copy the virtual environment with the installed dependencies +COPY --from=builder /opt/venv /opt/venv + +ENV PATH="/opt/venv/bin:$PATH" + +# Install the Chromium build that matches the installed Playwright version, which can be newer than the base image's RUN playwright install chromium +# Copy the Actor source code +COPY . ./ + CMD ["sh", "-c", "python server.py & python -m actor"] From e8c6571199ae9d1e9a4eff04c0c6d940fc449cf7 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 09:05:40 +0100 Subject: [PATCH 15/28] docs: merge the test READMEs into `tests/README.md` --- CONTRIBUTING.md | 8 +- tests/README.md | 171 ++++++++++++++++++++++++++++++++++++ tests/e2e/README.md | 100 --------------------- tests/integration/README.md | 35 -------- tests/unit/README.md | 19 ---- 5 files changed, 175 insertions(+), 158 deletions(-) create mode 100644 tests/README.md delete mode 100644 tests/e2e/README.md delete mode 100644 tests/integration/README.md delete mode 100644 tests/unit/README.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d4e10e7c..4822099f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,11 +30,11 @@ We use [ruff](https://docs.astral.sh/ruff/) for linting and formatting, [ty](htt ## Tests -Each test level has a README describing its fixtures, isolation, and required tokens. +[`tests/README.md`](tests/README.md) describes each test level, with its fixtures, isolation, and required tokens. -- **Unit tests** (`tests/unit`) mock the Apify API and need no credentials. See [`tests/unit/README.md`](tests/unit/README.md). Run them with `uv run poe unit-tests`, or a single test with `uv run pytest tests/unit/.py::`. -- **Integration tests** (`tests/integration`) call the real Apify API without deploying Actors. See [`tests/integration/README.md`](tests/integration/README.md). Run them with `uv run poe integration-tests`. -- **E2E tests** (`tests/e2e`) build and run Actors on the Apify platform. See [`tests/e2e/README.md`](tests/e2e/README.md). Run them with `uv run poe e2e-tests`. +- **Unit tests** (`tests/unit`) mock the Apify API and need no credentials. Run them with `uv run poe unit-tests`, or a single test with `uv run pytest tests/unit/.py::`. +- **Integration tests** (`tests/integration`) call the real Apify API without deploying Actors. Run them with `uv run poe integration-tests`. +- **E2E tests** (`tests/e2e`) build and run Actors on the Apify platform. Run them with `uv run poe e2e-tests`. ## Code style diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..3ab54f46 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,171 @@ +# Tests + +The tests are split into three levels: + +- [Unit tests](#unit-tests) (`tests/unit`) mock the Apify API and need no credentials. +- [Integration tests](#integration-tests) (`tests/integration`) call the real Apify API without deploying Actors. +- [E2E tests](#e2e-tests) (`tests/e2e`) build and run Actors on the Apify platform. + +Prefer the lowest level that can test what you need. Integration tests are faster than E2E tests and can be fully debugged locally, so write E2E tests only when the test needs an Actor built and running on the platform. + +## Fixtures in more than one level + +- **`prepare_test_env`** / **`_isolate_test_environment`** (autouse, all levels) — Resets global state (Actor initialization, service locator, storage) and sets `APIFY_LOCAL_STORAGE_DIR` to a temporary directory before each test. +- **`apify_client_async`** (integration and E2E) — A session-scoped `ApifyClientAsync` instance configured with the test token and API URL. + +## Unit tests + +These tests verify the SDK's internal logic in isolation, without making any real API calls. All external dependencies (Apify API, platform services) are mocked. They are fast and can be run without any environment variables or credentials. + +### Running unit tests + +```bash +uv run poe unit-tests +``` + +### Unit test fixtures + +- **`apify_client_async_patcher`** — Helper for patching `ApifyClientAsync` methods to return fixed values or replacement functions, with automatic call tracking. +- **`httpserver`** — Local HTTP server (via `pytest-httpserver`) for testing HTTP interactions without real network calls. + +### Troubleshooting + +If tests fail with `SystemExit: 0` raised from `_actor.py` when `async with Actor:` exits, the virtual environment lacks the optional extras. Without Scrapy installed, the Actor exits the process on teardown. Run `uv sync --all-extras` (or `uv run poe install-dev`). + +## Integration tests + +These tests make real requests to the Apify API, but do not build or deploy Actors on the platform. + +### Running integration tests + +```bash +export APIFY_TEST_USER_API_TOKEN= +uv run poe integration-tests +``` + +To run against a different environment, also set `APIFY_INTEGRATION_TESTS_API_URL`. + +### Structure + +| File | Description | +| --- | --- | +| `test_dataset.py` | Dataset operations | +| `test_key_value_store.py` | Key-value store operations | +| `test_request_queue.py` | Request queue operations | +| `test_storages.py` | Cross-storage tests (aliases, unnamed defaults, explicit init) | + +### Integration test fixtures + +- **`request_queue_apify`** — Creates a parametrized (`single`/`shared` access mode) Apify request queue on the platform, yields it, and drops it after the test. Defined in `conftest.py`. + +### Platform behavior to account for + +- Right after a request is added to a shared RQ, the head can still come back empty. Poll negative assertions (such as `is_finished()`) as well as positive ones. +- In the response of `update_request`, `wasAlreadyHandled` reflects the request's state before the update. +- RQ metadata counts (`handledRequestCount`, `pendingRequestCount`) lag a few seconds behind the operations that change them. + +## E2E tests + +These tests build and run Actors using the Python SDK on the Apify platform. They are slower than integration tests because they need to build and deploy Actors. + +### Running E2E tests + +```bash +export APIFY_TEST_USER_API_TOKEN= +uv run poe e2e-tests +``` + +To run against a different environment, also set `APIFY_INTEGRATION_TESTS_API_URL`. + +### E2E test fixtures + +- **`make_actor`** — Factory for creating temporary Actors on the Apify platform (built, then auto-deleted after the test). +- **`run_actor`** — Starts an Actor run and waits for completion (10 min timeout). + +### How test Actors are built + +Every test Actor gets the files from `tests/e2e/actor_source_base` and the SDK source code under `sdk/`. Its image builds in two stages (see `tests/e2e/actor_source_base/Dockerfile`): + +- The builder stage installs the SDK from `sdk/` with uv, together with the dependencies, all pinned to the project's `uv.lock`. +- The runner stage copies the resulting virtual environment and the Actor source code, and runs the `actor` package. + +The Python version of the test run reaches the build as the `PYTHON_VERSION` build argument. + +### How to write tests + +#### Creating an Actor from a Python function + +You can create Actors straight from a Python function. This is great because the test Actor source code gets checked by the linter. + +```python +async def test_something( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: + async def main() -> None: + async with Actor: + print('Hello!') + + actor = await make_actor(label='something', main_func=main) + run_result = await run_actor(actor) + + assert run_result.status == 'SUCCEEDED' +``` + +The `actor/main.py` file will be set to the function definition, prepended with `import asyncio` and `from apify import Actor`. You can add extra imports directly inside the function body. + +#### Creating an Actor from source files + +Pass the `main_py` argument for a single-file Actor: + +```python +async def test_something( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: + expected_output = f'ACTOR_OUTPUT_{crypto_random_object_id(5)}' + main_py_source = f""" + import asyncio + from datetime import datetime + from apify import Actor + async def main(): + async with Actor: + await Actor.set_value('OUTPUT', '{expected_output}') + """ + + actor = await make_actor(label='something', main_py=main_py_source) + await run_actor(actor) + + output_record = await actor.last_run().key_value_store().get_record('OUTPUT') + assert output_record is not None + assert output_record['value'] == expected_output +``` + +Or pass `source_files` for multi-file Actors: + +```python +actor_source_files = { + 'actor/utils.py': """ + from datetime import datetime, timezone + def get_current_datetime(): + return datetime.now(timezone.utc) + """, + 'actor/main.py': """ + import asyncio + from apify import Actor + from .utils import get_current_datetime + async def main(): + async with Actor: + print('Hello! It is ' + str(get_current_datetime())) + """, +} +actor = await make_actor(label='something', source_files=actor_source_files) +``` + +#### Assertions inside Actors + +Since test Actors are not executed as standard pytest tests, we don't get introspection of assertion expressions. In case of failure, only a bare `AssertionError` is shown. Always include explicit assertion messages: + +```python +assert is_finished is False, f'is_finished={is_finished}' +``` diff --git a/tests/e2e/README.md b/tests/e2e/README.md deleted file mode 100644 index 1c36a83c..00000000 --- a/tests/e2e/README.md +++ /dev/null @@ -1,100 +0,0 @@ -# E2E tests - -These tests build and run Actors using the Python SDK on the Apify platform. They are slower than integration tests (see [`tests/integration/`](../integration/)) because they need to build and deploy Actors. - -When writing new tests, prefer integration tests if possible. Only write E2E tests when you need to test something that requires building and running an Actor on the platform. - -## Running - -```bash -export APIFY_TEST_USER_API_TOKEN= -uv run poe e2e-tests -``` - -To run against a different environment, also set `APIFY_INTEGRATION_TESTS_API_URL`. - -## Key fixtures - -- **`apify_client_async`** — A session-scoped `ApifyClientAsync` instance configured with the test token and API URL. -- **`prepare_test_env`** / **`_isolate_test_environment`** (autouse) — Resets global state and sets `APIFY_LOCAL_STORAGE_DIR` to a temporary directory before each test. -- **`make_actor`** — Factory for creating temporary Actors on the Apify platform (built, then auto-deleted after the test). -- **`run_actor`** — Starts an Actor run and waits for completion (10 min timeout). - -## How to write tests - -### Creating an Actor from a Python function - -You can create Actors straight from a Python function. This is great because the test Actor source code gets checked by the linter. - -```python -async def test_something( - make_actor: MakeActorFunction, - run_actor: RunActorFunction, -) -> None: - async def main() -> None: - async with Actor: - print('Hello!') - - actor = await make_actor(label='something', main_func=main) - run_result = await run_actor(actor) - - assert run_result.status == 'SUCCEEDED' -``` - -The `actor/main.py` file will be set to the function definition, prepended with `import asyncio` and `from apify import Actor`. You can add extra imports directly inside the function body. - -### Creating an Actor from source files - -Pass the `main_py` argument for a single-file Actor: - -```python -async def test_something( - make_actor: MakeActorFunction, - run_actor: RunActorFunction, -) -> None: - expected_output = f'ACTOR_OUTPUT_{crypto_random_object_id(5)}' - main_py_source = f""" - import asyncio - from datetime import datetime - from apify import Actor - async def main(): - async with Actor: - await Actor.set_value('OUTPUT', '{expected_output}') - """ - - actor = await make_actor(label='something', main_py=main_py_source) - await run_actor(actor) - - output_record = await actor.last_run().key_value_store().get_record('OUTPUT') - assert output_record is not None - assert output_record['value'] == expected_output -``` - -Or pass `source_files` for multi-file Actors: - -```python -actor_source_files = { - 'actor/utils.py': """ - from datetime import datetime, timezone - def get_current_datetime(): - return datetime.now(timezone.utc) - """, - 'actor/main.py': """ - import asyncio - from apify import Actor - from .utils import get_current_datetime - async def main(): - async with Actor: - print('Hello! It is ' + str(get_current_datetime())) - """, -} -actor = await make_actor(label='something', source_files=actor_source_files) -``` - -### Assertions inside Actors - -Since test Actors are not executed as standard pytest tests, we don't get introspection of assertion expressions. In case of failure, only a bare `AssertionError` is shown. Always include explicit assertion messages: - -```python -assert is_finished is False, f'is_finished={is_finished}' -``` diff --git a/tests/integration/README.md b/tests/integration/README.md deleted file mode 100644 index ce49e197..00000000 --- a/tests/integration/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Integration tests - -These tests make real requests to the Apify API, but do not build or deploy Actors on the platform. They are faster than E2E tests and can be fully debugged locally. - -When writing new tests, prefer this level if possible. Only write E2E tests (see [`tests/e2e/`](../e2e/)) when you need to test something that requires building and running an Actor on the platform. - -## Running - -```bash -export APIFY_TEST_USER_API_TOKEN= -uv run poe integration-tests -``` - -To run against a different environment, also set `APIFY_INTEGRATION_TESTS_API_URL`. - -## Structure - -| File | Description | -| --- | --- | -| `test_dataset.py` | Dataset operations | -| `test_key_value_store.py` | Key-value store operations | -| `test_request_queue.py` | Request queue operations | -| `test_storages.py` | Cross-storage tests (aliases, unnamed defaults, explicit init) | - -## Key fixtures - -- **`apify_client_async`** — A session-scoped `ApifyClientAsync` instance configured with the test token and API URL. -- **`request_queue_apify`** — Creates a parametrized (`single`/`shared` access mode) Apify request queue on the platform, yields it, and drops it after the test. Defined in `conftest.py`. -- **`prepare_test_env`** / **`_isolate_test_environment`** (autouse) — Resets global state and sets `APIFY_LOCAL_STORAGE_DIR` to a temporary directory before each test. - -## Platform behavior to account for - -- Right after a request is added to a shared RQ, the head can still come back empty. Poll negative assertions (such as `is_finished()`) as well as positive ones. -- In the response of `update_request`, `wasAlreadyHandled` reflects the request's state before the update. -- RQ metadata counts (`handledRequestCount`, `pendingRequestCount`) lag a few seconds behind the operations that change them. diff --git a/tests/unit/README.md b/tests/unit/README.md deleted file mode 100644 index fb8b8c90..00000000 --- a/tests/unit/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# Unit tests - -These tests verify the SDK's internal logic in isolation, without making any real API calls. All external dependencies (Apify API, platform services) are mocked. They are fast and can be run without any environment variables or credentials. - -## Running - -```bash -uv run poe unit-tests -``` - -## Key fixtures - -- **`prepare_test_env`** / **`_isolate_test_environment`** (autouse) — Resets global state (Actor initialization, service locator, storage) and sets `APIFY_LOCAL_STORAGE_DIR` to a temporary directory before each test. -- **`apify_client_async_patcher`** — Helper for patching `ApifyClientAsync` methods to return fixed values or replacement functions, with automatic call tracking. -- **`httpserver`** — Local HTTP server (via `pytest-httpserver`) for testing HTTP interactions without real network calls. - -## Troubleshooting - -If tests fail with `SystemExit: 0` raised from `_actor.py` when `async with Actor:` exits, the virtual environment lacks the optional extras. Without Scrapy installed, the Actor exits the process on teardown. Run `uv sync --all-extras` (or `uv run poe install-dev`). From 900e39c711e465d9b738cf2e94b479326bd74098 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 09:08:45 +0100 Subject: [PATCH 16/28] docs: polish the wording of `tests/README.md` --- tests/README.md | 59 ++++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/tests/README.md b/tests/README.md index 3ab54f46..e97c8dbe 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,21 +1,21 @@ # Tests -The tests are split into three levels: +The tests have three levels: - [Unit tests](#unit-tests) (`tests/unit`) mock the Apify API and need no credentials. - [Integration tests](#integration-tests) (`tests/integration`) call the real Apify API without deploying Actors. - [E2E tests](#e2e-tests) (`tests/e2e`) build and run Actors on the Apify platform. -Prefer the lowest level that can test what you need. Integration tests are faster than E2E tests and can be fully debugged locally, so write E2E tests only when the test needs an Actor built and running on the platform. +Use the lowest level that can test what you need. Integration tests run faster than E2E tests and you can debug them locally, so write an E2E test only when the test needs an Actor built and running on the platform. -## Fixtures in more than one level +## Fixtures shared by several levels -- **`prepare_test_env`** / **`_isolate_test_environment`** (autouse, all levels) — Resets global state (Actor initialization, service locator, storage) and sets `APIFY_LOCAL_STORAGE_DIR` to a temporary directory before each test. -- **`apify_client_async`** (integration and E2E) — A session-scoped `ApifyClientAsync` instance configured with the test token and API URL. +- `prepare_test_env` and its autouse wrapper `_isolate_test_environment` (all levels): reset the global state (Actor initialization, service locator, storages) and point `APIFY_LOCAL_STORAGE_DIR` to a temporary directory before each test. +- `apify_client_async` (integration and E2E): a session-scoped `ApifyClientAsync` configured with the test token and API URL. ## Unit tests -These tests verify the SDK's internal logic in isolation, without making any real API calls. All external dependencies (Apify API, platform services) are mocked. They are fast and can be run without any environment variables or credentials. +Unit tests check the SDK's logic in isolation. They mock every external dependency, such as the Apify API and other platform services, so they're fast and need no environment variables or credentials. ### Running unit tests @@ -25,16 +25,16 @@ uv run poe unit-tests ### Unit test fixtures -- **`apify_client_async_patcher`** — Helper for patching `ApifyClientAsync` methods to return fixed values or replacement functions, with automatic call tracking. -- **`httpserver`** — Local HTTP server (via `pytest-httpserver`) for testing HTTP interactions without real network calls. +- `apify_client_async_patcher`: patches `ApifyClientAsync` methods to return fixed values or call replacement functions, and records the calls. +- `httpserver`: a local HTTP server (from `pytest-httpserver`) for testing HTTP interactions without real network calls. ### Troubleshooting -If tests fail with `SystemExit: 0` raised from `_actor.py` when `async with Actor:` exits, the virtual environment lacks the optional extras. Without Scrapy installed, the Actor exits the process on teardown. Run `uv sync --all-extras` (or `uv run poe install-dev`). +If a test fails with `SystemExit: 0` raised from `_actor.py` when `async with Actor:` exits, the virtual environment is missing the optional extras. Without Scrapy installed, the Actor exits the process on teardown. Run `uv sync --all-extras` or `uv run poe install-dev`. ## Integration tests -These tests make real requests to the Apify API, but do not build or deploy Actors on the platform. +Integration tests make real requests to the Apify API, but they don't build or deploy Actors. ### Running integration tests @@ -43,30 +43,30 @@ export APIFY_TEST_USER_API_TOKEN= uv run poe integration-tests ``` -To run against a different environment, also set `APIFY_INTEGRATION_TESTS_API_URL`. +To run them against a different environment, also set `APIFY_INTEGRATION_TESTS_API_URL`. -### Structure +### Integration test files -| File | Description | +| File | What it tests | | --- | --- | | `test_dataset.py` | Dataset operations | | `test_key_value_store.py` | Key-value store operations | | `test_request_queue.py` | Request queue operations | -| `test_storages.py` | Cross-storage tests (aliases, unnamed defaults, explicit init) | +| `test_storages.py` | Behavior across storages: aliases, unnamed default storages, explicit initialization | ### Integration test fixtures -- **`request_queue_apify`** — Creates a parametrized (`single`/`shared` access mode) Apify request queue on the platform, yields it, and drops it after the test. Defined in `conftest.py`. +- `request_queue_apify`: creates an Apify request queue on the platform, once for each access mode (`single` and `shared`), and drops it after the test. ### Platform behavior to account for -- Right after a request is added to a shared RQ, the head can still come back empty. Poll negative assertions (such as `is_finished()`) as well as positive ones. -- In the response of `update_request`, `wasAlreadyHandled` reflects the request's state before the update. +- Right after you add a request to a shared RQ, its head can still come back empty. Poll negative assertions, such as `is_finished()`, the same way as positive ones. +- In the response of `update_request`, `wasAlreadyHandled` describes the request's state before the update. - RQ metadata counts (`handledRequestCount`, `pendingRequestCount`) lag a few seconds behind the operations that change them. ## E2E tests -These tests build and run Actors using the Python SDK on the Apify platform. They are slower than integration tests because they need to build and deploy Actors. +E2E tests build Actors that use the SDK and run them on the Apify platform. Building and deploying the Actors makes them the slowest level. ### Running E2E tests @@ -75,27 +75,27 @@ export APIFY_TEST_USER_API_TOKEN= uv run poe e2e-tests ``` -To run against a different environment, also set `APIFY_INTEGRATION_TESTS_API_URL`. +To run them against a different environment, also set `APIFY_INTEGRATION_TESTS_API_URL`. ### E2E test fixtures -- **`make_actor`** — Factory for creating temporary Actors on the Apify platform (built, then auto-deleted after the test). -- **`run_actor`** — Starts an Actor run and waits for completion (10 min timeout). +- `make_actor`: creates a temporary Actor on the platform and builds it, retrying a failed build once. The Actor is deleted after the test. +- `run_actor`: starts an Actor run and waits up to 10 minutes for it to finish. ### How test Actors are built -Every test Actor gets the files from `tests/e2e/actor_source_base` and the SDK source code under `sdk/`. Its image builds in two stages (see `tests/e2e/actor_source_base/Dockerfile`): +Each test Actor gets the files from `tests/e2e/actor_source_base`, plus the SDK source code under `sdk/`. The image builds in two stages, defined in `tests/e2e/actor_source_base/Dockerfile`: -- The builder stage installs the SDK from `sdk/` with uv, together with the dependencies, all pinned to the project's `uv.lock`. +- The builder stage uses uv to install the SDK from `sdk/` and its dependencies, all pinned to the project's `uv.lock`. - The runner stage copies the resulting virtual environment and the Actor source code, and runs the `actor` package. The Python version of the test run reaches the build as the `PYTHON_VERSION` build argument. -### How to write tests +### Writing E2E tests #### Creating an Actor from a Python function -You can create Actors straight from a Python function. This is great because the test Actor source code gets checked by the linter. +You can create an Actor straight from a Python function, which means the linter also checks the Actor's code. ```python async def test_something( @@ -112,11 +112,11 @@ async def test_something( assert run_result.status == 'SUCCEEDED' ``` -The `actor/main.py` file will be set to the function definition, prepended with `import asyncio` and `from apify import Actor`. You can add extra imports directly inside the function body. +`make_actor` writes the function to `actor/main.py`, after `import asyncio` and `from apify import Actor`. Put any other imports inside the function body. #### Creating an Actor from source files -Pass the `main_py` argument for a single-file Actor: +For a single-file Actor, pass its code as `main_py`: ```python async def test_something( @@ -126,7 +126,6 @@ async def test_something( expected_output = f'ACTOR_OUTPUT_{crypto_random_object_id(5)}' main_py_source = f""" import asyncio - from datetime import datetime from apify import Actor async def main(): async with Actor: @@ -141,7 +140,7 @@ async def test_something( assert output_record['value'] == expected_output ``` -Or pass `source_files` for multi-file Actors: +For an Actor with several files, pass `source_files`: ```python actor_source_files = { @@ -164,7 +163,7 @@ actor = await make_actor(label='something', source_files=actor_source_files) #### Assertions inside Actors -Since test Actors are not executed as standard pytest tests, we don't get introspection of assertion expressions. In case of failure, only a bare `AssertionError` is shown. Always include explicit assertion messages: +Test Actors don't run under pytest, so a failed assertion inside one shows only a bare `AssertionError`, without the values involved. Always give the assertion a message: ```python assert is_finished is False, f'is_finished={is_finished}' From affc07944f504aa69fb14f2915cfa0ed25f9b070 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 09:10:19 +0100 Subject: [PATCH 17/28] test: end the continued lines of the E2E Dockerfiles with `&&` --- tests/e2e/actor_source_base/Dockerfile | 8 ++++---- tests/e2e/test_crawlee/actor_source/playwright.Dockerfile | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/e2e/actor_source_base/Dockerfile b/tests/e2e/actor_source_base/Dockerfile index a3204835..09577759 100644 --- a/tests/e2e/actor_source_base/Dockerfile +++ b/tests/e2e/actor_source_base/Dockerfile @@ -21,10 +21,10 @@ COPY sdk/ ./sdk/ # Install the Apify SDK and the dependencies into a virtual environment, pinned to the project lock file RUN uv export --project sdk --frozen --no-hashes --no-emit-project --all-extras --all-groups \ - --format requirements.txt --output-file constraints.txt \ - && uv venv /opt/venv \ - && uv pip install --python /opt/venv/bin/python ./sdk -r requirements.txt -c constraints.txt \ - && uv pip freeze --python /opt/venv/bin/python + --format requirements.txt --output-file constraints.txt && \ + uv venv /opt/venv && \ + uv pip install --python /opt/venv/bin/python ./sdk -r requirements.txt -c constraints.txt && \ + uv pip freeze --python /opt/venv/bin/python # ============================================================================= # Runtime stage diff --git a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile index 0f3667e3..05ff696c 100644 --- a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile +++ b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile @@ -21,10 +21,10 @@ COPY sdk/ ./sdk/ # Install the Apify SDK and the dependencies into a virtual environment, pinned to the project lock file RUN uv export --project sdk --frozen --no-hashes --no-emit-project --all-extras --all-groups \ - --format requirements.txt --output-file constraints.txt \ - && uv venv /opt/venv \ - && uv pip install --python /opt/venv/bin/python ./sdk -r requirements.txt -c constraints.txt \ - && uv pip freeze --python /opt/venv/bin/python + --format requirements.txt --output-file constraints.txt && \ + uv venv /opt/venv && \ + uv pip install --python /opt/venv/bin/python ./sdk -r requirements.txt -c constraints.txt && \ + uv pip freeze --python /opt/venv/bin/python # ============================================================================= # Runtime stage From 7af0aeba3162f85d19aa2d2f88590985419f2ee8 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 09:10:20 +0100 Subject: [PATCH 18/28] ci: run the E2E tests on Python 3.11 and 3.15 --- .github/workflows/_checks.yaml | 4 ++-- tests/e2e/actor_source_base/Dockerfile | 2 +- tests/e2e/test_crawlee/actor_source/playwright.Dockerfile | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_checks.yaml b/.github/workflows/_checks.yaml index ebcc662f..390578b6 100644 --- a/.github/workflows/_checks.yaml +++ b/.github/workflows/_checks.yaml @@ -218,7 +218,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ["3.11", "3.14"] + python-version: ["3.11", "3.15"] runs-on: ${{ matrix.os }} @@ -259,7 +259,7 @@ jobs: if: >- ${{ matrix.os == 'ubuntu-latest' && - matrix.python-version == '3.14' && + matrix.python-version == '3.15' && env.CODECOV_TOKEN != '' }} uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 diff --git a/tests/e2e/actor_source_base/Dockerfile b/tests/e2e/actor_source_base/Dockerfile index 09577759..67cdefcc 100644 --- a/tests/e2e/actor_source_base/Dockerfile +++ b/tests/e2e/actor_source_base/Dockerfile @@ -3,7 +3,7 @@ # - runner: Actor image with the virtual environment and the Actor source code # The test fixture passes the Python version of the test run as a build argument -ARG PYTHON_VERSION=3.14 +ARG PYTHON_VERSION=3.15 ARG UV_VERSION=0.13 FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv diff --git a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile index 05ff696c..83c2931f 100644 --- a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile +++ b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile @@ -3,7 +3,7 @@ # - runner: Actor image with Playwright browsers, the virtual environment and the Actor source code # The test fixture passes the Python version of the test run as a build argument -ARG PYTHON_VERSION=3.14 +ARG PYTHON_VERSION=3.15 ARG UV_VERSION=0.13 FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv From 6b4885e06d57d8df2f34cbb37234ed754cdd248c Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 09:11:04 +0100 Subject: [PATCH 19/28] test: require the Python version build argument in the E2E Dockerfiles --- tests/README.md | 2 +- tests/e2e/actor_source_base/Dockerfile | 4 ++-- tests/e2e/test_crawlee/actor_source/playwright.Dockerfile | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/README.md b/tests/README.md index e97c8dbe..17fb6111 100644 --- a/tests/README.md +++ b/tests/README.md @@ -89,7 +89,7 @@ Each test Actor gets the files from `tests/e2e/actor_source_base`, plus the SDK - The builder stage uses uv to install the SDK from `sdk/` and its dependencies, all pinned to the project's `uv.lock`. - The runner stage copies the resulting virtual environment and the Actor source code, and runs the `actor` package. -The Python version of the test run reaches the build as the `PYTHON_VERSION` build argument. +The Python version of the test run reaches the build as the `PYTHON_VERSION` build argument. The Dockerfiles have no default for it, so a build without it fails. ### Writing E2E tests diff --git a/tests/e2e/actor_source_base/Dockerfile b/tests/e2e/actor_source_base/Dockerfile index 67cdefcc..d12d01ad 100644 --- a/tests/e2e/actor_source_base/Dockerfile +++ b/tests/e2e/actor_source_base/Dockerfile @@ -2,8 +2,8 @@ # - builder: Installs the Apify SDK and the dependencies, pinned to the project lock file, into a virtual environment # - runner: Actor image with the virtual environment and the Actor source code -# The test fixture passes the Python version of the test run as a build argument -ARG PYTHON_VERSION=3.15 +# Required: the test fixture passes the Python version of the test run as a build argument +ARG PYTHON_VERSION ARG UV_VERSION=0.13 FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv diff --git a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile index 83c2931f..e50af0bc 100644 --- a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile +++ b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile @@ -2,8 +2,8 @@ # - builder: Installs the Apify SDK and the dependencies, pinned to the project lock file, into a virtual environment # - runner: Actor image with Playwright browsers, the virtual environment and the Actor source code -# The test fixture passes the Python version of the test run as a build argument -ARG PYTHON_VERSION=3.15 +# Required: the test fixture passes the Python version of the test run as a build argument +ARG PYTHON_VERSION ARG UV_VERSION=0.13 FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv From 1d21b2f8417ce1b9c34a5490d100bad25d50ee03 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 09:16:43 +0100 Subject: [PATCH 20/28] test: create the E2E virtual environment from the base image's Python --- tests/e2e/actor_source_base/Dockerfile | 3 +++ tests/e2e/test_crawlee/actor_source/playwright.Dockerfile | 3 +++ 2 files changed, 6 insertions(+) diff --git a/tests/e2e/actor_source_base/Dockerfile b/tests/e2e/actor_source_base/Dockerfile index d12d01ad..1e8c52d6 100644 --- a/tests/e2e/actor_source_base/Dockerfile +++ b/tests/e2e/actor_source_base/Dockerfile @@ -15,6 +15,9 @@ FROM apify/actor-python:${PYTHON_VERSION} AS builder COPY --from=uv /uv /bin/ +# Create the virtual environment from the base image's Python, which the runner stage has too +ENV UV_PYTHON_DOWNLOADS=never + # Copy the requirements and the Apify SDK source code with the project lock file COPY requirements.txt ./ COPY sdk/ ./sdk/ diff --git a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile index e50af0bc..4ac96659 100644 --- a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile +++ b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile @@ -15,6 +15,9 @@ FROM apify/actor-python-playwright:${PYTHON_VERSION} AS builder COPY --from=uv /uv /bin/ +# Create the virtual environment from the base image's Python, which the runner stage has too +ENV UV_PYTHON_DOWNLOADS=never + # Copy the requirements and the Apify SDK source code with the project lock file COPY requirements.txt ./ COPY sdk/ ./sdk/ From c6d22234a3cfee90e8d3c3f2bdfbecad3d24c612 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 09:16:44 +0100 Subject: [PATCH 21/28] test: check the installed E2E dependencies for conflicts --- tests/e2e/actor_source_base/Dockerfile | 1 + tests/e2e/test_crawlee/actor_source/playwright.Dockerfile | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/e2e/actor_source_base/Dockerfile b/tests/e2e/actor_source_base/Dockerfile index 1e8c52d6..40e64d1d 100644 --- a/tests/e2e/actor_source_base/Dockerfile +++ b/tests/e2e/actor_source_base/Dockerfile @@ -27,6 +27,7 @@ RUN uv export --project sdk --frozen --no-hashes --no-emit-project --all-extras --format requirements.txt --output-file constraints.txt && \ uv venv /opt/venv && \ uv pip install --python /opt/venv/bin/python ./sdk -r requirements.txt -c constraints.txt && \ + uv pip check --python /opt/venv/bin/python && \ uv pip freeze --python /opt/venv/bin/python # ============================================================================= diff --git a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile index 4ac96659..533b46bf 100644 --- a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile +++ b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile @@ -27,6 +27,7 @@ RUN uv export --project sdk --frozen --no-hashes --no-emit-project --all-extras --format requirements.txt --output-file constraints.txt && \ uv venv /opt/venv && \ uv pip install --python /opt/venv/bin/python ./sdk -r requirements.txt -c constraints.txt && \ + uv pip check --python /opt/venv/bin/python && \ uv pip freeze --python /opt/venv/bin/python # ============================================================================= From d0b9786b1b8058b67155a85a70f2215f775bb99e Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 09:18:12 +0100 Subject: [PATCH 22/28] test: stop listing the installed packages twice in the E2E image build --- tests/e2e/actor_source_base/Dockerfile | 3 +-- tests/e2e/test_crawlee/actor_source/playwright.Dockerfile | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/e2e/actor_source_base/Dockerfile b/tests/e2e/actor_source_base/Dockerfile index 40e64d1d..add17ef5 100644 --- a/tests/e2e/actor_source_base/Dockerfile +++ b/tests/e2e/actor_source_base/Dockerfile @@ -27,8 +27,7 @@ RUN uv export --project sdk --frozen --no-hashes --no-emit-project --all-extras --format requirements.txt --output-file constraints.txt && \ uv venv /opt/venv && \ uv pip install --python /opt/venv/bin/python ./sdk -r requirements.txt -c constraints.txt && \ - uv pip check --python /opt/venv/bin/python && \ - uv pip freeze --python /opt/venv/bin/python + uv pip check --python /opt/venv/bin/python # ============================================================================= # Runtime stage diff --git a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile index 533b46bf..0b32457c 100644 --- a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile +++ b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile @@ -27,8 +27,7 @@ RUN uv export --project sdk --frozen --no-hashes --no-emit-project --all-extras --format requirements.txt --output-file constraints.txt && \ uv venv /opt/venv && \ uv pip install --python /opt/venv/bin/python ./sdk -r requirements.txt -c constraints.txt && \ - uv pip check --python /opt/venv/bin/python && \ - uv pip freeze --python /opt/venv/bin/python + uv pip check --python /opt/venv/bin/python # ============================================================================= # Runtime stage From 217045cc29b6c4d9f6aa6fcc6b7b68fb633f20cb Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 09:19:49 +0100 Subject: [PATCH 23/28] test: put each `uv export` option on its own line in the E2E Dockerfiles --- tests/e2e/actor_source_base/Dockerfile | 11 +++++++++-- .../test_crawlee/actor_source/playwright.Dockerfile | 11 +++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/e2e/actor_source_base/Dockerfile b/tests/e2e/actor_source_base/Dockerfile index add17ef5..6712af2a 100644 --- a/tests/e2e/actor_source_base/Dockerfile +++ b/tests/e2e/actor_source_base/Dockerfile @@ -23,8 +23,15 @@ COPY requirements.txt ./ COPY sdk/ ./sdk/ # Install the Apify SDK and the dependencies into a virtual environment, pinned to the project lock file -RUN uv export --project sdk --frozen --no-hashes --no-emit-project --all-extras --all-groups \ - --format requirements.txt --output-file constraints.txt && \ +RUN uv export \ + --project sdk \ + --frozen \ + --no-hashes \ + --no-emit-project \ + --all-extras \ + --all-groups \ + --format requirements.txt \ + --output-file constraints.txt && \ uv venv /opt/venv && \ uv pip install --python /opt/venv/bin/python ./sdk -r requirements.txt -c constraints.txt && \ uv pip check --python /opt/venv/bin/python diff --git a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile index 0b32457c..57fd292b 100644 --- a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile +++ b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile @@ -23,8 +23,15 @@ COPY requirements.txt ./ COPY sdk/ ./sdk/ # Install the Apify SDK and the dependencies into a virtual environment, pinned to the project lock file -RUN uv export --project sdk --frozen --no-hashes --no-emit-project --all-extras --all-groups \ - --format requirements.txt --output-file constraints.txt && \ +RUN uv export \ + --project sdk \ + --frozen \ + --no-hashes \ + --no-emit-project \ + --all-extras \ + --all-groups \ + --format requirements.txt \ + --output-file constraints.txt && \ uv venv /opt/venv && \ uv pip install --python /opt/venv/bin/python ./sdk -r requirements.txt -c constraints.txt && \ uv pip check --python /opt/venv/bin/python From 290e248b9f7d4f70713a7a8f52d016d059ed5d56 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 09:23:50 +0100 Subject: [PATCH 24/28] test: say the base image Playwright version can differ in either direction --- tests/e2e/test_crawlee/actor_source/playwright.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile index 57fd292b..2dcd136f 100644 --- a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile +++ b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile @@ -46,7 +46,7 @@ COPY --from=builder /opt/venv /opt/venv ENV PATH="/opt/venv/bin:$PATH" -# Install the Chromium build that matches the installed Playwright version, which can be newer than the base image's +# Install the Chromium build that matches the installed Playwright version, which can differ from the base image's RUN playwright install chromium # Copy the Actor source code From 720a829d2453b738779c4e609ef35ac546e155f9 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 09:33:29 +0100 Subject: [PATCH 25/28] test: read the Playwright E2E Dockerfile with `read_actor_source` directly --- tests/e2e/test_crawlee/conftest.py | 4 ---- tests/e2e/test_crawlee/test_adaptive_playwright_crawler.py | 4 ++-- tests/e2e/test_crawlee/test_playwright_crawler.py | 4 ++-- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/e2e/test_crawlee/conftest.py b/tests/e2e/test_crawlee/conftest.py index 12413843..654a3885 100644 --- a/tests/e2e/test_crawlee/conftest.py +++ b/tests/e2e/test_crawlee/conftest.py @@ -16,10 +16,6 @@ def read_actor_source(filename: str) -> str: return (_ACTOR_SOURCE_DIR / filename).read_text() -def get_playwright_dockerfile() -> str: - return read_actor_source('playwright.Dockerfile') - - _EXPECTED_PRODUCTS = { 'Widget A': {'price': '$19.99', 'description': 'A basic widget for everyday use'}, 'Widget B': {'price': '$29.99', 'description': 'An advanced widget with extra features'}, diff --git a/tests/e2e/test_crawlee/test_adaptive_playwright_crawler.py b/tests/e2e/test_crawlee/test_adaptive_playwright_crawler.py index a4d5a992..1948e845 100644 --- a/tests/e2e/test_crawlee/test_adaptive_playwright_crawler.py +++ b/tests/e2e/test_crawlee/test_adaptive_playwright_crawler.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING -from .conftest import get_playwright_dockerfile, read_actor_source, verify_crawler_results +from .conftest import read_actor_source, verify_crawler_results if TYPE_CHECKING: from ..conftest import MakeActorFunction, RunActorFunction @@ -13,7 +13,7 @@ async def test_adaptive_playwright_crawler(make_actor: MakeActorFunction, run_ac label='crawl-adaptive', source_files={ 'actor/main.py': read_actor_source('main_adaptive_playwright_crawler.py'), - 'Dockerfile': get_playwright_dockerfile(), + 'Dockerfile': read_actor_source('playwright.Dockerfile'), }, additional_requirements=['crawlee[all]'], memory_mbytes=1024, diff --git a/tests/e2e/test_crawlee/test_playwright_crawler.py b/tests/e2e/test_crawlee/test_playwright_crawler.py index 4f586300..7c29faa3 100644 --- a/tests/e2e/test_crawlee/test_playwright_crawler.py +++ b/tests/e2e/test_crawlee/test_playwright_crawler.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING -from .conftest import get_playwright_dockerfile, read_actor_source, verify_crawler_results +from .conftest import read_actor_source, verify_crawler_results if TYPE_CHECKING: from ..conftest import MakeActorFunction, RunActorFunction @@ -13,7 +13,7 @@ async def test_playwright_crawler(make_actor: MakeActorFunction, run_actor: RunA label='crawl-playwright', source_files={ 'actor/main.py': read_actor_source('main_playwright_crawler.py'), - 'Dockerfile': get_playwright_dockerfile(), + 'Dockerfile': read_actor_source('playwright.Dockerfile'), }, additional_requirements=['crawlee[playwright]'], memory_mbytes=1024, From 92035ab819c1b06ece8753f248f43a4ba580fc8d Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 09:33:30 +0100 Subject: [PATCH 26/28] test: update the stale docstring and comment in the E2E fixture --- tests/e2e/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index c4c6f6be..2b6d11a3 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -174,7 +174,7 @@ def __call__( Args: label: The label which will be a part of the generated Actor name. main_func: The main function of the Actor. - main_py: The `src/main.py` file of the Actor. + main_py: The `actor/main.py` file of the Actor. source_files: A dictionary of the source files of the Actor. additional_requirements: A list of additional requirements to be added to the `requirements.txt`. memory_mbytes: The default memory allocation for the Actor run in MB. @@ -296,7 +296,7 @@ async def _make_actor( actor_client = client.actor(created_actor.id) - # Building the Actor on the platform involves network-dependent steps (installing system packages and + # Building the Actor on the platform involves network-dependent steps (pulling the base images and installing # Python dependencies in the Docker image), so a build can occasionally fail transiently. Retry a failed # build once before giving up. build_client_result = None From bbc9dff847aec717c439c9018869fa93b1996171 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 09:33:31 +0100 Subject: [PATCH 27/28] docs: mention the Playwright Dockerfile in `tests/README.md` --- tests/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/README.md b/tests/README.md index 17fb6111..f6d4150a 100644 --- a/tests/README.md +++ b/tests/README.md @@ -89,6 +89,8 @@ Each test Actor gets the files from `tests/e2e/actor_source_base`, plus the SDK - The builder stage uses uv to install the SDK from `sdk/` and its dependencies, all pinned to the project's `uv.lock`. - The runner stage copies the resulting virtual environment and the Actor source code, and runs the `actor` package. +The Playwright tests replace that Dockerfile with `tests/e2e/test_crawlee/actor_source/playwright.Dockerfile`, which builds the same way on top of the Playwright base image and installs the matching Chromium. + The Python version of the test run reaches the build as the `PYTHON_VERSION` build argument. The Dockerfiles have no default for it, so a build without it fails. ### Writing E2E tests From d22678371e780baa92d6f1dbbbd4d01231e91ebd Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 6 Nov 2026 09:37:20 +0100 Subject: [PATCH 28/28] docs: move the test README changes to a separate PR --- CONTRIBUTING.md | 8 +- tests/README.md | 172 ------------------------------------ tests/e2e/README.md | 100 +++++++++++++++++++++ tests/integration/README.md | 35 ++++++++ tests/unit/README.md | 19 ++++ 5 files changed, 158 insertions(+), 176 deletions(-) delete mode 100644 tests/README.md create mode 100644 tests/e2e/README.md create mode 100644 tests/integration/README.md create mode 100644 tests/unit/README.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4822099f..d4e10e7c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,11 +30,11 @@ We use [ruff](https://docs.astral.sh/ruff/) for linting and formatting, [ty](htt ## Tests -[`tests/README.md`](tests/README.md) describes each test level, with its fixtures, isolation, and required tokens. +Each test level has a README describing its fixtures, isolation, and required tokens. -- **Unit tests** (`tests/unit`) mock the Apify API and need no credentials. Run them with `uv run poe unit-tests`, or a single test with `uv run pytest tests/unit/.py::`. -- **Integration tests** (`tests/integration`) call the real Apify API without deploying Actors. Run them with `uv run poe integration-tests`. -- **E2E tests** (`tests/e2e`) build and run Actors on the Apify platform. Run them with `uv run poe e2e-tests`. +- **Unit tests** (`tests/unit`) mock the Apify API and need no credentials. See [`tests/unit/README.md`](tests/unit/README.md). Run them with `uv run poe unit-tests`, or a single test with `uv run pytest tests/unit/.py::`. +- **Integration tests** (`tests/integration`) call the real Apify API without deploying Actors. See [`tests/integration/README.md`](tests/integration/README.md). Run them with `uv run poe integration-tests`. +- **E2E tests** (`tests/e2e`) build and run Actors on the Apify platform. See [`tests/e2e/README.md`](tests/e2e/README.md). Run them with `uv run poe e2e-tests`. ## Code style diff --git a/tests/README.md b/tests/README.md deleted file mode 100644 index f6d4150a..00000000 --- a/tests/README.md +++ /dev/null @@ -1,172 +0,0 @@ -# Tests - -The tests have three levels: - -- [Unit tests](#unit-tests) (`tests/unit`) mock the Apify API and need no credentials. -- [Integration tests](#integration-tests) (`tests/integration`) call the real Apify API without deploying Actors. -- [E2E tests](#e2e-tests) (`tests/e2e`) build and run Actors on the Apify platform. - -Use the lowest level that can test what you need. Integration tests run faster than E2E tests and you can debug them locally, so write an E2E test only when the test needs an Actor built and running on the platform. - -## Fixtures shared by several levels - -- `prepare_test_env` and its autouse wrapper `_isolate_test_environment` (all levels): reset the global state (Actor initialization, service locator, storages) and point `APIFY_LOCAL_STORAGE_DIR` to a temporary directory before each test. -- `apify_client_async` (integration and E2E): a session-scoped `ApifyClientAsync` configured with the test token and API URL. - -## Unit tests - -Unit tests check the SDK's logic in isolation. They mock every external dependency, such as the Apify API and other platform services, so they're fast and need no environment variables or credentials. - -### Running unit tests - -```bash -uv run poe unit-tests -``` - -### Unit test fixtures - -- `apify_client_async_patcher`: patches `ApifyClientAsync` methods to return fixed values or call replacement functions, and records the calls. -- `httpserver`: a local HTTP server (from `pytest-httpserver`) for testing HTTP interactions without real network calls. - -### Troubleshooting - -If a test fails with `SystemExit: 0` raised from `_actor.py` when `async with Actor:` exits, the virtual environment is missing the optional extras. Without Scrapy installed, the Actor exits the process on teardown. Run `uv sync --all-extras` or `uv run poe install-dev`. - -## Integration tests - -Integration tests make real requests to the Apify API, but they don't build or deploy Actors. - -### Running integration tests - -```bash -export APIFY_TEST_USER_API_TOKEN= -uv run poe integration-tests -``` - -To run them against a different environment, also set `APIFY_INTEGRATION_TESTS_API_URL`. - -### Integration test files - -| File | What it tests | -| --- | --- | -| `test_dataset.py` | Dataset operations | -| `test_key_value_store.py` | Key-value store operations | -| `test_request_queue.py` | Request queue operations | -| `test_storages.py` | Behavior across storages: aliases, unnamed default storages, explicit initialization | - -### Integration test fixtures - -- `request_queue_apify`: creates an Apify request queue on the platform, once for each access mode (`single` and `shared`), and drops it after the test. - -### Platform behavior to account for - -- Right after you add a request to a shared RQ, its head can still come back empty. Poll negative assertions, such as `is_finished()`, the same way as positive ones. -- In the response of `update_request`, `wasAlreadyHandled` describes the request's state before the update. -- RQ metadata counts (`handledRequestCount`, `pendingRequestCount`) lag a few seconds behind the operations that change them. - -## E2E tests - -E2E tests build Actors that use the SDK and run them on the Apify platform. Building and deploying the Actors makes them the slowest level. - -### Running E2E tests - -```bash -export APIFY_TEST_USER_API_TOKEN= -uv run poe e2e-tests -``` - -To run them against a different environment, also set `APIFY_INTEGRATION_TESTS_API_URL`. - -### E2E test fixtures - -- `make_actor`: creates a temporary Actor on the platform and builds it, retrying a failed build once. The Actor is deleted after the test. -- `run_actor`: starts an Actor run and waits up to 10 minutes for it to finish. - -### How test Actors are built - -Each test Actor gets the files from `tests/e2e/actor_source_base`, plus the SDK source code under `sdk/`. The image builds in two stages, defined in `tests/e2e/actor_source_base/Dockerfile`: - -- The builder stage uses uv to install the SDK from `sdk/` and its dependencies, all pinned to the project's `uv.lock`. -- The runner stage copies the resulting virtual environment and the Actor source code, and runs the `actor` package. - -The Playwright tests replace that Dockerfile with `tests/e2e/test_crawlee/actor_source/playwright.Dockerfile`, which builds the same way on top of the Playwright base image and installs the matching Chromium. - -The Python version of the test run reaches the build as the `PYTHON_VERSION` build argument. The Dockerfiles have no default for it, so a build without it fails. - -### Writing E2E tests - -#### Creating an Actor from a Python function - -You can create an Actor straight from a Python function, which means the linter also checks the Actor's code. - -```python -async def test_something( - make_actor: MakeActorFunction, - run_actor: RunActorFunction, -) -> None: - async def main() -> None: - async with Actor: - print('Hello!') - - actor = await make_actor(label='something', main_func=main) - run_result = await run_actor(actor) - - assert run_result.status == 'SUCCEEDED' -``` - -`make_actor` writes the function to `actor/main.py`, after `import asyncio` and `from apify import Actor`. Put any other imports inside the function body. - -#### Creating an Actor from source files - -For a single-file Actor, pass its code as `main_py`: - -```python -async def test_something( - make_actor: MakeActorFunction, - run_actor: RunActorFunction, -) -> None: - expected_output = f'ACTOR_OUTPUT_{crypto_random_object_id(5)}' - main_py_source = f""" - import asyncio - from apify import Actor - async def main(): - async with Actor: - await Actor.set_value('OUTPUT', '{expected_output}') - """ - - actor = await make_actor(label='something', main_py=main_py_source) - await run_actor(actor) - - output_record = await actor.last_run().key_value_store().get_record('OUTPUT') - assert output_record is not None - assert output_record['value'] == expected_output -``` - -For an Actor with several files, pass `source_files`: - -```python -actor_source_files = { - 'actor/utils.py': """ - from datetime import datetime, timezone - def get_current_datetime(): - return datetime.now(timezone.utc) - """, - 'actor/main.py': """ - import asyncio - from apify import Actor - from .utils import get_current_datetime - async def main(): - async with Actor: - print('Hello! It is ' + str(get_current_datetime())) - """, -} -actor = await make_actor(label='something', source_files=actor_source_files) -``` - -#### Assertions inside Actors - -Test Actors don't run under pytest, so a failed assertion inside one shows only a bare `AssertionError`, without the values involved. Always give the assertion a message: - -```python -assert is_finished is False, f'is_finished={is_finished}' -``` diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 00000000..8f9d5ff1 --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,100 @@ +# E2E tests + +These tests build and run Actors using the Python SDK on the Apify platform. They are slower than integration tests (see [`tests/integration/`](../integration/)) because they need to build and deploy Actors. + +When writing new tests, prefer integration tests if possible. Only write E2E tests when you need to test something that requires building and running an Actor on the platform. + +## Running + +```bash +export APIFY_TEST_USER_API_TOKEN= +uv run poe e2e-tests +``` + +To run against a different environment, also set `APIFY_INTEGRATION_TESTS_API_URL`. + +## Key fixtures + +- **`apify_client_async`** — A session-scoped `ApifyClientAsync` instance configured with the test token and API URL. +- **`prepare_test_env`** / **`_isolate_test_environment`** (autouse) — Resets global state and sets `APIFY_LOCAL_STORAGE_DIR` to a temporary directory before each test. +- **`make_actor`** — Factory for creating temporary Actors on the Apify platform (built, then auto-deleted after the test). +- **`run_actor`** — Starts an Actor run and waits for completion (10 min timeout). + +## How to write tests + +### Creating an Actor from a Python function + +You can create Actors straight from a Python function. This is great because the test Actor source code gets checked by the linter. + +```python +async def test_something( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: + async def main() -> None: + async with Actor: + print('Hello!') + + actor = await make_actor(label='something', main_func=main) + run_result = await run_actor(actor) + + assert run_result.status == 'SUCCEEDED' +``` + +The `src/main.py` file will be set to the function definition, prepended with `import asyncio` and `from apify import Actor`. You can add extra imports directly inside the function body. + +### Creating an Actor from source files + +Pass the `main_py` argument for a single-file Actor: + +```python +async def test_something( + make_actor: MakeActorFunction, + run_actor: RunActorFunction, +) -> None: + expected_output = f'ACTOR_OUTPUT_{crypto_random_object_id(5)}' + main_py_source = f""" + import asyncio + from datetime import datetime + from apify import Actor + async def main(): + async with Actor: + await Actor.set_value('OUTPUT', '{expected_output}') + """ + + actor = await make_actor(label='something', main_py=main_py_source) + await run_actor(actor) + + output_record = await actor.last_run().key_value_store().get_record('OUTPUT') + assert output_record is not None + assert output_record['value'] == expected_output +``` + +Or pass `source_files` for multi-file Actors: + +```python +actor_source_files = { + 'src/utils.py': """ + from datetime import datetime, timezone + def get_current_datetime(): + return datetime.now(timezone.utc) + """, + 'src/main.py': """ + import asyncio + from apify import Actor + from .utils import get_current_datetime + async def main(): + async with Actor: + print('Hello! It is ' + str(get_current_datetime())) + """, +} +actor = await make_actor(label='something', source_files=actor_source_files) +``` + +### Assertions inside Actors + +Since test Actors are not executed as standard pytest tests, we don't get introspection of assertion expressions. In case of failure, only a bare `AssertionError` is shown. Always include explicit assertion messages: + +```python +assert is_finished is False, f'is_finished={is_finished}' +``` diff --git a/tests/integration/README.md b/tests/integration/README.md new file mode 100644 index 00000000..ce49e197 --- /dev/null +++ b/tests/integration/README.md @@ -0,0 +1,35 @@ +# Integration tests + +These tests make real requests to the Apify API, but do not build or deploy Actors on the platform. They are faster than E2E tests and can be fully debugged locally. + +When writing new tests, prefer this level if possible. Only write E2E tests (see [`tests/e2e/`](../e2e/)) when you need to test something that requires building and running an Actor on the platform. + +## Running + +```bash +export APIFY_TEST_USER_API_TOKEN= +uv run poe integration-tests +``` + +To run against a different environment, also set `APIFY_INTEGRATION_TESTS_API_URL`. + +## Structure + +| File | Description | +| --- | --- | +| `test_dataset.py` | Dataset operations | +| `test_key_value_store.py` | Key-value store operations | +| `test_request_queue.py` | Request queue operations | +| `test_storages.py` | Cross-storage tests (aliases, unnamed defaults, explicit init) | + +## Key fixtures + +- **`apify_client_async`** — A session-scoped `ApifyClientAsync` instance configured with the test token and API URL. +- **`request_queue_apify`** — Creates a parametrized (`single`/`shared` access mode) Apify request queue on the platform, yields it, and drops it after the test. Defined in `conftest.py`. +- **`prepare_test_env`** / **`_isolate_test_environment`** (autouse) — Resets global state and sets `APIFY_LOCAL_STORAGE_DIR` to a temporary directory before each test. + +## Platform behavior to account for + +- Right after a request is added to a shared RQ, the head can still come back empty. Poll negative assertions (such as `is_finished()`) as well as positive ones. +- In the response of `update_request`, `wasAlreadyHandled` reflects the request's state before the update. +- RQ metadata counts (`handledRequestCount`, `pendingRequestCount`) lag a few seconds behind the operations that change them. diff --git a/tests/unit/README.md b/tests/unit/README.md new file mode 100644 index 00000000..fb8b8c90 --- /dev/null +++ b/tests/unit/README.md @@ -0,0 +1,19 @@ +# Unit tests + +These tests verify the SDK's internal logic in isolation, without making any real API calls. All external dependencies (Apify API, platform services) are mocked. They are fast and can be run without any environment variables or credentials. + +## Running + +```bash +uv run poe unit-tests +``` + +## Key fixtures + +- **`prepare_test_env`** / **`_isolate_test_environment`** (autouse) — Resets global state (Actor initialization, service locator, storage) and sets `APIFY_LOCAL_STORAGE_DIR` to a temporary directory before each test. +- **`apify_client_async_patcher`** — Helper for patching `ApifyClientAsync` methods to return fixed values or replacement functions, with automatic call tracking. +- **`httpserver`** — Local HTTP server (via `pytest-httpserver`) for testing HTTP interactions without real network calls. + +## Troubleshooting + +If tests fail with `SystemExit: 0` raised from `_actor.py` when `async with Actor:` exits, the virtual environment lacks the optional extras. Without Scrapy installed, the Actor exits the process on teardown. Run `uv sync --all-extras` (or `uv run poe install-dev`).