test: install the E2E test Actor dependencies from the project lock file with uv

.github/workflows/_checks.yaml

@@ -218,7 +218,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ["3.11", "3.14"]
+        python-version: ["3.11", "3.15"]
 
     runs-on: ${{ matrix.os }}
 
@@ -259,7 +259,7 @@ jobs:
         if: >-
           ${{
             matrix.os == 'ubuntu-latest' &&
-            matrix.python-version == '3.14' &&
+            matrix.python-version == '3.15' &&
             env.CODECOV_TOKEN != ''
           }}
         uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1

pyproject.toml

@@ -70,7 +70,6 @@ dev = [
     # See https://github.com/apify/apify-client-python/pull/582/ for more details.
     # We explicitly constrain black>=24.3.0 to override the transitive dependency.
     "black>=24.3.0",
-    "build<2.0.0",
     "crawlee[parsel]",
     "griffe",
     "poethepoet<1.0.0",

tests/e2e/actor_source_base/Dockerfile

@@ -1,19 +1,52 @@
-# The test fixture will put the right Python version here
-FROM apify/actor-python:BASE_IMAGE_VERSION_PLACEHOLDER
+# Multi-stage build:
+#   - builder: Installs the Apify SDK and the dependencies, pinned to the project lock file, into a virtual environment
+#   - runner:  Actor image with the virtual environment and the Actor source code
 
-COPY . ./
+# Required: the test fixture passes the Python version of the test run as a build argument
+ARG PYTHON_VERSION
+ARG UV_VERSION=0.13
+
+FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv
+
+# =============================================================================
+# Build stage
+# =============================================================================
+FROM apify/actor-python:${PYTHON_VERSION} AS builder
+
+COPY --from=uv /uv /bin/
+
+# Create the virtual environment from the base image's Python, which the runner stage has too
+ENV UV_PYTHON_DOWNLOADS=never
 
-RUN apt-get update && apt-get install -y \
-    git \
- && rm -rf /var/lib/apt/lists/*
+# Copy the requirements and the Apify SDK source code with the project lock file
+COPY requirements.txt ./
+COPY sdk/ ./sdk/
 
-RUN echo "Python version:" \
- && python --version \
- && echo "Pip version:" \
- && pip --version \
- && echo "Installing dependencies:" \
- && pip install --force-reinstall -r requirements.txt \
- && echo "All installed Python packages:" \
- && pip freeze
+# Install the Apify SDK and the dependencies into a virtual environment, pinned to the project lock file
+RUN uv export \
+        --project sdk \
+        --frozen \
+        --no-hashes \
+        --no-emit-project \
+        --all-extras \
+        --all-groups \
+        --format requirements.txt \
+        --output-file constraints.txt && \
+    uv venv /opt/venv && \
+    uv pip install --python /opt/venv/bin/python ./sdk -r requirements.txt -c constraints.txt && \
+    uv pip check --python /opt/venv/bin/python
+
+# =============================================================================
+# Runtime stage
+# =============================================================================
+FROM apify/actor-python:${PYTHON_VERSION} AS runner
+
+# Copy the virtual environment with the installed dependencies
+COPY --from=builder /opt/venv /opt/venv
+
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Copy the Actor source code
+COPY . ./
 
-CMD ["sh", "-c", "python server.py & python -m src"]
+CMD ["sh", "-c", "python server.py & python -m actor"]

tests/e2e/actor_source_base/requirements.txt

@@ -1,4 +1,2 @@
-# The test fixture will put the Apify SDK wheel path on the next line
-APIFY_SDK_WHEEL_PLACEHOLDER
 uvicorn[standard]
-crawlee[parsel]>=1.0.0,<2.0.0,!=1.10.1
+crawlee[parsel]>=1.0.0,<2.0.0

tests/e2e/conftest.py

@@ -3,15 +3,13 @@
 import base64
 import inspect
 import os
-import subprocess
 import sys
 import textwrap
 from datetime import UTC, datetime, timedelta
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Protocol
 
 import pytest
-from filelock import FileLock
 
 from apify_client import ApifyClient, ApifyClientAsync
 from crawlee import service_locator
@@ -32,6 +30,7 @@
 _TOKEN_ENV_VAR = 'APIFY_TEST_USER_API_TOKEN'
 _API_URL_ENV_VAR = 'APIFY_INTEGRATION_TESTS_API_URL'
 _SDK_ROOT_PATH = Path(__file__).parent.parent.parent.resolve()
+_PYTHON_VERSION = f'{sys.version_info[0]}.{sys.version_info[1]}'
 _MAX_BUILD_ATTEMPTS = 2
 
 # The Actors in these suites install browsers or Scrapy into their images, so their platform builds take several
@@ -120,78 +119,34 @@ def _isolate_test_environment(prepare_test_env: Callable[[], None]) -> None:
     prepare_test_env()
 
 
-@pytest.fixture(scope='session')
-def sdk_wheel_path(tmp_path_factory: pytest.TempPathFactory, testrun_uid: str) -> Path:
-    """Build the package wheel if it hasn't been built yet, and return the path to the wheel."""
-    # `getbasetemp()` is per-xdist-worker, so both the lock and the indicator file live in its shared parent
-    # directory, which every worker sees.
-    shared_tmp_path = tmp_path_factory.getbasetemp().parent
-
-    # Make sure the wheel is not being built concurrently across all the pytest-xdist runners,
-    # through locking the building process with a temp file.
-    with FileLock(shared_tmp_path / 'sdk_wheel_build.lock'):
-        # Make sure the wheel is built exactly once across all the pytest-xdist runners, through an indicator
-        # file saying that the wheel was already built.
-        was_wheel_built_this_test_run_file = shared_tmp_path / f'wheel_was_built_in_run_{testrun_uid}'
-        if not was_wheel_built_this_test_run_file.exists():
-            subprocess.run(
-                args=[sys.executable, '-m', 'build'],
-                cwd=_SDK_ROOT_PATH,
-                check=True,
-                capture_output=True,
-            )
-            was_wheel_built_this_test_run_file.touch()
-
-        # Read the current package version, necessary for getting the right wheel filename.
-        pyproject_toml_file = (_SDK_ROOT_PATH / 'pyproject.toml').read_text(encoding='utf-8')
-        for line in pyproject_toml_file.splitlines():
-            if line.startswith('version = '):
-                delim = '"' if '"' in line else "'"
-                sdk_version = line.split(delim)[1]
-                break
-        else:
-            raise RuntimeError('Unable to find version string.')
-
-        wheel_path = _SDK_ROOT_PATH / 'dist' / f'apify-{sdk_version}-py3-none-any.whl'
-
-        # Just to be sure.
-        assert wheel_path.exists()
-
-        return wheel_path
+def _read_source_file(path: Path) -> str | bytes:
+    """Read a file to upload as an Actor source file, as text when it is valid UTF-8."""
+    try:
+        return path.read_text(encoding='utf-8')
+    except ValueError:
+        return path.read_bytes()
 
 
 @pytest.fixture(scope='session')
-def actor_base_source_files(sdk_wheel_path: Path) -> dict[str, str | bytes]:
+def actor_base_source_files() -> dict[str, str | bytes]:
     """Create a dictionary of the base source files for a testing Actor.
 
-    It takes the files from `tests/e2e/actor_source_base`, builds the Apify SDK wheel from
-    the current codebase, and adds them all together in a dictionary.
+    It takes the files from `tests/e2e/actor_source_base` and adds the Apify SDK source code under `sdk/`. The Actor
+    build installs the SDK from it, with every dependency pinned to the project lock file.
     """
     source_files: dict[str, str | bytes] = {}
 
-    # First read the actor_source_base files
     actor_source_base_path = _SDK_ROOT_PATH / 'tests/e2e/actor_source_base'
-
     for path in actor_source_base_path.glob('**/*'):
-        if not path.is_file():
-            continue
-        relative_path = str(path.relative_to(actor_source_base_path))
-        try:
-            source_files[relative_path] = path.read_text(encoding='utf-8')
-        except ValueError:
-            source_files[relative_path] = path.read_bytes()
-
-    sdk_wheel_file_name = sdk_wheel_path.name
-    source_files[sdk_wheel_file_name] = sdk_wheel_path.read_bytes()
-
-    source_files['requirements.txt'] = str(source_files['requirements.txt']).replace(
-        'APIFY_SDK_WHEEL_PLACEHOLDER', f'./{sdk_wheel_file_name}'
-    )
+        if path.is_file():
+            source_files[str(path.relative_to(actor_source_base_path))] = _read_source_file(path)
 
-    python_version = f'{sys.version_info[0]}.{sys.version_info[1]}'
-    source_files['Dockerfile'] = str(source_files['Dockerfile']).replace(
-        'BASE_IMAGE_VERSION_PLACEHOLDER', python_version
-    )
+    sdk_paths = [_SDK_ROOT_PATH / file_name for file_name in ('pyproject.toml', 'uv.lock', 'README.md', 'LICENSE')]
+    src_paths = (_SDK_ROOT_PATH / 'src').glob('**/*')
+    sdk_paths.extend(path for path in src_paths if path.is_file() and '__pycache__' not in path.parts)
+
+    for path in sdk_paths:
+        source_files[f'sdk/{path.relative_to(_SDK_ROOT_PATH)}'] = _read_source_file(path)
 
     return source_files
 
@@ -219,7 +174,7 @@ def __call__(
         Args:
             label: The label which will be a part of the generated Actor name.
             main_func: The main function of the Actor.
-            main_py: The `src/main.py` file of the Actor.
+            main_py: The `actor/main.py` file of the Actor.
             source_files: A dictionary of the source files of the Actor.
             additional_requirements: A list of additional requirements to be added to the `requirements.txt`.
             memory_mbytes: The default memory allocation for the Actor run in MB.
@@ -282,7 +237,7 @@ async def _make_actor(
             )
 
         if main_py:
-            source_files = {'src/main.py': main_py}
+            source_files = {'actor/main.py': main_py}
 
         assert source_files is not None
 
@@ -332,13 +287,16 @@ async def _make_actor(
                     'buildTag': 'latest',
                     'sourceType': 'SOURCE_FILES',
                     'sourceFiles': source_files_for_api,
+                    # The Dockerfile reads the Python version of the test run as a build argument.
+                    'envVars': [{'name': 'PYTHON_VERSION', 'value': _PYTHON_VERSION}],
+                    'applyEnvVarsToBuild': True,
                 }
             ],
         )
 
         actor_client = client.actor(created_actor.id)
 
-        # Building the Actor on the platform involves network-dependent steps (installing system packages and
+        # Building the Actor on the platform involves network-dependent steps (pulling the base images and installing
         # Python dependencies in the Docker image), so a build can occasionally fail transiently. Retry a failed
         # build once before giving up.
         build_client_result = None

tests/e2e/test_actor_key_value_store.py

@@ -30,7 +30,7 @@ async def test_actor_get_input_from_run(
                 "required": ["password"]
             }
         """,
-        'src/main.py': """
+        'actor/main.py': """
             import asyncio
             from apify import Actor
 

tests/e2e/test_actor_scrapy.py

@@ -13,16 +13,21 @@ async def test_actor_scrapy_title_spider(
     make_actor: MakeActorFunction,
     run_actor: RunActorFunction,
 ) -> None:
-    base_path = Path('docs/03_guides/code/scrapy_project')
+    docs_package_path = Path('docs/03_guides/code/scrapy_project/src')
+    file_names = (
+        '__init__.py',
+        '__main__.py',
+        'items.py',
+        'main.py',
+        'settings.py',
+        'spiders/__init__.py',
+        'spiders/title.py',
+    )
 
+    # The guide's project is the `src` package of the Apify Scrapy template, while test Actors run the `actor` package.
     actor_source_files = {
-        'src/__init__.py': (base_path / 'src/__init__.py').read_text(),
-        'src/__main__.py': (base_path / 'src/__main__.py').read_text(),
-        'src/items.py': (base_path / 'src/items.py').read_text(),
-        'src/main.py': (base_path / 'src/main.py').read_text(),
-        'src/settings.py': (base_path / 'src/settings.py').read_text(),
-        'src/spiders/__init__.py': (base_path / 'src/spiders/__init__.py').read_text(),
-        'src/spiders/title.py': (base_path / 'src/spiders/title.py').read_text(),
+        f'actor/{file_name}': (docs_package_path / file_name).read_text().replace("'src.", "'actor.")
+        for file_name in file_names
     }
 
     actor = await make_actor(
@@ -33,7 +38,7 @@ async def test_actor_scrapy_title_spider(
     run_result = await run_actor(
         actor,
         run_input={
-            'startUrls': [{'url': 'https://crawlee.dev'}],
+            'startUrls': [{'url': 'https://crawlee.dev/'}],
             'allowedDomains': ['crawlee.dev'],
             'proxyConfiguration': {'useApifyProxy': True},
         },
@@ -50,7 +55,7 @@ async def test_actor_scrapy_title_spider(
 
     # The start page and the pages it links to (`DEPTH_LIMIT` is 1, `CLOSESPIDER_PAGECOUNT` is 10), each scraped once.
     urls = [item['url'] for item in items.items]
-    assert 'https://crawlee.dev' in urls
+    assert 'https://crawlee.dev/' in urls
     assert len(urls) > 1
     assert len(urls) == len(set(urls)), urls
 

tests/e2e/test_crawlee/actor_source/playwright.Dockerfile

@@ -1,13 +1,55 @@
-FROM apify/actor-python-playwright:PYTHON_VERSION_PLACEHOLDER
+# Multi-stage build:
+#   - builder: Installs the Apify SDK and the dependencies, pinned to the project lock file, into a virtual environment
+#   - runner:  Actor image with Playwright browsers, the virtual environment and the Actor source code
 
-COPY . ./
+# Required: the test fixture passes the Python version of the test run as a build argument
+ARG PYTHON_VERSION
+ARG UV_VERSION=0.13
+
+FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv
+
+# =============================================================================
+# Build stage
+# =============================================================================
+FROM apify/actor-python-playwright:${PYTHON_VERSION} AS builder
+
+COPY --from=uv /uv /bin/
+
+# Create the virtual environment from the base image's Python, which the runner stage has too
+ENV UV_PYTHON_DOWNLOADS=never
 
-RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
+# Copy the requirements and the Apify SDK source code with the project lock file
+COPY requirements.txt ./
+COPY sdk/ ./sdk/
 
-RUN pip install --force-reinstall -r requirements.txt
+# Install the Apify SDK and the dependencies into a virtual environment, pinned to the project lock file
+RUN uv export \
+        --project sdk \
+        --frozen \
+        --no-hashes \
+        --no-emit-project \
+        --all-extras \
+        --all-groups \
+        --format requirements.txt \
+        --output-file constraints.txt && \
+    uv venv /opt/venv && \
+    uv pip install --python /opt/venv/bin/python ./sdk -r requirements.txt -c constraints.txt && \
+    uv pip check --python /opt/venv/bin/python
 
-# Reinstall the Chromium binary so it matches the just-installed Playwright version
-# (the base image's pre-installed browser can lag behind a newer Playwright pulled via crawlee[playwright]).
+# =============================================================================
+# Runtime stage
+# =============================================================================
+FROM apify/actor-python-playwright:${PYTHON_VERSION} AS runner
+
+# Copy the virtual environment with the installed dependencies
+COPY --from=builder /opt/venv /opt/venv
+
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Install the Chromium build that matches the installed Playwright version, which can differ from the base image's
 RUN playwright install chromium
 
-CMD ["sh", "-c", "python server.py & python -m src"]
+# Copy the Actor source code
+COPY . ./
+
+CMD ["sh", "-c", "python server.py & python -m actor"]