diff --git a/.github/workflows/_checks.yaml b/.github/workflows/_checks.yaml index ebcc662f9..390578b6a 100644 --- a/.github/workflows/_checks.yaml +++ b/.github/workflows/_checks.yaml @@ -218,7 +218,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ["3.11", "3.14"] + python-version: ["3.11", "3.15"] runs-on: ${{ matrix.os }} @@ -259,7 +259,7 @@ jobs: if: >- ${{ matrix.os == 'ubuntu-latest' && - matrix.python-version == '3.14' && + matrix.python-version == '3.15' && env.CODECOV_TOKEN != '' }} uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 diff --git a/pyproject.toml b/pyproject.toml index d9addafa5..b5ba89e9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,7 +70,6 @@ dev = [ # See https://github.com/apify/apify-client-python/pull/582/ for more details. # We explicitly constrain black>=24.3.0 to override the transitive dependency. "black>=24.3.0", - "build<2.0.0", "crawlee[parsel]", "griffe", "poethepoet<1.0.0", diff --git a/tests/e2e/actor_source_base/Dockerfile b/tests/e2e/actor_source_base/Dockerfile index 1e5df612a..6712af2a5 100644 --- a/tests/e2e/actor_source_base/Dockerfile +++ b/tests/e2e/actor_source_base/Dockerfile @@ -1,19 +1,52 @@ -# The test fixture will put the right Python version here -FROM apify/actor-python:BASE_IMAGE_VERSION_PLACEHOLDER +# Multi-stage build: +# - builder: Installs the Apify SDK and the dependencies, pinned to the project lock file, into a virtual environment +# - runner: Actor image with the virtual environment and the Actor source code -COPY . ./ +# Required: the test fixture passes the Python version of the test run as a build argument +ARG PYTHON_VERSION +ARG UV_VERSION=0.13 + +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv + +# ============================================================================= +# Build stage +# ============================================================================= +FROM apify/actor-python:${PYTHON_VERSION} AS builder + +COPY --from=uv /uv /bin/ + +# Create the virtual environment from the base image's Python, which the runner stage has too +ENV UV_PYTHON_DOWNLOADS=never -RUN apt-get update && apt-get install -y \ - git \ - && rm -rf /var/lib/apt/lists/* +# Copy the requirements and the Apify SDK source code with the project lock file +COPY requirements.txt ./ +COPY sdk/ ./sdk/ -RUN echo "Python version:" \ - && python --version \ - && echo "Pip version:" \ - && pip --version \ - && echo "Installing dependencies:" \ - && pip install --force-reinstall -r requirements.txt \ - && echo "All installed Python packages:" \ - && pip freeze +# Install the Apify SDK and the dependencies into a virtual environment, pinned to the project lock file +RUN uv export \ + --project sdk \ + --frozen \ + --no-hashes \ + --no-emit-project \ + --all-extras \ + --all-groups \ + --format requirements.txt \ + --output-file constraints.txt && \ + uv venv /opt/venv && \ + uv pip install --python /opt/venv/bin/python ./sdk -r requirements.txt -c constraints.txt && \ + uv pip check --python /opt/venv/bin/python + +# ============================================================================= +# Runtime stage +# ============================================================================= +FROM apify/actor-python:${PYTHON_VERSION} AS runner + +# Copy the virtual environment with the installed dependencies +COPY --from=builder /opt/venv /opt/venv + +ENV PATH="/opt/venv/bin:$PATH" + +# Copy the Actor source code +COPY . ./ -CMD ["sh", "-c", "python server.py & python -m src"] +CMD ["sh", "-c", "python server.py & python -m actor"] diff --git a/tests/e2e/actor_source_base/src/__init__.py b/tests/e2e/actor_source_base/actor/__init__.py similarity index 100% rename from tests/e2e/actor_source_base/src/__init__.py rename to tests/e2e/actor_source_base/actor/__init__.py diff --git a/tests/e2e/actor_source_base/src/__main__.py b/tests/e2e/actor_source_base/actor/__main__.py similarity index 100% rename from tests/e2e/actor_source_base/src/__main__.py rename to tests/e2e/actor_source_base/actor/__main__.py diff --git a/tests/e2e/actor_source_base/src/main.py b/tests/e2e/actor_source_base/actor/main.py similarity index 100% rename from tests/e2e/actor_source_base/src/main.py rename to tests/e2e/actor_source_base/actor/main.py diff --git a/tests/e2e/actor_source_base/requirements.txt b/tests/e2e/actor_source_base/requirements.txt index e70a7515b..bb282edbb 100644 --- a/tests/e2e/actor_source_base/requirements.txt +++ b/tests/e2e/actor_source_base/requirements.txt @@ -1,4 +1,2 @@ -# The test fixture will put the Apify SDK wheel path on the next line -APIFY_SDK_WHEEL_PLACEHOLDER uvicorn[standard] -crawlee[parsel]>=1.0.0,<2.0.0,!=1.10.1 +crawlee[parsel]>=1.0.0,<2.0.0 diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index a76dc4483..2b6d11a3c 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -3,7 +3,6 @@ import base64 import inspect import os -import subprocess import sys import textwrap from datetime import UTC, datetime, timedelta @@ -11,7 +10,6 @@ from typing import TYPE_CHECKING, Any, Protocol import pytest -from filelock import FileLock from apify_client import ApifyClient, ApifyClientAsync from crawlee import service_locator @@ -32,6 +30,7 @@ _TOKEN_ENV_VAR = 'APIFY_TEST_USER_API_TOKEN' _API_URL_ENV_VAR = 'APIFY_INTEGRATION_TESTS_API_URL' _SDK_ROOT_PATH = Path(__file__).parent.parent.parent.resolve() +_PYTHON_VERSION = f'{sys.version_info[0]}.{sys.version_info[1]}' _MAX_BUILD_ATTEMPTS = 2 # The Actors in these suites install browsers or Scrapy into their images, so their platform builds take several @@ -120,78 +119,34 @@ def _isolate_test_environment(prepare_test_env: Callable[[], None]) -> None: prepare_test_env() -@pytest.fixture(scope='session') -def sdk_wheel_path(tmp_path_factory: pytest.TempPathFactory, testrun_uid: str) -> Path: - """Build the package wheel if it hasn't been built yet, and return the path to the wheel.""" - # `getbasetemp()` is per-xdist-worker, so both the lock and the indicator file live in its shared parent - # directory, which every worker sees. - shared_tmp_path = tmp_path_factory.getbasetemp().parent - - # Make sure the wheel is not being built concurrently across all the pytest-xdist runners, - # through locking the building process with a temp file. - with FileLock(shared_tmp_path / 'sdk_wheel_build.lock'): - # Make sure the wheel is built exactly once across all the pytest-xdist runners, through an indicator - # file saying that the wheel was already built. - was_wheel_built_this_test_run_file = shared_tmp_path / f'wheel_was_built_in_run_{testrun_uid}' - if not was_wheel_built_this_test_run_file.exists(): - subprocess.run( - args=[sys.executable, '-m', 'build'], - cwd=_SDK_ROOT_PATH, - check=True, - capture_output=True, - ) - was_wheel_built_this_test_run_file.touch() - - # Read the current package version, necessary for getting the right wheel filename. - pyproject_toml_file = (_SDK_ROOT_PATH / 'pyproject.toml').read_text(encoding='utf-8') - for line in pyproject_toml_file.splitlines(): - if line.startswith('version = '): - delim = '"' if '"' in line else "'" - sdk_version = line.split(delim)[1] - break - else: - raise RuntimeError('Unable to find version string.') - - wheel_path = _SDK_ROOT_PATH / 'dist' / f'apify-{sdk_version}-py3-none-any.whl' - - # Just to be sure. - assert wheel_path.exists() - - return wheel_path +def _read_source_file(path: Path) -> str | bytes: + """Read a file to upload as an Actor source file, as text when it is valid UTF-8.""" + try: + return path.read_text(encoding='utf-8') + except ValueError: + return path.read_bytes() @pytest.fixture(scope='session') -def actor_base_source_files(sdk_wheel_path: Path) -> dict[str, str | bytes]: +def actor_base_source_files() -> dict[str, str | bytes]: """Create a dictionary of the base source files for a testing Actor. - It takes the files from `tests/e2e/actor_source_base`, builds the Apify SDK wheel from - the current codebase, and adds them all together in a dictionary. + It takes the files from `tests/e2e/actor_source_base` and adds the Apify SDK source code under `sdk/`. The Actor + build installs the SDK from it, with every dependency pinned to the project lock file. """ source_files: dict[str, str | bytes] = {} - # First read the actor_source_base files actor_source_base_path = _SDK_ROOT_PATH / 'tests/e2e/actor_source_base' - for path in actor_source_base_path.glob('**/*'): - if not path.is_file(): - continue - relative_path = str(path.relative_to(actor_source_base_path)) - try: - source_files[relative_path] = path.read_text(encoding='utf-8') - except ValueError: - source_files[relative_path] = path.read_bytes() - - sdk_wheel_file_name = sdk_wheel_path.name - source_files[sdk_wheel_file_name] = sdk_wheel_path.read_bytes() - - source_files['requirements.txt'] = str(source_files['requirements.txt']).replace( - 'APIFY_SDK_WHEEL_PLACEHOLDER', f'./{sdk_wheel_file_name}' - ) + if path.is_file(): + source_files[str(path.relative_to(actor_source_base_path))] = _read_source_file(path) - python_version = f'{sys.version_info[0]}.{sys.version_info[1]}' - source_files['Dockerfile'] = str(source_files['Dockerfile']).replace( - 'BASE_IMAGE_VERSION_PLACEHOLDER', python_version - ) + sdk_paths = [_SDK_ROOT_PATH / file_name for file_name in ('pyproject.toml', 'uv.lock', 'README.md', 'LICENSE')] + src_paths = (_SDK_ROOT_PATH / 'src').glob('**/*') + sdk_paths.extend(path for path in src_paths if path.is_file() and '__pycache__' not in path.parts) + + for path in sdk_paths: + source_files[f'sdk/{path.relative_to(_SDK_ROOT_PATH)}'] = _read_source_file(path) return source_files @@ -219,7 +174,7 @@ def __call__( Args: label: The label which will be a part of the generated Actor name. main_func: The main function of the Actor. - main_py: The `src/main.py` file of the Actor. + main_py: The `actor/main.py` file of the Actor. source_files: A dictionary of the source files of the Actor. additional_requirements: A list of additional requirements to be added to the `requirements.txt`. memory_mbytes: The default memory allocation for the Actor run in MB. @@ -282,7 +237,7 @@ async def _make_actor( ) if main_py: - source_files = {'src/main.py': main_py} + source_files = {'actor/main.py': main_py} assert source_files is not None @@ -332,13 +287,16 @@ async def _make_actor( 'buildTag': 'latest', 'sourceType': 'SOURCE_FILES', 'sourceFiles': source_files_for_api, + # The Dockerfile reads the Python version of the test run as a build argument. + 'envVars': [{'name': 'PYTHON_VERSION', 'value': _PYTHON_VERSION}], + 'applyEnvVarsToBuild': True, } ], ) actor_client = client.actor(created_actor.id) - # Building the Actor on the platform involves network-dependent steps (installing system packages and + # Building the Actor on the platform involves network-dependent steps (pulling the base images and installing # Python dependencies in the Docker image), so a build can occasionally fail transiently. Retry a failed # build once before giving up. build_client_result = None diff --git a/tests/e2e/test_actor_key_value_store.py b/tests/e2e/test_actor_key_value_store.py index 618613e76..47edf6886 100644 --- a/tests/e2e/test_actor_key_value_store.py +++ b/tests/e2e/test_actor_key_value_store.py @@ -30,7 +30,7 @@ async def test_actor_get_input_from_run( "required": ["password"] } """, - 'src/main.py': """ + 'actor/main.py': """ import asyncio from apify import Actor diff --git a/tests/e2e/test_actor_scrapy.py b/tests/e2e/test_actor_scrapy.py index 96a454c1c..7c288fd10 100644 --- a/tests/e2e/test_actor_scrapy.py +++ b/tests/e2e/test_actor_scrapy.py @@ -13,16 +13,21 @@ async def test_actor_scrapy_title_spider( make_actor: MakeActorFunction, run_actor: RunActorFunction, ) -> None: - base_path = Path('docs/03_guides/code/scrapy_project') + docs_package_path = Path('docs/03_guides/code/scrapy_project/src') + file_names = ( + '__init__.py', + '__main__.py', + 'items.py', + 'main.py', + 'settings.py', + 'spiders/__init__.py', + 'spiders/title.py', + ) + # The guide's project is the `src` package of the Apify Scrapy template, while test Actors run the `actor` package. actor_source_files = { - 'src/__init__.py': (base_path / 'src/__init__.py').read_text(), - 'src/__main__.py': (base_path / 'src/__main__.py').read_text(), - 'src/items.py': (base_path / 'src/items.py').read_text(), - 'src/main.py': (base_path / 'src/main.py').read_text(), - 'src/settings.py': (base_path / 'src/settings.py').read_text(), - 'src/spiders/__init__.py': (base_path / 'src/spiders/__init__.py').read_text(), - 'src/spiders/title.py': (base_path / 'src/spiders/title.py').read_text(), + f'actor/{file_name}': (docs_package_path / file_name).read_text().replace("'src.", "'actor.") + for file_name in file_names } actor = await make_actor( @@ -33,7 +38,7 @@ async def test_actor_scrapy_title_spider( run_result = await run_actor( actor, run_input={ - 'startUrls': [{'url': 'https://crawlee.dev'}], + 'startUrls': [{'url': 'https://crawlee.dev/'}], 'allowedDomains': ['crawlee.dev'], 'proxyConfiguration': {'useApifyProxy': True}, }, @@ -50,7 +55,7 @@ async def test_actor_scrapy_title_spider( # The start page and the pages it links to (`DEPTH_LIMIT` is 1, `CLOSESPIDER_PAGECOUNT` is 10), each scraped once. urls = [item['url'] for item in items.items] - assert 'https://crawlee.dev' in urls + assert 'https://crawlee.dev/' in urls assert len(urls) > 1 assert len(urls) == len(set(urls)), urls diff --git a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile index 90f7c246c..2dcd136f8 100644 --- a/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile +++ b/tests/e2e/test_crawlee/actor_source/playwright.Dockerfile @@ -1,13 +1,55 @@ -FROM apify/actor-python-playwright:PYTHON_VERSION_PLACEHOLDER +# Multi-stage build: +# - builder: Installs the Apify SDK and the dependencies, pinned to the project lock file, into a virtual environment +# - runner: Actor image with Playwright browsers, the virtual environment and the Actor source code -COPY . ./ +# Required: the test fixture passes the Python version of the test run as a build argument +ARG PYTHON_VERSION +ARG UV_VERSION=0.13 + +FROM ghcr.io/astral-sh/uv:${UV_VERSION} AS uv + +# ============================================================================= +# Build stage +# ============================================================================= +FROM apify/actor-python-playwright:${PYTHON_VERSION} AS builder + +COPY --from=uv /uv /bin/ + +# Create the virtual environment from the base image's Python, which the runner stage has too +ENV UV_PYTHON_DOWNLOADS=never -RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* +# Copy the requirements and the Apify SDK source code with the project lock file +COPY requirements.txt ./ +COPY sdk/ ./sdk/ -RUN pip install --force-reinstall -r requirements.txt +# Install the Apify SDK and the dependencies into a virtual environment, pinned to the project lock file +RUN uv export \ + --project sdk \ + --frozen \ + --no-hashes \ + --no-emit-project \ + --all-extras \ + --all-groups \ + --format requirements.txt \ + --output-file constraints.txt && \ + uv venv /opt/venv && \ + uv pip install --python /opt/venv/bin/python ./sdk -r requirements.txt -c constraints.txt && \ + uv pip check --python /opt/venv/bin/python -# Reinstall the Chromium binary so it matches the just-installed Playwright version -# (the base image's pre-installed browser can lag behind a newer Playwright pulled via crawlee[playwright]). +# ============================================================================= +# Runtime stage +# ============================================================================= +FROM apify/actor-python-playwright:${PYTHON_VERSION} AS runner + +# Copy the virtual environment with the installed dependencies +COPY --from=builder /opt/venv /opt/venv + +ENV PATH="/opt/venv/bin:$PATH" + +# Install the Chromium build that matches the installed Playwright version, which can differ from the base image's RUN playwright install chromium -CMD ["sh", "-c", "python server.py & python -m src"] +# Copy the Actor source code +COPY . ./ + +CMD ["sh", "-c", "python server.py & python -m actor"] diff --git a/tests/e2e/test_crawlee/conftest.py b/tests/e2e/test_crawlee/conftest.py index e5bbbbb3e..654a38858 100644 --- a/tests/e2e/test_crawlee/conftest.py +++ b/tests/e2e/test_crawlee/conftest.py @@ -1,6 +1,5 @@ from __future__ import annotations -import sys from pathlib import Path from typing import TYPE_CHECKING @@ -10,8 +9,6 @@ from apify_client._models import Run from apify_client._resource_clients import ActorClientAsync -_PYTHON_VERSION = f'{sys.version_info[0]}.{sys.version_info[1]}' - _ACTOR_SOURCE_DIR = Path(__file__).parent / 'actor_source' @@ -19,13 +16,6 @@ def read_actor_source(filename: str) -> str: return (_ACTOR_SOURCE_DIR / filename).read_text() -def get_playwright_dockerfile() -> str: - return read_actor_source('playwright.Dockerfile').replace( - 'PYTHON_VERSION_PLACEHOLDER', - _PYTHON_VERSION, - ) - - _EXPECTED_PRODUCTS = { 'Widget A': {'price': '$19.99', 'description': 'A basic widget for everyday use'}, 'Widget B': {'price': '$29.99', 'description': 'An advanced widget with extra features'}, diff --git a/tests/e2e/test_crawlee/test_adaptive_playwright_crawler.py b/tests/e2e/test_crawlee/test_adaptive_playwright_crawler.py index bc84a50a9..1948e8459 100644 --- a/tests/e2e/test_crawlee/test_adaptive_playwright_crawler.py +++ b/tests/e2e/test_crawlee/test_adaptive_playwright_crawler.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING -from .conftest import get_playwright_dockerfile, read_actor_source, verify_crawler_results +from .conftest import read_actor_source, verify_crawler_results if TYPE_CHECKING: from ..conftest import MakeActorFunction, RunActorFunction @@ -12,8 +12,8 @@ async def test_adaptive_playwright_crawler(make_actor: MakeActorFunction, run_ac actor = await make_actor( label='crawl-adaptive', source_files={ - 'src/main.py': read_actor_source('main_adaptive_playwright_crawler.py'), - 'Dockerfile': get_playwright_dockerfile(), + 'actor/main.py': read_actor_source('main_adaptive_playwright_crawler.py'), + 'Dockerfile': read_actor_source('playwright.Dockerfile'), }, additional_requirements=['crawlee[all]'], memory_mbytes=1024, diff --git a/tests/e2e/test_crawlee/test_basic_crawler.py b/tests/e2e/test_crawlee/test_basic_crawler.py index 8667f393a..5c14ffe28 100644 --- a/tests/e2e/test_crawlee/test_basic_crawler.py +++ b/tests/e2e/test_crawlee/test_basic_crawler.py @@ -12,7 +12,7 @@ async def test_basic_crawler(make_actor: MakeActorFunction, run_actor: RunActorF actor = await make_actor( label='crawl-basic', source_files={ - 'src/main.py': read_actor_source('main_basic_crawler.py'), + 'actor/main.py': read_actor_source('main_basic_crawler.py'), }, ) run_result = await run_actor(actor) diff --git a/tests/e2e/test_crawlee/test_beautifulsoup_crawler.py b/tests/e2e/test_crawlee/test_beautifulsoup_crawler.py index 56d0a4975..841ba8097 100644 --- a/tests/e2e/test_crawlee/test_beautifulsoup_crawler.py +++ b/tests/e2e/test_crawlee/test_beautifulsoup_crawler.py @@ -12,7 +12,7 @@ async def test_beautifulsoup_crawler(make_actor: MakeActorFunction, run_actor: R actor = await make_actor( label='crawl-bsoup', source_files={ - 'src/main.py': read_actor_source('main_beautifulsoup_crawler.py'), + 'actor/main.py': read_actor_source('main_beautifulsoup_crawler.py'), }, additional_requirements=['crawlee[beautifulsoup]'], ) diff --git a/tests/e2e/test_crawlee/test_http_crawler.py b/tests/e2e/test_crawlee/test_http_crawler.py index 86dc0e379..f29959d16 100644 --- a/tests/e2e/test_crawlee/test_http_crawler.py +++ b/tests/e2e/test_crawlee/test_http_crawler.py @@ -12,7 +12,7 @@ async def test_http_crawler(make_actor: MakeActorFunction, run_actor: RunActorFu actor = await make_actor( label='crawl-http', source_files={ - 'src/main.py': read_actor_source('main_http_crawler.py'), + 'actor/main.py': read_actor_source('main_http_crawler.py'), }, ) run_result = await run_actor(actor) diff --git a/tests/e2e/test_crawlee/test_parsel_crawler.py b/tests/e2e/test_crawlee/test_parsel_crawler.py index 0815c9f2d..aafccc558 100644 --- a/tests/e2e/test_crawlee/test_parsel_crawler.py +++ b/tests/e2e/test_crawlee/test_parsel_crawler.py @@ -12,7 +12,7 @@ async def test_parsel_crawler(make_actor: MakeActorFunction, run_actor: RunActor actor = await make_actor( label='crawl-parsel', source_files={ - 'src/main.py': read_actor_source('main_parsel_crawler.py'), + 'actor/main.py': read_actor_source('main_parsel_crawler.py'), }, ) run_result = await run_actor(actor) diff --git a/tests/e2e/test_crawlee/test_playwright_crawler.py b/tests/e2e/test_crawlee/test_playwright_crawler.py index 5d7d8244e..7c29faa33 100644 --- a/tests/e2e/test_crawlee/test_playwright_crawler.py +++ b/tests/e2e/test_crawlee/test_playwright_crawler.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING -from .conftest import get_playwright_dockerfile, read_actor_source, verify_crawler_results +from .conftest import read_actor_source, verify_crawler_results if TYPE_CHECKING: from ..conftest import MakeActorFunction, RunActorFunction @@ -12,8 +12,8 @@ async def test_playwright_crawler(make_actor: MakeActorFunction, run_actor: RunA actor = await make_actor( label='crawl-playwright', source_files={ - 'src/main.py': read_actor_source('main_playwright_crawler.py'), - 'Dockerfile': get_playwright_dockerfile(), + 'actor/main.py': read_actor_source('main_playwright_crawler.py'), + 'Dockerfile': read_actor_source('playwright.Dockerfile'), }, additional_requirements=['crawlee[playwright]'], memory_mbytes=1024, diff --git a/tests/e2e/test_fixtures.py b/tests/e2e/test_fixtures.py index ec2a02bfe..e4c7db6a1 100644 --- a/tests/e2e/test_fixtures.py +++ b/tests/e2e/test_fixtures.py @@ -66,13 +66,13 @@ async def test_actor_from_source_files( ) -> None: test_started_at = datetime.now(UTC) actor_source_files = { - 'src/utils.py': """ + 'actor/utils.py': """ from datetime import datetime, timezone def get_current_datetime(): return datetime.now(timezone.utc) """, - 'src/main.py': """ + 'actor/main.py': """ import asyncio from apify import Actor from .utils import get_current_datetime diff --git a/tests/e2e/test_schema_storages/test_schema_storages.py b/tests/e2e/test_schema_storages/test_schema_storages.py index 8ad98b7a6..8aecefaee 100644 --- a/tests/e2e/test_schema_storages/test_schema_storages.py +++ b/tests/e2e/test_schema_storages/test_schema_storages.py @@ -17,7 +17,7 @@ async def test_configuration_storages(make_actor: MakeActorFunction, run_actor: actor = await make_actor( label='schema_storages', source_files={ - 'src/main.py': read_actor_source('main.py'), + 'actor/main.py': read_actor_source('main.py'), '.actor/actor.json': read_actor_source('actor.json'), }, ) diff --git a/tests/e2e/test_scrapy/actor_source/__main__.py b/tests/e2e/test_scrapy/actor_source/__main__.py index 8443f09b4..537d8e03f 100644 --- a/tests/e2e/test_scrapy/actor_source/__main__.py +++ b/tests/e2e/test_scrapy/actor_source/__main__.py @@ -5,7 +5,7 @@ from .main import main from apify.scrapy import initialize_logging, run_scrapy_actor -os.environ['SCRAPY_SETTINGS_MODULE'] = 'src.settings' +os.environ['SCRAPY_SETTINGS_MODULE'] = 'actor.settings' if __name__ == '__main__': initialize_logging() diff --git a/tests/e2e/test_scrapy/actor_source/main_custom_pipeline.py b/tests/e2e/test_scrapy/actor_source/main_custom_pipeline.py index e29ae9144..a8cbcef07 100644 --- a/tests/e2e/test_scrapy/actor_source/main_custom_pipeline.py +++ b/tests/e2e/test_scrapy/actor_source/main_custom_pipeline.py @@ -11,7 +11,7 @@ async def main() -> None: async with Actor: - os.environ['SCRAPY_SETTINGS_MODULE'] = 'src.settings_custom_pipeline' + os.environ['SCRAPY_SETTINGS_MODULE'] = 'actor.settings_custom_pipeline' settings = apply_apify_settings() runner = AsyncCrawlerRunner(settings) await runner.crawl(Spider, start_urls=['http://localhost:8080/']) diff --git a/tests/e2e/test_scrapy/actor_source/settings.py b/tests/e2e/test_scrapy/actor_source/settings.py index 860f94323..37ddc1071 100644 --- a/tests/e2e/test_scrapy/actor_source/settings.py +++ b/tests/e2e/test_scrapy/actor_source/settings.py @@ -1,8 +1,8 @@ BOT_NAME = 'testbot' LOG_LEVEL = 'INFO' -NEWSPIDER_MODULE = 'src.spiders' +NEWSPIDER_MODULE = 'actor.spiders' ROBOTSTXT_OBEY = False -SPIDER_MODULES = ['src.spiders'] +SPIDER_MODULES = ['actor.spiders'] TELNETCONSOLE_ENABLED = False TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor' HTTPCACHE_ENABLED = False diff --git a/tests/e2e/test_scrapy/actor_source/settings_custom_pipeline.py b/tests/e2e/test_scrapy/actor_source/settings_custom_pipeline.py index 7dfa245dc..69a326796 100644 --- a/tests/e2e/test_scrapy/actor_source/settings_custom_pipeline.py +++ b/tests/e2e/test_scrapy/actor_source/settings_custom_pipeline.py @@ -1,5 +1,5 @@ -from src.settings import * # noqa: F403 # ty: ignore[unresolved-import] +from actor.settings import * # noqa: F403 # ty: ignore[unresolved-import] ITEM_PIPELINES = { - 'src.pipelines.PriceCleanerPipeline': 100, + 'actor.pipelines.PriceCleanerPipeline': 100, } diff --git a/tests/e2e/test_scrapy/actor_source/spider_delayed_chain.py b/tests/e2e/test_scrapy/actor_source/spider_delayed_chain.py index 074b784ba..89ae45820 100644 --- a/tests/e2e/test_scrapy/actor_source/spider_delayed_chain.py +++ b/tests/e2e/test_scrapy/actor_source/spider_delayed_chain.py @@ -30,7 +30,7 @@ class DelayedChainSpider(Spider): name = 'delayed_chain_spider' custom_settings = { # noqa: RUF012 - 'DOWNLOADER_MIDDLEWARES': {'src.spiders.spider.MarkInFlightMiddleware': 543}, + 'DOWNLOADER_MIDDLEWARES': {'actor.spiders.spider.MarkInFlightMiddleware': 543}, # One request at a time, so the test knows what is in flight when it interrupts the run. 'CONCURRENT_REQUESTS': 1, } diff --git a/tests/e2e/test_scrapy/actor_source/spider_in_flight.py b/tests/e2e/test_scrapy/actor_source/spider_in_flight.py index d1b1b3e46..f65034256 100644 --- a/tests/e2e/test_scrapy/actor_source/spider_in_flight.py +++ b/tests/e2e/test_scrapy/actor_source/spider_in_flight.py @@ -30,7 +30,7 @@ class InFlightSpider(Spider): name = 'in_flight_spider' custom_settings = { # noqa: RUF012 - 'DOWNLOADER_MIDDLEWARES': {'src.spiders.spider.MarkInFlightMiddleware': 543}, + 'DOWNLOADER_MIDDLEWARES': {'actor.spiders.spider.MarkInFlightMiddleware': 543}, # The page answers after 10 minutes; the download must not time out before the test aborts the run. 'DOWNLOAD_TIMEOUT': 600, } diff --git a/tests/e2e/test_scrapy/actor_source/spider_itemloader.py b/tests/e2e/test_scrapy/actor_source/spider_itemloader.py index fc79c588e..bb5d5dcca 100644 --- a/tests/e2e/test_scrapy/actor_source/spider_itemloader.py +++ b/tests/e2e/test_scrapy/actor_source/spider_itemloader.py @@ -6,7 +6,7 @@ from scrapy import Request, Spider from scrapy.loader import ItemLoader -from src.items import ProductItem # ty: ignore[unresolved-import] +from actor.items import ProductItem # ty: ignore[unresolved-import] if TYPE_CHECKING: from collections.abc import AsyncIterator, Generator diff --git a/tests/e2e/test_scrapy/actor_source/spider_reboot.py b/tests/e2e/test_scrapy/actor_source/spider_reboot.py index f3ca35d0c..f0ffa86b0 100644 --- a/tests/e2e/test_scrapy/actor_source/spider_reboot.py +++ b/tests/e2e/test_scrapy/actor_source/spider_reboot.py @@ -54,7 +54,7 @@ class RebootSpider(Spider): name = 'reboot_spider' custom_settings = { # noqa: RUF012 - 'DOWNLOADER_MIDDLEWARES': {'src.spiders.spider.RebootOnceMiddleware': 543}, + 'DOWNLOADER_MIDDLEWARES': {'actor.spiders.spider.RebootOnceMiddleware': 543}, 'CONCURRENT_REQUESTS': 1, } diff --git a/tests/e2e/test_scrapy/conftest.py b/tests/e2e/test_scrapy/conftest.py index ab3934254..048ae5d71 100644 --- a/tests/e2e/test_scrapy/conftest.py +++ b/tests/e2e/test_scrapy/conftest.py @@ -23,12 +23,12 @@ def get_scrapy_source_files( extra_source_files: dict[str, str] | None = None, ) -> dict[str, str]: source_files: dict[str, str] = { - 'src/__main__.py': read_actor_source('__main__.py'), - 'src/main.py': read_actor_source('main.py'), - 'src/settings.py': read_actor_source('settings.py'), - 'src/items.py': read_actor_source('items.py'), - 'src/spiders/__init__.py': f'from .spider import {spider_class_name} as Spider\n', - 'src/spiders/spider.py': read_actor_source(spider_file), + 'actor/__main__.py': read_actor_source('__main__.py'), + 'actor/main.py': read_actor_source('main.py'), + 'actor/settings.py': read_actor_source('settings.py'), + 'actor/items.py': read_actor_source('items.py'), + 'actor/spiders/__init__.py': f'from .spider import {spider_class_name} as Spider\n', + 'actor/spiders/spider.py': read_actor_source(spider_file), } if extra_source_files: source_files.update(extra_source_files) diff --git a/tests/e2e/test_scrapy/test_custom_pipeline_spider.py b/tests/e2e/test_scrapy/test_custom_pipeline_spider.py index 7c0e8307a..2ccf1cd7d 100644 --- a/tests/e2e/test_scrapy/test_custom_pipeline_spider.py +++ b/tests/e2e/test_scrapy/test_custom_pipeline_spider.py @@ -21,9 +21,9 @@ async def test_custom_pipeline_spider(make_actor: MakeActorFunction, run_actor: 'spider_custom_pipeline.py', 'CustomPipelineSpider', extra_source_files={ - 'src/main.py': read_actor_source('main_custom_pipeline.py'), - 'src/settings_custom_pipeline.py': read_actor_source('settings_custom_pipeline.py'), - 'src/pipelines.py': read_actor_source('pipelines.py'), + 'actor/main.py': read_actor_source('main_custom_pipeline.py'), + 'actor/settings_custom_pipeline.py': read_actor_source('settings_custom_pipeline.py'), + 'actor/pipelines.py': read_actor_source('pipelines.py'), }, ), additional_requirements=['scrapy>=2.14.0'], diff --git a/uv.lock b/uv.lock index c079a03ed..c23d73c94 100644 --- a/uv.lock +++ b/uv.lock @@ -246,7 +246,6 @@ scrapy = [ [package.dev-dependencies] dev = [ { name = "black" }, - { name = "build" }, { name = "crawlee", extra = ["parsel"] }, { name = "griffe" }, { name = "poethepoet" }, @@ -287,7 +286,6 @@ provides-extras = ["scrapy"] [package.metadata.requires-dev] dev = [ { name = "black", specifier = ">=24.3.0" }, - { name = "build", specifier = "<2.0.0" }, { name = "crawlee", extras = ["parsel"] }, { name = "griffe" }, { name = "poethepoet", specifier = "<1.0.0" }, @@ -580,20 +578,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/42/da/f77571a1e13025991a6e1e265997ea4b5d18dd5092e9fb309531cfb2a305/brotlicffi-1.2.1.0-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:52b99120543ffa3b0667e922c6a7dc6b6e01350f7fd38f21aaca511bb9f8b4f2", size = 465300, upload-time = "2026-10-22T16:19:30.988Z" }, ] -[[package]] -name = "build" -version = "1.6.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "os_name == 'nt'" }, - { name = "packaging" }, - { name = "pyproject-hooks" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/bd/67/4898a44ea4f3f8e213b0954ec0aa0a16971d62a6212d6ea3931e97115b99/build-1.6.1.tar.gz", hash = "sha256:51cc11666391ab6f092070437ac747002ff46f3e4113a3622177ee6b488bfc53", size = 113427, upload-time = "2026-09-10T07:56:00.417Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ad/9b/9fb3585dabcd73a1b2a6267f63f62649347c9e6d072c9fde365b105abb2c/build-1.6.1-py3-none-any.whl", hash = "sha256:ecd351a4be9d35a9eaaba244a7687143c9c7d4aea6ac964e7e7ddab20cbcf4e7", size = 31179, upload-time = "2026-09-10T07:55:59.148Z" }, -] - [[package]] name = "cachetools" version = "7.2.8" @@ -2704,15 +2688,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/ad/2cf6d3fa2fae5c79e1ed9960c0d42badd0f94d81dd12b50604cdc839e648/pyopenssl-26.4.0-py3-none-any.whl", hash = "sha256:f0eb0cb2d581d3ad2b9c489468485e7f2ab6727d08401bcf9d824c3caddf3c1c", size = 56026, upload-time = "2026-08-01T19:50:48.94Z" }, ] -[[package]] -name = "pyproject-hooks" -version = "1.3.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6d/5d/f2ddeef4a855a102aaae5e97826a0260007522ab504421b75addfdb1517c/pyproject_hooks-1.3.3.tar.gz", hash = "sha256:defda19b854fa0d3bd4f76ea4ddcba8abd7dcfcdd585a6690ade050744fc5f43", size = 21013, upload-time = "2026-09-16T08:58:03.999Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/85/11/044d1ae1b4ec0d7af88ee5bc91e081be1022533032b906a7bdabdbb60977/pyproject_hooks-1.3.3-py3-none-any.whl", hash = "sha256:5fc53fdac9f7bd63fbcdc868fb5f90b4784d78a53a3d3388cd738b807441a20b", size = 10724, upload-time = "2026-09-16T08:58:02.96Z" }, -] - [[package]] name = "pypydispatcher" version = "2.1.2"