diff --git a/docs/02_concepts/05_proxy_management.mdx b/docs/02_concepts/05_proxy_management.mdx index 5397714cd..f4ead4b1b 100644 --- a/docs/02_concepts/05_proxy_management.mdx +++ b/docs/02_concepts/05_proxy_management.mdx @@ -20,7 +20,7 @@ The Apify SDK provides built-in proxy management through the `Actor.create_proxy_configuration()` logs a warning and returns `None`, and the Actor runs without a proxy. ### Using Apify Proxy diff --git a/docs/02_concepts/13_exceptions.mdx b/docs/02_concepts/13_exceptions.mdx index 2bec78ed7..7fd54b47d 100644 --- a/docs/02_concepts/13_exceptions.mdx +++ b/docs/02_concepts/13_exceptions.mdx @@ -43,7 +43,7 @@ The SDK raises standard Python exceptions when it's used incorrectly or given in - [`RuntimeError`](https://docs.python.org/3/library/exceptions.html#RuntimeError) when an `Actor` method is used outside the `async with Actor:` block, either before initialization or after exit, or when the Actor is initialized twice. - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) for an invalid argument, such as a malformed `timeout`, an invalid proxy configuration, charging an automatically charged event by hand, or pushing data that is not JSON-serializable or is over the size limit. - [`TypeError`](https://docs.python.org/3/library/exceptions.html#TypeError) for an argument of the wrong type. -- [`ConnectionError`](https://docs.python.org/3/library/exceptions.html#ConnectionError) when `Actor.create_proxy_configuration` verifies Apify Proxy access and the proxy reports that you have none. +- [`ConnectionError`](https://docs.python.org/3/library/exceptions.html#ConnectionError) when `Actor.create_proxy_configuration` verifies Apify Proxy access on the Apify platform and the proxy reports that you have none. In a local run, the method logs a warning and returns `None` instead. ## Run failures diff --git a/docs/02_concepts/code/05_apify_proxy.py b/docs/02_concepts/code/05_apify_proxy.py index 12d1dcca2..902ec1f74 100644 --- a/docs/02_concepts/code/05_apify_proxy.py +++ b/docs/02_concepts/code/05_apify_proxy.py @@ -8,7 +8,8 @@ async def main() -> None: proxy_cfg = await Actor.create_proxy_configuration() if not proxy_cfg: - raise RuntimeError('No proxy configuration available.') + Actor.log.warning('Apify Proxy is not available, exiting...') + return proxy_url = await proxy_cfg.new_url() Actor.log.info(f'Using proxy URL: {proxy_url}') diff --git a/docs/02_concepts/code/05_apify_proxy_config.py b/docs/02_concepts/code/05_apify_proxy_config.py index 40dac7387..948c0fd79 100644 --- a/docs/02_concepts/code/05_apify_proxy_config.py +++ b/docs/02_concepts/code/05_apify_proxy_config.py @@ -12,7 +12,8 @@ async def main() -> None: ) if not proxy_cfg: - raise RuntimeError('No proxy configuration available.') + Actor.log.warning('Apify Proxy is not available, exiting...') + return proxy_url = await proxy_cfg.new_url() Actor.log.info(f'Proxy URL: {proxy_url}') diff --git a/docs/03_guides/code/05_crawlee_beautifulsoup.py b/docs/03_guides/code/05_crawlee_beautifulsoup.py index bdd1603b0..c7cadd8a8 100644 --- a/docs/03_guides/code/05_crawlee_beautifulsoup.py +++ b/docs/03_guides/code/05_crawlee_beautifulsoup.py @@ -42,10 +42,9 @@ async def main() -> None: Actor.log.info('No start URLs specified in Actor input, exiting...') await Actor.exit() - # Crawlee rotates the proxy URL per request on its own. + # Crawlee rotates the proxy URL per request on its own. A local run without + # Apify Proxy access gets no proxy configuration and crawls directly. proxy_configuration = await Actor.create_proxy_configuration() - if proxy_configuration is None: - raise RuntimeError('Failed to create the proxy configuration.') crawler = BeautifulSoupCrawler( proxy_configuration=proxy_configuration, diff --git a/docs/03_guides/code/05_crawlee_parsel.py b/docs/03_guides/code/05_crawlee_parsel.py index 8ccb24406..4f88fe852 100644 --- a/docs/03_guides/code/05_crawlee_parsel.py +++ b/docs/03_guides/code/05_crawlee_parsel.py @@ -42,10 +42,9 @@ async def main() -> None: Actor.log.info('No start URLs specified in Actor input, exiting...') await Actor.exit() - # Crawlee rotates the proxy URL per request on its own. + # Crawlee rotates the proxy URL per request on its own. A local run without + # Apify Proxy access gets no proxy configuration and crawls directly. proxy_configuration = await Actor.create_proxy_configuration() - if proxy_configuration is None: - raise RuntimeError('Failed to create the proxy configuration.') crawler = ParselCrawler( proxy_configuration=proxy_configuration, diff --git a/docs/03_guides/code/05_crawlee_playwright.py b/docs/03_guides/code/05_crawlee_playwright.py index df91b3f0a..1bd7ddf90 100644 --- a/docs/03_guides/code/05_crawlee_playwright.py +++ b/docs/03_guides/code/05_crawlee_playwright.py @@ -42,10 +42,9 @@ async def main() -> None: Actor.log.info('No start URLs specified in Actor input, exiting...') await Actor.exit() - # Crawlee rotates the proxy URL per request on its own. + # Crawlee rotates the proxy URL per request on its own. A local run without + # Apify Proxy access gets no proxy configuration and crawls directly. proxy_configuration = await Actor.create_proxy_configuration() - if proxy_configuration is None: - raise RuntimeError('Failed to create the proxy configuration.') # Common Chrome flags for running the browser in a container. browser_args = ['--no-sandbox', '--disable-dev-shm-usage', '--disable-gpu'] diff --git a/src/apify/_actor.py b/src/apify/_actor.py index d3db93951..bb17a6e7c 100644 --- a/src/apify/_actor.py +++ b/src/apify/_actor.py @@ -1899,7 +1899,8 @@ async def create_proxy_configuration( Returns: ProxyConfiguration object with the passed configuration, or None, if no proxy should be used based - on the configuration. + on the configuration, or if Apify Proxy was requested in a local run that cannot use it, for example + because no password is available or the proxy access is denied. """ if actor_proxy_input is not None: if actor_proxy_input.get('useApifyProxy', False): @@ -1922,7 +1923,9 @@ async def create_proxy_configuration( _apify_client=self._apify_client, ) - await proxy_configuration.initialize() + is_proxy_usable = await proxy_configuration.initialize() + if not is_proxy_usable: + return None return proxy_configuration diff --git a/src/apify/_proxy_configuration.py b/src/apify/_proxy_configuration.py index e21b884f4..656ff8914 100644 --- a/src/apify/_proxy_configuration.py +++ b/src/apify/_proxy_configuration.py @@ -185,7 +185,7 @@ def __init__( self._country_code = country_code self._subdivision_code = subdivision_code - async def initialize(self) -> None: + async def initialize(self) -> bool: """Check if using proxy, if so, check the access. Load the Apify Proxy password from API (only if not passed to constructor or through env var). @@ -193,22 +193,43 @@ async def initialize(self) -> None: Only called if Apify Proxy configuration is used. Also checks if country has access to Apify Proxy groups if the country code is provided. + On the Apify platform, a missing password, a failed password fetch, or a denied proxy access raises. In a + local run, each of them logs a warning and returns `False` instead, and the run should proceed without a + proxy. Without a password, `new_proxy_info` and `new_url` return `None`. + You should use the Actor.create_proxy_configuration function to create a pre-initialized `ProxyConfiguration` instance instead of calling this manually. + + Returns: + `True` if the proxy configuration is usable, `False` if the run should proceed without a proxy. """ if self._uses_apify_proxy: if not self._password: await self._maybe_fetch_password() if not self._password: - raise ValueError( - 'Apify Proxy password must be provided using the "password" constructor argument ' - f'or the "{ApifyEnvVars.PROXY_PASSWORD}" environment variable. ' - f'You can also provide your Apify token via the "${ApifyEnvVars.TOKEN}" environment variable, ' - f'so that the SDK can fetch the proxy password from Apify API, ' - f'when not provided through constructor or ${ApifyEnvVars.PROXY_PASSWORD}.' + if self._configuration.is_at_home: + raise ValueError( + 'Apify Proxy password must be provided using the "password" constructor argument ' + f'or the "{ApifyEnvVars.PROXY_PASSWORD}" environment variable. ' + f'You can also provide your Apify token via the "{ApifyEnvVars.TOKEN}" environment ' + 'variable, so that the SDK can fetch the proxy password from Apify API, ' + f'when not provided through constructor or {ApifyEnvVars.PROXY_PASSWORD}.' + ) + + logger.warning( + 'No Apify Proxy password available, running without proxy. To use Apify Proxy locally, ' + 'provide the "password" constructor argument or the "%s" environment variable. You can also ' + 'provide your Apify token via the "%s" environment variable, so that the SDK can fetch the ' + 'proxy password from Apify API, when %s is not defined.', + ApifyEnvVars.PROXY_PASSWORD, + ApifyEnvVars.TOKEN, + ApifyEnvVars.PROXY_PASSWORD, ) + return False - await self._check_access() + return await self._check_access() + + return True async def new_proxy_info( self, @@ -242,16 +263,19 @@ async def new_proxy_info( return None if self._uses_apify_proxy: + if not self._password: + return None + parsed_url = URL(proxy_info.url) username = self._get_username(session_id) return ProxyInfo( - url=f'http://{username}:{self._password or ""}@{parsed_url.host}:{parsed_url.port}', + url=f'http://{username}:{self._password}@{parsed_url.host}:{parsed_url.port}', scheme='http', hostname=proxy_info.hostname, port=proxy_info.port, username=username, - password=self._password or '', + password=self._password, session_id=proxy_info.session_id, proxy_tier=proxy_info.proxy_tier, groups=self._groups, @@ -274,16 +298,22 @@ async def _maybe_fetch_password(self) -> None: token = self._configuration.token if token and self._apify_client: - user_info = await self._apify_client.user().get() + try: + user_info = await self._apify_client.user().get() + except Exception: + if self._configuration.is_at_home: + raise + logger.warning('Failed to fetch the Apify Proxy password using the Apify token.', exc_info=True) + return if user_info and (proxy := getattr(user_info, 'proxy', None)): self._password = proxy.password - async def _check_access(self) -> None: + async def _check_access(self) -> bool: proxy_status_url = f'{self._configuration.proxy_status_url}/?format=json' proxy_info = await self.new_proxy_info() if proxy_info is None: - return + return True status = None async with impit.AsyncClient(proxy=proxy_info.url, timeout=10) as client: @@ -296,16 +326,22 @@ async def _check_access(self) -> None: # retry on connection errors pass - if status: - if not status['connected']: - raise ConnectionError(status['connectionError']) - - self.is_man_in_the_middle = status['isManInTheMiddle'] - else: + if not status: logger.warning( 'Apify Proxy access check timed out. Watch out for errors with status code 407. If you see some, it ' 'most likely means you do not have access to either all or some of the proxies you are trying to use.' ) + return True + + if not status['connected']: + # A local setup may not be complete yet, so only the platform treats a denied access as fatal. + if self._configuration.is_at_home: + raise ConnectionError(status['connectionError']) + logger.warning('%s Running without proxy.', status['connectionError']) + return False + + self.is_man_in_the_middle = status['isManInTheMiddle'] + return True def _get_username(self, session_id: int | str | None = None) -> str: if session_id is not None: diff --git a/src/apify/scrapy/middlewares/apify_proxy.py b/src/apify/scrapy/middlewares/apify_proxy.py index 6814bf4c4..2cf0a7368 100644 --- a/src/apify/scrapy/middlewares/apify_proxy.py +++ b/src/apify/scrapy/middlewares/apify_proxy.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio from typing import TYPE_CHECKING from urllib.parse import ParseResult, urlparse @@ -33,6 +34,8 @@ def __init__(self, proxy_settings: dict) -> None: """ self._proxy_settings = proxy_settings self._proxy_cfg_internal: ProxyConfiguration | None = None + self._proxy_cfg_lock = asyncio.Lock() + self._runs_without_proxy = False @classmethod def from_crawler(cls: type[ApifyHttpProxyMiddleware], crawler: Crawler) -> ApifyHttpProxyMiddleware: @@ -66,6 +69,8 @@ def from_crawler(cls: type[ApifyHttpProxyMiddleware], crawler: Crawler) -> Apify async def process_request(self, request: Request) -> None: """Process a Scrapy request by assigning a new proxy. + In a local run that cannot use Apify Proxy, the request is left without a proxy. + Args: request: Scrapy Request object. @@ -74,6 +79,9 @@ async def process_request(self, request: Request) -> None: """ url = await self._get_new_proxy_url() + if url is None: + return + if not (url.username and url.password): raise ValueError('Username and password must be provided in the proxy URL.') @@ -104,32 +112,23 @@ def process_exception( exception, ) - async def _get_new_proxy_url(self) -> ParseResult: + async def _get_new_proxy_url(self) -> ParseResult | None: """Get a new proxy URL. - Raises: - NotConfigured: If creation of the proxy configuration fails. - - Returns: New proxy URL. + Returns: New proxy URL, or None when the run goes without a proxy. """ - # Get proxy configuration, creating it if necessary - proxy_cfg = ( - self._proxy_cfg_internal - if isinstance(self._proxy_cfg_internal, ProxyConfiguration) - else await Actor.create_proxy_configuration(actor_proxy_input=self._proxy_settings) - ) - - # If the proxy configuration is still not available, raise an error. However, this should not happen due - # to the checks in the `from_crawler` method. - if proxy_cfg is None: - Actor.log.error( - 'Creation of proxy configuration failed. Check the field "proxyConfiguration" in the Actor input.' - ) - raise NotConfigured - - # Store the proxy configuration for future use - self._proxy_cfg_internal = proxy_cfg - - # Get a new proxy URL and return it - new_url = await proxy_cfg.new_url() - return urlparse(str(new_url)) + # The lock makes concurrent first requests create the proxy configuration only once. + async with self._proxy_cfg_lock: + if self._proxy_cfg_internal is None and not self._runs_without_proxy: + self._proxy_cfg_internal = await Actor.create_proxy_configuration( + actor_proxy_input=self._proxy_settings + ) + # `from_crawler` guarantees `useApifyProxy` is set, so None only comes from a local run that cannot + # use Apify Proxy. `ProxyConfiguration.initialize` has already logged a warning about it. + self._runs_without_proxy = self._proxy_cfg_internal is None + + if self._proxy_cfg_internal is None: + return None + + new_url = await self._proxy_cfg_internal.new_url() + return urlparse(new_url) if new_url else None diff --git a/tests/unit/actor/test_actor_create_proxy_configuration.py b/tests/unit/actor/test_actor_create_proxy_configuration.py index 9d409ee20..17213374e 100644 --- a/tests/unit/actor/test_actor_create_proxy_configuration.py +++ b/tests/unit/actor/test_actor_create_proxy_configuration.py @@ -165,3 +165,37 @@ def request_handler(request: Request, response: Response) -> Response: assert call_mock.call_count == 3 await Actor.exit() + + +async def test_create_proxy_configuration_without_password_locally( + monkeypatch: pytest.MonkeyPatch, httpserver: HTTPServer +) -> None: + """Locally, requesting Apify Proxy without a password yields no proxy configuration.""" + monkeypatch.delenv(ApifyEnvVars.PROXY_PASSWORD.value, raising=False) + monkeypatch.delenv(ApifyEnvVars.TOKEN.value, raising=False) + dummy_proxy_status_url = str(httpserver.url_for('/')).removesuffix('/') + monkeypatch.setenv(ApifyEnvVars.PROXY_STATUS_URL.value, dummy_proxy_status_url) + + async with Actor: + proxy_configuration = await Actor.create_proxy_configuration(groups=['GROUP1']) + + assert proxy_configuration is None + assert len(httpserver.log) == 0 + + +@pytest.mark.usefixtures('patched_impit_client') +async def test_create_proxy_configuration_when_not_connected_locally( + monkeypatch: pytest.MonkeyPatch, httpserver: HTTPServer +) -> None: + """Locally, a denied Apify Proxy access yields no proxy configuration.""" + dummy_proxy_status_url = str(httpserver.url_for('/')).removesuffix('/') + monkeypatch.setenv(ApifyEnvVars.PROXY_STATUS_URL.value, dummy_proxy_status_url) + httpserver.expect_oneshot_request('/').respond_with_json( + {'connected': False, 'connectionError': 'DUMMY_CONNECTION_ERROR'}, + status=200, + ) + + async with Actor: + proxy_configuration = await Actor.create_proxy_configuration(password=DUMMY_PASSWORD) + + assert proxy_configuration is None diff --git a/tests/unit/scrapy/middlewares/test_apify_proxy.py b/tests/unit/scrapy/middlewares/test_apify_proxy.py index e6ad25b65..fa4fffae1 100644 --- a/tests/unit/scrapy/middlewares/test_apify_proxy.py +++ b/tests/unit/scrapy/middlewares/test_apify_proxy.py @@ -1,5 +1,7 @@ from __future__ import annotations +import asyncio +from unittest.mock import AsyncMock from urllib.parse import ParseResult, urlparse import pytest @@ -8,7 +10,7 @@ from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured -from apify import ProxyConfiguration +from apify import Actor, ProxyConfiguration from apify.scrapy.middlewares import ApifyHttpProxyMiddleware @@ -133,6 +135,40 @@ async def mock_get_new_proxy_url() -> ParseResult: await middleware.process_request(dummy_request) +async def test_process_request_without_proxy_configuration( + monkeypatch: pytest.MonkeyPatch, + middleware: ApifyHttpProxyMiddleware, + dummy_request: Request, +) -> None: + """Without a proxy configuration, requests go without a proxy and the creation is not retried.""" + create_proxy_configuration = AsyncMock(return_value=None) + monkeypatch.setattr(Actor, 'create_proxy_configuration', create_proxy_configuration) + + await middleware.process_request(dummy_request) + await middleware.process_request(Request('https://example.com/other')) + + assert 'proxy' not in dummy_request.meta + assert b'Proxy-Authorization' not in dummy_request.headers + create_proxy_configuration.assert_awaited_once() + + +async def test_concurrent_requests_create_proxy_configuration_once( + monkeypatch: pytest.MonkeyPatch, + middleware: ApifyHttpProxyMiddleware, +) -> None: + """Concurrent first requests share one proxy configuration creation.""" + + async def slow_create_proxy_configuration(**_: object) -> None: + await asyncio.sleep(0.01) + + create_proxy_configuration = AsyncMock(side_effect=slow_create_proxy_configuration) + monkeypatch.setattr(Actor, 'create_proxy_configuration', create_proxy_configuration) + + await asyncio.gather(*(middleware.process_request(Request(f'https://example.com/{i}')) for i in range(3))) + + create_proxy_configuration.assert_awaited_once() + + @pytest.mark.parametrize( 'exception', [TunnelError(), ValueError()], diff --git a/tests/unit/test_proxy_configuration.py b/tests/unit/test_proxy_configuration.py index 2d6ed74ce..63e52ce83 100644 --- a/tests/unit/test_proxy_configuration.py +++ b/tests/unit/test_proxy_configuration.py @@ -438,7 +438,29 @@ def request_handler(request: Request, response: Response) -> Response: assert call_mock.call_count == 1 -async def test_initialize_without_password_or_token() -> None: +async def test_initialize_without_password_or_token( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, httpserver: HTTPServer +) -> None: + """Locally, a missing proxy password logs a warning, skips the access check and reports no usable proxy.""" + caplog.set_level('WARNING') + monkeypatch.delenv(ApifyEnvVars.PROXY_PASSWORD.value, raising=False) + monkeypatch.delenv(ApifyEnvVars.TOKEN.value, raising=False) + dummy_proxy_status_url = str(httpserver.url_for('/')).removesuffix('/') + monkeypatch.setenv(ApifyEnvVars.PROXY_STATUS_URL.value, dummy_proxy_status_url) + + proxy_configuration = ProxyConfiguration() + + assert await proxy_configuration.initialize() is False + assert any('No Apify Proxy password available, running without proxy.' in msg for msg in caplog.messages) + assert len(httpserver.log) == 0 + + +async def test_initialize_without_password_or_token_at_home(monkeypatch: pytest.MonkeyPatch) -> None: + """On the platform, a missing proxy password raises a `ValueError`.""" + monkeypatch.setenv(ApifyEnvVars.IS_AT_HOME.value, '1') + monkeypatch.delenv(ApifyEnvVars.PROXY_PASSWORD.value, raising=False) + monkeypatch.delenv(ApifyEnvVars.TOKEN.value, raising=False) + proxy_configuration = ProxyConfiguration() with pytest.raises(ValueError, match=r'Apify Proxy password must be provided'): @@ -526,9 +548,35 @@ async def test_initialize_with_manual_password_different_than_user_one( @pytest.mark.usefixtures('patched_impit_client') -async def test_initialize_when_not_connected(monkeypatch: pytest.MonkeyPatch, httpserver: HTTPServer) -> None: +async def test_initialize_when_not_connected( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, httpserver: HTTPServer +) -> None: + """Locally, a denied proxy access logs the connection error and reports no usable proxy.""" + caplog.set_level('WARNING') + dummy_connection_error = 'DUMMY_CONNECTION_ERROR' + dummy_proxy_status_url = str(httpserver.url_for('/')).removesuffix('/') + monkeypatch.setenv(ApifyEnvVars.PROXY_STATUS_URL.value, dummy_proxy_status_url) + + httpserver.expect_oneshot_request('/').respond_with_json( + { + 'connected': False, + 'connectionError': dummy_connection_error, + }, + status=200, + ) + + proxy_configuration = ProxyConfiguration(password=DUMMY_PASSWORD) + + assert await proxy_configuration.initialize() is False + assert any(dummy_connection_error in msg for msg in caplog.messages) + + +@pytest.mark.usefixtures('patched_impit_client') +async def test_initialize_when_not_connected_at_home(monkeypatch: pytest.MonkeyPatch, httpserver: HTTPServer) -> None: + """On the platform, a denied proxy access raises a `ConnectionError`.""" dummy_connection_error = 'DUMMY_CONNECTION_ERROR' dummy_proxy_status_url = str(httpserver.url_for('/')).removesuffix('/') + monkeypatch.setenv(ApifyEnvVars.IS_AT_HOME.value, '1') monkeypatch.setenv(ApifyEnvVars.PROXY_STATUS_URL.value, dummy_proxy_status_url) httpserver.expect_oneshot_request('/').respond_with_json( @@ -545,6 +593,52 @@ async def test_initialize_when_not_connected(monkeypatch: pytest.MonkeyPatch, ht await proxy_configuration.initialize() +async def test_initialize_when_password_fetch_fails( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + httpserver: HTTPServer, + apify_client_async_patcher: ApifyClientAsyncPatcher, +) -> None: + """Locally, a failed password fetch logs a warning and reports no usable proxy.""" + caplog.set_level('WARNING') + monkeypatch.delenv(ApifyEnvVars.PROXY_PASSWORD.value, raising=False) + monkeypatch.setenv(ApifyEnvVars.TOKEN.value, 'DUMMY_TOKEN') + dummy_proxy_status_url = str(httpserver.url_for('/')).removesuffix('/') + monkeypatch.setenv(ApifyEnvVars.PROXY_STATUS_URL.value, dummy_proxy_status_url) + apify_client_async_patcher.patch('user', 'get', replacement_method=Mock(side_effect=RuntimeError('Invalid token'))) + + proxy_configuration = ProxyConfiguration(_apify_client=ApifyClientAsync()) + + assert await proxy_configuration.initialize() is False + assert 'Failed to fetch the Apify Proxy password using the Apify token.' in caplog.messages + assert len(httpserver.log) == 0 + + +async def test_initialize_when_password_fetch_fails_at_home( + monkeypatch: pytest.MonkeyPatch, apify_client_async_patcher: ApifyClientAsyncPatcher +) -> None: + """On the platform, a failed password fetch raises the client error.""" + monkeypatch.setenv(ApifyEnvVars.IS_AT_HOME.value, '1') + monkeypatch.delenv(ApifyEnvVars.PROXY_PASSWORD.value, raising=False) + monkeypatch.setenv(ApifyEnvVars.TOKEN.value, 'DUMMY_TOKEN') + apify_client_async_patcher.patch('user', 'get', replacement_method=Mock(side_effect=RuntimeError('Invalid token'))) + + proxy_configuration = ProxyConfiguration(_apify_client=ApifyClientAsync()) + + with pytest.raises(RuntimeError, match='Invalid token'): + await proxy_configuration.initialize() + + +async def test_new_url_without_password(monkeypatch: pytest.MonkeyPatch) -> None: + """Without a password, Apify Proxy yields no URL and no proxy info.""" + monkeypatch.delenv(ApifyEnvVars.PROXY_PASSWORD.value, raising=False) + + proxy_configuration = ProxyConfiguration() + + assert await proxy_configuration.new_url() is None + assert await proxy_configuration.new_proxy_info() is None + + async def test_initialize_when_status_page_unavailable( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, httpserver: HTTPServer ) -> None: diff --git a/uv.lock b/uv.lock index 83eee90ea..111d5ea8c 100644 --- a/uv.lock +++ b/uv.lock @@ -998,7 +998,7 @@ wheels = [ [[package]] name = "crawlee" -version = "1.11.1" +version = "1.11.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "async-timeout" }, @@ -1014,9 +1014,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "yarl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bd/36/73371dda02f1b992171f1bcd8e0afa192c2dda29314d86c0618023608193/crawlee-1.11.1.tar.gz", hash = "sha256:7b504f8c63957a7fd2eacfaa14cbeb82b834f2e172fdcf3410349f42de46943b", size = 408277, upload-time = "2026-11-05T14:43:28.784Z" } +sdist = { url = "https://files.pythonhosted.org/packages/67/e7/3cf4c31a8595f554afeb5b6819d2cbd93e682bfccb34b2ff77d38227026c/crawlee-1.11.2.tar.gz", hash = "sha256:c87401f1378fe539afe4d2637a3a3fee659ea9bfcfab6495b0ea7494e5eeb4c2", size = 409471, upload-time = "2026-11-10T10:13:52.695Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b4/a5/c000be053571f40a9dbddfdb7f890144c5fbd9deec256026ad242f4e6ebf/crawlee-1.11.1-py3-none-any.whl", hash = "sha256:125ed497e4607e5e58c22e58c089e5d5bedc3b7392d601923b83bfe0ca722b79", size = 474625, upload-time = "2026-11-05T14:43:26.831Z" }, + { url = "https://files.pythonhosted.org/packages/5e/8c/710cb7beaac61172732f4ad42f3061fba1534ba2fe35048b586ecf8849e1/crawlee-1.11.2-py3-none-any.whl", hash = "sha256:be6bd8f555dac1894812d5473cc2aee3c86e6c8bea9395c717320a6a89e6e0cb", size = 474838, upload-time = "2026-11-10T10:13:50.881Z" }, ] [package.optional-dependencies]