fix: run without proxy locally when Apify Proxy is not available

docs/02_concepts/05_proxy_management.mdx

@@ -20,7 +20,7 @@ The Apify SDK provides built-in proxy management through the <ApiLink to="class/
 
 ## Quick start
 
-If you want to use Apify Proxy locally, make sure that you run your Actors via the Apify CLI and that you are [logged in](https://docs.apify.com/cli/docs/installation#login-with-your-apify-account) with your Apify account in the CLI.
+If you want to use Apify Proxy locally, make sure that you run your Actors via the Apify CLI and that you are [logged in](https://docs.apify.com/cli/docs/installation#login-with-your-apify-account) with your Apify account in the CLI. In a local run without Apify Proxy credentials, or when Apify Proxy denies access, <ApiLink to="class/Actor#create_proxy_configuration">`Actor.create_proxy_configuration()`</ApiLink> logs a warning and returns `None`, and the Actor runs without a proxy.
 
 ### Using Apify Proxy
 

docs/02_concepts/13_exceptions.mdx

@@ -43,7 +43,7 @@ The SDK raises standard Python exceptions when it's used incorrectly or given in
 - [`RuntimeError`](https://docs.python.org/3/library/exceptions.html#RuntimeError) when an `Actor` method is used outside the `async with Actor:` block, either before initialization or after exit, or when the Actor is initialized twice.
 - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError) for an invalid argument, such as a malformed `timeout`, an invalid proxy configuration, charging an automatically charged event by hand, or pushing data that is not JSON-serializable or is over the size limit.
 - [`TypeError`](https://docs.python.org/3/library/exceptions.html#TypeError) for an argument of the wrong type.
-- [`ConnectionError`](https://docs.python.org/3/library/exceptions.html#ConnectionError) when <ApiLink to="class/Actor#create_proxy_configuration">`Actor.create_proxy_configuration`</ApiLink> verifies Apify Proxy access and the proxy reports that you have none.
+- [`ConnectionError`](https://docs.python.org/3/library/exceptions.html#ConnectionError) when <ApiLink to="class/Actor#create_proxy_configuration">`Actor.create_proxy_configuration`</ApiLink> verifies Apify Proxy access on the Apify platform and the proxy reports that you have none. In a local run, the method logs a warning and returns `None` instead.
 
 ## Run failures
 

docs/02_concepts/code/05_apify_proxy.py

@@ -8,7 +8,8 @@ async def main() -> None:
         proxy_cfg = await Actor.create_proxy_configuration()
 
         if not proxy_cfg:
-            raise RuntimeError('No proxy configuration available.')
+            Actor.log.warning('Apify Proxy is not available, exiting...')
+            return
 
         proxy_url = await proxy_cfg.new_url()
         Actor.log.info(f'Using proxy URL: {proxy_url}')

docs/02_concepts/code/05_apify_proxy_config.py

@@ -12,7 +12,8 @@ async def main() -> None:
         )
 
         if not proxy_cfg:
-            raise RuntimeError('No proxy configuration available.')
+            Actor.log.warning('Apify Proxy is not available, exiting...')
+            return
 
         proxy_url = await proxy_cfg.new_url()
         Actor.log.info(f'Proxy URL: {proxy_url}')

docs/03_guides/code/05_crawlee_beautifulsoup.py

@@ -42,10 +42,9 @@ async def main() -> None:
             Actor.log.info('No start URLs specified in Actor input, exiting...')
             await Actor.exit()
 
-        # Crawlee rotates the proxy URL per request on its own.
+        # Crawlee rotates the proxy URL per request on its own. A local run without
+        # Apify Proxy access gets no proxy configuration and crawls directly.
         proxy_configuration = await Actor.create_proxy_configuration()
-        if proxy_configuration is None:
-            raise RuntimeError('Failed to create the proxy configuration.')
 
         crawler = BeautifulSoupCrawler(
             proxy_configuration=proxy_configuration,

docs/03_guides/code/05_crawlee_parsel.py

@@ -42,10 +42,9 @@ async def main() -> None:
             Actor.log.info('No start URLs specified in Actor input, exiting...')
             await Actor.exit()
 
-        # Crawlee rotates the proxy URL per request on its own.
+        # Crawlee rotates the proxy URL per request on its own. A local run without
+        # Apify Proxy access gets no proxy configuration and crawls directly.
         proxy_configuration = await Actor.create_proxy_configuration()
-        if proxy_configuration is None:
-            raise RuntimeError('Failed to create the proxy configuration.')
 
         crawler = ParselCrawler(
             proxy_configuration=proxy_configuration,

docs/03_guides/code/05_crawlee_playwright.py

@@ -42,10 +42,9 @@ async def main() -> None:
             Actor.log.info('No start URLs specified in Actor input, exiting...')
             await Actor.exit()
 
-        # Crawlee rotates the proxy URL per request on its own.
+        # Crawlee rotates the proxy URL per request on its own. A local run without
+        # Apify Proxy access gets no proxy configuration and crawls directly.
         proxy_configuration = await Actor.create_proxy_configuration()
-        if proxy_configuration is None:
-            raise RuntimeError('Failed to create the proxy configuration.')
 
         # Common Chrome flags for running the browser in a container.
         browser_args = ['--no-sandbox', '--disable-dev-shm-usage', '--disable-gpu']

src/apify/_actor.py

@@ -1899,7 +1899,8 @@ async def create_proxy_configuration(
 
         Returns:
             ProxyConfiguration object with the passed configuration, or None, if no proxy should be used based
-            on the configuration.
+            on the configuration, or if Apify Proxy was requested in a local run that cannot use it, for example
+            because no password is available or the proxy access is denied.
         """
         if actor_proxy_input is not None:
             if actor_proxy_input.get('useApifyProxy', False):
@@ -1922,7 +1923,9 @@ async def create_proxy_configuration(
             _apify_client=self._apify_client,
         )
 
-        await proxy_configuration.initialize()
+        is_proxy_usable = await proxy_configuration.initialize()
+        if not is_proxy_usable:
+            return None
 
         return proxy_configuration
 

src/apify/_proxy_configuration.py

@@ -185,30 +185,51 @@ def __init__(
         self._country_code = country_code
         self._subdivision_code = subdivision_code
 
-    async def initialize(self) -> None:
+    async def initialize(self) -> bool:
         """Check if using proxy, if so, check the access.
 
         Load the Apify Proxy password from API (only if not passed to constructor or through env var).
 
         Only called if Apify Proxy configuration is used. Also checks if country has access to Apify Proxy groups
         if the country code is provided.
 
+        On the Apify platform, a missing password, a failed password fetch, or a denied proxy access raises. In a
+        local run, each of them logs a warning and returns `False` instead, and the run should proceed without a
+        proxy. Without a password, `new_proxy_info` and `new_url` return `None`.
+
         You should use the Actor.create_proxy_configuration function to create a pre-initialized
         `ProxyConfiguration` instance instead of calling this manually.
+
+        Returns:
+            `True` if the proxy configuration is usable, `False` if the run should proceed without a proxy.
         """
         if self._uses_apify_proxy:
             if not self._password:
                 await self._maybe_fetch_password()
             if not self._password:
-                raise ValueError(
-                    'Apify Proxy password must be provided using the "password" constructor argument '
-                    f'or the "{ApifyEnvVars.PROXY_PASSWORD}" environment variable. '
-                    f'You can also provide your Apify token via the "${ApifyEnvVars.TOKEN}" environment variable, '
-                    f'so that the SDK can fetch the proxy password from Apify API, '
-                    f'when not provided through constructor or ${ApifyEnvVars.PROXY_PASSWORD}.'
+                if self._configuration.is_at_home:
+                    raise ValueError(
+                        'Apify Proxy password must be provided using the "password" constructor argument '
+                        f'or the "{ApifyEnvVars.PROXY_PASSWORD}" environment variable. '
+                        f'You can also provide your Apify token via the "{ApifyEnvVars.TOKEN}" environment '
+                        'variable, so that the SDK can fetch the proxy password from Apify API, '
+                        f'when not provided through constructor or {ApifyEnvVars.PROXY_PASSWORD}.'
+                    )
+
+                logger.warning(
+                    'No Apify Proxy password available, running without proxy. To use Apify Proxy locally, '
+                    'provide the "password" constructor argument or the "%s" environment variable. You can also '
+                    'provide your Apify token via the "%s" environment variable, so that the SDK can fetch the '
+                    'proxy password from Apify API, when %s is not defined.',
+                    ApifyEnvVars.PROXY_PASSWORD,
+                    ApifyEnvVars.TOKEN,
+                    ApifyEnvVars.PROXY_PASSWORD,
                 )
+                return False
 
-            await self._check_access()
+            return await self._check_access()
+
+        return True
 
     async def new_proxy_info(
         self,
@@ -242,16 +263,19 @@ async def new_proxy_info(
             return None
 
         if self._uses_apify_proxy:
+            if not self._password:
+                return None
+
             parsed_url = URL(proxy_info.url)
             username = self._get_username(session_id)
 
             return ProxyInfo(
-                url=f'http://{username}:{self._password or ""}@{parsed_url.host}:{parsed_url.port}',
+                url=f'http://{username}:{self._password}@{parsed_url.host}:{parsed_url.port}',
                 scheme='http',
                 hostname=proxy_info.hostname,
                 port=proxy_info.port,
                 username=username,
-                password=self._password or '',
+                password=self._password,
                 session_id=proxy_info.session_id,
                 proxy_tier=proxy_info.proxy_tier,
                 groups=self._groups,
@@ -274,16 +298,22 @@ async def _maybe_fetch_password(self) -> None:
         token = self._configuration.token
 
         if token and self._apify_client:
-            user_info = await self._apify_client.user().get()
+            try:
+                user_info = await self._apify_client.user().get()
+            except Exception:
+                if self._configuration.is_at_home:
+                    raise
+                logger.warning('Failed to fetch the Apify Proxy password using the Apify token.', exc_info=True)
+                return
             if user_info and (proxy := getattr(user_info, 'proxy', None)):
                 self._password = proxy.password
 
-    async def _check_access(self) -> None:
+    async def _check_access(self) -> bool:
         proxy_status_url = f'{self._configuration.proxy_status_url}/?format=json'
         proxy_info = await self.new_proxy_info()
 
         if proxy_info is None:
-            return
+            return True
 
         status = None
         async with impit.AsyncClient(proxy=proxy_info.url, timeout=10) as client:
@@ -296,16 +326,22 @@ async def _check_access(self) -> None:
                     # retry on connection errors
                     pass
 
-        if status:
-            if not status['connected']:
-                raise ConnectionError(status['connectionError'])
-
-            self.is_man_in_the_middle = status['isManInTheMiddle']
-        else:
+        if not status:
             logger.warning(
                 'Apify Proxy access check timed out. Watch out for errors with status code 407. If you see some, it '
                 'most likely means you do not have access to either all or some of the proxies you are trying to use.'
             )
+            return True
+
+        if not status['connected']:
+            # A local setup may not be complete yet, so only the platform treats a denied access as fatal.
+            if self._configuration.is_at_home:
+                raise ConnectionError(status['connectionError'])
+            logger.warning('%s Running without proxy.', status['connectionError'])
+            return False
+
+        self.is_man_in_the_middle = status['isManInTheMiddle']
+        return True
 
     def _get_username(self, session_id: int | str | None = None) -> str:
         if session_id is not None:

src/apify/scrapy/middlewares/apify_proxy.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 from typing import TYPE_CHECKING
 from urllib.parse import ParseResult, urlparse
 
@@ -33,6 +34,8 @@ def __init__(self, proxy_settings: dict) -> None:
         """
         self._proxy_settings = proxy_settings
         self._proxy_cfg_internal: ProxyConfiguration | None = None
+        self._proxy_cfg_lock = asyncio.Lock()
+        self._runs_without_proxy = False
 
     @classmethod
     def from_crawler(cls: type[ApifyHttpProxyMiddleware], crawler: Crawler) -> ApifyHttpProxyMiddleware:
@@ -66,6 +69,8 @@ def from_crawler(cls: type[ApifyHttpProxyMiddleware], crawler: Crawler) -> Apify
     async def process_request(self, request: Request) -> None:
         """Process a Scrapy request by assigning a new proxy.
 
+        In a local run that cannot use Apify Proxy, the request is left without a proxy.
+
         Args:
             request: Scrapy Request object.
 
@@ -74,6 +79,9 @@ async def process_request(self, request: Request) -> None:
         """
         url = await self._get_new_proxy_url()
 
+        if url is None:
+            return
+
         if not (url.username and url.password):
             raise ValueError('Username and password must be provided in the proxy URL.')
 
@@ -104,32 +112,23 @@ def process_exception(
                 exception,
             )
 
-    async def _get_new_proxy_url(self) -> ParseResult:
+    async def _get_new_proxy_url(self) -> ParseResult | None:
         """Get a new proxy URL.
 
-        Raises:
-            NotConfigured: If creation of the proxy configuration fails.
-
-        Returns: New proxy URL.
+        Returns: New proxy URL, or None when the run goes without a proxy.
         """
-        # Get proxy configuration, creating it if necessary
-        proxy_cfg = (
-            self._proxy_cfg_internal
-            if isinstance(self._proxy_cfg_internal, ProxyConfiguration)
-            else await Actor.create_proxy_configuration(actor_proxy_input=self._proxy_settings)
-        )
-
-        # If the proxy configuration is still not available, raise an error. However, this should not happen due
-        # to the checks in the `from_crawler` method.
-        if proxy_cfg is None:
-            Actor.log.error(
-                'Creation of proxy configuration failed. Check the field "proxyConfiguration" in the Actor input.'
-            )
-            raise NotConfigured
-
-        # Store the proxy configuration for future use
-        self._proxy_cfg_internal = proxy_cfg
-
-        # Get a new proxy URL and return it
-        new_url = await proxy_cfg.new_url()
-        return urlparse(str(new_url))
+        # The lock makes concurrent first requests create the proxy configuration only once.
+        async with self._proxy_cfg_lock:
+            if self._proxy_cfg_internal is None and not self._runs_without_proxy:
+                self._proxy_cfg_internal = await Actor.create_proxy_configuration(
+                    actor_proxy_input=self._proxy_settings
+                )
+                # `from_crawler` guarantees `useApifyProxy` is set, so None only comes from a local run that cannot
+                # use Apify Proxy. `ProxyConfiguration.initialize` has already logged a warning about it.
+                self._runs_without_proxy = self._proxy_cfg_internal is None
+
+        if self._proxy_cfg_internal is None:
+            return None
+
+        new_url = await self._proxy_cfg_internal.new_url()
+        return urlparse(new_url) if new_url else None

tests/unit/actor/test_actor_create_proxy_configuration.py

@@ -165,3 +165,37 @@ def request_handler(request: Request, response: Response) -> Response:
     assert call_mock.call_count == 3
 
     await Actor.exit()
+
+
+async def test_create_proxy_configuration_without_password_locally(
+    monkeypatch: pytest.MonkeyPatch, httpserver: HTTPServer
+) -> None:
+    """Locally, requesting Apify Proxy without a password yields no proxy configuration."""
+    monkeypatch.delenv(ApifyEnvVars.PROXY_PASSWORD.value, raising=False)
+    monkeypatch.delenv(ApifyEnvVars.TOKEN.value, raising=False)
+    dummy_proxy_status_url = str(httpserver.url_for('/')).removesuffix('/')
+    monkeypatch.setenv(ApifyEnvVars.PROXY_STATUS_URL.value, dummy_proxy_status_url)
+
+    async with Actor:
+        proxy_configuration = await Actor.create_proxy_configuration(groups=['GROUP1'])
+
+    assert proxy_configuration is None
+    assert len(httpserver.log) == 0
+
+
+@pytest.mark.usefixtures('patched_impit_client')
+async def test_create_proxy_configuration_when_not_connected_locally(
+    monkeypatch: pytest.MonkeyPatch, httpserver: HTTPServer
+) -> None:
+    """Locally, a denied Apify Proxy access yields no proxy configuration."""
+    dummy_proxy_status_url = str(httpserver.url_for('/')).removesuffix('/')
+    monkeypatch.setenv(ApifyEnvVars.PROXY_STATUS_URL.value, dummy_proxy_status_url)
+    httpserver.expect_oneshot_request('/').respond_with_json(
+        {'connected': False, 'connectionError': 'DUMMY_CONNECTION_ERROR'},
+        status=200,
+    )
+
+    async with Actor:
+        proxy_configuration = await Actor.create_proxy_configuration(password=DUMMY_PASSWORD)
+
+    assert proxy_configuration is None