refactor!: remove requestsFromUrl sources

docs/public-api/crawlee-basic.api.md

@@ -483,16 +483,13 @@ export interface IStatistics<StateExtension extends object = {}> {
 }
 
 // @public
-export type LabeledSource<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | Source : string | Request_2 | ({
-    requestsFromUrl?: string;
-    regex?: RegExp;
-} & ({
+export type LabeledSource<Routes extends Record<keyof Routes, Dictionary>> = string extends keyof Routes ? string | Source : string | Request_2 | {
     [Label in keyof Routes & string]: Omit<Partial<RequestOptions<Routes[Label]>>, 'label'> & {
         label: Label;
     };
 }[keyof Routes & string] | (Omit<Partial<RequestOptions>, 'label'> & {
     label?: undefined;
-})));
+});
 
 // @public
 export type LoadedRequest<R extends Request_2> = R & Required<Pick<R, 'id' | 'loadedUrl'>>;

docs/public-api/crawlee-core.api.md

@@ -8,7 +8,6 @@ import type { AllowedHttpMethods } from '@crawlee/types';
 import { ArgumentValidationError } from '@crawlee/utils';
 import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter';
 import type { Awaitable } from '@crawlee/types';
-import { BaseHttpClient } from '@crawlee/http-client';
 import type { BatchAddRequestsResult } from '@crawlee/types';
 import type { BetterIntervalID } from '@apify/utilities';
 import type { Constructor } from '@crawlee/types';
@@ -679,11 +678,9 @@ export class RequestList implements IRequestLoader {
 
 // @public (undocumented)
 export interface RequestListOptions {
-    httpClient?: BaseHttpClient;
     keepDuplicateUrls?: boolean;
     persistRequestsKey?: string;
     persistStateKey?: string;
-    proxyConfiguration?: IProxyConfiguration;
     sources?: RequestListSource[];
     sourcesFunction?: RequestListSourcesFunction;
     state?: RequestListState;
@@ -905,10 +902,7 @@ export class SessionRetiredError extends SessionError {
 }
 
 // @public (undocumented)
-export type Source = (Partial<RequestOptions> & {
-    requestsFromUrl?: string;
-    regex?: RegExp;
-}) | Request_2;
+export type Source = Partial<RequestOptions> | Request_2;
 
 // @public
 export type StateConversion<TFrom, TTo> = ((value: TFrom) => Awaitable<TTo>) | StandardSchemaV1<TFrom, TTo>;
@@ -939,8 +933,6 @@ export class StorageInstanceManager {
 // @public
 export interface StorageOpenOptions {
     configuration?: Configuration;
-    httpClient?: BaseHttpClient;
-    proxyConfiguration?: IProxyConfiguration;
     storageBackend?: StorageBackend;
 }
 

docs/upgrading/upgrading-v4/upgrading-v4.md

@@ -300,6 +300,34 @@ Reading the run input is an Apify platform concern, so Crawlee no longer has any
 - **`Configuration.inputKey` and the `CRAWLEE_INPUT_KEY` environment variable are removed.** The Apify SDK's `Configuration` defines `inputKey` itself.
 - **The default key-value store is purged in full**, `INPUT` included. Sparing a key, and claiming a bare `<key>` / `<key>.json` file as the record `<key>` (see [Out-of-band key-value files](#out-of-band-key-value-files-eg-a-hand-placed-inputjson)), is left to subclasses of `FileSystemStorageBackend` through its two protected hooks, `keyValueStoreAdoptionCandidates` and `purgeKeyValueStore`. The Apify SDK's `ApifyFileSystemStorageBackend` does this for its run input; plain Crawlee treats a hand-placed `INPUT.json` as any other file.
 
+### `requestsFromUrl` sources moved to the Apify SDK
+
+Downloading URLs from a `{ requestsFromUrl }` source served the "Link remote text file" option of Apify Actor inputs, so it moved to the Apify SDK. `RequestList`, `RequestQueue` and the crawlers' `run()` / `addRequests()` no longer accept `requestsFromUrl` (nor the `regex` option next to it), and the `proxyConfiguration` and `httpClient` options of `RequestList.open()` and `RequestQueue.open()`, which only served those downloads, are removed.
+
+In an Actor, turn the input into a `RequestList` with `ApifyRequestList` from `apify`:
+
+**Before:**
+```typescript
+const { startUrls } = await Actor.getInput();
+await crawler.run(startUrls);
+```
+
+**After:**
+```typescript
+import { ApifyRequestList } from 'apify';
+
+const { startUrls } = await Actor.getInput();
+await crawler.run(await ApifyRequestList.open({ requestListSourcesInput: startUrls }));
+```
+
+Elsewhere, download the list yourself with `downloadListOfUrls()`, whose `urlRegExp`, `proxyUrl` and `httpClient` options replace the removed ones:
+
+```typescript
+import { downloadListOfUrls } from 'crawlee';
+
+await crawler.run(await downloadListOfUrls({ url: 'https://example.com/urls.txt' }));
+```
+
 ### Service management moved from `Configuration` to `ServiceLocator`
 
 The service management functionality has been extracted from `Configuration` into a new `ServiceLocator` class.
@@ -876,7 +904,7 @@ The mechanism can be disabled entirely with `transactionalStorage: false` on any
 #### Removed symbols and options
 
 - `checkStorageAccess` and `withCheckedStorageAccess` are superseded by the transaction mechanism; the per-call-site helper is now `withDirectStorageAccess()`.
-- The experimental `AdaptivePlaywrightCrawler` no longer needs its bespoke write-buffering machinery: the `preventDirectStorageAccess` option is gone (direct storage calls are now captured by the per-attempt transaction instead of throwing), and `RequestHandlerResult` is replaced by the read-only `StorageTransactionView`, which the `resultChecker` / `resultComparator` callbacks (and `fullResultComparator`) now receive. The view keeps the familiar accessors (`datasetItems`, `enqueuedUrls`, `keyValueStoreChanges`), so most callbacks only need a type change. The `calls` and `enqueuedUrlLists` accessors are gone — `requestsFromUrl` sources are now expanded when added, so the fetched URLs appear in `enqueuedUrls` (and are what `fullResultComparator` compares). The `commitResult` override point is gone too; use an `afterStorageCommit()` callback to run logic once the winning attempt's writes have landed (or failed to).
+- The experimental `AdaptivePlaywrightCrawler` no longer needs its bespoke write-buffering machinery: the `preventDirectStorageAccess` option is gone (direct storage calls are now captured by the per-attempt transaction instead of throwing), and `RequestHandlerResult` is replaced by the read-only `StorageTransactionView`, which the `resultChecker` / `resultComparator` callbacks (and `fullResultComparator`) now receive. The view keeps the familiar accessors (`datasetItems`, `enqueuedUrls`, `keyValueStoreChanges`), so most callbacks only need a type change. The `calls` and `enqueuedUrlLists` accessors are gone. The `commitResult` override point is gone too; use an `afterStorageCommit()` callback to run logic once the winning attempt's writes have landed (or failed to).
 
 ### `storageObject` is removed from storage classes
 
@@ -1118,7 +1146,7 @@ With `respectRobotsTxtFile` enabled, a site that answers `/robots.txt` with a `5
 
 - A crawl that discovers more than `maxThrottledDomains` domains (100 by default) throws instead of quietly running out of steam. Pass your own `ThrottlingRequestManager` as `requestManager` to raise the ceiling — or crawl fewer sites.
 - Combined with a manager that paces on its own, a `ThrottlingRequestManager` with `domains: 'all'` and `throttleBy: 'registrableDomain'` takes the delay as its `minCrawlDelaySecs` floor, wherever it sits in a composition. One that paces only *some* domains throws instead — set `domains: 'all'`, or configure the delay there yourself and drop the option.
-- Requests that never pass through the request manager — those from a `requestsFromUrl` list — are not paced, and the crawler warns when it hands one out.
+- Requests that never pass through the request manager — those from a `RequestList` — are not paced, and the crawler warns when it hands one out.
 
 ### Internal KVS keys renamed
 

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -1532,7 +1532,8 @@ export class BasicCrawler<
             const newCrawlDepth = request!.crawlDepth + 1;
             const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth);
 
-            return await this.addRequests(requestsGenerator, options);
+            // The labels were checked by the typed `addRequests` signature, the generator only adds `crawlDepth`.
+            return await this.addRequests(requestsGenerator as TypedRequestsLike<Routes>, options);
         };
 
         const sendRequest = createSendRequest(this.httpClient, request!, session);

packages/basic-crawler/src/internals/crawlers/crawler-commons.ts

@@ -38,14 +38,12 @@ export type LabeledSource<Routes extends Record<keyof Routes, Dictionary>> = str
     :
           | string
           | Request
-          | ({ requestsFromUrl?: string; regex?: RegExp } & (
-                | {
-                      [Label in keyof Routes & string]: Omit<Partial<RequestOptions<Routes[Label]>>, 'label'> & {
-                          label: Label;
-                      };
-                  }[keyof Routes & string]
-                | (Omit<Partial<RequestOptions>, 'label'> & { label?: undefined })
-            ));
+          | {
+                [Label in keyof Routes & string]: Omit<Partial<RequestOptions<Routes[Label]>>, 'label'> & {
+                    label: Label;
+                };
+            }[keyof Routes & string]
+          | (Omit<Partial<RequestOptions>, 'label'> & { label?: undefined });
 
 /**
  * The iterable/array of {@apilink LabeledSource} inputs accepted by the label-aware `addRequests`/`run`

packages/basic-crawler/src/internals/throttling-request-manager.ts

@@ -439,20 +439,6 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManage
         return this.#resolvedInner;
     }
 
-    /** Warns once about sources that cannot be routed by domain, because their URLs are not known yet. */
-    #warnIfNotRoutable(requestLike: Source): void {
-        if ('requestsFromUrl' in requestLike && requestLike.requestsFromUrl !== undefined && this.#throttlingEnabled) {
-            // The URL list is only fetched once the owning manager expands it, so we cannot know which domains
-            // it covers and cannot route it. Warn instead of silently exempting those URLs from throttling.
-            this.#warnOnce(
-                'urlListNotRouted',
-                `Requests loaded via \`requestsFromUrl\` cannot be routed to a per-domain queue, because their URLs ` +
-                    `are not known at insertion time. They will be added to the inner request manager and will not ` +
-                    `be throttled, even if they belong to a configured domain.`,
-            );
-        }
-    }
-
     #warnOnce(key: string, message: string): void {
         if (this.#warnedAbout.has(key)) {
             return;
@@ -803,8 +789,6 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManage
     // --- IRequestManager Implementation ---
 
     async addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo> {
-        this.#warnIfNotRoutable(requestLike);
-
         const manager = await this.#selectManagerOrThrow(requestLike.url ?? '');
 
         return manager.addRequest(requestLike, options);
@@ -846,8 +830,6 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManage
                 const overflowing = new Set<string>();
 
                 for (const request of chunk) {
-                    this.#warnIfNotRoutable(request);
-
                     const url = request.url ?? '';
                     const manager = await this.#selectManager(url);
 
@@ -1146,15 +1128,13 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManage
         }
 
         if (request !== null && this.#throttlesEveryDomain) {
-            // Requests that were never routed by domain - a `RequestList`, a `requestsFromUrl` expansion - are
-            // handed out as fast as the crawler asks for them, because there is no per-domain queue to hold
-            // them back in.
+            // Requests that were never routed by domain - e.g. from a `RequestList` - are handed out as fast as the
+            // crawler asks for them, because there is no per-domain queue to hold them back in.
             this.#warnOnce(
                 'innerNotThrottled',
-                `Requests read directly from the wrapped request manager (for instance from a \`RequestList\` or a ` +
-                    `\`requestsFromUrl\` list) are not throttled, because they are not stored per domain. Enqueue ` +
-                    `them through the crawler - \`crawler.run(requests)\` or \`crawler.addRequests()\` - to have ` +
-                    `their domains paced.`,
+                `Requests read directly from the wrapped request manager (for instance from a \`RequestList\`) are ` +
+                    `not throttled, because they are not stored per domain. Enqueue them through the crawler - ` +
+                    `\`crawler.run(requests)\` or \`crawler.addRequests()\` - to have their domains paced.`,
             );
         }
 

packages/core/src/request.ts

@@ -493,12 +493,6 @@ interface ComputeUniqueKeyOptions {
     alwaysEnqueue?: boolean;
 }
 
-export type Source = (Partial<RequestOptions> & { requestsFromUrl?: string; regex?: RegExp }) | CrawleeRequest;
-
-/** @internal */
-export interface InternalSource {
-    requestsFromUrl: string;
-    regex?: RegExp;
-}
+export type Source = Partial<RequestOptions> | CrawleeRequest;
 
 export { CrawleeRequest as Request };