From 8b5aa0cd79cd64d429744a6cb90dd98d17ff5580 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Fri, 6 Nov 2026 09:48:40 +0100 Subject: [PATCH] refactor!: remove `requestsFromUrl` sources --- docs/public-api/crawlee-basic.api.md | 7 +- docs/public-api/crawlee-core.api.md | 10 +- docs/upgrading/upgrading-v4/upgrading-v4.md | 32 +++- .../src/internals/basic-crawler.ts | 3 +- .../src/internals/crawlers/crawler-commons.ts | 14 +- .../internals/throttling-request-manager.ts | 30 +-- packages/core/src/request.ts | 8 +- packages/core/src/storages/request-list.ts | 134 +------------ packages/core/src/storages/request-queue.ts | 110 +---------- packages/core/src/storages/utils.ts | 14 -- .../internals/adaptive-playwright-crawler.ts | 3 +- .../throttling-request-manager.test.ts | 33 +--- test/core/request-list.test.ts | 178 ------------------ test/core/storages/request-queue.test.ts | 140 +------------- .../core/storages/storage-transaction.test.ts | 46 ----- 15 files changed, 59 insertions(+), 703 deletions(-) diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.md index c31e969379f3..b9d9814c0005 100644 --- a/docs/public-api/crawlee-basic.api.md +++ b/docs/public-api/crawlee-basic.api.md @@ -483,16 +483,13 @@ export interface IStatistics { } // @public -export type LabeledSource> = string extends keyof Routes ? string | Source : string | Request_2 | ({ - requestsFromUrl?: string; - regex?: RegExp; -} & ({ +export type LabeledSource> = string extends keyof Routes ? string | Source : string | Request_2 | { [Label in keyof Routes & string]: Omit>, 'label'> & { label: Label; }; }[keyof Routes & string] | (Omit, 'label'> & { label?: undefined; -}))); +}); // @public export type LoadedRequest = R & Required>; diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.md index 7d55bfd05804..16fa61d4d74f 100644 --- a/docs/public-api/crawlee-core.api.md +++ b/docs/public-api/crawlee-core.api.md @@ -8,7 +8,6 @@ import type { AllowedHttpMethods } from '@crawlee/types'; import { ArgumentValidationError } from '@crawlee/utils'; import { AsyncEventEmitter } from '@vladfrangu/async_event_emitter'; import type { Awaitable } from '@crawlee/types'; -import { BaseHttpClient } from '@crawlee/http-client'; import type { BatchAddRequestsResult } from '@crawlee/types'; import type { BetterIntervalID } from '@apify/utilities'; import type { Constructor } from '@crawlee/types'; @@ -679,11 +678,9 @@ export class RequestList implements IRequestLoader { // @public (undocumented) export interface RequestListOptions { - httpClient?: BaseHttpClient; keepDuplicateUrls?: boolean; persistRequestsKey?: string; persistStateKey?: string; - proxyConfiguration?: IProxyConfiguration; sources?: RequestListSource[]; sourcesFunction?: RequestListSourcesFunction; state?: RequestListState; @@ -905,10 +902,7 @@ export class SessionRetiredError extends SessionError { } // @public (undocumented) -export type Source = (Partial & { - requestsFromUrl?: string; - regex?: RegExp; -}) | Request_2; +export type Source = Partial | Request_2; // @public export type StateConversion = ((value: TFrom) => Awaitable) | StandardSchemaV1; @@ -939,8 +933,6 @@ export class StorageInstanceManager { // @public export interface StorageOpenOptions { configuration?: Configuration; - httpClient?: BaseHttpClient; - proxyConfiguration?: IProxyConfiguration; storageBackend?: StorageBackend; } diff --git a/docs/upgrading/upgrading-v4/upgrading-v4.md b/docs/upgrading/upgrading-v4/upgrading-v4.md index 1866a0b89ef3..d610082ba2a3 100644 --- a/docs/upgrading/upgrading-v4/upgrading-v4.md +++ b/docs/upgrading/upgrading-v4/upgrading-v4.md @@ -300,6 +300,34 @@ Reading the run input is an Apify platform concern, so Crawlee no longer has any - **`Configuration.inputKey` and the `CRAWLEE_INPUT_KEY` environment variable are removed.** The Apify SDK's `Configuration` defines `inputKey` itself. - **The default key-value store is purged in full**, `INPUT` included. Sparing a key, and claiming a bare `` / `.json` file as the record `` (see [Out-of-band key-value files](#out-of-band-key-value-files-eg-a-hand-placed-inputjson)), is left to subclasses of `FileSystemStorageBackend` through its two protected hooks, `keyValueStoreAdoptionCandidates` and `purgeKeyValueStore`. The Apify SDK's `ApifyFileSystemStorageBackend` does this for its run input; plain Crawlee treats a hand-placed `INPUT.json` as any other file. +### `requestsFromUrl` sources moved to the Apify SDK + +Downloading URLs from a `{ requestsFromUrl }` source served the "Link remote text file" option of Apify Actor inputs, so it moved to the Apify SDK. `RequestList`, `RequestQueue` and the crawlers' `run()` / `addRequests()` no longer accept `requestsFromUrl` (nor the `regex` option next to it), and the `proxyConfiguration` and `httpClient` options of `RequestList.open()` and `RequestQueue.open()`, which only served those downloads, are removed. + +In an Actor, turn the input into a `RequestList` with `ApifyRequestList` from `apify`: + +**Before:** +```typescript +const { startUrls } = await Actor.getInput(); +await crawler.run(startUrls); +``` + +**After:** +```typescript +import { ApifyRequestList } from 'apify'; + +const { startUrls } = await Actor.getInput(); +await crawler.run(await ApifyRequestList.open({ requestListSourcesInput: startUrls })); +``` + +Elsewhere, download the list yourself with `downloadListOfUrls()`, whose `urlRegExp`, `proxyUrl` and `httpClient` options replace the removed ones: + +```typescript +import { downloadListOfUrls } from 'crawlee'; + +await crawler.run(await downloadListOfUrls({ url: 'https://example.com/urls.txt' })); +``` + ### Service management moved from `Configuration` to `ServiceLocator` The service management functionality has been extracted from `Configuration` into a new `ServiceLocator` class. @@ -876,7 +904,7 @@ The mechanism can be disabled entirely with `transactionalStorage: false` on any #### Removed symbols and options - `checkStorageAccess` and `withCheckedStorageAccess` are superseded by the transaction mechanism; the per-call-site helper is now `withDirectStorageAccess()`. -- The experimental `AdaptivePlaywrightCrawler` no longer needs its bespoke write-buffering machinery: the `preventDirectStorageAccess` option is gone (direct storage calls are now captured by the per-attempt transaction instead of throwing), and `RequestHandlerResult` is replaced by the read-only `StorageTransactionView`, which the `resultChecker` / `resultComparator` callbacks (and `fullResultComparator`) now receive. The view keeps the familiar accessors (`datasetItems`, `enqueuedUrls`, `keyValueStoreChanges`), so most callbacks only need a type change. The `calls` and `enqueuedUrlLists` accessors are gone — `requestsFromUrl` sources are now expanded when added, so the fetched URLs appear in `enqueuedUrls` (and are what `fullResultComparator` compares). The `commitResult` override point is gone too; use an `afterStorageCommit()` callback to run logic once the winning attempt's writes have landed (or failed to). +- The experimental `AdaptivePlaywrightCrawler` no longer needs its bespoke write-buffering machinery: the `preventDirectStorageAccess` option is gone (direct storage calls are now captured by the per-attempt transaction instead of throwing), and `RequestHandlerResult` is replaced by the read-only `StorageTransactionView`, which the `resultChecker` / `resultComparator` callbacks (and `fullResultComparator`) now receive. The view keeps the familiar accessors (`datasetItems`, `enqueuedUrls`, `keyValueStoreChanges`), so most callbacks only need a type change. The `calls` and `enqueuedUrlLists` accessors are gone. The `commitResult` override point is gone too; use an `afterStorageCommit()` callback to run logic once the winning attempt's writes have landed (or failed to). ### `storageObject` is removed from storage classes @@ -1118,7 +1146,7 @@ With `respectRobotsTxtFile` enabled, a site that answers `/robots.txt` with a `5 - A crawl that discovers more than `maxThrottledDomains` domains (100 by default) throws instead of quietly running out of steam. Pass your own `ThrottlingRequestManager` as `requestManager` to raise the ceiling — or crawl fewer sites. - Combined with a manager that paces on its own, a `ThrottlingRequestManager` with `domains: 'all'` and `throttleBy: 'registrableDomain'` takes the delay as its `minCrawlDelaySecs` floor, wherever it sits in a composition. One that paces only *some* domains throws instead — set `domains: 'all'`, or configure the delay there yourself and drop the option. -- Requests that never pass through the request manager — those from a `requestsFromUrl` list — are not paced, and the crawler warns when it hands one out. +- Requests that never pass through the request manager — those from a `RequestList` — are not paced, and the crawler warns when it hands one out. ### Internal KVS keys renamed diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts index 61a400937be8..07cdfd93070c 100644 --- a/packages/basic-crawler/src/internals/basic-crawler.ts +++ b/packages/basic-crawler/src/internals/basic-crawler.ts @@ -1532,7 +1532,8 @@ export class BasicCrawler< const newCrawlDepth = request!.crawlDepth + 1; const requestsGenerator = this.addCrawlDepthRequestGenerator(requests, newCrawlDepth); - return await this.addRequests(requestsGenerator, options); + // The labels were checked by the typed `addRequests` signature, the generator only adds `crawlDepth`. + return await this.addRequests(requestsGenerator as TypedRequestsLike, options); }; const sendRequest = createSendRequest(this.httpClient, request!, session); diff --git a/packages/basic-crawler/src/internals/crawlers/crawler-commons.ts b/packages/basic-crawler/src/internals/crawlers/crawler-commons.ts index 3bd891393d60..c9a1776381cd 100644 --- a/packages/basic-crawler/src/internals/crawlers/crawler-commons.ts +++ b/packages/basic-crawler/src/internals/crawlers/crawler-commons.ts @@ -38,14 +38,12 @@ export type LabeledSource> = str : | string | Request - | ({ requestsFromUrl?: string; regex?: RegExp } & ( - | { - [Label in keyof Routes & string]: Omit>, 'label'> & { - label: Label; - }; - }[keyof Routes & string] - | (Omit, 'label'> & { label?: undefined }) - )); + | { + [Label in keyof Routes & string]: Omit>, 'label'> & { + label: Label; + }; + }[keyof Routes & string] + | (Omit, 'label'> & { label?: undefined }); /** * The iterable/array of {@apilink LabeledSource} inputs accepted by the label-aware `addRequests`/`run` diff --git a/packages/basic-crawler/src/internals/throttling-request-manager.ts b/packages/basic-crawler/src/internals/throttling-request-manager.ts index 915956681ce8..d16ddce5d81a 100644 --- a/packages/basic-crawler/src/internals/throttling-request-manager.ts +++ b/packages/basic-crawler/src/internals/throttling-request-manager.ts @@ -439,20 +439,6 @@ export class ThrottlingRequestManager { - this.#warnIfNotRoutable(requestLike); - const manager = await this.#selectManagerOrThrow(requestLike.url ?? ''); return manager.addRequest(requestLike, options); @@ -846,8 +830,6 @@ export class ThrottlingRequestManager(); for (const request of chunk) { - this.#warnIfNotRoutable(request); - const url = request.url ?? ''; const manager = await this.#selectManager(url); @@ -1146,15 +1128,13 @@ export class ThrottlingRequestManager & { requestsFromUrl?: string; regex?: RegExp }) | CrawleeRequest; - -/** @internal */ -export interface InternalSource { - requestsFromUrl: string; - regex?: RegExp; -} +export type Source = Partial | CrawleeRequest; export { CrawleeRequest as Request }; diff --git a/packages/core/src/storages/request-list.ts b/packages/core/src/storages/request-list.ts index 9ced26e3063f..4c55dcfa5e51 100644 --- a/packages/core/src/storages/request-list.ts +++ b/packages/core/src/storages/request-list.ts @@ -1,16 +1,12 @@ -import type { BaseHttpClient } from '@crawlee/http-client'; -import type { Dictionary } from '@crawlee/types'; -import { downloadListOfUrls } from '@crawlee/utils'; import { z } from 'zod'; import type { Configuration } from '../configuration.js'; import type { CrawleeLogger } from '../log.js'; -import type { IProxyConfiguration } from '../proxy-configuration.js'; import { convertStateSync, RecoverableState } from '../recoverable-state.js'; -import { type InternalSource, Request, type RequestOptions, type Source } from '../request.js'; +import { Request, type RequestOptions, type Source } from '../request.js'; import { createDeserialize, serializeArray } from '../serialization.js'; import { serviceLocator } from '../service-locator.js'; -import { parseArgument, schemas, validators } from '../validators.js'; +import { parseArgument, schemas } from '../validators.js'; import { KeyValueStore } from './key-value-store.js'; import type { IRequestLoader, RequestLoaderStatus } from './request-loader.js'; import type { IRequestManager } from './request-manager.js'; @@ -47,8 +43,6 @@ const requestListOptionsSchema = z.strictObject({ persistRequestsKey: z.string().optional(), state: persistedRequestListState.optional(), keepDuplicateUrls: z.boolean().default(false), - proxyConfiguration: validators.proxyConfiguration.optional(), - httpClient: schemas.httpClient.optional(), }); const listNameSchema = z.string().nullish(); const openOptionsSchema = z.looseObject({}); @@ -136,10 +130,6 @@ export interface RequestListOptions { * This is a measure to prevent memory leaks in situations when millions of sources are * added. * - * Additionally, the `requestsFromUrl` property may be used instead of `url`, - * which will instruct `RequestList` to download the source URLs from a given remote location. - * The URLs will be parsed from the received response. - * * ``` * [ * // A single URL @@ -147,17 +137,6 @@ export interface RequestListOptions { * * // Modify Request options * { method: PUT, 'https://example.com/put, payload: { foo: 'bar' }} - * - * // Batch import of URLs from a file hosted on the web, - * // where the URLs should be requested using the HTTP POST request - * { method: 'POST', requestsFromUrl: 'http://example.com/urls.txt' }, - * - * // Batch import from remote file, using a specific regular expression to extract the URLs. - * { requestsFromUrl: 'http://example.com/urls.txt', regex: /https:\/\/example.com\/.+/ }, - * - * // Get list of URLs from a Google Sheets document. Just add "/gviz/tq?tqx=out:csv" to the Google Sheet URL. - * // For details, see https://help.apify.com/en/articles/2906022-scraping-a-list-of-urls-from-a-google-sheets-document - * { requestsFromUrl: 'https://docs.google.com/spreadsheets/d/1-2mUcRAiBbCTVA5KcpFdEYWflLMLp9DDU3iJutvES4w/gviz/tq?tqx=out:csv' } * ] * ``` */ @@ -203,13 +182,6 @@ export interface RequestListOptions { */ sourcesFunction?: RequestListSourcesFunction; - /** - * Used to pass the proxy configuration for the `requestsFromUrl` objects. - * Takes advantage of the internal address rotation and authentication process. - * If undefined, the `requestsFromUrl` requests will be made without proxy. - */ - proxyConfiguration?: IProxyConfiguration; - /** * Identifies the key in the default key-value store under which `RequestList` periodically stores its * state (i.e. which URLs were crawled and which not). @@ -274,18 +246,10 @@ export interface RequestListOptions { /** @internal */ configuration?: Configuration; - - /** - * The HTTP client to be used to download `requestsFromUrl` URLs. - * - * If not specified the `RequestList` will use the default HTTP client. - */ - httpClient?: BaseHttpClient; } /** * Represents a static list of URLs to crawl. - * The URLs can be provided either in code or parsed from a text file hosted on the web. * `RequestList` is used by {@apilink BasicCrawler}, {@apilink CheerioCrawler}, {@apilink PuppeteerCrawler} * and {@apilink PlaywrightCrawler} as a source of URLs to crawl. * @@ -322,7 +286,6 @@ export interface RequestListOptions { * const requestList = await RequestList.open('my-request-list', [ * 'http://www.example.com/page-1', * { url: 'http://www.example.com/page-2', method: 'POST', userData: { foo: 'bar' }}, - * { requestsFromUrl: 'http://www.example.com/my-url-list.txt', userData: { isFromUrl: true } }, * ]); * ``` * @@ -332,10 +295,6 @@ export interface RequestListOptions { * // Separate requests * { url: 'http://www.example.com/page-1', method: 'GET', headers: { ... } }, * { url: 'http://www.example.com/page-2', userData: { foo: 'bar' }}, - * - * // Bulk load of URLs from file `http://www.example.com/my-url-list.txt` - * // Note that all URLs must start with http:// or https:// - * { requestsFromUrl: 'http://www.example.com/my-url-list.txt', userData: { isFromUrl: true } }, * ], { * // Persist the state to avoid re-crawling which can lead to data duplications. * // Keep in mind that the sources have to be immutable or this will throw an error. @@ -393,8 +352,6 @@ export class RequestList implements IRequestLoader { #keepDuplicateUrls: boolean; #sources: RequestListSource[]; #sourcesFunction?: RequestListSourcesFunction; - #proxyConfiguration?: IProxyConfiguration; - #httpClient?: BaseHttpClient; /** * To create new instance of `RequestList` we need to use `RequestList.open()` factory method. @@ -402,16 +359,8 @@ export class RequestList implements IRequestLoader { * @internal */ private constructor(options: RequestListOptions = {}) { - const { - sources, - sourcesFunction, - persistStateKey, - persistRequestsKey, - state, - proxyConfiguration, - keepDuplicateUrls, - httpClient, - } = parseArgument(options, requestListOptionsSchema); + const { sources, sourcesFunction, persistStateKey, persistRequestsKey, state, keepDuplicateUrls } = + parseArgument(options, requestListOptionsSchema); if (!(sources || sourcesFunction)) { throw new Error('At least one of "sources" or "sourcesFunction" must be provided.'); @@ -419,7 +368,6 @@ export class RequestList implements IRequestLoader { this.#persistStateKey = persistStateKey ? `CRAWLEE_${persistStateKey}` : persistStateKey; this.#persistRequestsKey = persistRequestsKey ? `CRAWLEE_${persistRequestsKey}` : persistRequestsKey; - this.#httpClient = httpClient; this.#stateCodec = buildRequestListStateCodec({ requests: this.requests, @@ -452,9 +400,6 @@ export class RequestList implements IRequestLoader { // Will be empty after initialization to save memory. this.#sources = sources ? [...sources] : []; this.#sourcesFunction = sourcesFunction; - - // The proxy configuration used for `requestsFromUrl` requests. - this.#proxyConfiguration = proxyConfiguration; } /** @@ -532,12 +477,7 @@ export class RequestList implements IRequestLoader { // oxlint-disable-next-line typescript/no-array-delete delete this.#sources[i]; - if (typeof source === 'object' && (source as Dictionary).requestsFromUrl) { - const fetchedRequests = await this.#fetchRequestsFromUrl(source as InternalSource); - await this.#addFetchedRequests(source as InternalSource, fetchedRequests); - } else { - this.#addRequest(source); - } + this.#addRequest(source); } // Drop the original array full of empty indexes. @@ -681,28 +621,6 @@ export class RequestList implements IRequestLoader { this.inProgress.delete(uniqueKey); } - /** - * Adds all fetched requests from a URL from a remote resource. - */ - async #addFetchedRequests(source: InternalSource, fetchedRequests: RequestOptions[]) { - const { requestsFromUrl, regex } = source; - const originalLength = this.requests.length; - - fetchedRequests.forEach((request) => this.#addRequest(request)); - - const fetchedCount = fetchedRequests.length; - const importedCount = this.requests.length - originalLength; - - this.#log.info('Fetched and loaded Requests from a remote resource.', { - requestsFromUrl, - regex, - fetchedCount, - importedCount, - duplicateCount: fetchedCount - importedCount, - sample: JSON.stringify(fetchedRequests.slice(0, 5)), - }); - } - async #getPersistedState(key: string): Promise { this.#store ??= await KeyValueStore.open(); const state = await this.#store.getValue(key); @@ -710,33 +628,6 @@ export class RequestList implements IRequestLoader { return state!; } - /** - * Fetches URLs from requestsFromUrl and returns them in format of list of requests - */ - async #fetchRequestsFromUrl(source: InternalSource): Promise { - const { requestsFromUrl, regex, ...sharedOpts } = source; - - // Download remote resource and parse URLs. - let urlsArr; - try { - urlsArr = await this.#downloadListOfUrls({ - url: requestsFromUrl, - urlRegExp: regex, - proxyUrl: (await this.#proxyConfiguration?.newProxyInfo())?.url, - }); - } catch (err) { - throw new Error(`Cannot fetch a request list from ${requestsFromUrl}: ${err}`); - } - - // Skip if resource contained no URLs. - if (!urlsArr.length) { - this.#log.warning('The fetched list contains no valid URLs.', { requestsFromUrl, regex }); - return []; - } - - return urlsArr.map((url) => ({ url, ...sharedOpts })); - } - /** * Adds given request. * If the `source` parameter is a string or plain object and not an instance @@ -890,11 +781,6 @@ export class RequestList implements IRequestLoader { * This is a measure to prevent memory leaks in situations when millions of sources are * added. * - * Additionally, the `requestsFromUrl` property may be used instead of `url`, - * which will instruct {@apilink RequestList} to download the source URLs from a given remote location. - * The URLs will be parsed from the received response. In this case you can limit the URLs - * using `regex` parameter containing regular expression pattern for URLs to be included. - * * For details, see the {@apilink RequestListOptions.sources} * @param [options] * The {@apilink RequestList} options. Note that the `listName` parameter supersedes @@ -930,16 +816,6 @@ export class RequestList implements IRequestLoader { return rl; } - - /** - * @internal wraps public utility for mocking purposes - */ - async #downloadListOfUrls(options: { url: string; urlRegExp?: RegExp; proxyUrl?: string }): Promise { - return downloadListOfUrls({ - ...options, - httpClient: this.#httpClient, - }); - } } /** diff --git a/packages/core/src/storages/request-queue.ts b/packages/core/src/storages/request-queue.ts index 52cd50d9922a..ad19b3138d55 100644 --- a/packages/core/src/storages/request-queue.ts +++ b/packages/core/src/storages/request-queue.ts @@ -1,6 +1,5 @@ import { inspect } from 'node:util'; -import type { BaseHttpClient } from '@crawlee/http-client'; import type { BatchAddRequestsResult, Constructor, @@ -12,7 +11,6 @@ import type { RequestSchema, } from '@crawlee/types'; import { isAsyncIterable, isIterable } from '@crawlee/utils/internal'; -import { downloadListOfUrls } from '@crawlee/utils'; import type { ReadonlyDeep } from 'type-fest'; import { z } from 'zod'; @@ -24,8 +22,7 @@ import { getObjectType } from '../debug.js'; import type { EventManager } from '../events/event-manager.js'; import { EventType } from '../events/event-manager.js'; import type { CrawleeLogger } from '../log.js'; -import type { IProxyConfiguration } from '../proxy-configuration.js'; -import type { InternalSource, RequestOptions, Source } from '../request.js'; +import type { RequestOptions, Source } from '../request.js'; import { Request } from '../request.js'; import { serviceLocator } from '../service-locator.js'; import { parseArgument, schemas, validators } from '../validators.js'; @@ -89,8 +86,6 @@ const uniqueKeySchema = z.string(); const openOptionsSchema = z.strictObject({ configuration: z.instanceof(Configuration).optional(), storageBackend: validators.storageBackend.optional(), - proxyConfiguration: validators.proxyConfiguration.optional(), - httpClient: schemas.httpClient.optional(), }); /** @@ -131,7 +126,6 @@ export class RequestQueue implements IStorage, IRequestManager { readonly id: string; readonly name?: string; readonly backend: RequestQueueBackend; - #proxyConfiguration?: IProxyConfiguration; readonly log: CrawleeLogger; @@ -155,8 +149,6 @@ export class RequestQueue implements IStorage, IRequestManager { */ #expectedRequestProcessingSecs = 0; - #httpClient?: BaseHttpClient; - readonly #events: EventManager; readonly #statsTracker = new StorageStatsTracker({ @@ -181,8 +173,6 @@ export class RequestQueue implements IStorage, IRequestManager { this.#events = serviceLocator.getEventManager(); this.backend = options.backend; - this.#proxyConfiguration = options.proxyConfiguration; - this.#requestCache = new LruCache({ maxLength: MAX_CACHED_REQUESTS }); this.#requestSeenCache = new RequestDeduplicationCache(); this.log = serviceLocator.getLogger().child({ prefix: `RequestQueue(${this.id}, ${this.name ?? 'no-name'})` }); @@ -235,13 +225,6 @@ export class RequestQueue implements IStorage, IRequestManager { parseArgument(requestLike, schemas.anyObject); const { forefront } = parseArgument(options, operationOptionsSchema); - if ('requestsFromUrl' in requestLike) { - const requests = await this.#fetchRequestsFromUrl(requestLike as InternalSource); - const processedRequests = await this.#addFetchedRequests(requestLike as InternalSource, requests, options); - - return { ...processedRequests[0], forefront }; - } - parseArgument(requestLike, newRequestLikeSchema); const request = requestLike instanceof Request ? requestLike : new Request(requestLike as RequestOptions); @@ -503,9 +486,6 @@ export class RequestQueue implements IStorage, IRequestManager { for await (const requestLike of requestsLike) { if (typeof requestLike === 'string') { requests.push(new Request({ url: requestLike })); - } else if ('requestsFromUrl' in requestLike) { - const fetchedRequests = await this.#fetchRequestsFromUrl(requestLike as InternalSource); - await this.#addFetchedRequests(requestLike as InternalSource, fetchedRequests, options); } else { requests.push( requestLike instanceof Request ? requestLike : new Request(requestLike as RequestOptions), @@ -596,8 +576,6 @@ export class RequestQueue implements IStorage, IRequestManager { const { forefront, waitForAllRequestsToBeAdded, batchSize, waitBetweenBatchesMillis, maxNewRequests } = parseArgument(options, addRequestsBatchedOptionsSchema); - const addRequest = this.addRequest.bind(this); - async function* generateRequests() { for await (const opts of requests) { // Validate the input @@ -613,24 +591,9 @@ export class RequestQueue implements IStorage, IRequestManager { `Request options are not valid, the 'id' property must not be present. Input: ${inspect(opts)}`, ); } - - if ( - (opts as any).requestsFromUrl !== undefined && - typeof (opts as any).requestsFromUrl !== 'string' - ) { - throw new Error( - `Request options are not valid, the 'requestsFromUrl' property is not a string. Input: ${inspect(opts)}`, - ); - } } - if (opts && typeof opts === 'object' && 'requestsFromUrl' in opts) { - // Handle URL lists right away - await addRequest(opts, { forefront }); - } else { - // Yield valid requests - yield typeof opts === 'string' ? { url: opts } : (opts as RequestOptions); - } + yield typeof opts === 'string' ? { url: opts } : (opts as RequestOptions); } } @@ -991,66 +954,6 @@ export class RequestQueue implements IStorage, IRequestManager { return metadata; } - /** - * Fetches URLs from requestsFromUrl and returns them in format of list of requests - */ - async #fetchRequestsFromUrl(source: InternalSource): Promise { - const { requestsFromUrl, regex, ...sharedOpts } = source; - - // Download remote resource and parse URLs. - let urlsArr; - try { - urlsArr = await this.#downloadListOfUrls({ - url: requestsFromUrl, - urlRegExp: regex, - proxyUrl: (await this.#proxyConfiguration?.newProxyInfo())?.url, - }); - } catch (err) { - throw new Error(`Cannot fetch a request list from ${requestsFromUrl}: ${err}`); - } - - // Skip if resource contained no URLs. - if (!urlsArr.length) { - this.log.warning('The fetched list contains no valid URLs.', { requestsFromUrl, regex }); - return []; - } - - return urlsArr.map((url) => ({ url, ...sharedOpts })); - } - - /** - * Adds all fetched requests from a URL from a remote resource. - */ - async #addFetchedRequests( - source: InternalSource, - fetchedRequests: RequestOptions[], - options: RequestQueueOperationOptions, - ) { - const { requestsFromUrl, regex } = source; - const { addedRequests } = await this.addRequestsBatched(fetchedRequests, options); - - this.log.info('Fetched and loaded Requests from a remote resource.', { - requestsFromUrl, - regex, - fetchedCount: fetchedRequests.length, - importedCount: addedRequests.length, - duplicateCount: fetchedRequests.length - addedRequests.length, - sample: JSON.stringify(fetchedRequests.slice(0, 5)), - }); - - return addedRequests; - } - - /** - * @internal wraps public utility for mocking purposes - */ - async #downloadListOfUrls(options: { url: string; urlRegExp?: RegExp; proxyUrl?: string }): Promise { - return downloadListOfUrls({ - ...options, - httpClient: this.#httpClient, - }); - } - /** * Opens a request queue and returns a promise resolving to an instance * of the {@apilink RequestQueue} class. @@ -1090,8 +993,6 @@ export class RequestQueue implements IStorage, IRequestManager { backendOpener: () => storageBackend.createRequestQueueBackend(resolved), backendCacheKey: storageBackend.getStorageBackendCacheKey?.() ?? storageBackend.constructor.name, }); - queue.#proxyConfiguration = parsedOptions.proxyConfiguration; - queue.#httpClient = parsedOptions.httpClient; return queue; } @@ -1110,13 +1011,6 @@ export interface RequestQueueOptions { /** Resolved metadata for the request queue, as returned by the backend's `getMetadata()`. */ metadata: RequestQueueInfo; backend: RequestQueueBackend; - - /** - * Used to pass the proxy configuration for the `requestsFromUrl` objects. - * Takes advantage of the internal address rotation and authentication process. - * If undefined, the `requestsFromUrl` requests will be made without proxy. - */ - proxyConfiguration?: IProxyConfiguration; } export interface RequestQueueOperationOptions { diff --git a/packages/core/src/storages/utils.ts b/packages/core/src/storages/utils.ts index 7002118b0ea5..2f10e3c7621d 100644 --- a/packages/core/src/storages/utils.ts +++ b/packages/core/src/storages/utils.ts @@ -1,10 +1,8 @@ import crypto from 'node:crypto'; -import type { BaseHttpClient } from '@crawlee/http-client'; import type { Dictionary, StorageBackend } from '@crawlee/types'; import { Configuration } from '../configuration.js'; -import type { IProxyConfiguration } from '../proxy-configuration.js'; import { serviceLocator } from '../service-locator.js'; import { KeyValueStore } from './key-value-store.js'; @@ -241,16 +239,4 @@ export interface StorageOpenOptions { * Optional storage backend that should be used to open storages. */ storageBackend?: StorageBackend; - - /** - * Used to pass the proxy configuration for the `requestsFromUrl` objects. - * Takes advantage of the internal address rotation and authentication process. - * If undefined, the `requestsFromUrl` requests will be made without proxy. - */ - proxyConfiguration?: IProxyConfiguration; - - /** - * HTTP client to be used to download the list of URLs in `RequestQueue`. - */ - httpClient?: BaseHttpClient; } diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts index 551f938019cc..dceb67022bbb 100644 --- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts +++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts @@ -10,6 +10,7 @@ import type { RestrictedCrawlingContext, RouterRoutes, StatisticStateExtensionOptions, + TypedRequestsLike, } from '@crawlee/basic'; import { BasicCrawler, @@ -963,7 +964,7 @@ export class AdaptivePlaywrightCrawler< // The per-attempt transaction buffers these (the queue policy defaults to `deferred` here), // so a discarded attempt's enqueues never reach the queue. - return await this.addRequests(requestsWithDepth, { + return await this.addRequests(requestsWithDepth as TypedRequestsLike, { ...options, baseUrl, strategy: options.strategy ?? EnqueueStrategy.SameHostname, diff --git a/test/basic-crawler/throttling-request-manager.test.ts b/test/basic-crawler/throttling-request-manager.test.ts index eb292cb031bb..1d0a7b42bd77 100644 --- a/test/basic-crawler/throttling-request-manager.test.ts +++ b/test/basic-crawler/throttling-request-manager.test.ts @@ -15,7 +15,6 @@ import { } from '@crawlee/core'; import { sleep } from '@crawlee/utils'; import { FileSystemStorageBackend } from '@crawlee/fs-storage'; -import { BaseHttpClient } from '@crawlee/http-client'; describe('ThrottlingRequestManager', () => { beforeEach(() => { @@ -242,40 +241,12 @@ describe('ThrottlingRequestManager', () => { }); }); - test('warns that requestsFromUrl sources cannot be domain-routed', async () => { - // The manager logs through a `child()` of the shared logger; collapsing `child` onto the - // logger itself is the only way to see what it emits. - const logger = serviceLocator.getLogger(); - vitest.spyOn(logger, 'child').mockReturnValue(logger); - const warning = vitest.spyOn(logger, 'warning').mockImplementation(() => {}); - - // The inner queue downloads the lists, so serve them locally: whatever a live host returns (an empty - // list, an error) would be logged through the same spied logger. - const httpClient = Object.assign(Object.create(BaseHttpClient.prototype) as BaseHttpClient, { - sendRequest: vitest.fn(async () => new Response('https://example.com/1\nhttps://example.com/2')), - stream: vitest.fn(async () => new Response()), - }); - - const manager = new ThrottlingRequestManager({ - inner: await RequestQueue.open({ name: 'inner-queue' }, { httpClient }), - domains: ['example.com'], - }); - - await manager.addRequestsBatched([ - { requestsFromUrl: 'https://example.com/urls.txt' }, - { requestsFromUrl: 'https://example.com/more.txt' }, - ]); - - expect(warning).toHaveBeenCalledTimes(1); - expect(warning.mock.calls[0][0]).toMatch(/requestsFromUrl/); - }); - test('a request fetched from the inner manager is handed back to it', async () => { const inner = await createQueue(); const manager = new ThrottlingRequestManager({ inner, domains: ['example.com'] }); - // Bypasses routing, exactly as a `RequestList` transfer or a `requestsFromUrl` expansion does - so the - // request sits in the inner manager despite belonging to a throttled domain. + // Bypasses routing, exactly as a `RequestList` transfer does - so the request sits in the inner manager + // despite belonging to a throttled domain. await inner.addRequest({ url: 'https://example.com/1' }); const request = (await manager.fetchNextRequest())!; diff --git a/test/core/request-list.test.ts b/test/core/request-list.test.ts index d908dee24ed3..110502218ebf 100644 --- a/test/core/request-list.test.ts +++ b/test/core/request-list.test.ts @@ -1,11 +1,9 @@ import { ArgumentValidationError, Configuration, - deserializeArray, EventType, KeyValueStore, MemoryStorageBackend, - ProxyConfiguration, Request, RequestList, REQUESTS_PERSISTENCE_KEY, @@ -13,7 +11,6 @@ import { STATE_PERSISTENCE_KEY, StateValidationError, } from '@crawlee/core'; -import { BaseHttpClient } from '@crawlee/http-client'; import { sleep } from '@crawlee/utils'; import { beforeAll, type MockedFunction } from 'vitest'; @@ -31,20 +28,6 @@ function shuffle(array: unknown[]): unknown[] { return out; } -// `vitest.mockObject` clones the object and drops its prototype, so build the mock manually to -// keep it an `instanceof BaseHttpClient`. -const createMockHttpClient = () => - Object.assign(Object.create(BaseHttpClient.prototype) as BaseHttpClient, { - sendRequest: vitest.fn(async (_request?: any, _options?: any) => new Response()), - stream: vitest.fn(async () => new Response()), - }); - -let mockHttpClient = createMockHttpClient(); - -beforeEach(async () => { - mockHttpClient = createMockHttpClient(); -}); - describe('RequestList', () => { let ll: number; beforeAll(() => { @@ -159,132 +142,6 @@ describe('RequestList', () => { } }); - test('should correctly load list from hosted files in correct order', async () => { - const list1 = ['https://example.com', 'https://google.com', 'https://wired.com']; - const list2 = ['https://another.com', 'https://page.com']; - mockHttpClient.sendRequest - .mockImplementationOnce(async () => { - await sleep(100); - return new Response(list1.join('\n')); - }) - .mockResolvedValueOnce(new Response(list2.join('\n'))); - - const requestList = await RequestList.open({ - sources: [ - { method: 'GET', requestsFromUrl: 'http://example.com/list-1' }, - { method: 'POST', requestsFromUrl: 'http://example.com/list-2' }, - ], - httpClient: mockHttpClient, - }); - - expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list1[0] }); - expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list1[1] }); - expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list1[2] }); - expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'POST', url: list2[0] }); - expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'POST', url: list2[1] }); - - expect(mockHttpClient.sendRequest).toBeCalledTimes(2); - expect(mockHttpClient.sendRequest.mock.calls.map(([request]) => request.url)).toEqual([ - 'http://example.com/list-1', - 'http://example.com/list-2', - ]); - }); - - test('should use regex parameter to parse urls', async () => { - const listStr = 'kjnjkn"https://example.com/a/b/c?q=1#abc";,"HTTP://google.com/a/b/c";dgg:dd'; - const listArr = ['https://example.com', 'HTTP://google.com']; - - const regex = /(https:\/\/example.com|HTTP:\/\/google.com)/g; - - mockHttpClient.sendRequest.mockResolvedValueOnce(new Response(listStr)); - - const requestList = await RequestList.open({ - sources: [ - { - method: 'GET', - requestsFromUrl: 'http://example.com/list-1', - regex, - }, - ], - httpClient: mockHttpClient, - }); - - expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: listArr[0] }); - expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: listArr[1] }); - - expect(mockHttpClient.sendRequest).toBeCalled(); - expect(mockHttpClient.sendRequest.mock.calls[0][0].url).toBe('http://example.com/list-1'); - }); - - test('should fix gdoc sharing url in `requestsFromUrl` automatically (GH issue #639)', async () => { - const list = ['https://example.com', 'https://google.com', 'https://wired.com']; - const wrongUrls = [ - 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU', - 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/', - 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/edit?usp=sharing', - 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/123123132', - 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/?q=blablabla', - 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/edit#gid=0', - ]; - const correctUrl = - 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/gviz/tq?tqx=out:csv'; - - mockHttpClient.sendRequest.mockImplementation(async () => new Response(list.join('\n'))); - - const requestList = await RequestList.open({ - sources: wrongUrls.map((requestsFromUrl) => ({ requestsFromUrl })), - httpClient: mockHttpClient, - }); - - expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[0] }); - expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[1] }); - expect(await requestList.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[2] }); - - expect(mockHttpClient.sendRequest.mock.calls[0][0]?.url).toBe(correctUrl); - }); - - test('should handle requestsFromUrl with no URLs', async () => { - mockHttpClient.sendRequest.mockResolvedValueOnce(new Response('')); - - const requestList = await RequestList.open({ - sources: [ - { - method: 'GET', - requestsFromUrl: 'http://example.com/list-1', - }, - ], - httpClient: mockHttpClient, - }); - - expect(await requestList.fetchNextRequest()).toBe(null); - - expect(mockHttpClient.sendRequest).toBeCalledTimes(1); - expect(mockHttpClient.sendRequest.mock.calls[0][0].url).toBe('http://example.com/list-1'); - }); - - test('should use the defined proxy server when using `requestsFromUrl`', async () => { - const proxyUrls = ['http://proxyurl.usedforthe.download', 'http://another.proxy.url']; - - const proxyConfiguration = new ProxyConfiguration({ - proxyUrls, - }); - - await RequestList.open({ - sources: [ - { requestsFromUrl: 'http://example.com/list-1' }, - { requestsFromUrl: 'http://example.com/list-2' }, - { requestsFromUrl: 'http://example.com/list-3' }, - ], - proxyConfiguration, - httpClient: mockHttpClient, - }); - - expect(mockHttpClient.sendRequest).toBeCalledTimes(3); - for (const [, options] of mockHttpClient.sendRequest.mock.calls) { - expect(proxyUrls).toContain(options.proxyUrl); - } - }); - test('tracks in-progress requests through the crawl lifecycle', async () => { const requestList = await RequestList.open({ sources: [ @@ -454,41 +311,6 @@ describe('RequestList', () => { expect(requestList2.requests).toEqual(requestList.requests); }); - test('should correctly persist sources from requestsFromUrl if persistRequestsKey is set', async () => { - const PERSIST_REQUESTS_KEY = 'some-key'; - const getValueSpy = vitest.spyOn(KeyValueStore.prototype, 'getValue'); - const setValueSpy = vitest.spyOn(KeyValueStore.prototype, 'setValue'); - let persistedRequests: any; - - const opts = { - sources: [ - { url: 'https://example.com/1' }, - { url: 'https://example.com/2' }, - { requestsFromUrl: 'http://example.com/list-urls.txt', userData: { isFromUrl: true } }, - { url: 'https://example.com/5' }, - ], - persistRequestsKey: PERSIST_REQUESTS_KEY, - httpClient: mockHttpClient, - }; - - const urlsFromTxt = ['http://example.com/3', 'http://example.com/4']; - mockHttpClient.sendRequest.mockResolvedValueOnce(new Response(urlsFromTxt.join('\n'))); - - getValueSpy.mockResolvedValueOnce(null); - setValueSpy.mockImplementationOnce(async (_key, value) => { - persistedRequests = value; - }); - - const requestList = await RequestList.open(opts); - expect(requestList.areRequestsPersisted).toBe(true); - const requests = await deserializeArray(persistedRequests); - expect(requestList.requests).toHaveLength(5); - expect(requests).toEqual(requestList.requests); - - expect(mockHttpClient.sendRequest).toBeCalledTimes(1); - expect(mockHttpClient.sendRequest.mock.calls[0][0].url).toBe('http://example.com/list-urls.txt'); - }); - test('handles correctly inconsistent inProgress fields in state', async () => { // NOTE: This is a test for the deleteFromInProgress hotfix - see RequestList.initialize() diff --git a/test/core/storages/request-queue.test.ts b/test/core/storages/request-queue.test.ts index c8bb84f5d4b8..048b68a21a5a 100644 --- a/test/core/storages/request-queue.test.ts +++ b/test/core/storages/request-queue.test.ts @@ -4,25 +4,10 @@ import { resolve } from 'node:path'; import type { LoadedRequest } from '@crawlee/basic'; import { CrawlingRequest, NavigationSkippedError, skippedNavigationRequest } from '@crawlee/basic'; -import { MemoryStorageBackend, ProxyConfiguration, Request, RequestQueue, serviceLocator } from '@crawlee/core'; -import { BaseHttpClient } from '@crawlee/http-client'; +import { MemoryStorageBackend, Request, RequestQueue, serviceLocator } from '@crawlee/core'; import { FileSystemStorageBackend } from '@crawlee/fs-storage'; import { sleep } from '@crawlee/utils'; -// `vitest.mockObject` clones the object and drops its prototype, so build the mock manually to -// keep it an `instanceof BaseHttpClient`. -const createMockHttpClient = () => - Object.assign(Object.create(BaseHttpClient.prototype) as BaseHttpClient, { - sendRequest: vitest.fn(async (_request?: any, _options?: any) => new Response()), - stream: vitest.fn(async () => new Response()), - }); - -let mockHttpClient = createMockHttpClient(); - -beforeEach(async () => { - mockHttpClient = createMockHttpClient(); -}); - describe('RequestQueue remote', () => { beforeEach(async () => { serviceLocator.setStorageBackend(new MemoryStorageBackend()); @@ -477,129 +462,6 @@ describe('RequestQueue remote', () => { }); }); -describe('RequestQueue with requestsFromUrl', () => { - beforeEach(async () => { - serviceLocator.setStorageBackend(new MemoryStorageBackend()); - vitest.restoreAllMocks(); - }); - - test('should correctly load list from hosted files in correct order', async () => { - const list1 = ['https://example.com', 'https://google.com', 'https://wired.com']; - const list2 = ['https://another.com', 'https://page.com']; - mockHttpClient.sendRequest - .mockImplementationOnce(async () => { - await sleep(100); - return new Response(list1.join('\n')); - }) - .mockResolvedValueOnce(new Response(list2.join('\n'))); - - const queue = await RequestQueue.open(null, { httpClient: mockHttpClient }); - await queue.addRequests([ - { method: 'GET', requestsFromUrl: 'http://example.com/list-1' }, - { method: 'POST', requestsFromUrl: 'http://example.com/list-2' }, - ]); - - expect(await queue.fetchNextRequest()).toMatchObject({ method: 'GET', url: list1[0] }); - expect(await queue.fetchNextRequest()).toMatchObject({ method: 'GET', url: list1[1] }); - expect(await queue.fetchNextRequest()).toMatchObject({ method: 'GET', url: list1[2] }); - expect(await queue.fetchNextRequest()).toMatchObject({ method: 'POST', url: list2[0] }); - expect(await queue.fetchNextRequest()).toMatchObject({ method: 'POST', url: list2[1] }); - - expect(mockHttpClient.sendRequest).toHaveBeenCalledTimes(2); - expect(mockHttpClient.sendRequest.mock.calls.map(([request]) => request.url)).toEqual([ - 'http://example.com/list-1', - 'http://example.com/list-2', - ]); - }); - - test('should use regex parameter to parse urls', async () => { - const listStr = 'kjnjkn"https://example.com/a/b/c?q=1#abc";,"HTTP://google.com/a/b/c";dgg:dd'; - const listArr = ['https://example.com', 'HTTP://google.com']; - - mockHttpClient.sendRequest.mockResolvedValueOnce(new Response(listStr)); - - const regex = /(https:\/\/example.com|HTTP:\/\/google.com)/g; - const queue = await RequestQueue.open(null, { - httpClient: mockHttpClient, - }); - await queue.addRequest({ - method: 'GET', - requestsFromUrl: 'http://example.com/list-1', - regex, - }); - - expect(await queue.fetchNextRequest()).toMatchObject({ method: 'GET', url: listArr[0] }); - expect(await queue.fetchNextRequest()).toMatchObject({ method: 'GET', url: listArr[1] }); - await queue.drop(); - - expect(mockHttpClient.sendRequest).toHaveBeenCalled(); - expect(mockHttpClient.sendRequest.mock.calls[0][0].url).toBe('http://example.com/list-1'); - }); - - test('should fix gdoc sharing url in `requestsFromUrl` automatically (GH issue #639)', async () => { - const list = ['https://example.com', 'https://google.com', 'https://wired.com']; - const wrongUrls = [ - 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU', - 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/', - 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/edit?usp=sharing', - 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/123123132', - 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/?q=blablabla', - 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/edit#gid=0', - ]; - const correctUrl = - 'https://docs.google.com/spreadsheets/d/11UGSBOSXy5Ov2WEP9nr4kSIxQJmH18zh-5onKtBsovU/gviz/tq?tqx=out:csv'; - - mockHttpClient.sendRequest.mockImplementation(async () => new Response(list.join('\n'), { status: 200 })); - - const queue = await RequestQueue.open(null, { - httpClient: mockHttpClient, - }); - await queue.addRequests(wrongUrls.map((requestsFromUrl) => ({ requestsFromUrl }))); - - expect(await queue.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[0] }); - expect(await queue.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[1] }); - expect(await queue.fetchNextRequest()).toMatchObject({ method: 'GET', url: list[2] }); - - expect(mockHttpClient.sendRequest.mock.calls[0][0].url).toBe(correctUrl); - await queue.drop(); - }); - - test('should handle requestsFromUrl with no URLs', async () => { - mockHttpClient.sendRequest.mockResolvedValueOnce(new Response('')); - - const queue = await RequestQueue.open(null, { httpClient: mockHttpClient }); - await queue.addRequest({ - method: 'GET', - requestsFromUrl: 'http://example.com/list-1', - }); - - expect(await queue.fetchNextRequest()).toBe(null); - - expect(mockHttpClient.sendRequest).toHaveBeenCalledTimes(1); - expect(mockHttpClient.sendRequest.mock.calls[0][0].url).toBe('http://example.com/list-1'); - }); - - test('should use the defined proxy server when using `requestsFromUrl`', async () => { - const proxyUrls = ['http://proxyurl.usedforthe.download', 'http://another.proxy.url']; - - const proxyConfiguration = new ProxyConfiguration({ - proxyUrls, - }); - - const queue = await RequestQueue.open(null, { proxyConfiguration, httpClient: mockHttpClient }); - await queue.addRequests([ - { requestsFromUrl: 'http://example.com/list-1' }, - { requestsFromUrl: 'http://example.com/list-2' }, - { requestsFromUrl: 'http://example.com/list-3' }, - ]); - - expect(mockHttpClient.sendRequest).toHaveBeenCalledTimes(3); - for (const [, options] of mockHttpClient.sendRequest.mock.calls) { - expect(proxyUrls).toContain(options.proxyUrl); - } - }); -}); - describe('RequestQueue (request lifecycle)', () => { const totalRequestsPerTest = 50; diff --git a/test/core/storages/storage-transaction.test.ts b/test/core/storages/storage-transaction.test.ts index a28357010f88..c826c064d226 100644 --- a/test/core/storages/storage-transaction.test.ts +++ b/test/core/storages/storage-transaction.test.ts @@ -14,7 +14,6 @@ import { withDirectStorageAccess, withStorageTransaction, } from '@crawlee/core'; -import { BaseHttpClient } from '@crawlee/http-client'; import type { DatasetBackend } from '@crawlee/types'; import { storage as timeoutStorage } from '@apify/timeout'; @@ -972,51 +971,6 @@ describe('RequestQueue in a transaction', () => { await expect(queue.getTotalCount()).resolves.toBe(1500); }); - test('requestsFromUrl sources are observed through their expanded URLs in `enqueuedUrls`', async () => { - // Mocked at the HTTP client level: the list download goes through the queue's configured client. - const lists: Record = { - 'http://example.com/list-1': ['https://example.com/a', 'https://example.com/b'], - 'http://example.com/list-2': ['https://example.com/c'], - }; - // Carries the BaseHttpClient prototype so the queue's `z.instanceof` validation accepts it. - const sendRequest = vitest.fn(async (request: { url: string }) => { - return new Response(lists[request.url]?.join('\n') ?? ''); - }); - const httpClient = Object.assign(Object.create(BaseHttpClient.prototype) as BaseHttpClient, { - sendRequest, - stream: vitest.fn(async () => new Response()), - }); - - // There is no `enqueuedUrlLists` on the view - the list is downloaded at call time and the - // *fetched* URLs are journaled, under either policy. - const queue = await RequestQueue.open(null, { httpClient }); - - await withStorageTransaction(async (transaction) => { - await queue.addRequests([{ requestsFromUrl: 'http://example.com/list-1', label: 'from-list' }]); - - // writeThrough (default): applied immediately, and journaled for introspection. - expect(transaction.enqueuedUrls).toEqual([ - { url: 'https://example.com/a', label: 'from-list' }, - { url: 'https://example.com/b', label: 'from-list' }, - ]); - }); - - await expect(queue.getTotalCount()).resolves.toBe(2); - - await withStorageTransaction( - async (transaction) => { - await queue.addRequests([{ requestsFromUrl: 'http://example.com/list-2' }]); - - // deferred: journaled the same way, applied only at commit. - expect(transaction.enqueuedUrls).toEqual([{ url: 'https://example.com/c' }]); - }, - { policy: { requestQueue: 'deferred' } }, - ); - - expect(httpClient.sendRequest).toHaveBeenCalledTimes(2); - await expect(queue.getTotalCount()).resolves.toBe(3); - }); - test('crawler bookkeeping operations are rejected inside a transaction', async () => { const queue = await RequestQueue.open(); await queue.addRequest({ url: 'https://example.com' });