From 7fab574aea1667bcfe08982563eb49ccb8c4cffb Mon Sep 17 00:00:00 2001 From: nikitachapovskii-dev Date: Tue, 10 Nov 2026 15:32:57 +0100 Subject: [PATCH 1/4] test: drop dead e2e tests and fix silent passes in the runner --- test/e2e/runScraperTests.mjs | 53 +++++------ .../scrapers/cheerio-max-requests/test.mjs | 91 ------------------- test/e2e/scrapers/jsdom-max-requests/test.mjs | 91 ------------------- test/e2e/scrapers/web-dev-mode/test.mjs | 81 ----------------- test/e2e/tools.mjs | 10 -- 5 files changed, 23 insertions(+), 303 deletions(-) delete mode 100644 test/e2e/scrapers/cheerio-max-requests/test.mjs delete mode 100644 test/e2e/scrapers/jsdom-max-requests/test.mjs delete mode 100644 test/e2e/scrapers/web-dev-mode/test.mjs diff --git a/test/e2e/runScraperTests.mjs b/test/e2e/runScraperTests.mjs index b3b244fb..572a6772 100644 --- a/test/e2e/runScraperTests.mjs +++ b/test/e2e/runScraperTests.mjs @@ -6,7 +6,7 @@ import { isMainThread, Worker, workerData } from 'node:worker_threads'; import log from '@apify/log'; -import { clearStorage, colors, SKIPPED_TEST_CLOSE_CODE } from './tools.mjs'; +import { clearStorage, colors } from './tools.mjs'; const basePath = join(dirname(fileURLToPath(import.meta.url)), 'scrapers'); @@ -33,6 +33,7 @@ async function run() { }); /** @type Map */ const allLogs = new Map(); + let passedAssertions = 0; worker.stderr.on('data', (data) => { const str = data.toString(); const taskLogs = allLogs.get(dir.name) ?? []; @@ -45,43 +46,35 @@ async function run() { allLogs.set(dir.name, taskLogs); taskLogs.push(str); - if (str.startsWith('[test skipped]')) { - return; - } - const match = str.match(/\[assertion] (passed|failed): (.*)/); if (match) { + if (match[1] === 'passed') passedAssertions++; const c = match[1] === 'passed' ? colors.green : colors.red; log.info(`${colors.yellow(`[${dir.name}] `)}${match[2]}: ${c(match[1])}`); } }); - // eslint-disable-next-line no-loop-func -- we actually want to closure the reference to `failure` - worker.on('exit', async (code) => { - if (code === SKIPPED_TEST_CLOSE_CODE) { - log.info(`Test ${colors.yellow(`[${dir.name}]`)} was skipped`); - return; - } - - const took = (Date.now() - now) / 1000; - const status = code === 0 ? 'success' : 'failure'; - const color = code === 0 ? 'green' : 'red'; - log.info( - `${colors.yellow(`[${dir.name}] `)}${colors[color](`Test finished with status: ${status} `)}${colors.grey(`[took ${took}s]`)}`, - ); - - await clearStorage(`${basePath}/${dir.name}`); - const taskLogs = allLogs.get(dir.name); - - if (code !== 0 && taskLogs?.length > 0) { - // Each log line already end with '\n', - // So we join them with an empty string. - log.info(taskLogs.join('')); - } + const [code] = await once(worker, 'exit'); + + const took = (Date.now() - now) / 1000; + // A worker can exit with 0 without reaching the test's assertions. + const passed = code === 0 && passedAssertions > 0; + const status = passed ? 'success' : 'failure'; + const color = passed ? 'green' : 'red'; + log.info( + `${colors.yellow(`[${dir.name}] `)}${colors[color](`Test finished with status: ${status} `)}${colors.grey(`[took ${took}s]`)}`, + ); + + await clearStorage(`${basePath}/${dir.name}`); + const taskLogs = allLogs.get(dir.name); + + if (!passed && taskLogs?.length > 0) { + // Each log line already end with '\n', + // So we join them with an empty string. + log.info(taskLogs.join('')); + } - if (status === 'failure') failure = true; - }); - await once(worker, 'exit'); + if (!passed) failure = true; } } diff --git a/test/e2e/scrapers/cheerio-max-requests/test.mjs b/test/e2e/scrapers/cheerio-max-requests/test.mjs deleted file mode 100644 index 9954138e..00000000 --- a/test/e2e/scrapers/cheerio-max-requests/test.mjs +++ /dev/null @@ -1,91 +0,0 @@ -import { expect, getDatasetItems, getStats, getTestDir, run, skipTest, validateDataset } from '../../tools.mjs'; - -void skipTest('broken test'); - -const testDir = getTestDir(import.meta.url); - -const { exit } = process; -process.exit = () => {}; - -await run(testDir, 'cheerio-scraper', { - startUrls: [ - { - url: 'https://apify.com/apify', - method: 'GET', - userData: { label: 'START' }, - }, - ], - keepUrlFragments: false, - maxPagesPerCrawl: 10, - pageFunction: async function pageFunction(context) { - switch (context.request.userData.label) { - case 'START': - return handleStart(context); - case 'DETAIL': - return handleDetail(context); - default: - throw new Error(`Unrecognized request label: ${context.request.userData.label}`); - } - - async function handleStart({ enqueueRequest, $ }) { - const links = $('[data-test="actorCard"] > a') - .toArray() - .map((item) => $(item).attr('href')); - for (const link of links) { - const actorDetailUrl = `https://apify.com${link}`; - await enqueueRequest({ - url: actorDetailUrl, - userData: { label: 'DETAIL' }, - }); - } - } - - async function handleDetail({ request, log, skipLinks, $ }) { - const { url } = request; - log.info(`Scraping ${url}`); - await skipLinks(); - - const uniqueIdentifier = url.split('/').slice(-2).join('/'); - const title = $('header h1').text(); - const description = $('div.Section-body > div > p').text(); - const modifiedDate = $('div:nth-of-type(2) > ul > li:nth-of-type(2)').text(); - const runCount = $('div:nth-of-type(2) > ul > li:nth-of-type(1)').text(); - - return { - url, - uniqueIdentifier, - title, - description, - modifiedDate, - runCount, - }; - } - }, - proxyConfiguration: { useApifyProxy: false }, - proxyRotation: 'RECOMMENDED', - forceResponseEncoding: false, - ignoreSslErrors: false, - debugLog: false, -}); - -process.exit = exit; - -const stats = await getStats(testDir); -await expect(stats.requestsFinished >= 10, 'All requests finished'); - -const datasetItems = await getDatasetItems(testDir); -await expect(datasetItems.length > 5 && datasetItems.length < 15, 'Number of dataset items'); -await expect( - validateDataset(datasetItems, [ - 'url', - 'title', - 'uniqueIdentifier', - 'description', - // Skip modifiedAt and runCount since they changed - // 'modifiedDate', - // 'runCount', - ]), - 'Dataset items validation', -); - -process.exit(0); diff --git a/test/e2e/scrapers/jsdom-max-requests/test.mjs b/test/e2e/scrapers/jsdom-max-requests/test.mjs deleted file mode 100644 index 722e8a48..00000000 --- a/test/e2e/scrapers/jsdom-max-requests/test.mjs +++ /dev/null @@ -1,91 +0,0 @@ -import { expect, getDatasetItems, getStats, getTestDir, run, skipTest, validateDataset } from '../../tools.mjs'; - -void skipTest('broken test'); - -const testDir = getTestDir(import.meta.url); - -const { exit } = process; -process.exit = () => {}; - -await run(testDir, 'jsdom-scraper', { - startUrls: [ - { - url: 'https://apify.com/apify', - method: 'GET', - userData: { label: 'START' }, - }, - ], - keepUrlFragments: false, - maxPagesPerCrawl: 10, - pageFunction: async function pageFunction(context) { - switch (context.request.userData.label) { - case 'START': - return handleStart(context); - case 'DETAIL': - return handleDetail(context); - default: - throw new Error(`Unrecognized request label: ${context.request.userData.label}`); - } - - async function handleStart({ enqueueRequest, window }) { - const { document } = window; - const links = Array.from(document.querySelectorAll('[data-test="actorCard"] > a')).map((x) => x.href); - for (const link of links) { - await enqueueRequest({ - url: link, - userData: { label: 'DETAIL' }, - }); - } - } - - async function handleDetail({ request, log, skipLinks, window }) { - const { url } = request; - const { document } = window; - log.info(`Scraping ${url}`); - await skipLinks(); - - const uniqueIdentifier = url.split('/').slice(-2).join('/'); - const title = document.querySelector('header h1').textContent; - const description = document.querySelector('div.Section-body > div > p').textContent; - const modifiedDate = document.querySelector('div:nth-of-type(2) > ul > li:nth-of-type(2)').textContent; - const runCount = document.querySelector('div:nth-of-type(2) > ul > li:nth-of-type(1)').textContent; - - return { - url, - uniqueIdentifier, - title, - description, - modifiedDate, - runCount, - }; - } - }, - proxyConfiguration: { useApifyProxy: false }, - proxyRotation: 'RECOMMENDED', - forceResponseEncoding: false, - ignoreSslErrors: false, - debugLog: false, - runScripts: false, -}); - -process.exit = exit; - -const stats = await getStats(testDir); -await expect(stats.requestsFinished >= 10, 'All requests finished'); - -const datasetItems = await getDatasetItems(testDir); -await expect(datasetItems.length > 5 && datasetItems.length < 15, 'Number of dataset items'); -await expect( - validateDataset(datasetItems, [ - 'url', - 'title', - 'uniqueIdentifier', - 'description', - // Skip modifiedAt and runCount since they changed - // 'modifiedDate', - // 'runCount', - ]), - 'Dataset items validation', -); - -process.exit(0); diff --git a/test/e2e/scrapers/web-dev-mode/test.mjs b/test/e2e/scrapers/web-dev-mode/test.mjs deleted file mode 100644 index 8cc32091..00000000 --- a/test/e2e/scrapers/web-dev-mode/test.mjs +++ /dev/null @@ -1,81 +0,0 @@ -import { expect, getDatasetItems, getStats, getTestDir, run, skipTest, validateDataset } from '../../tools.mjs'; - -void skipTest('httpstat.us is very unstable'); - -const testDir = getTestDir(import.meta.url); - -const { exit } = process; -process.exit = () => {}; - -await run(testDir, 'web-scraper', { - runMode: 'DEVELOPMENT', - startUrls: [ - { - url: 'http://httpstat.us/', - method: 'GET', - userData: { label: 'START' }, - }, - ], - pseudoUrls: [ - { - purl: 'http://httpstat.us/[200|401|403|429|500]', - method: 'GET', - userData: { label: 'DETAIL' }, - }, - ], - linkSelector: 'a[href]', - keepUrlFragments: false, - pageFunction: async function pageFunction(context) { - const { - request: { - userData: { label }, - }, - } = context; - - switch (label) { - case 'START': - return handleStart(context); - case 'DETAIL': - return handleDetail(context); - default: - throw new Error(`Unrecognized request label: ${label}`); - } - - async function handleStart({ log }) { - log.info('The https stats page successfully opened!'); - } - - async function handleDetail({ request, log, jQuery: $ }) { - const { url } = request; - log.info(`Scraping ${url}`); - - const text = $('pre').text(); - - return { url, text }; - } - }, - injectJQuery: true, - injectUnderscore: false, - proxyConfiguration: { useApifyProxy: false }, - proxyRotation: 'RECOMMENDED', - useChrome: false, - ignoreSslErrors: false, - ignoreCorsAndCsp: false, - downloadMedia: true, - downloadCss: true, - waitUntil: ['networkidle2'], - breakpointLocation: 'NONE', - debugLog: false, - browserLog: false, -}); - -process.exit = exit; - -const stats = await getStats(testDir); -await expect(stats.requestsFinished > 5, 'All requests finished'); - -const datasetItems = await getDatasetItems(testDir); -await expect(datasetItems.length >= 5, 'Minimum number of dataset items'); -await expect(validateDataset(datasetItems, ['url', 'text']), 'Dataset items validation'); - -process.exit(0); diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjs index 1a5dff4d..51df5fab 100644 --- a/test/e2e/tools.mjs +++ b/test/e2e/tools.mjs @@ -10,8 +10,6 @@ import { Configuration, KeyValueStore, log } from 'apify'; import { sleep, URL_NO_COMMAS_REGEX } from 'crawlee'; import fs from 'fs-extra'; -export const SKIPPED_TEST_CLOSE_CODE = 404; - /** @type {Record string>} */ export const colors = { red: (text) => `\x1B[31m${text}\x1B[39m`, @@ -159,14 +157,6 @@ export async function expect(bool, message) { } } -/** - * @param {string} reason - */ -export async function skipTest(reason) { - log.warning(`[test skipped] ${reason}`); - process.exit(SKIPPED_TEST_CLOSE_CODE); -} - /** * @param {Record} item * @param {string} propName From 3d0e87fe204b709ea0861b6b84d055f781dbcbc0 Mon Sep 17 00:00:00 2001 From: nikitachapovskii-dev Date: Wed, 11 Nov 2026 09:20:51 +0100 Subject: [PATCH 2/4] test: fix silent e2e passes and revive the skipped suites --- test/MemoryStorageEmulator.ts | 50 ------------ test/e2e/runScraperTests.mjs | 2 +- .../scrapers/cheerio-max-requests/test.mjs | 77 ++++++++++++++++++ test/e2e/scrapers/jsdom-max-requests/test.mjs | 77 ++++++++++++++++++ test/e2e/scrapers/web-dev-mode/test.mjs | 79 +++++++++++++++++++ test/e2e/tools.mjs | 27 ++----- 6 files changed, 239 insertions(+), 73 deletions(-) delete mode 100644 test/MemoryStorageEmulator.ts create mode 100644 test/e2e/scrapers/cheerio-max-requests/test.mjs create mode 100644 test/e2e/scrapers/jsdom-max-requests/test.mjs create mode 100644 test/e2e/scrapers/web-dev-mode/test.mjs diff --git a/test/MemoryStorageEmulator.ts b/test/MemoryStorageEmulator.ts deleted file mode 100644 index 9947d3f6..00000000 --- a/test/MemoryStorageEmulator.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { rm } from 'node:fs/promises'; -import { resolve } from 'node:path'; - -import { StorageManager } from '@crawlee/core'; -import { MemoryStorage } from '@crawlee/memory-storage'; -import { Configuration } from 'apify'; -import { ensureDir } from 'fs-extra'; - -import log from '@apify/log'; -import { cryptoRandomObjectId } from '@apify/utilities'; - -const LOCAL_EMULATION_DIR = resolve(__dirname, '..', 'tmp', 'memory-emulation-dir'); - -export class MemoryStorageEmulator { - protected localStorageDirectories: string[] = []; - protected storage?: MemoryStorage; - - async init(dirName = cryptoRandomObjectId(10)) { - StorageManager.clearCache(); - const localStorageDir = resolve(LOCAL_EMULATION_DIR, dirName); - this.localStorageDirectories.push(localStorageDir); - await ensureDir(localStorageDir); - - this.storage = new MemoryStorage({ - localDataDirectory: localStorageDir, - }); - Configuration.getGlobalConfig().useStorageClient(this.storage); - log.debug(`Initialized emulated memory storage in folder ${localStorageDir}`); - } - - reapplyStorageClient() { - if (!this.storage) { - throw new Error('Storage not initialized. Call init() first.'); - } - Configuration.getGlobalConfig().useStorageClient(this.storage); - } - - async destroy() { - const promises = this.localStorageDirectories.map(async (dir) => { - return rm(dir, { force: true, recursive: true }); - }); - - await Promise.all(promises); - StorageManager.clearCache(); - } - - static toString() { - return '@crawlee/memory-storage'; - } -} diff --git a/test/e2e/runScraperTests.mjs b/test/e2e/runScraperTests.mjs index 572a6772..bbd3cddf 100644 --- a/test/e2e/runScraperTests.mjs +++ b/test/e2e/runScraperTests.mjs @@ -57,7 +57,7 @@ async function run() { const [code] = await once(worker, 'exit'); const took = (Date.now() - now) / 1000; - // A worker can exit with 0 without reaching the test's assertions. + // A test that exits with 0 before calling expect() is not a pass. const passed = code === 0 && passedAssertions > 0; const status = passed ? 'success' : 'failure'; const color = passed ? 'green' : 'red'; diff --git a/test/e2e/scrapers/cheerio-max-requests/test.mjs b/test/e2e/scrapers/cheerio-max-requests/test.mjs new file mode 100644 index 00000000..ace1d2b1 --- /dev/null +++ b/test/e2e/scrapers/cheerio-max-requests/test.mjs @@ -0,0 +1,77 @@ +import { expect, getDatasetItems, getStats, getTestDir, run, validateDataset } from '../../tools.mjs'; + +const testDir = getTestDir(import.meta.url); + +const { exit } = process; +process.exit = () => {}; + +await run(testDir, 'cheerio-scraper', { + startUrls: [ + { + url: 'https://crawlee.dev/js/docs/guides/request-storage', + method: 'GET', + userData: { label: 'START' }, + }, + ], + keepUrlFragments: false, + maxPagesPerCrawl: 10, + pageFunction: async function pageFunction(context) { + switch (context.request.userData.label) { + case 'START': + return handleStart(context); + case 'DETAIL': + return handleDetail(context); + default: + throw new Error(`Unrecognized request label: ${context.request.userData.label}`); + } + + async function handleStart({ enqueueRequest, $ }) { + const links = $('a[href^="/js/docs/guides/"]') + .toArray() + .map((item) => $(item).attr('href')); + for (const link of links) { + const actorDetailUrl = `https://crawlee.dev${link}`; + await enqueueRequest({ + url: actorDetailUrl, + userData: { label: 'DETAIL' }, + }); + } + } + + async function handleDetail({ request, log, skipLinks, $ }) { + const { url } = request; + log.info(`Scraping ${url}`); + await skipLinks(); + + const uniqueIdentifier = url.split('/').slice(-2).join('/'); + const title = $('h1').first().text(); + const description = $('article p').first().text(); + + return { + url, + uniqueIdentifier, + title, + description, + }; + } + }, + proxyConfiguration: { useApifyProxy: false }, + proxyRotation: 'RECOMMENDED', + forceResponseEncoding: false, + ignoreSslErrors: false, + debugLog: false, +}); + +process.exit = exit; + +const stats = await getStats(testDir); +await expect(stats.requestsFinished >= 10, 'All requests finished'); + +const datasetItems = await getDatasetItems(testDir); +await expect(datasetItems.length > 5 && datasetItems.length < 15, 'Number of dataset items'); +await expect( + validateDataset(datasetItems, ['url', 'title', 'uniqueIdentifier', 'description']), + 'Dataset items validation', +); + +process.exit(0); diff --git a/test/e2e/scrapers/jsdom-max-requests/test.mjs b/test/e2e/scrapers/jsdom-max-requests/test.mjs new file mode 100644 index 00000000..2d6f59c6 --- /dev/null +++ b/test/e2e/scrapers/jsdom-max-requests/test.mjs @@ -0,0 +1,77 @@ +import { expect, getDatasetItems, getStats, getTestDir, run, validateDataset } from '../../tools.mjs'; + +const testDir = getTestDir(import.meta.url); + +const { exit } = process; +process.exit = () => {}; + +await run(testDir, 'jsdom-scraper', { + startUrls: [ + { + url: 'https://crawlee.dev/js/docs/guides/request-storage', + method: 'GET', + userData: { label: 'START' }, + }, + ], + keepUrlFragments: false, + maxPagesPerCrawl: 10, + pageFunction: async function pageFunction(context) { + switch (context.request.userData.label) { + case 'START': + return handleStart(context); + case 'DETAIL': + return handleDetail(context); + default: + throw new Error(`Unrecognized request label: ${context.request.userData.label}`); + } + + async function handleStart({ enqueueRequest, window }) { + const { document } = window; + const links = Array.from(document.querySelectorAll('a[href^="/js/docs/guides/"]')).map((x) => x.href); + for (const link of links) { + await enqueueRequest({ + url: link, + userData: { label: 'DETAIL' }, + }); + } + } + + async function handleDetail({ request, log, skipLinks, window }) { + const { url } = request; + const { document } = window; + log.info(`Scraping ${url}`); + await skipLinks(); + + const uniqueIdentifier = url.split('/').slice(-2).join('/'); + const title = document.querySelector('h1').textContent; + const description = document.querySelector('article p').textContent; + + return { + url, + uniqueIdentifier, + title, + description, + }; + } + }, + proxyConfiguration: { useApifyProxy: false }, + proxyRotation: 'RECOMMENDED', + forceResponseEncoding: false, + ignoreSslErrors: false, + debugLog: false, + runScripts: false, +}); + +process.exit = exit; + +const stats = await getStats(testDir); +await expect(stats.requestsFinished >= 10, 'All requests finished'); + +const datasetItems = await getDatasetItems(testDir); +await expect(datasetItems.length > 5 && datasetItems.length < 15, 'Number of dataset items'); +await expect( + validateDataset(datasetItems, ['url', 'title', 'uniqueIdentifier', 'description']), + 'Dataset items validation', +); + +process.exit(0); diff --git a/test/e2e/scrapers/web-dev-mode/test.mjs b/test/e2e/scrapers/web-dev-mode/test.mjs new file mode 100644 index 00000000..acf69e93 --- /dev/null +++ b/test/e2e/scrapers/web-dev-mode/test.mjs @@ -0,0 +1,79 @@ +import { expect, getDatasetItems, getStats, getTestDir, run, validateDataset } from '../../tools.mjs'; + +const testDir = getTestDir(import.meta.url); + +const { exit } = process; +process.exit = () => {}; + +await run(testDir, 'web-scraper', { + runMode: 'DEVELOPMENT', + startUrls: [ + { + url: 'https://crawlee.dev/js/docs/guides/request-storage', + method: 'GET', + userData: { label: 'START' }, + }, + ], + pseudoUrls: [ + { + purl: 'https://crawlee.dev/js/docs/guides/[.+]', + method: 'GET', + userData: { label: 'DETAIL' }, + }, + ], + linkSelector: 'a[href]', + keepUrlFragments: false, + pageFunction: async function pageFunction(context) { + const { + request: { + userData: { label }, + }, + } = context; + + switch (label) { + case 'START': + return handleStart(context); + case 'DETAIL': + return handleDetail(context); + default: + throw new Error(`Unrecognized request label: ${label}`); + } + + async function handleStart({ log }) { + log.info('The https stats page successfully opened!'); + } + + async function handleDetail({ request, log, jQuery: $ }) { + const { url } = request; + log.info(`Scraping ${url}`); + + const text = $('pre').text(); + + return { url, text }; + } + }, + injectJQuery: true, + injectUnderscore: false, + proxyConfiguration: { useApifyProxy: false }, + proxyRotation: 'RECOMMENDED', + useChrome: false, + ignoreSslErrors: false, + ignoreCorsAndCsp: false, + downloadMedia: true, + downloadCss: true, + waitUntil: ['networkidle2'], + breakpointLocation: 'NONE', + debugLog: false, + browserLog: false, +}); + +process.exit = exit; + +const stats = await getStats(testDir); +await expect(stats.requestsFinished > 5, 'All requests finished'); + +const datasetItems = await getDatasetItems(testDir); +await expect(datasetItems.length >= 5, 'Minimum number of dataset items'); +await expect(validateDataset(datasetItems, ['url', 'text']), 'Dataset items validation'); + +process.exit(0); diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjs index 51df5fab..fc50f03b 100644 --- a/test/e2e/tools.mjs +++ b/test/e2e/tools.mjs @@ -61,25 +61,12 @@ export async function run(url, scraper, input) { const inputKey = Configuration.getGlobalConfig().get('inputKey'); await KeyValueStore.setValue(inputKey, input); + // Actor.exit() calls process.exit() only after the run is fully closed, so that call ends the run. + const actorExited = new Promise((resolve) => { + process.exit = resolve; + }); await import(`../../packages/actor-scraper/${scraper}/dist/main.js`); - // Some runs don't save the final stats, and stats.crawlerFinishedAt - // is always null, therefore waitForFinish never resolves. - // However, logs do say that actor finished with exit code 0, - // i.e. dataset items are there, etc. Honestly, no idea why - - // hanging test is always random. So adding Promise.race() - // to make sure all tests will run and finish. - await Promise.race([waitForFinish(url), setTimeout(120e3)]); -} - -export async function waitForFinish(dir) { - while (!(await isFinished(dir))) { - await setTimeout(1e3); - } -} - -async function isFinished(dir) { - const stats = await getStats(dir); - return !!stats.crawlerFinishedAt; + await Promise.race([actorExited, setTimeout(120e3)]); } /** @@ -170,10 +157,6 @@ function checkDatasetItem(item, propName) { switch (propName) { case 'url': return item.url.match(URL_NO_COMMAS_REGEX); - case 'modifiedDate': - return !Number.isNaN(Date.parse(item.modifiedDate)); - case 'runCount': - return Number.isInteger(item.runCount); default: return ['string', 'number', 'boolean'].includes(typeof item[propName]); } From 6cf2022f8abe86a79734879d04658ec829ec1766 Mon Sep 17 00:00:00 2001 From: nikitachapovskii-dev Date: Wed, 11 Nov 2026 10:34:48 +0100 Subject: [PATCH 3/4] test: give slow e2e suites up to 5 minutes --- test/e2e/tools.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjs index fc50f03b..21ce988a 100644 --- a/test/e2e/tools.mjs +++ b/test/e2e/tools.mjs @@ -66,7 +66,8 @@ export async function run(url, scraper, input) { process.exit = resolve; }); await import(`../../packages/actor-scraper/${scraper}/dist/main.js`); - await Promise.race([actorExited, setTimeout(120e3)]); + // The limit only catches an Actor that never exits; the badssl.com suites can take over two minutes. + await Promise.race([actorExited, setTimeout(300e3)]); } /** From 45ddb1f39e40106337a6a3dc9091fc8f27a903f1 Mon Sep 17 00:00:00 2001 From: nikitachapovskii-dev Date: Wed, 11 Nov 2026 12:12:35 +0100 Subject: [PATCH 4/4] test: move the HTTP page-info suites to crawlee.dev --- test/e2e/scrapers/cheerio-page-info/test.mjs | 38 ++++---------------- test/e2e/scrapers/jsdom-page-info/test.mjs | 31 ++++------------ 2 files changed, 14 insertions(+), 55 deletions(-) diff --git a/test/e2e/scrapers/cheerio-page-info/test.mjs b/test/e2e/scrapers/cheerio-page-info/test.mjs index aa8c6156..5a5a14b2 100644 --- a/test/e2e/scrapers/cheerio-page-info/test.mjs +++ b/test/e2e/scrapers/cheerio-page-info/test.mjs @@ -8,7 +8,7 @@ process.exit = () => {}; await run(testDir, 'cheerio-scraper', { startUrls: [ { - url: 'https://warehouse-theme-metal.myshopify.com/collections/all-tvs', + url: 'https://crawlee.dev/js/docs/guides/request-storage', method: 'GET', userData: { label: 'START' }, }, @@ -16,7 +16,7 @@ await run(testDir, 'cheerio-scraper', { keepUrlFragments: false, pseudoUrls: [ { - purl: 'https://warehouse-theme-metal.myshopify.com/products/sony-xbr-65x950g-65-class-64-5-diag-bravia-4k-hdr-ultra-hd-tv', + purl: 'https://crawlee.dev/js/docs/guides/result-storage', method: 'GET', userData: { label: 'DETAIL' }, }, @@ -30,7 +30,7 @@ await run(testDir, 'cheerio-scraper', { } = request; if (label === 'START') { - log.info('Store opened!'); + log.info('Start page opened!'); } if (label === 'DETAIL') { @@ -38,31 +38,10 @@ await run(testDir, 'cheerio-scraper', { log.info(`Scraping ${url}`); await skipLinks(); - const urlPart = url.split('/').slice(-1); // ['sennheiser-mke-440-professional-stereo-shotgun-microphone-mke-440'] - const manufacturer = urlPart[0].split('-')[0]; // 'sennheiser' - const title = $('.product-meta h1').text(); - const sku = $('span.product-meta__sku-number').text(); + const title = $('h1').first().text(); + const description = $('article p').first().text(); - const rawPrice = $('span.price') - .filter((_, el) => $(el).text().includes('$')) - .first() - .text() - .split('$')[1]; - const price = Number(rawPrice.replaceAll(',', '')); - - const inStock = - $('span.product-form__inventory') - .first() - .filter((_, el) => $(el).text().includes('In stock')).length !== 0; - - return { - url, - manufacturer, - title, - sku, - currentPrice: price, - availableInStock: inStock, - }; + return { url, title, description }; } }, proxyConfiguration: { useApifyProxy: false }, @@ -79,9 +58,6 @@ await expect(stats.requestsFinished === 2, 'All requests finished'); const datasetItems = await getDatasetItems(testDir); await expect(datasetItems.length === 1, 'Number of dataset items'); -await expect( - validateDataset(datasetItems, ['url', 'manufacturer', 'title', 'sku', 'currentPrice', 'availableInStock']), - 'Dataset items validation', -); +await expect(validateDataset(datasetItems, ['url', 'title', 'description']), 'Dataset items validation'); process.exit(0); diff --git a/test/e2e/scrapers/jsdom-page-info/test.mjs b/test/e2e/scrapers/jsdom-page-info/test.mjs index 8a15686e..cb9a2e0e 100644 --- a/test/e2e/scrapers/jsdom-page-info/test.mjs +++ b/test/e2e/scrapers/jsdom-page-info/test.mjs @@ -8,7 +8,7 @@ process.exit = () => {}; await run(testDir, 'jsdom-scraper', { startUrls: [ { - url: 'https://warehouse-theme-metal.myshopify.com/collections/all-tvs', + url: 'https://crawlee.dev/js/docs/guides/request-storage', method: 'GET', userData: { label: 'START' }, }, @@ -16,7 +16,7 @@ await run(testDir, 'jsdom-scraper', { keepUrlFragments: false, pseudoUrls: [ { - purl: 'https://warehouse-theme-metal.myshopify.com/products/sony-xbr-65x950g-65-class-64-5-diag-bravia-4k-hdr-ultra-hd-tv', + purl: 'https://crawlee.dev/js/docs/guides/result-storage', method: 'GET', userData: { label: 'DETAIL' }, }, @@ -31,7 +31,7 @@ await run(testDir, 'jsdom-scraper', { const { document } = window; if (label === 'START') { - log.info('Store opened!'); + log.info('Start page opened!'); } if (label === 'DETAIL') { @@ -39,24 +39,10 @@ await run(testDir, 'jsdom-scraper', { log.info(`Scraping ${url}`); await skipLinks(); - const urlPart = url.split('/').slice(-1); // ['sennheiser-mke-440-professional-stereo-shotgun-microphone-mke-440'] - const manufacturer = urlPart[0].split('-')[0]; // 'sennheiser' - const title = document.querySelector('.product-meta h1').textContent; - const sku = document.querySelector('span.product-meta__sku-number').textContent; + const title = document.querySelector('h1').textContent; + const description = document.querySelector('article p').textContent; - const rawPrice = document.querySelector('span.price').textContent.split('$')[1]; - const price = Number(rawPrice.replaceAll(',', '')); - - const inStock = document.querySelector('span.product-form__inventory').textContent.includes('In stock'); - - return { - url, - manufacturer, - title, - sku, - currentPrice: price, - availableInStock: inStock, - }; + return { url, title, description }; } }, proxyConfiguration: { useApifyProxy: false }, @@ -74,9 +60,6 @@ await expect(stats.requestsFinished === 2, 'All requests finished'); const datasetItems = await getDatasetItems(testDir); await expect(datasetItems.length === 1, 'Number of dataset items'); -await expect( - validateDataset(datasetItems, ['url', 'manufacturer', 'title', 'sku', 'currentPrice', 'availableInStock']), - 'Dataset items validation', -); +await expect(validateDataset(datasetItems, ['url', 'title', 'description']), 'Dataset items validation'); process.exit(0);