diff --git a/test/MemoryStorageEmulator.ts b/test/MemoryStorageEmulator.ts deleted file mode 100644 index 9947d3f6..00000000 --- a/test/MemoryStorageEmulator.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { rm } from 'node:fs/promises'; -import { resolve } from 'node:path'; - -import { StorageManager } from '@crawlee/core'; -import { MemoryStorage } from '@crawlee/memory-storage'; -import { Configuration } from 'apify'; -import { ensureDir } from 'fs-extra'; - -import log from '@apify/log'; -import { cryptoRandomObjectId } from '@apify/utilities'; - -const LOCAL_EMULATION_DIR = resolve(__dirname, '..', 'tmp', 'memory-emulation-dir'); - -export class MemoryStorageEmulator { - protected localStorageDirectories: string[] = []; - protected storage?: MemoryStorage; - - async init(dirName = cryptoRandomObjectId(10)) { - StorageManager.clearCache(); - const localStorageDir = resolve(LOCAL_EMULATION_DIR, dirName); - this.localStorageDirectories.push(localStorageDir); - await ensureDir(localStorageDir); - - this.storage = new MemoryStorage({ - localDataDirectory: localStorageDir, - }); - Configuration.getGlobalConfig().useStorageClient(this.storage); - log.debug(`Initialized emulated memory storage in folder ${localStorageDir}`); - } - - reapplyStorageClient() { - if (!this.storage) { - throw new Error('Storage not initialized. Call init() first.'); - } - Configuration.getGlobalConfig().useStorageClient(this.storage); - } - - async destroy() { - const promises = this.localStorageDirectories.map(async (dir) => { - return rm(dir, { force: true, recursive: true }); - }); - - await Promise.all(promises); - StorageManager.clearCache(); - } - - static toString() { - return '@crawlee/memory-storage'; - } -} diff --git a/test/e2e/runScraperTests.mjs b/test/e2e/runScraperTests.mjs index b3b244fb..bbd3cddf 100644 --- a/test/e2e/runScraperTests.mjs +++ b/test/e2e/runScraperTests.mjs @@ -6,7 +6,7 @@ import { isMainThread, Worker, workerData } from 'node:worker_threads'; import log from '@apify/log'; -import { clearStorage, colors, SKIPPED_TEST_CLOSE_CODE } from './tools.mjs'; +import { clearStorage, colors } from './tools.mjs'; const basePath = join(dirname(fileURLToPath(import.meta.url)), 'scrapers'); @@ -33,6 +33,7 @@ async function run() { }); /** @type Map */ const allLogs = new Map(); + let passedAssertions = 0; worker.stderr.on('data', (data) => { const str = data.toString(); const taskLogs = allLogs.get(dir.name) ?? []; @@ -45,43 +46,35 @@ async function run() { allLogs.set(dir.name, taskLogs); taskLogs.push(str); - if (str.startsWith('[test skipped]')) { - return; - } - const match = str.match(/\[assertion] (passed|failed): (.*)/); if (match) { + if (match[1] === 'passed') passedAssertions++; const c = match[1] === 'passed' ? colors.green : colors.red; log.info(`${colors.yellow(`[${dir.name}] `)}${match[2]}: ${c(match[1])}`); } }); - // eslint-disable-next-line no-loop-func -- we actually want to closure the reference to `failure` - worker.on('exit', async (code) => { - if (code === SKIPPED_TEST_CLOSE_CODE) { - log.info(`Test ${colors.yellow(`[${dir.name}]`)} was skipped`); - return; - } - - const took = (Date.now() - now) / 1000; - const status = code === 0 ? 'success' : 'failure'; - const color = code === 0 ? 'green' : 'red'; - log.info( - `${colors.yellow(`[${dir.name}] `)}${colors[color](`Test finished with status: ${status} `)}${colors.grey(`[took ${took}s]`)}`, - ); - - await clearStorage(`${basePath}/${dir.name}`); - const taskLogs = allLogs.get(dir.name); - - if (code !== 0 && taskLogs?.length > 0) { - // Each log line already end with '\n', - // So we join them with an empty string. - log.info(taskLogs.join('')); - } + const [code] = await once(worker, 'exit'); + + const took = (Date.now() - now) / 1000; + // A test that exits with 0 before calling expect() is not a pass. + const passed = code === 0 && passedAssertions > 0; + const status = passed ? 'success' : 'failure'; + const color = passed ? 'green' : 'red'; + log.info( + `${colors.yellow(`[${dir.name}] `)}${colors[color](`Test finished with status: ${status} `)}${colors.grey(`[took ${took}s]`)}`, + ); + + await clearStorage(`${basePath}/${dir.name}`); + const taskLogs = allLogs.get(dir.name); + + if (!passed && taskLogs?.length > 0) { + // Each log line already end with '\n', + // So we join them with an empty string. + log.info(taskLogs.join('')); + } - if (status === 'failure') failure = true; - }); - await once(worker, 'exit'); + if (!passed) failure = true; } } diff --git a/test/e2e/scrapers/cheerio-max-requests/test.mjs b/test/e2e/scrapers/cheerio-max-requests/test.mjs index 9954138e..ace1d2b1 100644 --- a/test/e2e/scrapers/cheerio-max-requests/test.mjs +++ b/test/e2e/scrapers/cheerio-max-requests/test.mjs @@ -1,6 +1,4 @@ -import { expect, getDatasetItems, getStats, getTestDir, run, skipTest, validateDataset } from '../../tools.mjs'; - -void skipTest('broken test'); +import { expect, getDatasetItems, getStats, getTestDir, run, validateDataset } from '../../tools.mjs'; const testDir = getTestDir(import.meta.url); @@ -10,7 +8,7 @@ process.exit = () => {}; await run(testDir, 'cheerio-scraper', { startUrls: [ { - url: 'https://apify.com/apify', + url: 'https://crawlee.dev/js/docs/guides/request-storage', method: 'GET', userData: { label: 'START' }, }, @@ -28,11 +26,11 @@ await run(testDir, 'cheerio-scraper', { } async function handleStart({ enqueueRequest, $ }) { - const links = $('[data-test="actorCard"] > a') + const links = $('a[href^="/js/docs/guides/"]') .toArray() .map((item) => $(item).attr('href')); for (const link of links) { - const actorDetailUrl = `https://apify.com${link}`; + const actorDetailUrl = `https://crawlee.dev${link}`; await enqueueRequest({ url: actorDetailUrl, userData: { label: 'DETAIL' }, @@ -46,18 +44,14 @@ await run(testDir, 'cheerio-scraper', { await skipLinks(); const uniqueIdentifier = url.split('/').slice(-2).join('/'); - const title = $('header h1').text(); - const description = $('div.Section-body > div > p').text(); - const modifiedDate = $('div:nth-of-type(2) > ul > li:nth-of-type(2)').text(); - const runCount = $('div:nth-of-type(2) > ul > li:nth-of-type(1)').text(); + const title = $('h1').first().text(); + const description = $('article p').first().text(); return { url, uniqueIdentifier, title, description, - modifiedDate, - runCount, }; } }, @@ -76,15 +70,7 @@ await expect(stats.requestsFinished >= 10, 'All requests finished'); const datasetItems = await getDatasetItems(testDir); await expect(datasetItems.length > 5 && datasetItems.length < 15, 'Number of dataset items'); await expect( - validateDataset(datasetItems, [ - 'url', - 'title', - 'uniqueIdentifier', - 'description', - // Skip modifiedAt and runCount since they changed - // 'modifiedDate', - // 'runCount', - ]), + validateDataset(datasetItems, ['url', 'title', 'uniqueIdentifier', 'description']), 'Dataset items validation', ); diff --git a/test/e2e/scrapers/cheerio-page-info/test.mjs b/test/e2e/scrapers/cheerio-page-info/test.mjs index aa8c6156..5a5a14b2 100644 --- a/test/e2e/scrapers/cheerio-page-info/test.mjs +++ b/test/e2e/scrapers/cheerio-page-info/test.mjs @@ -8,7 +8,7 @@ process.exit = () => {}; await run(testDir, 'cheerio-scraper', { startUrls: [ { - url: 'https://warehouse-theme-metal.myshopify.com/collections/all-tvs', + url: 'https://crawlee.dev/js/docs/guides/request-storage', method: 'GET', userData: { label: 'START' }, }, @@ -16,7 +16,7 @@ await run(testDir, 'cheerio-scraper', { keepUrlFragments: false, pseudoUrls: [ { - purl: 'https://warehouse-theme-metal.myshopify.com/products/sony-xbr-65x950g-65-class-64-5-diag-bravia-4k-hdr-ultra-hd-tv', + purl: 'https://crawlee.dev/js/docs/guides/result-storage', method: 'GET', userData: { label: 'DETAIL' }, }, @@ -30,7 +30,7 @@ await run(testDir, 'cheerio-scraper', { } = request; if (label === 'START') { - log.info('Store opened!'); + log.info('Start page opened!'); } if (label === 'DETAIL') { @@ -38,31 +38,10 @@ await run(testDir, 'cheerio-scraper', { log.info(`Scraping ${url}`); await skipLinks(); - const urlPart = url.split('/').slice(-1); // ['sennheiser-mke-440-professional-stereo-shotgun-microphone-mke-440'] - const manufacturer = urlPart[0].split('-')[0]; // 'sennheiser' - const title = $('.product-meta h1').text(); - const sku = $('span.product-meta__sku-number').text(); + const title = $('h1').first().text(); + const description = $('article p').first().text(); - const rawPrice = $('span.price') - .filter((_, el) => $(el).text().includes('$')) - .first() - .text() - .split('$')[1]; - const price = Number(rawPrice.replaceAll(',', '')); - - const inStock = - $('span.product-form__inventory') - .first() - .filter((_, el) => $(el).text().includes('In stock')).length !== 0; - - return { - url, - manufacturer, - title, - sku, - currentPrice: price, - availableInStock: inStock, - }; + return { url, title, description }; } }, proxyConfiguration: { useApifyProxy: false }, @@ -79,9 +58,6 @@ await expect(stats.requestsFinished === 2, 'All requests finished'); const datasetItems = await getDatasetItems(testDir); await expect(datasetItems.length === 1, 'Number of dataset items'); -await expect( - validateDataset(datasetItems, ['url', 'manufacturer', 'title', 'sku', 'currentPrice', 'availableInStock']), - 'Dataset items validation', -); +await expect(validateDataset(datasetItems, ['url', 'title', 'description']), 'Dataset items validation'); process.exit(0); diff --git a/test/e2e/scrapers/jsdom-max-requests/test.mjs b/test/e2e/scrapers/jsdom-max-requests/test.mjs index 722e8a48..2d6f59c6 100644 --- a/test/e2e/scrapers/jsdom-max-requests/test.mjs +++ b/test/e2e/scrapers/jsdom-max-requests/test.mjs @@ -1,6 +1,4 @@ -import { expect, getDatasetItems, getStats, getTestDir, run, skipTest, validateDataset } from '../../tools.mjs'; - -void skipTest('broken test'); +import { expect, getDatasetItems, getStats, getTestDir, run, validateDataset } from '../../tools.mjs'; const testDir = getTestDir(import.meta.url); @@ -10,7 +8,7 @@ process.exit = () => {}; await run(testDir, 'jsdom-scraper', { startUrls: [ { - url: 'https://apify.com/apify', + url: 'https://crawlee.dev/js/docs/guides/request-storage', method: 'GET', userData: { label: 'START' }, }, @@ -29,7 +27,7 @@ await run(testDir, 'jsdom-scraper', { async function handleStart({ enqueueRequest, window }) { const { document } = window; - const links = Array.from(document.querySelectorAll('[data-test="actorCard"] > a')).map((x) => x.href); + const links = Array.from(document.querySelectorAll('a[href^="/js/docs/guides/"]')).map((x) => x.href); for (const link of links) { await enqueueRequest({ url: link, @@ -45,18 +43,14 @@ await run(testDir, 'jsdom-scraper', { await skipLinks(); const uniqueIdentifier = url.split('/').slice(-2).join('/'); - const title = document.querySelector('header h1').textContent; - const description = document.querySelector('div.Section-body > div > p').textContent; - const modifiedDate = document.querySelector('div:nth-of-type(2) > ul > li:nth-of-type(2)').textContent; - const runCount = document.querySelector('div:nth-of-type(2) > ul > li:nth-of-type(1)').textContent; + const title = document.querySelector('h1').textContent; + const description = document.querySelector('article p').textContent; return { url, uniqueIdentifier, title, description, - modifiedDate, - runCount, }; } }, @@ -76,15 +70,7 @@ await expect(stats.requestsFinished >= 10, 'All requests finished'); const datasetItems = await getDatasetItems(testDir); await expect(datasetItems.length > 5 && datasetItems.length < 15, 'Number of dataset items'); await expect( - validateDataset(datasetItems, [ - 'url', - 'title', - 'uniqueIdentifier', - 'description', - // Skip modifiedAt and runCount since they changed - // 'modifiedDate', - // 'runCount', - ]), + validateDataset(datasetItems, ['url', 'title', 'uniqueIdentifier', 'description']), 'Dataset items validation', ); diff --git a/test/e2e/scrapers/jsdom-page-info/test.mjs b/test/e2e/scrapers/jsdom-page-info/test.mjs index 8a15686e..cb9a2e0e 100644 --- a/test/e2e/scrapers/jsdom-page-info/test.mjs +++ b/test/e2e/scrapers/jsdom-page-info/test.mjs @@ -8,7 +8,7 @@ process.exit = () => {}; await run(testDir, 'jsdom-scraper', { startUrls: [ { - url: 'https://warehouse-theme-metal.myshopify.com/collections/all-tvs', + url: 'https://crawlee.dev/js/docs/guides/request-storage', method: 'GET', userData: { label: 'START' }, }, @@ -16,7 +16,7 @@ await run(testDir, 'jsdom-scraper', { keepUrlFragments: false, pseudoUrls: [ { - purl: 'https://warehouse-theme-metal.myshopify.com/products/sony-xbr-65x950g-65-class-64-5-diag-bravia-4k-hdr-ultra-hd-tv', + purl: 'https://crawlee.dev/js/docs/guides/result-storage', method: 'GET', userData: { label: 'DETAIL' }, }, @@ -31,7 +31,7 @@ await run(testDir, 'jsdom-scraper', { const { document } = window; if (label === 'START') { - log.info('Store opened!'); + log.info('Start page opened!'); } if (label === 'DETAIL') { @@ -39,24 +39,10 @@ await run(testDir, 'jsdom-scraper', { log.info(`Scraping ${url}`); await skipLinks(); - const urlPart = url.split('/').slice(-1); // ['sennheiser-mke-440-professional-stereo-shotgun-microphone-mke-440'] - const manufacturer = urlPart[0].split('-')[0]; // 'sennheiser' - const title = document.querySelector('.product-meta h1').textContent; - const sku = document.querySelector('span.product-meta__sku-number').textContent; + const title = document.querySelector('h1').textContent; + const description = document.querySelector('article p').textContent; - const rawPrice = document.querySelector('span.price').textContent.split('$')[1]; - const price = Number(rawPrice.replaceAll(',', '')); - - const inStock = document.querySelector('span.product-form__inventory').textContent.includes('In stock'); - - return { - url, - manufacturer, - title, - sku, - currentPrice: price, - availableInStock: inStock, - }; + return { url, title, description }; } }, proxyConfiguration: { useApifyProxy: false }, @@ -74,9 +60,6 @@ await expect(stats.requestsFinished === 2, 'All requests finished'); const datasetItems = await getDatasetItems(testDir); await expect(datasetItems.length === 1, 'Number of dataset items'); -await expect( - validateDataset(datasetItems, ['url', 'manufacturer', 'title', 'sku', 'currentPrice', 'availableInStock']), - 'Dataset items validation', -); +await expect(validateDataset(datasetItems, ['url', 'title', 'description']), 'Dataset items validation'); process.exit(0); diff --git a/test/e2e/scrapers/web-dev-mode/test.mjs b/test/e2e/scrapers/web-dev-mode/test.mjs index 8cc32091..acf69e93 100644 --- a/test/e2e/scrapers/web-dev-mode/test.mjs +++ b/test/e2e/scrapers/web-dev-mode/test.mjs @@ -1,6 +1,4 @@ -import { expect, getDatasetItems, getStats, getTestDir, run, skipTest, validateDataset } from '../../tools.mjs'; - -void skipTest('httpstat.us is very unstable'); +import { expect, getDatasetItems, getStats, getTestDir, run, validateDataset } from '../../tools.mjs'; const testDir = getTestDir(import.meta.url); @@ -11,14 +9,14 @@ await run(testDir, 'web-scraper', { runMode: 'DEVELOPMENT', startUrls: [ { - url: 'http://httpstat.us/', + url: 'https://crawlee.dev/js/docs/guides/request-storage', method: 'GET', userData: { label: 'START' }, }, ], pseudoUrls: [ { - purl: 'http://httpstat.us/[200|401|403|429|500]', + purl: 'https://crawlee.dev/js/docs/guides/[.+]', method: 'GET', userData: { label: 'DETAIL' }, }, diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjs index 1a5dff4d..21ce988a 100644 --- a/test/e2e/tools.mjs +++ b/test/e2e/tools.mjs @@ -10,8 +10,6 @@ import { Configuration, KeyValueStore, log } from 'apify'; import { sleep, URL_NO_COMMAS_REGEX } from 'crawlee'; import fs from 'fs-extra'; -export const SKIPPED_TEST_CLOSE_CODE = 404; - /** @type {Record string>} */ export const colors = { red: (text) => `\x1B[31m${text}\x1B[39m`, @@ -63,25 +61,13 @@ export async function run(url, scraper, input) { const inputKey = Configuration.getGlobalConfig().get('inputKey'); await KeyValueStore.setValue(inputKey, input); + // Actor.exit() calls process.exit() only after the run is fully closed, so that call ends the run. + const actorExited = new Promise((resolve) => { + process.exit = resolve; + }); await import(`../../packages/actor-scraper/${scraper}/dist/main.js`); - // Some runs don't save the final stats, and stats.crawlerFinishedAt - // is always null, therefore waitForFinish never resolves. - // However, logs do say that actor finished with exit code 0, - // i.e. dataset items are there, etc. Honestly, no idea why - - // hanging test is always random. So adding Promise.race() - // to make sure all tests will run and finish. - await Promise.race([waitForFinish(url), setTimeout(120e3)]); -} - -export async function waitForFinish(dir) { - while (!(await isFinished(dir))) { - await setTimeout(1e3); - } -} - -async function isFinished(dir) { - const stats = await getStats(dir); - return !!stats.crawlerFinishedAt; + // The limit only catches an Actor that never exits; the badssl.com suites can take over two minutes. + await Promise.race([actorExited, setTimeout(300e3)]); } /** @@ -159,14 +145,6 @@ export async function expect(bool, message) { } } -/** - * @param {string} reason - */ -export async function skipTest(reason) { - log.warning(`[test skipped] ${reason}`); - process.exit(SKIPPED_TEST_CLOSE_CODE); -} - /** * @param {Record} item * @param {string} propName @@ -180,10 +158,6 @@ function checkDatasetItem(item, propName) { switch (propName) { case 'url': return item.url.match(URL_NO_COMMAS_REGEX); - case 'modifiedDate': - return !Number.isNaN(Date.parse(item.modifiedDate)); - case 'runCount': - return Number.isInteger(item.runCount); default: return ['string', 'number', 'boolean'].includes(typeof item[propName]); }