From e5a0b8b20dd5dc2fb00808445844e863f18e8bb2 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 14 Sep 2026 11:53:52 +0200 Subject: [PATCH 1/8] feat!: make request-body compression configurable BREAKING CHANGE: The client runs only the configured compressor (brotli by default) and lets its error fail the request. It no longer falls back to gzip, or to an uncompressed body, when brotli fails. Pass `compression: 'gzip'` on a runtime whose `node:zlib` lacks brotli. --- docs/02_concepts/07_http-compression.md | 136 ++++++++++++++++++ docs/04_upgrading/upgrading_v3.md | 16 +++ docs/public-api/apify-client.api.md | 41 ++++++ rsbuild.config.ts | 2 +- src/apify_client.ts | 13 ++ src/http_client.ts | 10 +- src/http_compressors/base.ts | 51 +++++++ src/http_compressors/brotli.ts | 65 +++++++++ src/http_compressors/gzip.ts | 61 ++++++++ src/http_compressors/index.ts | 6 + src/http_compressors/resolve.ts | 38 +++++ src/index.ts | 1 + src/interceptors.ts | 48 +++++-- src/utils.ts | 75 +--------- test/http_compressors.test.ts | 182 ++++++++++++++++++++++++ test/utils.test.ts | 28 ---- 16 files changed, 660 insertions(+), 113 deletions(-) create mode 100644 docs/02_concepts/07_http-compression.md create mode 100644 src/http_compressors/base.ts create mode 100644 src/http_compressors/brotli.ts create mode 100644 src/http_compressors/gzip.ts create mode 100644 src/http_compressors/index.ts create mode 100644 src/http_compressors/resolve.ts create mode 100644 test/http_compressors.test.ts diff --git a/docs/02_concepts/07_http-compression.md b/docs/02_concepts/07_http-compression.md new file mode 100644 index 000000000..7a37a0eec --- /dev/null +++ b/docs/02_concepts/07_http-compression.md @@ -0,0 +1,136 @@ +--- +id: http-compression +title: HTTP compression +sidebar_label: HTTP compression +description: 'Configure how the Apify API client for JavaScript compresses request bodies: brotli by default, gzip, or a custom compressor.' +--- + +import ApiLink from '@theme/ApiLink'; + +The client compresses request bodies before sending them to the API. Compression reduces the amount of data transferred over the network, which makes requests faster and saves bandwidth, especially for large payloads such as Actor inputs, dataset items, or key-value store records. + +## How it works + +The client compresses request bodies with the compressor configured via the `compression` option of the `ApifyClient` constructor, brotli by default. The API accepts both brotli and gzip and decompresses the body transparently. A body is compressed only when it's large enough to benefit, its content type isn't already compressed, and the request carries no `Content-Encoding` of its own. For details, see [Minimum body size](#minimum-body-size), [Already-compressed payloads](#already-compressed-payloads), and [Pre-compressed bodies](#pre-compressed-bodies). + +Compression runs on Node.js only. In a browser, the client sends every body as it is, and the `compression` option has no effect. + +## Minimum body size + +The client sends bodies smaller than 1024 bytes without compression and without the `Content-Encoding` header. A body of this size fits in one network packet, so compression doesn't remove a network round trip and only costs CPU time. For very small bodies, the compression format adds bytes and can make the body larger. + +## Already-compressed payloads + +Some payloads carry their own compression, so compressing them again costs CPU and memory while making the request slightly larger. The client skips compression when the request's `Content-Type` is one of these: + +- any `image/*`, `audio/*`, or `video/*` type +- archives such as `application/zip`, `application/gzip`, or `application/x-7z-compressed` +- office documents and packages built on ZIP, such as `.docx`, `.xlsx`, `.epub`, or `.apk` +- web fonts (`font/woff`, `font/woff2`) + +Two kinds of media type are compressed anyway: raw formats such as `image/bmp`, `image/tiff`, and `audio/wav`, and subtypes with a structured syntax suffix such as `image/svg+xml`. Set an accurate `contentType` when uploading media to a key-value store: + +```js +import { readFile } from 'node:fs/promises'; + +const screenshot = await readFile('screenshot.png'); + +// The explicit content type lets the client skip compressing the PNG. +await client.keyValueStore('my-store-id').setRecord({ + key: 'screenshot.png', + value: screenshot, + contentType: 'image/png', +}); +``` + +Without an explicit content type, a `Buffer` is sent as `application/octet-stream`, which the client can't tell apart from uncompressed binary data and therefore still compresses. A stream is sent as it is, whatever its content type. + +## Pre-compressed bodies + +A payload can reach the client already encoded, for example a gzipped file read from disk. Set the `Content-Encoding` header to name the encoding the payload carries. The client then sends the body as it is and forwards the header, so nothing gets compressed twice. The resource clients don't expose the header, so send such a request through `client.httpClient.call()`: + +```js +import { readFile } from 'node:fs/promises'; + +// A file that is gzipped on disk already. +const report = await readFile('report.json.gz'); + +// The explicit content encoding stops the client from compressing the bytes again. +await client.httpClient.call({ + url: `${client.baseUrl}/key-value-stores/my-store-id/records/report.json`, + method: 'PUT', + data: report, + headers: { 'content-type': 'application/json', 'content-encoding': 'gzip' }, + timeoutSecs: 'long', +}); +``` + +The header is forwarded verbatim, so it also covers encodings the client ships no compressor for, such as `deflate`. The API accepts `gzip`, `br`, `deflate`, and `identity`. Passing `identity` turns compression off for a single request without changing how the client is configured. + +The client can't verify that the bytes match the header, so set `Content-Encoding` only when the payload really is encoded that way. Key-value store records are stored exactly as you upload them, which makes the header part of the stored record rather than a transport detail. + +## Configuration + +To choose the compression algorithm, pass `compression` to the client constructor: + +```js +import { ApifyClient } from 'apify-client'; + +// Default: brotli +const client = new ApifyClient({ token: 'MY-APIFY-TOKEN' }); + +// Gzip +const gzipClient = new ApifyClient({ token: 'MY-APIFY-TOKEN', compression: 'gzip' }); +``` + +Both algorithms come from `node:zlib`, so neither needs an extra dependency. If the runtime's `node:zlib` doesn't implement brotli, which can happen with a Node.js compatibility shim or a bundler polyfill, the request fails with the error from the runtime. The client doesn't fall back to gzip on its own, so pass `compression: 'gzip'` on such a runtime. + +## Custom quality and advanced control + +For fine-grained control over compression quality, pass an `HttpCompressor` instance instead of a name: + +```js +import { ApifyClient, BrotliHttpCompressor, GzipHttpCompressor } from 'apify-client'; + +// Brotli at maximum quality +const client = new ApifyClient({ + token: 'MY-APIFY-TOKEN', + compression: new BrotliHttpCompressor({ quality: 11 }), +}); + +// Gzip at the fastest level +const fastClient = new ApifyClient({ + token: 'MY-APIFY-TOKEN', + compression: new GzipHttpCompressor({ quality: 1 }), +}); +``` + +You can also implement a fully custom compressor by extending `HttpCompressor`. The client calls it only for bodies that reach the [minimum body size](#minimum-body-size) and aren't [already compressed](#already-compressed-payloads) or [pre-compressed by the caller](#pre-compressed-bodies): + +```js +import { ApifyClient, HttpCompressor } from 'apify-client'; + +class IdentityCompressor extends HttpCompressor { + /** Value sent in the `Content-Encoding` header. */ + contentEncoding = 'identity'; + + /** Returns the body as it is, so nothing gets compressed. */ + async compress(data) { + return data; + } +} + +const client = new ApifyClient({ token: 'MY-APIFY-TOKEN', compression: new IdentityCompressor() }); +``` + +## Comparison + +| | Brotli | Gzip | +|-------------------------------|-----------------------------------------|---------------------------------------| +| **Compression ratio** | Typically better than gzip | Good | +| **CPU cost** | Moderate, depends on quality | Low | +| **`Content-Encoding` header** | `br` | `gzip` | +| **Quality range** | `0` to `11` | `1` to `9` | +| **Default quality** | `6` | `9` | +| **Enable via config** | `compression: 'brotli'` (default) | `compression: 'gzip'` | +| **Best for** | Large payloads where bandwidth matters | Low CPU cost, runtimes without brotli | diff --git a/docs/04_upgrading/upgrading_v3.md b/docs/04_upgrading/upgrading_v3.md index 6991f3465..04d3a1583 100644 --- a/docs/04_upgrading/upgrading_v3.md +++ b/docs/04_upgrading/upgrading_v3.md @@ -308,6 +308,22 @@ The `timeoutSecs` option of `K `client.httpClient.call()` takes `timeoutSecs` where it read the axios `timeout` in milliseconds. A direct call that passed `timeout: 30000` has to become `timeoutSecs: 30`. +## Request compression is configurable + +The `ApifyClient` constructor takes a `compression` option: `'brotli'`, `'gzip'`, or an instance of `HttpCompressor` for a custom quality or algorithm. The default stays brotli at quality 6, so a client constructed without the option sends the same requests as in v2. For the rules that decide which bodies get compressed, see [HTTP compression](../02_concepts/07_http-compression.md). + +```js +import { ApifyClient, BrotliHttpCompressor } from 'apify-client'; + +const client = new ApifyClient({ token: 'MY-APIFY-TOKEN', compression: 'gzip' }); +const bestClient = new ApifyClient({ + token: 'MY-APIFY-TOKEN', + compression: new BrotliHttpCompressor({ quality: 11 }), +}); +``` + +The silent fallback is gone. In v2, a body was sent gzipped when brotli failed, and uncompressed when gzip failed too, which could happen on a runtime whose `node:zlib` is a polyfill or a Node.js compatibility shim. In v3 the configured compressor is the only one the client runs, and its error fails the request. On such a runtime, pass `compression: 'gzip'` or a custom compressor. + ## `versions().list()` and `envVars().list()` lose their pagination options `ActorVersionCollectionClient.list()` and `ActorEnvVarCollectionClient.list()` accept only the `timeoutSecs` option. Neither endpoint reads `offset`, `limit` or `desc`, and both return every item in one response, so `chunkSize` had nothing to size either. The `ActorVersionCollectionListOptions` and `ActorEnvVarCollectionListOptions` types that declared those four options, deprecated since v2.21.0, are gone from the package. A call that passed any of them no longer compiles, and throws an `ArgumentValidationError` about an unrecognized key in JavaScript. Drop them and the call returns the same items as before. diff --git a/docs/public-api/apify-client.api.md b/docs/public-api/apify-client.api.md index 18597835b..e8c4d9099 100644 --- a/docs/public-api/apify-client.api.md +++ b/docs/public-api/apify-client.api.md @@ -610,6 +610,7 @@ export class ApifyClient { // @public export interface ApifyClientOptions { baseUrl?: string; + compression?: HttpCompressionAlgorithm | HttpCompressor; // (undocumented) maxRetries?: number; minDelayBetweenRetriesMillis?: number; @@ -666,6 +667,20 @@ interface BaseOptions { publicBaseUrl: string; } +// @public +export class BrotliHttpCompressor extends HttpCompressor { + constructor(options?: BrotliHttpCompressorOptions); + // (undocumented) + compress(data: Buffer): Promise; + // (undocumented) + readonly contentEncoding = "br"; +} + +// @public +export interface BrotliHttpCompressorOptions { + quality?: number; +} + // @public export interface Build extends Omit, BuildRePointed { } @@ -2849,6 +2864,20 @@ export interface GetStreamedLogOptions extends TimeoutOptions { toLog?: Log | null | 'default'; } +// @public +export class GzipHttpCompressor extends HttpCompressor { + constructor(options?: GzipHttpCompressorOptions); + // (undocumented) + compress(data: Buffer): Promise; + // (undocumented) + readonly contentEncoding = "gzip"; +} + +// @public +export interface GzipHttpCompressorOptions { + quality?: number; +} + // Not exported by the entry point; reachable only as a referenced type. // @public (undocumented) class HttpClient { @@ -2859,6 +2888,7 @@ class HttpClient { call(config: ApifyRequestConfig): Promise>; // (undocumented) httpAgent?: http.Agent; + httpCompressor: HttpCompressor; // (undocumented) httpsAgent?: https.Agent; // (undocumented) @@ -2883,6 +2913,8 @@ interface HttpClientOptions { // (undocumented) apifyClientStats: Statistics; // (undocumented) + httpCompressor: HttpCompressor; + // (undocumented) logger: Log; // (undocumented) maxRetries: number; @@ -2904,6 +2936,15 @@ interface HttpClientOptions { workflowKey?: string; } +// @public +export type HttpCompressionAlgorithm = 'brotli' | 'gzip'; + +// @public +export abstract class HttpCompressor { + abstract compress(data: Buffer): Promise; + abstract readonly contentEncoding: string; +} + // @public export class InvalidRequestError extends ApifyApiError { } diff --git a/rsbuild.config.ts b/rsbuild.config.ts index 00e717da0..1772009f2 100644 --- a/rsbuild.config.ts +++ b/rsbuild.config.ts @@ -6,7 +6,7 @@ import { version } from './package.json'; const MAX_BUNDLE_BYTES = 360 * 1024; const nodeOnlyModules = /^proxy-agent$/; -const unusedInBrowserBuiltins = ['os', 'zlib', 'util']; +const unusedInBrowserBuiltins = ['os', 'zlib']; const builtinAliases = Object.fromEntries( unusedInBrowserBuiltins.flatMap((m) => [ [m, false], diff --git a/src/apify_client.ts b/src/apify_client.ts index fe1b92842..ac5dc8ee5 100644 --- a/src/apify_client.ts +++ b/src/apify_client.ts @@ -6,6 +6,9 @@ import type { Log } from '@apify/log'; import logger from '@apify/log'; import { HttpClient } from './http_client.js'; +import type { HttpCompressor } from './http_compressors/base.js'; +import type { HttpCompressionAlgorithm } from './http_compressors/resolve.js'; +import { compressionSchema, resolveCompressor } from './http_compressors/resolve.js'; import type { RequestInterceptorFunction } from './interceptors.js'; import { ActorClient } from './resource_clients/actor.js'; import { ActorCollectionClient } from './resource_clients/actor_collection.js'; @@ -42,6 +45,7 @@ import { parseArgument } from './utils.js'; const clientOptionsSchema = z.strictObject({ baseUrl: z.string().default('https://api.apify.com'), + compression: compressionSchema.default('brotli'), publicBaseUrl: z.string().default('https://api.apify.com'), maxRetries: z.number().default(8), minDelayBetweenRetriesMillis: z.number().default(500), @@ -101,6 +105,7 @@ export class ApifyClient { const { baseUrl, + compression, publicBaseUrl, maxRetries, minDelayBetweenRetriesMillis, @@ -126,6 +131,7 @@ export class ApifyClient { maxRetries, minDelayBetweenRetriesMillis, requestInterceptors, + httpCompressor: resolveCompressor(compression), timeoutShortSecs, timeoutMediumSecs, timeoutLongSecs, @@ -584,6 +590,13 @@ export class ApifyClient { export interface ApifyClientOptions { /** @default https://api.apify.com */ baseUrl?: string; + /** + * Compression of request bodies: the name of a built-in algorithm, or an {@link HttpCompressor} instance for + * a custom quality or algorithm. Bodies are compressed only in Node.js, and only when they are large enough + * to benefit and their content type does not already carry its own compression. + * @default 'brotli' + */ + compression?: HttpCompressionAlgorithm | HttpCompressor; /** * @default https://api.apify.com * @since Added in 2.17.0 diff --git a/src/http_client.ts b/src/http_client.ts index bcf2c0e87..48b5d6c3c 100644 --- a/src/http_client.ts +++ b/src/http_client.ts @@ -11,8 +11,9 @@ import { APIFY_ENV_VARS } from '@apify/consts'; import type { Log } from '@apify/log'; import { ApifyApiError } from './apify_api_error.js'; +import type { HttpCompressor } from './http_compressors/base.js'; import type { RequestInterceptorFunction } from './interceptors.js'; -import { InvalidResponseBodyError, requestInterceptors, responseInterceptors } from './interceptors.js'; +import { createRequestInterceptors, InvalidResponseBodyError, responseInterceptors } from './interceptors.js'; import type { Statistics } from './statistics.js'; import type { Timeout, TimeoutTier } from './timeouts.js'; import { asArray, cast, getVersionData, isNode, isStream } from './utils.js'; @@ -30,6 +31,9 @@ export class HttpClient { userProvidedRequestInterceptors: RequestInterceptorFunction[]; + /** Compressor applied to request bodies that are worth compressing. */ + httpCompressor: HttpCompressor; + logger: Log; /** Duration of each timeout tier, in milliseconds. */ @@ -56,6 +60,7 @@ export class HttpClient { this.maxRetries = options.maxRetries; this.minDelayBetweenRetriesMillis = options.minDelayBetweenRetriesMillis; this.userProvidedRequestInterceptors = options.requestInterceptors; + this.httpCompressor = options.httpCompressor; this.timeoutMillis = { short: options.timeoutShortSecs * 1000, medium: options.timeoutMediumSecs * 1000, @@ -111,7 +116,7 @@ export class HttpClient { this.axios.defaults.headers.Authorization = `Bearer ${token}`; } - requestInterceptors.forEach((i) => this.axios.interceptors.request.use(i as any)); + createRequestInterceptors(this.httpCompressor).forEach((i) => this.axios.interceptors.request.use(i as any)); this.userProvidedRequestInterceptors.forEach((i) => this.axios.interceptors.request.use(i as any)); responseInterceptors.forEach((i) => this.axios.interceptors.response.use(i as any)); } @@ -377,6 +382,7 @@ export interface HttpClientOptions { maxRetries: number; minDelayBetweenRetriesMillis: number; requestInterceptors: RequestInterceptorFunction[]; + httpCompressor: HttpCompressor; timeoutShortSecs: number; timeoutMediumSecs: number; timeoutLongSecs: number; diff --git a/src/http_compressors/base.ts b/src/http_compressors/base.ts new file mode 100644 index 000000000..0e5bd0ce3 --- /dev/null +++ b/src/http_compressors/base.ts @@ -0,0 +1,51 @@ +import { z } from 'zod'; + +/** + * Strategy for compressing HTTP request bodies. + * + * Extend this class to create a custom compressor. Set `contentEncoding` to the value that should be sent in the + * `Content-Encoding` header and implement `compress()`. The client calls it only for bodies that are large enough + * to benefit from compression, whose content type does not already carry its own compression, and which the caller + * did not send with a `Content-Encoding` of their own. + * + * @example + * ```javascript + * import { ApifyClient, HttpCompressor } from 'apify-client'; + * + * class IdentityCompressor extends HttpCompressor { + * contentEncoding = 'identity'; + * + * async compress(data) { + * return data; + * } + * } + * + * const client = new ApifyClient({ token: 'my-token', compression: new IdentityCompressor() }); + * ``` + */ +export abstract class HttpCompressor { + /** Value sent in the `Content-Encoding` header, for example `gzip` or `br`. */ + abstract readonly contentEncoding: string; + + /** + * Compresses a request body. + * + * @param data - The raw bytes to compress. + * @returns The compressed bytes. + */ + abstract compress(data: Buffer): Promise; +} + +/** + * Schema accepting an {@link HttpCompressor} by shape, so an instance of the class from another copy of the package + * passes too. + * @internal + */ +export const httpCompressorSchema = z.custom( + (value) => + typeof value === 'object' && + value !== null && + typeof (value as HttpCompressor).contentEncoding === 'string' && + typeof (value as HttpCompressor).compress === 'function', + { error: 'Invalid input: expected an HttpCompressor' }, +); diff --git a/src/http_compressors/brotli.ts b/src/http_compressors/brotli.ts new file mode 100644 index 000000000..1cfd3b7d5 --- /dev/null +++ b/src/http_compressors/brotli.ts @@ -0,0 +1,65 @@ +import { z } from 'zod'; + +import { parseArgument } from '../utils.js'; +import { HttpCompressor } from './base.js'; + +/** Lowest valid brotli quality, the fastest with the least compression. */ +const MIN_QUALITY = 0; + +/** Highest valid brotli quality, the slowest with the best compression. */ +const MAX_QUALITY = 11; + +/** Middle of the range, where brotli already beats gzip at a comparable CPU cost. */ +const DEFAULT_QUALITY = 6; + +const optionsSchema = z.strictObject({ + quality: z.int().min(MIN_QUALITY).max(MAX_QUALITY).default(DEFAULT_QUALITY), +}); + +/** + * Compresses request bodies using brotli. + * + * Uses the `node:zlib` module, so it works wherever the client runs on Node.js. + * + * @example + * ```javascript + * import { ApifyClient, BrotliHttpCompressor } from 'apify-client'; + * + * const client = new ApifyClient({ token: 'my-token', compression: new BrotliHttpCompressor({ quality: 11 }) }); + * ``` + */ +export class BrotliHttpCompressor extends HttpCompressor { + readonly contentEncoding = 'br'; + + readonly #quality: number; + + /** + * @param options - Compressor options. + * @throws {ArgumentValidationError} If `quality` is out of the valid range. + */ + constructor(options: BrotliHttpCompressorOptions = {}) { + super(); + const { quality } = parseArgument(options, optionsSchema, 'BrotliHttpCompressorOptions'); + this.#quality = quality; + } + + async compress(data: Buffer): Promise { + const { brotliCompress, constants } = await import('node:zlib'); + const options = { params: { [constants.BROTLI_PARAM_QUALITY]: this.#quality } }; + + return new Promise((resolve, reject) => { + brotliCompress(data, options, (error, result) => (error ? reject(error) : resolve(result))); + }); + } +} + +/** + * Options for {@link BrotliHttpCompressor}. + */ +export interface BrotliHttpCompressorOptions { + /** + * Compression level, from `0` (the fastest) to `11` (the best compression). + * @default 6 + */ + quality?: number; +} diff --git a/src/http_compressors/gzip.ts b/src/http_compressors/gzip.ts new file mode 100644 index 000000000..1a255f878 --- /dev/null +++ b/src/http_compressors/gzip.ts @@ -0,0 +1,61 @@ +import { z } from 'zod'; + +import { parseArgument } from '../utils.js'; +import { HttpCompressor } from './base.js'; + +/** Lowest valid gzip quality, the fastest with the least compression. */ +const MIN_QUALITY = 1; + +/** Highest valid gzip quality, the slowest with the best compression. */ +const MAX_QUALITY = 9; + +const optionsSchema = z.strictObject({ + quality: z.int().min(MIN_QUALITY).max(MAX_QUALITY).default(MAX_QUALITY), +}); + +/** + * Compresses request bodies using gzip. + * + * Uses the `node:zlib` module, so it works wherever the client runs on Node.js. + * + * @example + * ```javascript + * import { ApifyClient, GzipHttpCompressor } from 'apify-client'; + * + * const client = new ApifyClient({ token: 'my-token', compression: new GzipHttpCompressor({ quality: 6 }) }); + * ``` + */ +export class GzipHttpCompressor extends HttpCompressor { + readonly contentEncoding = 'gzip'; + + readonly #quality: number; + + /** + * @param options - Compressor options. + * @throws {ArgumentValidationError} If `quality` is out of the valid range. + */ + constructor(options: GzipHttpCompressorOptions = {}) { + super(); + const { quality } = parseArgument(options, optionsSchema, 'GzipHttpCompressorOptions'); + this.#quality = quality; + } + + async compress(data: Buffer): Promise { + const { gzip } = await import('node:zlib'); + + return new Promise((resolve, reject) => { + gzip(data, { level: this.#quality }, (error, result) => (error ? reject(error) : resolve(result))); + }); + } +} + +/** + * Options for {@link GzipHttpCompressor}. + */ +export interface GzipHttpCompressorOptions { + /** + * Compression level, from `1` (the fastest) to `9` (the best compression). + * @default 9 + */ + quality?: number; +} diff --git a/src/http_compressors/index.ts b/src/http_compressors/index.ts new file mode 100644 index 000000000..e19dc390e --- /dev/null +++ b/src/http_compressors/index.ts @@ -0,0 +1,6 @@ +export { HttpCompressor } from './base.js'; +export { BrotliHttpCompressor } from './brotli.js'; +export type { BrotliHttpCompressorOptions } from './brotli.js'; +export { GzipHttpCompressor } from './gzip.js'; +export type { GzipHttpCompressorOptions } from './gzip.js'; +export type { HttpCompressionAlgorithm } from './resolve.js'; diff --git a/src/http_compressors/resolve.ts b/src/http_compressors/resolve.ts new file mode 100644 index 000000000..6dbb7c39a --- /dev/null +++ b/src/http_compressors/resolve.ts @@ -0,0 +1,38 @@ +import { z } from 'zod'; + +import type { HttpCompressor } from './base.js'; +import { httpCompressorSchema } from './base.js'; +import { BrotliHttpCompressor } from './brotli.js'; +import { GzipHttpCompressor } from './gzip.js'; + +/** + * Compression algorithms the `compression` option of {@link ApifyClient} accepts by name. Each name selects the + * matching built-in compressor with its default quality: `'brotli'` a {@link BrotliHttpCompressor}, `'gzip'` a + * {@link GzipHttpCompressor}. + */ +export type HttpCompressionAlgorithm = 'brotli' | 'gzip'; + +/** + * Schema of the `compression` option: an algorithm name or an {@link HttpCompressor} instance. + * @internal + */ +export const compressionSchema = z.union([z.enum(['brotli', 'gzip']), httpCompressorSchema]); + +/** + * Turns the `compression` option into a ready-to-use {@link HttpCompressor}: a name into the matching built-in + * compressor with its default quality, an instance into itself. + * @internal + */ +export function resolveCompressor(compression: HttpCompressionAlgorithm | HttpCompressor): HttpCompressor { + if (typeof compression !== 'string') return compression; + + switch (compression) { + case 'gzip': + return new GzipHttpCompressor(); + case 'brotli': + return new BrotliHttpCompressor(); + default: + // The API also accepts `deflate` and `identity`; a custom compressor covers those. + throw new Error(`Unsupported compression algorithm: ${String(compression)}`); + } +} diff --git a/src/index.ts b/src/index.ts index 1377384e1..fdd2838ee 100644 --- a/src/index.ts +++ b/src/index.ts @@ -30,5 +30,6 @@ export * from './apify_api_error.js'; export { ArgumentValidationError } from '@apify/validations'; export * from './response_validation_error.js'; export { InvalidResponseBodyError } from './interceptors.js'; +export * from './http_compressors/index.js'; export type { PaginatedList, Dictionary } from './utils.js'; export type { Timeout, TimeoutOptions, TimeoutTier } from './timeouts.js'; diff --git a/src/interceptors.ts b/src/interceptors.ts index dd747533e..b69ce4710 100644 --- a/src/interceptors.ts +++ b/src/interceptors.ts @@ -5,7 +5,8 @@ import type { JsonObject } from 'type-fest'; import { maybeParseBody } from './body_parser.js'; import type { ApifyRequestConfig, ApifyResponse } from './http_client.js'; -import { isCompressibleContentType, isNode, maybeCompressValue } from './utils.js'; +import type { HttpCompressor } from './http_compressors/base.js'; +import { isCompressibleContentType, isNode, MIN_COMPRESS_BYTES } from './utils.js'; /** * This error exists for the quite common situation, where only a partial JSON response is received and @@ -90,18 +91,32 @@ function stringifyWithFunctions(obj: JsonObject) { }); } -async function maybeCompressRequest(config: ApifyRequestConfig): Promise { +/** + * Compresses the request body with the client's compressor and labels it with the compressor's `Content-Encoding`. + * + * Runs after `serializeRequest`, so a JSON body is already a string here. The body is sent as it is when it is not + * a string or a `Buffer`, when it is smaller than `MIN_COMPRESS_BYTES`, when its content type says the payload is + * already compressed, or when the caller set a `Content-Encoding` of their own. That header is forwarded verbatim, + * which is how a pre-encoded body is uploaded, and `Content-Encoding: identity` opts a single request out of + * compression. Browsers have no `node:zlib`, so there the body is always sent as it is. + */ +async function maybeCompressRequest( + config: ApifyRequestConfig, + compressor: HttpCompressor, +): Promise { + if (!isNode()) return config; + + const { data } = config; + if (typeof data !== 'string' && !Buffer.isBuffer(data)) return config; + if (Buffer.byteLength(data) < MIN_COMPRESS_BYTES) return config; + // A caller-supplied encoding means the body is already encoded and the header describes it, so leave both alone. if (getHeader(config, 'content-encoding')) return config; - if (!isCompressibleContentType(getHeader(config, 'content-type'))) return config; - const maybeCompressed = await maybeCompressValue(config.data); - if (maybeCompressed) { - config.headers ??= {}; - config.headers['content-encoding'] = maybeCompressed.encoding; - config.data = maybeCompressed.data; - } + config.data = await compressor.compress(typeof data === 'string' ? Buffer.from(data) : data); + config.headers ??= {}; + config.headers['content-encoding'] = compressor.contentEncoding; return config; } @@ -135,9 +150,14 @@ function parseResponseData(response: ApifyResponse): ApifyResponse { export type RequestInterceptorFunction = Parameters['use']>[0]; export type ResponseInterceptorFunction = Parameters['use']>[0]; -export const requestInterceptors: RequestInterceptorFunction[] = [ - maybeCompressRequest, - serializeRequest, - ensureHeadersPrototype, -]; +/** + * The client's own request interceptors, in registration order. Axios runs request interceptors in the reverse + * order of registration, so the body is serialized before it is compressed, and interceptors registered later, + * such as the user-provided ones, run before both. + * @internal + */ +export function createRequestInterceptors(compressor: HttpCompressor): RequestInterceptorFunction[] { + return [async (config) => maybeCompressRequest(config, compressor), serializeRequest, ensureHeadersPrototype]; +} + export const responseInterceptors: ResponseInterceptorFunction[] = [parseResponseData]; diff --git a/src/utils.ts b/src/utils.ts index 5781a064f..1c4249345 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -17,7 +17,13 @@ import type { WebhookUpdateData } from './resource_clients/webhook.js'; // @ts-ignore if we enable `resolveJsonModule`, we end up with a `src` folder in `dist` import packageJson from '../package.json' with { type: 'json' }; -const MIN_COMPRESS_BYTES = 1024; +/** + * Smallest request body, in bytes, that is worth compressing. A smaller body already fits in a single network + * packet, so compressing it costs CPU time without saving a round trip, and the compression format itself can + * make it larger. + * @internal + */ +export const MIN_COMPRESS_BYTES = 1024; /** Media type prefixes whose payloads carry their own compression, so compressing the request body is wasted work. */ const ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES = ['audio/', 'image/', 'video/']; @@ -167,43 +173,6 @@ export function stringifyWebhooksToBase64(webhooks?: readonly WebhookUpdateData[ return btoa(String.fromCharCode(...uint8Array)); } -let brotliCompressPromisified: ((arg: string | Buffer) => Promise) | undefined; - -/** - * Brotli-compress the provided value. - */ -async function brotliValue(value: string | Buffer): Promise { - if (!brotliCompressPromisified) { - const { promisify } = await import('node:util'); - const { brotliCompress, constants } = await import('node:zlib'); - const compress = promisify(brotliCompress); - const options = { params: { [constants.BROTLI_PARAM_QUALITY]: 6 } }; - brotliCompressPromisified = async (input) => compress(input, options); - } - - return brotliCompressPromisified(value); -} - -let gzipPromisified: ((arg: string | Buffer) => Promise) | undefined; - -/** - * Gzip-compress the provided value. - */ -async function gzipValue(value: string | Buffer): Promise { - if (!gzipPromisified) { - const { promisify } = await import('node:util'); - const { gzip } = await import('node:zlib'); - gzipPromisified = promisify(gzip); - } - - return gzipPromisified(value); -} - -export interface CompressedValue { - data: Buffer; - encoding: 'br' | 'gzip'; -} - /** * Decides whether a request body with the given content type is worth compressing. * @@ -226,36 +195,6 @@ export function isCompressibleContentType(contentType?: string): boolean { return !ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES.some((prefix) => mediaType.startsWith(prefix)); } -/** - * Compress the passed value using brotli, falling back to gzip. Returns undefined if the data is - * too small / wrong type, or if neither algorithm is available. - */ -export async function maybeCompressValue(value: unknown): Promise { - if (!isNode()) return undefined; - - // Request compression is not that important so let's - // skip it instead of throwing for unsupported types. - if (typeof value !== 'string' && !Buffer.isBuffer(value)) return undefined; - - const areDataLargeEnough = Buffer.byteLength(value) >= MIN_COMPRESS_BYTES; - if (!areDataLargeEnough) return undefined; - - try { - return { data: await brotliValue(value), encoding: 'br' }; - } catch { - // Runtimes that only provide a partial `node:zlib` (bundler polyfills, edge runtimes with - // Node compatibility shims) may not implement brotli, but usually do implement gzip. - } - - try { - return { data: await gzipValue(value), encoding: 'gzip' }; - } catch { - // Same reasoning as above: compression is a best-effort optimization, so skip it instead - // of failing the request. - return undefined; - } -} - /** * Helper function slice the items from array to fit the max byte length. */ diff --git a/test/http_compressors.test.ts b/test/http_compressors.test.ts new file mode 100644 index 000000000..de6d49230 --- /dev/null +++ b/test/http_compressors.test.ts @@ -0,0 +1,182 @@ +import type { AddressInfo } from 'node:net'; +import { brotliDecompressSync, gunzipSync } from 'node:zlib'; + +import type { HttpCompressionAlgorithm } from 'apify-client'; +import { + ApifyClient, + ArgumentValidationError, + BrotliHttpCompressor, + GzipHttpCompressor, + HttpCompressor, +} from 'apify-client'; +import { afterAll, beforeAll, describe, expect, test, vi } from 'vitest'; + +import { resolveCompressor } from '../src/http_compressors/resolve.js'; +import { MIN_COMPRESS_BYTES } from '../src/utils.js'; +import { mockServer } from './mock_server/server.js'; + +const payload = Buffer.from('hello world'); + +describe('GzipHttpCompressor', () => { + test('reports gzip as its content encoding', () => { + expect(new GzipHttpCompressor().contentEncoding).toBe('gzip'); + }); + + test.each([ + { name: 'the fastest', quality: 1 }, + { name: 'the best', quality: 9 }, + ])('round-trips data at $name quality', async ({ quality }) => { + const compressed = await new GzipHttpCompressor({ quality }).compress(payload); + expect(gunzipSync(compressed)).toEqual(payload); + }); + + test.each([ + { name: 'below the minimum', quality: 0 }, + { name: 'above the maximum', quality: 10 }, + { name: 'negative', quality: -1 }, + { name: 'fractional', quality: 5.5 }, + ])('rejects a quality $name', ({ quality }) => { + expect(() => new GzipHttpCompressor({ quality })).toThrow(ArgumentValidationError); + }); +}); + +describe('BrotliHttpCompressor', () => { + test('reports br as its content encoding', () => { + expect(new BrotliHttpCompressor().contentEncoding).toBe('br'); + }); + + test.each([ + { name: 'the fastest', quality: 0 }, + { name: 'the best', quality: 11 }, + ])('round-trips data at $name quality', async ({ quality }) => { + const compressed = await new BrotliHttpCompressor({ quality }).compress(payload); + expect(brotliDecompressSync(compressed)).toEqual(payload); + }); + + test.each([ + { name: 'negative', quality: -1 }, + { name: 'above the maximum', quality: 12 }, + { name: 'fractional', quality: 5.5 }, + ])('rejects a quality $name', ({ quality }) => { + expect(() => new BrotliHttpCompressor({ quality })).toThrow(ArgumentValidationError); + }); +}); + +describe('resolveCompressor()', () => { + test("resolves 'gzip' to a GzipHttpCompressor", () => { + expect(resolveCompressor('gzip')).toBeInstanceOf(GzipHttpCompressor); + }); + + test("resolves 'brotli' to a BrotliHttpCompressor", () => { + expect(resolveCompressor('brotli')).toBeInstanceOf(BrotliHttpCompressor); + }); + + test('passes an HttpCompressor instance through unchanged', () => { + const compressor = new BrotliHttpCompressor({ quality: 11 }); + expect(resolveCompressor(compressor)).toBe(compressor); + }); + + test('rejects an unknown algorithm', () => { + expect(() => resolveCompressor('deflate' as HttpCompressionAlgorithm)).toThrow( + 'Unsupported compression algorithm', + ); + }); +}); + +describe('ApifyClient compression option', () => { + let baseUrl: string; + + beforeAll(async () => { + const server = await mockServer.start(); + baseUrl = `http://localhost:${(server.address() as AddressInfo).port}`; + }); + + afterAll(async () => { + await mockServer.close(); + }); + + class IdentityCompressor extends HttpCompressor { + readonly contentEncoding = 'identity'; + + async compress(data: Buffer): Promise { + return data; + } + } + + // Serializes to well above the compression threshold, so the body goes through the compressor. + const largeBody = Array.from({ length: 200 }, (_, index) => ({ index, url: `https://example.com/item/${index}` })); + + test('defaults to brotli', () => { + expect(new ApifyClient().httpClient.httpCompressor).toBeInstanceOf(BrotliHttpCompressor); + }); + + test.each([ + { compression: 'gzip', contentEncoding: 'gzip' }, + { compression: 'brotli', contentEncoding: 'br' }, + ] as const)('sends a large body compressed with $compression', async ({ compression, contentEncoding }) => { + const client = new ApifyClient({ baseUrl, compression, maxRetries: 0 }); + + await client.dataset('some-id').pushItems(largeBody); + + // The mock server decodes the body by its `Content-Encoding`, so a body in any other encoding would not + // parse back into the items. + const request = mockServer.getLastRequest(); + expect(request?.headers['content-encoding']).toBe(contentEncoding); + expect(request?.body).toEqual(largeBody); + }); + + test('sends a large body through a custom compressor and labels it with its content encoding', async () => { + const compressor = new IdentityCompressor(); + const compress = vi.spyOn(compressor, 'compress'); + const client = new ApifyClient({ baseUrl, compression: compressor, maxRetries: 0 }); + expect(client.httpClient.httpCompressor).toBe(compressor); + + await client.dataset('some-id').pushItems(largeBody); + + expect(compress).toHaveBeenCalledOnce(); + const [data] = compress.mock.calls[0]; + expect(Buffer.isBuffer(data)).toBe(true); + expect(JSON.parse(data.toString())).toEqual(largeBody); + const request = mockServer.getLastRequest(); + expect(request?.headers['content-encoding']).toBe('identity'); + expect(request?.body).toEqual(largeBody); + }); + + test('compresses a body of exactly the threshold size', async () => { + const compressor = new IdentityCompressor(); + const compress = vi.spyOn(compressor, 'compress'); + const client = new ApifyClient({ baseUrl, compression: compressor, maxRetries: 0 }); + + await client + .keyValueStore('some-id') + .setRecord({ key: 'some-key', value: Buffer.alloc(MIN_COMPRESS_BYTES, 'a') }); + + expect(compress).toHaveBeenCalledOnce(); + expect(mockServer.getLastRequest()?.headers['content-encoding']).toBe('identity'); + }); + + test.each([ + { + name: 'a body below the threshold', + value: Buffer.alloc(MIN_COMPRESS_BYTES - 1, 'a'), + contentType: 'application/octet-stream', + }, + { name: 'an already-compressed content type', value: Buffer.alloc(4096, 'a'), contentType: 'image/png' }, + ])('leaves the compressor out for $name', async ({ value, contentType }) => { + const compressor = new IdentityCompressor(); + const compress = vi.spyOn(compressor, 'compress'); + const client = new ApifyClient({ baseUrl, compression: compressor, maxRetries: 0 }); + + await client.keyValueStore('some-id').setRecord({ key: 'some-key', value, contentType }); + + expect(compress).not.toHaveBeenCalled(); + expect(mockServer.getLastRequest()?.headers['content-encoding']).toBeUndefined(); + }); + + test.each([ + { name: 'an unknown algorithm', compression: 'deflate' }, + { name: 'an object that is not a compressor', compression: { contentEncoding: 'gzip' } }, + ])('rejects $name', ({ compression }) => { + expect(() => new ApifyClient({ compression: compression as any })).toThrow(ArgumentValidationError); + }); +}); diff --git a/test/utils.test.ts b/test/utils.test.ts index 110b4dfdf..b2784f920 100644 --- a/test/utils.test.ts +++ b/test/utils.test.ts @@ -98,34 +98,6 @@ describe('utils.parseResponse()', () => { }); }); -describe('utils.maybeCompressValue()', () => { - test('returns undefined for small values', async () => { - expect(await utils.maybeCompressValue('small')).toBeUndefined(); - }); - - test('returns undefined for non-string non-Buffer values', async () => { - expect(await utils.maybeCompressValue({ foo: 'bar' })).toBeUndefined(); - }); - - test('compresses large string using brotli in Node.js', async () => { - const largeValue = 'x'.repeat(2048); - const result = await utils.maybeCompressValue(largeValue); - expect(result).not.toBeUndefined(); - expect(result!.encoding).toBe('br'); - expect(result!.data).toBeInstanceOf(Buffer); - expect(result!.data.length).toBeLessThan(Buffer.byteLength(largeValue)); - }); - - test('compresses large Buffer using brotli in Node.js', async () => { - const largeValue = Buffer.alloc(2048, 'a'); - const result = await utils.maybeCompressValue(largeValue); - expect(result).not.toBeUndefined(); - expect(result!.encoding).toBe('br'); - expect(result!.data).toBeInstanceOf(Buffer); - expect(result!.data.length).toBeLessThan(largeValue.length); - }); -}); - describe('utils.isCompressibleContentType()', () => { test.each([ { name: 'a missing content type', contentType: undefined, compressible: true }, From 469ecca97cbb660706453489b8f499a8c4a269c5 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 14 Sep 2026 13:08:42 +0200 Subject: [PATCH 2/8] fix: drop a stale Content-Length when compressing a request body --- src/interceptors.ts | 23 ++++++++++++++++++----- test/http_compressors.test.ts | 20 ++++++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/interceptors.ts b/src/interceptors.ts index b69ce4710..2bf62dfbe 100644 --- a/src/interceptors.ts +++ b/src/interceptors.ts @@ -43,6 +43,17 @@ function getHeader(config: ApifyRequestConfig, name: string): string | undefined return typeof value === 'string' ? value : undefined; } +/** Removes a request header regardless of the casing it was set with. */ +function deleteHeader(config: ApifyRequestConfig, name: string): void { + const { headers } = config; + if (!headers) return; + + const wanted = name.toLowerCase(); + for (const key of Object.keys(headers)) { + if (key.toLowerCase() === wanted) delete headers[key]; + } +} + function serializeRequest(config: ApifyRequestConfig): ApifyRequestConfig { const [defaultTransform] = axios.defaults.transformRequest as AxiosRequestTransformer[]; @@ -94,11 +105,12 @@ function stringifyWithFunctions(obj: JsonObject) { /** * Compresses the request body with the client's compressor and labels it with the compressor's `Content-Encoding`. * - * Runs after `serializeRequest`, so a JSON body is already a string here. The body is sent as it is when it is not - * a string or a `Buffer`, when it is smaller than `MIN_COMPRESS_BYTES`, when its content type says the payload is - * already compressed, or when the caller set a `Content-Encoding` of their own. That header is forwarded verbatim, - * which is how a pre-encoded body is uploaded, and `Content-Encoding: identity` opts a single request out of - * compression. Browsers have no `node:zlib`, so there the body is always sent as it is. + * Runs after `serializeRequest`, so a JSON body is already a string here. A caller-set `Content-Encoding` is + * forwarded verbatim, which is how a pre-encoded body is uploaded, and `Content-Encoding: identity` opts a single + * request out of compression. Browsers have no `node:zlib`, so there the body is always sent as it is. + * + * Compressing changes the body length, so any `Content-Length` the caller set describes the wrong body and has to + * go. Axios keeps a caller-set one over the size it computes, which would stall the request until it times out. */ async function maybeCompressRequest( config: ApifyRequestConfig, @@ -117,6 +129,7 @@ async function maybeCompressRequest( config.data = await compressor.compress(typeof data === 'string' ? Buffer.from(data) : data); config.headers ??= {}; config.headers['content-encoding'] = compressor.contentEncoding; + deleteHeader(config, 'content-length'); return config; } diff --git a/test/http_compressors.test.ts b/test/http_compressors.test.ts index de6d49230..ef4f77734 100644 --- a/test/http_compressors.test.ts +++ b/test/http_compressors.test.ts @@ -125,6 +125,26 @@ describe('ApifyClient compression option', () => { expect(request?.body).toEqual(largeBody); }); + test('replaces a caller-set content-length with the size of the compressed body', async () => { + const client = new ApifyClient({ baseUrl, maxRetries: 0 }); + const body = JSON.stringify(largeBody); + + await client.httpClient.call({ + url: `${baseUrl}/v2/key-value-stores/some-id/records/some-key`, + method: 'PUT', + data: body, + headers: { 'content-type': 'application/json', 'content-length': String(Buffer.byteLength(body)) }, + // A stale content-length leaves the server waiting for bytes that never arrive, so a short timeout + // turns that into a fast failure instead of a hang. + timeoutSecs: 5, + }); + + const request = mockServer.getLastRequest(); + expect(request?.headers['content-encoding']).toBe('br'); + expect(Number(request?.headers['content-length'])).toBeLessThan(Buffer.byteLength(body)); + expect(request?.body).toEqual(largeBody); + }); + test('sends a large body through a custom compressor and labels it with its content encoding', async () => { const compressor = new IdentityCompressor(); const compress = vi.spyOn(compressor, 'compress'); From 81ab6cc91ea9d58d7a878bc0aeccff45f099eba1 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 14 Sep 2026 13:08:47 +0200 Subject: [PATCH 3/8] refactor: make HttpClient.httpCompressor readonly --- docs/public-api/apify-client.api.md | 2 +- src/http_client.ts | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/public-api/apify-client.api.md b/docs/public-api/apify-client.api.md index e8c4d9099..0ba3cdb11 100644 --- a/docs/public-api/apify-client.api.md +++ b/docs/public-api/apify-client.api.md @@ -2888,7 +2888,7 @@ class HttpClient { call(config: ApifyRequestConfig): Promise>; // (undocumented) httpAgent?: http.Agent; - httpCompressor: HttpCompressor; + readonly httpCompressor: HttpCompressor; // (undocumented) httpsAgent?: https.Agent; // (undocumented) diff --git a/src/http_client.ts b/src/http_client.ts index 48b5d6c3c..757c42f7f 100644 --- a/src/http_client.ts +++ b/src/http_client.ts @@ -31,8 +31,11 @@ export class HttpClient { userProvidedRequestInterceptors: RequestInterceptorFunction[]; - /** Compressor applied to request bodies that are worth compressing. */ - httpCompressor: HttpCompressor; + /** + * Compressor applied to request bodies that are worth compressing. The request interceptor captures it at + * construction, so it is fixed for the lifetime of the client. + */ + readonly httpCompressor: HttpCompressor; logger: Log; From 9d62e5f6491e70847ff5053183c0ff2836f939b0 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 14 Sep 2026 13:08:47 +0200 Subject: [PATCH 4/8] refactor: derive the compression schema enum from HttpCompressionAlgorithm --- src/http_compressors/resolve.ts | 11 ++++++----- test/http_compressors.test.ts | 7 ------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/http_compressors/resolve.ts b/src/http_compressors/resolve.ts index 6dbb7c39a..75f77fee6 100644 --- a/src/http_compressors/resolve.ts +++ b/src/http_compressors/resolve.ts @@ -8,7 +8,8 @@ import { GzipHttpCompressor } from './gzip.js'; /** * Compression algorithms the `compression` option of {@link ApifyClient} accepts by name. Each name selects the * matching built-in compressor with its default quality: `'brotli'` a {@link BrotliHttpCompressor}, `'gzip'` a - * {@link GzipHttpCompressor}. + * {@link GzipHttpCompressor}. The Apify API also accepts `deflate` and `identity`, which a custom + * {@link HttpCompressor} covers. */ export type HttpCompressionAlgorithm = 'brotli' | 'gzip'; @@ -16,7 +17,10 @@ export type HttpCompressionAlgorithm = 'brotli' | 'gzip'; * Schema of the `compression` option: an algorithm name or an {@link HttpCompressor} instance. * @internal */ -export const compressionSchema = z.union([z.enum(['brotli', 'gzip']), httpCompressorSchema]); +export const compressionSchema = z.union([ + z.enum(['brotli', 'gzip'] as const satisfies readonly HttpCompressionAlgorithm[]), + httpCompressorSchema, +]); /** * Turns the `compression` option into a ready-to-use {@link HttpCompressor}: a name into the matching built-in @@ -31,8 +35,5 @@ export function resolveCompressor(compression: HttpCompressionAlgorithm | HttpCo return new GzipHttpCompressor(); case 'brotli': return new BrotliHttpCompressor(); - default: - // The API also accepts `deflate` and `identity`; a custom compressor covers those. - throw new Error(`Unsupported compression algorithm: ${String(compression)}`); } } diff --git a/test/http_compressors.test.ts b/test/http_compressors.test.ts index ef4f77734..4db2cbf55 100644 --- a/test/http_compressors.test.ts +++ b/test/http_compressors.test.ts @@ -1,7 +1,6 @@ import type { AddressInfo } from 'node:net'; import { brotliDecompressSync, gunzipSync } from 'node:zlib'; -import type { HttpCompressionAlgorithm } from 'apify-client'; import { ApifyClient, ArgumentValidationError, @@ -75,12 +74,6 @@ describe('resolveCompressor()', () => { const compressor = new BrotliHttpCompressor({ quality: 11 }); expect(resolveCompressor(compressor)).toBe(compressor); }); - - test('rejects an unknown algorithm', () => { - expect(() => resolveCompressor('deflate' as HttpCompressionAlgorithm)).toThrow( - 'Unsupported compression algorithm', - ); - }); }); describe('ApifyClient compression option', () => { From 50957ccae2f113dd0e2060ef6f29c2675d16139f Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 14 Sep 2026 13:08:47 +0200 Subject: [PATCH 5/8] fix: default GzipHttpCompressor to quality 6 --- docs/02_concepts/07_http-compression.md | 2 +- src/http_compressors/gzip.ts | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/02_concepts/07_http-compression.md b/docs/02_concepts/07_http-compression.md index 7a37a0eec..f09364f35 100644 --- a/docs/02_concepts/07_http-compression.md +++ b/docs/02_concepts/07_http-compression.md @@ -131,6 +131,6 @@ const client = new ApifyClient({ token: 'MY-APIFY-TOKEN', compression: new Ident | **CPU cost** | Moderate, depends on quality | Low | | **`Content-Encoding` header** | `br` | `gzip` | | **Quality range** | `0` to `11` | `1` to `9` | -| **Default quality** | `6` | `9` | +| **Default quality** | `6` | `6` | | **Enable via config** | `compression: 'brotli'` (default) | `compression: 'gzip'` | | **Best for** | Large payloads where bandwidth matters | Low CPU cost, runtimes without brotli | diff --git a/src/http_compressors/gzip.ts b/src/http_compressors/gzip.ts index 1a255f878..727d405d7 100644 --- a/src/http_compressors/gzip.ts +++ b/src/http_compressors/gzip.ts @@ -9,8 +9,11 @@ const MIN_QUALITY = 1; /** Highest valid gzip quality, the slowest with the best compression. */ const MAX_QUALITY = 9; +/** zlib's own default. The levels above it roughly double the CPU cost for about a percent fewer bytes. */ +const DEFAULT_QUALITY = 6; + const optionsSchema = z.strictObject({ - quality: z.int().min(MIN_QUALITY).max(MAX_QUALITY).default(MAX_QUALITY), + quality: z.int().min(MIN_QUALITY).max(MAX_QUALITY).default(DEFAULT_QUALITY), }); /** @@ -22,7 +25,7 @@ const optionsSchema = z.strictObject({ * ```javascript * import { ApifyClient, GzipHttpCompressor } from 'apify-client'; * - * const client = new ApifyClient({ token: 'my-token', compression: new GzipHttpCompressor({ quality: 6 }) }); + * const client = new ApifyClient({ token: 'my-token', compression: new GzipHttpCompressor({ quality: 1 }) }); * ``` */ export class GzipHttpCompressor extends HttpCompressor { @@ -55,7 +58,7 @@ export class GzipHttpCompressor extends HttpCompressor { export interface GzipHttpCompressorOptions { /** * Compression level, from `1` (the fastest) to `9` (the best compression). - * @default 9 + * @default 6 */ quality?: number; } From a75c0b3a78d4f91eaf7e2472e962ab1552edfdc5 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 14 Sep 2026 13:08:51 +0200 Subject: [PATCH 6/8] test: cover compressor quality passthrough and compression errors --- test/http_compressors.test.ts | 43 ++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/test/http_compressors.test.ts b/test/http_compressors.test.ts index 4db2cbf55..02e46849d 100644 --- a/test/http_compressors.test.ts +++ b/test/http_compressors.test.ts @@ -1,5 +1,5 @@ import type { AddressInfo } from 'node:net'; -import { brotliDecompressSync, gunzipSync } from 'node:zlib'; +import { brotliCompressSync, brotliDecompressSync, constants, gunzipSync, gzipSync } from 'node:zlib'; import { ApifyClient, @@ -16,6 +16,9 @@ import { mockServer } from './mock_server/server.js'; const payload = Buffer.from('hello world'); +/** Long and repetitive, so the compression quality changes the output bytes. */ +const repetitivePayload = Buffer.from('lorem ipsum dolor sit amet '.repeat(500)); + describe('GzipHttpCompressor', () => { test('reports gzip as its content encoding', () => { expect(new GzipHttpCompressor().contentEncoding).toBe('gzip'); @@ -29,6 +32,19 @@ describe('GzipHttpCompressor', () => { expect(gunzipSync(compressed)).toEqual(payload); }); + test('passes the quality through to zlib', async () => { + const fastest = await new GzipHttpCompressor({ quality: 1 }).compress(repetitivePayload); + const best = await new GzipHttpCompressor({ quality: 9 }).compress(repetitivePayload); + + expect(fastest).toEqual(gzipSync(repetitivePayload, { level: 1 })); + expect(best).toEqual(gzipSync(repetitivePayload, { level: 9 })); + }); + + test('defaults to level 6', async () => { + const compressed = await new GzipHttpCompressor().compress(repetitivePayload); + expect(compressed).toEqual(gzipSync(repetitivePayload, { level: 6 })); + }); + test.each([ { name: 'below the minimum', quality: 0 }, { name: 'above the maximum', quality: 10 }, @@ -52,6 +68,21 @@ describe('BrotliHttpCompressor', () => { expect(brotliDecompressSync(compressed)).toEqual(payload); }); + test('passes the quality through to zlib', async () => { + const brotli = (quality: number) => + brotliCompressSync(repetitivePayload, { params: { [constants.BROTLI_PARAM_QUALITY]: quality } }); + + expect(await new BrotliHttpCompressor({ quality: 0 }).compress(repetitivePayload)).toEqual(brotli(0)); + expect(await new BrotliHttpCompressor({ quality: 11 }).compress(repetitivePayload)).toEqual(brotli(11)); + }); + + test('defaults to quality 6', async () => { + const compressed = await new BrotliHttpCompressor().compress(repetitivePayload); + expect(compressed).toEqual( + brotliCompressSync(repetitivePayload, { params: { [constants.BROTLI_PARAM_QUALITY]: 6 } }), + ); + }); + test.each([ { name: 'negative', quality: -1 }, { name: 'above the maximum', quality: 12 }, @@ -155,6 +186,16 @@ describe('ApifyClient compression option', () => { expect(request?.body).toEqual(largeBody); }); + test('fails the request with the error the compressor throws', async () => { + const compressor = new IdentityCompressor(); + const compress = vi.spyOn(compressor, 'compress').mockRejectedValue(new Error('compression failed')); + const client = new ApifyClient({ baseUrl, compression: compressor, maxRetries: 2 }); + + await expect(client.dataset('some-id').pushItems(largeBody)).rejects.toThrow('compression failed'); + // Retries are on, so a single call is what proves a compressor error is not retried. + expect(compress).toHaveBeenCalledOnce(); + }); + test('compresses a body of exactly the threshold size', async () => { const compressor = new IdentityCompressor(); const compress = vi.spyOn(compressor, 'compress'); From e33050ea7d0846dc2ef60cdca9befb208fe54efe Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 14 Sep 2026 13:08:51 +0200 Subject: [PATCH 7/8] docs: tighten the HTTP compression page wording --- docs/02_concepts/07_http-compression.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/02_concepts/07_http-compression.md b/docs/02_concepts/07_http-compression.md index f09364f35..818ab697f 100644 --- a/docs/02_concepts/07_http-compression.md +++ b/docs/02_concepts/07_http-compression.md @@ -85,9 +85,9 @@ const gzipClient = new ApifyClient({ token: 'MY-APIFY-TOKEN', compression: 'gzip Both algorithms come from `node:zlib`, so neither needs an extra dependency. If the runtime's `node:zlib` doesn't implement brotli, which can happen with a Node.js compatibility shim or a bundler polyfill, the request fails with the error from the runtime. The client doesn't fall back to gzip on its own, so pass `compression: 'gzip'` on such a runtime. -## Custom quality and advanced control +## Compression quality and custom compressors -For fine-grained control over compression quality, pass an `HttpCompressor` instance instead of a name: +To set the compression quality, pass an `HttpCompressor` instance instead of a name: ```js import { ApifyClient, BrotliHttpCompressor, GzipHttpCompressor } from 'apify-client'; @@ -105,7 +105,7 @@ const fastClient = new ApifyClient({ }); ``` -You can also implement a fully custom compressor by extending `HttpCompressor`. The client calls it only for bodies that reach the [minimum body size](#minimum-body-size) and aren't [already compressed](#already-compressed-payloads) or [pre-compressed by the caller](#pre-compressed-bodies): +You can also implement a custom compressor by extending `HttpCompressor`. The client calls it only for bodies that reach the [minimum body size](#minimum-body-size) and aren't [already compressed](#already-compressed-payloads) or [pre-compressed by the caller](#pre-compressed-bodies): ```js import { ApifyClient, HttpCompressor } from 'apify-client'; From 0d317bacb4f68fa5bed43254825e72de8db0378b Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 15 Sep 2026 21:54:19 +0200 Subject: [PATCH 8/8] refactor: turn HttpCompressor into an interface --- docs/02_concepts/07_http-compression.md | 16 +++++++------- docs/04_upgrading/upgrading_v3.md | 2 +- docs/public-api/apify-client.api.md | 10 ++++----- src/apify_client.ts | 6 ++--- src/http_compressors/base.ts | 29 ++++++++++++------------- src/http_compressors/brotli.ts | 5 ++--- src/http_compressors/gzip.ts | 5 ++--- src/http_compressors/index.ts | 2 +- src/http_compressors/resolve.ts | 4 ++-- test/http_compressors.test.ts | 13 ++++------- 10 files changed, 42 insertions(+), 50 deletions(-) diff --git a/docs/02_concepts/07_http-compression.md b/docs/02_concepts/07_http-compression.md index 92f1cc10e..2a0cbea2b 100644 --- a/docs/02_concepts/07_http-compression.md +++ b/docs/02_concepts/07_http-compression.md @@ -87,7 +87,7 @@ Both algorithms come from `node:zlib`, so neither needs an extra dependency. If ## Compression quality and custom compressors -To set the compression quality, pass an `HttpCompressor` instance instead of a name: +To set the compression quality, pass an `HttpCompressor` instead of a name: ```js import { ApifyClient, BrotliHttpCompressor, GzipHttpCompressor } from 'apify-client'; @@ -105,22 +105,22 @@ const fastClient = new ApifyClient({ }); ``` -You can also implement a custom compressor by extending `HttpCompressor`. The client calls it only for bodies that reach the [minimum body size](#minimum-body-size) and aren't [already compressed](#already-compressed-payloads) or [pre-compressed by the caller](#pre-compressed-bodies): +A custom compressor is any object with a `contentEncoding` string and an async `compress()` method, which is what the `HttpCompressor` interface describes. The client calls it only for bodies that reach the [minimum body size](#minimum-body-size) and aren't [already compressed](#already-compressed-payloads) or [pre-compressed by the caller](#pre-compressed-bodies): ```js -import { ApifyClient, HttpCompressor } from 'apify-client'; +import { ApifyClient } from 'apify-client'; -class IdentityCompressor extends HttpCompressor { +const identityCompressor = { /** Value sent in the `Content-Encoding` header. */ - contentEncoding = 'identity'; + contentEncoding: 'identity', /** Returns the body as it is, so nothing gets compressed. */ async compress(data) { return data; - } -} + }, +}; -const client = new ApifyClient({ token: 'MY-APIFY-TOKEN', compression: new IdentityCompressor() }); +const client = new ApifyClient({ token: 'MY-APIFY-TOKEN', compression: identityCompressor }); ``` ## Comparison diff --git a/docs/04_upgrading/upgrading_v3.md b/docs/04_upgrading/upgrading_v3.md index aa0fe38c1..6ce414050 100644 --- a/docs/04_upgrading/upgrading_v3.md +++ b/docs/04_upgrading/upgrading_v3.md @@ -341,7 +341,7 @@ The `timeoutSecs` option of `K ## Request compression is configurable -The `ApifyClient` constructor takes a `compression` option: `'brotli'`, `'gzip'`, or an instance of `HttpCompressor` for a custom quality or algorithm. The default stays brotli at quality 6, so a client constructed without the option sends the same requests as in v2. For the rules that decide which bodies get compressed, see [HTTP compression](../02_concepts/07_http-compression.md). +The `ApifyClient` constructor takes a `compression` option: `'brotli'`, `'gzip'`, or an `HttpCompressor` for a custom quality or algorithm. The default stays brotli at quality 6, so a client constructed without the option sends the same requests as in v2. For the rules that decide which bodies get compressed, see [HTTP compression](../02_concepts/07_http-compression.md). ```js import { ApifyClient, BrotliHttpCompressor } from 'apify-client'; diff --git a/docs/public-api/apify-client.api.md b/docs/public-api/apify-client.api.md index c442873af..551664a97 100644 --- a/docs/public-api/apify-client.api.md +++ b/docs/public-api/apify-client.api.md @@ -668,7 +668,7 @@ interface BaseOptions { } // @public -export class BrotliHttpCompressor extends HttpCompressor { +export class BrotliHttpCompressor implements HttpCompressor { constructor(options?: BrotliHttpCompressorOptions); // (undocumented) compress(data: Buffer): Promise; @@ -2865,7 +2865,7 @@ export interface GetStreamedLogOptions extends TimeoutOptions { } // @public -export class GzipHttpCompressor extends HttpCompressor { +export class GzipHttpCompressor implements HttpCompressor { constructor(options?: GzipHttpCompressorOptions); // (undocumented) compress(data: Buffer): Promise; @@ -2940,9 +2940,9 @@ interface HttpClientOptions { export type HttpCompressionAlgorithm = 'brotli' | 'gzip'; // @public -export abstract class HttpCompressor { - abstract compress(data: Buffer): Promise; - abstract readonly contentEncoding: string; +export interface HttpCompressor { + compress(data: Buffer): Promise; + readonly contentEncoding: string; } // @public diff --git a/src/apify_client.ts b/src/apify_client.ts index e0a719ff2..34a6878e9 100644 --- a/src/apify_client.ts +++ b/src/apify_client.ts @@ -591,9 +591,9 @@ export interface ApifyClientOptions { /** @default https://api.apify.com */ baseUrl?: string; /** - * Compression of request bodies: the name of a built-in algorithm, or an {@link HttpCompressor} instance for - * a custom quality or algorithm. Bodies are compressed only in Node.js, and only when they are large enough - * to benefit and their content type does not already carry its own compression. + * Compression of request bodies: the name of a built-in algorithm, or an {@link HttpCompressor} for a custom + * quality or algorithm. Bodies are compressed only in Node.js, and only when they are large enough to benefit + * and their content type does not already carry its own compression. * @default 'brotli' */ compression?: HttpCompressionAlgorithm | HttpCompressor; diff --git a/src/http_compressors/base.ts b/src/http_compressors/base.ts index 0e5bd0ce3..9b333a9b1 100644 --- a/src/http_compressors/base.ts +++ b/src/http_compressors/base.ts @@ -3,29 +3,29 @@ import { z } from 'zod'; /** * Strategy for compressing HTTP request bodies. * - * Extend this class to create a custom compressor. Set `contentEncoding` to the value that should be sent in the - * `Content-Encoding` header and implement `compress()`. The client calls it only for bodies that are large enough - * to benefit from compression, whose content type does not already carry its own compression, and which the caller - * did not send with a `Content-Encoding` of their own. + * Implement this interface to create a custom compressor. Set `contentEncoding` to the value that should be sent in + * the `Content-Encoding` header and implement `compress()`. The client calls it only for bodies that are large + * enough to benefit from compression, whose content type does not already carry its own compression, and which the + * caller did not send with a `Content-Encoding` of their own. * * @example * ```javascript - * import { ApifyClient, HttpCompressor } from 'apify-client'; + * import { ApifyClient } from 'apify-client'; * - * class IdentityCompressor extends HttpCompressor { - * contentEncoding = 'identity'; + * const identityCompressor = { + * contentEncoding: 'identity', * * async compress(data) { * return data; - * } - * } + * }, + * }; * - * const client = new ApifyClient({ token: 'my-token', compression: new IdentityCompressor() }); + * const client = new ApifyClient({ token: 'my-token', compression: identityCompressor }); * ``` */ -export abstract class HttpCompressor { +export interface HttpCompressor { /** Value sent in the `Content-Encoding` header, for example `gzip` or `br`. */ - abstract readonly contentEncoding: string; + readonly contentEncoding: string; /** * Compresses a request body. @@ -33,12 +33,11 @@ export abstract class HttpCompressor { * @param data - The raw bytes to compress. * @returns The compressed bytes. */ - abstract compress(data: Buffer): Promise; + compress(data: Buffer): Promise; } /** - * Schema accepting an {@link HttpCompressor} by shape, so an instance of the class from another copy of the package - * passes too. + * Schema accepting anything shaped like an {@link HttpCompressor}. * @internal */ export const httpCompressorSchema = z.custom( diff --git a/src/http_compressors/brotli.ts b/src/http_compressors/brotli.ts index e8e74481e..a20b9364e 100644 --- a/src/http_compressors/brotli.ts +++ b/src/http_compressors/brotli.ts @@ -2,7 +2,7 @@ import { z } from 'zod'; import { runtime } from '#runtime'; import { parseArgument } from '../utils.js'; -import { HttpCompressor } from './base.js'; +import type { HttpCompressor } from './base.js'; /** Lowest valid brotli quality, the fastest with the least compression. */ const MIN_QUALITY = 0; @@ -29,7 +29,7 @@ const optionsSchema = z.strictObject({ * const client = new ApifyClient({ token: 'my-token', compression: new BrotliHttpCompressor({ quality: 11 }) }); * ``` */ -export class BrotliHttpCompressor extends HttpCompressor { +export class BrotliHttpCompressor implements HttpCompressor { readonly contentEncoding = 'br'; readonly #quality: number; @@ -39,7 +39,6 @@ export class BrotliHttpCompressor extends HttpCompressor { * @throws {ArgumentValidationError} If `quality` is out of the valid range. */ constructor(options: BrotliHttpCompressorOptions = {}) { - super(); const { quality } = parseArgument(options, optionsSchema, 'BrotliHttpCompressorOptions'); this.#quality = quality; } diff --git a/src/http_compressors/gzip.ts b/src/http_compressors/gzip.ts index 22dba13f3..f99587763 100644 --- a/src/http_compressors/gzip.ts +++ b/src/http_compressors/gzip.ts @@ -2,7 +2,7 @@ import { z } from 'zod'; import { runtime } from '#runtime'; import { parseArgument } from '../utils.js'; -import { HttpCompressor } from './base.js'; +import type { HttpCompressor } from './base.js'; /** Lowest valid gzip quality, the fastest with the least compression. */ const MIN_QUALITY = 1; @@ -29,7 +29,7 @@ const optionsSchema = z.strictObject({ * const client = new ApifyClient({ token: 'my-token', compression: new GzipHttpCompressor({ quality: 1 }) }); * ``` */ -export class GzipHttpCompressor extends HttpCompressor { +export class GzipHttpCompressor implements HttpCompressor { readonly contentEncoding = 'gzip'; readonly #quality: number; @@ -39,7 +39,6 @@ export class GzipHttpCompressor extends HttpCompressor { * @throws {ArgumentValidationError} If `quality` is out of the valid range. */ constructor(options: GzipHttpCompressorOptions = {}) { - super(); const { quality } = parseArgument(options, optionsSchema, 'GzipHttpCompressorOptions'); this.#quality = quality; } diff --git a/src/http_compressors/index.ts b/src/http_compressors/index.ts index e19dc390e..71efb22ac 100644 --- a/src/http_compressors/index.ts +++ b/src/http_compressors/index.ts @@ -1,4 +1,4 @@ -export { HttpCompressor } from './base.js'; +export type { HttpCompressor } from './base.js'; export { BrotliHttpCompressor } from './brotli.js'; export type { BrotliHttpCompressorOptions } from './brotli.js'; export { GzipHttpCompressor } from './gzip.js'; diff --git a/src/http_compressors/resolve.ts b/src/http_compressors/resolve.ts index 75f77fee6..d365c0975 100644 --- a/src/http_compressors/resolve.ts +++ b/src/http_compressors/resolve.ts @@ -14,7 +14,7 @@ import { GzipHttpCompressor } from './gzip.js'; export type HttpCompressionAlgorithm = 'brotli' | 'gzip'; /** - * Schema of the `compression` option: an algorithm name or an {@link HttpCompressor} instance. + * Schema of the `compression` option: an algorithm name or an {@link HttpCompressor}. * @internal */ export const compressionSchema = z.union([ @@ -24,7 +24,7 @@ export const compressionSchema = z.union([ /** * Turns the `compression` option into a ready-to-use {@link HttpCompressor}: a name into the matching built-in - * compressor with its default quality, an instance into itself. + * compressor with its default quality, a compressor into itself. * @internal */ export function resolveCompressor(compression: HttpCompressionAlgorithm | HttpCompressor): HttpCompressor { diff --git a/test/http_compressors.test.ts b/test/http_compressors.test.ts index f28dcf03e..b941c336a 100644 --- a/test/http_compressors.test.ts +++ b/test/http_compressors.test.ts @@ -1,13 +1,8 @@ import type { AddressInfo } from 'node:net'; import { brotliCompressSync, brotliDecompressSync, constants, gunzipSync, gzipSync } from 'node:zlib'; -import { - ApifyClient, - ArgumentValidationError, - BrotliHttpCompressor, - GzipHttpCompressor, - HttpCompressor, -} from 'apify-client'; +import type { HttpCompressor } from 'apify-client'; +import { ApifyClient, ArgumentValidationError, BrotliHttpCompressor, GzipHttpCompressor } from 'apify-client'; import { afterAll, beforeAll, describe, expect, test, vi } from 'vitest'; import { resolveCompressor } from '../src/http_compressors/resolve.js'; @@ -101,7 +96,7 @@ describe('resolveCompressor()', () => { expect(resolveCompressor('brotli')).toBeInstanceOf(BrotliHttpCompressor); }); - test('passes an HttpCompressor instance through unchanged', () => { + test('passes a compressor through unchanged', () => { const compressor = new BrotliHttpCompressor({ quality: 11 }); expect(resolveCompressor(compressor)).toBe(compressor); }); @@ -119,7 +114,7 @@ describe('ApifyClient compression option', () => { await mockServer.close(); }); - class IdentityCompressor extends HttpCompressor { + class IdentityCompressor implements HttpCompressor { readonly contentEncoding = 'identity'; async compress(data: Buffer): Promise {