From 3ec6bc4474c46bc58cdee513c0ae6acd531861d0 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 9 Sep 2026 16:28:42 +0200 Subject: [PATCH 1/4] perf: skip request-body compression for already-compressed content types --- src/interceptors.ts | 6 ++- src/resource_clients/key_value_store.ts | 5 ++ src/utils.ts | 71 +++++++++++++++++++++++++ test/key_value_stores.test.ts | 36 +++++++++++++ test/utils.test.ts | 34 ++++++++++++ 5 files changed, 151 insertions(+), 1 deletion(-) diff --git a/src/interceptors.ts b/src/interceptors.ts index 0696195c8..de28df2c7 100644 --- a/src/interceptors.ts +++ b/src/interceptors.ts @@ -5,7 +5,7 @@ import type { JsonObject } from 'type-fest'; import { maybeParseBody } from './body_parser.js'; import type { ApifyRequestConfig, ApifyResponse } from './http_client.js'; -import { isNode, maybeCompressValue } from './utils.js'; +import { isCompressibleContentType, isNode, maybeCompressValue } from './utils.js'; /** * This error exists for the quite common situation, where only a partial JSON response is received and @@ -79,8 +79,12 @@ function stringifyWithFunctions(obj: JsonObject) { } async function maybeCompressRequest(config: ApifyRequestConfig): Promise { + // A caller-supplied encoding means the body is already encoded and the header describes it, so leave both alone. if (config.headers?.['content-encoding']) return config; + const contentTypeHeader = config.headers?.['Content-Type'] || config.headers?.['content-type']; + if (!isCompressibleContentType(contentTypeHeader)) return config; + const maybeCompressed = await maybeCompressValue(config.data); if (maybeCompressed) { config.headers ??= {}; diff --git a/src/resource_clients/key_value_store.ts b/src/resource_clients/key_value_store.ts index 7d8a46354..a9da0ac0f 100644 --- a/src/resource_clients/key_value_store.ts +++ b/src/resource_clients/key_value_store.ts @@ -429,6 +429,11 @@ export class KeyValueStoreClient extends ResourceClient { * - Objects: `'application/json; charset=utf-8'` * - Strings: `'text/plain; charset=utf-8'` * - Buffers/Streams: `'application/octet-stream'` + * + * Worth setting for media and archives: the client skips compressing a body + * whose content type already carries its own compression. The + * `'application/octet-stream'` fallback is still compressed, because it + * could be any binary data. * @param options - Storage options * @param options.timeoutSecs - Timeout for the upload in seconds. Default varies by value size. * @param options.doNotRetryTimeouts - If `true`, don't retry on timeout errors. Default is `false`. diff --git a/src/utils.ts b/src/utils.ts index 3fc0c7d3c..fc30701ea 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -19,6 +19,55 @@ import packageJson from '../package.json' with { type: 'json' }; const MIN_COMPRESS_BYTES = 1024; +/** Media type prefixes whose payloads carry their own compression, so compressing the request body is wasted work. */ +const ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES = ['audio/', 'image/', 'video/']; + +/** Exact media types whose payloads carry their own compression. */ +const ALREADY_COMPRESSED_MEDIA_TYPES = new Set([ + 'application/epub+zip', + 'application/gzip', + 'application/java-archive', + 'application/vnd.android.package-archive', + 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'application/vnd.rar', + 'application/x-7z-compressed', + 'application/x-bzip', + 'application/x-bzip2', + 'application/x-gzip', + 'application/x-rar-compressed', + 'application/x-xz', + 'application/x-zip-compressed', + 'application/zip', + 'application/zstd', + 'font/woff', + 'font/woff2', +]); + +/** Uncompressed media types that sit under an already-compressed prefix, so compressing them still pays off. */ +const COMPRESSIBLE_MEDIA_TYPES = new Set([ + 'audio/aiff', + 'audio/basic', + 'audio/l16', + 'audio/l24', + 'audio/midi', + 'audio/vnd.wave', + 'audio/wav', + 'audio/wave', + 'audio/x-aiff', + 'audio/x-wav', + 'image/bmp', + 'image/tiff', + 'image/vnd.adobe.photoshop', + 'image/vnd.microsoft.icon', + 'image/x-icon', + 'image/x-ms-bmp', +]); + +/** Structured syntax suffixes marking a media type as text even under an already-compressed prefix (`image/svg+xml`). */ +const COMPRESSIBLE_MEDIA_TYPE_SUFFIXES = ['+json', '+xml']; + export { parseArgument }; /** @@ -200,6 +249,28 @@ export interface CompressedValue { encoding: 'br' | 'gzip'; } +/** + * Decides whether a request body with the given content type is worth compressing. + * + * Images, audio, video and archives already carry their own compression. Running them through brotli or gzip + * burns CPU, holds a second full copy of the body in memory, and usually produces output slightly larger than + * the input. Formats that are raw despite such a media type, for example `image/bmp` or `audio/wav`, are still + * compressed. A body with no content type is assumed to be compressible. + * @internal + */ +export function isCompressibleContentType(contentType?: string): boolean { + if (!contentType) return true; + + // `Content-Type` is case-insensitive and may carry parameters, for example `text/plain; charset=utf-8`. + const mediaType = contentType.split(';', 1)[0].trim().toLowerCase(); + + if (COMPRESSIBLE_MEDIA_TYPES.has(mediaType)) return true; + if (COMPRESSIBLE_MEDIA_TYPE_SUFFIXES.some((suffix) => mediaType.endsWith(suffix))) return true; + + if (ALREADY_COMPRESSED_MEDIA_TYPES.has(mediaType)) return false; + return !ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES.some((prefix) => mediaType.startsWith(prefix)); +} + /** * Compress the passed value using brotli, falling back to gzip. Returns undefined if the data is * too small / wrong type, or if neither algorithm is available. diff --git a/test/key_value_stores.test.ts b/test/key_value_stores.test.ts index 048955a06..6e1cb9ec0 100644 --- a/test/key_value_stores.test.ts +++ b/test/key_value_stores.test.ts @@ -607,6 +607,42 @@ describe('Key-Value Store methods', () => { }); }); + test.each([ + { name: 'a PNG', contentType: 'image/png' }, + { name: 'a ZIP archive', contentType: 'application/zip' }, + { name: 'a web font', contentType: 'font/woff2' }, + ])('setRecord() sends $name uncompressed', async ({ contentType }) => { + const key = 'some-key'; + const storeId = 'some-id'; + // Well above the 1 KiB compression threshold and trivially compressible, so an unchanged + // content length is proof the body was not run through the compressor. + const value = Buffer.alloc(4096, 'a'); + + const res = await client.keyValueStore(storeId).setRecord({ key, value: value as any, contentType }); + expect(res).toBeUndefined(); + + const request = mockServer.getLastRequest(); + expect(request?.headers['content-type']).toBe(contentType); + expect(request?.headers['content-encoding']).toBeUndefined(); + expect(request?.headers['content-length']).toBe(String(value.length)); + }); + + test.each([ + { name: 'SVG', contentType: 'image/svg+xml' }, + { name: 'a raw bitmap', contentType: 'image/bmp' }, + ])('setRecord() compresses $name despite its already-compressed prefix', async ({ contentType }) => { + const key = 'some-key'; + const storeId = 'some-id'; + const value = Buffer.alloc(4096, 'a'); + + const res = await client.keyValueStore(storeId).setRecord({ key, value: value as any, contentType }); + expect(res).toBeUndefined(); + + const request = mockServer.getLastRequest(); + expect(request?.headers['content-type']).toBe(contentType); + expect(request?.headers['content-encoding']).toBe('br'); + }); + test('deleteRecord() works', async () => { const key = 'some-key'; const storeId = '204'; diff --git a/test/utils.test.ts b/test/utils.test.ts index c11cbbf0a..88f404cc7 100644 --- a/test/utils.test.ts +++ b/test/utils.test.ts @@ -158,6 +158,40 @@ describe('utils.maybeCompressValue()', () => { }); }); +describe('utils.isCompressibleContentType()', () => { + test.each([ + { name: 'a missing content type', contentType: undefined, compressible: true }, + { name: 'an empty content type', contentType: '', compressible: true }, + { name: 'JSON', contentType: 'application/json', compressible: true }, + { name: 'text with parameters', contentType: 'text/plain; charset=utf-8', compressible: true }, + { name: 'unknown binary', contentType: 'application/octet-stream', compressible: true }, + { name: 'a structured JSON suffix', contentType: 'application/vnd.api+json', compressible: true }, + { name: 'SVG under a compressed prefix', contentType: 'image/svg+xml', compressible: true }, + { name: 'SVG uppercase with parameters', contentType: 'IMAGE/SVG+XML; charset=utf-8', compressible: true }, + { name: 'a raw bitmap under a compressed prefix', contentType: 'image/bmp', compressible: true }, + { name: 'TIFF under a compressed prefix', contentType: 'image/tiff', compressible: true }, + { name: 'raw audio under a compressed prefix', contentType: 'audio/wav', compressible: true }, + { name: 'raw PCM audio in its registered casing', contentType: 'audio/L24', compressible: true }, + { name: 'MIDI event data under a compressed prefix', contentType: 'audio/midi', compressible: true }, + { name: 'the image prefix', contentType: 'image/png', compressible: false }, + { name: 'the video prefix', contentType: 'video/mp4', compressible: false }, + { name: 'the audio prefix', contentType: 'audio/mpeg', compressible: false }, + { name: 'an archive', contentType: 'application/zip', compressible: false }, + { name: 'a gzip archive', contentType: 'application/x-gzip', compressible: false }, + { name: 'a windows zip archive', contentType: 'application/x-zip-compressed', compressible: false }, + { name: 'a zip container with a suffix', contentType: 'application/epub+zip', compressible: false }, + { + name: 'an office open xml document', + contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + compressible: false, + }, + { name: 'a web font', contentType: 'font/woff2', compressible: false }, + { name: 'surrounding whitespace and mixed case', contentType: ' Image/PNG ', compressible: false }, + ])('reports $name as compressible: $compressible', ({ contentType, compressible }) => { + expect(utils.isCompressibleContentType(contentType)).toBe(compressible); + }); +}); + describe('utils.isBuffer()', () => { test('accepts binary values', () => { expect(utils.isBuffer(Buffer.from('abc'))).toBe(true); From 459fa8138d1793ec6dea15e36a77bcd489ad48a8 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 10 Sep 2026 09:59:30 +0200 Subject: [PATCH 2/4] fix: read request headers case-insensitively Axios stores a header under whatever casing the caller used, and bracket access is case-sensitive, so `Content-Encoding: gzip` missed the guard that skips compression for an already-encoded body. The body was then compressed a second time, and axios collapsed the two header keys last-write-wins, leaving `br` on a brotli-wrapped gzip stream. The same lookup in `serializeRequest` dropped function-valued Actor input fields when the content type carried an unconventional casing. --- src/interceptors.ts | 21 +++++++++++++----- test/http_client.test.ts | 46 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/src/interceptors.ts b/src/interceptors.ts index de28df2c7..dd747533e 100644 --- a/src/interceptors.ts +++ b/src/interceptors.ts @@ -30,6 +30,18 @@ export class InvalidResponseBodyError extends Error { } } +/** + * Reads a request header regardless of the casing it was set with, since HTTP header names are case-insensitive + * while the config keeps whatever casing the caller used. + */ +function getHeader(config: ApifyRequestConfig, name: string): string | undefined { + const wanted = name.toLowerCase(); + const key = Object.keys(config.headers ?? {}).find((candidate) => candidate.toLowerCase() === wanted); + const value = key === undefined ? undefined : config.headers?.[key]; + + return typeof value === 'string' ? value : undefined; +} + function serializeRequest(config: ApifyRequestConfig): ApifyRequestConfig { const [defaultTransform] = axios.defaults.transformRequest as AxiosRequestTransformer[]; @@ -42,9 +54,9 @@ function serializeRequest(config: ApifyRequestConfig): ApifyRequestConfig { // it's a small price to pay. The axios default transform does a lot // of body type checks and we would have to copy all of them to the resource clients. if (config.stringifyFunctions) { - const contentTypeHeader = config.headers?.['Content-Type'] || config.headers?.['content-type']; + const contentTypeHeader = getHeader(config, 'content-type'); try { - const { type } = contentTypeParser.parse(contentTypeHeader); + const type = contentTypeHeader ? contentTypeParser.parse(contentTypeHeader).type : undefined; if (type === 'application/json' && typeof config.data === 'object') { config.data = stringifyWithFunctions(config.data); } else { @@ -80,10 +92,9 @@ function stringifyWithFunctions(obj: JsonObject) { async function maybeCompressRequest(config: ApifyRequestConfig): Promise { // A caller-supplied encoding means the body is already encoded and the header describes it, so leave both alone. - if (config.headers?.['content-encoding']) return config; + if (getHeader(config, 'content-encoding')) return config; - const contentTypeHeader = config.headers?.['Content-Type'] || config.headers?.['content-type']; - if (!isCompressibleContentType(contentTypeHeader)) return config; + if (!isCompressibleContentType(getHeader(config, 'content-type'))) return config; const maybeCompressed = await maybeCompressValue(config.data); if (maybeCompressed) { diff --git a/test/http_client.test.ts b/test/http_client.test.ts index 31a788894..f10a4b0c5 100644 --- a/test/http_client.test.ts +++ b/test/http_client.test.ts @@ -1,5 +1,7 @@ +import { randomBytes } from 'node:crypto'; import type { AddressInfo } from 'node:net'; import os from 'node:os'; +import { gzipSync } from 'node:zlib'; import { ApifyClient } from 'apify-client'; import type { Page } from 'puppeteer'; @@ -51,4 +53,48 @@ describe('HttpClient', () => { // this is failing after axios upgrade, the error is returned with a wrong name and message // expect(err.message).toMatch('timeout of 1000ms exceeded'); }); + + test.each([ + { name: 'lowercase', header: 'content-encoding' }, + { name: 'canonical casing', header: 'Content-Encoding' }, + { name: 'uppercase', header: 'CONTENT-ENCODING' }, + ])('forwards a body pre-encoded with a $name header as it is', async ({ header }) => { + // Gzipped random bytes: incompressible, and above the 1 KiB threshold that would otherwise + // send the body through the compressor. + const payload = randomBytes(8192); + const encoded = gzipSync(payload); + + await client.httpClient.call({ + url: `${baseUrl}/v2/key-value-stores/some-id/records/some-key`, + method: 'PUT', + data: encoded, + headers: { 'content-type': 'application/octet-stream', [header]: 'gzip' }, + }); + + const request = mockServer.getLastRequest(); + expect(request?.headers['content-encoding']).toBe('gzip'); + expect(request?.headers['content-length']).toBe(String(encoded.length)); + expect(request?.body).toEqual(payload); + }); + + test.each([ + { name: 'lowercase', header: 'content-type' }, + { name: 'canonical casing', header: 'Content-Type' }, + { name: 'uppercase', header: 'CONTENT-TYPE' }, + ])('skips compression for an already-compressed type sent with a $name header', async ({ header }) => { + // Trivially compressible and well above the 1 KiB threshold, so an unchanged content length + // is proof the body was not run through the compressor. + const payload = Buffer.alloc(4096, 'a'); + + await client.httpClient.call({ + url: `${baseUrl}/v2/key-value-stores/some-id/records/some-key`, + method: 'PUT', + data: payload, + headers: { [header]: 'image/png' }, + }); + + const request = mockServer.getLastRequest(); + expect(request?.headers['content-encoding']).toBeUndefined(); + expect(request?.headers['content-length']).toBe(String(payload.length)); + }); }); From b3d177f959216865eddcd50093057c22d1841b71 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 10 Sep 2026 09:59:35 +0200 Subject: [PATCH 3/4] docs: clarify which content types setRecord compresses --- src/resource_clients/key_value_store.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/resource_clients/key_value_store.ts b/src/resource_clients/key_value_store.ts index 79593775b..16b7e8aec 100644 --- a/src/resource_clients/key_value_store.ts +++ b/src/resource_clients/key_value_store.ts @@ -432,8 +432,8 @@ export class KeyValueStoreClient extends ResourceClient { * * Worth setting for media and archives: the client skips compressing a body * whose content type already carries its own compression. The - * `'application/octet-stream'` fallback is still compressed, because it - * could be any binary data. + * `'application/octet-stream'` fallback is treated as compressible, because + * it could hold any binary data. * @param options - Storage options * @param options.timeoutSecs - Timeout for the upload in seconds. Default varies by value size. * @param options.doNotRetryTimeouts - If `true`, don't retry on timeout errors. Default is `false`. From 5fc97a6d6d850bb6613ae7f21db01065de5740d2 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 10 Sep 2026 09:59:39 +0200 Subject: [PATCH 4/4] test: cover compression of an unknown-binary content type --- test/key_value_stores.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/key_value_stores.test.ts b/test/key_value_stores.test.ts index 0f9451f50..c39a4388c 100644 --- a/test/key_value_stores.test.ts +++ b/test/key_value_stores.test.ts @@ -669,9 +669,10 @@ describe('Key-Value Store methods', () => { }); test.each([ - { name: 'SVG', contentType: 'image/svg+xml' }, - { name: 'a raw bitmap', contentType: 'image/bmp' }, - ])('setRecord() compresses $name despite its already-compressed prefix', async ({ contentType }) => { + { name: 'SVG under a compressed prefix', contentType: 'image/svg+xml' }, + { name: 'a raw bitmap under a compressed prefix', contentType: 'image/bmp' }, + { name: 'unknown binary', contentType: 'application/octet-stream' }, + ])('setRecord() compresses $name', async ({ contentType }) => { const key = 'some-key'; const storeId = 'some-id'; const value = Buffer.alloc(4096, 'a');