perf: skip request-body compression for already-compressed content types

src/interceptors.ts

@@ -5,7 +5,7 @@ import type { JsonObject } from 'type-fest';
 
 import { maybeParseBody } from './body_parser.js';
 import type { ApifyRequestConfig, ApifyResponse } from './http_client.js';
-import { isNode, maybeCompressValue } from './utils.js';
+import { isCompressibleContentType, isNode, maybeCompressValue } from './utils.js';
 
 /**
  * This error exists for the quite common situation, where only a partial JSON response is received and
@@ -30,6 +30,18 @@ export class InvalidResponseBodyError extends Error {
     }
 }
 
+/**
+ * Reads a request header regardless of the casing it was set with, since HTTP header names are case-insensitive
+ * while the config keeps whatever casing the caller used.
+ */
+function getHeader(config: ApifyRequestConfig, name: string): string | undefined {
+    const wanted = name.toLowerCase();
+    const key = Object.keys(config.headers ?? {}).find((candidate) => candidate.toLowerCase() === wanted);
+    const value = key === undefined ? undefined : config.headers?.[key];
+
+    return typeof value === 'string' ? value : undefined;
+}
+
 function serializeRequest(config: ApifyRequestConfig): ApifyRequestConfig {
     const [defaultTransform] = axios.defaults.transformRequest as AxiosRequestTransformer[];
 
@@ -42,9 +54,9 @@ function serializeRequest(config: ApifyRequestConfig): ApifyRequestConfig {
     // it's a small price to pay. The axios default transform does a lot
     // of body type checks and we would have to copy all of them to the resource clients.
     if (config.stringifyFunctions) {
-        const contentTypeHeader = config.headers?.['Content-Type'] || config.headers?.['content-type'];
+        const contentTypeHeader = getHeader(config, 'content-type');
         try {
-            const { type } = contentTypeParser.parse(contentTypeHeader);
+            const type = contentTypeHeader ? contentTypeParser.parse(contentTypeHeader).type : undefined;
             if (type === 'application/json' && typeof config.data === 'object') {
                 config.data = stringifyWithFunctions(config.data);
             } else {
@@ -79,7 +91,10 @@ function stringifyWithFunctions(obj: JsonObject) {
 }
 
 async function maybeCompressRequest(config: ApifyRequestConfig): Promise<ApifyRequestConfig> {
-    if (config.headers?.['content-encoding']) return config;
+    // A caller-supplied encoding means the body is already encoded and the header describes it, so leave both alone.
+    if (getHeader(config, 'content-encoding')) return config;
+
+    if (!isCompressibleContentType(getHeader(config, 'content-type'))) return config;
 
     const maybeCompressed = await maybeCompressValue(config.data);
     if (maybeCompressed) {

src/resource_clients/key_value_store.ts

@@ -429,6 +429,11 @@ export class KeyValueStoreClient extends ResourceClient {
      *                             - Objects: `'application/json; charset=utf-8'`
      *                             - Strings: `'text/plain; charset=utf-8'`
      *                             - Binary values and streams: `'application/octet-stream'`
+     *
+     *                             Worth setting for media and archives: the client skips compressing a body
+     *                             whose content type already carries its own compression. The
+     *                             `'application/octet-stream'` fallback is treated as compressible, because
+     *                             it could hold any binary data.
      * @param options - Storage options
      * @param options.timeoutSecs - Timeout for the upload in seconds. Default varies by value size.
      * @param options.doNotRetryTimeouts - If `true`, don't retry on timeout errors. Default is `false`.

src/utils.ts

@@ -19,6 +19,55 @@ import packageJson from '../package.json' with { type: 'json' };
 
 const MIN_COMPRESS_BYTES = 1024;
 
+/** Media type prefixes whose payloads carry their own compression, so compressing the request body is wasted work. */
+const ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES = ['audio/', 'image/', 'video/'];
+
+/** Exact media types whose payloads carry their own compression. */
+const ALREADY_COMPRESSED_MEDIA_TYPES = new Set([
+    'application/epub+zip',
+    'application/gzip',
+    'application/java-archive',
+    'application/vnd.android.package-archive',
+    'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+    'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+    'application/vnd.rar',
+    'application/x-7z-compressed',
+    'application/x-bzip',
+    'application/x-bzip2',
+    'application/x-gzip',
+    'application/x-rar-compressed',
+    'application/x-xz',
+    'application/x-zip-compressed',
+    'application/zip',
+    'application/zstd',
+    'font/woff',
+    'font/woff2',
+]);
+
+/** Uncompressed media types that sit under an already-compressed prefix, so compressing them still pays off. */
+const COMPRESSIBLE_MEDIA_TYPES = new Set([
+    'audio/aiff',
+    'audio/basic',
+    'audio/l16',
+    'audio/l24',
+    'audio/midi',
+    'audio/vnd.wave',
+    'audio/wav',
+    'audio/wave',
+    'audio/x-aiff',
+    'audio/x-wav',
+    'image/bmp',
+    'image/tiff',
+    'image/vnd.adobe.photoshop',
+    'image/vnd.microsoft.icon',
+    'image/x-icon',
+    'image/x-ms-bmp',
+]);
+
+/** Structured syntax suffixes marking a media type as text even under an already-compressed prefix (`image/svg+xml`). */
+const COMPRESSIBLE_MEDIA_TYPE_SUFFIXES = ['+json', '+xml'];
+
 export { parseArgument };
 
 /**
@@ -212,6 +261,28 @@ export interface CompressedValue {
     encoding: 'br' | 'gzip';
 }
 
+/**
+ * Decides whether a request body with the given content type is worth compressing.
+ *
+ * Images, audio, video and archives already carry their own compression. Running them through brotli or gzip
+ * burns CPU, holds a second full copy of the body in memory, and usually produces output slightly larger than
+ * the input. Formats that are raw despite such a media type, for example `image/bmp` or `audio/wav`, are still
+ * compressed. A body with no content type is assumed to be compressible.
+ * @internal
+ */
+export function isCompressibleContentType(contentType?: string): boolean {
+    if (!contentType) return true;
+
+    // `Content-Type` is case-insensitive and may carry parameters, for example `text/plain; charset=utf-8`.
+    const mediaType = contentType.split(';', 1)[0].trim().toLowerCase();
+
+    if (COMPRESSIBLE_MEDIA_TYPES.has(mediaType)) return true;
+    if (COMPRESSIBLE_MEDIA_TYPE_SUFFIXES.some((suffix) => mediaType.endsWith(suffix))) return true;
+
+    if (ALREADY_COMPRESSED_MEDIA_TYPES.has(mediaType)) return false;
+    return !ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES.some((prefix) => mediaType.startsWith(prefix));
+}
+
 /**
  * Compress the passed value using brotli, falling back to gzip. Returns undefined if the data is
  * too small / wrong type, or if neither algorithm is available.

test/http_client.test.ts

@@ -1,6 +1,8 @@
+import { randomBytes } from 'node:crypto';
 import http from 'node:http';
 import type { AddressInfo } from 'node:net';
 import os from 'node:os';
+import { gzipSync } from 'node:zlib';
 
 import { ApifyClient } from 'apify-client';
 import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test, vi } from 'vitest';
@@ -113,4 +115,48 @@ describe('HttpClient', () => {
             await new Promise<void>((done) => proxy.close(() => done()));
         }
     });
+
+    test.each([
+        { name: 'lowercase', header: 'content-encoding' },
+        { name: 'canonical casing', header: 'Content-Encoding' },
+        { name: 'uppercase', header: 'CONTENT-ENCODING' },
+    ])('forwards a body pre-encoded with a $name header as it is', async ({ header }) => {
+        // Gzipped random bytes: incompressible, and above the 1 KiB threshold that would otherwise
+        // send the body through the compressor.
+        const payload = randomBytes(8192);
+        const encoded = gzipSync(payload);
+
+        await client.httpClient.call({
+            url: `${baseUrl}/v2/key-value-stores/some-id/records/some-key`,
+            method: 'PUT',
+            data: encoded,
+            headers: { 'content-type': 'application/octet-stream', [header]: 'gzip' },
+        });
+
+        const request = mockServer.getLastRequest();
+        expect(request?.headers['content-encoding']).toBe('gzip');
+        expect(request?.headers['content-length']).toBe(String(encoded.length));
+        expect(request?.body).toEqual(payload);
+    });
+
+    test.each([
+        { name: 'lowercase', header: 'content-type' },
+        { name: 'canonical casing', header: 'Content-Type' },
+        { name: 'uppercase', header: 'CONTENT-TYPE' },
+    ])('skips compression for an already-compressed type sent with a $name header', async ({ header }) => {
+        // Trivially compressible and well above the 1 KiB threshold, so an unchanged content length
+        // is proof the body was not run through the compressor.
+        const payload = Buffer.alloc(4096, 'a');
+
+        await client.httpClient.call({
+            url: `${baseUrl}/v2/key-value-stores/some-id/records/some-key`,
+            method: 'PUT',
+            data: payload,
+            headers: { [header]: 'image/png' },
+        });
+
+        const request = mockServer.getLastRequest();
+        expect(request?.headers['content-encoding']).toBeUndefined();
+        expect(request?.headers['content-length']).toBe(String(payload.length));
+    });
 });

test/key_value_stores.test.ts

@@ -648,6 +648,43 @@ describe('Key-Value Store methods', () => {
             });
         });
 
+        test.each([
+            { name: 'a PNG', contentType: 'image/png' },
+            { name: 'a ZIP archive', contentType: 'application/zip' },
+            { name: 'a web font', contentType: 'font/woff2' },
+        ])('setRecord() sends $name uncompressed', async ({ contentType }) => {
+            const key = 'some-key';
+            const storeId = 'some-id';
+            // Well above the 1 KiB compression threshold and trivially compressible, so an unchanged
+            // content length is proof the body was not run through the compressor.
+            const value = Buffer.alloc(4096, 'a');
+
+            const res = await client.keyValueStore(storeId).setRecord({ key, value, contentType });
+            expect(res).toBeUndefined();
+
+            const request = mockServer.getLastRequest();
+            expect(request?.headers['content-type']).toBe(contentType);
+            expect(request?.headers['content-encoding']).toBeUndefined();
+            expect(request?.headers['content-length']).toBe(String(value.length));
+        });
+
+        test.each([
+            { name: 'SVG under a compressed prefix', contentType: 'image/svg+xml' },
+            { name: 'a raw bitmap under a compressed prefix', contentType: 'image/bmp' },
+            { name: 'unknown binary', contentType: 'application/octet-stream' },
+        ])('setRecord() compresses $name', async ({ contentType }) => {
+            const key = 'some-key';
+            const storeId = 'some-id';
+            const value = Buffer.alloc(4096, 'a');
+
+            const res = await client.keyValueStore(storeId).setRecord({ key, value, contentType });
+            expect(res).toBeUndefined();
+
+            const request = mockServer.getLastRequest();
+            expect(request?.headers['content-type']).toBe(contentType);
+            expect(request?.headers['content-encoding']).toBe('br');
+        });
+
         test('deleteRecord() works', async () => {
             const key = 'some-key';
             const storeId = '204';

test/utils.test.ts

@@ -158,6 +158,40 @@ describe('utils.maybeCompressValue()', () => {
     });
 });
 
+describe('utils.isCompressibleContentType()', () => {
+    test.each([
+        { name: 'a missing content type', contentType: undefined, compressible: true },
+        { name: 'an empty content type', contentType: '', compressible: true },
+        { name: 'JSON', contentType: 'application/json', compressible: true },
+        { name: 'text with parameters', contentType: 'text/plain; charset=utf-8', compressible: true },
+        { name: 'unknown binary', contentType: 'application/octet-stream', compressible: true },
+        { name: 'a structured JSON suffix', contentType: 'application/vnd.api+json', compressible: true },
+        { name: 'SVG under a compressed prefix', contentType: 'image/svg+xml', compressible: true },
+        { name: 'SVG uppercase with parameters', contentType: 'IMAGE/SVG+XML; charset=utf-8', compressible: true },
+        { name: 'a raw bitmap under a compressed prefix', contentType: 'image/bmp', compressible: true },
+        { name: 'TIFF under a compressed prefix', contentType: 'image/tiff', compressible: true },
+        { name: 'raw audio under a compressed prefix', contentType: 'audio/wav', compressible: true },
+        { name: 'raw PCM audio in its registered casing', contentType: 'audio/L24', compressible: true },
+        { name: 'MIDI event data under a compressed prefix', contentType: 'audio/midi', compressible: true },
+        { name: 'the image prefix', contentType: 'image/png', compressible: false },
+        { name: 'the video prefix', contentType: 'video/mp4', compressible: false },
+        { name: 'the audio prefix', contentType: 'audio/mpeg', compressible: false },
+        { name: 'an archive', contentType: 'application/zip', compressible: false },
+        { name: 'a gzip archive', contentType: 'application/x-gzip', compressible: false },
+        { name: 'a windows zip archive', contentType: 'application/x-zip-compressed', compressible: false },
+        { name: 'a zip container with a suffix', contentType: 'application/epub+zip', compressible: false },
+        {
+            name: 'an office open xml document',
+            contentType: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+            compressible: false,
+        },
+        { name: 'a web font', contentType: 'font/woff2', compressible: false },
+        { name: 'surrounding whitespace and mixed case', contentType: '  Image/PNG  ', compressible: false },
+    ])('reports $name as compressible: $compressible', ({ contentType, compressible }) => {
+        expect(utils.isCompressibleContentType(contentType)).toBe(compressible);
+    });
+});
+
 describe('utils.isBuffer()', () => {
     test('accepts binary values', () => {
         expect(utils.isBuffer(Buffer.from('abc'))).toBe(true);