feat: Compress requests using brotli algo

- apify/apify-core#28971 added support for brotli compression to BE.
- Pros: higher compression, cons: more CPU intensive.
- The code could be running on too old Node.js, so brotli compression is applied only if possible; the code otherwise falls back to gzip. Too small payloads and unsupported types are still not compressed - no change.

Author: mixalturek

src/interceptors.ts

@@ -5,7 +5,7 @@ import type { JsonObject } from 'type-fest';
 
 import { maybeParseBody } from './body_parser';
 import type { ApifyRequestConfig, ApifyResponse } from './http_client';
-import { isNode, maybeGzipValue } from './utils';
+import { isNode, maybeCompressValue } from './utils';
 
 /**
  * This error exists for the quite common situation, where only a partial JSON response is received and
@@ -78,14 +78,14 @@ function stringifyWithFunctions(obj: JsonObject) {
     });
 }
 
-async function maybeGzipRequest(config: ApifyRequestConfig): Promise<ApifyRequestConfig> {
+async function maybeCompressRequest(config: ApifyRequestConfig): Promise<ApifyRequestConfig> {
     if (config.headers?.['content-encoding']) return config;
 
-    const maybeZippedData = await maybeGzipValue(config.data);
-    if (maybeZippedData) {
+    const maybeCompressed = await maybeCompressValue(config.data);
+    if (maybeCompressed) {
         config.headers ??= {};
-        config.headers['content-encoding'] = 'gzip';
-        config.data = maybeZippedData;
+        config.headers['content-encoding'] = maybeCompressed.encoding;
+        config.data = maybeCompressed.data;
     }
 
     return config;
@@ -121,7 +121,7 @@ export type RequestInterceptorFunction = Parameters<AxiosInterceptorManager<Apif
 export type ResponseInterceptorFunction = Parameters<AxiosInterceptorManager<ApifyResponse>['use']>[0];
 
 export const requestInterceptors: RequestInterceptorFunction[] = [
-    maybeGzipRequest,
+    maybeCompressRequest,
     serializeRequest,
     ensureHeadersPrototype,
 ];

src/utils.ts

@@ -13,7 +13,7 @@ import type { WebhookUpdateData } from './resource_clients/webhook';
 const NOT_FOUND_STATUS_CODE = 404;
 const RECORD_NOT_FOUND_TYPE = 'record-not-found';
 const RECORD_OR_TOKEN_NOT_FOUND_TYPE = 'record-or-token-not-found';
-const MIN_GZIP_BYTES = 1024;
+const MIN_COMPRESS_BYTES = 1024;
 
 /**
  * Generic interface for objects that may contain a data property.
@@ -112,28 +112,65 @@ export function stringifyWebhooksToBase64(webhooks: WebhookUpdateData[]): string
 let gzipPromisified: ((arg: string | Buffer<ArrayBufferLike>) => Promise<Buffer>) | undefined;
 
 /**
- * Gzip provided value, otherwise returns undefined.
+ * Gzip-compress the provided value.
  */
-export async function maybeGzipValue(value: unknown): Promise<Buffer | undefined> {
-    if (!isNode()) return;
-    if (typeof value !== 'string' && !Buffer.isBuffer(value)) return;
+async function gzipValue(value: string | Buffer<ArrayBufferLike>): Promise<Buffer> {
+    if (!gzipPromisified) {
+        const { promisify } = await import('node:util');
+        const { gzip } = await import('node:zlib');
+        gzipPromisified = promisify(gzip);
+    }
 
-    // Request compression is not that important so let's
-    // skip it instead of throwing for unsupported types.
-    const areDataLargeEnough = Buffer.byteLength(value as string) >= MIN_GZIP_BYTES;
-    if (areDataLargeEnough) {
-        if (!gzipPromisified) {
-            const { promisify } = await import('node:util');
-            const { gzip } = await import('node:zlib');
-            gzipPromisified = promisify(gzip);
-        }
+    return gzipPromisified(value);
+}
+
+// null = confirmed unavailable; undefined = not yet checked
+let brotliCompressPromisified: ((arg: string | Buffer<ArrayBufferLike>) => Promise<Buffer>) | null | undefined;
 
-        return gzipPromisified(value);
+/**
+ * Brotli-compress the provided value, or return undefined if brotli is unavailable
+ * (Node.js < v10.16.0), this is a strict defensive guard.
+ */
+async function maybeBrotliValue(value: string | Buffer<ArrayBufferLike>): Promise<Buffer | undefined> {
+    if (brotliCompressPromisified === undefined) {
+        const { promisify } = await import('node:util');
+        const { brotliCompress } = await import('node:zlib');
+        brotliCompressPromisified = typeof brotliCompress === 'function' ? promisify(brotliCompress) : null;
+    }
+
+    if (brotliCompressPromisified !== null) {
+        return brotliCompressPromisified(value);
     }
 
     return undefined;
 }
 
+export interface CompressedValue {
+    data: Buffer;
+    encoding: 'br' | 'gzip';
+}
+
+/**
+ * Compress the passed value using brotli if available or using gzip as a fallback. Returns undefined
+ * if the data is too small / wrong type.
+ */
+export async function maybeCompressValue(value: unknown): Promise<CompressedValue | undefined> {
+    if (!isNode()) return undefined;
+
+    // Request compression is not that important so let's
+    // skip it instead of throwing for unsupported types.
+    if (typeof value !== 'string' && !Buffer.isBuffer(value)) return undefined;
+
+    const areDataLargeEnough = Buffer.byteLength(value) >= MIN_COMPRESS_BYTES;
+    if (!areDataLargeEnough) return undefined;
+
+    const brotli = await maybeBrotliValue(value);
+    if (brotli) return { data: brotli, encoding: 'br' };
+
+    const gzipped = await gzipValue(value);
+    return { data: gzipped, encoding: 'gzip' };
+}
+
 /**
  * Helper function slice the items from array to fit the max byte length.
  */

test/datasets.test.ts

@@ -369,7 +369,7 @@ describe('Dataset methods', () => {
 
             const expectedHeaders = {
                 'content-type': 'application/json; charset=utf-8',
-                'content-encoding': 'gzip',
+                'content-encoding': 'br',
             };
 
             const res = await client.dataset(datasetId).pushItems(data);

test/key_value_stores.test.ts

@@ -557,7 +557,7 @@ describe('Key-Value Store methods', () => {
             validateRequest({ params: { storeId, key }, body: JSON.parse(value), additionalHeaders: expectedHeaders });
         });
 
-        test('setRecord() uploads gzipped buffer in node context', async () => {
+        test('setRecord() uploads compressed buffer in node context', async () => {
             const key = 'some-key';
             const storeId = 'some-id';
             const value = [];
@@ -573,7 +573,7 @@ describe('Key-Value Store methods', () => {
                 body: value,
                 additionalHeaders: {
                     'content-type': contentType,
-                    'content-encoding': 'gzip',
+                    'content-encoding': 'br',
                 },
             });
 

test/utils.test.ts

@@ -1,6 +1,6 @@
 import type { WebhookUpdateData } from 'apify-client';
 import { ApifyApiError } from 'apify-client';
-import { describe, expect, test } from 'vitest';
+import { describe, expect, test, vi } from 'vitest';
 
 import * as utils from '../src/utils';
 
@@ -124,6 +124,54 @@ describe('utils.parseDateFields()', () => {
     });
 });
 
+describe('utils.maybeCompressValue()', () => {
+    test('returns undefined for small values', async () => {
+        expect(await utils.maybeCompressValue('small')).toBeUndefined();
+    });
+
+    test('returns undefined for non-string non-Buffer values', async () => {
+        expect(await utils.maybeCompressValue({ foo: 'bar' })).toBeUndefined();
+    });
+
+    test('compresses large string using brotli in Node.js', async () => {
+        const largeValue = 'x'.repeat(2048);
+        const result = await utils.maybeCompressValue(largeValue);
+        expect(result).not.toBeUndefined();
+        expect(result!.encoding).toBe('br');
+        expect(result!.data).toBeInstanceOf(Buffer);
+        expect(result!.data.length).toBeLessThan(Buffer.byteLength(largeValue));
+    });
+
+    test('compresses large Buffer using brotli in Node.js', async () => {
+        const largeValue = Buffer.alloc(2048, 'a');
+        const result = await utils.maybeCompressValue(largeValue);
+        expect(result).not.toBeUndefined();
+        expect(result!.encoding).toBe('br');
+        expect(result!.data).toBeInstanceOf(Buffer);
+        expect(result!.data.length).toBeLessThan(largeValue.length);
+    });
+
+    test('falls back to gzip when brotli is unavailable', async () => {
+        vi.resetModules();
+        vi.doMock('node:zlib', async () => {
+            const actual = await vi.importActual<typeof import('node:zlib')>('node:zlib');
+            return { ...actual, brotliCompress: undefined };
+        });
+
+        const { maybeCompressValue } = await import('../src/utils.js');
+        const largeValue = 'x'.repeat(2048);
+        const result = await maybeCompressValue(largeValue);
+
+        expect(result).not.toBeUndefined();
+        expect(result!.encoding).toBe('gzip');
+        expect(result!.data).toBeInstanceOf(Buffer);
+        expect(result!.data.length).toBeLessThan(Buffer.byteLength(largeValue));
+
+        vi.doUnmock('node:zlib');
+        vi.resetModules();
+    });
+});
+
 describe('utils.stringifyWebhooksToBase64()', () => {
     test('works', () => {
         const webhooks: WebhookUpdateData[] = [