Migrate node-fetch to built-in fetch (undici)

.changeset/pro-9597-fetch-apostrophe.md

@@ -0,0 +1,17 @@
+---
+"apostrophe": minor
+---
+
+The server-side HTTP client (`apos.http`) now uses Node's built-in `fetch` instead of `node-fetch`.
+
+Most code that calls `apos.http.get()`, `apos.http.post()`, etc. needs no changes. A few things to be aware of if you use advanced options or read raw responses:
+
+- The `agent` option is no longer supported (the built-in `fetch` has no equivalent). Pass an undici `dispatcher` instead; `apos.http` throws if `agent` is given.
+- A `Host` request header can no longer be set (it is disallowed by the fetch standard and is silently ignored).
+- `originalResponse: true` now resolves with the built-in `fetch` `Response`. Its `body` is a web `ReadableStream` (use `require('node:stream').Readable.fromWeb()` to read it as a Node stream), and node-fetch-only helpers such as `.buffer()` are no longer available.
+- Requests that send a conditional header (`If-None-Match` / `If-Modified-Since`) now also send `Cache-Control: no-cache`, as required by the fetch standard. An endpoint that returns `304 Not Modified` based on those headers may return `200` to such a request.
+
+New capabilities:
+
+- The `timeout` option (in milliseconds) and the standard `signal` (`AbortSignal`) and undici `dispatcher` options are supported.
+- A request `body` may be a native `FormData`, in addition to a `form-data` package instance.

.changeset/pro-9597-fetch-oembetter.md

@@ -0,0 +1,5 @@
+---
+"oembetter": patch
+---
+
+Replaced the `node-fetch` dependency with Node's built-in `fetch`. This is an internal change with no effect on the public API.

packages/apostrophe/package.json

@@ -23,7 +23,7 @@
   },
   "homepage": "https://github.com/apostrophecms/apostrophe/tree/main/packages/apostrophe",
   "engines": {
-    "node": ">=16.0.0"
+    "node": ">=22.0.0"
   },
   "keywords": [
     "apostrophe",
@@ -102,7 +102,6 @@
     "minimatch": "^3.1.4",
     "mkdirp": "^0.5.5",
     "multer": "^2.1.1",
-    "node-fetch": "^2.6.1",
     "nodemailer": "^8.0.5",
     "nunjucks": "^3.2.1",
     "oembetter": "workspace:^",

packages/apostrophe/test-lib/test.js

@@ -1,5 +1,6 @@
 const fs = require('fs-extra');
 const path = require('path');
+const http = require('node:http');
 
 const setupPackages = ({ folder = 'test' }) => {
   const testNodeModules = path.join(__dirname, '../', folder, 'node_modules/');
@@ -57,5 +58,37 @@ const setupPackages = ({ folder = 'test' }) => {
 };
 setupPackages({ folder: 'test' });
 
+// Performs a GET via the raw node:http client, sending `headers` to the server
+// verbatim. Use this in tests that must control headers the built-in fetch
+// (used by apos.http) would otherwise refuse or rewrite: e.g. a forbidden
+// `Host` header, or the `Cache-Control: no-cache` it adds to any request that
+// carries a conditional header (If-None-Match / If-Modified-Since). `url` may
+// be absolute or site-relative (resolved against `apos.http.getBase()`).
+// Resolves with a fullResponse-shaped { status, headers, body }.
+const rawGet = (apos, url, headers = {}) => {
+  const target = url.startsWith('/') ? `${apos.http.getBase()}${url}` : url;
+  const parsed = new URL(target);
+  return new Promise((resolve, reject) => {
+    const req = http.request({
+      hostname: parsed.hostname,
+      port: parsed.port,
+      path: parsed.pathname + parsed.search,
+      method: 'GET',
+      headers
+    }, (res) => {
+      const chunks = [];
+      res.on('data', (chunk) => chunks.push(chunk));
+      res.on('end', () => resolve({
+        status: res.statusCode,
+        headers: res.headers,
+        body: Buffer.concat(chunks).toString()
+      }));
+    });
+    req.on('error', reject);
+    req.end();
+  });
+};
+
 module.exports = require('./util.js');
 module.exports.setupPackages = setupPackages;
+module.exports.rawGet = rawGet;

packages/apostrophe/test/files.js

@@ -1,6 +1,9 @@
 const t = require('../test-lib/test.js');
 const assert = require('assert/strict');
 const fs = require('fs');
+// rawGet (raw node:http) lets this suite send a spoofed `Host` header, which
+// the built-in fetch used by apos.http would drop as a forbidden header.
+const { rawGet } = t;
 
 describe('Files', function() {
 
@@ -142,17 +145,13 @@ describe('Files', function() {
       const attachment = apos.attachment.first(file);
       const url = apos.attachment.url(attachment);
       assert(url);
-      // Send an attacker-controlled Host header (e.g. the cloud metadata
-      // address from the advisory). The upstream fetch must be resolved
-      // against the server-trusted baseUrl, not this header, so the
-      // legitimate content is still served and the request is never
-      // steered at the spoofed host.
-      const response = await apos.http.get(url, {
-        headers: {
-          Host: '169.254.169.254'
-        },
-        fullResponse: true
-      });
+      // Spoof the Host header (the cloud-metadata address from the advisory)
+      // over a raw request: apos.http uses the built-in fetch, which drops a
+      // forbidden `Host` header and so cannot deliver the spoof. The server
+      // must resolve the upstream fetch against its trusted baseUrl, not this
+      // header, so the legitimate content is still served and the request is
+      // never steered at the spoofed host.
+      const response = await rawGet(apos, url, { Host: '169.254.169.254' });
       assert.strictEqual(response.status, 200);
       assert.strictEqual(response.body, attachment.data);
     } finally {

packages/apostrophe/test/pages.js

@@ -1,6 +1,12 @@
 const t = require('../test-lib/test.js');
 const assert = require('assert');
 const _ = require('lodash');
+// The REST API etag tests below issue their conditional request via rawGet
+// (raw node:http): the built-in fetch used by apos.http adds Cache-Control:
+// no-cache to any request carrying a conditional header (Fetch standard), which
+// would suppress the asserted 304s. The page-serving etag tests stay on
+// apos.http (those routes set 304 explicitly).
+const { rawGet } = t;
 
 describe('Pages', function() {
   let apos;
@@ -1087,11 +1093,8 @@ describe('Pages', function() {
     };
 
     const response1 = await apos.http.get(`/api/v1/@apostrophecms/page/${homeId}`, { fullResponse: true });
-    const response2 = await apos.http.get(`/api/v1/@apostrophecms/page/${homeId}`, {
-      fullResponse: true,
-      headers: {
-        'if-none-match': response1.headers.etag
-      }
+    const response2 = await rawGet(apos, `/api/v1/@apostrophecms/page/${homeId}`, {
+      'if-none-match': response1.headers.etag
     });
 
     assert(response1.status === 200);
@@ -1125,11 +1128,8 @@ describe('Pages', function() {
     // so requesting it again should not return a 304 status code
     const pageUpdateResponse = await apos.doc.update(apos.task.getReq(), pageDoc);
 
-    const response2 = await apos.http.get(`/api/v1/@apostrophecms/page/${homeId}`, {
-      fullResponse: true,
-      headers: {
-        'if-none-match': response1.headers.etag
-      }
+    const response2 = await rawGet(apos, `/api/v1/@apostrophecms/page/${homeId}`, {
+      'if-none-match': response1.headers.etag
     });
 
     const eTag1Parts = response1.headers.etag.split(':');
@@ -1165,11 +1165,8 @@ describe('Pages', function() {
     outOfDateETagParts[2] = Number(outOfDateETagParts[2]) -
       (4444 + 1) * 1000; // 1s outdated
 
-    const response2 = await apos.http.get(`/api/v1/@apostrophecms/page/${homeId}`, {
-      fullResponse: true,
-      headers: {
-        'if-none-match': outOfDateETagParts.join(':')
-      }
+    const response2 = await rawGet(apos, `/api/v1/@apostrophecms/page/${homeId}`, {
+      'if-none-match': outOfDateETagParts.join(':')
     });
 
     const eTag1Parts = response1.headers.etag.split(':');

packages/apostrophe/test/pieces.js

@@ -6,6 +6,12 @@ const _ = require('lodash');
 const FormData = require('form-data');
 const t = require('../test-lib/test.js');
 
+// The etag tests below issue their conditional request via rawGet (raw
+// node:http): the built-in fetch used by apos.http adds Cache-Control: no-cache
+// to any request carrying a conditional header (Fetch standard), which would
+// suppress the asserted 304s.
+const { rawGet } = t;
+
 describe('Pieces', function() {
 
   let apos;
@@ -1807,11 +1813,8 @@ describe('Pieces', function() {
     };
 
     const response1 = await apos.http.get('/api/v1/thing/testThing:en:published', { fullResponse: true });
-    const response2 = await apos.http.get('/api/v1/thing/testThing:en:published', {
-      fullResponse: true,
-      headers: {
-        'if-none-match': response1.headers.etag
-      }
+    const response2 = await rawGet(apos, '/api/v1/thing/testThing:en:published', {
+      'if-none-match': response1.headers.etag
     });
 
     assert(response1.status === 200);
@@ -1845,11 +1848,8 @@ describe('Pieces', function() {
     // so requesting it again should not return a 304 status code
     const pieceUpdateResponse = await apos.doc.update(apos.task.getReq(), pieceDoc);
 
-    const response2 = await apos.http.get('/api/v1/thing/testThing:en:published', {
-      fullResponse: true,
-      headers: {
-        'if-none-match': response1.headers.etag
-      }
+    const response2 = await rawGet(apos, '/api/v1/thing/testThing:en:published', {
+      'if-none-match': response1.headers.etag
     });
 
     const eTag1Parts = response1.headers.etag.split(':');
@@ -1885,11 +1885,8 @@ describe('Pieces', function() {
     outOfDateETagParts[2] = Number(outOfDateETagParts[2]) -
       (4444 + 1) * 1000; // 1s outdated
 
-    const response2 = await apos.http.get('/api/v1/thing/testThing:en:published', {
-      fullResponse: true,
-      headers: {
-        'if-none-match': outOfDateETagParts.join(':')
-      }
+    const response2 = await rawGet(apos, '/api/v1/thing/testThing:en:published', {
+      'if-none-match': outOfDateETagParts.join(':')
     });
 
     const eTag1Parts = response1.headers.etag.split(':');

packages/oembetter/oembed.js

@@ -1,4 +1,3 @@
-const fetch = require('node-fetch');
 const { XMLParser } = require('fast-xml-parser');
 
 const cheerio = require('cheerio');

packages/oembetter/package.json

@@ -30,7 +30,6 @@
     "async": "^0.9.0",
     "cheerio": "^1.1.0",
     "fast-xml-parser": "^5.7.0",
-    "node-fetch": "^2.6.7",
     "urls": "0.0.4"
   },
   "devDependencies": {