fix: prerender and crawl pages whose content-type header carries a charset parameter (#16567)

The prerenderer compares `content-type` with `===`, so `text/html;
charset=utf-8` is not treated as HTML: the page is written without a
`.html` extension and its links are never crawled. `is_content_type`
already normalizes the header for this reason, from #7195 (strip
parameters) and GHSA-gv7g-x59x-wf8f (case-insensitive compare); the two
prerender checks predate both.

Reproduces #13612, closed as unreproducible, using the charset
workaround from #3184. The root `+server.js` error added in #15994 is
gated on the same value, so it can fire for an app that has none.

Co-authored-by: Nic Polumeyv <nicolas.polum@gmail.com>
Co-authored-by: Rich Harris <richard.a.harris@gmail.com>

Author: 3 people

.changeset/wild-donkeys-listen.md

@@ -0,0 +1,5 @@
+---
+'@sveltejs/kit': patch
+---
+
+fix: prerender and crawl pages whose `content-type` header carries a `charset` parameter

packages/kit/src/core/postbuild/prerender.js

@@ -17,6 +17,7 @@ import { createReadableStream } from '@sveltejs/kit/node';
 import generate_fallback from './fallback.js';
 import { stringify_remote_arg } from '../../runtime/shared.js';
 import { log_response } from '../../exports/vite/utils.js';
+import { matches_content_type } from '../../utils/http.js';
 
 export default forked(import.meta.url, prerender);
 
@@ -387,7 +388,11 @@ async function prerender({ hash, out, manifest_path, metadata, verbose, env, vit
 		const headers = Object.fromEntries(response.headers);
 
 		// if it's a 200 HTML response, crawl it. Skip error responses, as we don't save those
-		if (response.ok && config.prerender.crawl && headers['content-type'] === 'text/html') {
+		if (
+			response.ok &&
+			config.prerender.crawl &&
+			matches_content_type(headers['content-type'], 'text/html')
+		) {
 			const { ids, hrefs, invalid } = crawl(body.toString(), decoded);
 
 			for (const href of invalid) {
@@ -444,7 +449,7 @@ async function prerender({ hash, out, manifest_path, metadata, verbose, env, vit
 		const headers = Object.fromEntries(response.headers);
 
 		const type = headers['content-type'];
-		const is_html = response_type === REDIRECT || type === 'text/html';
+		const is_html = response_type === REDIRECT || matches_content_type(type, 'text/html');
 
 		if (!is_html && response.status === 200 && decoded.slice(config.paths.base.length + 1) === '') {
 			throw new Error(

packages/kit/src/utils/http.js

@@ -57,12 +57,13 @@ export function negotiate(accept, types) {
 }
 
 /**
- * Returns `true` if the request contains a `content-type` header with the given type
- * @param {Request} request
+ * Returns `true` if a `content-type` header value is one of the given types, ignoring
+ * parameters such as `charset` and comparing case-insensitively
+ * @param {string | null | undefined} header
  * @param  {...string} types
  */
-function is_content_type(request, ...types) {
-	const type = request.headers.get('content-type')?.split(';', 1)[0].trim() ?? '';
+export function matches_content_type(header, ...types) {
+	const type = header?.split(';', 1)[0].trim() ?? '';
 	return types.includes(type.toLowerCase());
 }
 
@@ -72,8 +73,8 @@ function is_content_type(request, ...types) {
 export function is_form_content_type(request) {
 	// These content types must be protected against CSRF
 	// https://developer.mozilla.org/en-US/docs/Web/API/HTMLFormElement/enctype
-	return is_content_type(
-		request,
+	return matches_content_type(
+		request.headers.get('content-type'),
 		'application/x-www-form-urlencoded',
 		'multipart/form-data',
 		'text/plain',

packages/kit/src/utils/http.spec.js

@@ -1,5 +1,5 @@
 import { assert, test } from 'vitest';
-import { negotiate } from './http.js';
+import { matches_content_type, negotiate } from './http.js';
 
 test('handle valid accept header value', () => {
 	const accept = 'text/html';
@@ -18,6 +18,12 @@ test('handle invalid accept header value', () => {
 	assert.equal(negotiate(accept, ['text/html']), 'text/html');
 });
 
+test('matches content types regardless of parameters and casing', () => {
+	assert.isTrue(matches_content_type('text/html; charset=utf-8', 'text/html'));
+	assert.isTrue(matches_content_type('TEXT/HTML ; charset=UTF-8', 'text/html'));
+	assert.isFalse(matches_content_type('text/html', 'text/plain'));
+});
+
 test('ignores an accept segment with no slash without catastrophic backtracking', () => {
 	assert.equal(negotiate('a'.repeat(200_000), ['text/html']), undefined);
 }, 100);

packages/kit/test/prerendering/basics/src/hooks.server.js

@@ -12,9 +12,16 @@ export const handle = async ({ event, resolve }) => {
 					.replace('__PRERENDERING__', String(building))
 		});
 	}
-	return await resolve(event, {
+
+	const response = await resolve(event, {
 		filterSerializedResponseHeaders: (name) => name === 'content-type'
 	});
+
+	if (event.url.pathname.startsWith('/content-type-charset')) {
+		response.headers.set('content-type', 'text/html; charset=utf-8');
+	}
+
+	return response;
 };
 
 // this code is here to make sure that we kill the process

packages/kit/test/prerendering/basics/src/routes/content-type-charset/+page.svelte

@@ -0,0 +1 @@
+<a href="/content-type-charset/dynamic">Please crawl this</a>

packages/kit/test/prerendering/basics/src/routes/content-type-charset/[dynamic]/+page.svelte

@@ -0,0 +1,3 @@
+<h1>
+	This page will only be discovered if pages whose content-type has a charset parameter are crawled
+</h1>

packages/kit/test/prerendering/basics/test/tests.spec.js

@@ -281,6 +281,11 @@ test('crawls links that start with config.paths.origin', () => {
 	expect(content).toBeTruthy();
 });
 
+test('crawls pages whose content-type has a charset parameter', () => {
+	expect(read('content-type-charset.html')).toBeTruthy();
+	expect(read('content-type-charset/dynamic.html')).toBeTruthy();
+});
+
 test('identifies missing ids', () => {
 	const missing_ids_file = fileURLToPath(new URL('../missing_ids/index.jsonl', import.meta.url));
 	const missing_ids_content = fs.readFileSync(missing_ids_file, 'utf-8');