test(fieldNorm): cover full separator set; narrow "any whitespace" comment

Follow-up to #836. The isWordSeparator comment claimed "any whitespace
character" but the predicate only covers common ASCII whitespace plus NBSP,
not the full Unicode \s set (ideographic space, en/em spaces, U+2028/2029,
etc.). Narrow the wording so the scope reads as deliberate rather than an
oversight, and add assertions for CR, VT, FF, and NBSP to lock in the set
the predicate actually claims.

Author: krisk

src/tools/fieldNorm.ts

@@ -1,8 +1,10 @@
 import type { NormInterface } from '../types'
 
-// Whitespace charCodes treated as word separators by the counter below:
-// tab (9), newline (10), vertical tab (11), form feed (12), CR (13),
-// space (32), non-breaking space (160).
+// charCodes we treat as word separators: the common ASCII whitespace
+// (tab 9, LF 10, VT 11, FF 12, CR 13, space 32) plus NBSP (160). This is
+// deliberately narrower than the full Unicode whitespace set (\s); it skips
+// rarer separators like the ideographic space and en/em spaces, which the
+// field-norm heuristic is too coarse to benefit from distinguishing.
 function isWordSeparator(code: number): boolean {
   return (code >= 9 && code <= 13) || code === 32 || code === 160
 }
@@ -24,10 +26,10 @@ export default function norm(
       // transition-counter (starting at 1 and incrementing on every boundary)
       // would over-count by 1 for each stray boundary.
       //
-      // A separator is any whitespace character (space, tab, newline, CR,
-      // vertical tab, form feed, or non-breaking space) — not just plain
-      // ASCII space. Checking charCode 32 alone missed tabs and newlines, so
-      // e.g. a tab- or newline-joined field was scored as a single word.
+      // A separator here is common ASCII whitespace (space, tab, newline, CR,
+      // vertical tab, form feed) plus NBSP, not the full Unicode set. Checking
+      // charCode 32 alone missed tabs and newlines, so a tab- or newline-joined
+      // field was scored as a single word.
       let numTokens = 0
       let inWord = false
       for (let i = 0; i < value.length; i++) {

test/internals.test.ts

@@ -56,6 +56,16 @@ describe('fieldNorm', () => {
     expect(n.get('one\ttwo\tthree')).toBe(n.get('one two three'))
     expect(n.get('one\ntwo\nthree')).toBe(n.get('one two three'))
   })
+
+  test('covers the rest of the separator set: CR, VT, FF, and NBSP', () => {
+    const n = norm(1, 3)
+    // Lock in the full set isWordSeparator claims: carriage return, vertical
+    // tab, form feed, and non-breaking space all count as word boundaries.
+    expect(n.get('hello\rworld')).toBe(n.get('hello world'))
+    expect(n.get('hello\vworld')).toBe(n.get('hello world'))
+    expect(n.get('hello\fworld')).toBe(n.get('hello world'))
+    expect(n.get('hello world')).toBe(n.get('hello world'))
+  })
 })
 
 describe('InvertedIndex', () => {