perf(runtime): #6812 — object-owned overflow storage (spill buffer) replaces the TLS side table

changelog.d/6812-object-spill.md

@@ -0,0 +1 @@
+perf(runtime): #6812 — object-owned overflow storage. Properties past an object's inline slot capacity move from the thread-local side table (TLS fetch + RefCell + pointer-keyed hash probe per access; ~250B map+Vec per wide object, visited/re-keyed/finalized by every GC cycle) into a GC-traced buffer hung off the object's ObjectMeta record: reads are two dependent loads, stores are a raw slot write + layout note + generational barrier, and marking/evacuation/death ride the ordinary object graph (object → meta → buffer). Semantics (absolute field indexing, delete-compaction, enumeration) are unchanged through the same overflow_get/overflow_set entry points. Legacy side table kept one release behind PERRY_OBJECT_SPILL=0 for bisection.

crates/perry-runtime/src/gc/layout.rs

@@ -362,7 +362,9 @@ pub(super) unsafe fn layout_header_for_user(user_ptr: usize) -> Option<*mut GcHe
         GcLayoutSlotKind::ArrayElements
         | GcLayoutSlotKind::ObjectFields
         | GcLayoutSlotKind::ClosureCaptures => Some(header),
-        GcLayoutSlotKind::None => None,
+        // #6812: meta records keep no layout mask — their two child slots
+        // (prototype, spill) are enumerated unconditionally.
+        GcLayoutSlotKind::None | GcLayoutSlotKind::ObjectMeta => None,
     }
 }
 
@@ -1047,6 +1049,9 @@ pub(super) enum HeapPayloadSlotSelection {
 
 pub(crate) struct HeapChildSlotIterator {
     pub(super) prefix_slot: Option<*mut u64>,
+    /// #6812: second prefix — the object's `meta` header edge. Kept
+    /// separate from `prefix_slot` so payload indices stay mask-aligned.
+    pub(super) meta_slot: Option<*mut u64>,
     pub(super) payload: HeapSlotRange,
     pub(super) selection: HeapPayloadSlotSelection,
 }
@@ -1055,6 +1060,7 @@ impl HeapChildSlotIterator {
     pub(super) fn empty() -> Self {
         Self {
             prefix_slot: None,
+            meta_slot: None,
             payload: HeapSlotRange::new(std::ptr::null_mut(), 0),
             selection: HeapPayloadSlotSelection::Empty,
         }
@@ -1068,11 +1074,21 @@ impl HeapChildSlotIterator {
         let selection = unsafe { heap_payload_slot_selection(header, payload) };
         Self {
             prefix_slot,
+            meta_slot: None,
             payload,
             selection,
         }
     }
 
+    pub(super) fn with_meta_slot(mut self, slot: Option<*mut u64>) -> Self {
+        self.meta_slot = slot;
+        self
+    }
+
+    pub(super) fn take_meta_child_slot(&mut self) -> Option<*mut u64> {
+        self.meta_slot.take()
+    }
+
     pub(super) fn take_prefix_child_slot(&mut self) -> Option<*mut u64> {
         self.prefix_slot.take()
     }
@@ -1101,6 +1117,9 @@ impl Iterator for HeapChildSlotIterator {
         if let Some(slot) = self.prefix_slot.take() {
             return Some(HeapChildSlot::Child(slot, HeapChildSlotReadKind::Prefix));
         }
+        if let Some(slot) = self.meta_slot.take() {
+            return Some(HeapChildSlot::Child(slot, HeapChildSlotReadKind::Prefix));
+        }
         match &mut self.selection {
             HeapPayloadSlotSelection::Empty => None,
             HeapPayloadSlotSelection::PointerFree {
@@ -1246,7 +1265,23 @@ pub(super) unsafe fn gc_child_slots(header: *mut GcHeader) -> HeapChildSlotItera
                 return HeapChildSlotIterator::empty();
             };
             let keys_slot = crate::object::gc_keys_array_slot(obj);
+            // #6812: the meta record is a raw-pointer child edge; before the
+            // spill buffer it was enumerated only on the rewrite path, which
+            // left it invisible to MARKING (latent for custom prototypes,
+            // which are usually rooted elsewhere; fatal for the spill
+            // buffer, reachable through meta alone). A second prefix slot
+            // keeps payload slot indices aligned with the layout masks.
             HeapChildSlotIterator::new(header, keys_slot, range)
+                .with_meta_slot(crate::object::gc_object_meta_slot(user_ptr as usize))
+        }
+        GcLayoutSlotKind::ObjectMeta => {
+            // #6812: prototype (NaN-boxed / raw / sentinel) as the prefix
+            // slot, the raw spill-buffer pointer as a 1-slot range. Mirrors
+            // the rewrite descriptor arm — marking must see the same edges.
+            let meta = user_ptr as *mut crate::object::ObjectMeta;
+            let proto_slot = Some(&mut (*meta).prototype as *mut u64);
+            let range = HeapSlotRange::new(&mut (*meta).spill as *mut u64, 1);
+            HeapChildSlotIterator::new(header, proto_slot, range)
         }
         GcLayoutSlotKind::ClosureCaptures => {
             let closure = user_ptr as *mut crate::closure::ClosureHeader;
@@ -1324,6 +1359,9 @@ pub(super) unsafe fn visit_gc_layout_slot_descriptors(
     if let Some(slot) = child_slots.take_prefix_child_slot() {
         visit(fixed_slot(slot).with_layout(HeapChildSlotReadKind::Prefix));
     }
+    if let Some(slot) = child_slots.take_meta_child_slot() {
+        visit(fixed_slot(slot).with_layout(HeapChildSlotReadKind::Prefix));
+    }
 
     match child_slots.payload_scan() {
         HeapPayloadSlotScan::Empty => {}
@@ -1384,16 +1422,14 @@ pub(super) unsafe fn visit_gc_rewrite_slot_descriptors(
             visit_gc_layout_slot_descriptors(header, &mut visit);
         }
         GcRewriteDescriptorKind::Object => {
+            // #6759 Phase B / #6812: the per-object meta record is a raw-
+            // pointer child edge exactly like `keys_array`'s prefix slot.
+            // Since the child-slot iterator gained the meta second-prefix
+            // (so MARKING sees it too), the layout-descriptor visit below
+            // already emits it — no explicit `gc_object_meta_slot` visit
+            // here, or the rewrite pass would hand the same slot to the
+            // visitor twice and double-count in verification statistics.
             visit_gc_layout_slot_descriptors(header, &mut visit);
-            // #6759 Phase B: the per-object meta record is a GC allocation
-            // reachable ONLY through this header slot — a raw-pointer child
-            // edge exactly like `keys_array`'s prefix slot (marked live here,
-            // rewritten when the record itself is evacuated). The accessor
-            // returns `None` for RegExp headers, whose bytes at the meta
-            // offset are native data.
-            if let Some(slot) = crate::object::gc_object_meta_slot(user_ptr as usize) {
-                visit(fixed_slot(slot));
-            }
             crate::object::visit_overflow_field_slots_mut(user_ptr as usize, |slot| {
                 visit(fixed_slot(slot));
             });
@@ -1515,6 +1551,9 @@ pub(super) unsafe fn visit_gc_rewrite_slot_descriptors(
             // 0-unset sentinels, which the slot visitor ignores).
             let meta = user_ptr as *mut crate::object::ObjectMeta;
             visit(fixed_slot(&mut (*meta).prototype as *mut u64));
+            // #6812: the object-owned overflow buffer is a raw-pointer child
+            // edge (0 = none), traced and rewritten exactly like `prototype`.
+            visit(fixed_slot(&mut (*meta).spill as *mut u64));
         }
         GcRewriteDescriptorKind::Leaf => {}
     }

crates/perry-runtime/src/gc/tests/layout_trace.rs

@@ -1043,6 +1043,11 @@ fn test_heap_child_iterator_pointer_free_object_yields_no_child_slots() {
 fn test_layout_mask_overflow_fields_and_array_grow_transfer() {
     clear_marks();
     clear_mark_seeds();
+    // #6812 spill: overflow writes now allocate GC memory (meta record +
+    // spill buffer), so an automatic minor GC mid-build could move `obj`
+    // out from under this test's raw pointers. The test asserts layout and
+    // tracing, not move-resilience — pin the heap while building.
+    let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers();
 
     let child = crate::string::js_string_from_bytes(b"overflow-child".as_ptr(), 14) as *mut u8;
     let child_header = unsafe { header_from_user_ptr(child) };
@@ -1058,11 +1063,29 @@ fn test_layout_mask_overflow_fields_and_array_grow_transfer() {
         crate::object::js_object_set_field_by_name(obj, key, value);
     }
 
-    assert_eq!(test_layout_pointer_slot_count(obj as usize, 9), Some(1));
+    // #6812 spill: the k8 pointer lives in the object-owned spill buffer
+    // (owner inline slots hold only k0..k3 numerics), so the pointer-slot
+    // count moves from the owner's mask to the buffer's. Legacy mode keeps
+    // the original owner-mask expectation.
+    if crate::object::test_object_spill_enabled() {
+        let spill = crate::object::test_spill_buffer_addr(obj as usize);
+        assert_ne!(spill, 0, "overflow write must have created a spill buffer");
+        assert_eq!(test_layout_pointer_slot_count(spill, 9), Some(1));
+    } else {
+        assert_eq!(test_layout_pointer_slot_count(obj as usize, 9), Some(1));
+    }
     let valid_ptrs = build_valid_pointer_set();
     let mut worklist = Vec::new();
     unsafe {
         trace_object(obj as *mut u8, &valid_ptrs, &mut worklist);
+        // #6812 spill: the overflow value is no longer owner-adjacent — the
+        // chain is obj → meta record → spill buffer → child, so drain the
+        // worklist exactly like production marking does instead of relying
+        // on a single hop.
+        while let Some(queued) = worklist.pop() {
+            let user = (queued as *mut u8).add(crate::gc::GC_HEADER_SIZE);
+            trace_object(user, &valid_ptrs, &mut worklist);
+        }
     }
     unsafe {
         assert_ne!((*child_header).gc_flags & GC_FLAG_MARKED, 0);

crates/perry-runtime/src/gc/types.rs

@@ -107,6 +107,13 @@ pub(crate) enum GcLayoutSlotKind {
     ArrayElements,
     ObjectFields,
     ClosureCaptures,
+    /// #6812: ObjectMeta records carry two live edges — the custom
+    /// `[[Prototype]]` value and the raw spill-buffer pointer. Before the
+    /// spill buffer these were enumerated only on the REWRITE path, which
+    /// left them invisible to marking (latent for prototypes, which are
+    /// normally rooted elsewhere; fatal for the spill buffer, which is
+    /// reachable through meta alone).
+    ObjectMeta,
 }
 
 #[allow(dead_code)]
@@ -548,7 +555,7 @@ pub(super) static GC_TYPE_INFO_BY_ID: [Option<GcTypeInfo>; MALLOC_KIND_BUCKET_CO
         GcAllocationPolicy::Arena,
         true,
         GcRewriteDescriptorKind::ObjectMeta,
-        GcLayoutSlotKind::None,
+        GcLayoutSlotKind::ObjectMeta,
         // Movable: the owner's `meta` header slot is a raw-pointer child
         // edge (visited in the Object rewrite descriptor), so evacuation
         // rewrites it like any other reference — no address-keyed side
@@ -789,14 +796,22 @@ pub(crate) fn validate_gc_type_info(info: &GcTypeInfo) -> Result<(), &'static st
         | GcRewriteDescriptorKind::Error
         | GcRewriteDescriptorKind::Map
         | GcRewriteDescriptorKind::LazyArray
-        | GcRewriteDescriptorKind::Set
-        | GcRewriteDescriptorKind::ObjectMeta => {
+        | GcRewriteDescriptorKind::Set => {
             if info.layout_slot_kind != GcLayoutSlotKind::None {
                 return Err(
                     "external-backed rewrite descriptor must not expose payload layout slots",
                 );
             }
         }
+        GcRewriteDescriptorKind::ObjectMeta => {
+            // #6812: meta records expose their two child edges (prototype,
+            // spill buffer) to MARKING via GcLayoutSlotKind::ObjectMeta —
+            // the spill buffer is reachable through meta alone, so a
+            // rewrite-only descriptor would leave it invisible to liveness.
+            if info.layout_slot_kind != GcLayoutSlotKind::ObjectMeta {
+                return Err("object-meta descriptor must expose its child edges to marking");
+            }
+        }
         GcRewriteDescriptorKind::NativeTypedView | GcRewriteDescriptorKind::NativePodView => {
             if info.layout_slot_kind != GcLayoutSlotKind::None {
                 return Err("native view rewrite descriptor must use fixed slots only");