diff --git a/changelog.d/6795-runtime-state-phase-a.md b/changelog.d/6795-runtime-state-phase-a.md new file mode 100644 index 0000000000..6e769d8f7a --- /dev/null +++ b/changelog.d/6795-runtime-state-phase-a.md @@ -0,0 +1 @@ +perf(runtime): #6759 Phase A — one heap-allocated per-thread `RuntimeState` behind a single const-init TLS pointer replaces 12 hot `thread_local!` object-op side tables (descriptor/accessor tables + gates, overflow fields + last-cache, keys-index sidecar, shape inline/overflow caches, transition cache, field cache, wide-key index). One TLS fetch per operation instead of one per table; isolation, lifetime, and borrow discipline unchanged; arm64_32 transition-cache boxing preserved. diff --git a/crates/perry-runtime/src/lib.rs b/crates/perry-runtime/src/lib.rs index 269fa7ad73..d0c6f360a1 100644 --- a/crates/perry-runtime/src/lib.rs +++ b/crates/perry-runtime/src/lib.rs @@ -126,6 +126,7 @@ pub mod regex; pub mod safe_area; pub mod set; pub mod shared_sab; +pub(crate) mod state; pub mod string; pub mod symbol; /// TC39 Temporal API (#4686): `Temporal.Duration`, `Temporal.Instant`, diff --git a/crates/perry-runtime/src/object/array_object_ops.rs b/crates/perry-runtime/src/object/array_object_ops.rs index 08baa77bff..eeb4092c68 100644 --- a/crates/perry-runtime/src/object/array_object_ops.rs +++ b/crates/perry-runtime/src/object/array_object_ops.rs @@ -545,9 +545,11 @@ pub(crate) unsafe fn define_array_property( // Redefining an index that was previously an accessor back to a data // property: drop the stale accessor entry. - ACCESSOR_DESCRIPTORS.with(|m| { - m.borrow_mut().remove(&(obj as usize, key_name.to_string())); - }); + crate::state::state() + .descriptors + .accessor_descriptors + .borrow_mut() + .remove(&(obj as usize, key_name.to_string())); // [[DefineOwnProperty]] writes the slot directly — clear any stale // attrs first so the extend helper's [[Set]]-side writability check // (added for ordinary `arr[i] = v` writes) can't reject this store. @@ -726,9 +728,11 @@ pub(crate) unsafe fn define_array_property( // Redefining a former accessor back to a data property drops the stale // accessor entry (the non-configurable case already threw above). if cur_accessor.is_some() { - ACCESSOR_DESCRIPTORS.with(|m| { - m.borrow_mut().remove(&(obj as usize, key_name.to_string())); - }); + crate::state::state() + .descriptors + .accessor_descriptors + .borrow_mut() + .remove(&(obj as usize, key_name.to_string())); } // Write the value: an explicit `value` wins; a NEW property with no value diff --git a/crates/perry-runtime/src/object/class_registry/prototype_objects.rs b/crates/perry-runtime/src/object/class_registry/prototype_objects.rs index 335d3e53c1..14460033f3 100644 --- a/crates/perry-runtime/src/object/class_registry/prototype_objects.rs +++ b/crates/perry-runtime/src/object/class_registry/prototype_objects.rs @@ -314,7 +314,7 @@ unsafe fn inherited_proto_accessor_value( key: *const crate::StringHeader, receiver: f64, ) -> Option { - if key.is_null() || !ACCESSORS_IN_USE.with(|c| c.get()) { + if key.is_null() || !crate::state::state().descriptors.accessors_in_use.get() { return None; } let key_ptr = (key as *const u8).add(std::mem::size_of::()); diff --git a/crates/perry-runtime/src/object/delete_rest.rs b/crates/perry-runtime/src/object/delete_rest.rs index 71bcb1f89b..ec1399bcd5 100644 --- a/crates/perry-runtime/src/object/delete_rest.rs +++ b/crates/perry-runtime/src/object/delete_rest.rs @@ -380,9 +380,11 @@ pub extern "C" fn js_object_delete_field( // slot map is now stale (entries past `i` have shifted). // The next lookup at threshold will rebuild from current // keys_array. - KEYS_INDEX.with(|m| { - m.borrow_mut().remove(&(obj as usize)); - }); + crate::state::state() + .object_hot + .keys_index + .borrow_mut() + .remove(&(obj as usize)); 1 } diff --git a/crates/perry-runtime/src/object/descriptor_state.rs b/crates/perry-runtime/src/object/descriptor_state.rs index 61dd65d1c7..2e6a67a4f9 100644 --- a/crates/perry-runtime/src/object/descriptor_state.rs +++ b/crates/perry-runtime/src/object/descriptor_state.rs @@ -4,6 +4,7 @@ use super::*; use crate::fast_hash::{new_fast_key_hash_map, FastKeyHashMap}; +use crate::state::state; use std::cell::{Cell, RefCell}; use std::collections::HashMap; use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; @@ -47,12 +48,46 @@ impl PropertyAttrs { } } -thread_local! { - // Hasher: `FastKeyHasher` (FNV-1a) rather than std's SipHash `RandomState`. - // The key is `(owner_addr, key_string)` — a runtime heap pointer plus a - // program-supplied property name, so no external input reaches it and - // DoS-resistant hashing buys nothing on this hot property-access path. - pub(crate) static PROPERTY_DESCRIPTORS: RefCell> = RefCell::new(new_fast_key_hash_map()); +/// #6759 Phase A: the descriptor side tables and their per-thread fast-path +/// gates, grouped as the `descriptors` field of +/// [`crate::state::RuntimeState`]. Previously four separate `thread_local!`s; +/// reach them via `crate::state::state().descriptors` (one TLS fetch for the +/// whole group). +pub(crate) struct DescriptorTables { + /// Per-property attribute flags set by `Object.defineProperty` / + /// `Object.freeze` / `Object.seal`, keyed `(owner_addr, key_string)`. + /// + /// Hasher: `FastKeyHasher` (FNV-1a) rather than std's SipHash + /// `RandomState`. The key is a runtime heap pointer plus a + /// program-supplied property name, so no external input reaches it and + /// DoS-resistant hashing buys nothing on this hot property-access path. + pub(crate) property_descriptors: RefCell>, + /// Accessor descriptor storage: maps `(owner_addr, key_string)` to the + /// getter/setter closure bits. Same hasher rationale as + /// `property_descriptors`. + pub(crate) accessor_descriptors: RefCell>, + /// Fast-path gate: `false` when no accessor descriptors have ever been + /// installed on this thread, so hot `js_object_get_field_by_name` / + /// `set_field_by_name` can skip the `accessor_descriptors` HashMap + /// lookup entirely. + pub(crate) accessors_in_use: Cell, + /// Fast-path gate for `property_descriptors` — flipped the first time + /// `Object.defineProperty` (or freeze/seal via `set_property_attrs`) + /// installs a per-property descriptor. Lets the hot object-write path + /// skip the `.to_string()` allocation required to look up a descriptor + /// that almost never exists. + pub(crate) property_attrs_in_use: Cell, +} + +impl DescriptorTables { + pub(crate) fn new() -> Self { + DescriptorTables { + property_descriptors: RefCell::new(new_fast_key_hash_map()), + accessor_descriptors: RefCell::new(new_fast_key_hash_map()), + accessors_in_use: Cell::new(false), + property_attrs_in_use: Cell::new(false), + } + } } /// Accessor descriptor storage: maps (obj_ptr, key) -> (get_closure_bits, set_closure_bits). @@ -66,22 +101,6 @@ pub(crate) struct AccessorDescriptor { pub set: u64, // NaN-boxed closure f64 bits, 0 = absent } -thread_local! { - // Hasher: `FastKeyHasher` (FNV-1a); see `PROPERTY_DESCRIPTORS` above for the - // same-shape `(owner_addr, key_string)` key and rationale. - pub(crate) static ACCESSOR_DESCRIPTORS: RefCell> = RefCell::new(new_fast_key_hash_map()); - /// Fast-path gate: `false` when no accessor descriptors have ever been installed - /// on this thread, so hot `js_object_get_field_by_name` / `set_field_by_name` - /// can skip the `ACCESSOR_DESCRIPTORS` HashMap lookup entirely. - pub(crate) static ACCESSORS_IN_USE: Cell = const { Cell::new(false) }; - /// Fast-path gate for `PROPERTY_DESCRIPTORS` — flipped the first time - /// `Object.defineProperty` (or freeze/seal via `set_property_attrs`) - /// installs a per-property descriptor. Lets the hot object-write path - /// skip the `.to_string()` allocation required to look up a descriptor - /// that almost never exists. - pub(crate) static PROPERTY_ATTRS_IN_USE: Cell = const { Cell::new(false) }; -} - /// Global monotonic flag: set once any accessor or property descriptor is /// installed. Checked on every dynamic property write via a single /// `Relaxed` load (no TLS overhead, no fence on aarch64/x86). @@ -319,7 +338,12 @@ pub(crate) fn note_descriptor_target(obj: usize) { /// Look up the property descriptor for (obj, key). Returns None if no entry exists, /// in which case the JS default `{ writable: true, enumerable: true, configurable: true }` applies. pub(crate) fn get_property_attrs(obj: usize, key: &str) -> Option { - PROPERTY_DESCRIPTORS.with(|m| m.borrow().get(&(obj, key.to_string())).copied()) + state() + .descriptors + .property_descriptors + .borrow() + .get(&(obj, key.to_string())) + .copied() } /// Whether this specific object has ever had a property descriptor installed on @@ -416,38 +440,47 @@ pub(crate) unsafe fn plain_data_write_may_intercept(addr: usize, class_id: u32, pub(crate) fn set_property_attrs(obj: usize, key: String, attrs: PropertyAttrs) { super::prop_plan::prop_plan_epoch_bump(); note_descriptor_target(obj); - PROPERTY_ATTRS_IN_USE.with(|c| c.set(true)); + let st = state(); + st.descriptors.property_attrs_in_use.set(true); GLOBAL_DESCRIPTORS_IN_USE.store(true, Ordering::Relaxed); disable_class_field_inline_guard_for_target(obj); - PROPERTY_DESCRIPTORS.with(|m| { - m.borrow_mut().insert((obj, key), attrs); - }); + st.descriptors + .property_descriptors + .borrow_mut() + .insert((obj, key), attrs); } /// Remove a customized property descriptor for (obj, key), restoring default /// data-property attributes for subsequent writes and reflection. pub(crate) fn clear_property_attrs(obj: usize, key: &str) { super::prop_plan::prop_plan_epoch_bump(); - PROPERTY_DESCRIPTORS.with(|m| { - m.borrow_mut().remove(&(obj, key.to_string())); - }); + state() + .descriptors + .property_descriptors + .borrow_mut() + .remove(&(obj, key.to_string())); } /// Look up the accessor descriptor (get/set) for (obj, key). pub(crate) fn get_accessor_descriptor(obj: usize, key: &str) -> Option { - ACCESSOR_DESCRIPTORS.with(|m| m.borrow().get(&(obj, key.to_string())).copied()) + state() + .descriptors + .accessor_descriptors + .borrow() + .get(&(obj, key.to_string())) + .copied() } pub(crate) fn accessor_descriptor_keys_for_obj(obj: usize) -> Vec { - ACCESSOR_DESCRIPTORS.with(|m| { - let mut keys = m - .borrow() - .keys() - .filter_map(|(owner, key)| (*owner == obj).then(|| key.clone())) - .collect::>(); - keys.sort(); - keys - }) + let mut keys = state() + .descriptors + .accessor_descriptors + .borrow() + .keys() + .filter_map(|(owner, key)| (*owner == obj).then(|| key.clone())) + .collect::>(); + keys.sort(); + keys } /// #2766: resolve an accessor *getter* closure for `(value, key)` if one is @@ -458,7 +491,7 @@ pub(crate) fn accessor_descriptor_keys_for_obj(obj: usize) -> Vec { /// invoking it. Returns `None` (rather than reading the field) when there is no /// accessor at all, so the caller falls back to an ordinary field read. pub(crate) fn reflect_getter_closure_bits(value: f64, key: f64) -> Option { - if !ACCESSORS_IN_USE.with(|c| c.get()) { + if !state().descriptors.accessors_in_use.get() { return None; } let key_str = crate::builtins::js_string_coerce(key); @@ -567,22 +600,26 @@ fn note_accessor_descriptor_key(key: &str) { pub(crate) fn set_accessor_descriptor(obj: usize, key: String, acc: AccessorDescriptor) { super::prop_plan::prop_plan_epoch_bump(); note_descriptor_target(obj); - ACCESSORS_IN_USE.with(|c| c.set(true)); + let st = state(); + st.descriptors.accessors_in_use.set(true); GLOBAL_DESCRIPTORS_IN_USE.store(true, Ordering::Relaxed); disable_class_field_inline_guard_for_target(obj); note_accessor_descriptor_key(&key); - ACCESSOR_DESCRIPTORS.with(|m| { - m.borrow_mut().insert((obj, key), acc); - }); + st.descriptors + .accessor_descriptors + .borrow_mut() + .insert((obj, key), acc); } /// Remove an accessor descriptor for (obj, key), letting ordinary data-property /// reads and writes use the object's stored field again. pub(crate) fn clear_accessor_descriptor(obj: usize, key: &str) { super::prop_plan::prop_plan_epoch_bump(); - ACCESSOR_DESCRIPTORS.with(|m| { - m.borrow_mut().remove(&(obj, key.to_string())); - }); + state() + .descriptors + .accessor_descriptors + .borrow_mut() + .remove(&(obj, key.to_string())); } /// Install a built-in *reflection-only* accessor descriptor for (obj, key) @@ -607,12 +644,15 @@ pub(crate) fn set_builtin_accessor_descriptor( ) { super::prop_plan::prop_plan_epoch_bump(); note_accessor_descriptor_key(&key); - ACCESSOR_DESCRIPTORS.with(|m| { - m.borrow_mut().insert((obj, key.clone()), acc); - }); - PROPERTY_DESCRIPTORS.with(|m| { - m.borrow_mut().insert((obj, key), attrs); - }); + let st = state(); + st.descriptors + .accessor_descriptors + .borrow_mut() + .insert((obj, key.clone()), acc); + st.descriptors + .property_descriptors + .borrow_mut() + .insert((obj, key), attrs); } /// Install a built-in *reflection-only* data-property descriptor for (obj, key) @@ -633,9 +673,11 @@ pub(crate) fn set_builtin_accessor_descriptor( pub(crate) fn set_builtin_property_attrs(obj: usize, key: String, attrs: PropertyAttrs) { super::prop_plan::prop_plan_epoch_bump(); note_descriptor_target(obj); - PROPERTY_DESCRIPTORS.with(|m| { - m.borrow_mut().insert((obj, key), attrs); - }); + state() + .descriptors + .property_descriptors + .borrow_mut() + .insert((obj, key), attrs); } /// Walk the keys array of `obj` and apply the given attribute mask AND filter to every existing key. @@ -703,18 +745,19 @@ pub(crate) fn prune_dead_descriptor_owner_entries(is_dead_owner: &dyn Fn(usize) .entry(owner) .or_insert_with(|| is_dead_owner(owner)) }; - PROPERTY_DESCRIPTORS.with(|m| { - let mut m = m.borrow_mut(); + let st = state(); + { + let mut m = st.descriptors.property_descriptors.borrow_mut(); if !m.is_empty() { m.retain(|(owner, _), _| !is_dead(*owner)); } - }); - ACCESSOR_DESCRIPTORS.with(|m| { - let mut m = m.borrow_mut(); + } + { + let mut m = st.descriptors.accessor_descriptors.borrow_mut(); if !m.is_empty() { m.retain(|(owner, _), _| !is_dead(*owner)); } - }); + } } /// #6710: drop every property-attr + accessor descriptor owned by `obj`. @@ -731,18 +774,19 @@ pub(crate) fn clear_object_descriptors(obj: usize) { if !HANDLE_HAS_DESCRIPTORS.load(Ordering::Relaxed) { return; } - PROPERTY_DESCRIPTORS.with(|m| { - let mut m = m.borrow_mut(); + let st = state(); + { + let mut m = st.descriptors.property_descriptors.borrow_mut(); if !m.is_empty() { m.retain(|(owner, _), _| *owner != obj); } - }); - ACCESSOR_DESCRIPTORS.with(|m| { - let mut m = m.borrow_mut(); + } + { + let mut m = st.descriptors.accessor_descriptors.borrow_mut(); if !m.is_empty() { m.retain(|(owner, _), _| *owner != obj); } - }); + } } /// Rewrite a descriptor table's owner ADDRESS during the GC metadata-rewrite @@ -771,8 +815,9 @@ fn rewrite_descriptor_owner( /// attrs and accessors don't silently detach (or fire on a new tenant at a /// reused address). pub(crate) fn scan_descriptor_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { - PROPERTY_DESCRIPTORS.with(|descriptors| { - let mut descriptors = descriptors.borrow_mut(); + let st = state(); + { + let mut descriptors = st.descriptors.property_descriptors.borrow_mut(); let needs_rebuild = descriptors .keys() .any(|(owner, _)| rewrite_descriptor_owner(visitor, *owner) != *owner); @@ -783,10 +828,10 @@ pub(crate) fn scan_descriptor_roots_mut(visitor: &mut crate::gc::RuntimeRootVisi descriptors.insert((owner, key), attrs); } } - }); + } - ACCESSOR_DESCRIPTORS.with(|descriptors| { - let mut descriptors = descriptors.borrow_mut(); + { + let mut descriptors = st.descriptors.accessor_descriptors.borrow_mut(); let needs_rebuild = descriptors .keys() .any(|(owner, _)| rewrite_descriptor_owner(visitor, *owner) != *owner); @@ -812,5 +857,5 @@ pub(crate) fn scan_descriptor_roots_mut(visitor: &mut crate::gc::RuntimeRootVisi } } } - }); + } } diff --git a/crates/perry-runtime/src/object/field_get_set.rs b/crates/perry-runtime/src/object/field_get_set.rs index 84bee096f9..a635a603f5 100644 --- a/crates/perry-runtime/src/object/field_get_set.rs +++ b/crates/perry-runtime/src/object/field_get_set.rs @@ -128,6 +128,34 @@ mod has_property; mod ic_miss; mod map_set_receiver; +/// Size of the direct-mapped `(keys_ptr, key_hash, field_index)` inline +/// cache backing `js_object_get_field_by_name`'s slow tail. +pub(crate) const FIELD_CACHE_SIZE: usize = 1024; + +/// #6759 Phase A: property-lookup inline caches, grouped as the +/// `field_lookup` field of [`crate::state::RuntimeState`]. Previously a +/// function-local `thread_local!` in `get_field_by_name_tail` plus a module +/// `thread_local!` in `has_property`; reach them via +/// `crate::state::state().field_lookup`. +pub(crate) struct FieldLookupCaches { + /// Fixed-size direct-mapped cache (no allocation, no HashMap): each + /// entry stores `(keys_ptr, key_hash, field_index)`. Copied-minor + /// nursery reset can reuse a keys-array address, so cache hits still + /// validate the key slot before returning a field. + pub(crate) field_cache: std::cell::UnsafeCell<[(usize, u32, u32); FIELD_CACHE_SIZE]>, + /// #5054 wide-object key→index map entries (see `has_property.rs`). + pub(crate) wide_key_index: std::cell::RefCell>, +} + +impl FieldLookupCaches { + pub(crate) fn new() -> Self { + FieldLookupCaches { + field_cache: std::cell::UnsafeCell::new([(0usize, 0u32, 0u32); FIELD_CACHE_SIZE]), + wide_key_index: std::cell::RefCell::new(Vec::new()), + } + } +} + // Explicit named re-exports so existing `crate::object::…` / `super::…` // paths keep resolving (a glob re-export does not reliably propagate through // `object/mod.rs`'s `pub use field_get_set::*`), and so sibling modules can diff --git a/crates/perry-runtime/src/object/field_get_set/accessors.rs b/crates/perry-runtime/src/object/field_get_set/accessors.rs index cb8ba19aae..df5d159f5f 100644 --- a/crates/perry-runtime/src/object/field_get_set/accessors.rs +++ b/crates/perry-runtime/src/object/field_get_set/accessors.rs @@ -387,7 +387,7 @@ pub(crate) unsafe fn primitive_object_prototype_accessor( name: &str, receiver: f64, ) -> Option { - if !ACCESSORS_IN_USE.with(|c| c.get()) { + if !crate::state::state().descriptors.accessors_in_use.get() { return None; } let object_ctor = super::super::js_get_global_this_builtin_value(b"Object".as_ptr(), 6); @@ -449,7 +449,7 @@ pub(crate) unsafe fn primitive_builtin_prototype_property( // with the ORIGINAL primitive receiver — boxed/raw per getter strictness // inside `invoke_accessor_getter` — not the prototype object the accessor // happens to live on (which a plain field read below would hand it). - if ACCESSORS_IN_USE.with(|c| c.get()) { + if crate::state::state().descriptors.accessors_in_use.get() { let key_ptr = (key as *const u8).add(std::mem::size_of::()); let key_len = (*key).byte_len as usize; if let Ok(name) = std::str::from_utf8(std::slice::from_raw_parts(key_ptr, key_len)) { diff --git a/crates/perry-runtime/src/object/field_get_set/enumeration.rs b/crates/perry-runtime/src/object/field_get_set/enumeration.rs index 09f2315839..2594e1a503 100644 --- a/crates/perry-runtime/src/object/field_get_set/enumeration.rs +++ b/crates/perry-runtime/src/object/field_get_set/enumeration.rs @@ -911,8 +911,12 @@ pub extern "C" fn js_object_keys(obj: *const ObjectHeader) -> *mut ArrayHeader { // actually has descriptor entries, so the common all-default // array stays on the fast path. let owner = stripped as usize; - let has_idx_descriptors = - PROPERTY_DESCRIPTORS.with(|m| m.borrow().keys().any(|(ptr, _)| *ptr == owner)); + let has_idx_descriptors = crate::state::state() + .descriptors + .property_descriptors + .borrow() + .keys() + .any(|(ptr, _)| *ptr == owner); let result = crate::array::js_array_alloc(length); for i in 0..length { if std::ptr::read(elements.add(i as usize)) == crate::value::TAG_HOLE { @@ -982,8 +986,12 @@ pub extern "C" fn js_object_keys(obj: *const ObjectHeader) -> *mut ArrayHeader { // the wrong slot's value. The slow path below already builds a // fresh array; the fast path now mirrors it, just without the // per-key descriptor check. - let has_descriptors = - PROPERTY_DESCRIPTORS.with(|m| m.borrow().keys().any(|(ptr, _)| *ptr == obj as usize)); + let has_descriptors = crate::state::state() + .descriptors + .property_descriptors + .borrow() + .keys() + .any(|(ptr, _)| *ptr == obj as usize); let len = crate::array::js_array_length(keys) as usize; // #2438: enumerate in ECMA-262 OrdinaryOwnPropertyKeys order — // array-index keys first (ascending numeric), then string keys in diff --git a/crates/perry-runtime/src/object/field_get_set/field_ops.rs b/crates/perry-runtime/src/object/field_get_set/field_ops.rs index e62e702078..07b15cc0c6 100644 --- a/crates/perry-runtime/src/object/field_get_set/field_ops.rs +++ b/crates/perry-runtime/src/object/field_get_set/field_ops.rs @@ -341,7 +341,7 @@ pub extern "C" fn js_object_set_field_by_index( // address must not pick up the previous tenant's stale accessor // (it would silently drop `obj.k = v` for a getter-only stale // entry). A fresh allocation has the flag clear. - if ACCESSORS_IN_USE.with(|c| c.get()) + if crate::state::state().descriptors.accessors_in_use.get() && super::super::object_has_descriptors(obj as usize) { if let Some(acc) = get_accessor_descriptor(obj as usize, name) { diff --git a/crates/perry-runtime/src/object/field_get_set/get_field_by_name_tail.rs b/crates/perry-runtime/src/object/field_get_set/get_field_by_name_tail.rs index 585ff67046..0da0ca1d2d 100644 --- a/crates/perry-runtime/src/object/field_get_set/get_field_by_name_tail.rs +++ b/crates/perry-runtime/src/object/field_get_set/get_field_by_name_tail.rs @@ -915,7 +915,7 @@ pub(crate) fn get_field_by_name_object_tail( } if let Ok(name) = std::str::from_utf8(key_bytes) { if let Some(index) = super::super::canonical_array_index(name) { - if ACCESSORS_IN_USE.with(|c| c.get()) { + if crate::state::state().descriptors.accessors_in_use.get() { if let Some(acc) = get_accessor_descriptor(obj as usize, name) { if acc.get != 0 { let receiver = crate::value::js_nanbox_pointer(obj as i64); @@ -936,7 +936,7 @@ pub(crate) fn get_field_by_name_object_tail( } // Named (non-index) accessor installed via // `Object.defineProperty(arr, "prop", {get,set})`. - if ACCESSORS_IN_USE.with(|c| c.get()) { + if crate::state::state().descriptors.accessors_in_use.get() { if let Some(acc) = get_accessor_descriptor(obj as usize, name) { if acc.get != 0 { let receiver = crate::value::js_nanbox_pointer(obj as i64); @@ -1532,25 +1532,22 @@ pub(crate) fn get_field_by_name_object_tail( // arrays. let key_count = crate::array::keys_array_len_capped_to_capacity(keys); - // Thread-local inline cache: fixed-size direct-mapped cache (no allocation, no HashMap) + // Per-thread inline cache (`state().field_lookup.field_cache`): + // fixed-size direct-mapped cache (no allocation, no HashMap). // Each entry stores (keys_ptr, key_hash, field_index). Copied-minor // nursery reset can reuse a keys-array address, so cache hits still // validate the key slot before returning a field. - const FIELD_CACHE_SIZE: usize = 1024; - thread_local! { - static FIELD_CACHE: std::cell::UnsafeCell<[(usize, u32, u32); FIELD_CACHE_SIZE]> = - const { std::cell::UnsafeCell::new([(0usize, 0u32, 0u32); FIELD_CACHE_SIZE]) }; - } - let cache_idx = (keys_id.wrapping_add(key_hash as usize)) % FIELD_CACHE_SIZE; - let cached = FIELD_CACHE.with(|c| { - let cache = &*c.get(); + let st = crate::state::state(); + let cache_idx = (keys_id.wrapping_add(key_hash as usize)) % super::FIELD_CACHE_SIZE; + let cached = { + let cache = &*st.field_lookup.field_cache.get(); let entry = cache[cache_idx]; if entry.0 == keys_id && entry.1 == key_hash { Some(entry.2) } else { None } - }); + }; if let Some(field_idx) = cached { let idx = field_idx as usize; let cache_hit_valid = if idx < key_count { @@ -1563,17 +1560,15 @@ pub(crate) fn get_field_by_name_object_tail( false }; if !cache_hit_valid { - FIELD_CACHE.with(|c| { - let cache = &mut *c.get(); - cache[cache_idx] = (0, 0, 0); - }); + let cache = &mut *st.field_lookup.field_cache.get(); + cache[cache_idx] = (0, 0, 0); } else { // Accessor short-circuit: if this (obj, key) has a getter installed, // invoke it instead of reading the slot. The `ACCESSORS_IN_USE` // thread-local gate keeps this off the hot path in the common case; // the per-object flag gate avoids invoking a stale getter left by a // freed object whose address this fresh object reused. - if ACCESSORS_IN_USE.with(|c| c.get()) + if crate::state::state().descriptors.accessors_in_use.get() && super::super::object_has_descriptors(obj as usize) { if let Ok(name) = std::str::from_utf8(key_bytes) { @@ -1602,7 +1597,7 @@ pub(crate) fn get_field_by_name_object_tail( // linear scan below (the index is an accelerator, not authoritative). if key_count >= WIDE_KEY_INDEX_MIN_KEYS { if let Some(i) = wide_key_index_lookup(keys_id, key_bytes, key, keys, key_count) { - if ACCESSORS_IN_USE.with(|c| c.get()) + if crate::state::state().descriptors.accessors_in_use.get() && super::super::object_has_descriptors(obj as usize) { if let Ok(name) = std::str::from_utf8(key_bytes) { @@ -1637,15 +1632,15 @@ pub(crate) fn get_field_by_name_object_tail( // keys after a field-cache miss. if crate::string::js_string_key_matches(key_val, key) { // Cache this lookup for next time - FIELD_CACHE.with(|c| { - let cache = &mut *c.get(); + { + let cache = &mut *st.field_lookup.field_cache.get(); cache[cache_idx] = (keys_id, key_hash, i as u32); - }); + } if key_count >= WIDE_KEY_INDEX_MIN_KEYS { wide_key_index_note_hit(keys_id, key_bytes, i as u32); } // Accessor short-circuit (see fast path above). - if ACCESSORS_IN_USE.with(|c| c.get()) + if crate::state::state().descriptors.accessors_in_use.get() && super::super::object_has_descriptors(obj as usize) { if let Ok(name) = std::str::from_utf8(key_bytes) { diff --git a/crates/perry-runtime/src/object/field_get_set/has_property.rs b/crates/perry-runtime/src/object/field_get_set/has_property.rs index 90f8860f0b..610d6ea9eb 100644 --- a/crates/perry-runtime/src/object/field_get_set/has_property.rs +++ b/crates/perry-runtime/src/object/field_get_set/has_property.rs @@ -1060,7 +1060,7 @@ unsafe fn ordinary_has_property( // EVERY `in`, dominating its profile (~60% of samples on a // descriptor-less receiver). if let Some(name) = key_name { - if crate::object::descriptor_state::ACCESSORS_IN_USE.with(|c| c.get()) + if crate::state::state().descriptors.accessors_in_use.get() && crate::object::descriptor_state::object_has_descriptors(cur as usize) && get_accessor_descriptor(cur as usize, name).is_some() { @@ -1276,16 +1276,13 @@ pub(crate) unsafe fn native_module_own_field_by_key( pub(crate) const WIDE_KEY_INDEX_MIN_KEYS: usize = 257; const WIDE_KEY_INDEX_CAPACITY: usize = 4; -struct WideKeyIndexEntry { +pub(crate) struct WideKeyIndexEntry { keys_id: usize, indexed_len: u32, map: std::collections::HashMap, u32>, } -thread_local! { - static WIDE_KEY_INDEX: std::cell::RefCell> = - const { std::cell::RefCell::new(Vec::new()) }; -} +// Storage: `FieldLookupCaches::wide_key_index` (`state().field_lookup`). /// Probe the wide-object index for `key_bytes` in the keys array identified by /// `keys_id`. Returns a slot index whose stored key has been re-validated @@ -1298,8 +1295,11 @@ pub(crate) unsafe fn wide_key_index_lookup( keys: *const crate::array::ArrayHeader, key_count: usize, ) -> Option { - WIDE_KEY_INDEX.with(|cell| { - let mut table = cell.borrow_mut(); + { + let mut table = crate::state::state() + .field_lookup + .wide_key_index + .borrow_mut(); let pos = table.iter().position(|e| e.keys_id == keys_id); let pos = match pos { Some(p) => p, @@ -1372,16 +1372,17 @@ pub(crate) unsafe fn wide_key_index_lookup( } _ => None, } - }) + } } /// Back-fill a linear-scan hit into the wide-object index (no-op when the /// keys array has no entry — the next lookup builds it wholesale). pub(crate) fn wide_key_index_note_hit(keys_id: usize, key_bytes: &[u8], index: u32) { - WIDE_KEY_INDEX.with(|cell| { - let mut table = cell.borrow_mut(); - if let Some(e) = table.iter_mut().find(|e| e.keys_id == keys_id) { - e.map.entry(key_bytes.to_vec()).or_insert(index); - } - }); + let mut table = crate::state::state() + .field_lookup + .wide_key_index + .borrow_mut(); + if let Some(e) = table.iter_mut().find(|e| e.keys_id == keys_id) { + e.map.entry(key_bytes.to_vec()).or_insert(index); + } } diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs index 428b46280a..cdcb5b6947 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs @@ -382,7 +382,7 @@ pub extern "C" fn js_object_get_field_ic_miss( // would silently return the raw slot value instead of calling the // getter. The slow path through js_object_get_field_by_name handles // accessors correctly. - let can_cache = !ACCESSORS_IN_USE.with(|c| c.get()); + let can_cache = !crate::state::state().descriptors.accessors_in_use.get(); unsafe { // Issue #72: validate this really is a GC_TYPE_OBJECT before reading // (*obj).keys_array — otherwise an Array/String/Buffer/etc. receiver diff --git a/crates/perry-runtime/src/object/field_set_by_name.rs b/crates/perry-runtime/src/object/field_set_by_name.rs index 490a9328fa..d1731b493f 100644 --- a/crates/perry-runtime/src/object/field_set_by_name.rs +++ b/crates/perry-runtime/src/object/field_set_by_name.rs @@ -907,7 +907,7 @@ pub extern "C" fn js_object_set_field_by_name( // fresh array reusing a freed address (its `_reserved` zeroed at // allocation) skips this lookup and can't fire a previous tenant's // stale accessor. - if ACCESSORS_IN_USE.with(|c| c.get()) + if crate::state::state().descriptors.accessors_in_use.get() && (*gc_header)._reserved & crate::gc::OBJ_FLAG_ARRAY_DESCRIPTORS != 0 { if let Some(acc) = get_accessor_descriptor(obj as usize, name) { @@ -1351,7 +1351,11 @@ pub extern "C" fn js_object_set_field_by_name( // honored), so the descriptor key string can never be consulted: skip // the per-store String allocation entirely. let needs_descriptor_key = !plan_fast - && (ACCESSORS_IN_USE.with(|c| c.get()) || PROPERTY_ATTRS_IN_USE.with(|c| c.get())); + && (crate::state::state().descriptors.accessors_in_use.get() + || crate::state::state() + .descriptors + .property_attrs_in_use + .get()); let incoming_key_str: Option = if needs_descriptor_key && !key.is_null() { let name_ptr = (key as *const u8).add(std::mem::size_of::()); let name_len = (*key).byte_len as usize; @@ -1379,7 +1383,7 @@ pub extern "C" fn js_object_set_field_by_name( // app-page-turbo runtime's `exports.Fragment = …`). A fresh allocation // has the flag clear, so it skips the stale lookup entirely. if !plan_fast - && ACCESSORS_IN_USE.with(|c| c.get()) + && crate::state::state().descriptors.accessors_in_use.get() && super::object_has_descriptors(obj as usize) { if let Some(ref k) = incoming_key_str { @@ -1585,7 +1589,10 @@ pub extern "C" fn js_object_set_field_by_name( // read only property" on a plain `{}` (Next.js app-page-turbo // runtime's `exports.Fragment = …`). A fresh allocation has the // flag clear, so it skips the lookup entirely. - if PROPERTY_ATTRS_IN_USE.with(|c| c.get()) + if crate::state::state() + .descriptors + .property_attrs_in_use + .get() && super::object_has_descriptors(obj as usize) { if let Some(ref k) = incoming_key_str { diff --git a/crates/perry-runtime/src/object/handle_expando.rs b/crates/perry-runtime/src/object/handle_expando.rs index c838b0099d..edcd06c2a4 100644 --- a/crates/perry-runtime/src/object/handle_expando.rs +++ b/crates/perry-runtime/src/object/handle_expando.rs @@ -33,10 +33,7 @@ //! every phase (keeping e.g. a stored array and its elements alive) and rewrites //! the stored bits when a copying collection moves the value. -use super::descriptor_state::{ - get_accessor_descriptor, get_property_attrs, PropertyAttrs, ACCESSOR_DESCRIPTORS, - PROPERTY_DESCRIPTORS, -}; +use super::descriptor_state::{get_accessor_descriptor, get_property_attrs, PropertyAttrs}; use std::cell::RefCell; use std::collections::HashMap; @@ -231,12 +228,15 @@ pub(crate) fn handle_expando_delete(handle: i64, name: &str) -> bool { props.retain(|(k, _)| k != name); } }); - PROPERTY_DESCRIPTORS.with(|m| { - m.borrow_mut().remove(&(handle as usize, name.to_string())); - }); - ACCESSOR_DESCRIPTORS.with(|m| { - m.borrow_mut().remove(&(handle as usize, name.to_string())); - }); + let st = crate::state::state(); + st.descriptors + .property_descriptors + .borrow_mut() + .remove(&(handle as usize, name.to_string())); + st.descriptors + .accessor_descriptors + .borrow_mut() + .remove(&(handle as usize, name.to_string())); true } @@ -436,8 +436,10 @@ mod tests { HANDLE_EXPANDO_PROPS.with(|cell| { cell.borrow_mut().remove(&h); }); - PROPERTY_DESCRIPTORS.with(|m| { - m.borrow_mut().remove(&(h as usize, "hid".to_string())); - }); + crate::state::state() + .descriptors + .property_descriptors + .borrow_mut() + .remove(&(h as usize, "hid".to_string())); } } diff --git a/crates/perry-runtime/src/object/mod.rs b/crates/perry-runtime/src/object/mod.rs index 1c8929c5ef..a0584df524 100644 --- a/crates/perry-runtime/src/object/mod.rs +++ b/crates/perry-runtime/src/object/mod.rs @@ -169,9 +169,9 @@ pub(crate) use descriptor_state::{ object_proto_may_intercept_key, plain_data_write_may_intercept, prune_dead_descriptor_owner_entries, reflect_getter_closure_bits, set_accessor_descriptor, set_builtin_accessor_descriptor, set_builtin_property_attrs, set_property_attrs, - AccessorDescriptor, PropertyAttrs, ACCESSORS_IN_USE, ACCESSOR_DESCRIPTORS, - PROPERTY_ATTRS_IN_USE, PROPERTY_DESCRIPTORS, + AccessorDescriptor, DescriptorTables, PropertyAttrs, }; +pub(crate) use field_get_set::FieldLookupCaches; pub use this_binding::{ js_implicit_this_get, js_implicit_this_get_sloppy, js_implicit_this_set, js_new_target_get, js_new_target_set, js_static_this_arm_classref, js_static_this_arm_value, @@ -228,15 +228,29 @@ pub(crate) static SESSION_STORAGE_PTR: AtomicI64 = AtomicI64::new(0); // This handles cases like Object.assign() adding many fields to an object // that was allocated with only 8 slots (e.g., @noble/curves Fp field with 21 properties). thread_local! { + static CLASS_PROTOTYPE_METHOD_VALUES: RefCell> = + RefCell::new(HashMap::new()); +} + +/// #6759 Phase A: object field-storage side tables and the shape/transition +/// caches, grouped as the `object_hot` field of +/// [`crate::state::RuntimeState`]. Previously five separate +/// `thread_local!`s; reach them via `crate::state::state().object_hot` +/// (one TLS fetch for the whole group). +pub(crate) struct ObjectHotTables { + /// Extra properties for objects that exceeded their pre-allocated + /// inline slot count. + /// /// Heap-pointer keyed; PtrHasher avoids the per-call SipHash on /// every overflow read/write. `clear_overflow_for_ptr` was 0.7% /// leaf samples on perf-comprehensive (called from object dispatch /// + arena_walk_objects in the GC path). - static OVERFLOW_FIELDS: RefCell>> = - RefCell::new(crate::fast_hash::new_ptr_hash_map()); - static CLASS_PROTOTYPE_METHOD_VALUES: RefCell> = - RefCell::new(HashMap::new()); - + pub(crate) overflow_fields: RefCell>>, + /// Last-accessed overflow Vec cache — one entry, keyed by `obj_ptr`. + /// Skips the outer HashMap lookup on consecutive writes to the same + /// object (the row-build pattern). See `overflow_set` for the safety + /// argument behind the cached raw `Vec` pointer. + pub(crate) overflow_last: Cell<(usize, *mut Vec)>, /// Sidecar hash index for object key lookup. The on-object /// `keys_array` only supports O(N) linear scan; for objects that /// grow beyond `KEYS_INDEX_THRESHOLD` keys, the linear scan @@ -255,8 +269,52 @@ thread_local! { /// unrelated array) are tolerated: lookup just misses, content /// validation against the actual stored key on the linear-scan /// fallback ensures correctness. - static KEYS_INDEX: RefCell>)>> = - RefCell::new(crate::fast_hash::new_ptr_hash_map()); + #[allow(clippy::type_complexity)] + pub(crate) keys_index: RefCell< + crate::fast_hash::PtrHashMap>)>, + >, + /// Direct-mapped inline shape cache. Empty entries have shape_id == 0 + /// and keys_array == null. + pub(crate) shape_inline_cache: + std::cell::UnsafeCell<[ShapeCacheEntry; SHAPE_INLINE_CACHE_SIZE]>, + /// Overflow map for shape_ids that collide in the inline cache. + pub(crate) shape_cache_overflow: RefCell>, + /// Per-thread shape-transition cache for the dynamic-key write path; + /// see the doc block above `with_transition_cache`. HEAP-allocated + /// (`Box`) — oversized inline storage overflowed the arm64_32 ILP32 + /// TLS layout when this lived in a `thread_local!`, and keeping it + /// boxed inside the heap-allocated `RuntimeState` preserves that. + pub(crate) transition_cache: std::cell::UnsafeCell>, +} + +impl ObjectHotTables { + pub(crate) fn new() -> Self { + ObjectHotTables { + overflow_fields: RefCell::new(crate::fast_hash::new_ptr_hash_map()), + overflow_last: Cell::new((0, std::ptr::null_mut())), + keys_index: RefCell::new(crate::fast_hash::new_ptr_hash_map()), + shape_inline_cache: std::cell::UnsafeCell::new( + [ShapeCacheEntry { + shape_id: 0, + keys_array: std::ptr::null_mut(), + }; SHAPE_INLINE_CACHE_SIZE], + ), + shape_cache_overflow: RefCell::new(HashMap::new()), + transition_cache: std::cell::UnsafeCell::new( + vec![ + TransitionEntry { + prev_keys: 0, + key_ptr: 0, + next_keys: 0, + slot_idx: 0, + target_len: 0, + }; + TRANSITION_CACHE_SIZE + ] + .into_boxed_slice(), + ), + } + } } /// When keys_array length exceeds this, build the sidecar hash index @@ -327,13 +385,14 @@ unsafe fn keys_index_lookup( // Look up the cached index. If absent OR stale (length doesn't // match — caller appended without going through `keys_index_insert`), // rebuild. - let needs_rebuild = KEYS_INDEX.with(|m| { - let m = m.borrow(); + let st = crate::state::state(); + let needs_rebuild = { + let m = st.object_hot.keys_index.borrow(); match m.get(&obj_addr) { Some((cached_len, _)) => *cached_len != key_count, None => true, } - }); + }; if needs_rebuild { let mut map: std::collections::HashMap> = std::collections::HashMap::with_capacity(key_count as usize); @@ -351,12 +410,13 @@ unsafe fn keys_index_lookup( map.entry(h).or_default().push(i); } } - KEYS_INDEX.with(|m| { - m.borrow_mut().insert(obj_addr, (key_count, map)); - }); + st.object_hot + .keys_index + .borrow_mut() + .insert(obj_addr, (key_count, map)); } - KEYS_INDEX.with(|m| { - let m = m.borrow(); + { + let m = st.object_hot.keys_index.borrow(); let (_, map) = m.get(&obj_addr)?; let candidates = map.get(&key_hash)?; let mut sso = [0u8; crate::value::SHORT_STRING_MAX_LEN]; @@ -374,7 +434,7 @@ unsafe fn keys_index_lookup( } } None - }) + } } /// Record a new (key_hash → slot) entry in the sidecar after a key @@ -385,15 +445,13 @@ fn keys_index_insert(obj_addr: usize, new_count: u32, key_hash: u64, slot: u32) if new_count < KEYS_INDEX_THRESHOLD { return; } - KEYS_INDEX.with(|m| { - let mut m = m.borrow_mut(); - if let Some(entry) = m.get_mut(&obj_addr) { - if entry.0 + 1 == new_count { - entry.0 = new_count; - entry.1.entry(key_hash).or_default().push(slot); - } + let mut m = crate::state::state().object_hot.keys_index.borrow_mut(); + if let Some(entry) = m.get_mut(&obj_addr) { + if entry.0 + 1 == new_count { + entry.0 = new_count; + entry.1.entry(key_hash).or_default().push(slot); } - }); + } } // Last-accessed overflow Vec cache — one entry, keyed by `obj_ptr`. @@ -407,13 +465,10 @@ fn keys_index_insert(obj_addr: usize, new_count: u32, key_hash: u64, slot: u32) // inside a HashMap bucket. That struct only moves when the HashMap // resizes, which only happens on `entry().or_default()` inserting a // fresh key. The slow path below does both the potentially-resizing -// call and the cache refresh inside a single `OVERFLOW_FIELDS.with` -// closure, so no other thread-local mutation can interleave between +// call and the cache refresh while holding the `overflow_fields` +// borrow, so no other thread-local mutation can interleave between // obtaining `&mut Vec` and caching its address. -thread_local! { - static OVERFLOW_LAST: std::cell::UnsafeCell<(usize, *mut Vec)> = - const { std::cell::UnsafeCell::new((0, std::ptr::null_mut())) }; -} +// (Storage: `ObjectHotTables::overflow_last`.) /// Read the u64 bits stored at `field_index` for `obj`, or `None` if absent. /// Positions never written are stored as `TAG_UNDEFINED`; this helper reports @@ -421,12 +476,13 @@ thread_local! { /// "no Vec entry at all" case. #[inline] fn overflow_get(obj_ptr: usize, field_index: usize) -> Option { - OVERFLOW_FIELDS.with(|m| { - m.borrow() - .get(&obj_ptr) - .and_then(|v| v.get(field_index).copied()) - .filter(|&bits| bits != crate::value::TAG_UNDEFINED) - }) + crate::state::state() + .object_hot + .overflow_fields + .borrow() + .get(&obj_ptr) + .and_then(|v| v.get(field_index).copied()) + .filter(|&bits| bits != crate::value::TAG_UNDEFINED) } /// Write `vbits` to the overflow slot `field_index` for `obj`. Grows the @@ -506,8 +562,9 @@ fn overflow_set(obj_ptr: usize, field_index: usize, vbits: u64) { let hdr = obj_ptr as *const ObjectHeader; note_learned_inline_fields((*hdr).class_id, (field_index as u32).saturating_add(1)); } - let cached_slot = OVERFLOW_LAST.with(|c| unsafe { - let (cached_obj, cached_vec) = *c.get(); + let st = crate::state::state(); + let cached_slot = unsafe { + let (cached_obj, cached_vec) = st.object_hot.overflow_last.get(); if cached_obj == obj_ptr && !cached_vec.is_null() { let v = &mut *cached_vec; if v.len() <= field_index { @@ -519,15 +576,15 @@ fn overflow_set(obj_ptr: usize, field_index: usize, vbits: u64) { } else { None } - }); + }; if let Some(slot_addr) = cached_slot { crate::gc::layout_note_slot(obj_ptr, field_index, vbits); crate::gc::runtime_write_barrier_external_slot(obj_ptr, slot_addr, vbits); return; } - let mut slot_addr = 0; - OVERFLOW_FIELDS.with(|m| { - let mut map = m.borrow_mut(); + let slot_addr; + { + let mut map = st.object_hot.overflow_fields.borrow_mut(); let v = map.entry(obj_ptr).or_default(); if v.len() <= field_index { v.resize(field_index + 1, crate::value::TAG_UNDEFINED); @@ -535,10 +592,8 @@ fn overflow_set(obj_ptr: usize, field_index: usize, vbits: u64) { v[field_index] = vbits; slot_addr = (&mut v[field_index]) as *mut u64 as usize; let vec_ptr = v as *mut Vec; - OVERFLOW_LAST.with(|c| unsafe { - *c.get() = (obj_ptr, vec_ptr); - }); - }); + st.object_hot.overflow_last.set((obj_ptr, vec_ptr)); + } crate::gc::layout_note_slot(obj_ptr, field_index, vbits); crate::gc::runtime_write_barrier_external_slot(obj_ptr, slot_addr, vbits); } @@ -670,7 +725,7 @@ const SHAPE_INLINE_CACHE_SIZE: usize = 256; #[repr(C)] #[derive(Clone, Copy)] -struct ShapeCacheEntry { +pub(crate) struct ShapeCacheEntry { shape_id: u32, keys_array: *mut ArrayHeader, } @@ -691,39 +746,27 @@ thread_local! { std::cell::RefCell::new(std::collections::HashMap::new()); } -thread_local! { - /// Direct-mapped inline cache. Empty entries have shape_id == 0 - /// and keys_array == null. - static SHAPE_INLINE_CACHE: std::cell::UnsafeCell<[ShapeCacheEntry; SHAPE_INLINE_CACHE_SIZE]> = - const { std::cell::UnsafeCell::new([ShapeCacheEntry { - shape_id: 0, - keys_array: std::ptr::null_mut(), - }; SHAPE_INLINE_CACHE_SIZE]) }; - - /// Overflow map for shape_ids that collide in the inline cache. - static SHAPE_CACHE_OVERFLOW: RefCell> = RefCell::new(HashMap::new()); -} +// Storage: `ObjectHotTables::{shape_inline_cache, shape_cache_overflow}`. /// Look up a keys_array by shape_id. Returns `null` on miss. /// Hot-path: ~3 ALU ops + 1 load + 1 cmp + 1 branch (no RefCell, no HashMap). #[inline(always)] fn shape_cache_get(shape_id: u32) -> *mut ArrayHeader { - SHAPE_INLINE_CACHE.with(|cache| { - let slot = (shape_id as usize) & (SHAPE_INLINE_CACHE_SIZE - 1); - // Safety: this thread-local is single-threaded by definition; - // the UnsafeCell allows zero-overhead reads on the hot path. - let entry = unsafe { (*cache.get())[slot] }; - if entry.shape_id == shape_id { - return entry.keys_array; - } - // Miss — check the overflow map. - SHAPE_CACHE_OVERFLOW.with(|m| { - m.borrow() - .get(&shape_id) - .copied() - .unwrap_or(std::ptr::null_mut()) - }) - }) + let st = crate::state::state(); + let slot = (shape_id as usize) & (SHAPE_INLINE_CACHE_SIZE - 1); + // Safety: the state is per-thread by construction; the UnsafeCell + // allows zero-overhead reads on the hot path. + let entry = unsafe { (*st.object_hot.shape_inline_cache.get())[slot] }; + if entry.shape_id == shape_id { + return entry.keys_array; + } + // Miss — check the overflow map. + st.object_hot + .shape_cache_overflow + .borrow() + .get(&shape_id) + .copied() + .unwrap_or(std::ptr::null_mut()) } /// Insert a keys_array into the cache. Updates the inline slot @@ -745,19 +788,19 @@ fn shape_cache_insert(shape_id: u32, keys_array: *mut ArrayHeader) { (*gc_header).gc_flags |= crate::gc::GC_FLAG_SHAPE_SHARED; } } - SHAPE_INLINE_CACHE.with(|cache| { - let slot = (shape_id as usize) & (SHAPE_INLINE_CACHE_SIZE - 1); - unsafe { - // GC_STORE_AUDIT(ROOT): SHAPE_INLINE_CACHE entries are scanned by scan_shape_cache_roots_mut. - let entry = &mut (*cache.get())[slot]; - entry.shape_id = shape_id; - crate::gc::runtime_store_root_raw_mut_ptr_slot(&mut entry.keys_array, keys_array); - } - }); - SHAPE_CACHE_OVERFLOW.with(|m| { - m.borrow_mut().insert(shape_id, keys_array); - crate::gc::runtime_write_barrier_root_raw_ptr(keys_array); - }); + let st = crate::state::state(); + let slot = (shape_id as usize) & (SHAPE_INLINE_CACHE_SIZE - 1); + unsafe { + // GC_STORE_AUDIT(ROOT): shape_inline_cache entries are scanned by scan_shape_cache_roots_mut. + let entry = &mut (*st.object_hot.shape_inline_cache.get())[slot]; + entry.shape_id = shape_id; + crate::gc::runtime_store_root_raw_mut_ptr_slot(&mut entry.keys_array, keys_array); + } + st.object_hot + .shape_cache_overflow + .borrow_mut() + .insert(shape_id, keys_array); + crate::gc::runtime_write_barrier_root_raw_ptr(keys_array); } /// Thread-local shape-transition cache for the dynamic-key write path @@ -788,7 +831,7 @@ fn shape_cache_insert(shape_id: u32, keys_array: *mut ArrayHeader) { /// first write — no per-row allocation of a 1-entry keys_array. #[derive(Clone, Copy)] #[repr(C)] -struct TransitionEntry { +pub(crate) struct TransitionEntry { prev_keys: usize, // offset 0 key_ptr: usize, // offset 8 — interned string pointer (pointer identity) next_keys: usize, // offset 16 @@ -805,43 +848,28 @@ const TRANSITION_CACHE_SIZE: usize = 16384; #[allow(dead_code)] const TRANSITION_CACHE_MASK: usize = TRANSITION_CACHE_SIZE - 1; -/// Per-thread transition cache. Was a process-wide `static mut`, but with -/// `perry/thread` user code allocating objects on worker threads each -/// thread has its own arena — cached `next_keys` / `key_ptr` pointers -/// from another thread are use-after-free in our address space. The -/// previous `#[no_mangle]` exposed the symbol for inline LLVM lookups -/// but a grep across crates/perry-codegen confirms no codegen path ever -/// resolved against it, so the export was dead. -thread_local! { - // arm64_32 fix: HEAP-allocate the 320KB cache (Box) instead of storing it - // inline in TLS. Oversized `#[thread_local]` storage overflows the ILP32 - // TLS layout and its writes corrupt adjacent thread-locals (confirmed on a - // real Series 7: shrinking OR boxing removes the corruption). `vec!` builds - // directly on the heap (no 320KB stack temporary). - static TRANSITION_CACHE_GLOBAL: std::cell::UnsafeCell> = - std::cell::UnsafeCell::new( - vec![ - TransitionEntry { - prev_keys: 0, - key_ptr: 0, - next_keys: 0, - slot_idx: 0, - target_len: 0, - }; - TRANSITION_CACHE_SIZE - ] - .into_boxed_slice(), - ); -} - +// Per-thread transition cache (`ObjectHotTables::transition_cache`). Was a +// process-wide `static mut`, but with `perry/thread` user code allocating +// objects on worker threads each thread has its own arena — cached +// `next_keys` / `key_ptr` pointers from another thread are use-after-free +// in our address space. The one-time `#[no_mangle]` exposed the symbol for +// inline LLVM lookups but a grep across crates/perry-codegen confirms no +// codegen path ever resolved against it, so the export was dead. +// +// arm64_32 note: the cache stays HEAP-allocated (Box, now inside the +// heap-allocated `RuntimeState`). Oversized `#[thread_local]` storage +// overflowed the ILP32 TLS layout and its writes corrupted adjacent +// thread-locals (confirmed on a real Series 7: shrinking OR boxing removes +// the corruption). `vec!` builds directly on the heap (no 320KB stack +// temporary). #[inline] fn with_transition_cache( f: impl FnOnce(*mut [TransitionEntry; TRANSITION_CACHE_SIZE]) -> R, ) -> R { - TRANSITION_CACHE_GLOBAL.with(|c| unsafe { - let boxed = &mut *c.get(); + unsafe { + let boxed = &mut *crate::state::state().object_hot.transition_cache.get(); f(boxed.as_mut_ptr() as *mut [TransitionEntry; TRANSITION_CACHE_SIZE]) - }) + } } /// FNV-1a content hash for a property-name string. @@ -1087,18 +1115,19 @@ pub fn scan_shape_cache_roots(mark: &mut dyn FnMut(f64)) { } pub fn scan_shape_cache_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { - SHAPE_INLINE_CACHE.with(|cache| { - let entries = unsafe { &mut *cache.get() }; + let st = crate::state::state(); + { + let entries = unsafe { &mut *st.object_hot.shape_inline_cache.get() }; for entry in entries.iter_mut() { visitor.visit_raw_mut_ptr_slot(&mut entry.keys_array); } - }); - SHAPE_CACHE_OVERFLOW.with(|cache| { - let mut cache = cache.borrow_mut(); + } + { + let mut cache = st.object_hot.shape_cache_overflow.borrow_mut(); for arr_ptr in cache.values_mut() { visitor.visit_raw_mut_ptr_slot(arr_ptr); } - }); + } } /// GC root scanner: mark all JSValues stored in OVERFLOW_FIELDS. @@ -1112,10 +1141,11 @@ pub fn scan_overflow_fields_roots(mark: &mut dyn FnMut(f64)) { } pub fn scan_overflow_fields_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { + let st = crate::state::state(); let mut moved = Vec::new(); let mut moved_any = false; - OVERFLOW_FIELDS.with(|m| { - let mut m = m.borrow_mut(); + { + let mut m = st.object_hot.overflow_fields.borrow_mut(); for (&owner, fields) in m.iter_mut() { let mut new_owner = owner; if visitor.visit_metadata_usize_slot(&mut new_owner) { @@ -1134,11 +1164,9 @@ pub fn scan_overflow_fields_roots_mut(visitor: &mut crate::gc::RuntimeRootVisito moved_any = true; } } - }); + } if moved_any { - OVERFLOW_LAST.with(|c| unsafe { - *c.get() = (0, std::ptr::null_mut()); - }); + st.object_hot.overflow_last.set((0, std::ptr::null_mut())); } } @@ -1146,14 +1174,8 @@ pub(crate) fn visit_overflow_field_slots_mut(owner: usize, mut visit: impl FnMut if owner == 0 { return; } - let slots = OVERFLOW_FIELDS.with(|m| { - let map = m.borrow(); - let Some(fields) = map.get(&owner) else { - return Vec::new(); - }; - if fields.is_empty() { - return Vec::new(); - } + let slots = { + let map = crate::state::state().object_hot.overflow_fields.borrow(); // #6495: visit EVERY overflow slot — never the layout-mask subset. // The per-object slot mask is maintained by `layout_note_slot` at // store time, but not every overflow write path notes (GC owner @@ -1164,15 +1186,20 @@ pub(crate) fn visit_overflow_field_slots_mut(owner: usize, mut visit: impl FnMut // overflow region, and objects with large overflow populations are // in UNKNOWN layout state in practice (dynamic-shape stores degrade // the layout), so the mask bought little here. - let mut slots = Vec::with_capacity(fields.len()); - let base = fields.as_ptr() as *mut u64; - for i in 0..fields.len() { - unsafe { - slots.push(base.add(i)); + match map.get(&owner) { + Some(fields) if !fields.is_empty() => { + let mut slots = Vec::with_capacity(fields.len()); + let base = fields.as_ptr() as *mut u64; + for i in 0..fields.len() { + unsafe { + slots.push(base.add(i)); + } + } + slots } + _ => Vec::new(), } - slots - }); + }; for slot in slots { visit(slot); } @@ -1193,23 +1220,21 @@ pub(crate) fn overflow_fields_owner_moved(old_owner: usize, new_owner: usize) { if old_owner == 0 || new_owner == 0 || old_owner == new_owner { return; } - OVERFLOW_FIELDS.with(|m| { - let mut map = m.borrow_mut(); - let Some(old_fields) = map.remove(&old_owner) else { - return; - }; - match map.entry(new_owner) { - std::collections::hash_map::Entry::Occupied(mut entry) => { - merge_overflow_fields(entry.get_mut(), old_fields); - } - std::collections::hash_map::Entry::Vacant(entry) => { - entry.insert(old_fields); + let st = crate::state::state(); + { + let mut map = st.object_hot.overflow_fields.borrow_mut(); + if let Some(old_fields) = map.remove(&old_owner) { + match map.entry(new_owner) { + std::collections::hash_map::Entry::Occupied(mut entry) => { + merge_overflow_fields(entry.get_mut(), old_fields); + } + std::collections::hash_map::Entry::Vacant(entry) => { + entry.insert(old_fields); + } } } - }); - OVERFLOW_LAST.with(|c| unsafe { - *c.get() = (0, std::ptr::null_mut()); - }); + } + st.object_hot.overflow_last.set((0, std::ptr::null_mut())); } pub fn scan_object_cache_roots(mark: &mut dyn FnMut(f64)) { @@ -1303,35 +1328,34 @@ pub fn scan_object_cache_roots_mut(visitor: &mut crate::gc::RuntimeRootVisitor<' #[cfg(test)] pub(crate) fn test_seed_shape_cache_root(shape_id: u32, keys_array: *mut ArrayHeader) { - SHAPE_INLINE_CACHE.with(|cache| { - let slot = (shape_id as usize) & (SHAPE_INLINE_CACHE_SIZE - 1); - unsafe { - // GC_STORE_AUDIT(ROOT): test seed mirrors SHAPE_INLINE_CACHE roots scanned by scan_shape_cache_roots_mut. - let entry = &mut (*cache.get())[slot]; - entry.shape_id = shape_id; - crate::gc::runtime_store_root_raw_mut_ptr_slot(&mut entry.keys_array, keys_array); - } - }); - SHAPE_CACHE_OVERFLOW.with(|cache| { - cache.borrow_mut().clear(); - cache.borrow_mut().insert(shape_id, keys_array); - crate::gc::runtime_write_barrier_root_raw_ptr(keys_array); - }); + let st = crate::state::state(); + let slot = (shape_id as usize) & (SHAPE_INLINE_CACHE_SIZE - 1); + unsafe { + // GC_STORE_AUDIT(ROOT): test seed mirrors shape_inline_cache roots scanned by scan_shape_cache_roots_mut. + let entry = &mut (*st.object_hot.shape_inline_cache.get())[slot]; + entry.shape_id = shape_id; + crate::gc::runtime_store_root_raw_mut_ptr_slot(&mut entry.keys_array, keys_array); + } + { + let mut cache = st.object_hot.shape_cache_overflow.borrow_mut(); + cache.clear(); + cache.insert(shape_id, keys_array); + } + crate::gc::runtime_write_barrier_root_raw_ptr(keys_array); } #[cfg(test)] pub(crate) fn test_shape_cache_root(shape_id: u32) -> (usize, usize) { - let inline = SHAPE_INLINE_CACHE.with(|cache| { - let slot = (shape_id as usize) & (SHAPE_INLINE_CACHE_SIZE - 1); - unsafe { (*cache.get())[slot].keys_array as usize } - }); - let overflow = SHAPE_CACHE_OVERFLOW.with(|cache| { - cache - .borrow() - .get(&shape_id) - .map(|ptr| *ptr as usize) - .unwrap_or(0) - }); + let st = crate::state::state(); + let slot = (shape_id as usize) & (SHAPE_INLINE_CACHE_SIZE - 1); + let inline = unsafe { (*st.object_hot.shape_inline_cache.get())[slot].keys_array as usize }; + let overflow = st + .object_hot + .shape_cache_overflow + .borrow() + .get(&shape_id) + .map(|ptr| *ptr as usize) + .unwrap_or(0); (inline, overflow) } @@ -1371,72 +1395,80 @@ pub(crate) fn test_clear_transition_cache_root() { #[cfg(test)] pub(crate) fn test_seed_overflow_fields_root(owner: usize, value_bits: u64) { - OVERFLOW_FIELDS.with(|m| { - let mut m = m.borrow_mut(); + let st = crate::state::state(); + { + let mut m = st.object_hot.overflow_fields.borrow_mut(); m.clear(); m.insert(owner, vec![value_bits]); - }); + } crate::gc::layout_note_slot(owner, 0, value_bits); - OVERFLOW_LAST.with(|c| unsafe { - *c.get() = (0, std::ptr::null_mut()); - }); + st.object_hot.overflow_last.set((0, std::ptr::null_mut())); } #[cfg(test)] pub(crate) fn debug_overflow_entry_len(owner: usize) -> Option { - OVERFLOW_FIELDS.with(|m| m.borrow().get(&owner).map(|v| v.len())) + crate::state::state() + .object_hot + .overflow_fields + .borrow() + .get(&owner) + .map(|v| v.len()) } #[cfg(test)] pub(crate) fn test_seed_overflow_fields_vec(owner: usize, values: Vec) { - OVERFLOW_FIELDS.with(|m| { - m.borrow_mut().insert(owner, values); - }); - OVERFLOW_LAST.with(|c| unsafe { - *c.get() = (0, std::ptr::null_mut()); - }); + let st = crate::state::state(); + st.object_hot + .overflow_fields + .borrow_mut() + .insert(owner, values); + st.object_hot.overflow_last.set((0, std::ptr::null_mut())); } #[cfg(test)] pub(crate) fn test_clear_overflow_fields_root() { - OVERFLOW_FIELDS.with(|m| m.borrow_mut().clear()); - OVERFLOW_LAST.with(|c| unsafe { - *c.get() = (0, std::ptr::null_mut()); - }); + let st = crate::state::state(); + st.object_hot.overflow_fields.borrow_mut().clear(); + st.object_hot.overflow_last.set((0, std::ptr::null_mut())); } #[cfg(test)] pub(crate) fn test_overflow_fields_root() -> (usize, u64) { - OVERFLOW_FIELDS.with(|m| { - let m = m.borrow(); - let Some((&owner, fields)) = m.iter().next() else { - return (0, 0); - }; - (owner, fields.first().copied().unwrap_or(0)) - }) + let m = crate::state::state().object_hot.overflow_fields.borrow(); + let Some((&owner, fields)) = m.iter().next() else { + return (0, 0); + }; + (owner, fields.first().copied().unwrap_or(0)) } #[cfg(test)] pub(crate) fn test_overflow_field_bits(owner: usize, index: usize) -> u64 { - OVERFLOW_FIELDS.with(|m| { - m.borrow() - .get(&owner) - .and_then(|fields| fields.get(index).copied()) - .unwrap_or(0) - }) + crate::state::state() + .object_hot + .overflow_fields + .borrow() + .get(&owner) + .and_then(|fields| fields.get(index).copied()) + .unwrap_or(0) } #[cfg(test)] pub(crate) fn test_seed_keys_index_entry(owner: usize) { - KEYS_INDEX.with(|m| { - m.borrow_mut() - .insert(owner, (0, std::collections::HashMap::new())); - }); + crate::state::state() + .object_hot + .keys_index + .borrow_mut() + .insert(owner, (0, std::collections::HashMap::new())); } #[cfg(test)] pub(crate) fn test_keys_index_entry_exists(owner: usize) -> bool { - KEYS_INDEX.with(|m| m.borrow().get(&owner).is_some()) + crate::state::state() + .object_hot + .keys_index + .borrow() + .get(&owner) + .is_some() } #[cfg(test)] @@ -1549,16 +1581,13 @@ pub(crate) fn test_clear_object_cache_roots() { /// Called from GC sweep when an ObjectHeader is collected, to prevent stale entries /// from "infecting" new objects allocated at the same address. pub fn clear_overflow_for_ptr(obj_ptr: usize) { - OVERFLOW_FIELDS.with(|m| { - m.borrow_mut().remove(&obj_ptr); - }); + let st = crate::state::state(); + st.object_hot.overflow_fields.borrow_mut().remove(&obj_ptr); // If the freed object is the one our last-accessed cache points at, // the cached `Vec` pointer is now dangling — clear it. - OVERFLOW_LAST.with(|c| unsafe { - if (*c.get()).0 == obj_ptr { - *c.get() = (0, std::ptr::null_mut()); - } - }); + if st.object_hot.overflow_last.get().0 == obj_ptr { + st.object_hot.overflow_last.set((0, std::ptr::null_mut())); + } } /// Remove the `KEYS_INDEX` sidecar entry for a freed object pointer. @@ -1571,9 +1600,11 @@ pub fn clear_overflow_for_ptr(obj_ptr: usize) { /// read. Unlike `clear_overflow_for_ptr` there is no last-accessed /// cache to invalidate: `keys_index_lookup` always goes through the map. pub fn clear_keys_index_for_ptr(obj_ptr: usize) { - KEYS_INDEX.with(|m| { - m.borrow_mut().remove(&obj_ptr); - }); + crate::state::state() + .object_hot + .keys_index + .borrow_mut() + .remove(&obj_ptr); } /// Cheap check used by the GC sweep to short-circuit per-object @@ -1591,7 +1622,11 @@ pub fn clear_keys_index_for_ptr(obj_ptr: usize) { /// matching obj_ptr without first writing to OVERFLOW_FIELDS. #[inline] pub fn overflow_fields_is_empty() -> bool { - OVERFLOW_FIELDS.with(|m| m.borrow().is_empty()) + crate::state::state() + .object_hot + .overflow_fields + .borrow() + .is_empty() } // `is_valid_obj_ptr` moved to `value/addr_class.rs` (the centralized diff --git a/crates/perry-runtime/src/object/native_call_method.rs b/crates/perry-runtime/src/object/native_call_method.rs index 754a3755f6..45732c83a6 100644 --- a/crates/perry-runtime/src/object/native_call_method.rs +++ b/crates/perry-runtime/src/object/native_call_method.rs @@ -1096,7 +1096,7 @@ pub unsafe extern "C" fn js_native_call_method( // yield-star-* with `get next()`). `get_accessor_descriptor` is a cheap // keyed HashMap lookup (no deref), gated on the accessor hot-path flag so // non-accessor programs skip it entirely. - if jsval.is_pointer() && crate::object::ACCESSORS_IN_USE.with(|c| c.get()) { + if jsval.is_pointer() && crate::state::state().descriptors.accessors_in_use.get() { let obj_usize = crate::value::js_nanbox_get_pointer(object) as usize; if crate::value::addr_class::is_above_handle_band(obj_usize) { if let Some(acc) = crate::object::get_accessor_descriptor(obj_usize, method_name) { diff --git a/crates/perry-runtime/src/object/object_ops/define_property.rs b/crates/perry-runtime/src/object/object_ops/define_property.rs index cd37fe63f3..8a30373834 100644 --- a/crates/perry-runtime/src/object/object_ops/define_property.rs +++ b/crates/perry-runtime/src/object/object_ops/define_property.rs @@ -235,9 +235,11 @@ unsafe fn define_property_on_handle( // (`{ enumerable: true }` alone). Drop any accessor that used to occupy // the key so the store can't fire a stale setter. if had_accessor { - crate::object::descriptor_state::ACCESSOR_DESCRIPTORS.with(|m| { - m.borrow_mut().remove(&(hid as usize, key.clone())); - }); + crate::state::state() + .descriptors + .accessor_descriptors + .borrow_mut() + .remove(&(hid as usize, key.clone())); } // [[Value]] is the descriptor's when present; otherwise it defaults to // `undefined` for a BRAND-NEW key or an accessor→data conversion (neither @@ -686,9 +688,11 @@ pub extern "C" fn js_object_define_property( let value_key = crate::string::js_string_from_bytes(b"value".as_ptr(), 5); let value_field = js_object_get_field_by_name(desc_ptr as *const ObjectHeader, value_key); - ACCESSOR_DESCRIPTORS.with(|m| { - m.borrow_mut().remove(&(closure_ptr, key_rust.clone())); - }); + crate::state::state() + .descriptors + .accessor_descriptors + .borrow_mut() + .remove(&(closure_ptr, key_rust.clone())); if !value_field.is_undefined() { crate::closure::closure_set_dynamic_prop( closure_ptr, @@ -1100,9 +1104,11 @@ pub extern "C" fn js_object_define_property( // `writable: false` doesn't reject the forced store below. The // final attributes are (re)applied a few lines down. if let Some(ref k) = key_rust { - ACCESSOR_DESCRIPTORS.with(|m| { - m.borrow_mut().remove(&(obj as usize, k.clone())); - }); + crate::state::state() + .descriptors + .accessor_descriptors + .borrow_mut() + .remove(&(obj as usize, k.clone())); clear_property_attrs(obj as usize, k); } let value_field = match &desc_view { diff --git a/crates/perry-runtime/src/state.rs b/crates/perry-runtime/src/state.rs new file mode 100644 index 0000000000..10a7e4f0fe --- /dev/null +++ b/crates/perry-runtime/src/state.rs @@ -0,0 +1,110 @@ +//! #6759 Phase A: explicit per-thread runtime state (perry's `Isolate`). +//! +//! Historically every piece of object-model metadata lived in its own +//! `thread_local!` side table, so answering "what are property X's +//! attributes on this receiver" cost one TLS resolution (`_tlv_get_addr` +//! plus `LocalKey::with`'s lazy-init/destructor bookkeeping) *per table +//! probed*. This module concentrates the hot tables into one heap-allocated +//! [`RuntimeState`] reached through a single const-initialized TLS pointer: +//! the fast path of [`state`] is one TLS address computation and one load — +//! no init flag, no destructor registration — and a hot function that +//! probes several tables can fetch the state once into a local and reuse +//! it. +//! +//! Isolation semantics are unchanged: each OS thread that touches the +//! runtime (the main JS thread, every `perry/thread` worker, and any tokio +//! callback that reaches an object helper) lazily allocates its own +//! `RuntimeState` on first use, exactly as each `thread_local!` table used +//! to lazily initialize per thread. The state is freed at thread exit via +//! [`StateOwner`]'s TLS destructor, mirroring the drop the old per-table +//! TLS values received. +//! +//! Borrow discipline is also unchanged for now (fields keep their +//! `RefCell`/`Cell`/`UnsafeCell` wrappers); relaxing it is explicitly a +//! later step in #6759. + +use std::cell::{Cell, RefCell}; + +/// The per-thread runtime state. Grows one field group at a time as tables +/// migrate out of module-level `thread_local!`s (#6759 Phase A is +/// explicitly incremental); each group struct lives next to the code that +/// owns it so the table types stay private to their module. +pub(crate) struct RuntimeState { + /// Property/accessor descriptor tables + their fast-path gates + /// (previously `object::descriptor_state`'s four `thread_local!`s). + pub(crate) descriptors: crate::object::DescriptorTables, + /// Object field storage side tables: overflow fields, keys-index + /// sidecar, shape and transition caches (previously five + /// `thread_local!`s in `object::mod`). + pub(crate) object_hot: crate::object::ObjectHotTables, + /// Property-lookup inline caches: the direct-mapped field cache and + /// the wide-object key index (previously `thread_local!`s in + /// `object::field_get_set`). + pub(crate) field_lookup: crate::object::FieldLookupCaches, +} + +impl RuntimeState { + fn new_boxed() -> Box { + Box::new(RuntimeState { + descriptors: crate::object::DescriptorTables::new(), + object_hot: crate::object::ObjectHotTables::new(), + field_lookup: crate::object::FieldLookupCaches::new(), + }) + } +} + +thread_local! { + /// Fast-path pointer to this thread's state. `Cell<*mut _>` has no drop + /// glue, so this TLS slot never registers a destructor — `with` on it + /// compiles down to the raw TLS address computation + load, and it + /// remains accessible from other TLS destructors during thread + /// teardown. + static STATE_PTR: Cell<*mut RuntimeState> = const { Cell::new(std::ptr::null_mut()) }; + /// Owns the allocation behind [`STATE_PTR`]; its destructor frees the + /// state at thread exit (and nulls the fast-path pointer first, so a + /// late access from another TLS destructor re-initializes instead of + /// dereferencing a freed pointer). + static STATE_OWNER: RefCell> = const { RefCell::new(None) }; +} + +struct StateOwner(*mut RuntimeState); + +impl Drop for StateOwner { + fn drop(&mut self) { + STATE_PTR.with(|c| c.set(std::ptr::null_mut())); + // Safety: `self.0` came from `Box::into_raw` in `init_state` and is + // only freed here; nulling STATE_PTR above keeps any later `state()` + // call on this thread from handing out the dangling pointer. + unsafe { drop(Box::from_raw(self.0)) }; + } +} + +/// Fetch this thread's [`RuntimeState`], allocating it on first use. +/// +/// Hot paths that touch several tables should call this once and keep the +/// reference in a local. The returned `&'static` is sound for runtime code +/// because every caller runs on the thread that owns the state and cannot +/// outlive it: the state lives until thread exit, and runtime entry points +/// never park a reference across threads. +#[inline] +pub(crate) fn state() -> &'static RuntimeState { + let p = STATE_PTR.with(|c| c.get()); + if p.is_null() { + init_state() + } else { + unsafe { &*p } + } +} + +#[cold] +#[inline(never)] +fn init_state() -> &'static RuntimeState { + let raw = Box::into_raw(RuntimeState::new_boxed()); + // Register the owner so the state is freed at thread exit. During thread + // teardown STATE_OWNER may already be destroyed (`try_with` fails); the + // state then leaks for the remainder of teardown, which is the safe + // choice — only other TLS destructors can still reach it. + let _ = STATE_OWNER.try_with(|o| *o.borrow_mut() = Some(StateOwner(raw))); + STATE_PTR.with(|c| c.set(raw)); + unsafe { &*raw } +}