fix(emacs-core): type byte-code structural keys

GNU Emacs compares PVEC_CLOSURE objects element-wise and requires structurally equal closures to produce the same sxhash. The PR's shared structural view established that contract, but its equal-table projection encoded an absent field as the ordinary text key #<absent>. A byte-code object with a present Lisp string of that value therefore aliased an unequal object whose environment was absent.

Represent byte-code table keys as a dedicated HashKey::ByteCode containing domain-tagged ByteCodeKeyPart values. Preserve observable shape, Lisp values, retained bytes, decoded operations, constant vectors, GNU string contents, and absence as distinct enum variants, and teach the pdump conversion and object codec to round-trip that typed representation.

Replace the allocating structural Vec with an ExactSizeIterator over a fixed array plus extra slots. Derive Eq and Hash for decoded Op values so sxhash no longer allocates a debug-formatted string, and keep observable closure shape in the same structural evidence stream consumed by equality, sxhash, and equal-table keys.

Add a red-first regression proving absent state cannot collide with Lisp-reachable sentinel text, correct the tests to honor the one-way sxhash contract instead of demanding distinct hashes for unequal values, and add a GNU 31.0.90 differential oracle covering byte-code objects, make-closure, equal tables, and delete.

Verified with release-mode neovm-core checks, seven focused release tests including pdump codec round-trip, and the live release differential oracle against the pinned GNU Emacs 31.0.90 binary.

Author: eval-exec

crates/neovm-core/src/emacs_core/runtime/bytecode/chunk.rs

@@ -122,9 +122,18 @@ fn arglist_value_from_params(params: &LambdaParams) -> Value {
     Value::list(elements)
 }
 
-/// One element of [`ByteCodeFunction::structural_slots`].
+/// One piece of evidence used for structural byte-code equality and hashing.
+///
+/// This is deliberately named a "part", not a GNU closure "slot": Neomacs
+/// keeps some execution state (notably the captured environment) beside the
+/// GNU-compatible slot projection.  Consumers must compare that state too,
+/// because it affects execution even when it has no independently observable
+/// GNU slot.
 #[derive(Clone, Copy, Debug)]
-pub enum ByteCodeSlot<'a> {
+pub enum ByteCodeStructuralPart<'a> {
+    /// GNU's observable pseudovector size.  Keeping shape in the sequence
+    /// makes it impossible for a consumer to forget the `ASIZE` comparison.
+    ObservableSlotCount(usize),
     /// A Lisp value compared, hashed and keyed as any other.
     Value(Value),
     /// The GNU bytecode string, byte-exact.
@@ -135,10 +144,37 @@ pub enum ByteCodeSlot<'a> {
     Values(&'a [Value]),
     /// A docstring, under GNU string equality: characters, bytes, contents.
     Text(&'a LispString),
-    /// An optional slot that is not there.
+    /// An optional structural part that is not present.
     Absent,
 }
 
+/// Allocation-free iterator over a byte-code function's structural evidence.
+pub(crate) struct ByteCodeStructuralParts<'a> {
+    fixed: std::array::IntoIter<ByteCodeStructuralPart<'a>, 8>,
+    extras: std::slice::Iter<'a, Value>,
+}
+
+impl<'a> Iterator for ByteCodeStructuralParts<'a> {
+    type Item = ByteCodeStructuralPart<'a>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.fixed.next().or_else(|| {
+            self.extras
+                .next()
+                .copied()
+                .map(ByteCodeStructuralPart::Value)
+        })
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let remaining = self.fixed.len() + self.extras.len();
+        (remaining, Some(remaining))
+    }
+}
+
+impl ExactSizeIterator for ByteCodeStructuralParts<'_> {}
+impl std::iter::FusedIterator for ByteCodeStructuralParts<'_> {}
+
 /// A compiled bytecode function.
 #[derive(Debug)]
 pub struct ByteCodeFunction {
@@ -634,9 +670,9 @@ impl ByteCodeFunction {
         }
     }
 
-    /// The structural view of this function: GNU's closure slots, in GNU's
-    /// order, plus the one field GNU keeps inside a slot and this port keeps
-    /// beside it.
+    /// The structural evidence for this function: GNU's observable shape and
+    /// closure slots, in GNU's order, plus the environment this port keeps
+    /// beside the constants vector.
     ///
     /// GNU's `internal_equal_1` compares a `PVEC_CLOSURE` element-wise
     /// (src/fns.c:2984-2998 in emacs-31.1) and `sxhash_obj` hashes it through
@@ -645,46 +681,48 @@ impl ByteCodeFunction {
     /// `equal`, its fallible twin, `sxhash-equal` and the `equal`-table key
     /// all read THIS sequence rather than each re-deriving the schema:
     ///
-    /// | index | GNU slot            | here                                   |
-    /// |-------|---------------------|----------------------------------------|
-    /// | 0     | arglist             | `Value`                                |
-    /// | 1     | bytecode string     | `Bytes`, or `Ops` when none is retained|
-    /// | 2     | constants vector    | `Values`                               |
-    /// | 3     | (inside slot 2)     | captured `env`: `Value`, else `Absent` |
-    /// | 4     | max stack depth     | `Value` (a fixnum)                     |
-    /// | 5     | docstring / form    | `Text`, `Value`, or `Absent`           |
-    /// | 6     | interactive spec    | `Value` or `Absent`                    |
-    /// | 7..   | `&rest ELEMENTS`    | `Value` each                           |
+    /// | part | GNU representation  | Neomacs structural evidence             |
+    /// |------|---------------------|------------------------------------------|
+    /// | 0    | pseudovector ASIZE  | `ObservableSlotCount`                    |
+    /// | 1    | arglist             | `Value`                                  |
+    /// | 2    | bytecode string     | `Bytes`, or `Ops` when none is retained  |
+    /// | 3    | constants vector    | `Values`                                 |
+    /// | 4    | inside constants    | captured `env`: `Value`, else `Absent`   |
+    /// | 5    | max stack depth     | `Value` (a fixnum)                       |
+    /// | 6    | docstring / form    | `Text`, `Value`, or `Absent`             |
+    /// | 7    | interactive spec    | `Value` or `Absent`                      |
+    /// | 8..  | `&rest ELEMENTS`    | `Value` each                             |
     ///
     /// `Absent` is distinct from a present `nil`: `Op::MakeClosure` stores
     /// `Some(lexenv)`, and a closure over no variables has a present, `nil`
     /// environment that `(aref FN 2)` reports as `nil` rather than as the
     /// constants vector.  A function with no retained GNU bytes compares its
     /// decoded instructions instead, so two functions with different code are
     /// never equal even though GNU's slot 1 would be `nil` for both.
-    pub(crate) fn structural_slots(&self) -> Vec<ByteCodeSlot<'_>> {
-        let mut slots = Vec::with_capacity(7 + self.extra_slots.len());
-        slots.push(ByteCodeSlot::Value(self.arglist));
-        slots.push(match &self.gnu_bytecode_bytes {
-            Some(bytes) => ByteCodeSlot::Bytes(bytes.as_slice()),
-            None => ByteCodeSlot::Ops(&self.ops),
-        });
-        slots.push(ByteCodeSlot::Values(self.constants.as_slice()));
-        slots.push(self.env.map_or(ByteCodeSlot::Absent, ByteCodeSlot::Value));
-        slots.push(ByteCodeSlot::Value(Value::fixnum(i64::from(
-            self.max_stack,
-        ))));
-        slots.push(match (self.doc_form, &self.docstring) {
-            (Some(form), _) => ByteCodeSlot::Value(form),
-            (None, Some(doc)) => ByteCodeSlot::Text(doc),
-            (None, None) => ByteCodeSlot::Absent,
-        });
-        slots.push(
-            self.interactive
-                .map_or(ByteCodeSlot::Absent, ByteCodeSlot::Value),
-        );
-        slots.extend(self.extra_slots.iter().copied().map(ByteCodeSlot::Value));
-        slots
+    pub(crate) fn structural_parts(&self) -> ByteCodeStructuralParts<'_> {
+        use ByteCodeStructuralPart as Part;
+
+        ByteCodeStructuralParts {
+            fixed: [
+                Part::ObservableSlotCount(self.observable_closure_slot_count()),
+                Part::Value(self.arglist),
+                match &self.gnu_bytecode_bytes {
+                    Some(bytes) => Part::Bytes(bytes.as_slice()),
+                    None => Part::Ops(&self.ops),
+                },
+                Part::Values(self.constants.as_slice()),
+                self.env.map_or(Part::Absent, Part::Value),
+                Part::Value(Value::fixnum(i64::from(self.max_stack))),
+                match (self.doc_form, &self.docstring) {
+                    (Some(form), _) => Part::Value(form),
+                    (None, Some(doc)) => Part::Text(doc),
+                    (None, None) => Part::Absent,
+                },
+                self.interactive.map_or(Part::Absent, Part::Value),
+            ]
+            .into_iter(),
+            extras: self.extra_slots.iter(),
+        }
     }
 
     pub fn observable_closure_slot_count(&self) -> usize {

crates/neovm-core/src/emacs_core/runtime/bytecode/mod.rs

@@ -13,6 +13,6 @@ pub mod vm;
 
 // Re-export main types
 pub(crate) use chunk::fresh_bytecode_source_id;
-pub use chunk::{ByteCodeFunction, ByteCodeSlot};
+pub use chunk::{ByteCodeFunction, ByteCodeStructuralPart};
 pub use opcode::Op;
 pub use vm::Vm;

crates/neovm-core/src/emacs_core/runtime/bytecode/opcode.rs

@@ -7,7 +7,7 @@
 use serde::{Deserialize, Serialize};
 
 /// A single bytecode instruction.
-#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub enum Op {
     // -- Constants and stack --------------------------------------------------
     /// Push a constant from the constant pool.

crates/neovm-core/src/emacs_core/runtime/hashtab/mod.rs

@@ -73,6 +73,7 @@ fn hash_key_to_value(key: &HashKey) -> Value {
             let vals: Vec<Value> = items.iter().map(hash_key_to_value).collect();
             Value::vector(vals)
         }
+        HashKey::ByteCode(_) => Value::NIL,
         HashKey::Marker(_) | HashKey::Overlay(_) => Value::NIL,
         HashKey::BoolVec(parts) => {
             let (len, bits) = **parts;
@@ -251,21 +252,32 @@ fn emacs_sxhash_overlay(value: &Value, depth: usize) -> Option<u64> {
 /// GNU `sxhash_obj` (src/fns.c:5525-5536) sends a `PVEC_CLOSURE` through
 /// `sxhash_vector` like any pseudovector `internal_equal` walks, so two
 /// `equal` byte-code objects hash alike and share an `equal` hash-table
-/// bucket.  The elements are [`ByteCodeFunction::structural_slots`] -- the
+/// bucket.  The elements are [`ByteCodeFunction::structural_parts`] -- the
 /// same sequence `equal` compares, so everything `equal` distinguishes,
 /// including a captured environment beside the constants, moves the hash --
 /// folded like `sxhash_vector`: seeded with the slot count, at most
 /// `SXHASH_MAX_LEN` elements.
 fn emacs_sxhash_bytecode(value: &Value, depth: usize) -> Option<u64> {
-    use crate::emacs_core::bytecode::ByteCodeSlot;
+    use crate::emacs_core::bytecode::ByteCodeStructuralPart as Part;
     let bc = value.get_bytecode_data()?;
-    let mut hash = bc.observable_closure_slot_count() as u64;
-    for slot in bc.structural_slots().into_iter().take(SXHASH_MAX_LEN) {
-        let slot_hash = match slot {
-            ByteCodeSlot::Value(value) => emacs_sxhash_obj_with_fallback(&value, depth + 1),
-            ByteCodeSlot::Bytes(bytes) => emacs_hash_char_array(bytes),
-            ByteCodeSlot::Ops(ops) => emacs_hash_char_array(format!("{ops:?}").as_bytes()),
-            ByteCodeSlot::Values(values) => {
+    let mut parts = bc.structural_parts();
+    let Part::ObservableSlotCount(slot_count) = parts.next()? else {
+        unreachable!("byte-code structure starts with its observable shape")
+    };
+    let mut hash = slot_count as u64;
+    for part in parts.take(SXHASH_MAX_LEN) {
+        let part_hash = match part {
+            Part::ObservableSlotCount(_) => {
+                unreachable!("byte-code structure contains one leading shape")
+            }
+            Part::Value(value) => emacs_sxhash_obj_with_fallback(&value, depth + 1),
+            Part::Bytes(bytes) => emacs_hash_char_array(bytes),
+            Part::Ops(ops) => {
+                let mut hasher = DefaultHasher::new();
+                ops.hash(&mut hasher);
+                hasher.finish() & SXHASH_FALLBACK_NONNEG_MASK
+            }
+            Part::Values(values) => {
                 let mut vector_hash = values.len() as u64;
                 for value in values.iter().take(SXHASH_MAX_LEN) {
                     vector_hash = sxhash_combine(
@@ -277,10 +289,10 @@ fn emacs_sxhash_bytecode(value: &Value, depth: usize) -> Option<u64> {
             }
             // GNU `sxhash_string` hashes the bytes; equal strings have equal
             // bytes whatever their representation.
-            ByteCodeSlot::Text(text) => emacs_hash_char_array(text.as_bytes()),
-            ByteCodeSlot::Absent => 42,
+            Part::Text(text) => emacs_hash_char_array(text.as_bytes()),
+            Part::Absent => 42,
         };
-        hash = sxhash_combine(hash, slot_hash);
+        hash = sxhash_combine(hash, part_hash);
     }
     Some(hash)
 }

crates/neovm-core/src/emacs_core/runtime/pdump/convert.rs

@@ -51,8 +51,8 @@ use crate::emacs_core::rect::RectangleState;
 use crate::emacs_core::register::{RegisterContent, RegisterManager};
 use crate::emacs_core::symbol::{LispSymbol, Obarray, SymbolTrappedWrite};
 use crate::emacs_core::value::{
-    HashKey, HashTableTest, HashTableWeakness, LambdaParams, LispHashTable, RuntimeBindingValue,
-    StringTextPropertyRun, Value,
+    ByteCodeKeyPart, HashKey, HashTableTest, HashTableWeakness, LambdaParams, LispHashTable,
+    RuntimeBindingValue, StringTextPropertyRun, Value,
 };
 use crate::emacs_core::value::{ValueKind, VecLikeType};
 use crate::emacs_core::value::{
@@ -3046,6 +3046,12 @@ pub(crate) fn dump_hash_key(encoder: &mut DumpEncoder, k: &HashKey) -> DumpHashK
         HashKey::EqualVec(v) => {
             DumpHashKey::EqualVec(v.iter().map(|key| dump_hash_key(encoder, key)).collect())
         }
+        HashKey::ByteCode(parts) => DumpHashKey::ByteCode(
+            parts
+                .iter()
+                .map(|part| dump_byte_code_key_part(encoder, part))
+                .collect(),
+        ),
         HashKey::Marker(parts) => DumpHashKey::Marker(parts.0, parts.1.get()),
         HashKey::Overlay(parts) => DumpHashKey::Overlay {
             buffer: parts.0,
@@ -3066,6 +3072,31 @@ pub(crate) fn dump_hash_key(encoder: &mut DumpEncoder, k: &HashKey) -> DumpHashK
     }
 }
 
+fn dump_byte_code_key_part(
+    encoder: &mut DumpEncoder,
+    part: &ByteCodeKeyPart,
+) -> DumpByteCodeKeyPart {
+    match part {
+        ByteCodeKeyPart::ObservableSlotCount(count) => {
+            DumpByteCodeKeyPart::ObservableSlotCount(*count)
+        }
+        ByteCodeKeyPart::Value(value) => DumpByteCodeKeyPart::Value(dump_hash_key(encoder, value)),
+        ByteCodeKeyPart::Bytes(bytes) => DumpByteCodeKeyPart::Bytes(bytes.to_vec()),
+        ByteCodeKeyPart::Ops(ops) => DumpByteCodeKeyPart::Ops(ops.to_vec()),
+        ByteCodeKeyPart::Values(values) => DumpByteCodeKeyPart::Values(
+            values
+                .iter()
+                .map(|value| dump_hash_key(encoder, value))
+                .collect(),
+        ),
+        ByteCodeKeyPart::Text { char_count, bytes } => DumpByteCodeKeyPart::Text {
+            char_count: *char_count,
+            bytes: bytes.to_vec(),
+        },
+        ByteCodeKeyPart::Absent => DumpByteCodeKeyPart::Absent,
+    }
+}
+
 pub(crate) fn dump_hash_table_test(t: &HashTableTest) -> DumpHashTableTest {
     match t {
         HashTableTest::Eq => DumpHashTableTest::Eq,
@@ -4701,6 +4732,13 @@ pub(crate) fn load_hash_key(decoder: &mut LoadDecoder, k: &DumpHashKey) -> HashK
                 .collect::<Vec<_>>()
                 .into_boxed_slice(),
         ),
+        DumpHashKey::ByteCode(parts) => HashKey::ByteCode(
+            parts
+                .iter()
+                .map(|part| load_byte_code_key_part(decoder, part))
+                .collect::<Vec<_>>()
+                .into_boxed_slice(),
+        ),
         DumpHashKey::Marker(buffer, bytepos) => {
             HashKey::Marker(Box::new((*buffer, EmacsBytePos::new(*bytepos))))
         }
@@ -4725,6 +4763,34 @@ pub(crate) fn load_hash_key(decoder: &mut LoadDecoder, k: &DumpHashKey) -> HashK
     }
 }
 
+fn load_byte_code_key_part(
+    decoder: &mut LoadDecoder,
+    part: &DumpByteCodeKeyPart,
+) -> ByteCodeKeyPart {
+    match part {
+        DumpByteCodeKeyPart::ObservableSlotCount(count) => {
+            ByteCodeKeyPart::ObservableSlotCount(*count)
+        }
+        DumpByteCodeKeyPart::Value(value) => ByteCodeKeyPart::Value(load_hash_key(decoder, value)),
+        DumpByteCodeKeyPart::Bytes(bytes) => {
+            ByteCodeKeyPart::Bytes(bytes.clone().into_boxed_slice())
+        }
+        DumpByteCodeKeyPart::Ops(ops) => ByteCodeKeyPart::Ops(ops.clone().into_boxed_slice()),
+        DumpByteCodeKeyPart::Values(values) => ByteCodeKeyPart::Values(
+            values
+                .iter()
+                .map(|value| load_hash_key(decoder, value))
+                .collect::<Vec<_>>()
+                .into_boxed_slice(),
+        ),
+        DumpByteCodeKeyPart::Text { char_count, bytes } => ByteCodeKeyPart::Text {
+            char_count: *char_count,
+            bytes: bytes.clone().into_boxed_slice(),
+        },
+        DumpByteCodeKeyPart::Absent => ByteCodeKeyPart::Absent,
+    }
+}
+
 fn load_hash_key_owned(decoder: &mut LoadDecoder, k: DumpHashKey) -> HashKey {
     match k {
         DumpHashKey::Nil => HashKey::Nil,
@@ -4755,6 +4821,13 @@ fn load_hash_key_owned(decoder: &mut LoadDecoder, k: DumpHashKey) -> HashKey {
                 .collect::<Vec<_>>()
                 .into_boxed_slice(),
         ),
+        DumpHashKey::ByteCode(parts) => HashKey::ByteCode(
+            parts
+                .into_iter()
+                .map(|part| load_byte_code_key_part_owned(decoder, part))
+                .collect::<Vec<_>>()
+                .into_boxed_slice(),
+        ),
         DumpHashKey::Marker(buffer, bytepos) => {
             HashKey::Marker(Box::new((buffer, EmacsBytePos::new(bytepos))))
         }
@@ -4779,6 +4852,34 @@ fn load_hash_key_owned(decoder: &mut LoadDecoder, k: DumpHashKey) -> HashKey {
     }
 }
 
+fn load_byte_code_key_part_owned(
+    decoder: &mut LoadDecoder,
+    part: DumpByteCodeKeyPart,
+) -> ByteCodeKeyPart {
+    match part {
+        DumpByteCodeKeyPart::ObservableSlotCount(count) => {
+            ByteCodeKeyPart::ObservableSlotCount(count)
+        }
+        DumpByteCodeKeyPart::Value(value) => {
+            ByteCodeKeyPart::Value(load_hash_key_owned(decoder, value))
+        }
+        DumpByteCodeKeyPart::Bytes(bytes) => ByteCodeKeyPart::Bytes(bytes.into_boxed_slice()),
+        DumpByteCodeKeyPart::Ops(ops) => ByteCodeKeyPart::Ops(ops.into_boxed_slice()),
+        DumpByteCodeKeyPart::Values(values) => ByteCodeKeyPart::Values(
+            values
+                .into_iter()
+                .map(|value| load_hash_key_owned(decoder, value))
+                .collect::<Vec<_>>()
+                .into_boxed_slice(),
+        ),
+        DumpByteCodeKeyPart::Text { char_count, bytes } => ByteCodeKeyPart::Text {
+            char_count,
+            bytes: bytes.into_boxed_slice(),
+        },
+        DumpByteCodeKeyPart::Absent => ByteCodeKeyPart::Absent,
+    }
+}
+
 pub(crate) fn load_hash_table_test(t: &DumpHashTableTest) -> HashTableTest {
     match t {
         DumpHashTableTest::Eq => HashTableTest::Eq,