Iteration improvements

Cargo.toml

@@ -42,7 +42,7 @@ signet-storage = { version = "0.0.1", path = "./crates/storage" }
 signet-storage-types = { version = "0.0.1", path = "./crates/types" }
 
 # External, in-house
-signet-libmdbx = "0.6.0"
+signet-libmdbx = { version = "0.8.0" }
 
 signet-zenith = "0.16.0-rc.5"
 
@@ -52,6 +52,7 @@ trevm = { version = "0.33.0", features = ["full_env_cfg"] }
 # alloy
 alloy = { version = "1.0.9", default-features = false, features = ["consensus", "rlp"] }
 
+ahash = "0.8"
 auto_impl = "1.3.0"
 bytes = "1.11.0"
 byteorder = "1.5.0"
@@ -64,4 +65,5 @@ tempfile = "3.20.0"
 thiserror = "2.0.18"
 tokio = { version = "1.45.0", features = ["full"] }
 tokio-util = { version = "0.7", features = ["rt"] }
+itertools = "0.14"
 tracing = "0.1.44"

TODOS

@@ -0,0 +1 @@
+- Append optimization for tables keyed by blocknumbers

crates/hot-mdbx/src/cursor.rs

@@ -3,9 +3,12 @@
 use crate::{FixedSizeInfo, MdbxError};
 use signet_hot::{
     MAX_FIXED_VAL_SIZE, MAX_KEY_SIZE,
-    model::{DualKeyTraverse, KvTraverse, KvTraverseMut, RawDualKeyValue, RawKeyValue, RawValue},
+    model::{
+        DualKeyItem, DualKeyTraverse, DualKeyTraverseMut, KvTraverse, KvTraverseMut,
+        RawDualKeyItem, RawDualKeyValue, RawKeyValue, RawValue,
+    },
 };
-use signet_libmdbx::{Ro, Rw, RwSync, TransactionKind, tx::WriteMarker};
+use signet_libmdbx::{DupItem, Ro, Rw, RwSync, TransactionKind, tx::WriteMarker};
 use std::{
     borrow::Cow,
     ops::{Deref, DerefMut},
@@ -104,7 +107,7 @@ where
 
 impl<K: TransactionKind + WriteMarker> KvTraverseMut<MdbxError> for Cursor<'_, K> {
     fn delete_current(&mut self) -> Result<(), MdbxError> {
-        self.inner.del(Default::default()).map_err(MdbxError::Mdbx)
+        self.inner.del().map_err(MdbxError::Mdbx)
     }
 }
 
@@ -501,4 +504,80 @@ where
             None => Ok(None),
         }
     }
+
+    fn iter_items(
+        &mut self,
+    ) -> Result<impl Iterator<Item = Result<RawDualKeyItem<'_>, MdbxError>> + '_, MdbxError> {
+        if !self.fsi.is_dupsort() {
+            return Err(MdbxError::NotDupSort);
+        }
+
+        let key2_size = self.fsi.key2_size().ok_or(MdbxError::UnknownFixedSize)?;
+
+        // Use iter_dup_start which yields DupItem::NewKey/SameKey
+        let iter_dup = self.inner.iter_dup_start::<Cow<'_, [u8]>, Cow<'_, [u8]>>()?;
+        Ok(MdbxDualKeyItemIter { iter_dup, key2_size })
+    }
+}
+
+/// Iterator adapter that converts mdbx's [`DupItem`] to [`DualKeyItem`].
+///
+/// This iterator returns owned data to avoid lifetime complexities between
+/// the transaction lifetime and the iterator borrow lifetime.
+struct MdbxDualKeyItemIter<'tx, 'cur, K: TransactionKind> {
+    iter_dup: signet_libmdbx::tx::iter::IterDup<'tx, 'cur, K, Cow<'tx, [u8]>, Cow<'tx, [u8]>>,
+    key2_size: usize,
+}
+
+impl<'a, K: TransactionKind> Iterator for MdbxDualKeyItemIter<'_, 'a, K> {
+    type Item = Result<RawDualKeyItem<'a>, MdbxError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.iter_dup.borrow_next() {
+            Ok(Some(item)) => {
+                let result = match item {
+                    DupItem::NewKey(k1, v) => {
+                        let (k2, val) = split_cow_at_owned(v, self.key2_size);
+                        DualKeyItem::NewK1(Cow::Owned(k1.into_owned()), k2, val)
+                    }
+                    DupItem::SameKey(v) => {
+                        let (k2, val) = split_cow_at_owned(v, self.key2_size);
+                        DualKeyItem::SameK1(k2, val)
+                    }
+                };
+                Some(Ok(result))
+            }
+            Ok(None) => None,
+            Err(e) => Some(Err(MdbxError::from(e))),
+        }
+    }
+}
+
+/// Splits a [`Cow`] at the given index and returns owned [`Cow`]s.
+#[inline]
+fn split_cow_at_owned(cow: Cow<'_, [u8]>, at: usize) -> (Cow<'static, [u8]>, Cow<'static, [u8]>) {
+    let vec = cow.into_owned();
+    let (left, right) = vec.split_at(at);
+    (Cow::Owned(left.to_vec()), Cow::Owned(right.to_vec()))
+}
+
+impl<K: TransactionKind + WriteMarker> DualKeyTraverseMut<MdbxError> for Cursor<'_, K> {
+    fn delete_current(&mut self) -> Result<(), MdbxError> {
+        // For DUPSORT tables, del() deletes only the current duplicate
+        self.inner.del().map_err(MdbxError::Mdbx)
+    }
+
+    fn clear_k1(&mut self, key1: &[u8]) -> Result<(), MdbxError> {
+        if !self.fsi.is_dupsort() {
+            return Err(MdbxError::NotDupSort);
+        }
+
+        // Position at the K1 - if it doesn't exist, nothing to delete
+        if self.inner.set::<()>(key1)?.is_none() {
+            return Ok(());
+        }
+        // Delete all K2 entries for this K1
+        self.inner.del_all_dups()?;
+        Ok(())
+    }
 }