trie memmory optimized

trie/committer.go

@@ -86,38 +86,33 @@ func (c *committer) commit(n node, db *Database) (node, error) {
 	// Commit children, then parent, and remove the dirty flag.
 	switch cn := n.(type) {
 	case *shortNode:
-		// Commit child
-		collapsed := cn.copy()
-
 		// If the child is fullnode, recursively commit.
 		// Otherwise it can only be hashNode or valueNode.
 		if _, ok := cn.Val.(*fullNode); ok {
 			childV, err := c.commit(cn.Val, db)
 			if err != nil {
 				return nil, err
 			}
-			collapsed.Val = childV
+			cn.Val = childV
 		}
 		// The key needs to be copied, since we're delivering it to database
-		collapsed.Key = hexToCompact(cn.Key)
-		hashedNode := c.store(collapsed, db)
+		cn.Key = hexToCompact(cn.Key)
+		hashedNode := c.store(cn, db)
 		if hn, ok := hashedNode.(hashNode); ok {
 			return hn, nil
 		}
-		return collapsed, nil
+		return cn, nil
 	case *fullNode:
 		hashedKids, err := c.commitChildren(cn, db)
 		if err != nil {
 			return nil, err
 		}
-		collapsed := cn.copy()
-		collapsed.Children = hashedKids
-
-		hashedNode := c.store(collapsed, db)
+		cn.Children = hashedKids
+		hashedNode := c.store(cn, db)
 		if hn, ok := hashedNode.(hashNode); ok {
 			return hn, nil
 		}
-		return collapsed, nil
+		return cn, nil
 	case hashNode:
 		return cn, nil
 	default:

trie/errors.go

@@ -17,11 +17,17 @@
 package trie
 
 import (
+	"errors"
 	"fmt"
 
 	"github.com/CortexFoundation/CortexTheseus/common"
 )
 
+// ErrCommitted is returned when an already committed trie is requested for usage.
+// The potential usages can be `Get`, `Update`, `Delete`, `NodeIterator`, `Prove`
+// and so on.
+var ErrCommitted = errors.New("trie is already committed")
+
 // MissingNodeError is returned by the trie functions (TryGet, Update, TryDelete)
 // in the case where a trie node is not present in the local database. It contains
 // information necessary for retrieving the missing node.

trie/hasher.go

@@ -55,70 +55,72 @@ func returnHasherToPool(h *hasher) {
 
 // hash collapses a node down into a hash node, also returning a copy of the
 // original node initialized with the computed hash to replace the original one.
-func (h *hasher) hash(n node, force bool) (hashed node, cached node) {
+func (h *hasher) hash(n node, force bool) node {
 	// Return the cached hash if it's available
 	if hash, _ := n.cache(); hash != nil {
-		return hash, n
+		return hash
 	}
 	// Trie not processed yet, walk the children
 	switch n := n.(type) {
 	case *shortNode:
-		collapsed, cached := h.hashShortNodeChildren(n)
+		collapsed := h.hashShortNodeChildren(n)
 		hashed := h.shortnodeToHash(collapsed, force)
 		// We need to retain the possibly _not_ hashed node, in case it was too
 		// small to be hashed
 		if hn, ok := hashed.(hashNode); ok {
-			cached.flags.hash = hn
+			n.flags.hash = hn
 		} else {
-			cached.flags.hash = nil
+			n.flags.hash = nil
 		}
-		return hashed, cached
+		return hashed
 	case *fullNode:
-		collapsed, cached := h.hashFullNodeChildren(n)
-		hashed = h.fullnodeToHash(collapsed, force)
+		collapsed := h.hashFullNodeChildren(n)
+		hashed := h.fullnodeToHash(collapsed, force)
 		if hn, ok := hashed.(hashNode); ok {
-			cached.flags.hash = hn
+			n.flags.hash = hn
 		} else {
-			cached.flags.hash = nil
+			n.flags.hash = nil
 		}
-		return hashed, cached
+		return hashed
 	default:
 		// Value and hash nodes don't have children, so they're left as were
-		return n, n
+		return n
 	}
 }
 
 // hashShortNodeChildren collapses the short node. The returned collapsed node
 // holds a live reference to the Key, and must not be modified.
-func (h *hasher) hashShortNodeChildren(n *shortNode) (collapsed, cached *shortNode) {
+func (h *hasher) hashShortNodeChildren(n *shortNode) *shortNode {
 	// Hash the short node's child, caching the newly hashed subtree
-	collapsed, cached = n.copy(), n.copy()
+	//collapsed, cached = n.copy(), n.copy()
+	var collapsed shortNode
 	// Previously, we did copy this one. We don't seem to need to actually
 	// do that, since we don't overwrite/reuse keys
 	// cached.Key = common.CopyBytes(n.Key)
 	collapsed.Key = hexToCompact(n.Key)
 	// Unless the child is a valuenode or hashnode, hash it
 	switch n.Val.(type) {
 	case *fullNode, *shortNode:
-		collapsed.Val, cached.Val = h.hash(n.Val, false)
+		collapsed.Val = h.hash(n.Val, false)
+	default:
+		collapsed.Val = n.Val
 	}
-	return collapsed, cached
+	return &collapsed
 }
 
-func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached *fullNode) {
+func (h *hasher) hashFullNodeChildren(n *fullNode) *fullNode {
 	// Hash the full node's children, caching the newly hashed subtrees
-	cached = n.copy()
-	collapsed = n.copy()
+	var children [17]node
 	if h.parallel {
 		var wg sync.WaitGroup
 		wg.Add(16)
 		for i := 0; i < 16; i++ {
 			go func(i int) {
 				hasher := newHasher(false)
 				if child := n.Children[i]; child != nil {
-					collapsed.Children[i], cached.Children[i] = hasher.hash(child, false)
+					children[i] = hasher.hash(child, false)
 				} else {
-					collapsed.Children[i] = nilValueNode
+					children[i] = nilValueNode
 				}
 				returnHasherToPool(hasher)
 				wg.Done()
@@ -128,13 +130,16 @@ func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached
 	} else {
 		for i := 0; i < 16; i++ {
 			if child := n.Children[i]; child != nil {
-				collapsed.Children[i], cached.Children[i] = h.hash(child, false)
+				children[i] = h.hash(child, false)
 			} else {
-				collapsed.Children[i] = nilValueNode
+				children[i] = nilValueNode
 			}
 		}
 	}
-	return collapsed, cached
+	if n.Children[16] != nil {
+		children[16] = n.Children[16]
+	}
+	return &fullNode{flags: nodeFlag{}, Children: children}
 }
 
 // shortnodeToHash creates a hashNode from a shortNode. The supplied shortnode
@@ -203,10 +208,10 @@ func (h *hasher) hashDataTo(dst, data []byte) {
 func (h *hasher) proofHash(original node) (collapsed, hashed node) {
 	switch n := original.(type) {
 	case *shortNode:
-		sn, _ := h.hashShortNodeChildren(n)
+		sn := h.hashShortNodeChildren(n)
 		return sn, h.shortnodeToHash(sn, false)
 	case *fullNode:
-		fn, _ := h.hashFullNodeChildren(n)
+		fn := h.hashFullNodeChildren(n)
 		return fn, h.fullnodeToHash(fn, false)
 	default:
 		// Value and hash nodes don't have children, so they're left as were

trie/iterator_test.go

@@ -59,7 +59,7 @@ func TestIterator(t *testing.T) {
 		all[val.k] = val.v
 		trie.Update([]byte(val.k), []byte(val.v))
 	}
-	trie.Commit(nil)
+	//trie.Commit(nil)
 
 	found := make(map[string]string)
 	it := NewIterator(trie.NodeIterator(nil))
@@ -123,7 +123,7 @@ func TestNodeIteratorCoverage(t *testing.T) {
 
 	// Gather all the node hashes found by the iterator
 	hashes := make(map[common.Hash]struct{})
-	for it := trie.NodeIterator(nil); it.Next(true); {
+	for it := trie.MustNodeIterator(nil); it.Next(true); {
 		if it.Hash() != (common.Hash{}) {
 			hashes[it.Hash()] = struct{}{}
 		}

trie/node.go

@@ -79,15 +79,19 @@ func (n *fullNode) EncodeRLP(w io.Writer) error {
 	return eb.Flush()
 }
 
-func (n *fullNode) copy() *fullNode   { copy := *n; return &copy }
-func (n *shortNode) copy() *shortNode { copy := *n; return &copy }
-
 // nodeFlag contains caching-related metadata about a node.
 type nodeFlag struct {
 	hash  hashNode // cached hash of the node (may be nil)
 	dirty bool     // whether the node has changes that must be written to the database
 }
 
+func (n nodeFlag) copy() nodeFlag {
+	return nodeFlag{
+		hash:  common.CopyBytes(n.hash),
+		dirty: n.dirty,
+	}
+}
+
 func (n *fullNode) cache() (hashNode, bool)  { return n.flags.hash, n.flags.dirty }
 func (n *shortNode) cache() (hashNode, bool) { return n.flags.hash, n.flags.dirty }
 func (n hashNode) cache() (hashNode, bool)   { return nil, true }
@@ -130,6 +134,16 @@ func mustDecodeNode(hash, buf []byte) node {
 	return n
 }
 
+// mustDecodeNodeUnsafe is a wrapper of decodeNodeUnsafe and panic if any error is
+// encountered.
+func mustDecodeNodeUnsafe(hash, buf []byte) node {
+	n, err := decodeNodeUnsafe(hash, buf)
+	if err != nil {
+		panic(fmt.Sprintf("node %x: %v", hash, err))
+	}
+	return n
+}
+
 // decodeNode parses the RLP encoding of a trie node.
 func decodeNode(hash, buf []byte) (node, error) {
 	if len(buf) == 0 {
@@ -151,6 +165,29 @@ func decodeNode(hash, buf []byte) (node, error) {
 	}
 }
 
+// decodeNodeUnsafe parses the RLP encoding of a trie node. The passed byte slice
+// will be directly referenced by node without bytes deep copy, so the input MUST
+// not be changed after.
+func decodeNodeUnsafe(hash, buf []byte) (node, error) {
+	if len(buf) == 0 {
+		return nil, io.ErrUnexpectedEOF
+	}
+	elems, _, err := rlp.SplitList(buf)
+	if err != nil {
+		return nil, fmt.Errorf("decode error: %v", err)
+	}
+	switch c, _ := rlp.CountValues(elems); c {
+	case 2:
+		n, err := decodeShort(hash, elems)
+		return n, wrapError(err, "short")
+	case 17:
+		n, err := decodeFull(hash, elems)
+		return n, wrapError(err, "full")
+	default:
+		return nil, fmt.Errorf("invalid number of list elements: %v", c)
+	}
+}
+
 func decodeShort(hash, elems []byte) (node, error) {
 	kbuf, rest, err := rlp.SplitString(elems)
 	if err != nil {
@@ -207,7 +244,8 @@ func decodeRef(buf []byte) (node, []byte, error) {
 			err := fmt.Errorf("oversized embedded node (size is %d bytes, want size < %d)", size, hashLen)
 			return nil, buf, err
 		}
-		n, err := decodeNode(nil, buf)
+		//n, err := decodeNode(nil, buf)
+		n, err := decodeNodeUnsafe(nil, buf)
 		return n, rest, err
 	case kind == rlp.String && len(val) == 0:
 		// empty node

trie/secure_trie.go

@@ -273,6 +273,12 @@ func (t *StateTrie) NodeIterator(start []byte) NodeIterator {
 	return t.trie.NodeIterator(start)
 }
 
+// MustNodeIterator is a wrapper of NodeIterator and will omit any encountered
+// error but just print out an error message.
+func (t *StateTrie) MustNodeIterator(start []byte) NodeIterator {
+	return t.trie.MustNodeIterator(start)
+}
+
 // hashKey returns the hash of key as an ephemeral buffer.
 // The caller must not hold onto the return value because it will become
 // invalid on the next call to hashKey or secKey.