trie, les, tests, core: implement trie tracer #24403

all: rework trie committer #25320

core/state/database.go

@@ -87,7 +87,14 @@ type Trie interface {
 	// TryDeleteAccount abstracts an account deletion from the trie.
 	TryDeleteAccount(key []byte) error
 
-	Commit(onleaf trie.LeafCallback) (common.Hash, int, error)
+	// Commit collects all dirty nodes in the trie and replace them with the
+	// corresponding node hash. All collected nodes(including dirty leaves if
+	// collectLeaf is true) will be encapsulated into a nodeset for return.
+	// The returned nodeset can be nil if the trie is clean(nothing to commit).
+	// Once the trie is committed, it's not usable anymore. A new trie must
+	// be created with new root and updated trie database for following usage
+	Commit(collectLeaf bool) (common.Hash, *trie.NodeSet, error)
+
 	Hash() common.Hash
 	NodeIterator(startKey []byte) trie.NodeIterator
 	GetKey([]byte) []byte // TODO(fjl): remove this when SecureTrie is removed

core/state/metrics.go

@@ -19,10 +19,10 @@ package state
 import "github.com/PlatONnetwork/PlatON-Go/metrics"
 
 var (
-	accountUpdatedMeter   = metrics.NewRegisteredMeter("state/update/account", nil)
-	storageUpdatedMeter   = metrics.NewRegisteredMeter("state/update/storage", nil)
-	accountDeletedMeter   = metrics.NewRegisteredMeter("state/delete/account", nil)
-	storageDeletedMeter   = metrics.NewRegisteredMeter("state/delete/storage", nil)
-	accountCommittedMeter = metrics.NewRegisteredMeter("state/commit/account", nil)
-	storageCommittedMeter = metrics.NewRegisteredMeter("state/commit/storage", nil)
+	accountUpdatedMeter        = metrics.NewRegisteredMeter("state/update/account", nil)
+	storageUpdatedMeter        = metrics.NewRegisteredMeter("state/update/storage", nil)
+	accountDeletedMeter        = metrics.NewRegisteredMeter("state/delete/account", nil)
+	storageDeletedMeter        = metrics.NewRegisteredMeter("state/delete/storage", nil)
+	accountTrieCommittedMeter  = metrics.NewRegisteredMeter("state/commit/accountnodes", nil)
+	storageTriesCommittedMeter = metrics.NewRegisteredMeter("state/commit/storagenodes", nil)
 )

core/state/snapshot/generate.go

@@ -26,14 +26,15 @@ import (
 	"github.com/PlatONnetwork/PlatON-Go/common/hexutil"
 	"github.com/PlatONnetwork/PlatON-Go/ethdb/memorydb"
 
+	"github.com/VictoriaMetrics/fastcache"
+
 	"github.com/PlatONnetwork/PlatON-Go/common"
 	"github.com/PlatONnetwork/PlatON-Go/core/rawdb"
 	"github.com/PlatONnetwork/PlatON-Go/crypto"
 	"github.com/PlatONnetwork/PlatON-Go/ethdb"
 	"github.com/PlatONnetwork/PlatON-Go/log"
 	"github.com/PlatONnetwork/PlatON-Go/rlp"
 	"github.com/PlatONnetwork/PlatON-Go/trie"
-	"github.com/VictoriaMetrics/fastcache"
 )
 
 var (
@@ -73,7 +74,7 @@ func generateSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache i
 	rawdb.WriteSnapshotRoot(batch, root)
 	journalProgress(batch, genMarker, stats)
 	if err := batch.Write(); err != nil {
-		log.Crit("Failed to write initialized state marker ", "err", err)
+		log.Crit("Failed to write initialized state marker", "err", err)
 	}
 	base := &diskLayer{
 		diskdb:     diskdb,
@@ -368,7 +369,10 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, owner common.Hash, roo
 		for i, key := range result.keys {
 			snapTrie.Update(key, result.vals[i])
 		}
-		root, _, _ := snapTrie.Commit(nil)
+		root, nodes, _ := snapTrie.Commit(false)
+		if nodes != nil {
+			snapTrieDb.Update(trie.NewWithNodeSet(nodes))
+		}
 		snapTrieDb.Commit(root, false, false)
 	}
 	// Construct the trie for state iteration, reuse the trie
@@ -608,7 +612,6 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er
 		if accMarker != nil && bytes.Equal(marker, accMarker) && len(dl.genMarker) > common.HashLength {
 			marker = dl.genMarker[:]
 		}
-
 		// If we've exceeded our batch allowance or termination was requested, flush to disk
 		if err := dl.checkAndFlush(ctx, marker); err != nil {
 			return err

core/state/state_object.go

@@ -32,6 +32,7 @@ import (
 	"github.com/PlatONnetwork/PlatON-Go/crypto"
 	"github.com/PlatONnetwork/PlatON-Go/metrics"
 	"github.com/PlatONnetwork/PlatON-Go/rlp"
+	"github.com/PlatONnetwork/PlatON-Go/trie"
 )
 
 var emptyCodeHash = crypto.Keccak256(nil)
@@ -390,31 +391,30 @@ func (s *stateObject) updateRoot(db Database) {
 	if metrics.EnabledExpensive {
 		defer func(start time.Time) { s.db.StorageHashes += time.Since(start) }(time.Now())
 	}
-	//s.data.Root = s.trie.Hash()
 	s.data.Root = s.trie.Hash()
 }
 
 // CommitTrie the storage trie of the object to db.
 // This updates the trie root.
-func (s *stateObject) CommitTrie(db Database) (int, error) {
+func (s *stateObject) CommitTrie(db Database) (*trie.NodeSet, error) {
 	// If nothing changed, don't bother with hashing anything
 	if s.updateTrie(db) == nil {
-		return 0, nil
+		return nil, nil
 	}
 	if s.dbErr != nil {
-		return 0, s.dbErr
+		return nil, s.dbErr
 	}
 
 	// Track the amount of time wasted on committing the storage trie
 	if metrics.EnabledExpensive {
 		defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now())
 	}
-	root, committed, err := s.trie.Commit(nil)
+	root, nodes, err := s.trie.Commit(false)
 
 	if err == nil {
 		s.data.Root = root
 	}
-	return committed, err
+	return nodes, err
 }
 
 // AddBalance adds amount to s's balance.

core/state/statedb.go

@@ -1112,7 +1112,7 @@ func (s *StateDB) GetRefund() uint64 {
 	return s.refund
 }
 
-// Finalise finalises the state by removing the self destructed objects and clears
+// Finalise finalises the state by removing the destructed objects and clears
 // the journal as well as the refunds. Finalise, however, will not push any updates
 // into the tries just yet. Only IntermediateRoot or Commit will do that.
 func (s *StateDB) Finalise(deleteEmptyObjects bool) {
@@ -1314,7 +1314,11 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
 	s.db.TrieDB().IncrVersion()
 
 	// Commit objects to the trie, measuring the elapsed time
-	var storageCommitted int
+	var (
+		accountTrieNodes int
+		storageTrieNodes int
+		nodes            = trie.NewMergedNodeSet()
+	)
 	codeWriter := s.db.DiskDB().NewBatch()
 	for addr := range s.stateObjectsDirty {
 		if obj := s.stateObjects[addr]; !obj.deleted {
@@ -1324,11 +1328,17 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
 				obj.dirtyCode = false
 			}
 			// Write any storage changes in the state object to its storage trie
-			committed, err := obj.CommitTrie(s.db)
+			set, err := obj.CommitTrie(s.db)
 			if err != nil {
 				return common.Hash{}, err
 			}
-			storageCommitted += committed
+			// Merge the dirty nodes of storage trie into global set
+			if set != nil {
+				if err := nodes.Merge(set); err != nil {
+					return common.Hash{}, err
+				}
+				storageTrieNodes += set.Len()
+			}
 		}
 	}
 	if len(s.stateObjectsDirty) > 0 {
@@ -1345,28 +1355,26 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
 		start = time.Now()
 	}
 	// Write trie changes.
-	root, accountCommitted, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash, _ []byte) error {
-		var account types.StateAccount
-		if err := rlp.DecodeBytes(leaf, &account); err != nil {
-			return nil
-		}
-		if account.Root != emptyRoot {
-			s.db.TrieDB().Reference(account.Root, parent)
-		}
-		return nil
-	})
+	root, set, err := s.trie.Commit(true)
 	if err != nil {
 		return common.Hash{}, err
 	}
+	// Merge the dirty nodes of account trie into global set
+	if set != nil {
+		if err := nodes.Merge(set); err != nil {
+			return common.Hash{}, err
+		}
+		accountTrieNodes = set.Len()
+	}
 	if metrics.EnabledExpensive {
 		s.AccountCommits += time.Since(start)
 
 		accountUpdatedMeter.Mark(int64(s.AccountUpdated))
 		storageUpdatedMeter.Mark(int64(s.StorageUpdated))
 		accountDeletedMeter.Mark(int64(s.AccountDeleted))
 		storageDeletedMeter.Mark(int64(s.StorageDeleted))
-		accountCommittedMeter.Mark(int64(accountCommitted))
-		storageCommittedMeter.Mark(int64(storageCommitted))
+		accountTrieCommittedMeter.Mark(int64(accountTrieNodes))
+		storageTriesCommittedMeter.Mark(int64(storageTrieNodes))
 		s.AccountUpdated, s.AccountDeleted = 0, 0
 		s.StorageUpdated, s.StorageDeleted = 0, 0
 	}
@@ -1393,47 +1401,15 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
 	if len(s.stateObjectsDestruct) > 0 {
 		s.stateObjectsDestruct = make(map[common.Address]struct{})
 	}
+	if err := s.db.TrieDB().Update(nodes); err != nil {
+		return common.Hash{}, err
+	}
 	return root, err
 }
 
-func (s *StateDB) SetInt32(addr common.Address, key []byte, value int32) {
-	s.SetState(addr, key, common.Int32ToBytes(value))
-}
-func (s *StateDB) SetInt64(addr common.Address, key []byte, value int64) {
-	s.SetState(addr, key, common.Int64ToBytes(value))
-}
-func (s *StateDB) SetFloat32(addr common.Address, key []byte, value float32) {
-	s.SetState(addr, key, common.Float32ToBytes(value))
-}
-func (s *StateDB) SetFloat64(addr common.Address, key []byte, value float64) {
-	s.SetState(addr, key, common.Float64ToBytes(value))
-}
 func (s *StateDB) SetString(addr common.Address, key []byte, value string) {
 	s.SetState(addr, key, []byte(value))
 }
-func (s *StateDB) SetByte(addr common.Address, key []byte, value byte) {
-	s.SetState(addr, key, []byte{value})
-}
-
-func (s *StateDB) GetInt32(addr common.Address, key []byte) int32 {
-	return common.BytesToInt32(s.GetState(addr, key))
-}
-func (s *StateDB) GetInt64(addr common.Address, key []byte) int64 {
-	return common.BytesToInt64(s.GetState(addr, key))
-}
-func (s *StateDB) GetFloat32(addr common.Address, key []byte) float32 {
-	return common.BytesToFloat32(s.GetState(addr, key))
-}
-func (s *StateDB) GetFloat64(addr common.Address, key []byte) float64 {
-	return common.BytesToFloat64(s.GetState(addr, key))
-}
-func (s *StateDB) GetString(addr common.Address, key []byte) string {
-	return string(s.GetState(addr, key))
-}
-func (s *StateDB) GetByte(addr common.Address, key []byte) byte {
-	ret := s.GetState(addr, key)
-	return ret[0]
-}
 
 func (s *StateDB) AddMinerEarnings(addr common.Address, amount *big.Int) {
 	stateObject := s.GetOrNewStateObject(addr)