From b37708a09479314cf3c5fbf9a8f06c1a79911abe Mon Sep 17 00:00:00 2001
From: niuxiaojie81 <85773309@qq.com>
Date: Wed, 20 Nov 2024 15:02:56 +0800
Subject: [PATCH] trie, les, tests, core: implement trie tracer #24403 all:
rework trie committer #25320
---
core/state/database.go | 9 +-
core/state/metrics.go | 12 +-
core/state/snapshot/generate.go | 11 +-
core/state/snapshot/generate_test.go | 170 +++++++-------------
core/state/state_object.go | 12 +-
core/state/statedb.go | 78 ++++-----
eth/protocols/snap/sync_test.go | 113 +++++++++----
tests/fuzzers/trie/trie-fuzzer.go | 29 ++--
trie/committer.go | 153 ++++++++++--------
trie/database.go | 48 +++++-
trie/iterator.go | 3 +-
trie/iterator_test.go | 58 ++++---
trie/nodeset.go | 94 +++++++++++
trie/proof.go | 16 +-
trie/secure_trie.go | 4 +-
trie/secure_trie_test.go | 20 ++-
trie/sync_test.go | 13 +-
trie/trie.go | 118 +++++++++-----
trie/trie_test.go | 227 ++++++++++++++++++---------
trie/util_test.go | 124 +++++++++++++++
trie/utils.go | 167 ++++++++++++++++++++
21 files changed, 1027 insertions(+), 452 deletions(-)
create mode 100644 trie/nodeset.go
create mode 100644 trie/util_test.go
create mode 100644 trie/utils.go
diff --git a/core/state/database.go b/core/state/database.go
index 865ecae2c3..9c7abe0e7c 100644
--- a/core/state/database.go
+++ b/core/state/database.go
@@ -87,7 +87,14 @@ type Trie interface {
// TryDeleteAccount abstracts an account deletion from the trie.
TryDeleteAccount(key []byte) error
- Commit(onleaf trie.LeafCallback) (common.Hash, int, error)
+ // Commit collects all dirty nodes in the trie and replace them with the
+ // corresponding node hash. All collected nodes(including dirty leaves if
+ // collectLeaf is true) will be encapsulated into a nodeset for return.
+ // The returned nodeset can be nil if the trie is clean(nothing to commit).
+ // Once the trie is committed, it's not usable anymore. A new trie must
+ // be created with new root and updated trie database for following usage
+ Commit(collectLeaf bool) (common.Hash, *trie.NodeSet, error)
+
Hash() common.Hash
NodeIterator(startKey []byte) trie.NodeIterator
GetKey([]byte) []byte // TODO(fjl): remove this when SecureTrie is removed
diff --git a/core/state/metrics.go b/core/state/metrics.go
index da5df35cd4..6e6e407123 100644
--- a/core/state/metrics.go
+++ b/core/state/metrics.go
@@ -19,10 +19,10 @@ package state
import "github.com/PlatONnetwork/PlatON-Go/metrics"
var (
- accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil)
- storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil)
- accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil)
- storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil)
- accountCommittedMeter = metrics.NewRegisteredMeter("state/commit/account", nil)
- storageCommittedMeter = metrics.NewRegisteredMeter("state/commit/storage", nil)
+ accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil)
+ storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil)
+ accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil)
+ storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil)
+ accountTrieCommittedMeter = metrics.NewRegisteredMeter("state/commit/accountnodes", nil)
+ storageTriesCommittedMeter = metrics.NewRegisteredMeter("state/commit/storagenodes", nil)
)
diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go
index 9c6f62d87f..532c756403 100644
--- a/core/state/snapshot/generate.go
+++ b/core/state/snapshot/generate.go
@@ -26,6 +26,8 @@ import (
"github.com/PlatONnetwork/PlatON-Go/common/hexutil"
"github.com/PlatONnetwork/PlatON-Go/ethdb/memorydb"
+ "github.com/VictoriaMetrics/fastcache"
+
"github.com/PlatONnetwork/PlatON-Go/common"
"github.com/PlatONnetwork/PlatON-Go/core/rawdb"
"github.com/PlatONnetwork/PlatON-Go/crypto"
@@ -33,7 +35,6 @@ import (
"github.com/PlatONnetwork/PlatON-Go/log"
"github.com/PlatONnetwork/PlatON-Go/rlp"
"github.com/PlatONnetwork/PlatON-Go/trie"
- "github.com/VictoriaMetrics/fastcache"
)
var (
@@ -73,7 +74,7 @@ func generateSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache i
rawdb.WriteSnapshotRoot(batch, root)
journalProgress(batch, genMarker, stats)
if err := batch.Write(); err != nil {
- log.Crit("Failed to write initialized state marker ", "err", err)
+ log.Crit("Failed to write initialized state marker", "err", err)
}
base := &diskLayer{
diskdb: diskdb,
@@ -368,7 +369,10 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, owner common.Hash, roo
for i, key := range result.keys {
snapTrie.Update(key, result.vals[i])
}
- root, _, _ := snapTrie.Commit(nil)
+ root, nodes, _ := snapTrie.Commit(false)
+ if nodes != nil {
+ snapTrieDb.Update(trie.NewWithNodeSet(nodes))
+ }
snapTrieDb.Commit(root, false, false)
}
// Construct the trie for state iteration, reuse the trie
@@ -608,7 +612,6 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er
if accMarker != nil && bytes.Equal(marker, accMarker) && len(dl.genMarker) > common.HashLength {
marker = dl.genMarker[:]
}
-
// If we've exceeded our batch allowance or termination was requested, flush to disk
if err := dl.checkAndFlush(ctx, marker); err != nil {
return err
diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go
index 197bba9f5a..cef9dea35d 100644
--- a/core/state/snapshot/generate_test.go
+++ b/core/state/snapshot/generate_test.go
@@ -28,7 +28,6 @@ import (
"github.com/PlatONnetwork/PlatON-Go/common"
"github.com/PlatONnetwork/PlatON-Go/core/rawdb"
"github.com/PlatONnetwork/PlatON-Go/ethdb"
- "github.com/PlatONnetwork/PlatON-Go/ethdb/memorydb"
"github.com/PlatONnetwork/PlatON-Go/log"
"github.com/PlatONnetwork/PlatON-Go/rlp"
"github.com/PlatONnetwork/PlatON-Go/trie"
@@ -145,6 +144,7 @@ type testHelper struct {
diskdb ethdb.Database
triedb *trie.Database
accTrie *trie.StateTrie
+ nodes *trie.MergedNodeSet
}
func newHelper() *testHelper {
@@ -155,6 +155,7 @@ func newHelper() *testHelper {
diskdb: diskdb,
triedb: triedb,
accTrie: accTrie,
+ nodes: trie.NewMergedNodeSet(),
}
}
@@ -186,17 +187,22 @@ func (t *testHelper) makeStorageTrie(stateRoot, owner common.Hash, keys []string
for i, k := range keys {
stTrie.Update([]byte(k), []byte(vals[i]))
}
- var root common.Hash
if !commit {
- root = stTrie.Hash()
- } else {
- root, _, _ = stTrie.Commit(nil)
+ return stTrie.Hash().Bytes()
+ }
+ root, nodes, _ := stTrie.Commit(false)
+ if nodes != nil {
+ t.nodes.Merge(nodes)
}
return root.Bytes()
}
func (t *testHelper) Commit() common.Hash {
- root, _, _ := t.accTrie.Commit(nil)
+ root, nodes, _ := t.accTrie.Commit(true)
+ if nodes != nil {
+ t.nodes.Merge(nodes)
+ }
+ t.triedb.Update(t.nodes)
t.triedb.Commit(root, false, true)
return root
}
@@ -376,29 +382,19 @@ func TestGenerateCorruptAccountTrie(t *testing.T) {
// We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts,
// without any storage slots to keep the test smaller.
- var (
- diskdb = memorydb.New()
- triedb = trie.NewDatabase(diskdb)
- )
- tr, _ := trie.NewSecure(common.Hash{}, common.Hash{}, triedb)
- acc := &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
- val, _ := rlp.EncodeToBytes(acc)
- tr.Update([]byte("acc-1"), val) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074
+ helper := newHelper()
- acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
- val, _ = rlp.EncodeToBytes(acc)
- tr.Update([]byte("acc-2"), val) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
+ helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074
+ helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
+ helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4
- acc = &Account{Balance: big.NewInt(3), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
- val, _ = rlp.EncodeToBytes(acc)
- tr.Update([]byte("acc-3"), val) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4
- tr.Commit(nil) // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978
+ helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978
// Delete an account trie leaf and ensure the generator chokes
- triedb.Commit(common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978"), false, true)
- diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes())
+ helper.triedb.Commit(common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978"), false, true)
+ helper.diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes())
- snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978"))
+ snap := generateSnapshot(helper.diskdb, helper.triedb, 16, common.HexToHash("0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978"))
select {
case <-snap.genPending:
// Snapshot generation succeeded
@@ -420,45 +416,20 @@ func TestGenerateMissingStorageTrie(t *testing.T) {
// We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts,
// two of which also has the same 3-slot storage trie attached.
- var (
- diskdb = memorydb.New()
- triedb = trie.NewDatabase(diskdb)
- )
- stTrie, _ := trie.NewSecure(common.Hash{}, common.Hash{}, triedb)
- stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0
- stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371
- stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78
- stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
-
- accTrie, _ := trie.NewSecure(common.Hash{}, common.Hash{}, triedb)
- acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
- val, _ := rlp.EncodeToBytes(acc)
- accTrie.Update([]byte("acc-1"), val) // 0x547b07c3a71669c00eda14077d85c7fd14575b92d459572540b25b9a11914dcb
-
- acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
- val, _ = rlp.EncodeToBytes(acc)
- accTrie.Update([]byte("acc-2"), val) // 0x70da4ebd7602dd313c936b39000ed9ab7f849986a90ea934f0c3ec4cc9840441
-
- acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
- val, _ = rlp.EncodeToBytes(acc)
- accTrie.Update([]byte("acc-3"), val) // 0xf73118e0254ce091588d66038744a0afae5f65a194de67cff310c683ae43329e
- accTrie.Commit(nil) // Root: 0xa819054cfef894169a5b56ccc4e5e06f14829d4a57498e8b9fb13ff21491828d
-
- // We can only corrupt the disk database, so flush the tries out
- triedb.Reference(
- common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"),
- common.HexToHash("0x547b07c3a71669c00eda14077d85c7fd14575b92d459572540b25b9a11914dcb"),
- )
- triedb.Reference(
- common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"),
- common.HexToHash("0xf73118e0254ce091588d66038744a0afae5f65a194de67cff310c683ae43329e"),
- )
- triedb.Commit(common.HexToHash("0xa819054cfef894169a5b56ccc4e5e06f14829d4a57498e8b9fb13ff21491828d"), false, true)
+ helper := newHelper()
+
+ stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
+ helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
+ helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
+ stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
+ helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
+
+ root := helper.Commit()
// Delete a storage trie root and ensure the generator chokes
- diskdb.Delete(common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67").Bytes())
+ helper.diskdb.Delete(stRoot)
- snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xa819054cfef894169a5b56ccc4e5e06f14829d4a57498e8b9fb13ff21491828d"))
+ snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
select {
case <-snap.genPending:
// Snapshot generation succeeded
@@ -479,45 +450,20 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
// We can't use statedb to make a test trie (circular dependency), so make
// a fake one manually. We're going with a small account trie of 3 accounts,
// two of which also has the same 3-slot storage trie attached.
- var (
- diskdb = memorydb.New()
- triedb = trie.NewDatabase(diskdb)
- )
- stTrie, _ := trie.NewSecure(common.Hash{}, common.Hash{}, triedb)
- stTrie.Update([]byte("key-1"), []byte("val-1")) // 0x1314700b81afc49f94db3623ef1df38f3ed18b73a1b7ea2f6c095118cf6118a0
- stTrie.Update([]byte("key-2"), []byte("val-2")) // 0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371
- stTrie.Update([]byte("key-3"), []byte("val-3")) // 0x51c71a47af0695957647fb68766d0becee77e953df17c29b3c2f25436f055c78
- stTrie.Commit(nil) // Root: 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
-
- accTrie, _ := trie.NewSecure(common.Hash{}, common.Hash{}, triedb)
- acc := &Account{Balance: big.NewInt(1), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
- val, _ := rlp.EncodeToBytes(acc)
- accTrie.Update([]byte("acc-1"), val) // 0x547b07c3a71669c00eda14077d85c7fd14575b92d459572540b25b9a11914dcb
-
- acc = &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}
- val, _ = rlp.EncodeToBytes(acc)
- accTrie.Update([]byte("acc-2"), val) // 0x70da4ebd7602dd313c936b39000ed9ab7f849986a90ea934f0c3ec4cc9840441
-
- acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()}
- val, _ = rlp.EncodeToBytes(acc)
- accTrie.Update([]byte("acc-3"), val) // 0xf73118e0254ce091588d66038744a0afae5f65a194de67cff310c683ae43329e
- accTrie.Commit(nil) // Root: 0xa819054cfef894169a5b56ccc4e5e06f14829d4a57498e8b9fb13ff21491828d
-
- // We can only corrupt the disk database, so flush the tries out
- triedb.Reference(
- common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"),
- common.HexToHash("0x547b07c3a71669c00eda14077d85c7fd14575b92d459572540b25b9a11914dcb"),
- )
- triedb.Reference(
- common.HexToHash("0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67"),
- common.HexToHash("0xf73118e0254ce091588d66038744a0afae5f65a194de67cff310c683ae43329e"),
- )
- triedb.Commit(common.HexToHash("0xa819054cfef894169a5b56ccc4e5e06f14829d4a57498e8b9fb13ff21491828d"), false, true)
+ helper := newHelper()
+
+ stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
+ helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
+ helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
+ stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
+ helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2
+
+ root := helper.Commit()
// Delete a storage trie leaf and ensure the generator chokes
- diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes())
+ helper.diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes())
- snap := generateSnapshot(diskdb, triedb, 16, common.HexToHash("0xa819054cfef894169a5b56ccc4e5e06f14829d4a57498e8b9fb13ff21491828d"))
+ snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
select {
case <-snap.genPending:
// Snapshot generation succeeded
@@ -548,12 +494,12 @@ func TestGenerateWithExtraAccounts(t *testing.T) {
// Identical in the snap
key := hashData([]byte("acc-1"))
- rawdb.WriteAccountSnapshot(helper.diskdb, key, val)
- rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-1")), []byte("val-1"))
- rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-2")), []byte("val-2"))
- rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-3")), []byte("val-3"))
- rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-4")), []byte("val-4"))
- rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("key-5")), []byte("val-5"))
+ rawdb.WriteAccountSnapshot(helper.triedb.DiskDB(), key, val)
+ rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-1")), []byte("val-1"))
+ rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-2")), []byte("val-2"))
+ rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-3")), []byte("val-3"))
+ rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-4")), []byte("val-4"))
+ rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("key-5")), []byte("val-5"))
}
{
// Account two exists only in the snapshot
@@ -565,15 +511,15 @@ func TestGenerateWithExtraAccounts(t *testing.T) {
acc := &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}
val, _ := rlp.EncodeToBytes(acc)
key := hashData([]byte("acc-2"))
- rawdb.WriteAccountSnapshot(helper.diskdb, key, val)
- rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("b-key-1")), []byte("b-val-1"))
- rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("b-key-2")), []byte("b-val-2"))
- rawdb.WriteStorageSnapshot(helper.diskdb, key, hashData([]byte("b-key-3")), []byte("b-val-3"))
+ rawdb.WriteAccountSnapshot(helper.triedb.DiskDB(), key, val)
+ rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-1")), []byte("b-val-1"))
+ rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-2")), []byte("b-val-2"))
+ rawdb.WriteStorageSnapshot(helper.triedb.DiskDB(), key, hashData([]byte("b-key-3")), []byte("b-val-3"))
}
root := helper.Commit()
// To verify the test: If we now inspect the snap db, there should exist extraneous storage items
- if data := rawdb.ReadStorageSnapshot(helper.diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data == nil {
+ if data := rawdb.ReadStorageSnapshot(helper.triedb.DiskDB(), hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data == nil {
t.Fatalf("expected snap storage to exist")
}
snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
@@ -591,7 +537,7 @@ func TestGenerateWithExtraAccounts(t *testing.T) {
snap.genAbort <- stop
<-stop
// If we now inspect the snap db, there should exist no extraneous storage items
- if data := rawdb.ReadStorageSnapshot(helper.diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil {
+ if data := rawdb.ReadStorageSnapshot(helper.triedb.DiskDB(), hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil {
t.Fatalf("expected slot to be removed, got %v", string(data))
}
}
@@ -868,10 +814,12 @@ func populateDangling(disk ethdb.KeyValueStore) {
// This test will populate some dangling storages to see if they can be cleaned up.
func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
var helper = newHelper()
- stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
+ stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
+
+ helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"})
@@ -901,10 +849,12 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
// This test will populate some dangling storages to see if they can be cleaned up.
func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) {
var helper = newHelper()
- stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
+ stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()})
helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()})
+
+ helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()})
populateDangling(helper.diskdb)
diff --git a/core/state/state_object.go b/core/state/state_object.go
index ae66ace569..a3a18de2fb 100644
--- a/core/state/state_object.go
+++ b/core/state/state_object.go
@@ -32,6 +32,7 @@ import (
"github.com/PlatONnetwork/PlatON-Go/crypto"
"github.com/PlatONnetwork/PlatON-Go/metrics"
"github.com/PlatONnetwork/PlatON-Go/rlp"
+ "github.com/PlatONnetwork/PlatON-Go/trie"
)
var emptyCodeHash = crypto.Keccak256(nil)
@@ -390,31 +391,30 @@ func (s *stateObject) updateRoot(db Database) {
if metrics.EnabledExpensive {
defer func(start time.Time) { s.db.StorageHashes += time.Since(start) }(time.Now())
}
- //s.data.Root = s.trie.Hash()
s.data.Root = s.trie.Hash()
}
// CommitTrie the storage trie of the object to db.
// This updates the trie root.
-func (s *stateObject) CommitTrie(db Database) (int, error) {
+func (s *stateObject) CommitTrie(db Database) (*trie.NodeSet, error) {
// If nothing changed, don't bother with hashing anything
if s.updateTrie(db) == nil {
- return 0, nil
+ return nil, nil
}
if s.dbErr != nil {
- return 0, s.dbErr
+ return nil, s.dbErr
}
// Track the amount of time wasted on committing the storage trie
if metrics.EnabledExpensive {
defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now())
}
- root, committed, err := s.trie.Commit(nil)
+ root, nodes, err := s.trie.Commit(false)
if err == nil {
s.data.Root = root
}
- return committed, err
+ return nodes, err
}
// AddBalance adds amount to s's balance.
diff --git a/core/state/statedb.go b/core/state/statedb.go
index 26389b36fe..cf7892d0cc 100644
--- a/core/state/statedb.go
+++ b/core/state/statedb.go
@@ -1112,7 +1112,7 @@ func (s *StateDB) GetRefund() uint64 {
return s.refund
}
-// Finalise finalises the state by removing the self destructed objects and clears
+// Finalise finalises the state by removing the destructed objects and clears
// the journal as well as the refunds. Finalise, however, will not push any updates
// into the tries just yet. Only IntermediateRoot or Commit will do that.
func (s *StateDB) Finalise(deleteEmptyObjects bool) {
@@ -1314,7 +1314,11 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
s.db.TrieDB().IncrVersion()
// Commit objects to the trie, measuring the elapsed time
- var storageCommitted int
+ var (
+ accountTrieNodes int
+ storageTrieNodes int
+ nodes = trie.NewMergedNodeSet()
+ )
codeWriter := s.db.DiskDB().NewBatch()
for addr := range s.stateObjectsDirty {
if obj := s.stateObjects[addr]; !obj.deleted {
@@ -1324,11 +1328,17 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
obj.dirtyCode = false
}
// Write any storage changes in the state object to its storage trie
- committed, err := obj.CommitTrie(s.db)
+ set, err := obj.CommitTrie(s.db)
if err != nil {
return common.Hash{}, err
}
- storageCommitted += committed
+ // Merge the dirty nodes of storage trie into global set
+ if set != nil {
+ if err := nodes.Merge(set); err != nil {
+ return common.Hash{}, err
+ }
+ storageTrieNodes += set.Len()
+ }
}
}
if len(s.stateObjectsDirty) > 0 {
@@ -1345,19 +1355,17 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
start = time.Now()
}
// Write trie changes.
- root, accountCommitted, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash, _ []byte) error {
- var account types.StateAccount
- if err := rlp.DecodeBytes(leaf, &account); err != nil {
- return nil
- }
- if account.Root != emptyRoot {
- s.db.TrieDB().Reference(account.Root, parent)
- }
- return nil
- })
+ root, set, err := s.trie.Commit(true)
if err != nil {
return common.Hash{}, err
}
+ // Merge the dirty nodes of account trie into global set
+ if set != nil {
+ if err := nodes.Merge(set); err != nil {
+ return common.Hash{}, err
+ }
+ accountTrieNodes = set.Len()
+ }
if metrics.EnabledExpensive {
s.AccountCommits += time.Since(start)
@@ -1365,8 +1373,8 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
storageUpdatedMeter.Mark(int64(s.StorageUpdated))
accountDeletedMeter.Mark(int64(s.AccountDeleted))
storageDeletedMeter.Mark(int64(s.StorageDeleted))
- accountCommittedMeter.Mark(int64(accountCommitted))
- storageCommittedMeter.Mark(int64(storageCommitted))
+ accountTrieCommittedMeter.Mark(int64(accountTrieNodes))
+ storageTriesCommittedMeter.Mark(int64(storageTrieNodes))
s.AccountUpdated, s.AccountDeleted = 0, 0
s.StorageUpdated, s.StorageDeleted = 0, 0
}
@@ -1393,47 +1401,15 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) {
if len(s.stateObjectsDestruct) > 0 {
s.stateObjectsDestruct = make(map[common.Address]struct{})
}
+ if err := s.db.TrieDB().Update(nodes); err != nil {
+ return common.Hash{}, err
+ }
return root, err
}
-func (s *StateDB) SetInt32(addr common.Address, key []byte, value int32) {
- s.SetState(addr, key, common.Int32ToBytes(value))
-}
-func (s *StateDB) SetInt64(addr common.Address, key []byte, value int64) {
- s.SetState(addr, key, common.Int64ToBytes(value))
-}
-func (s *StateDB) SetFloat32(addr common.Address, key []byte, value float32) {
- s.SetState(addr, key, common.Float32ToBytes(value))
-}
-func (s *StateDB) SetFloat64(addr common.Address, key []byte, value float64) {
- s.SetState(addr, key, common.Float64ToBytes(value))
-}
func (s *StateDB) SetString(addr common.Address, key []byte, value string) {
s.SetState(addr, key, []byte(value))
}
-func (s *StateDB) SetByte(addr common.Address, key []byte, value byte) {
- s.SetState(addr, key, []byte{value})
-}
-
-func (s *StateDB) GetInt32(addr common.Address, key []byte) int32 {
- return common.BytesToInt32(s.GetState(addr, key))
-}
-func (s *StateDB) GetInt64(addr common.Address, key []byte) int64 {
- return common.BytesToInt64(s.GetState(addr, key))
-}
-func (s *StateDB) GetFloat32(addr common.Address, key []byte) float32 {
- return common.BytesToFloat32(s.GetState(addr, key))
-}
-func (s *StateDB) GetFloat64(addr common.Address, key []byte) float64 {
- return common.BytesToFloat64(s.GetState(addr, key))
-}
-func (s *StateDB) GetString(addr common.Address, key []byte) string {
- return string(s.GetState(addr, key))
-}
-func (s *StateDB) GetByte(addr common.Address, key []byte) byte {
- ret := s.GetState(addr, key)
- return ret[0]
-}
func (s *StateDB) AddMinerEarnings(addr common.Address, amount *big.Int) {
stateObject := s.GetOrNewStateObject(addr)
diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go
index 3ebb7653fb..cd87a16dda 100644
--- a/eth/protocols/snap/sync_test.go
+++ b/eth/protocols/snap/sync_test.go
@@ -1,4 +1,4 @@
-// Copyright 2020 The go-ethereum Authors
+// Copyright 2021 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
@@ -369,8 +369,7 @@ func createStorageRequestResponse(t *testPeer, root common.Hash, accounts []comm
return hashes, slots, proofs
}
-// the createStorageRequestResponseAlwaysProve tests a cornercase, where it always
-//
+// the createStorageRequestResponseAlwaysProve tests a cornercase, where it always
// supplies the proof for the last account, even if it is 'complete'.h
func createStorageRequestResponseAlwaysProve(t *testPeer, root common.Hash, accounts []common.Hash, bOrigin, bLimit []byte, max uint64) (hashes [][]common.Hash, slots [][][]byte, proofs [][]byte) {
var size uint64
@@ -1350,9 +1349,11 @@ func getCodeByHash(hash common.Hash) []byte {
// makeAccountTrieNoStorage spits out a trie, along with the leafs
func makeAccountTrieNoStorage(n int) (*trie.Trie, entrySlice) {
- db := trie.NewDatabase(rawdb.NewMemoryDatabase())
- accTrie := trie.NewEmpty(db)
- var entries entrySlice
+ var (
+ db = trie.NewDatabase(rawdb.NewMemoryDatabase())
+ accTrie = trie.NewEmpty(db)
+ entries entrySlice
+ )
for i := uint64(1); i <= uint64(n); i++ {
value, _ := rlp.EncodeToBytes(&types.StateAccount{
Nonce: i,
@@ -1367,7 +1368,13 @@ func makeAccountTrieNoStorage(n int) (*trie.Trie, entrySlice) {
entries = append(entries, elem)
}
sort.Sort(entries)
- accTrie.Commit(nil)
+
+ // Commit the state changes into db and re-create the trie
+ // for accessing later.
+ root, nodes, _ := accTrie.Commit(false)
+ db.Update(trie.NewWithNodeSet(nodes))
+
+ accTrie, _ = trie.New(common.Hash{}, root, db)
return accTrie, entries
}
@@ -1379,8 +1386,8 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) {
entries entrySlice
boundaries []common.Hash
- db = trie.NewDatabase(rawdb.NewMemoryDatabase())
- trie = trie.NewEmpty(db)
+ db = trie.NewDatabase(rawdb.NewMemoryDatabase())
+ accTrie = trie.NewEmpty(db)
)
// Initialize boundaries
var next common.Hash
@@ -1408,7 +1415,7 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) {
StorageKeyPrefix: big.NewInt(int64(i)).Bytes(),
})
elem := &kv{boundaries[i].Bytes(), value}
- trie.Update(elem.k, elem.v)
+ accTrie.Update(elem.k, elem.v)
entries = append(entries, elem)
}
// Fill other accounts if required
@@ -1421,12 +1428,18 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) {
StorageKeyPrefix: big.NewInt(int64(i)).Bytes(),
})
elem := &kv{key32(i), value}
- trie.Update(elem.k, elem.v)
+ accTrie.Update(elem.k, elem.v)
entries = append(entries, elem)
}
sort.Sort(entries)
- trie.Commit(nil)
- return trie, entries
+
+ // Commit the state changes into db and re-create the trie
+ // for accessing later.
+ root, nodes, _ := accTrie.Commit(false)
+ db.Update(trie.NewWithNodeSet(nodes))
+
+ accTrie, _ = trie.New(common.Hash{}, root, db)
+ return accTrie, entries
}
// makeAccountTrieWithStorageWithUniqueStorage creates an account trie where each accounts
@@ -1436,8 +1449,10 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool)
db = trie.NewDatabase(rawdb.NewMemoryDatabase())
accTrie = trie.NewEmpty(db)
entries entrySlice
+ storageRoots = make(map[common.Hash]common.Hash)
storageTries = make(map[common.Hash]*trie.Trie)
storageEntries = make(map[common.Hash]entrySlice)
+ nodes = trie.NewMergedNodeSet()
)
// Create n accounts in the trie
for i := uint64(1); i <= uint64(accounts); i++ {
@@ -1447,9 +1462,9 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool)
codehash = getCodeHash(i)
}
// Create a storage trie
- stTrie, stEntries := makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), i, db)
- stRoot := stTrie.Hash()
- stTrie.Commit(nil)
+ stRoot, stNodes, stEntries := makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), i, db)
+ nodes.Merge(stNodes)
+
value, _ := rlp.EncodeToBytes(&types.StateAccount{
Nonce: i,
Balance: big.NewInt(int64(i)),
@@ -1461,12 +1476,25 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool)
accTrie.Update(elem.k, elem.v)
entries = append(entries, elem)
- storageTries[common.BytesToHash(key)] = stTrie
+ storageRoots[common.BytesToHash(key)] = stRoot
storageEntries[common.BytesToHash(key)] = stEntries
}
sort.Sort(entries)
- accTrie.Commit(nil)
+ // Commit account trie
+ root, set, _ := accTrie.Commit(true)
+ nodes.Merge(set)
+
+ // Commit gathered dirty nodes into database
+ db.Update(nodes)
+
+ // Re-create tries with new root
+ accTrie, _ = trie.New(common.Hash{}, root, db)
+ for i := uint64(1); i <= uint64(accounts); i++ {
+ key := key32(i)
+ trie, _ := trie.New(common.BytesToHash(key), storageRoots[common.BytesToHash(key)], db)
+ storageTries[common.BytesToHash(key)] = trie
+ }
return accTrie, entries, storageTries, storageEntries
}
@@ -1476,8 +1504,10 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie
db = trie.NewDatabase(rawdb.NewMemoryDatabase())
accTrie = trie.NewEmpty(db)
entries entrySlice
+ storageRoots = make(map[common.Hash]common.Hash)
storageTries = make(map[common.Hash]*trie.Trie)
storageEntries = make(map[common.Hash]entrySlice)
+ nodes = trie.NewMergedNodeSet()
)
// Create n accounts in the trie
for i := uint64(1); i <= uint64(accounts); i++ {
@@ -1488,16 +1518,16 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie
}
// Make a storage trie
var (
- stTrie *trie.Trie
+ stRoot common.Hash
+ stNodes *trie.NodeSet
stEntries entrySlice
)
if boundary {
- stTrie, stEntries = makeBoundaryStorageTrie(common.BytesToHash(key), slots, db)
+ stRoot, stNodes, stEntries = makeBoundaryStorageTrie(common.BytesToHash(key), slots, db)
} else {
- stTrie, stEntries = makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), 0, db)
+ stRoot, stNodes, stEntries = makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), 0, db)
}
- stRoot := stTrie.Hash()
- stTrie.Commit(nil)
+ nodes.Merge(stNodes)
value, _ := rlp.EncodeToBytes(&types.StateAccount{
Nonce: i,
@@ -1509,19 +1539,40 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie
elem := &kv{key, value}
accTrie.Update(elem.k, elem.v)
entries = append(entries, elem)
+
// we reuse the same one for all accounts
- storageTries[common.BytesToHash(key)] = stTrie
+ storageRoots[common.BytesToHash(key)] = stRoot
storageEntries[common.BytesToHash(key)] = stEntries
}
sort.Sort(entries)
- accTrie.Commit(nil)
+
+ // Commit account trie
+ root, set, _ := accTrie.Commit(true)
+ nodes.Merge(set)
+
+ // Commit gathered dirty nodes into database
+ db.Update(nodes)
+
+ // Re-create tries with new root
+ accTrie, err := trie.New(common.Hash{}, root, db)
+ if err != nil {
+ panic(err)
+ }
+ for i := uint64(1); i <= uint64(accounts); i++ {
+ key := key32(i)
+ trie, err := trie.New(common.BytesToHash(key), storageRoots[common.BytesToHash(key)], db)
+ if err != nil {
+ panic(err)
+ }
+ storageTries[common.BytesToHash(key)] = trie
+ }
return accTrie, entries, storageTries, storageEntries
}
// makeStorageTrieWithSeed fills a storage trie with n items, returning the
// not-yet-committed trie and the sorted entries. The seeds can be used to ensure
// that tries are unique.
-func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Database) (*trie.Trie, entrySlice) {
+func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Database) (common.Hash, *trie.NodeSet, entrySlice) {
trie, _ := trie.New(owner, common.Hash{}, db)
var entries entrySlice
for i := uint64(1); i <= n; i++ {
@@ -1537,14 +1588,14 @@ func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Databas
entries = append(entries, elem)
}
sort.Sort(entries)
- trie.Commit(nil)
- return trie, entries
+ root, nodes, _ := trie.Commit(false)
+ return root, nodes, entries
}
// makeBoundaryStorageTrie constructs a storage trie. Instead of filling
// storage slots normally, this function will fill a few slots which have
// boundary hash.
-func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (*trie.Trie, entrySlice) {
+func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (common.Hash, *trie.NodeSet, entrySlice) {
var (
entries entrySlice
boundaries []common.Hash
@@ -1588,8 +1639,8 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (*trie
entries = append(entries, elem)
}
sort.Sort(entries)
- trie.Commit(nil)
- return trie, entries
+ root, nodes, _ := trie.Commit(false)
+ return root, nodes, entries
}
func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) {
diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go
index 89687db2bd..2bdcc55229 100644
--- a/tests/fuzzers/trie/trie-fuzzer.go
+++ b/tests/fuzzers/trie/trie-fuzzer.go
@@ -20,6 +20,7 @@ import (
"bytes"
"encoding/binary"
"fmt"
+
"github.com/PlatONnetwork/PlatON-Go/common"
"github.com/PlatONnetwork/PlatON-Go/ethdb/memorydb"
"github.com/PlatONnetwork/PlatON-Go/trie"
@@ -50,9 +51,8 @@ const (
opUpdate = iota
opDelete
opGet
- opCommit
opHash
- opReset
+ opCommit
opItercheckhash
opProve
opMax // boundary value, not an actual op
@@ -83,11 +83,9 @@ func (ds *dataSource) Ended() bool {
}
func Generate(input []byte) randTest {
-
var allKeys [][]byte
r := newDataSource(input)
genKey := func() []byte {
-
if len(allKeys) < 2 || r.readByte() < 0x0f {
// new key
key := make([]byte, r.readByte()%50)
@@ -102,7 +100,6 @@ func Generate(input []byte) randTest {
var steps randTest
for i := 0; !r.Ended(); i++ {
-
step := randTestStep{op: int(r.readByte()) % opMax}
switch step.op {
case opUpdate:
@@ -123,10 +120,8 @@ func Generate(input []byte) randTest {
// The function must return
// 1 if the fuzzer should increase priority of the
-//
-// given input during subsequent fuzzing (for example, the input is lexically
-// correct and was parsed successfully);
-//
+// given input during subsequent fuzzing (for example, the input is lexically
+// correct and was parsed successfully);
// -1 if the input must not be added to corpus even if gives new coverage; and
// 0 otherwise
// other values are reserved for future use.
@@ -142,10 +137,9 @@ func Fuzz(input []byte) int {
}
func runRandTest(rt randTest) error {
-
triedb := trie.NewDatabase(memorydb.New())
- tr, _ := trie.New(common.Hash{}, common.Hash{}, triedb)
+ tr := trie.NewEmpty(triedb)
values := make(map[string]string) // tracks content of the trie
for i, step := range rt {
@@ -162,22 +156,25 @@ func runRandTest(rt randTest) error {
if string(v) != want {
rt[i].err = fmt.Errorf("mismatch for key %#x, got %#x want %#x", step.key, v, want)
}
- case opCommit:
- _, _, rt[i].err = tr.Commit(nil)
case opHash:
tr.Hash()
- case opReset:
- hash, _, err := tr.Commit(nil)
+ case opCommit:
+ hash, nodes, err := tr.Commit(false)
if err != nil {
return err
}
+ if nodes != nil {
+ if err := triedb.Update(trie.NewWithNodeSet(nodes)); err != nil {
+ return err
+ }
+ }
newtr, err := trie.New(common.Hash{}, hash, triedb)
if err != nil {
return err
}
tr = newtr
case opItercheckhash:
- checktr, _ := trie.New(common.Hash{}, common.Hash{}, triedb)
+ checktr := trie.NewEmpty(triedb)
it := trie.NewIterator(tr.NodeIterator(nil))
for it.Next() {
checktr.Update(it.Key, it.Value)
diff --git a/trie/committer.go b/trie/committer.go
index 8cb05d9360..1614723745 100644
--- a/trie/committer.go
+++ b/trie/committer.go
@@ -1,4 +1,4 @@
-// Copyright 2019 The go-ethereum Authors
+// Copyright 2020 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
@@ -18,7 +18,6 @@ package trie
import (
"fmt"
- "sync"
"golang.org/x/crypto/sha3"
@@ -27,64 +26,55 @@ import (
"github.com/PlatONnetwork/PlatON-Go/rlp"
)
-// committer is a type used for the trie Commit operation. A committer has some
-// internal preallocated temp space, and also a callback that is invoked when
-// leaves are committed. The leafs are passed through the `leafCh`, to allow
-// some level of parallelism.
-// By 'some level' of parallelism, it's still the case that all leaves will be
-// processed sequentially - onleaf will never be called in parallel or out of order.
-type committer struct {
- sha crypto.KeccakState
- tmp sliceBuffer
- encbuf rlp.EncoderBuffer
- onleaf LeafCallback
+// leaf represents a trie leaf node
+type leaf struct {
+ blob []byte // raw blob of leaf
+ parent common.Hash // the hash of parent node
}
-// committers live in a global sync.Pool
-var committerPool = sync.Pool{
- New: func() interface{} {
- return &committer{
- tmp: make(sliceBuffer, 0, 550), // cap is as large as a full fullNode.
- sha: sha3.NewLegacyKeccak256().(crypto.KeccakState),
- encbuf: rlp.NewEncoderBuffer(nil),
- }
- },
+// committer is the tool used for the trie Commit operation. The committer will
+// capture all dirty nodes during the commit process and keep them cached in
+// insertion order.
+type committer struct {
+ sha crypto.KeccakState
+ tmp sliceBuffer
+ encbuf rlp.EncoderBuffer
+ nodes *NodeSet
+ collectLeaf bool
}
// newCommitter creates a new committer or picks one from the pool.
-func newCommitter(onleaf LeafCallback) *committer {
- c := committerPool.Get().(*committer)
- c.onleaf = onleaf
- return c
-}
-
-func returnCommitterToPool(c *committer) {
- c.onleaf = nil
- committerPool.Put(c)
+func newCommitter(owner common.Hash, collectLeaf bool) *committer {
+ return &committer{
+ tmp: make(sliceBuffer, 0, 550), // cap is as large as a full fullNode.
+ sha: sha3.NewLegacyKeccak256().(crypto.KeccakState),
+ encbuf: rlp.NewEncoderBuffer(nil),
+ nodes: NewNodeSet(owner),
+ collectLeaf: collectLeaf,
+ }
}
// commit collapses a node down into a hash node and inserts it into the database
-func (c *committer) commit(path []byte, n node, db *Database, force bool) (node, node, int, error) {
+func (c *committer) commit(path []byte, n node, force bool) (node, node, error) {
// If we're not storing the node, just hashing, use available cached data
if hash, dirty := n.cache(); len(hash) != 0 {
if !dirty {
switch n.(type) {
case *fullNode, *shortNode:
- return hash, hash, 0, nil
+ return hash, hash, nil
default:
- return hash, n, 0, nil
+ return hash, n, nil
}
}
}
- var committed int
// Trie not processed yet or needs storage, walk the children
- collapsed, cached, committed, err := c.commitChildren(path, n, db)
+ collapsed, cached, err := c.commitChildren(path, n)
if err != nil {
- return hashNode{}, n, 0, err
+ return hashNode{}, n, err
}
- hashed, err := c.store(collapsed, db, force)
+ hashed, err := c.store(path, collapsed, force)
if err != nil {
- return hashNode{}, n, 0, err
+ return hashNode{}, n, err
}
// Cache the hash of the node for later reuse and remove
// the dirty flag in commit mode. It's fine to assign these values directly
@@ -99,15 +89,11 @@ func (c *committer) commit(path []byte, n node, db *Database, force bool) (node,
*cn.flags.hash = cachedHash
*cn.flags.dirty = false
}
- return hashed, cached, committed + 1, nil
+ return hashed, cached, nil
}
-func (c *committer) commitChildren(path []byte, original node, db *Database) (node, node, int, error) {
- var (
- err error
- committed int
- childCommitted int
- )
+func (c *committer) commitChildren(path []byte, original node) (node, node, error) {
+ var err error
switch n := original.(type) {
case *shortNode:
@@ -117,12 +103,12 @@ func (c *committer) commitChildren(path []byte, original node, db *Database) (no
cached.Key = common.CopyBytes(n.Key)
if _, ok := n.Val.(valueNode); !ok {
- collapsed.Val, cached.Val, committed, err = c.commit(append(path, n.Key...), n.Val, db, false)
+ collapsed.Val, cached.Val, err = c.commit(append(path, n.Key...), n.Val, false)
if err != nil {
- return original, original, 0, err
+ return original, original, err
}
}
- return collapsed, cached, committed, nil
+ return collapsed, cached, nil
case *fullNode:
// Hash the full node's children, caching the newly hashed subtrees
@@ -130,26 +116,25 @@ func (c *committer) commitChildren(path []byte, original node, db *Database) (no
for i := 0; i < 16; i++ {
if n.Children[i] != nil {
- collapsed.Children[i], cached.Children[i], childCommitted, err = c.commit(path, n.Children[i], db, false)
+ collapsed.Children[i], cached.Children[i], err = c.commit(append(path, byte(i)), n.Children[i], false)
if err != nil {
- return original, original, 0, err
+ return original, original, err
}
- committed += childCommitted
}
}
cached.Children[16] = n.Children[16]
- return collapsed, cached, committed, nil
+ return collapsed, cached, nil
default:
// Value and hash nodes don't have children so they're left as were
- return n, original, 0, nil
+ return n, original, nil
}
}
// store hashes the node n and if we have a storage layer specified, it writes
// the key/value pair to it and tracks any node->child references as well as any
// node->external trie references.
-func (c *committer) store(n node, db *Database, force bool) (node, error) {
+func (c *committer) store(path []byte, n node, force bool) (node, error) {
// Don't store hashes or empty nodes.
if _, isHash := n.(hashNode); n == nil || isHash {
return n, nil
@@ -167,28 +152,60 @@ func (c *committer) store(n node, db *Database, force bool) (node, error) {
hash = c.makeHashNode(enc)
}
- // We are pooling the trie nodes into an intermediate memory cache
- hash2 := common.BytesToHash(hash)
- db.lock.Lock()
- db.insert(hash2, estimateSize(n), n)
- db.insertFreshNode(hash2)
- db.lock.Unlock()
+ // We have the hash already, estimate the RLP encoding-size of the node.
+ // The size is used for mem tracking, does not need to be exact
+ var (
+ size = estimateSize(n)
+ nhash = common.BytesToHash(hash)
+ mnode = &memoryNode{
+ hash: nhash,
+ node: simplifyNode(n),
+ size: uint16(size),
+ }
+ )
+ // Collect the dirty node to nodeset for return.
+ c.nodes.add(string(path), mnode)
- // Track external references from account->storage trie
- if c.onleaf != nil {
+ // Collect the corresponding leaf node if it's required. We don't check
+ // full node since it's impossible to store value in fullNode. The key
+ // length of leaves should be exactly same.
+ if c.collectLeaf {
switch n := n.(type) {
case *shortNode:
- if child, ok := n.Val.(valueNode); ok {
- c.onleaf(nil, nil, child, hash2, nil)
+ if val, ok := n.Val.(valueNode); ok {
+ c.nodes.addLeaf(&leaf{blob: val, parent: nhash})
}
case *fullNode:
for i := 0; i < 16; i++ {
- if child, ok := n.Children[i].(valueNode); ok {
- c.onleaf(nil, nil, child, hash2, nil)
+ if val, ok := n.Children[i].(valueNode); ok {
+ c.nodes.addLeaf(&leaf{blob: val, parent: nhash})
}
}
}
}
+ /*
+ // We are pooling the trie nodes into an intermediate memory cache
+ //hash2 := common.BytesToHash(hash)
+ db.lock.Lock()
+ db.insert(hash2, estimateSize(n), n)
+ db.insertFreshNode(hash2)
+ db.lock.Unlock()
+
+ // Track external references from account->storage trie
+ if c.onleaf != nil {
+ switch n := n.(type) {
+ case *shortNode:
+ if child, ok := n.Val.(valueNode); ok {
+ c.onleaf(nil, nil, child, hash2, nil)
+ }
+ case *fullNode:
+ for i := 0; i < 16; i++ {
+ if child, ok := n.Children[i].(valueNode); ok {
+ c.onleaf(nil, nil, child, hash2, nil)
+ }
+ }
+ }
+ }*/
return hash, nil
}
@@ -220,7 +237,7 @@ func (c *committer) makeHashNode(data []byte) hashNode {
// estimateSize estimates the size of an rlp-encoded node, without actually
// rlp-encoding it (zero allocs). This method has been experimentally tried, and with a trie
-// with 1000 leafs, the only errors above 1% are on small shortnodes, where this
+// with 1000 leaves, the only errors above 1% are on small shortnodes, where this
// method overestimates by 2 or 3 bytes (e.g. 37 instead of 35)
func estimateSize(n node) int {
switch n := n.(type) {
diff --git a/trie/database.go b/trie/database.go
index b774e956a1..ef42383ad8 100644
--- a/trie/database.go
+++ b/trie/database.go
@@ -26,6 +26,7 @@ import (
"time"
"github.com/PlatONnetwork/PlatON-Go/core/rawdb"
+ "github.com/PlatONnetwork/PlatON-Go/core/types"
"github.com/VictoriaMetrics/fastcache"
@@ -320,6 +321,10 @@ func NewDatabaseWithConfig(diskdb ethdb.KeyValueStore, config *Config) *Database
return db
}
+func (db *Database) DiskDB() ethdb.KeyValueStore {
+ return db.diskdb
+}
+
func (db *Database) NodeVersion() uint64 {
return db.nodeVersion
}
@@ -336,8 +341,7 @@ func (db *Database) resetFreshNode() {
db.freshNodes = make(map[common.Hash]struct{})
}
-// insert inserts a collapsed trie node into the memory database.
-// The blob size must be specified to allow proper size tracking.
+// insert inserts a simplified trie node into the memory database.
// All nodes inserted by this function will be reference tracked
// and in theory should only used for **trie nodes** insertion.
func (db *Database) insert(hash common.Hash, size int, node node) {
@@ -349,7 +353,8 @@ func (db *Database) insert(hash common.Hash, size int, node node) {
// Create the cached entry for this node
entry := &cachedNode{
- node: simplifyNode(node),
+ //node: simplifyNode(node),
+ node: node,
size: uint16(size),
flushPrev: db.newest,
version: db.NodeVersion(),
@@ -966,6 +971,43 @@ func (c *cleaner) Delete(key []byte) error {
panic("not implemented")
}
+// Update inserts the dirty nodes in provided nodeset into database and
+// link the account trie with multiple storage tries if necessary.
+func (db *Database) Update(nodes *MergedNodeSet) error {
+ db.lock.Lock()
+ defer db.lock.Unlock()
+
+ // Insert dirty nodes into the database. In the same tree, it must be
+ // ensured that children are inserted first, then parent so that children
+ // can be linked with their parent correctly. The order of writing between
+ // different tries(account trie, storage tries) is not required.
+ for owner, subset := range nodes.sets {
+ for _, path := range subset.paths {
+ n, ok := subset.nodes[path]
+ if !ok {
+ return fmt.Errorf("missing node %x %v", owner, path)
+ }
+ db.insert(n.hash, int(n.size), n.node)
+ db.insertFreshNode(n.hash)
+ }
+ }
+ // Link up the account trie and storage trie if the node points
+ // to an account trie leaf.
+ if set, present := nodes.sets[common.Hash{}]; present {
+ for _, n := range set.leaves {
+ var account types.StateAccount
+ if err := rlp.DecodeBytes(n.blob, &account); err != nil {
+ return err
+ }
+ if account.Root != emptyRoot {
+ //db.Reference(account.Root, n.parent)
+ db.reference(account.Root, n.parent)
+ }
+ }
+ }
+ return nil
+}
+
// Size returns the current storage size of the memory cache in front of the
// persistent database layer.
func (db *Database) Size() (common.StorageSize, common.StorageSize) {
diff --git a/trie/iterator.go b/trie/iterator.go
index dc3f727fa0..49d7923c29 100644
--- a/trie/iterator.go
+++ b/trie/iterator.go
@@ -377,8 +377,7 @@ func (it *nodeIterator) resolveHash(hash hashNode, path []byte) (node, error) {
}
}
}
- resolved, err := it.trie.resolveHash(hash, path)
- return resolved, err
+ return it.trie.resolveHash(hash, path)
}
func (it *nodeIterator) resolveBlob(hash hashNode, path []byte) ([]byte, error) {
diff --git a/trie/iterator_test.go b/trie/iterator_test.go
index 164f91d2e9..491128790d 100644
--- a/trie/iterator_test.go
+++ b/trie/iterator_test.go
@@ -33,7 +33,7 @@ import (
)
func TestEmptyIterator(t *testing.T) {
- trie := newEmpty()
+ trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
iter := trie.NodeIterator(nil)
seen := make(map[string]struct{})
@@ -46,7 +46,8 @@ func TestEmptyIterator(t *testing.T) {
}
func TestIterator(t *testing.T) {
- trie := newEmpty()
+ db := NewDatabase(rawdb.NewMemoryDatabase())
+ trie := NewEmpty(db)
vals := []struct{ k, v string }{
{"do", "verb"},
{"ether", "wookiedoo"},
@@ -61,8 +62,13 @@ func TestIterator(t *testing.T) {
all[val.k] = val.v
trie.Update([]byte(val.k), []byte(val.v))
}
- trie.Commit(nil)
+ root, nodes, err := trie.Commit(false)
+ if err != nil {
+ t.Fatalf("Failed to commit trie %v", err)
+ }
+ db.Update(NewWithNodeSet(nodes))
+ trie, _ = New(common.Hash{}, root, db)
found := make(map[string]string)
it := NewIterator(trie.NodeIterator(nil))
for it.Next() {
@@ -82,7 +88,7 @@ type kv struct {
}
func TestIteratorLargeData(t *testing.T) {
- trie := newEmpty()
+ trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
vals := make(map[string]*kv)
for i := byte(0); i < 255; i++ {
@@ -175,7 +181,7 @@ var testdata2 = []kvs{
}
func TestIteratorSeek(t *testing.T) {
- trie := newEmpty()
+ trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
for _, val := range testdata1 {
trie.Update([]byte(val.k), []byte(val.v))
}
@@ -216,17 +222,23 @@ func checkIteratorOrder(want []kvs, it *Iterator) error {
}
func TestDifferenceIterator(t *testing.T) {
- triea := newEmpty()
+ dba := NewDatabase(rawdb.NewMemoryDatabase())
+ triea := NewEmpty(dba)
for _, val := range testdata1 {
triea.Update([]byte(val.k), []byte(val.v))
}
- triea.Commit(nil)
+ rootA, nodesA, _ := triea.Commit(false)
+ dba.Update(NewWithNodeSet(nodesA))
+ triea, _ = New(common.Hash{}, rootA, dba)
- trieb := newEmpty()
+ dbb := NewDatabase(rawdb.NewMemoryDatabase())
+ trieb := NewEmpty(dbb)
for _, val := range testdata2 {
trieb.Update([]byte(val.k), []byte(val.v))
}
- trieb.Commit(nil)
+ rootB, nodesB, _ := trieb.Commit(false)
+ dbb.Update(NewWithNodeSet(nodesB))
+ trieb, _ = New(common.Hash{}, rootB, dbb)
found := make(map[string]string)
di, _ := NewDifferenceIterator(triea.NodeIterator(nil), trieb.NodeIterator(nil))
@@ -252,17 +264,23 @@ func TestDifferenceIterator(t *testing.T) {
}
func TestUnionIterator(t *testing.T) {
- triea := newEmpty()
+ dba := NewDatabase(rawdb.NewMemoryDatabase())
+ triea := NewEmpty(dba)
for _, val := range testdata1 {
triea.Update([]byte(val.k), []byte(val.v))
}
- triea.Commit(nil)
+ rootA, nodesA, _ := triea.Commit(false)
+ dba.Update(NewWithNodeSet(nodesA))
+ triea, _ = New(common.Hash{}, rootA, dba)
- trieb := newEmpty()
+ dbb := NewDatabase(rawdb.NewMemoryDatabase())
+ trieb := NewEmpty(dbb)
for _, val := range testdata2 {
trieb.Update([]byte(val.k), []byte(val.v))
}
- trieb.Commit(nil)
+ rootB, nodesB, _ := trieb.Commit(false)
+ dbb.Update(NewWithNodeSet(nodesB))
+ trieb, _ = New(common.Hash{}, rootB, dbb)
di, _ := NewUnionIterator([]NodeIterator{triea.NodeIterator(nil), trieb.NodeIterator(nil)})
it := NewIterator(di)
@@ -318,7 +336,8 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) {
for _, val := range testdata1 {
tr.Update([]byte(val.k), []byte(val.v))
}
- tr.Commit(nil)
+ _, nodes, _ := tr.Commit(false)
+ triedb.Update(NewWithNodeSet(nodes))
if !memonly {
triedb.Commit(tr.Hash(), true, true)
}
@@ -409,7 +428,8 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) {
for _, val := range testdata1 {
ctr.Update([]byte(val.k), []byte(val.v))
}
- root, _, _ := ctr.Commit(nil)
+ root, nodes, _ := ctr.Commit(false)
+ triedb.Update(NewWithNodeSet(nodes))
if !memonly {
triedb.Commit(root, true, true)
}
@@ -527,14 +547,13 @@ func makeLargeTestTrie() (*Database, *StateTrie, *loggingDb) {
val = crypto.Keccak256(val)
trie.Update(key, val)
}
- trie.Commit(nil)
+ _, nodes, _ := trie.Commit(false)
+ triedb.Update(NewWithNodeSet(nodes))
// Return the generated trie
return triedb, trie, logDb
}
// Tests that the node iterator indeed walks over the entire database contents.
-// After the trie is submitted, the root hangs on the real node instead of the hashnode
-// so the node iterator indeed walks over the entire database contents
func TestNodeIteratorLargeTrie(t *testing.T) {
// Create some arbitrary test trie to iterate
db, trie, logDb := makeLargeTestTrie()
@@ -569,7 +588,8 @@ func TestIteratorNodeBlob(t *testing.T) {
all[val.k] = val.v
trie.Update([]byte(val.k), []byte(val.v))
}
- trie.Commit(nil)
+ _, nodes, _ := trie.Commit(false)
+ triedb.Update(NewWithNodeSet(nodes))
triedb.Cap(0)
found := make(map[common.Hash][]byte)
diff --git a/trie/nodeset.go b/trie/nodeset.go
new file mode 100644
index 0000000000..1a3aebca47
--- /dev/null
+++ b/trie/nodeset.go
@@ -0,0 +1,94 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package trie
+
+import (
+ "fmt"
+
+ "github.com/PlatONnetwork/PlatON-Go/common"
+)
+
+// memoryNode is all the information we know about a single cached trie node
+// in the memory.
+type memoryNode struct {
+ hash common.Hash // Node hash, computed by hashing rlp value
+ size uint16 // Byte size of the useful cached data
+ node node // Cached collapsed trie node, or raw rlp data
+}
+
+// NodeSet contains all dirty nodes collected during the commit operation.
+// Each node is keyed by path. It's not thread-safe to use.
+type NodeSet struct {
+ owner common.Hash // the identifier of the trie
+ paths []string // the path of dirty nodes, sort by insertion order
+ nodes map[string]*memoryNode // the map of dirty nodes, keyed by node path
+ leaves []*leaf // the list of dirty leaves
+}
+
+// NewNodeSet initializes an empty node set to be used for tracking dirty nodes
+// from a specific account or storage trie. The owner is zero for the account
+// trie and the owning account address hash for storage tries.
+func NewNodeSet(owner common.Hash) *NodeSet {
+ return &NodeSet{
+ owner: owner,
+ nodes: make(map[string]*memoryNode),
+ }
+}
+
+// add caches node with provided path and node object.
+func (set *NodeSet) add(path string, node *memoryNode) {
+ set.paths = append(set.paths, path)
+ set.nodes[path] = node
+}
+
+// addLeaf caches the provided leaf node.
+func (set *NodeSet) addLeaf(node *leaf) {
+ set.leaves = append(set.leaves, node)
+}
+
+// Len returns the number of dirty nodes contained in the set.
+func (set *NodeSet) Len() int {
+ return len(set.nodes)
+}
+
+// MergedNodeSet represents a merged dirty node set for a group of tries.
+type MergedNodeSet struct {
+ sets map[common.Hash]*NodeSet
+}
+
+// NewMergedNodeSet initializes an empty merged set.
+func NewMergedNodeSet() *MergedNodeSet {
+ return &MergedNodeSet{sets: make(map[common.Hash]*NodeSet)}
+}
+
+// NewWithNodeSet constructs a merged nodeset with the provided single set.
+func NewWithNodeSet(set *NodeSet) *MergedNodeSet {
+ merged := NewMergedNodeSet()
+ merged.Merge(set)
+ return merged
+}
+
+// Merge merges the provided dirty nodes of a trie into the set. The assumption
+// is held that no duplicated set belonging to the same trie will be merged twice.
+func (set *MergedNodeSet) Merge(other *NodeSet) error {
+ _, present := set.sets[other.owner]
+ if present {
+ return fmt.Errorf("duplicate trie for owner %#x", other.owner)
+ }
+ set.sets[other.owner] = other
+ return nil
+}
diff --git a/trie/proof.go b/trie/proof.go
index f7b3f35939..b485e659df 100644
--- a/trie/proof.go
+++ b/trie/proof.go
@@ -22,6 +22,7 @@ import (
"fmt"
"github.com/PlatONnetwork/PlatON-Go/common"
+ "github.com/PlatONnetwork/PlatON-Go/core/rawdb"
"github.com/PlatONnetwork/PlatON-Go/crypto"
"github.com/PlatONnetwork/PlatON-Go/ethdb"
"github.com/PlatONnetwork/PlatON-Go/log"
@@ -38,9 +39,12 @@ var True = true
// with the node that proves the absence of the key.
func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error {
// Collect all nodes on the path to key.
+ var (
+ prefix []byte
+ nodes []node
+ tn = t.root
+ )
key = keybytesToHex(key)
- var nodes []node
- tn := t.root
for len(key) > 0 && tn != nil {
switch n := tn.(type) {
case *shortNode:
@@ -49,18 +53,20 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e
tn = nil
} else {
tn = n.Val
+ prefix = append(prefix, n.Key...)
key = key[len(n.Key):]
}
nodes = append(nodes, n)
case *fullNode:
tn = n.Children[key[0]]
+ prefix = append(prefix, key[0])
key = key[1:]
nodes = append(nodes, n)
case hashNode:
var err error
- tn, err = t.resolveHash(n, nil)
+ tn, err = t.resolveHash(n, prefix)
if err != nil {
- log.Error("Unhandled trie error in Trie.Prove", "err", err)
+ log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
return err
}
default:
@@ -558,7 +564,7 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, key
}
// Rebuild the trie with the leaf stream, the shape of trie
// should be same with the original one.
- tr := newWithRootNode(root)
+ tr := &Trie{root: root, db: NewDatabase(rawdb.NewMemoryDatabase())}
if empty {
tr.root = nil
}
diff --git a/trie/secure_trie.go b/trie/secure_trie.go
index d23f9aff96..769df7a168 100644
--- a/trie/secure_trie.go
+++ b/trie/secure_trie.go
@@ -207,7 +207,7 @@ func (t *StateTrie) GetKey(shaKey []byte) []byte {
// All cached preimages will be also flushed if preimages recording is enabled.
// Once the trie is committed, it's not usable anymore. A new trie must
// be created with new root and updated trie database for following usage
-func (t *StateTrie) Commit(onleaf LeafCallback) (root common.Hash, committed int, err error) {
+func (t *StateTrie) Commit(collectLeaf bool) (common.Hash, *NodeSet, error) {
// Write all the pre-images to the actual disk database
if len(t.getSecKeyCache()) > 0 {
if t.preimages != nil {
@@ -220,7 +220,7 @@ func (t *StateTrie) Commit(onleaf LeafCallback) (root common.Hash, committed int
t.secKeyCache = make(map[string][]byte)
}
// Commit the trie to its intermediate node database
- return t.trie.Commit(onleaf)
+ return t.trie.Commit(collectLeaf)
}
// Hash returns the root hash of StateTrie. It does not write to the
diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go
index 29ab822dc4..8eaf8cf3cf 100644
--- a/trie/secure_trie_test.go
+++ b/trie/secure_trie_test.go
@@ -18,6 +18,7 @@ package trie
import (
"bytes"
+ "fmt"
"runtime"
"sync"
"testing"
@@ -58,9 +59,15 @@ func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) {
trie.Update(key, val)
}
}
- trie.Commit(nil)
-
- // Return the generated trie
+ root, nodes, err := trie.Commit(false)
+ if err != nil {
+ panic(fmt.Errorf("failed to commit trie %v", err))
+ }
+ if err := triedb.Update(NewWithNodeSet(nodes)); err != nil {
+ panic(fmt.Errorf("failed to commit db %v", err))
+ }
+ // Re-create the trie based on the new state
+ trie, _ = NewSecure(common.Hash{}, root, triedb)
return triedb, trie, content
}
@@ -113,10 +120,9 @@ func TestStateTrieConcurrency(t *testing.T) {
threads := runtime.NumCPU()
tries := make([]*StateTrie, threads)
for i := 0; i < threads; i++ {
- cpy := *trie
- tries[i] = &cpy
+ tries[i] = trie.Copy()
}
- // Start a batch of goroutines interactng with the trie
+ // Start a batch of goroutines interacting with the trie
pend := new(sync.WaitGroup)
pend.Add(threads)
for i := 0; i < threads; i++ {
@@ -137,7 +143,7 @@ func TestStateTrieConcurrency(t *testing.T) {
tries[index].Update(key, val)
}
}
- tries[index].Commit(nil)
+ tries[index].Commit(false)
}(i)
}
// Wait for all threads to finish
diff --git a/trie/sync_test.go b/trie/sync_test.go
index b067642b85..298bcb8f17 100644
--- a/trie/sync_test.go
+++ b/trie/sync_test.go
@@ -18,6 +18,7 @@ package trie
import (
"bytes"
+ "fmt"
"testing"
"github.com/PlatONnetwork/PlatON-Go/ethdb/memorydb"
@@ -51,9 +52,15 @@ func makeTestTrie() (*Database, *StateTrie, map[string][]byte) {
trie.Update(key, val)
}
}
- trie.Commit(nil)
-
- // Return the generated trie
+ root, nodes, err := trie.Commit(false)
+ if err != nil {
+ panic(fmt.Errorf("failed to commit trie %v", err))
+ }
+ if err := triedb.Update(NewWithNodeSet(nodes)); err != nil {
+ panic(fmt.Errorf("failed to commit db %v", err))
+ }
+ // Re-create the trie based on the new state
+ trie, _ = NewSecure(common.Hash{}, root, triedb)
return triedb, trie, content
}
diff --git a/trie/trie.go b/trie/trie.go
index 0615241f66..5ebaf172fd 100644
--- a/trie/trie.go
+++ b/trie/trie.go
@@ -23,7 +23,6 @@ import (
"fmt"
"github.com/PlatONnetwork/PlatON-Go/common"
- "github.com/PlatONnetwork/PlatON-Go/core/rawdb"
"github.com/PlatONnetwork/PlatON-Go/crypto"
"github.com/PlatONnetwork/PlatON-Go/log"
)
@@ -52,15 +51,24 @@ var (
// for extracting the raw states(leaf nodes) with corresponding paths.
type LeafCallback func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error
-// Trie is a Merkle Patricia Trie.
-// The zero value is an empty trie with no database.
-// Use New to create a trie that sits on top of a database.
+// Trie is a Merkle Patricia Trie. Use New to create a trie that sits on
+// top of a database. Whenever trie performs a commit operation, the generated
+// nodes will be gathered and returned in a set. Once the trie is committed,
+// it's not usable anymore. Callers have to re-create the trie with new root
+// based on the updated trie database.
//
// Trie is not safe for concurrent use.
type Trie struct {
- db *Database
root node
owner common.Hash
+
+ // db is the handler trie can retrieve nodes from. It's
+ // only for reading purpose and not available for writing.
+ db *Database
+
+ // tracer is the tool to track the trie changes.
+ // It will be reset after each commit operation.
+ tracer *tracer
}
// newFlag returns the cache flag value for a newly created node.
@@ -69,6 +77,16 @@ func (t *Trie) newFlag() nodeFlag {
return nodeFlag{hash: &hashNode{}, dirty: &dirty}
}
+// Copy returns a copy of Trie.
+func (t *Trie) Copy() *Trie {
+ return &Trie{
+ root: t.root,
+ owner: t.owner,
+ db: t.db,
+ tracer: t.tracer.copy(),
+ }
+}
+
// New creates a trie with an existing root node from db and an assigned
// owner for storage proximity.
//
@@ -77,33 +95,13 @@ func (t *Trie) newFlag() nodeFlag {
// New will panic if db is nil and returns a MissingNodeError if root does
// not exist in the database. Accessing the trie loads nodes from db on demand.
func New(owner common.Hash, root common.Hash, db *Database) (*Trie, error) {
- return newTrie(owner, root, db)
-}
-
-// NewEmpty is a shortcut to create empty tree. It's mostly used in tests.
-func NewEmpty(db *Database) *Trie {
- tr, _ := newTrie(common.Hash{}, common.Hash{}, db)
- return tr
-}
-
-// newWithRootNode initializes the trie with the given root node.
-// It's only used by range prover.
-func newWithRootNode(root node) *Trie {
- return &Trie{
- root: root,
- //tracer: newTracer(),
- db: NewDatabase(rawdb.NewMemoryDatabase()),
- }
-}
-
-// newTrie is the internal function used to construct the trie with given parameters.
-func newTrie(owner common.Hash, root common.Hash, db *Database) (*Trie, error) {
if db == nil {
panic("trie.New called without a database")
}
trie := &Trie{
- db: db,
owner: owner,
+ db: db,
+ //tracer: newTracer(),
}
if root != (common.Hash{}) && root != emptyRoot {
rootnode, err := trie.resolveHash(root[:], nil)
@@ -115,6 +113,12 @@ func newTrie(owner common.Hash, root common.Hash, db *Database) (*Trie, error) {
return trie, nil
}
+// NewEmpty is a shortcut to create empty tree. It's mostly used in tests.
+func NewEmpty(db *Database) *Trie {
+ tr, _ := New(common.Hash{}, common.Hash{}, db)
+ return tr
+}
+
// NodeIterator returns an iterator that returns nodes of the trie. Iteration starts at
// the key after the given start key.
func (t *Trie) NodeIterator(start []byte) NodeIterator {
@@ -333,7 +337,12 @@ func (t *Trie) insert(n node, prefix, key []byte, value node) (bool, node, error
if matchlen == 0 {
return true, branch, nil
}
- // Otherwise, replace it with a short node leading up to the branch.
+ // New branch node is created as a child of the original short node.
+ // Track the newly inserted node in the tracer. The node identifier
+ // passed is the path from the root node.
+ t.tracer.onInsert(append(prefix, key[:matchlen]...))
+
+ // Replace it with a short node leading up to the branch.
return true, &shortNode{key[:matchlen], branch, t.newFlag()}, nil
case *fullNode:
@@ -347,6 +356,11 @@ func (t *Trie) insert(n node, prefix, key []byte, value node) (bool, node, error
return true, n, nil
case nil:
+ // New short node is created and track it in the tracer. The node identifier
+ // passed is the path from the root node. Note the valueNode won't be tracked
+ // since it's always embedded in its parent.
+ t.tracer.onInsert(prefix)
+
return true, &shortNode{key, value, t.newFlag()}, nil
case hashNode:
@@ -398,6 +412,11 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
return false, n, nil // don't replace n on mismatch
}
if matchlen == len(key) {
+ // The matched short node is deleted entirely and track
+ // it in the deletion set. The same the valueNode doesn't
+ // need to be tracked at all since it's always embedded.
+ t.tracer.onDelete(prefix)
+
return true, nil, nil // remove n entirely for whole matches
}
// The key is longer than n.Key. Remove the remaining suffix
@@ -410,6 +429,10 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
}
switch child := child.(type) {
case *shortNode:
+ // The child shortNode is merged into its parent, track
+ // is deleted as well.
+ t.tracer.onDelete(append(prefix, n.Key...))
+
// Deleting from the subtrie reduced it to another
// short node. Merge the nodes to avoid creating a
// shortNode{..., shortNode{...}}. Use concat (which
@@ -466,11 +489,17 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
// shortNode{..., shortNode{...}}. Since the entry
// might not be loaded yet, resolve it just for this
// check.
- cnode, err := t.resolve(n.Children[pos], prefix)
+ //cnode, err := t.resolve(n.Children[pos], prefix)
+ cnode, err := t.resolve(n.Children[pos], append(prefix, byte(pos)))
if err != nil {
return false, nil, err
}
if cnode, ok := cnode.(*shortNode); ok {
+ // Replace the entire full node with the short node.
+ // Mark the original short node as deleted since the
+ // value is embedded into the parent now.
+ t.tracer.onDelete(append(prefix, byte(pos)))
+
k := append([]byte{byte(pos)}, cnode.Key...)
return true, &shortNode{k, cnode.Val, t.newFlag()}, nil
}
@@ -521,6 +550,8 @@ func (t *Trie) resolve(n node, prefix []byte) (node, error) {
return n, nil
}
+// resolveHash loads node from the underlying database with the provided
+// node hash and path prefix.
func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) {
hash := common.BytesToHash(n)
if node := t.db.node(hash); node != nil {
@@ -529,10 +560,8 @@ func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) {
return nil, &MissingNodeError{Owner: t.owner, NodeHash: hash, Path: prefix}
}
-// Root returns the root hash of the trie.
-// Deprecated: use Hash instead.
-func (t *Trie) Root() []byte { return t.Hash().Bytes() }
-
+// resolveHash loads rlp-encoded node blob from the underlying database
+// with the provided node hash and path prefix.
func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) {
hash := common.BytesToHash(n)
blob, _ := t.db.Node(hash)
@@ -552,17 +581,19 @@ func (t *Trie) Hash() common.Hash {
// Commit writes all nodes to the trie's memory database, tracking the internal
// and external (for account tries) references.
-func (t *Trie) Commit(onleaf LeafCallback) (common.Hash, int, error) {
+func (t *Trie) Commit(collectLeaf bool) (common.Hash, *NodeSet, error) {
+ defer t.tracer.reset()
+
if t.db == nil {
panic("commit called on trie with nil database")
}
- hash, cached, committed, err := t.commitRoot(t.db, onleaf)
+ hash, cached, set, err := t.commitRoot(collectLeaf)
if err != nil {
- return common.Hash{}, 0, err
+ return common.Hash{}, nil, err
}
t.root = cached
- return common.BytesToHash(hash.(hashNode)), committed, nil
+ return common.BytesToHash(hash.(hashNode)), set, nil
}
func (t *Trie) hashRoot() (node, node, error) {
@@ -578,6 +609,7 @@ func (t *Trie) hashRoot() (node, node, error) {
func (t *Trie) Reset() {
t.root = nil
t.owner = common.Hash{}
+ t.tracer.reset()
}
// Owner returns the associated trie owner.
@@ -585,15 +617,14 @@ func (t *Trie) Owner() common.Hash {
return t.owner
}
-func (t *Trie) commitRoot(db *Database, onleaf LeafCallback) (node, node, int, error) {
+func (t *Trie) commitRoot(collectLeaf bool) (node, node, *NodeSet, error) {
if t.root == nil {
- return hashNode(emptyRoot.Bytes()), nil, 0, nil
+ return hashNode(emptyRoot.Bytes()), nil, nil, nil
}
- c := newCommitter(onleaf)
- defer returnCommitterToPool(c)
-
- return c.commit(nil, t.root, db, true)
+ c := newCommitter(t.owner, collectLeaf)
+ hashed, cached, err := c.commit(nil, t.root, true)
+ return hashed, cached, c.nodes, err
}
func (t *Trie) DeepCopyTrie() *Trie {
@@ -609,6 +640,7 @@ func (t *Trie) DeepCopyTrie() *Trie {
db: t.db,
root: cpyRoot,
owner: t.owner,
+ //tracer: newTracer(),
}
}
diff --git a/trie/trie_test.go b/trie/trie_test.go
index f2c4250eef..5395e99773 100644
--- a/trie/trie_test.go
+++ b/trie/trie_test.go
@@ -22,7 +22,6 @@ import (
"fmt"
"math/big"
"math/rand"
- "os"
"reflect"
"testing"
"testing/quick"
@@ -33,8 +32,6 @@ import (
"github.com/PlatONnetwork/PlatON-Go/common/byteutil"
"github.com/PlatONnetwork/PlatON-Go/core/rawdb"
- "github.com/PlatONnetwork/PlatON-Go/ethdb/leveldb"
-
"github.com/PlatONnetwork/PlatON-Go/ethdb/memorydb"
"github.com/stretchr/testify/assert"
@@ -51,14 +48,8 @@ func init() {
spew.Config.DisableMethods = false
}
-// Used for testing
-func newEmpty() *Trie {
- trie := NewEmpty(NewDatabase(memorydb.New()))
- return trie
-}
-
func TestEmptyTrie(t *testing.T) {
- var trie Trie
+ trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
res := trie.Hash()
exp := emptyRoot
if res != exp {
@@ -67,7 +58,7 @@ func TestEmptyTrie(t *testing.T) {
}
func TestNull(t *testing.T) {
- var trie Trie
+ trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
key := make([]byte, 32)
value := []byte("test")
trie.Update(key, value)
@@ -96,7 +87,8 @@ func testMissingNode(t *testing.T, memonly bool) {
trie := NewEmpty(triedb)
updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer")
updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf")
- root, _, _ := trie.Commit(nil)
+ root, nodes, _ := trie.Commit(false)
+ triedb.Update(NewWithNodeSet(nodes))
if !memonly {
triedb.Commit(root, true, true)
}
@@ -162,7 +154,7 @@ func testMissingNode(t *testing.T, memonly bool) {
}
func TestInsert(t *testing.T) {
- trie := newEmpty()
+ trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
updateString(trie, "doe", "reindeer")
updateString(trie, "dog", "puppy")
@@ -174,11 +166,11 @@ func TestInsert(t *testing.T) {
t.Errorf("exp %x got %x", exp, root)
}
- trie = newEmpty()
+ trie = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
updateString(trie, "A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
exp = common.HexToHash("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab")
- root, _, err := trie.Commit(nil)
+ root, _, err := trie.Commit(false)
if err != nil {
t.Fatalf("commit error: %v", err)
}
@@ -188,7 +180,8 @@ func TestInsert(t *testing.T) {
}
func TestGet(t *testing.T) {
- trie := newEmpty()
+ db := NewDatabase(rawdb.NewMemoryDatabase())
+ trie := NewEmpty(db)
updateString(trie, "doe", "reindeer")
updateString(trie, "dog", "puppy")
updateString(trie, "dogglesworth", "cat")
@@ -198,7 +191,6 @@ func TestGet(t *testing.T) {
if !bytes.Equal(res, []byte("puppy")) {
t.Errorf("expected puppy got %x", res)
}
-
unknown := getString(trie, "unknown")
if unknown != nil {
t.Errorf("expected nil got %x", unknown)
@@ -207,12 +199,14 @@ func TestGet(t *testing.T) {
if i == 1 {
return
}
- trie.Commit(nil)
+ root, nodes, _ := trie.Commit(false)
+ db.Update(NewWithNodeSet(nodes))
+ trie, _ = New(common.Hash{}, root, db)
}
}
func TestDelete(t *testing.T) {
- trie := newEmpty()
+ trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
vals := []struct{ k, v string }{
{"do", "verb"},
{"ether", "wookiedoo"},
@@ -239,7 +233,7 @@ func TestDelete(t *testing.T) {
}
func TestEmptyValues(t *testing.T) {
- trie := newEmpty()
+ trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
vals := []struct{ k, v string }{
{"do", "verb"},
@@ -263,7 +257,8 @@ func TestEmptyValues(t *testing.T) {
}
func TestReplication(t *testing.T) {
- trie := newEmpty()
+ triedb := NewDatabase(rawdb.NewMemoryDatabase())
+ trie := NewEmpty(triedb)
vals := []struct{ k, v string }{
{"do", "verb"},
{"ether", "wookiedoo"},
@@ -276,13 +271,14 @@ func TestReplication(t *testing.T) {
for _, val := range vals {
updateString(trie, val.k, val.v)
}
- exp, _, err := trie.Commit(nil)
+ exp, nodes, err := trie.Commit(false)
if err != nil {
t.Fatalf("commit error: %v", err)
}
+ triedb.Update(NewWithNodeSet(nodes))
// create a new trie on top of the database and check that lookups work.
- trie2, err := New(common.Hash{}, exp, trie.db)
+ trie2, err := New(common.Hash{}, exp, triedb)
if err != nil {
t.Fatalf("can't recreate trie at %x: %v", exp, err)
}
@@ -291,7 +287,7 @@ func TestReplication(t *testing.T) {
t.Errorf("trie2 doesn't have %q => %q", kv.k, kv.v)
}
}
- hash, _, err := trie2.Commit(nil)
+ hash, nodes, err := trie2.Commit(false)
if err != nil {
t.Fatalf("commit error: %v", err)
}
@@ -299,6 +295,14 @@ func TestReplication(t *testing.T) {
t.Errorf("root failure. expected %x got %x", exp, hash)
}
+ // recreate the trie after commit
+ if nodes != nil {
+ triedb.Update(NewWithNodeSet(nodes))
+ }
+ trie2, err = New(common.Hash{}, hash, triedb)
+ if err != nil {
+ t.Fatalf("can't recreate trie at %x: %v", exp, err)
+ }
// perform some insertions on the new trie.
vals2 := []struct{ k, v string }{
{"do", "verb"},
@@ -320,7 +324,7 @@ func TestReplication(t *testing.T) {
}
func TestLargeValue(t *testing.T) {
- trie := newEmpty()
+ trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
trie.Update([]byte("key1"), []byte{99, 99, 99, 99})
trie.Update([]byte("key2"), bytes.Repeat([]byte{1}, 32))
trie.Hash()
@@ -374,10 +378,10 @@ const (
opUpdate = iota
opDelete
opGet
- opCommit
opHash
- opReset
+ opCommit
opItercheckhash
+ opNodeDiff
opMax // boundary value, not an actual op
)
@@ -412,10 +416,14 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value {
}
func runRandTest(rt randTest) bool {
- triedb := NewDatabase(memorydb.New())
-
- tr := NewEmpty(triedb)
- values := make(map[string]string) // tracks content of the trie
+ var (
+ triedb = NewDatabase(memorydb.New())
+ tr = NewEmpty(triedb)
+ values = make(map[string]string) // tracks content of the trie
+ origTrie = NewEmpty(triedb)
+ )
+ tr.tracer = newTracer()
+ tr.tracer = newTracer()
for i, step := range rt {
switch step.op {
@@ -431,22 +439,26 @@ func runRandTest(rt randTest) bool {
if string(v) != want {
rt[i].err = fmt.Errorf("mismatch for key %#x, got %#x want %#x", step.key, v, want)
}
- case opCommit:
- _, _, rt[i].err = tr.Commit(nil)
case opHash:
tr.Hash()
- case opReset:
- hash, _, err := tr.Commit(nil)
+ case opCommit:
+ hash, nodes, err := tr.Commit(false)
if err != nil {
rt[i].err = err
return false
}
+ if nodes != nil {
+ triedb.Update(NewWithNodeSet(nodes))
+ }
newtr, err := New(common.Hash{}, hash, triedb)
if err != nil {
rt[i].err = err
return false
}
tr = newtr
+ tr.tracer = newTracer()
+
+ origTrie = tr.Copy()
case opItercheckhash:
checktr := NewEmpty(triedb)
it := NewIterator(tr.NodeIterator(nil))
@@ -457,6 +469,59 @@ func runRandTest(rt randTest) bool {
//fmt.Printf("phash: %x, chash: %x\n", tr.Hash(), checktr.Hash())
rt[i].err = fmt.Errorf("hash mismatch in opItercheckhash")
}
+ case opNodeDiff:
+ var (
+ inserted = tr.tracer.insertList()
+ deleted = tr.tracer.deleteList()
+ origIter = origTrie.NodeIterator(nil)
+ curIter = tr.NodeIterator(nil)
+ origSeen = make(map[string]struct{})
+ curSeen = make(map[string]struct{})
+ )
+ for origIter.Next(true) {
+ if origIter.Leaf() {
+ continue
+ }
+ origSeen[string(origIter.Path())] = struct{}{}
+ }
+ for curIter.Next(true) {
+ if curIter.Leaf() {
+ continue
+ }
+ curSeen[string(curIter.Path())] = struct{}{}
+ }
+ var (
+ insertExp = make(map[string]struct{})
+ deleteExp = make(map[string]struct{})
+ )
+ for path := range curSeen {
+ _, present := origSeen[path]
+ if !present {
+ insertExp[path] = struct{}{}
+ }
+ }
+ for path := range origSeen {
+ _, present := curSeen[path]
+ if !present {
+ deleteExp[path] = struct{}{}
+ }
+ }
+ if len(insertExp) != len(inserted) {
+ rt[i].err = fmt.Errorf("insert set mismatch")
+ }
+ if len(deleteExp) != len(deleted) {
+ rt[i].err = fmt.Errorf("delete set mismatch")
+ }
+ for _, insert := range inserted {
+ if _, present := insertExp[string(insert)]; !present {
+ rt[i].err = fmt.Errorf("missing inserted node")
+ }
+ }
+ for _, del := range deleted {
+ if _, present := deleteExp[string(del)]; !present {
+ rt[i].err = fmt.Errorf("missing deleted node")
+ }
+ }
}
// Abort the test on error.
if rt[i].err != nil {
@@ -482,44 +547,31 @@ func TestRandom(t *testing.T) {
}
}
-func BenchmarkGet(b *testing.B) { benchGet(b, false) }
-func BenchmarkGetDB(b *testing.B) { benchGet(b, true) }
+func BenchmarkGet(b *testing.B) { benchGet(b) }
func BenchmarkUpdateBE(b *testing.B) { benchUpdate(b, binary.BigEndian) }
func BenchmarkUpdateLE(b *testing.B) { benchUpdate(b, binary.LittleEndian) }
const benchElemCount = 20000
-func benchGet(b *testing.B, commit bool) {
- trie := new(Trie)
- if commit {
- tmpdb := tempDB(b)
- trie = NewEmpty(tmpdb)
- }
+func benchGet(b *testing.B) {
+ triedb := NewDatabase(rawdb.NewMemoryDatabase())
+ trie := NewEmpty(triedb)
k := make([]byte, 32)
for i := 0; i < benchElemCount; i++ {
binary.LittleEndian.PutUint64(k, uint64(i))
trie.Update(k, k)
}
binary.LittleEndian.PutUint64(k, benchElemCount/2)
- if commit {
- trie.Commit(nil)
- }
b.ResetTimer()
for i := 0; i < b.N; i++ {
trie.Get(k)
}
b.StopTimer()
-
- if commit {
- ldb := trie.db.diskdb.(*leveldb.Database)
- ldb.Close()
- os.RemoveAll(ldb.Path())
- }
}
func benchUpdate(b *testing.B, e binary.ByteOrder) *Trie {
- trie := newEmpty()
+ trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
k := make([]byte, 32)
for i := 0; i < b.N; i++ {
e.PutUint64(k, uint64(i))
@@ -554,7 +606,7 @@ func BenchmarkHash(b *testing.B) {
accounts[i], _ = rlp.EncodeToBytes([]interface{}{nonce, balance, root, code})
}
// Insert the accounts into the trie and hash it
- trie := newEmpty()
+ trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
for i := 0; i < len(addresses); i++ {
trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i])
}
@@ -563,15 +615,6 @@ func BenchmarkHash(b *testing.B) {
trie.Hash()
}
-func tempDB(tb testing.TB) *Database {
- dir := tb.TempDir()
- diskdb, err := leveldb.New(dir, 256, 0, "", false)
- if err != nil {
- panic(fmt.Sprintf("can't create temporary database: %v", err))
- }
- return NewDatabase(diskdb)
-}
-
func getString(trie *Trie, k string) []byte {
return trie.Get([]byte(k))
}
@@ -598,6 +641,12 @@ func TestDecodeNode(t *testing.T) {
}
func TestDeepCopy(t *testing.T) {
+ // handle leaf
+ var (
+ path [][]byte
+ hexpath, parentPath []byte
+ nodes *NodeSet
+ )
memdb := memorydb.New()
triedb := NewDatabase(memdb)
root := common.Hash{}
@@ -626,7 +675,14 @@ func TestDeepCopy(t *testing.T) {
kv[common.BytesToHash(tr.hashKey(k))] = v
}
- root, _, _ = tr.Commit(leafCB)
+ root, nodes, _ = tr.Commit(true)
+ mergedNodeSet := NewWithNodeSet(nodes)
+ triedb.Update(mergedNodeSet)
+ if set, present := mergedNodeSet.sets[common.Hash{}]; present {
+ for _, n := range set.leaves {
+ leafCB(path, hexpath, n.blob, n.parent, parentPath)
+ }
+ }
if codeWriter.ValueSize() > 0 {
if err := codeWriter.Write(); err != nil {
t.Fatal("Failed to commit dirty codes", "error", err)
@@ -684,7 +740,15 @@ func TestDeepCopy(t *testing.T) {
count++
}
assert.Equal(t, len(kv), keys)
- root, _, _ = tr2.Commit(leafCB)
+ root, nodes, _ = tr2.Commit(true)
+ mergedNodeSet := NewWithNodeSet(nodes)
+ triedb.Update(mergedNodeSet)
+ if set, present := mergedNodeSet.sets[common.Hash{}]; present {
+ for _, n := range set.leaves {
+ leafCB(path, hexpath, n.blob, n.parent, parentPath)
+ }
+ }
+
if codeWriter.ValueSize() > 0 {
if err := codeWriter.Write(); err != nil {
t.Fatal("Failed to commit dirty codes", "error", err)
@@ -693,7 +757,14 @@ func TestDeepCopy(t *testing.T) {
triedb.Reference(root, common.Hash{})
assert.Nil(t, triedb.Commit(root, false, false))
triedb.DereferenceDB(parent)
- cpyRoot, _, _ := cpy.Commit(leafCB)
+ cpyRoot, nodes, _ := cpy.Commit(true)
+ mergedNodeSet = NewWithNodeSet(nodes)
+ triedb.Update(mergedNodeSet)
+ if set, present := mergedNodeSet.sets[common.Hash{}]; present {
+ for _, n := range set.leaves {
+ leafCB(path, hexpath, n.blob, n.parent, parentPath)
+ }
+ }
if root != cpyRoot {
t.Fatal("cpyroot failed")
}
@@ -738,7 +809,8 @@ func TestOneTrieCollision(t *testing.T) {
for _, d := range trieData1 {
trie.Update(d.hash, d.value)
}
- root, _, _ := trie.Commit(nil)
+ root, nodes, _ := trie.Commit(false)
+ memdb.Update(NewWithNodeSet(nodes))
memdb.Commit(root, false, false)
assert.Nil(t, checkTrie(trie))
@@ -747,16 +819,19 @@ func TestOneTrieCollision(t *testing.T) {
reopenTrie, _ := New(common.Hash{}, root, reopenMemdb)
reopenTrie.Delete(trieData1[0].hash)
- reopenRoot, _, _ := reopenTrie.Commit(nil)
+ reopenRoot, nodes, _ := reopenTrie.Commit(false)
+ reopenMemdb.Update(NewWithNodeSet(nodes))
reopenMemdb.Commit(reopenRoot, false, false)
reopenTrie.Update(trieData1[0].hash, trieData1[0].value)
- reopenRoot, _, _ = reopenTrie.Commit(nil)
+ reopenRoot, nodes, _ = reopenTrie.Commit(false)
+ reopenMemdb.Update(NewWithNodeSet(nodes))
reopenMemdb.IncrVersion()
reopenMemdb.Commit(reopenRoot, false, false)
reopenMemdb.ReferenceVersion(root)
reopenTrie.Delete(trieData1[0].hash)
- reopenRoot, _, _ = reopenTrie.Commit(nil)
+ reopenRoot, nodes, _ = reopenTrie.Commit(false)
+ reopenMemdb.Update(NewWithNodeSet(nodes))
reopenMemdb.IncrVersion()
reopenMemdb.Commit(reopenRoot, false, false)
reopenMemdb.ReferenceVersion(reopenRoot)
@@ -791,8 +866,10 @@ func TestTwoTrieCollision(t *testing.T) {
trie2.Update(d.hash, d.value)
}
- root1, _, _ := trie1.Commit(nil)
- root2, _, _ := trie2.Commit(nil)
+ root1, nodes, _ := trie1.Commit(false)
+ memdb1.Update(NewWithNodeSet(nodes))
+ root2, nodes, _ := trie2.Commit(false)
+ memdb2.Update(NewWithNodeSet(nodes))
memdb1.Commit(root1, false, false)
memdb2.Commit(root2, false, false)
@@ -811,19 +888,19 @@ func TestTwoTrieCollision(t *testing.T) {
func TestCommitAfterHash(t *testing.T) {
// Create a realistic account trie to hash
addresses, accounts := makeAccounts(1000)
- trie := newEmpty()
+ trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
for i := 0; i < len(addresses); i++ {
trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i])
}
// Insert the accounts into the trie and hash it
trie.Hash()
- trie.Commit(nil)
+ trie.Commit(false)
root := trie.Hash()
exp := common.HexToHash("1ad36b758576e29b9917ed99765036cb37732ac61956d96872b1bb278a4fe2b9")
if exp != root {
t.Errorf("got %x, exp %x", root, exp)
}
- root, _, _ = trie.Commit(nil)
+ root, _, _ = trie.Commit(false)
if exp != root {
t.Errorf("got %x, exp %x", root, exp)
}
diff --git a/trie/util_test.go b/trie/util_test.go
new file mode 100644
index 0000000000..bd670aa67c
--- /dev/null
+++ b/trie/util_test.go
@@ -0,0 +1,124 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package trie
+
+import (
+ "testing"
+
+ "github.com/PlatONnetwork/PlatON-Go/common"
+ "github.com/PlatONnetwork/PlatON-Go/core/rawdb"
+)
+
+// Tests if the trie diffs are tracked correctly.
+func TestTrieTracer(t *testing.T) {
+ db := NewDatabase(rawdb.NewMemoryDatabase())
+ trie := NewEmpty(db)
+ trie.tracer = newTracer()
+
+ // Insert a batch of entries, all the nodes should be marked as inserted
+ vals := []struct{ k, v string }{
+ {"do", "verb"},
+ {"ether", "wookiedoo"},
+ {"horse", "stallion"},
+ {"shaman", "horse"},
+ {"doge", "coin"},
+ {"dog", "puppy"},
+ {"somethingveryoddindeedthis is", "myothernodedata"},
+ }
+ for _, val := range vals {
+ trie.Update([]byte(val.k), []byte(val.v))
+ }
+ trie.Hash()
+
+ seen := make(map[string]struct{})
+ it := trie.NodeIterator(nil)
+ for it.Next(true) {
+ if it.Leaf() {
+ continue
+ }
+ seen[string(it.Path())] = struct{}{}
+ }
+ inserted := trie.tracer.insertList()
+ if len(inserted) != len(seen) {
+ t.Fatalf("Unexpected inserted node tracked want %d got %d", len(seen), len(inserted))
+ }
+ for _, k := range inserted {
+ _, ok := seen[string(k)]
+ if !ok {
+ t.Fatalf("Unexpected inserted node")
+ }
+ }
+ deleted := trie.tracer.deleteList()
+ if len(deleted) != 0 {
+ t.Fatalf("Unexpected deleted node tracked %d", len(deleted))
+ }
+
+ // Commit the changes and re-create with new root
+ root, nodes, _ := trie.Commit(false)
+ db.Update(NewWithNodeSet(nodes))
+ trie, _ = New(common.Hash{}, root, db)
+ trie.tracer = newTracer()
+
+ // Delete all the elements, check deletion set
+ for _, val := range vals {
+ trie.Delete([]byte(val.k))
+ }
+ trie.Hash()
+
+ inserted = trie.tracer.insertList()
+ if len(inserted) != 0 {
+ t.Fatalf("Unexpected inserted node tracked %d", len(inserted))
+ }
+ deleted = trie.tracer.deleteList()
+ if len(deleted) != len(seen) {
+ t.Fatalf("Unexpected deleted node tracked want %d got %d", len(seen), len(deleted))
+ }
+ for _, k := range deleted {
+ _, ok := seen[string(k)]
+ if !ok {
+ t.Fatalf("Unexpected inserted node")
+ }
+ }
+}
+
+func TestTrieTracerNoop(t *testing.T) {
+ trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase()))
+ trie.tracer = newTracer()
+
+ // Insert a batch of entries, all the nodes should be marked as inserted
+ vals := []struct{ k, v string }{
+ {"do", "verb"},
+ {"ether", "wookiedoo"},
+ {"horse", "stallion"},
+ {"shaman", "horse"},
+ {"doge", "coin"},
+ {"dog", "puppy"},
+ {"somethingveryoddindeedthis is", "myothernodedata"},
+ }
+ for _, val := range vals {
+ trie.Update([]byte(val.k), []byte(val.v))
+ }
+ for _, val := range vals {
+ trie.Delete([]byte(val.k))
+ }
+ if len(trie.tracer.insertList()) != 0 {
+ t.Fatalf("Unexpected inserted node tracked %d", len(trie.tracer.insertList()))
+ }
+ if len(trie.tracer.deleteList()) != 0 {
+ t.Fatalf("Unexpected deleted node tracked %d", len(trie.tracer.deleteList()))
+ }
+}
diff --git a/trie/utils.go b/trie/utils.go
new file mode 100644
index 0000000000..7e26915041
--- /dev/null
+++ b/trie/utils.go
@@ -0,0 +1,167 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package trie
+
+// tracer tracks the changes of trie nodes. During the trie operations,
+// some nodes can be deleted from the trie, while these deleted nodes
+// won't be captured by trie.Hasher or trie.Committer. Thus, these deleted
+// nodes won't be removed from the disk at all. Tracer is an auxiliary tool
+// used to track all insert and delete operations of trie and capture all
+// deleted nodes eventually.
+//
+// The changed nodes can be mainly divided into two categories: the leaf
+// node and intermediate node. The former is inserted/deleted by callers
+// while the latter is inserted/deleted in order to follow the rule of trie.
+// This tool can track all of them no matter the node is embedded in its
+// parent or not, but valueNode is never tracked.
+//
+// Besides, it's also used for recording the original value of the nodes
+// when they are resolved from the disk. The pre-value of the nodes will
+// be used to construct reverse-diffs in the future.
+//
+// Note tracer is not thread-safe, callers should be responsible for handling
+// the concurrency issues by themselves.
+type tracer struct {
+ insert map[string]struct{}
+ delete map[string]struct{}
+ origin map[string][]byte
+}
+
+// newTracer initializes the tracer for capturing trie changes.
+func newTracer() *tracer {
+ return &tracer{
+ insert: make(map[string]struct{}),
+ delete: make(map[string]struct{}),
+ origin: make(map[string][]byte),
+ }
+}
+
+/*
+// onRead tracks the newly loaded trie node and caches the rlp-encoded blob internally.
+// Don't change the value outside of function since it's not deep-copied.
+func (t *tracer) onRead(key []byte, val []byte) {
+ // Tracer isn't used right now, remove this check later.
+ if t == nil {
+ return
+ }
+ t.origin[string(key)] = val
+}
+*/
+
+// onInsert tracks the newly inserted trie node. If it's already in the deletion set
+// (resurrected node), then just wipe it from the deletion set as the "untouched".
+func (t *tracer) onInsert(key []byte) {
+ // Tracer isn't used right now, remove this check later.
+ if t == nil {
+ return
+ }
+ if _, present := t.delete[string(key)]; present {
+ delete(t.delete, string(key))
+ return
+ }
+ t.insert[string(key)] = struct{}{}
+}
+
+// onDelete tracks the newly deleted trie node. If it's already
+// in the addition set, then just wipe it from the addition set
+// as it's untouched.
+func (t *tracer) onDelete(key []byte) {
+ // Tracer isn't used right now, remove this check later.
+ if t == nil {
+ return
+ }
+ if _, present := t.insert[string(key)]; present {
+ delete(t.insert, string(key))
+ return
+ }
+ t.delete[string(key)] = struct{}{}
+}
+
+// insertList returns the tracked inserted trie nodes in list format.
+func (t *tracer) insertList() [][]byte {
+ // Tracer isn't used right now, remove this check later.
+ if t == nil {
+ return nil
+ }
+ var ret [][]byte
+ for key := range t.insert {
+ ret = append(ret, []byte(key))
+ }
+ return ret
+}
+
+// deleteList returns the tracked deleted trie nodes in list format.
+func (t *tracer) deleteList() [][]byte {
+ // Tracer isn't used right now, remove this check later.
+ if t == nil {
+ return nil
+ }
+ var ret [][]byte
+ for key := range t.delete {
+ ret = append(ret, []byte(key))
+ }
+ return ret
+}
+
+/*
+// getPrev returns the cached original value of the specified node.
+func (t *tracer) getPrev(key []byte) []byte {
+ // Don't panic on uninitialized tracer, it's possible in testing.
+ if t == nil {
+ return nil
+ }
+ return t.origin[string(key)]
+}
+*/
+
+// reset clears the content tracked by tracer.
+func (t *tracer) reset() {
+ // Tracer isn't used right now, remove this check later.
+ if t == nil {
+ return
+ }
+ t.insert = make(map[string]struct{})
+ t.delete = make(map[string]struct{})
+ t.origin = make(map[string][]byte)
+}
+
+// copy returns a deep copied tracer instance.
+func (t *tracer) copy() *tracer {
+ // Tracer isn't used right now, remove this check later.
+ if t == nil {
+ return nil
+ }
+ var (
+ insert = make(map[string]struct{})
+ delete = make(map[string]struct{})
+ origin = make(map[string][]byte)
+ )
+ for key := range t.insert {
+ insert[key] = struct{}{}
+ }
+ for key := range t.delete {
+ delete[key] = struct{}{}
+ }
+ for key, val := range t.origin {
+ origin[key] = val
+ }
+ return &tracer{
+ insert: insert,
+ delete: delete,
+ origin: origin,
+ }
+}