From 359906c0dc8d734aea197ff7f8ed98da159b5c05 Mon Sep 17 00:00:00 2001 From: RealUncle Date: Thu, 2 Dec 2021 17:40:47 +0800 Subject: [PATCH] implement the framework of fast node --- cmd/geth/main.go | 1 + cmd/geth/snapshot.go | 76 ++++++++++++++++++++++- cmd/utils/flags.go | 7 +++ core/block_validator.go | 6 -- core/blockchain.go | 13 +++- core/state/database.go | 24 +++++++- core/state/pruner/pruner.go | 104 +++++++++++++++++++++++++++++++- core/state/snapshot/journal.go | 9 ++- core/state/snapshot/snapshot.go | 4 +- core/state/statedb.go | 87 ++++++++++++++++++-------- core/state_processor.go | 3 +- eth/backend.go | 3 +- eth/ethconfig/config.go | 2 + eth/ethconfig/gen_config.go | 69 ++++++++++++++++----- light/trie.go | 8 +++ tests/state_test_util.go | 2 +- trie/database.go | 1 + trie/dummy_trie.go | 96 +++++++++++++++++++++++++++++ 18 files changed, 454 insertions(+), 61 deletions(-) create mode 100644 trie/dummy_trie.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index e0f29bce77..db00cf19e1 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -115,6 +115,7 @@ var ( utils.WhitelistFlag, utils.BloomFilterSizeFlag, utils.TriesInMemoryFlag, + utils.AllowInsecureNoTriesFlag, utils.CacheFlag, utils.CacheDatabaseFlag, utils.CacheTrieFlag, diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 20920a0f94..d829abbc2a 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -18,6 +18,7 @@ package main import ( "bytes" + "encoding/json" "errors" "fmt" "os" @@ -29,13 +30,16 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/pruner" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" @@ -129,6 +133,32 @@ geth snapshot verify-state will traverse the whole accounts and storages set based on the specified snapshot and recalculate the root hash of state for verification. In other words, this command does the snapshot to trie conversion. +`, + }, + { + Name: "insecure-prune-all", + Usage: "Prune all trie state data except genesis block, it will break storage for fullnode, only suitable for fast node " + + "who do not need trie storage at all", + ArgsUsage: "", + Action: utils.MigrateFlags(pruneAllState), + Category: "MISCELLANEOUS COMMANDS", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.AncientFlag, + utils.RopstenFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + }, + Description: ` +will prune all historical trie state data except genesis block. +All trie nodes will be deleted from the database. + +It expects the genesis file as argument. + +WARNING: It's necessary to delete the trie clean cache after the pruning. +If you specify another directory for the trie clean cache via "--cache.trie.journal" +during the use of Geth, please also specify it here for correct deletion. Otherwise +the trie clean cache with default directory will be deleted. `, }, { @@ -195,7 +225,7 @@ func accessDb(ctx *cli.Context, stack *node.Node) (ethdb.Database, error) { } headHeader := headBlock.Header() //Make sure the MPT and snapshot matches before pruning, otherwise the node can not start. - snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, TriesInMemory, headBlock.Root(), false, false, false) + snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, TriesInMemory, headBlock.Root(), false, false, false, false) if err != nil { log.Error("snaptree error", "err", err) return nil, err // The relevant snapshot(s) might not exist @@ -362,6 +392,48 @@ func pruneState(ctx *cli.Context) error { return nil } +func pruneAllState(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + genesisPath := ctx.Args().First() + if len(genesisPath) == 0 { + utils.Fatalf("Must supply path to genesis JSON file") + } + file, err := os.Open(genesisPath) + if err != nil { + utils.Fatalf("Failed to read genesis file: %v", err) + } + defer file.Close() + + g := new(core.Genesis) + if err := json.NewDecoder(file).Decode(g); err != nil { + cfg := gethConfig{ + Eth: ethconfig.Defaults, + Node: defaultNodeConfig(), + Metrics: metrics.DefaultConfig, + } + + // Load config file. + if err := loadConfig(genesisPath, &cfg); err != nil { + utils.Fatalf("%v", err) + } + g = cfg.Eth.Genesis + } + + chaindb := utils.MakeChainDatabase(ctx, stack, false, false) + pruner, err := pruner.NewAllPruner(chaindb) + if err != nil { + log.Error("Failed to open snapshot tree", "err", err) + return err + } + if err = pruner.PruneAll(g); err != nil { + log.Error("Failed to prune state", "err", err) + return err + } + return nil +} + func verifyState(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() @@ -372,7 +444,7 @@ func verifyState(ctx *cli.Context) error { log.Error("Failed to load head block") return errors.New("no head block") } - snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, 128, headBlock.Root(), false, false, false) + snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, 128, headBlock.Root(), false, false, false, false) if err != nil { log.Error("Failed to open snapshot tree", "err", err) return err diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 8f5141907f..fbca81a15d 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -264,6 +264,10 @@ var ( Usage: "The layer of tries trees that keep in memory", Value: 128, } + AllowInsecureNoTriesFlag = cli.BoolTFlag{ + Name: "allow-insecure-no-tries", + Usage: `Disable the tries state root verification, the state consistency is no longer 100% guaranteed, diffsync is not allowed if enabled. Do not enable it unless you know exactly what the consequence it will cause.`, + } OverrideBerlinFlag = cli.Uint64Flag{ Name: "override.berlin", Usage: "Manually specify Berlin fork-block, overriding the bundled setting", @@ -1666,6 +1670,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.GlobalIsSet(TriesInMemoryFlag.Name) { cfg.TriesInMemory = ctx.GlobalUint64(TriesInMemoryFlag.Name) } + if ctx.GlobalIsSet(AllowInsecureNoTriesFlag.Name) { + cfg.NoTries = ctx.GlobalBool(AllowInsecureNoTriesFlag.Name) + } if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheSnapshotFlag.Name) { cfg.SnapshotCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheSnapshotFlag.Name) / 100 } diff --git a/core/block_validator.go b/core/block_validator.go index b109c1e54b..3ea6615b61 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -70,12 +70,6 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { } validateFuns := []func() error{ - func() error { - if v.bc.HasBlockAndState(block.Hash(), block.NumberU64()) { - return ErrKnownBlock - } - return nil - }, func() error { if hash := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); hash != header.TxHash { return fmt.Errorf("transaction root hash mismatch: have %x, want %x", hash, header.TxHash) diff --git a/core/blockchain.go b/core/blockchain.go index 6c87ffc708..e286a1e124 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -141,6 +141,7 @@ type CacheConfig struct { SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory Preimages bool // Whether to store preimage of trie key to the disk TriesInMemory uint64 // How many tries keeps in memory + NoTries bool // Insecure settings. Do not have any tries in databases if enabled. SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it } @@ -284,6 +285,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par Cache: cacheConfig.TrieCleanLimit, Journal: cacheConfig.TrieCleanJournal, Preimages: cacheConfig.Preimages, + NoTries: cacheConfig.NoTries, }), triesInMemory: cacheConfig.TriesInMemory, quit: make(chan struct{}), @@ -439,7 +441,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par log.Warn("Enabling snapshot recovery", "chainhead", head.NumberU64(), "diskbase", *layer) recover = true } - bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, int(bc.cacheConfig.TriesInMemory), head.Root(), !bc.cacheConfig.SnapshotWait, true, recover) + bc.snaps, _ = snapshot.New(bc.db, bc.stateCache.TrieDB(), bc.cacheConfig.SnapshotLimit, int(bc.cacheConfig.TriesInMemory), head.Root(), !bc.cacheConfig.SnapshotWait, true, recover, bc.stateCache.NoTries()) } // do options before start any routine for _, option := range options { @@ -1097,6 +1099,9 @@ func (bc *BlockChain) HasState(hash common.Hash) bool { return true } } + if bc.stateCache.NoTries() { + return bc.snaps != nil && bc.snaps.Snapshot(hash) != nil + } _, err := bc.stateCache.OpenTrie(hash) return err == nil } @@ -1109,6 +1114,9 @@ func (bc *BlockChain) HasBlockAndState(hash common.Hash, number uint64) bool { if block == nil { return false } + if bc.stateCache.NoTries() { + return bc.snaps != nil && bc.snaps.Snapshot(block.Root()) != nil + } return bc.HasState(block.Root()) } @@ -2105,6 +2113,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er if err != nil { return it.index, err } + if statedb.NoTrie() { + statedb.SetCurrentRoot(block.Root()) + } bc.updateHighestVerifiedHeader(block.Header()) // Enable prefetching to pull in trie node paths while processing transactions diff --git a/core/state/database.go b/core/state/database.go index 487589324c..dd114dc6ad 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -74,6 +74,9 @@ type Database interface { // Purge cache Purge() + + // NoTries returns whether the database has tries storage. + NoTries() bool } // Trie is a Ethereum Merkle Patricia trie. @@ -134,10 +137,12 @@ func NewDatabase(db ethdb.Database) Database { func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database { csc, _ := lru.New(codeSizeCacheSize) cc, _ := lru.New(codeCacheSize) + noTries := config != nil && config.NoTries return &cachingDB{ db: trie.NewDatabaseWithConfig(db, config), codeSizeCache: csc, codeCache: cc, + noTries: noTries, } } @@ -146,6 +151,7 @@ func NewDatabaseWithConfigAndCache(db ethdb.Database, config *trie.Config) Datab cc, _ := lru.New(codeCacheSize) atc, _ := lru.New(accountTrieCacheSize) stc, _ := lru.New(storageTrieCacheSize) + noTries := config != nil && config.NoTries database := &cachingDB{ db: trie.NewDatabaseWithConfig(db, config), @@ -153,8 +159,11 @@ func NewDatabaseWithConfigAndCache(db ethdb.Database, config *trie.Config) Datab codeCache: cc, accountTrieCache: atc, storageTrieCache: stc, + noTries: noTries, + } + if !noTries { + go database.purgeLoop() } - go database.purgeLoop() return database } @@ -164,6 +173,7 @@ type cachingDB struct { codeCache *lru.Cache accountTrieCache *lru.Cache storageTrieCache *lru.Cache + noTries bool } type triePair struct { @@ -187,6 +197,9 @@ func (db *cachingDB) purgeLoop() { // OpenTrie opens the main account trie at a specific root hash. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { + if db.noTries { + return trie.NewEmptyTrie(), nil + } if db.accountTrieCache != nil { if tr, exist := db.accountTrieCache.Get(root); exist { return tr.(Trie).(*trie.SecureTrie).Copy(), nil @@ -201,6 +214,9 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { // OpenStorageTrie opens the storage trie of an account. func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { + if db.noTries { + return trie.NewEmptyTrie(), nil + } if db.storageTrieCache != nil { if tries, exist := db.storageTrieCache.Get(addrHash); exist { triesPairs := tries.([3]*triePair) @@ -246,6 +262,10 @@ func (db *cachingDB) CacheStorage(addrHash common.Hash, root common.Hash, t Trie } } +func (db *cachingDB) NoTries() bool { + return db.noTries +} + func (db *cachingDB) Purge() { if db.storageTrieCache != nil { db.storageTrieCache.Purge() @@ -263,6 +283,8 @@ func (db *cachingDB) CopyTrie(t Trie) Trie { switch t := t.(type) { case *trie.SecureTrie: return t.Copy() + case *trie.EmptyTrie: + return t.Copy() default: panic(fmt.Errorf("unknown trie type %T", t)) } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 5b070f3afa..6d89d6b78a 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/snapshot" @@ -103,7 +104,7 @@ func NewPruner(db ethdb.Database, datadir, trieCachePath string, bloomSize, trie if headBlock == nil { return nil, errors.New("Failed to load head block") } - snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, int(triesInMemory), headBlock.Root(), false, false, false) + snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, int(triesInMemory), headBlock.Root(), false, false, false, false) if err != nil { return nil, err // The relevant snapshot(s) might not exist } @@ -138,6 +139,105 @@ func NewBlockPruner(db ethdb.Database, n *node.Node, oldAncientPath, newAncientP } } +func NewAllPruner(db ethdb.Database) (*Pruner, error) { + headBlock := rawdb.ReadHeadBlock(db) + if headBlock == nil { + return nil, errors.New("Failed to load head block") + } + return &Pruner{ + db: db, + }, nil +} + +func (p *Pruner) PruneAll(genesis *core.Genesis) error { + deleteCleanTrieCache(p.trieCachePath) + return pruneAll(p.db, genesis) +} + +func pruneAll(maindb ethdb.Database, g *core.Genesis) error { + var ( + count int + size common.StorageSize + pstart = time.Now() + logged = time.Now() + batch = maindb.NewBatch() + iter = maindb.NewIterator(nil, nil) + ) + start := time.Now() + for iter.Next() { + key := iter.Key() + if len(key) == common.HashLength { + count += 1 + size += common.StorageSize(len(key) + len(iter.Value())) + batch.Delete(key) + + var eta time.Duration // Realistically will never remain uninited + if done := binary.BigEndian.Uint64(key[:8]); done > 0 { + var ( + left = math.MaxUint64 - binary.BigEndian.Uint64(key[:8]) + speed = done/uint64(time.Since(pstart)/time.Millisecond+1) + 1 // +1s to avoid division by zero + ) + eta = time.Duration(left/speed) * time.Millisecond + } + if time.Since(logged) > 8*time.Second { + log.Info("Pruning state data", "nodes", count, "size", size, + "elapsed", common.PrettyDuration(time.Since(pstart)), "eta", common.PrettyDuration(eta)) + logged = time.Now() + } + // Recreate the iterator after every batch commit in order + // to allow the underlying compactor to delete the entries. + if batch.ValueSize() >= ethdb.IdealBatchSize { + batch.Write() + batch.Reset() + + iter.Release() + iter = maindb.NewIterator(nil, key) + } + } + } + if batch.ValueSize() > 0 { + batch.Write() + batch.Reset() + } + iter.Release() + log.Info("Pruned state data", "nodes", count, "size", size, "elapsed", common.PrettyDuration(time.Since(pstart))) + + // Start compactions, will remove the deleted data from the disk immediately. + // Note for small pruning, the compaction is skipped. + if count >= rangeCompactionThreshold { + cstart := time.Now() + for b := 0x00; b <= 0xf0; b += 0x10 { + var ( + start = []byte{byte(b)} + end = []byte{byte(b + 0x10)} + ) + if b == 0xf0 { + end = nil + } + log.Info("Compacting database", "range", fmt.Sprintf("%#x-%#x", start, end), "elapsed", common.PrettyDuration(time.Since(cstart))) + if err := maindb.Compact(start, end); err != nil { + log.Error("Database compaction failed", "error", err) + return err + } + } + log.Info("Database compaction finished", "elapsed", common.PrettyDuration(time.Since(cstart))) + } + statedb, _ := state.New(common.Hash{}, state.NewDatabase(maindb), nil) + for addr, account := range g.Alloc { + statedb.AddBalance(addr, account.Balance) + statedb.SetCode(addr, account.Code) + statedb.SetNonce(addr, account.Nonce) + for key, value := range account.Storage { + statedb.SetState(addr, key, value) + } + } + root := statedb.IntermediateRoot(false) + statedb.Commit(nil) + statedb.Database().TrieDB().Commit(root, true, nil) + log.Info("State pruning successful", "pruned", size, "elapsed", common.PrettyDuration(time.Since(start))) + return nil +} + func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, stateBloom *stateBloom, bloomPath string, middleStateRoots map[common.Hash]struct{}, start time.Time) error { // Delete all stale trie nodes in the disk. With the help of state bloom // the trie nodes(and codes) belong to the active state will be filtered @@ -585,7 +685,7 @@ func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string, tri // - The state HEAD is rewound already because of multiple incomplete `prune-state` // In this case, even the state HEAD is not exactly matched with snapshot, it // still feasible to recover the pruning correctly. - snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, int(triesInMemory), headBlock.Root(), false, false, true) + snaptree, err := snapshot.New(db, trie.NewDatabase(db), 256, int(triesInMemory), headBlock.Root(), false, false, true, false) if err != nil { return err // The relevant snapshot(s) might not exist } diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index 35c69cfd6b..3c18294cc1 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -126,7 +126,7 @@ func loadAndParseJournal(db ethdb.KeyValueStore, base *diskLayer) (snapshot, jou } // loadSnapshot loads a pre-existing state snapshot backed by a key-value store. -func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, recovery bool) (snapshot, bool, error) { +func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root common.Hash, recovery, withoutTrie bool) (snapshot, bool, error) { // If snapshotting is disabled (initial sync in progress), don't do anything, // wait for the chain to permit us to do something meaningful if rawdb.ReadSnapshotDisabled(diskdb) { @@ -145,6 +145,7 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root: baseRoot, } snapshot, generator, err := loadAndParseJournal(diskdb, base) + if err != nil { log.Warn("Failed to load new-format journal", "error", err) return nil, false, err @@ -158,6 +159,11 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, // which is below the snapshot. In this case the snapshot can be recovered // by re-executing blocks but right now it's unavailable. if head := snapshot.Root(); head != root { + log.Warn("Snapshot is not continuous with chain", "snaproot", head, "chainroot", root) + + if withoutTrie { + return snapshot, false, nil + } // If it's legacy snapshot, or it's new-format snapshot but // it's not in recovery mode, returns the error here for // rebuilding the entire snapshot forcibly. @@ -168,7 +174,6 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, // the disk layer is always higher than chain head. It can // be eventually recovered when the chain head beyonds the // disk layer. - log.Warn("Snapshot is not continuous with chain", "snaproot", head, "chainroot", root) } // Everything loaded correctly, resume any suspended operations if !generator.Done { diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 8ac93f28e4..38f52acced 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -184,7 +184,7 @@ type Tree struct { // store, on a background thread. If the memory layers from the journal is not // continuous with disk layer or the journal is missing, all diffs will be discarded // iff it's in "recovery" mode, otherwise rebuild is mandatory. -func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache, cap int, root common.Hash, async bool, rebuild bool, recovery bool) (*Tree, error) { +func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache, cap int, root common.Hash, async bool, rebuild bool, recovery, withoutTrie bool) (*Tree, error) { // Create a new, empty snapshot tree snap := &Tree{ diskdb: diskdb, @@ -197,7 +197,7 @@ func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache, cap int, root defer snap.waitBuild() } // Attempt to load a previously persisted snapshot and rebuild one if failed - head, disabled, err := loadSnapshot(diskdb, triedb, cache, root, recovery) + head, disabled, err := loadSnapshot(diskdb, triedb, cache, root, recovery, withoutTrie) if disabled { log.Warn("Snapshot maintenance disabled (syncing)") return snap, nil diff --git a/core/state/statedb.go b/core/state/statedb.go index 5ea84f4032..1bb1db7ba1 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -77,10 +77,12 @@ type StateDB struct { prefetcherLock sync.Mutex prefetcher *triePrefetcher originalRoot common.Hash // The pre-state root, before any changes were made + currentRoot common.Hash // only used when noTrie is true expectedRoot common.Hash // The state root in the block header stateRoot common.Hash // The calculation result of IntermediateRoot trie Trie + noTrie bool hasher crypto.KeccakState diffLayer *types.DiffLayer diffTries map[common.Address]Trie @@ -160,6 +162,7 @@ func newStateDB(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, journal: newJournal(), hasher: crypto.NewKeccakState(), } + if sdb.snaps != nil { if sdb.snap = sdb.snaps.Snapshot(root); sdb.snap != nil { sdb.snapDestructs = make(map[common.Address]struct{}) @@ -174,6 +177,7 @@ func newStateDB(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, if err != nil && (sdb.snap == nil || snapVerified) { return nil, err } + _, sdb.noTrie = tr.(*trie.EmptyTrie) sdb.trie = tr return sdb, nil } @@ -184,6 +188,9 @@ func newStateDB(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, func (s *StateDB) StartPrefetcher(namespace string) { s.prefetcherLock.Lock() defer s.prefetcherLock.Unlock() + if s.noTrie { + return + } if s.prefetcher != nil { s.prefetcher.close() s.prefetcher = nil @@ -198,6 +205,9 @@ func (s *StateDB) StartPrefetcher(namespace string) { func (s *StateDB) StopPrefetcher() { s.prefetcherLock.Lock() defer s.prefetcherLock.Unlock() + if s.noTrie { + return + } if s.prefetcher != nil { s.prefetcher.close() s.prefetcher = nil @@ -237,6 +247,14 @@ func (s *StateDB) setError(err error) { } } +func (s *StateDB) NoTrie() bool { + return s.noTrie +} + +func (s *StateDB) SetCurrentRoot(root common.Hash) { + s.currentRoot = root +} + func (s *StateDB) Error() error { return s.dbErr } @@ -549,6 +567,9 @@ func (s *StateDB) Suicide(addr common.Address) bool { // updateStateObject writes the given object to the trie. func (s *StateDB) updateStateObject(obj *StateObject) { + if s.noTrie { + return + } // Track the amount of time wasted on updating the account from the trie if metrics.EnabledExpensive { defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) @@ -570,6 +591,9 @@ func (s *StateDB) updateStateObject(obj *StateObject) { // deleteStateObject removes the given object from the state trie. func (s *StateDB) deleteStateObject(obj *StateObject) { + if s.noTrie { + return + } // Track the amount of time wasted on deleting the account from the trie if metrics.EnabledExpensive { defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) @@ -1013,6 +1037,7 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { // It is called in between transactions to get the root hash that // goes into transaction receipts. func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { + // light process is not allowed when there is no trie if s.lightProcessed { s.StopPrefetcher() return s.trie.Hash() @@ -1109,16 +1134,18 @@ func (s *StateDB) StateIntermediateRoot() common.Hash { s.trie = tr } usedAddrs := make([][]byte, 0, len(s.stateObjectsPending)) - for addr := range s.stateObjectsPending { - if obj := s.stateObjects[addr]; obj.deleted { - s.deleteStateObject(obj) - } else { - s.updateStateObject(obj) + if !s.noTrie { + for addr := range s.stateObjectsPending { + if obj := s.stateObjects[addr]; obj.deleted { + s.deleteStateObject(obj) + } else { + s.updateStateObject(obj) + } + usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure + } + if prefetcher != nil { + prefetcher.used(s.originalRoot, usedAddrs) } - usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure - } - if prefetcher != nil { - prefetcher.used(s.originalRoot, usedAddrs) } if len(s.stateObjectsPending) > 0 { s.stateObjectsPending = make(map[common.Address]struct{}) @@ -1127,8 +1154,11 @@ func (s *StateDB) StateIntermediateRoot() common.Hash { if metrics.EnabledExpensive { defer func(start time.Time) { s.AccountHashes += time.Since(start) }(time.Now()) } - root := s.trie.Hash() - return root + if s.noTrie { + return s.currentRoot + } else { + return s.trie.Hash() + } } // Prepare sets the current transaction hash and index and block hash which is @@ -1351,8 +1381,10 @@ func (s *StateDB) Commit(failPostCommitFunc func(), postCommitFuncs ...func() er // Write any contract code associated with the state object tasks <- func() { // Write any storage changes in the state object to its storage trie - if err := obj.CommitTrie(s.db); err != nil { - taskResults <- err + if !s.noTrie { + if err := obj.CommitTrie(s.db); err != nil { + taskResults <- err + } } taskResults <- nil } @@ -1371,24 +1403,27 @@ func (s *StateDB) Commit(failPostCommitFunc func(), postCommitFuncs ...func() er // The onleaf func is called _serially_, so we can reuse the same account // for unmarshalling every time. - var account Account - root, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error { - if err := rlp.DecodeBytes(leaf, &account); err != nil { + if !s.noTrie { + var account Account + root, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error { + if err := rlp.DecodeBytes(leaf, &account); err != nil { + return nil + } + if account.Root != emptyRoot { + s.db.TrieDB().Reference(account.Root, parent) + } return nil + }) + if err != nil { + return err } - if account.Root != emptyRoot { - s.db.TrieDB().Reference(account.Root, parent) + if root != emptyRoot { + s.db.CacheAccount(root, s.trie) } - return nil - }) - if err != nil { - return err - } - if root != emptyRoot { - s.db.CacheAccount(root, s.trie) } + for _, postFunc := range postCommitFuncs { - err = postFunc() + err := postFunc() if err != nil { return err } diff --git a/core/state_processor.go b/core/state_processor.go index 14fe9b4b92..a21d68c504 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -408,6 +408,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg // usually do have two tx, one for validator set contract, another for system reward contract. systemTxs := make([]*types.Transaction, 0, 2) + for i, tx := range block.Transactions() { if isPoSA { if isSystemTx, err := posa.IsSystemTransaction(tx, block.Header()); err != nil { @@ -423,11 +424,11 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return statedb, nil, nil, 0, err } statedb.Prepare(tx.Hash(), block.Hash(), i) + receipt, err := applyTransaction(msg, p.config, p.bc, nil, gp, statedb, header, tx, usedGas, vmenv, bloomProcessors) if err != nil { return statedb, nil, nil, 0, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } - commonTxs = append(commonTxs, tx) receipts = append(receipts, receipt) } diff --git a/eth/backend.go b/eth/backend.go index ab93006437..285bb6f7a7 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -194,13 +194,14 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { TrieDirtyLimit: config.TrieDirtyCache, TrieDirtyDisabled: config.NoPruning, TrieTimeLimit: config.TrieTimeout, + NoTries: config.NoTries, SnapshotLimit: config.SnapshotCache, TriesInMemory: config.TriesInMemory, Preimages: config.Preimages, } ) bcOps := make([]core.BlockChainOption, 0) - if config.DiffSync { + if config.DiffSync && !config.NoTries { bcOps = append(bcOps, core.EnableLightProcessor) } if config.PipeCommit { diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 09baad1e1c..4f2c87c52e 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -79,6 +79,7 @@ var Defaults = Config{ TrieDirtyCache: 256, TrieTimeout: 60 * time.Minute, TriesInMemory: 128, + NoTries: false, SnapshotCache: 102, DiffBlock: uint64(86400), Miner: miner.Config{ @@ -175,6 +176,7 @@ type Config struct { TrieTimeout time.Duration SnapshotCache int TriesInMemory uint64 + NoTries bool Preimages bool // Mining options diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index f192a1aace..ba2996279d 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -3,6 +3,7 @@ package ethconfig import ( + "math/big" "time" "github.com/ethereum/go-ethereum/common" @@ -25,6 +26,10 @@ func (c Config) MarshalTOML() (interface{}, error) { SnapDiscoveryURLs []string NoPruning bool NoPrefetch bool + DirectBroadcast bool + DisableSnapProtocol bool + DiffSync bool + RangeLimit bool TxLookupLimit uint64 `toml:",omitempty"` Whitelist map[uint64]common.Hash `toml:"-"` LightServ int `toml:",omitempty"` @@ -42,28 +47,30 @@ func (c Config) MarshalTOML() (interface{}, error) { DatabaseCache int DatabaseFreezer string DatabaseDiff string + PersistDiff bool + DiffBlock uint64 TrieCleanCache int TrieCleanCacheJournal string `toml:",omitempty"` TrieCleanCacheRejournal time.Duration `toml:",omitempty"` TrieDirtyCache int TrieTimeout time.Duration - TriesInMemory uint64 `toml:",omitempty"` SnapshotCache int + TriesInMemory uint64 + NoTries bool Preimages bool - PersistDiff bool - DiffBlock uint64 `toml:",omitempty"` Miner miner.Config - Ethash ethash.Config + Ethash ethash.Config `toml:",omitempty"` TxPool core.TxPoolConfig GPO gasprice.Config EnablePreimageRecording bool DocRoot string `toml:"-"` EWASMInterpreter string EVMInterpreter string - RPCGasCap uint64 `toml:",omitempty"` - RPCTxFeeCap float64 `toml:",omitempty"` + RPCGasCap uint64 + RPCTxFeeCap float64 Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + OverrideBerlin *big.Int `toml:",omitempty"` } var enc Config enc.Genesis = c.Genesis @@ -73,6 +80,10 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.EthDiscoveryURLs = c.EthDiscoveryURLs enc.SnapDiscoveryURLs = c.SnapDiscoveryURLs enc.NoPruning = c.NoPruning + enc.DirectBroadcast = c.DirectBroadcast + enc.DisableSnapProtocol = c.DisableSnapProtocol + enc.DiffSync = c.DiffSync + enc.RangeLimit = c.RangeLimit enc.TxLookupLimit = c.TxLookupLimit enc.Whitelist = c.Whitelist enc.LightServ = c.LightServ @@ -90,16 +101,17 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.DatabaseCache = c.DatabaseCache enc.DatabaseFreezer = c.DatabaseFreezer enc.DatabaseDiff = c.DatabaseDiff + enc.PersistDiff = c.PersistDiff + enc.DiffBlock = c.DiffBlock enc.TrieCleanCache = c.TrieCleanCache enc.TrieCleanCacheJournal = c.TrieCleanCacheJournal enc.TrieCleanCacheRejournal = c.TrieCleanCacheRejournal enc.TrieDirtyCache = c.TrieDirtyCache enc.TrieTimeout = c.TrieTimeout - enc.TriesInMemory = c.TriesInMemory enc.SnapshotCache = c.SnapshotCache + enc.TriesInMemory = c.TriesInMemory + enc.NoTries = c.NoTries enc.Preimages = c.Preimages - enc.PersistDiff = c.PersistDiff - enc.DiffBlock = c.DiffBlock enc.Miner = c.Miner enc.Ethash = c.Ethash enc.TxPool = c.TxPool @@ -112,6 +124,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.RPCTxFeeCap = c.RPCTxFeeCap enc.Checkpoint = c.Checkpoint enc.CheckpointOracle = c.CheckpointOracle + enc.OverrideBerlin = c.OverrideBerlin return &enc, nil } @@ -126,6 +139,10 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { SnapDiscoveryURLs []string NoPruning *bool NoPrefetch *bool + DirectBroadcast *bool + DisableSnapProtocol *bool + DiffSync *bool + RangeLimit *bool TxLookupLimit *uint64 `toml:",omitempty"` Whitelist map[uint64]common.Hash `toml:"-"` LightServ *int `toml:",omitempty"` @@ -144,27 +161,29 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { DatabaseFreezer *string DatabaseDiff *string PersistDiff *bool - DiffBlock *uint64 `toml:",omitempty"` + DiffBlock *uint64 TrieCleanCache *int TrieCleanCacheJournal *string `toml:",omitempty"` TrieCleanCacheRejournal *time.Duration `toml:",omitempty"` TrieDirtyCache *int TrieTimeout *time.Duration - TriesInMemory *uint64 `toml:",omitempty"` SnapshotCache *int + TriesInMemory *uint64 + NoTries *bool Preimages *bool Miner *miner.Config - Ethash *ethash.Config + Ethash *ethash.Config `toml:",omitempty"` TxPool *core.TxPoolConfig GPO *gasprice.Config EnablePreimageRecording *bool DocRoot *string `toml:"-"` EWASMInterpreter *string EVMInterpreter *string - RPCGasCap *uint64 `toml:",omitempty"` - RPCTxFeeCap *float64 `toml:",omitempty"` + RPCGasCap *uint64 + RPCTxFeeCap *float64 Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + OverrideBerlin *big.Int `toml:",omitempty"` } var dec Config if err := unmarshal(&dec); err != nil { @@ -191,6 +210,18 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.NoPruning != nil { c.NoPruning = *dec.NoPruning } + if dec.DirectBroadcast != nil { + c.DirectBroadcast = *dec.DirectBroadcast + } + if dec.DisableSnapProtocol != nil { + c.DisableSnapProtocol = *dec.DisableSnapProtocol + } + if dec.DiffSync != nil { + c.DiffSync = *dec.DiffSync + } + if dec.RangeLimit != nil { + c.RangeLimit = *dec.RangeLimit + } if dec.TxLookupLimit != nil { c.TxLookupLimit = *dec.TxLookupLimit } @@ -263,11 +294,14 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.TrieTimeout != nil { c.TrieTimeout = *dec.TrieTimeout } + if dec.SnapshotCache != nil { + c.SnapshotCache = *dec.SnapshotCache + } if dec.TriesInMemory != nil { c.TriesInMemory = *dec.TriesInMemory } - if dec.SnapshotCache != nil { - c.SnapshotCache = *dec.SnapshotCache + if dec.NoTries != nil { + c.NoTries = *dec.NoTries } if dec.Preimages != nil { c.Preimages = *dec.Preimages @@ -308,5 +342,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.CheckpointOracle != nil { c.CheckpointOracle = dec.CheckpointOracle } + if dec.OverrideBerlin != nil { + c.OverrideBerlin = dec.OverrideBerlin + } return nil } diff --git a/light/trie.go b/light/trie.go index 3896b73c4d..3f942f3607 100644 --- a/light/trie.go +++ b/light/trie.go @@ -49,6 +49,10 @@ type odrDatabase struct { backend OdrBackend } +func (db *odrDatabase) NoTries() bool { + return false +} + func (db *odrDatabase) OpenTrie(root common.Hash) (state.Trie, error) { return &odrTrie{db: db, id: db.id}, nil } @@ -178,6 +182,10 @@ func (t *odrTrie) do(key []byte, fn func() error) error { } } +func (db *odrTrie) NoTries() bool { + return false +} + type nodeIterator struct { trie.NodeIterator t *odrTrie diff --git a/tests/state_test_util.go b/tests/state_test_util.go index a688254a20..1de4a787dd 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -234,7 +234,7 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter boo var snaps *snapshot.Tree if snapshotter { - snaps, _ = snapshot.New(db, sdb.TrieDB(), 1, 128, root, false, true, false) + snaps, _ = snapshot.New(db, sdb.TrieDB(), 1, 128, root, false, true, false, false) } statedb, _ = state.New(root, sdb, snaps) return snaps, statedb diff --git a/trie/database.go b/trie/database.go index b6a3154d48..fa8b746c58 100644 --- a/trie/database.go +++ b/trie/database.go @@ -282,6 +282,7 @@ type Config struct { Cache int // Memory allowance (MB) to use for caching trie nodes in memory Journal string // Journal of clean cache to survive node restarts Preimages bool // Flag whether the preimage of trie key is recorded + NoTries bool } // NewDatabase creates a new trie database to store ephemeral trie content before diff --git a/trie/dummy_trie.go b/trie/dummy_trie.go new file mode 100644 index 0000000000..99eb79fbd4 --- /dev/null +++ b/trie/dummy_trie.go @@ -0,0 +1,96 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/ethdb" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" +) + +type EmptyTrie struct{} + +func (t *EmptyTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { + return nil +} + +// NewSecure creates a dummy trie +func NewEmptyTrie() *EmptyTrie { + return &EmptyTrie{} +} + +func (t *EmptyTrie) Get(key []byte) []byte { + return nil +} + +func (t *EmptyTrie) TryGet(key []byte) ([]byte, error) { + return nil, nil +} + +func (t *EmptyTrie) TryGetNode(path []byte) ([]byte, int, error) { + return nil, 0, nil +} +func (t *EmptyTrie) Update(key, value []byte) {} + +func (t *EmptyTrie) TryUpdate(key, value []byte) error { + return nil +} + +// Delete removes any existing value for key from the trie. +func (t *EmptyTrie) Delete(key []byte) { + if err := t.TryDelete(key); err != nil { + log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + } +} + +func (t *EmptyTrie) TryDelete(key []byte) error { + + return nil +} + +func (t *EmptyTrie) GetKey(shaKey []byte) []byte { + return nil +} + +func (t *EmptyTrie) Commit(onleaf LeafCallback) (root common.Hash, err error) { + + return common.Hash{}, nil +} + +func (t *EmptyTrie) Hash() common.Hash { + return common.Hash{} +} + +// Copy returns a copy of SecureTrie. +func (t *EmptyTrie) Copy() *EmptyTrie { + cpy := *t + return &cpy +} + +func (t *EmptyTrie) ResetCopy() *EmptyTrie { + cpy := *t + return &cpy +} + +// NodeIterator returns an iterator that returns nodes of the underlying trie. Iteration +// starts at the key after the given start key. +func (t *EmptyTrie) NodeIterator(start []byte) NodeIterator { + return nil +}