[R4R] Separate Processing and State Verification on BSC (#926)

Implement Separate Processing and State Verification on BSC

cmd/geth/main.go

@@ -71,6 +71,8 @@ var (
 		utils.NoUSBFlag,
 		utils.DirectBroadcastFlag,
 		utils.DisableSnapProtocolFlag,
+		utils.DisableDiffProtocolFlag,
+		utils.EnableTrustProtocolFlag,
 		utils.DiffSyncFlag,
 		utils.PipeCommitFlag,
 		utils.RangeLimitFlag,
@@ -98,6 +100,7 @@ var (
 		utils.TxPoolLifetimeFlag,
 		utils.TxPoolReannounceTimeFlag,
 		utils.SyncModeFlag,
+		utils.TriesVerifyModeFlag,
 		utils.ExitWhenSyncedFlag,
 		utils.GCModeFlag,
 		utils.SnapshotFlag,

cmd/geth/snapshot.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -29,13 +30,16 @@ import (
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/state/pruner"
 	"github.com/ethereum/go-ethereum/core/state/snapshot"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
@@ -129,6 +133,32 @@ geth snapshot verify-state <state-root>
 will traverse the whole accounts and storages set based on the specified
 snapshot and recalculate the root hash of state for verification.
 In other words, this command does the snapshot to trie conversion.
+`,
+			},
+			{
+				Name: "insecure-prune-all",
+				Usage: "Prune all trie state data except genesis block, it will break storage for fullnode, only suitable for fast node " +
+					"who do not need trie storage at all",
+				ArgsUsage: "<genesisPath>",
+				Action:    utils.MigrateFlags(pruneAllState),
+				Category:  "MISCELLANEOUS COMMANDS",
+				Flags: []cli.Flag{
+					utils.DataDirFlag,
+					utils.AncientFlag,
+					utils.RopstenFlag,
+					utils.RinkebyFlag,
+					utils.GoerliFlag,
+				},
+				Description: `
+will prune all historical trie state data except genesis block.
+All trie nodes will be deleted from the database. 
+
+It expects the genesis file as argument.
+
+WARNING: It's necessary to delete the trie clean cache after the pruning.
+If you specify another directory for the trie clean cache via "--cache.trie.journal"
+during the use of Geth, please also specify it here for correct deletion. Otherwise
+the trie clean cache with default directory will be deleted.
 `,
 			},
 			{
@@ -195,7 +225,7 @@ func accessDb(ctx *cli.Context, stack *node.Node) (ethdb.Database, error) {
 	}
 	headHeader := headBlock.Header()
 	//Make sure the MPT and snapshot matches before pruning, otherwise the node can not start.
-	snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, TriesInMemory, headBlock.Root(), false, false, false)
+	snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, TriesInMemory, headBlock.Root(), false, false, false, false)
 	if err != nil {
 		log.Error("snaptree error", "err", err)
 		return nil, err // The relevant snapshot(s) might not exist
@@ -363,6 +393,48 @@ func pruneState(ctx *cli.Context) error {
 	return nil
 }
 
+func pruneAllState(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	genesisPath := ctx.Args().First()
+	if len(genesisPath) == 0 {
+		utils.Fatalf("Must supply path to genesis JSON file")
+	}
+	file, err := os.Open(genesisPath)
+	if err != nil {
+		utils.Fatalf("Failed to read genesis file: %v", err)
+	}
+	defer file.Close()
+
+	g := new(core.Genesis)
+	if err := json.NewDecoder(file).Decode(g); err != nil {
+		cfg := gethConfig{
+			Eth:     ethconfig.Defaults,
+			Node:    defaultNodeConfig(),
+			Metrics: metrics.DefaultConfig,
+		}
+
+		// Load config file.
+		if err := loadConfig(genesisPath, &cfg); err != nil {
+			utils.Fatalf("%v", err)
+		}
+		g = cfg.Eth.Genesis
+	}
+
+	chaindb := utils.MakeChainDatabase(ctx, stack, false, false)
+	pruner, err := pruner.NewAllPruner(chaindb)
+	if err != nil {
+		log.Error("Failed to open snapshot tree", "err", err)
+		return err
+	}
+	if err = pruner.PruneAll(g); err != nil {
+		log.Error("Failed to prune state", "err", err)
+		return err
+	}
+	return nil
+}
+
 func verifyState(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
@@ -373,7 +445,7 @@ func verifyState(ctx *cli.Context) error {
 		log.Error("Failed to load head block")
 		return errors.New("no head block")
 	}
-	snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, 128, headBlock.Root(), false, false, false)
+	snaptree, err := snapshot.New(chaindb, trie.NewDatabase(chaindb), 256, 128, headBlock.Root(), false, false, false, false)
 	if err != nil {
 		log.Error("Failed to open snapshot tree", "err", err)
 		return err

cmd/geth/usage.go

@@ -41,6 +41,8 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.NoUSBFlag,
 			utils.DirectBroadcastFlag,
 			utils.DisableSnapProtocolFlag,
+			utils.DisableDiffProtocolFlag,
+			utils.EnableTrustProtocolFlag,
 			utils.RangeLimitFlag,
 			utils.SmartCardDaemonPathFlag,
 			utils.NetworkIdFlag,
@@ -50,6 +52,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.YoloV3Flag,
 			utils.RopstenFlag,
 			utils.SyncModeFlag,
+			utils.TriesVerifyModeFlag,
 			utils.ExitWhenSyncedFlag,
 			utils.GCModeFlag,
 			utils.TxLookupLimitFlag,

cmd/utils/flags.go

@@ -122,6 +122,14 @@ var (
 		Name:  "disablesnapprotocol",
 		Usage: "Disable snap protocol",
 	}
+	DisableDiffProtocolFlag = cli.BoolFlag{
+		Name:  "disablediffprotocol",
+		Usage: "Disable diff protocol",
+	}
+	EnableTrustProtocolFlag = cli.BoolFlag{
+		Name:  "enabletrustprotocol",
+		Usage: "Enable trust protocol",
+	}
 	DiffSyncFlag = cli.BoolFlag{
 		Name: "diffsync",
 		Usage: "Enable diffy sync, Please note that enable diffsync will improve the syncing speed, " +
@@ -264,6 +272,20 @@ var (
 		Usage: "The layer of tries trees that keep in memory",
 		Value: 128,
 	}
+	defaultVerifyMode   = ethconfig.Defaults.TriesVerifyMode
+	TriesVerifyModeFlag = TextMarshalerFlag{
+		Name: "tries-verify-mode",
+		Usage: `tries verify mode: 
+				"local(default): a normal full node with complete state world(both MPT and snapshot), merkle state root will 
+				                 be verified against the block header.", 
+				"full: a fast node with only snapshot state world. Merkle state root is verified by the trustworthy remote verify node 
+					   by comparing the diffhash(an identify of difflayer generated by the block) and state root.", 
+				"insecure: same as full mode, except that it can tolerate without verifying the diffhash when verify node does not have it.", 
+				"none: no merkle state root verification at all, there is no need to setup or connect remote verify node at all,
+				       it is more light comparing to full and insecure mode, but get a very small chance that the state is not consistent
+						with other peers."`,
+		Value: &defaultVerifyMode,
+	}
 	OverrideBerlinFlag = cli.Uint64Flag{
 		Name:  "override.berlin",
 		Usage: "Manually specify Berlin fork-block, overriding the bundled setting",
@@ -1644,6 +1666,12 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if ctx.GlobalIsSet(DisableSnapProtocolFlag.Name) {
 		cfg.DisableSnapProtocol = ctx.GlobalBool(DisableSnapProtocolFlag.Name)
 	}
+	if ctx.GlobalIsSet(DisableDiffProtocolFlag.Name) {
+		cfg.DisableDiffProtocol = ctx.GlobalIsSet(DisableDiffProtocolFlag.Name)
+	}
+	if ctx.GlobalIsSet(EnableTrustProtocolFlag.Name) {
+		cfg.EnableTrustProtocol = ctx.GlobalIsSet(EnableTrustProtocolFlag.Name)
+	}
 	if ctx.GlobalIsSet(DiffSyncFlag.Name) {
 		cfg.DiffSync = ctx.GlobalBool(DiffSyncFlag.Name)
 	}
@@ -1677,6 +1705,14 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if ctx.GlobalIsSet(TriesInMemoryFlag.Name) {
 		cfg.TriesInMemory = ctx.GlobalUint64(TriesInMemoryFlag.Name)
 	}
+	if ctx.GlobalIsSet(TriesVerifyModeFlag.Name) {
+		cfg.TriesVerifyMode = *GlobalTextMarshaler(ctx, TriesVerifyModeFlag.Name).(*core.VerifyMode)
+		// If a node sets verify mode to full or insecure, it's a fast node and need
+		// to verify blocks from verify nodes, then it should enable trust protocol.
+		if cfg.TriesVerifyMode.NeedRemoteVerify() {
+			cfg.EnableTrustProtocol = true
+		}
+	}
 	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheSnapshotFlag.Name) {
 		cfg.SnapshotCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheSnapshotFlag.Name) / 100
 	}
@@ -1716,7 +1752,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		cfg.RPCTxFeeCap = ctx.GlobalFloat64(RPCGlobalTxFeeCapFlag.Name)
 	}
 	if ctx.GlobalIsSet(NoDiscoverFlag.Name) {
-		cfg.EthDiscoveryURLs, cfg.SnapDiscoveryURLs = []string{}, []string{}
+		cfg.EthDiscoveryURLs, cfg.SnapDiscoveryURLs, cfg.TrustDiscoveryURLs = []string{}, []string{}, []string{}
 	} else if ctx.GlobalIsSet(DNSDiscoveryFlag.Name) {
 		urls := ctx.GlobalString(DNSDiscoveryFlag.Name)
 		if urls == "" {
@@ -1822,6 +1858,7 @@ func SetDNSDiscoveryDefaults(cfg *ethconfig.Config, genesis common.Hash) {
 	if url := params.KnownDNSNetwork(genesis, protocol); url != "" {
 		cfg.EthDiscoveryURLs = []string{url}
 		cfg.SnapDiscoveryURLs = cfg.EthDiscoveryURLs
+		cfg.TrustDiscoveryURLs = cfg.EthDiscoveryURLs
 	}
 }
 

core/block_validator.go

@@ -30,23 +30,38 @@ import (
 
 const badBlockCacheExpire = 30 * time.Second
 
+type BlockValidatorOption func(*BlockValidator) *BlockValidator
+
+func EnableRemoteVerifyManager(remoteValidator *remoteVerifyManager) BlockValidatorOption {
+	return func(bv *BlockValidator) *BlockValidator {
+		bv.remoteValidator = remoteValidator
+		return bv
+	}
+}
+
 // BlockValidator is responsible for validating block headers, uncles and
 // processed state.
 //
 // BlockValidator implements Validator.
 type BlockValidator struct {
-	config *params.ChainConfig // Chain configuration options
-	bc     *BlockChain         // Canonical block chain
-	engine consensus.Engine    // Consensus engine used for validating
+	config          *params.ChainConfig // Chain configuration options
+	bc              *BlockChain         // Canonical block chain
+	engine          consensus.Engine    // Consensus engine used for validating
+	remoteValidator *remoteVerifyManager
 }
 
 // NewBlockValidator returns a new block validator which is safe for re-use
-func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engine consensus.Engine) *BlockValidator {
+func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engine consensus.Engine, opts ...BlockValidatorOption) *BlockValidator {
 	validator := &BlockValidator{
 		config: config,
 		engine: engine,
 		bc:     blockchain,
 	}
+
+	for _, opt := range opts {
+		validator = opt(validator)
+	}
+
 	return validator
 }
 
@@ -92,6 +107,12 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 			}
 			return nil
 		},
+		func() error {
+			if v.remoteValidator != nil && !v.remoteValidator.AncestorVerified(block.Header()) {
+				return fmt.Errorf("%w, number: %s, hash: %s", ErrAncestorHasNotBeenVerified, block.Number(), block.Hash())
+			}
+			return nil
+		},
 	}
 	validateRes := make(chan error, len(validateFuns))
 	for _, f := range validateFuns {
@@ -171,6 +192,10 @@ func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateD
 	return err
 }
 
+func (v *BlockValidator) RemoteVerifyManager() *remoteVerifyManager {
+	return v.remoteValidator
+}
+
 // CalcGasLimit computes the gas limit of the next block after parent. It aims
 // to keep the baseline gas above the provided floor, and increase it towards the
 // ceil if the blocks are full. If the ceil is exceeded, it will always decrease