Enable maverick node for e2e test (#301)

* Update `e2e.App` and `marverick.App` with `Ostracon` features

* Enable maverick node

test/e2e/app/config.go

@@ -22,7 +22,6 @@ type Config struct {
 	PrivValKey       string                      `toml:"privval_key"`
 	PrivValState     string                      `toml:"privval_state"`
 	Misbehaviors     map[string]string           `toml:"misbehaviors"`
-	KeyType          string                      `toml:"key_type"`
 }
 
 // LoadConfig loads the configuration from disk.

test/e2e/app/main.go

@@ -158,8 +158,9 @@ func startMaverick(cfg *Config) error {
 		misbehaviors[height] = mcs.MisbehaviorList[misbehaviorString]
 	}
 
+	privKey, _ := maverick.LoadOrGenFilePV(tmcfg.PrivValidatorKeyFile(), tmcfg.PrivValidatorStateFile(), tmcfg.PrivKeyType)
 	n, err := maverick.NewNode(tmcfg,
-		maverick.LoadOrGenFilePV(tmcfg.PrivValidatorKeyFile(), tmcfg.PrivValidatorStateFile()),
+		privKey,
 		nodeKey,
 		proxy.NewLocalClientCreator(app),
 		maverick.DefaultGenesisDocProviderFunc(tmcfg),

test/e2e/networks/ci.toml

@@ -36,7 +36,7 @@ seeds = ["seed01"]
 seeds = ["seed01"]
 snapshot_interval = 5
 perturb = ["disconnect"]
-#misbehaviors = { 1018 = "double-prevote" }
+misbehaviors = { 1018 = "double-prevote" }
 
 [node.validator02]
 seeds = ["seed02"]

test/maverick/consensus/misbehavior.go

@@ -378,9 +378,11 @@ func defaultReceiveProposal(cs *State, proposal *types.Proposal) error {
 		return ErrInvalidProposalPOLRound
 	}
 
+	proposer := cs.Validators.SelectProposer(cs.state.LastProofHash, proposal.Height, proposal.Round)
+
 	p := proposal.ToProto()
 	// Verify signature
-	if !cs.Proposer.PubKey.VerifySignature(
+	if !proposer.PubKey.VerifySignature(
 		types.ProposalSignBytes(cs.state.ChainID, p), proposal.Signature) {
 		return ErrInvalidProposalSignature
 	}

test/maverick/consensus/reactor.go

@@ -55,13 +55,13 @@ type ReactorOption func(*Reactor)
 
 // NewReactor returns a new Reactor with the given
 // consensusState.
-func NewReactor(consensusState *State, waitSync bool, options ...ReactorOption) *Reactor {
+func NewReactor(consensusState *State, waitSync bool, async bool, recvBufSize int, options ...ReactorOption) *Reactor {
 	conR := &Reactor{
 		conS:     consensusState,
 		waitSync: waitSync,
 		Metrics:  tmcon.NopMetrics(),
 	}
-	conR.BaseReactor = *p2p.NewBaseReactor("Consensus", conR, true, 1000)
+	conR.BaseReactor = *p2p.NewBaseReactor("Consensus", conR, async, recvBufSize)
 
 	for _, option := range options {
 		option(conR)
@@ -75,6 +75,12 @@ func NewReactor(consensusState *State, waitSync bool, options ...ReactorOption)
 func (conR *Reactor) OnStart() error {
 	conR.Logger.Info("Reactor ", "waitSync", conR.WaitSync())
 
+	// call BaseReactor's OnStart()
+	err := conR.BaseReactor.OnStart()
+	if err != nil {
+		return err
+	}
+
 	// start routine that computes peer statistics for evaluating peer quality
 	go conR.peerStatsRoutine()
 
@@ -331,9 +337,9 @@ func (conR *Reactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) {
 		case *tmcon.VoteMessage:
 			cs := conR.conS
 			cs.mtx.RLock()
-			height, valSize, lastCommitSize := cs.Height, cs.Validators.Size(), cs.LastCommit.Size()
+			height, voterSize, lastCommitSize := cs.Height, cs.Voters.Size(), cs.LastCommit.Size()
 			cs.mtx.RUnlock()
-			ps.EnsureVoteBitArrays(height, valSize)
+			ps.EnsureVoteBitArrays(height, voterSize)
 			ps.EnsureVoteBitArrays(height-1, lastCommitSize)
 			ps.SetHasVote(msg.Vote)
 
@@ -1135,7 +1141,7 @@ func (ps *PeerState) getVoteBitArray(height int64, round int32, votesType tmprot
 }
 
 // 'round': A round for which we have a +2/3 commit.
-func (ps *PeerState) ensureCatchupCommitRound(height int64, round int32, numValidators int) {
+func (ps *PeerState) ensureCatchupCommitRound(height int64, round int32, numVoters int) {
 	if ps.PRS.Height != height {
 		return
 	}
@@ -1159,37 +1165,37 @@ func (ps *PeerState) ensureCatchupCommitRound(height int64, round int32, numVali
 	if round == ps.PRS.Round {
 		ps.PRS.CatchupCommit = ps.PRS.Precommits
 	} else {
-		ps.PRS.CatchupCommit = bits.NewBitArray(numValidators)
+		ps.PRS.CatchupCommit = bits.NewBitArray(numVoters)
 	}
 }
 
 // EnsureVoteBitArrays ensures the bit-arrays have been allocated for tracking
 // what votes this peer has received.
-// NOTE: It's important to make sure that numValidators actually matches
-// what the node sees as the number of validators for height.
-func (ps *PeerState) EnsureVoteBitArrays(height int64, numValidators int) {
+// NOTE: It's important to make sure that numVoters actually matches
+// what the node sees as the number of voters for height.
+func (ps *PeerState) EnsureVoteBitArrays(height int64, numVoters int) {
 	ps.mtx.Lock()
 	defer ps.mtx.Unlock()
-	ps.ensureVoteBitArrays(height, numValidators)
+	ps.ensureVoteBitArrays(height, numVoters)
 }
 
-func (ps *PeerState) ensureVoteBitArrays(height int64, numValidators int) {
+func (ps *PeerState) ensureVoteBitArrays(height int64, numVoters int) {
 	if ps.PRS.Height == height {
 		if ps.PRS.Prevotes == nil {
-			ps.PRS.Prevotes = bits.NewBitArray(numValidators)
+			ps.PRS.Prevotes = bits.NewBitArray(numVoters)
 		}
 		if ps.PRS.Precommits == nil {
-			ps.PRS.Precommits = bits.NewBitArray(numValidators)
+			ps.PRS.Precommits = bits.NewBitArray(numVoters)
 		}
 		if ps.PRS.CatchupCommit == nil {
-			ps.PRS.CatchupCommit = bits.NewBitArray(numValidators)
+			ps.PRS.CatchupCommit = bits.NewBitArray(numVoters)
 		}
 		if ps.PRS.ProposalPOL == nil {
-			ps.PRS.ProposalPOL = bits.NewBitArray(numValidators)
+			ps.PRS.ProposalPOL = bits.NewBitArray(numVoters)
 		}
 	} else if ps.PRS.Height == height+1 {
 		if ps.PRS.LastCommit == nil {
-			ps.PRS.LastCommit = bits.NewBitArray(numValidators)
+			ps.PRS.LastCommit = bits.NewBitArray(numVoters)
 		}
 	}
 }

test/maverick/consensus/replay.go

@@ -314,7 +314,7 @@ func (h *Handshaker) ReplayBlocks(
 			ChainId:         h.genDoc.ChainID,
 			InitialHeight:   h.genDoc.InitialHeight,
 			ConsensusParams: csParams,
-			Validators:      nextVals,
+			Validators:      nextVals, // ValidatorOrVoter: validator
 			AppStateBytes:   h.genDoc.AppState,
 		}
 		res, err := proxyApp.Consensus().InitChainSync(req)
@@ -338,7 +338,9 @@ func (h *Handshaker) ReplayBlocks(
 					return nil, err
 				}
 				state.Validators = types.NewValidatorSet(vals)
-				state.NextValidators = types.NewValidatorSet(vals).CopyIncrementProposerPriority(1)
+				state.Voters = types.SelectVoter(state.Validators, h.genDoc.Hash(), state.VoterParams)
+				// Should sync it with MakeGenesisState()
+				state.NextValidators = types.NewValidatorSet(vals)
 			} else if len(h.genDoc.Validators) == 0 {
 				// If validator set is not set in genesis and still empty after InitChain, exit.
 				return nil, fmt.Errorf("validator set is nil in genesis and still empty after InitChain")

test/maverick/consensus/replay_file.go

@@ -258,8 +258,8 @@ func (pb *playback) replayConsoleLoop() int {
 				switch tokens[1] {
 				case "short":
 					fmt.Printf("%v/%v/%v\n", rs.Height, rs.Round, rs.Step)
-				case "validators":
-					fmt.Println(rs.Validators)
+				case "voters":
+					fmt.Println(rs.Voters)
 				case "proposal":
 					fmt.Println(rs.Proposal)
 				case "proposal_block":

test/maverick/consensus/state.go

@@ -534,6 +534,7 @@ func (cs *State) GetRoundStateSimpleJSON() ([]byte, error) {
 }
 
 // GetValidators returns a copy of the current validators.
+// ValidatorOrVoter: validator
 func (cs *State) GetValidators() (int64, []*types.Validator) {
 	cs.mtx.RLock()
 	defer cs.mtx.RUnlock()
@@ -899,7 +900,8 @@ func (cs *State) updateToState(state sm.State) {
 		cs.StartTime = cs.config.Commit(cs.CommitTime)
 	}
 
-	cs.Validators = validators
+	cs.Validators = state.Validators.Copy()
+	cs.Voters = state.Voters.Copy()
 	cs.Proposal = nil
 	cs.ProposalBlock = nil
 	cs.ProposalBlockParts = nil
@@ -913,6 +915,7 @@ func (cs *State) updateToState(state sm.State) {
 	cs.CommitRound = -1
 	cs.LastVoters = state.LastVoters
 	cs.TriggeredTimeoutPrecommit = false
+	cs.Proposer = validators.SelectProposer(state.LastProofHash, cs.Height, cs.Round)
 
 	cs.state = state
 
@@ -1120,18 +1123,13 @@ func (cs *State) enterNewRound(height int64, round int32) {
 
 	logger.Info(fmt.Sprintf("enterNewRound(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 
-	// Increment validators if necessary
-	validators := cs.Validators
-	if cs.Round < round {
-		validators = validators.Copy()
-		validators.IncrementProposerPriority(tmmath.SafeSubInt32(round, cs.Round))
-	}
+	// Select the current height and round Proposer
+	cs.Proposer = cs.Validators.SelectProposer(cs.state.LastProofHash, height, round)
 
 	// Setup new round
 	// we don't fire newStep for this step,
 	// but we fire an event, so update the round step first
 	cs.updateRoundStep(round, cstypes.RoundStepNewRound)
-	cs.Validators = validators
 	if round == 0 {
 		// We've already reset these upon new height,
 		// and meanwhile we might have received a proposal
@@ -1192,7 +1190,7 @@ func (cs *State) defaultDecideProposal(height int64, round int32) {
 		block, blockParts = cs.ValidBlock, cs.ValidBlockParts
 	} else {
 		// Create a new proposal block from state/txs from the mempool.
-		block, blockParts = cs.createProposalBlock()
+		block, blockParts = cs.createProposalBlock(round)
 		if block == nil {
 			return
 		}
@@ -1247,7 +1245,7 @@ func (cs *State) isProposalComplete() bool {
 //
 // NOTE: keep it side-effect free for clarity.
 // CONTRACT: cs.privValidator is not nil.
-func (cs *State) createProposalBlock() (block *types.Block, blockParts *types.PartSet) {
+func (cs *State) createProposalBlock(round int32) (block *types.Block, blockParts *types.PartSet) {
 	if cs.privValidator == nil {
 		panic("entered createProposalBlock with privValidator being nil")
 	}
@@ -1274,13 +1272,13 @@ func (cs *State) createProposalBlock() (block *types.Block, blockParts *types.Pa
 	}
 	proposerAddr := cs.privValidatorPubKey.Address()
 
-	message := cs.GetState().MakeHashMessage(cs.Round)
+	message := cs.state.MakeHashMessage(round)
 	proof, err := cs.privValidator.GenerateVRFProof(message)
 	if err != nil {
 		cs.Logger.Error(fmt.Sprintf("enterPropose: %v", err))
 		return
 	}
-	return cs.blockExec.CreateProposalBlock(cs.Height, cs.state, commit, proposerAddr, cs.Round, proof)
+	return cs.blockExec.CreateProposalBlock(cs.Height, cs.state, commit, proposerAddr, round, proof)
 }
 
 // Enter: any +2/3 prevotes at next round.
@@ -1858,8 +1856,8 @@ func (cs *State) signAddVote(msgType tmproto.SignedMsgType, hash []byte, header
 		return nil
 	}
 
-	// If the node not in the validator set, do nothing.
-	if !cs.Validators.HasAddress(cs.privValidatorPubKey.Address()) {
+	// If the node not in the voter set, do nothing.
+	if !cs.Voters.HasAddress(cs.privValidatorPubKey.Address()) {
 		return nil
 	}
 

test/maverick/main.go

@@ -181,7 +181,7 @@ func initFilesWithConfig(config *cfg.Config) error {
 		logger.Info("Found private validator", "keyFile", privValKeyFile,
 			"stateFile", privValStateFile)
 	} else {
-		pv = nd.GenFilePV(privValKeyFile, privValStateFile)
+		pv, _ = nd.GenFilePV(privValKeyFile, privValStateFile, config.PrivKeyType)
 		pv.Save()
 		logger.Info("Generated private validator", "keyFile", privValKeyFile,
 			"stateFile", privValStateFile)

test/maverick/node/node.go

@@ -121,11 +121,19 @@ type Provider func(*cfg.Config, log.Logger) (*Node, error)
 func DefaultNewNode(config *cfg.Config, logger log.Logger, misbehaviors map[int64]cs.Misbehavior) (*Node, error) {
 	nodeKey, err := p2p.LoadOrGenNodeKey(config.NodeKeyFile())
 	if err != nil {
-		return nil, fmt.Errorf("failed to load or gen node key %s, err: %w", config.NodeKeyFile(), err)
+		return nil, fmt.Errorf("failed to load or gen node key %s: %w", config.NodeKeyFile(), err)
+	}
+
+	privKey, err := LoadOrGenFilePV(
+		config.PrivValidatorKeyFile(),
+		config.PrivValidatorStateFile(),
+		config.PrivValidatorKeyType())
+	if err != nil {
+		return nil, fmt.Errorf("failed to create a private key: %s", err)
 	}
 
 	return NewNode(config,
-		LoadOrGenFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()),
+		privKey,
 		nodeKey,
 		proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()),
 		DefaultGenesisDocProviderFunc(config),
@@ -466,7 +474,8 @@ func createConsensusReactor(config *cfg.Config,
 	if privValidator != nil {
 		consensusState.SetPrivValidator(privValidator)
 	}
-	consensusReactor := cs.NewReactor(consensusState, waitSync, cs.ReactorMetrics(csMetrics))
+	consensusReactor := cs.NewReactor(consensusState, waitSync, config.P2P.RecvAsync, config.P2P.ConsensusRecvBufSize,
+		cs.ReactorMetrics(csMetrics))
 	consensusReactor.SetLogger(consensusLogger)
 	// services which will be publishing and/or subscribing for messages (events)
 	// consensusReactor will set it on consensusState and blockExecutor
@@ -618,6 +627,7 @@ func createPEXReactorAndAddToSwitch(addrBook pex.AddrBook, config *cfg.Config,
 			// https://github.com/tendermint/tendermint/issues/3523
 			SeedDisconnectWaitPeriod:     28 * time.Hour,
 			PersistentPeersMaxDialPeriod: config.P2P.PersistentPeersMaxDialPeriod,
+			RecvBufSize:                  config.P2P.PexRecvBufSize,
 		})
 	pexReactor.SetLogger(logger.With("module", "pex"))
 	sw.AddReactor("PEX", pexReactor)
@@ -648,11 +658,18 @@ func startStateSync(ssR *statesync.Reactor, bcR fastSyncReactor, conR *cs.Reacto
 	}
 
 	go func() {
-		state, _, commit, err := ssR.Sync(stateProvider, config.DiscoveryTime)
+		state, previousState, commit, err := ssR.Sync(stateProvider, config.DiscoveryTime)
 		if err != nil {
 			ssR.Logger.Error("State sync failed", "err", err)
 			return
 		}
+		if previousState.LastBlockHeight > 0 {
+			err = stateStore.Bootstrap(previousState)
+			if err != nil {
+				ssR.Logger.Error("Failed to bootstrap node with previous state", "err", err)
+				return
+			}
+		}
 		err = stateStore.Bootstrap(state)
 		if err != nil {
 			ssR.Logger.Error("Failed to bootstrap node with new state", "err", err)
@@ -813,7 +830,7 @@ func NewNode(config *cfg.Config,
 	// we should clean this whole thing up. See:
 	// https://github.com/tendermint/tendermint/issues/4644
 	stateSyncReactor := statesync.NewReactor(proxyApp.Snapshot(), proxyApp.Query(),
-		config.P2P.RecvAsync, config.P2P.BlockchainRecvBufSize)
+		config.P2P.RecvAsync, config.P2P.StatesyncRecvBufSize)
 	stateSyncReactor.SetLogger(logger.With("module", "statesync"))
 
 	nodeInfo, err := makeNodeInfo(config, nodeKey, txIndexer, genDoc, state)