diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 674099c7c4..18c4a43dbf 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -28,6 +28,7 @@ jobs: github.com/ChainSafe/gossamer/dot/rpc/modules, github.com/ChainSafe/gossamer/lib/babe, github.com/ChainSafe/gossamer/dot/sync, + github.com/ChainSafe/gossamer/lib/grandpa, ] runs-on: ubuntu-latest steps: diff --git a/chain/westend-local/westend-local-spec.json b/chain/westend-local/westend-local-spec.json index 6f7474b4ff..59eadb959b 100644 --- a/chain/westend-local/westend-local-spec.json +++ b/chain/westend-local/westend-local-spec.json @@ -86,7 +86,9 @@ "minimumValidatorCount": 1, "invulnerables": [ "5GNJqTPyNqANBkUVMN1LPPrxXnFouWXoe2wNSmmEoLctxiZY", - "5HpG9w8EBLe5XCrbczpwq5TSXvedjrBGCwqxK1iQ7qUsSWFc" + "5HpG9w8EBLe5XCrbczpwq5TSXvedjrBGCwqxK1iQ7qUsSWFc", + "5Ck5SLSHYac6WFt5UZRSsdJjwmpSZq85fd5TRNAdZQVzEAPT", + "5HKPmK9GYtE1PSLsS1qiYU9xQ9Si1NcEhdeCq9sw5bqu4ns8" ], "forceEra": "NotForcing", "slashRewardFraction": 100000000, @@ -103,6 +105,18 @@ "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty", 100000000000000, "Validator" + ], + [ + "5Ck5SLSHYac6WFt5UZRSsdJjwmpSZq85fd5TRNAdZQVzEAPT", + "5FLSigC9HGRKVhB9FiEo4Y3koPsNmBmLJbpXg2mp1hXcS59Y", + 100000000000000, + "Validator" + ], + [ + "5HKPmK9GYtE1PSLsS1qiYU9xQ9Si1NcEhdeCq9sw5bqu4ns8", + "5DAAnrj7VHTznn2AWBemMuyBwZWs6FNFjdyVXUeYum3PTXFy", + 100000000000000, + "Validator" ] ], "minNominatorBond": 0, @@ -135,6 +149,30 @@ "para_assignment": "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty", "authority_discovery": "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty" } + ], + [ + "5Ck5SLSHYac6WFt5UZRSsdJjwmpSZq85fd5TRNAdZQVzEAPT", + "5Ck5SLSHYac6WFt5UZRSsdJjwmpSZq85fd5TRNAdZQVzEAPT", + { + "grandpa": "5DbKjhNLpqX3zqZdNBc9BGb4fHU1cRBaDhJUskrvkwfraDi6", + "babe": "5FLSigC9HGRKVhB9FiEo4Y3koPsNmBmLJbpXg2mp1hXcS59Y", + "im_online": "5FLSigC9HGRKVhB9FiEo4Y3koPsNmBmLJbpXg2mp1hXcS59Y", + "para_validator": "5FLSigC9HGRKVhB9FiEo4Y3koPsNmBmLJbpXg2mp1hXcS59Y", + "para_assignment": "5FLSigC9HGRKVhB9FiEo4Y3koPsNmBmLJbpXg2mp1hXcS59Y", + "authority_discovery": "5FLSigC9HGRKVhB9FiEo4Y3koPsNmBmLJbpXg2mp1hXcS59Y" + } + ], + [ + "5HKPmK9GYtE1PSLsS1qiYU9xQ9Si1NcEhdeCq9sw5bqu4ns8", + "5HKPmK9GYtE1PSLsS1qiYU9xQ9Si1NcEhdeCq9sw5bqu4ns8", + { + "grandpa": "5ECTwv6cZ5nJQPk6tWfaTrEk8YH2L7X1VT4EL5Tx2ikfFwb7", + "babe": "5DAAnrj7VHTznn2AWBemMuyBwZWs6FNFjdyVXUeYum3PTXFy", + "im_online": "5DAAnrj7VHTznn2AWBemMuyBwZWs6FNFjdyVXUeYum3PTXFy", + "para_validator": "5DAAnrj7VHTznn2AWBemMuyBwZWs6FNFjdyVXUeYum3PTXFy", + "para_assignment": "5DAAnrj7VHTznn2AWBemMuyBwZWs6FNFjdyVXUeYum3PTXFy", + "authority_discovery": "5DAAnrj7VHTznn2AWBemMuyBwZWs6FNFjdyVXUeYum3PTXFy" + } ] ] }, diff --git a/lib/grandpa/errors.go b/lib/grandpa/errors.go index 2bf1d07d3a..e6c1a2bb47 100644 --- a/lib/grandpa/errors.go +++ b/lib/grandpa/errors.go @@ -5,16 +5,10 @@ package grandpa import ( "errors" - "fmt" "github.com/ChainSafe/gossamer/lib/blocktree" ) -// errRoundMismatch is returned when trying to validate a vote message that isn't for the current round -func errRoundMismatch(got, want uint64) error { - return fmt.Errorf("rounds do not match: got %d, want %d", got, want) -} - var ( // ErrBlockDoesNotExist is returned when trying to validate a vote for a block that doesn't exist ErrBlockDoesNotExist = errors.New("block does not exist") @@ -60,6 +54,8 @@ var ( ErrBlockHashMismatch = errors.New("block hash does not correspond to given block number") + ErrBlockNumbersMismatch = errors.New("block numbers mismatch") + // ErrMinVotesNotMet is returned when the number of votes is less than the required minimum in a Justification ErrMinVotesNotMet = errors.New("minimum number of votes not met in a Justification") @@ -76,9 +72,6 @@ var ( // ErrCatchUpResponseNotCompletable is returned when the round represented by the catch up response is not completable ErrCatchUpResponseNotCompletable = errors.New("catch up response is not completable") - // ErrServicePaused is returned if the service is paused and waiting for catch up messages - ErrServicePaused = errors.New("service is paused") - // ErrPrecommitSignatureMismatch is returned when the number of precommits // and signatures in a CommitMessage do not match ErrPrecommitSignatureMismatch = errors.New("number of precommits does not match number of signatures") @@ -93,4 +86,6 @@ var ( errVoteToSignatureMismatch = errors.New("votes and authority count mismatch") errVoteBlockMismatch = errors.New("block in vote is not descendant of previously finalised block") errVoteFromSelf = errors.New("got vote from ourselves") + errRoundOutOfBounds = errors.New("round out of bounds") + errRoundsMismatch = errors.New("rounds mismatch") ) diff --git a/lib/grandpa/finalisation.go b/lib/grandpa/finalisation.go new file mode 100644 index 0000000000..cae82c6f9f --- /dev/null +++ b/lib/grandpa/finalisation.go @@ -0,0 +1,478 @@ +// Copyright 2022 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +package grandpa + +import ( + "errors" + "fmt" + "sync" + "time" + + "github.com/ChainSafe/gossamer/dot/telemetry" +) + +var ( + errServicesStopFailed = errors.New("services stop failed") + errvotingRoundHandlerFailed = errors.New("voting round ephemeral failed") + errfinalisationEngineFailed = errors.New("finalisation engine ephemeral failed") +) + +type ephemeralService interface { + Run() error + Stop() error +} + +type finalisationHandler struct { + servicesLock sync.Mutex + finalisationEngine ephemeralService + votingRound ephemeralService + + // newServices is a constructor function which takes care to instantiate + // and return the services needed to finalize a round, those services + // are ephemeral services with a lifetime of a round + newServices func() (engine, voting ephemeralService) + initiateRound func() error + + stopCh chan struct{} + handlerDone chan struct{} + + firstRun bool +} + +func newFinalisationHandler(service *Service) *finalisationHandler { + return &finalisationHandler{ + newServices: func() (engine, voting ephemeralService) { + finalisationEngine := newfinalisationEngine(service) + votingRound := newvotingRoundHandler(service, finalisationEngine.actionCh) + return finalisationEngine, votingRound + }, + initiateRound: service.initiateRound, + stopCh: make(chan struct{}), + handlerDone: make(chan struct{}), + firstRun: true, + } +} + +func (fh *finalisationHandler) Start() (<-chan error, error) { + errorCh := make(chan error) + ready := make(chan struct{}) + + go fh.run(errorCh, ready) + <-ready + + return errorCh, nil +} + +func (fh *finalisationHandler) run(errorCh chan<- error, ready chan<- struct{}) { + defer func() { + close(errorCh) + close(fh.handlerDone) + }() + + for { + select { + case <-fh.stopCh: + return + default: + } + + err := fh.initiateRound() + if err != nil { + errorCh <- fmt.Errorf("initiating round: %w", err) + return + } + + err = fh.runEphemeralServices(ready) + if err != nil { + errorCh <- fmt.Errorf("running ephemeral services: %w", err) + return + } + } +} + +func (fh *finalisationHandler) stop() (err error) { + fh.servicesLock.Lock() + defer fh.servicesLock.Unlock() + + finalisationEngineErrCh := make(chan error) + go func() { + finalisationEngineErrCh <- fh.finalisationEngine.Stop() + }() + + votingRoundErrCh := make(chan error) + go func() { + votingRoundErrCh <- fh.votingRound.Stop() + }() + + finalisationEngErr := <-finalisationEngineErrCh + votingRoundErr := <-votingRoundErrCh + + switch { + case finalisationEngErr != nil && votingRoundErr != nil: + return fmt.Errorf("%w: %s; %s", errServicesStopFailed, finalisationEngErr, votingRoundErr) + case finalisationEngErr != nil: + return fmt.Errorf("%w: %s", errServicesStopFailed, finalisationEngErr) + case votingRoundErr != nil: + return fmt.Errorf("%w: %s", errServicesStopFailed, votingRoundErr) + } + + return nil +} + +func (fh *finalisationHandler) Stop() (err error) { + close(fh.stopCh) + <-fh.handlerDone + + return fh.stop() +} + +// runEphemeralServices starts the two ephemeral services that handle the +// votes for the current round, and returns with nil when the two +// service runs succeed. +// If any service run fails, the other service run is stopped and +// an error is returned. The function returns nil is the finalisation +// handler is stopped. +func (fh *finalisationHandler) runEphemeralServices(ready chan<- struct{}) error { + fh.servicesLock.Lock() + fh.finalisationEngine, fh.votingRound = fh.newServices() + fh.servicesLock.Unlock() + + if fh.firstRun { + fh.firstRun = false + ready <- struct{}{} + } + + finalisationEngineErr := make(chan error) + go func() { + finalisationEngineErr <- fh.finalisationEngine.Run() + }() + + votingRoundErr := make(chan error) + go func() { + votingRoundErr <- fh.votingRound.Run() + }() + + for { + select { + case <-fh.stopCh: + return nil + + case err := <-votingRoundErr: + if err == nil { + votingRoundErr = nil + // go out from the select case + break + } + + stopErr := fh.finalisationEngine.Stop() + if stopErr != nil { + logger.Warnf("stopping finalisation engine: %s", stopErr) + } + return fmt.Errorf("%w: %s", errvotingRoundHandlerFailed, err) + + case err := <-finalisationEngineErr: + if err == nil { + finalisationEngineErr = nil + // go out from the select case + break + } + + stopErr := fh.votingRound.Stop() + if stopErr != nil { + logger.Warnf("stopping voting round: %s", stopErr) + } + + return fmt.Errorf("%w: %s", errfinalisationEngineFailed, err) + } + + finish := votingRoundErr == nil && finalisationEngineErr == nil + if finish { + return nil + } + } +} + +// votingRoundHandler interacts with finalisationEngine service +// executing the actions based on what it receives throuhg channel +type votingRoundHandler struct { + grandpaService *Service + finalisationEngineCh <-chan engineAction + stopCh chan struct{} + engineDone chan struct{} +} + +func newvotingRoundHandler(service *Service, finalisationEngineCh <-chan engineAction) *votingRoundHandler { + return &votingRoundHandler{ + grandpaService: service, + stopCh: make(chan struct{}), + engineDone: make(chan struct{}), + finalisationEngineCh: finalisationEngineCh, + } +} + +func (h *votingRoundHandler) Stop() (err error) { + close(h.stopCh) + <-h.engineDone + + return nil +} + +func (h *votingRoundHandler) Run() (err error) { + defer close(h.engineDone) + + start := time.Now() + + logger.Debugf("starting round %d with set id %d", + h.grandpaService.state.round, h.grandpaService.state.setID) + + for { + select { + case <-h.stopCh: + return nil + + case action := <-h.finalisationEngineCh: + switch action { + case determinePrevote: + isPrimary, err := h.grandpaService.handleIsPrimary() + if err != nil { + return fmt.Errorf("handling primary: %w", err) + } + + preVote, err := h.grandpaService.determinePreVote() + if err != nil { + return fmt.Errorf("determining pre-vote: %w", err) + } + + signedpreVote, prevoteMessage, err := + h.grandpaService.createSignedVoteAndVoteMessage(preVote, prevote) + if err != nil { + return fmt.Errorf("creating signed vote: %w", err) + } + + if !isPrimary { + h.grandpaService.prevotes.Store(h.grandpaService.publicKeyBytes(), signedpreVote) + } + + logger.Debugf("sending pre-vote message: {%v}", prevoteMessage) + err = h.grandpaService.sendPrevoteMessage(prevoteMessage) + if err != nil { + return fmt.Errorf("sending pre-vote message: %w", err) + } + + case determinePrecommit: + preCommit, err := h.grandpaService.determinePreCommit() + if err != nil { + return fmt.Errorf("determining pre-commit: %w", err) + } + + signedPreCommit, precommitMessage, err := + h.grandpaService.createSignedVoteAndVoteMessage(preCommit, precommit) + if err != nil { + return fmt.Errorf("creating signed vote: %w", err) + } + + h.grandpaService.precommits.Store(h.grandpaService.publicKeyBytes(), signedPreCommit) + logger.Debugf("sending pre-commit message: {%v}", precommitMessage) + err = h.grandpaService.sendPrecommitMessage(precommitMessage) + if err != nil { + logger.Errorf("sending pre-commit message: %s", err) + } + + case finalize: + commitMessage, err := h.grandpaService.newCommitMessage( + h.grandpaService.head, h.grandpaService.state.round, h.grandpaService.state.setID) + if err != nil { + return fmt.Errorf("creating commit message: %w", err) + } + + commitConsensusMessage, err := commitMessage.ToConsensusMessage() + if err != nil { + return fmt.Errorf("transforming commit into consensus message: %w", err) + } + + logger.Debugf("sending commit message: %v", commitMessage) + h.grandpaService.network.GossipMessage(commitConsensusMessage) + h.grandpaService.telemetry.SendMessage(telemetry.NewAfgFinalizedBlocksUpTo( + h.grandpaService.head.Hash(), + fmt.Sprint(h.grandpaService.head.Number), + )) + + logger.Debugf("round completed in %s", time.Since(start)) + return nil + + case alreadyFinalized: + logger.Debugf("round completed in %s", time.Since(start)) + return nil + } + } + } +} + +// actions that should take place accordingly to votes the +// finalisation engine knows about +type engineAction byte + +const ( + determinePrevote engineAction = iota + determinePrecommit + alreadyFinalized + finalize +) + +type finalisationEngine struct { + grandpaService *Service + stopCh chan struct{} + engineDone chan struct{} + actionCh chan engineAction +} + +func newfinalisationEngine(service *Service) *finalisationEngine { + return &finalisationEngine{ + grandpaService: service, + actionCh: make(chan engineAction), + stopCh: make(chan struct{}), + engineDone: make(chan struct{}), + } +} + +func (f *finalisationEngine) Stop() (err error) { + close(f.stopCh) + <-f.engineDone + + close(f.actionCh) + return nil +} + +func (f *finalisationEngine) Run() (err error) { + defer close(f.engineDone) + + err = f.defineRoundVotes() + if err != nil { + return fmt.Errorf("defining round votes: %w", err) + } + + err = f.finalizeRound() + if err != nil { + return fmt.Errorf("finalising round: %w", err) + } + + return nil +} + +func (f *finalisationEngine) defineRoundVotes() error { + gossipInterval := f.grandpaService.interval + determinePrevoteTimer := time.NewTimer(2 * gossipInterval) + determinePrecommitTimer := time.NewTimer(4 * gossipInterval) + + precommited := false + + for !precommited { + select { + case <-f.stopCh: + if !determinePrevoteTimer.Stop() { + <-determinePrevoteTimer.C + } + + if !determinePrecommitTimer.Stop() { + <-determinePrecommitTimer.C + } + + return nil + + case <-determinePrevoteTimer.C: + alreadyCompletable, err := f.grandpaService.checkRoundCompletable() + if err != nil { + return fmt.Errorf("checking round is completable: %w", err) + } + + if alreadyCompletable { + f.actionCh <- alreadyFinalized + + if !determinePrecommitTimer.Stop() { + <-determinePrecommitTimer.C + } + + return nil + } + + f.actionCh <- determinePrevote + + case <-determinePrecommitTimer.C: + alreadyCompletable, err := f.grandpaService.checkRoundCompletable() + if err != nil { + return fmt.Errorf("checking round is completable: %w", err) + } + + if alreadyCompletable { + f.actionCh <- alreadyFinalized + return nil + } + + prevoteGrandpaGhost, err := f.grandpaService.getPreVotedBlock() + if err != nil { + return fmt.Errorf("getting grandpa ghost: %w", err) + } + + total, err := f.grandpaService.getTotalVotesForBlock(prevoteGrandpaGhost.Hash, prevote) + if err != nil { + return fmt.Errorf("getting grandpa ghost: %w", err) + } + + if total <= f.grandpaService.state.threshold() { + determinePrecommitTimer.Reset(4 * gossipInterval) + continue + } + + latestFinalizedHash := f.grandpaService.head.Hash() + isDescendant, err := f.grandpaService.blockState.IsDescendantOf( + latestFinalizedHash, prevoteGrandpaGhost.Hash) + if err != nil { + return fmt.Errorf("checking grandpa ghost ancestry: %w", err) + } + + if !isDescendant { + panic("block with supermajority does not belong to the latest finalized block chain") + } + + f.actionCh <- determinePrecommit + precommited = true + } + } + + return nil +} + +func (f *finalisationEngine) finalizeRound() error { + gossipInterval := f.grandpaService.interval + attemptfinalisationTicker := time.NewTicker(gossipInterval / 2) + defer attemptfinalisationTicker.Stop() + + for { + completable, err := f.grandpaService.checkRoundCompletable() + if err != nil { + return fmt.Errorf("checking round is completable: %w", err) + } + + if completable { + f.actionCh <- alreadyFinalized + return nil + } + + finalizable, err := f.grandpaService.attemptToFinalize() + if err != nil { + return fmt.Errorf("attempting to finalize: %w", err) + } + + if finalizable { + f.actionCh <- finalize + return nil + } + + select { + case <-f.stopCh: + return nil + case <-attemptfinalisationTicker.C: + } + } +} diff --git a/lib/grandpa/finalisation_integration_test.go b/lib/grandpa/finalisation_integration_test.go new file mode 100644 index 0000000000..8e6908630f --- /dev/null +++ b/lib/grandpa/finalisation_integration_test.go @@ -0,0 +1,308 @@ +// Copyright 2022 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +//go:build integration + +package grandpa + +import ( + "errors" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" +) + +func Test_finalisationHandler_runEphemeralServices(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + callHandlerStop bool + createfinalisationHandler func(*gomock.Controller) *finalisationHandler + wantErr error + errString string + }{ + "voting_round_finalisation_engine_finishes_successfully": { + createfinalisationHandler: func(ctrl *gomock.Controller) *finalisationHandler { + builder := func() (engine ephemeralService, voting ephemeralService) { + mockVoting := NewMockephemeralService(ctrl) + mockVoting.EXPECT().Run().DoAndReturn(func() error { + return nil + }) + + mockEngine := NewMockephemeralService(ctrl) + mockEngine.EXPECT().Run().DoAndReturn(func() error { + return nil + }) + + return mockEngine, mockVoting + } + + return &finalisationHandler{ + newServices: builder, + stopCh: make(chan struct{}), + handlerDone: make(chan struct{}), + firstRun: false, + } + }, + }, + + "voting_round_fails_should_stop_engine_service": { + errString: "voting round ephemeral failed: mocked voting round failed", + wantErr: errvotingRoundHandlerFailed, + createfinalisationHandler: func(ctrl *gomock.Controller) *finalisationHandler { + builder := func() (engine ephemeralService, voting ephemeralService) { + mockVoting := NewMockephemeralService(ctrl) + mockVoting.EXPECT().Run().DoAndReturn(func() error { + time.Sleep(time.Second) + return errors.New("mocked voting round failed") + }) + + // once the voting round fails the finalisation handler + // should be awere of the error and call the stop method from + // the engine which will release the start method from engine service + engineStopCh := make(chan struct{}) + mockEngine := NewMockephemeralService(ctrl) + mockEngine.EXPECT().Run().DoAndReturn(func() error { + <-engineStopCh + return nil + }) + mockEngine.EXPECT().Stop().DoAndReturn(func() error { + close(engineStopCh) + return nil + }) + + return mockEngine, mockVoting + } + + return &finalisationHandler{ + newServices: builder, + stopCh: make(chan struct{}), + handlerDone: make(chan struct{}), + firstRun: false, + } + }, + }, + + "engine_fails_should_stop_voting_round_service": { + errString: "finalisation engine ephemeral failed: mocked finalisation engine failed", + wantErr: errfinalisationEngineFailed, + createfinalisationHandler: func(ctrl *gomock.Controller) *finalisationHandler { + builder := func() (engine ephemeralService, voting ephemeralService) { + mockEngine := NewMockephemeralService(ctrl) + mockEngine.EXPECT().Run().DoAndReturn(func() error { + time.Sleep(time.Second) + return errors.New("mocked finalisation engine failed") + }) + + // once the finalisation engine fails the finalisation handler + // should be awere of the error and call the stop method from the + // voting round which will release the start method from voting round service + votingStopChannel := make(chan struct{}) + mockVoting := NewMockephemeralService(ctrl) + mockVoting.EXPECT().Run().DoAndReturn(func() error { + <-votingStopChannel + return nil + }) + mockVoting.EXPECT().Stop().DoAndReturn(func() error { + close(votingStopChannel) + return nil + }) + + return mockEngine, mockVoting + } + + return &finalisationHandler{ + newServices: builder, + stopCh: make(chan struct{}), + handlerDone: make(chan struct{}), + firstRun: false, + } + }, + }, + } + + for tname, tt := range tests { + tt := tt + + t.Run(tname, func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + finalisationHandler := tt.createfinalisationHandler(ctrl) + + // passing the ready channel as nil since the first run is false + // and we ensure the method fh.newServices() is being called + err := finalisationHandler.runEphemeralServices(nil) + require.ErrorIs(t, err, tt.wantErr) + if tt.wantErr != nil { + require.EqualError(t, err, tt.errString) + } + }) + } +} + +func Test_finalisationHandler_Stop_ShouldHalt_Services(t *testing.T) { + t.Parallel() + + testcases := map[string]struct { + wantErr error + errString string + newHandler func(*gomock.Controller) *finalisationHandler + }{ + "halt_ephemeral_services_after_calling_stop": { + // when we start the finalisation handler we instantiate + // and call the Run method from each ephemeral services + // (votingHandler, finalisationEngine) since they are mocked + // they will wait until the Stop method being called to release + // the blocking channel and return from the function + newHandler: func(ctrl *gomock.Controller) *finalisationHandler { + builder := func() (engine ephemeralService, voting ephemeralService) { + engineStopCh := make(chan struct{}) + mockEngine := NewMockephemeralService(ctrl) + + mockEngine.EXPECT().Run().DoAndReturn(func() error { + <-engineStopCh + return nil + }) + mockEngine.EXPECT().Stop().DoAndReturn(func() error { + close(engineStopCh) + return nil + }) + + votingStopCh := make(chan struct{}) + mockVoting := NewMockephemeralService(ctrl) + mockVoting.EXPECT().Run().DoAndReturn(func() error { + <-votingStopCh + return nil + }) + mockVoting.EXPECT().Stop().DoAndReturn(func() error { + close(votingStopCh) + return nil + }) + return mockEngine, mockVoting + } + + return &finalisationHandler{ + newServices: builder, + // mocked initiate round function + initiateRound: func() error { return nil }, + stopCh: make(chan struct{}), + handlerDone: make(chan struct{}), + firstRun: true, + } + }, + }, + "halt_fails_to_stop_one_ephemeral_service": { + wantErr: errServicesStopFailed, + errString: "services stop failed: cannot stop finalisation engine test", + newHandler: func(ctrl *gomock.Controller) *finalisationHandler { + builder := func() (engine ephemeralService, voting ephemeralService) { + engineStopCh := make(chan struct{}) + mockEngine := NewMockephemeralService(ctrl) + + mockEngine.EXPECT().Run().DoAndReturn(func() error { + <-engineStopCh + return nil + }) + mockEngine.EXPECT().Stop().DoAndReturn(func() error { + close(engineStopCh) + return errors.New("cannot stop finalisation engine test") + }) + + votingStopCh := make(chan struct{}) + mockVoting := NewMockephemeralService(ctrl) + mockVoting.EXPECT().Run().DoAndReturn(func() error { + <-votingStopCh + return nil + }) + mockVoting.EXPECT().Stop().DoAndReturn(func() error { + close(votingStopCh) + return nil + }) + + return mockEngine, mockVoting + } + + return &finalisationHandler{ + newServices: builder, + // mocked initiate round function + initiateRound: func() error { return nil }, + stopCh: make(chan struct{}), + handlerDone: make(chan struct{}), + firstRun: true, + } + }, + }, + "halt_fails_to_stop_both_ephemeral_service": { + wantErr: errServicesStopFailed, + errString: "services stop failed: cannot stop finalisation engine test; " + + "cannot stop voting handler test", + newHandler: func(ctrl *gomock.Controller) *finalisationHandler { + builder := func() (engine ephemeralService, voting ephemeralService) { + engineStopCh := make(chan struct{}) + mockEngine := NewMockephemeralService(ctrl) + + mockEngine.EXPECT().Run().DoAndReturn(func() error { + <-engineStopCh + return nil + }) + mockEngine.EXPECT().Stop().DoAndReturn(func() error { + close(engineStopCh) + return errors.New("cannot stop finalisation engine test") + }) + + votingStopCh := make(chan struct{}) + mockVoting := NewMockephemeralService(ctrl) + mockVoting.EXPECT().Run().DoAndReturn(func() error { + <-votingStopCh + return nil + }) + mockVoting.EXPECT().Stop().DoAndReturn(func() error { + close(votingStopCh) + return errors.New("cannot stop voting handler test") + }) + + return mockEngine, mockVoting + } + + return &finalisationHandler{ + newServices: builder, + // mocked initiate round function + initiateRound: func() error { return nil }, + stopCh: make(chan struct{}), + handlerDone: make(chan struct{}), + firstRun: true, + } + }, + }, + } + + for tname, tt := range testcases { + tt := tt + t.Run(tname, func(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + handler := tt.newHandler(ctrl) + + errorCh, err := handler.Start() + require.NoError(t, err) + + // wait enough time to start subservices + // and then call stop + time.Sleep(2 * time.Second) + err = handler.Stop() + require.ErrorIs(t, err, tt.wantErr) + if tt.errString != "" { + require.EqualError(t, err, tt.errString) + } + + // since we are stopping the finalisation handler we expect + // the errorCh to be closed without any error + err, ok := <-errorCh + require.Falsef(t, ok, + "expected channel to be closed, got an unexpected error: %s", err) + }) + } +} diff --git a/lib/grandpa/grandpa.go b/lib/grandpa/grandpa.go index 7ae823bb2e..30c401822a 100644 --- a/lib/grandpa/grandpa.go +++ b/lib/grandpa/grandpa.go @@ -13,6 +13,7 @@ import ( "sync/atomic" "time" + "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/dot/state" "github.com/ChainSafe/gossamer/dot/telemetry" "github.com/ChainSafe/gossamer/dot/types" @@ -21,6 +22,7 @@ import ( "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/crypto/ed25519" "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/libp2p/go-libp2p-core/peer" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) @@ -74,7 +76,6 @@ type Service struct { bestFinalCandidate map[uint64]*Vote // map of round number -> best final candidate // channels for communication with other services - in chan *networkVoteMessage // only used to receive *VoteMessage finalisedCh chan *types.FinalisationInfo telemetry telemetry.Client @@ -144,7 +145,6 @@ func NewService(cfg *Config) (*Service, error) { preVotedBlock: make(map[uint64]*Vote), bestFinalCandidate: make(map[uint64]*Vote), head: head, - in: make(chan *networkVoteMessage, 1024), resumed: make(chan struct{}), network: cfg.Network, finalisedCh: finalisedCh, @@ -173,18 +173,12 @@ func (s *Service) Start() error { s.tracker.start() go func() { - for { - // TODO: sometimes grandpa fails to initiate due to a "Key not found" - // error, this shouldn't happen. - if err := s.initiate(); err != nil { - logger.Criticalf("failed to initiate: %s", err) - } - time.Sleep(s.interval) + err := s.initiate() + if err != nil { + panic(fmt.Sprintf("running grandpa service: %s", err)) } }() - go s.sendNeighbourMessage(neighbourMessageInterval) - return nil } @@ -303,7 +297,7 @@ func (s *Service) initiateRound() error { s.state.round = round } - s.head, err = s.blockState.GetFinalisedHeader(s.state.round, s.state.setID) + s.head, err = s.blockState.GetFinalisedHeader(round, setID) if err != nil { logger.Criticalf("failed to get finalised header for round %d: %s", round, err) return err @@ -321,69 +315,32 @@ func (s *Service) initiateRound() error { // make sure no votes can be validated while we are incrementing rounds s.roundLock.Lock() + defer s.roundLock.Unlock() + s.state.round++ logger.Debugf("incrementing grandpa round, next round will be %d", s.state.round) s.prevotes = new(sync.Map) s.precommits = new(sync.Map) s.pvEquivocations = make(map[ed25519.PublicKeyBytes][]*SignedVote) s.pcEquivocations = make(map[ed25519.PublicKeyBytes][]*SignedVote) - s.roundLock.Unlock() - - best, err := s.blockState.BestBlockHeader() - if err != nil { - return err - } - if best.Number > 0 { - return nil - } - - // don't begin grandpa until we are at block 1 - s.waitForFirstBlock() return nil } // initiate initates the grandpa service to begin voting in sequential rounds func (s *Service) initiate() error { - for { - err := s.initiateRound() - if err != nil { - logger.Warnf("failed to initiate round for round %d: %s", s.state.round, err) - return err - } - - err = s.playGrandpaRound() - if errors.Is(err, ErrServicePaused) { - logger.Info("service paused") - // wait for service to un-pause - <-s.resumed - err = s.initiate() - } - - if err != nil { - logger.Warnf("failed to play grandpa round: %s", err) - continue - } - - if s.ctx.Err() != nil { - return errors.New("context cancelled") - } + finalisationHandler := newFinalisationHandler(s) + errorCh, err := finalisationHandler.Start() + if err != nil { + return fmt.Errorf("starting finalisation handler: %w", err) } -} -func (s *Service) waitForFirstBlock() { - ch := s.blockState.GetImportedBlockNotifierChannel() - defer s.blockState.FreeImportedBlockNotifierChannel(ch) - - // loop until block 1 for { select { - case block := <-ch: - if block != nil && block.Header.Number > 0 { - return - } case <-s.ctx.Done(): - return + return finalisationHandler.Stop() + case err := <-errorCh: + return err } } } @@ -432,7 +389,7 @@ func (s *Service) handleIsPrimary() (bool, error) { // broadcast commit message from the previous round to the network // ignore errors, since it's not critical to broadcast func (s *Service) primaryBroadcastCommitMessage() { - cm, err := s.newCommitMessage(s.head, s.state.round-1) + cm, err := s.newCommitMessage(s.head, s.state.round-1, s.state.setID) if err != nil { return } @@ -446,191 +403,112 @@ func (s *Service) primaryBroadcastCommitMessage() { s.network.GossipMessage(msg) } -// playGrandpaRound executes a round of GRANDPA -// at the end of this round, a block will be finalised. -func (s *Service) playGrandpaRound() error { - logger.Debugf("starting round %d with set id %d", - s.state.round, s.state.setID) - start := time.Now() - - ctx, cancel := context.WithCancel(s.ctx) - defer cancel() - - isPrimary, err := s.handleIsPrimary() +func (s *Service) checkRoundCompletable() (bool, error) { + // check if the current round contains a finalized block + has, err := s.blockState.HasFinalisedBlock(s.state.round, s.state.setID) if err != nil { - return err + return false, fmt.Errorf("checking for finalised block in block state: %w", err) } - logger.Debug("receiving pre-vote messages...") - go s.receiveVoteMessages(ctx) - time.Sleep(s.interval) - - if s.paused.Load().(bool) { - return ErrServicePaused + if has { + logger.Debugf("block was finalised for round %d", s.state.round) + return true, nil } - // broadcast pre-vote - pv, err := s.determinePreVote() + // a block was finalised, seems like we missed some messages + highestRound, highestSetID, err := s.blockState.GetHighestRoundAndSetID() if err != nil { - return err + return false, fmt.Errorf("getting highest round and set id: %w", err) } - spv, vm, err := s.createSignedVoteAndVoteMessage(pv, prevote) - if err != nil { - return err + if highestRound > s.state.round { + logger.Debugf("block was finalised for round %d and set id %d", + highestRound, highestSetID) + return true, nil } - if !isPrimary { - s.prevotes.Store(s.publicKeyBytes(), spv) + // a block was finalised, seems like we missed some messages + if highestSetID > s.state.setID { + logger.Debugf("block was finalised for round %d and set id %d", + highestRound, highestSetID) + return true, nil } - logger.Debugf("sending pre-vote message %s...", pv) - roundComplete := make(chan struct{}) - // roundComplete is a signal channel which is closed when the round completes - // (will receive the default value of channel's type), so we don't need to - // explicitly send a value. - defer close(roundComplete) - - // continue to send prevote messages until round is done - go s.sendVoteMessage(prevote, vm, roundComplete) - - logger.Debug("receiving pre-commit messages...") - // through goroutine s.receiveVoteMessages(ctx) - time.Sleep(s.interval) + return false, nil +} - if s.paused.Load().(bool) { - return ErrServicePaused - } +func (s *Service) retrieveBestFinalCandidate() (bestFinalCandidate *types.GrandpaVote, + precommitCount uint64, err error) { - // broadcast pre-commit - pc, err := s.determinePreCommit() + bestFinalCandidate, err = s.getBestFinalCandidate() if err != nil { - return err + return nil, 0, fmt.Errorf("getting best final candidate: %w", err) } - spc, pcm, err := s.createSignedVoteAndVoteMessage(pc, precommit) - if err != nil { - return err + if bestFinalCandidate.Number < uint32(s.head.Number) { + return nil, 0, fmt.Errorf("%w: candidate number %d, latest finalized block number %d", + errBeforeFinalizedBlock, bestFinalCandidate.Number, s.head.Number) } - s.precommits.Store(s.publicKeyBytes(), spc) - logger.Debugf("sending pre-commit message %s...", pc) - - // continue to send precommit messages until round is done - go s.sendVoteMessage(precommit, pcm, roundComplete) - - if err = s.attemptToFinalize(); err != nil { - logger.Errorf("failed to finalise: %s", err) - return err + precommitCount, err = s.getTotalVotesForBlock(bestFinalCandidate.Hash, precommit) + if err != nil { + return nil, 0, fmt.Errorf("getting total votes for block %s: %w", + bestFinalCandidate.Hash.Short(), err) } - logger.Debugf("round completed in %s", time.Since(start)) - return nil + return bestFinalCandidate, precommitCount, nil } -func (s *Service) sendVoteMessage(stage Subround, msg *VoteMessage, roundComplete <-chan struct{}) { - ticker := time.NewTicker(s.interval * 4) - defer ticker.Stop() - - // Though this looks like we are sending messages multiple times, - // caching would make sure that they are being sent only once. - for { - if s.paused.Load().(bool) { - return - } - - if err := s.sendMessage(msg); err != nil { - logger.Warnf("could not send message for stage %s: %s", stage, err) - } else { - logger.Tracef("sent vote message for stage %s: %s", stage, msg.Message) - } - - select { - case <-roundComplete: - return - case <-ticker.C: - } +// attemptToFinalize check if we should finalize the current round +func (s *Service) attemptToFinalize() (isFinalizable bool, err error) { + bestFinalCandidate, precommitCount, err := s.retrieveBestFinalCandidate() + if err != nil { + return false, fmt.Errorf("getting best final candidate: %w", err) } -} -// attemptToFinalize loops until the round is finalisable -func (s *Service) attemptToFinalize() error { - ticker := time.NewTicker(s.interval / 100) - - for { - select { - case <-s.ctx.Done(): - return errors.New("context cancelled") - case <-ticker.C: - } - - if s.paused.Load().(bool) { - return ErrServicePaused - } - - has, _ := s.blockState.HasFinalisedBlock(s.state.round, s.state.setID) - if has { - logger.Debugf("block was finalised for round %d", s.state.round) - return nil // a block was finalised, seems like we missed some messages - } - - highestRound, highestSetID, _ := s.blockState.GetHighestRoundAndSetID() - if highestRound > s.state.round { - logger.Debugf("block was finalised for round %d and set id %d", - highestRound, highestSetID) - return nil // a block was finalised, seems like we missed some messages - } - - if highestSetID > s.state.setID { - logger.Debugf("block was finalised for round %d and set id %d", - highestRound, highestSetID) - return nil // a block was finalised, seems like we missed some messages - } - - bfc, err := s.getBestFinalCandidate() - if err != nil { - return err - } - - pc, err := s.getTotalVotesForBlock(bfc.Hash, precommit) - if err != nil { - return err - } + // once we reach the threshold we should stop sending precommit messages to other peers + if bestFinalCandidate.Number < uint32(s.head.Number) || precommitCount <= s.state.threshold() { + return false, nil + } - if bfc.Number < uint32(s.head.Number) || pc <= s.state.threshold() { - continue - } + err = s.finalise() + if err != nil { + return false, fmt.Errorf("finalising: %w", err) + } - if err = s.finalise(); err != nil { - return err - } + // if we haven't received a finalisation message for this block yet, broadcast a finalisation message + votes := s.getDirectVotes(precommit) + logger.Debugf("block was finalised for round %d and set id %d. "+ + "Head hash is %s, %d direct votes for bfc and %d total votes for bfc", + s.state.round, s.state.setID, s.head.Hash(), votes[*bestFinalCandidate], precommitCount) - // if we haven't received a finalisation message for this block yet, broadcast a finalisation message - votes := s.getDirectVotes(precommit) - logger.Debugf("block was finalised for round %d and set id %d. "+ - "Head hash is %s, %d direct votes for bfc and %d total votes for bfc", - s.state.round, s.state.setID, s.head.Hash(), votes[*bfc], pc) + return true, nil +} - cm, err := s.newCommitMessage(s.head, s.state.round) - if err != nil { - return err - } +func (s *Service) sendPrecommitMessage(voteMessage *VoteMessage) (err error) { + logger.Debugf("sending pre-commit message %s...", voteMessage.Message) - msg, err := cm.ToConsensusMessage() - if err != nil { - return err - } + consensusMessage, err := voteMessage.ToConsensusMessage() + if err != nil { + return fmt.Errorf("transforming pre-commit into consensus message: %w", err) + } - logger.Debugf("sending CommitMessage: %v", cm) - s.network.GossipMessage(msg) + s.network.GossipMessage(consensusMessage) + logger.Tracef("sent pre-commit message: %v", consensusMessage) + return nil +} - s.telemetry.SendMessage(telemetry.NewAfgFinalizedBlocksUpTo( - s.head.Hash(), - fmt.Sprint(s.head.Number), - )) +func (s *Service) sendPrevoteMessage(vm *VoteMessage) (err error) { + logger.Debugf("sending pre-vote message %s...", vm) - return nil + consensusMessage, err := vm.ToConsensusMessage() + if err != nil { + return fmt.Errorf("transforming pre-vote into consensus message: %w", err) } + + s.network.GossipMessage(consensusMessage) + logger.Tracef("sent pre-vote message: %v", consensusMessage) + return nil } func (s *Service) loadVote(key ed25519.PublicKeyBytes, stage Subround) (*SignedVote, bool) { @@ -713,12 +591,7 @@ func (s *Service) determinePreCommit() (*Vote, error) { s.preVotedBlock[s.state.round] = &pvb s.mapLock.Unlock() - bestBlockHeader, err := s.blockState.BestBlockHeader() - if err != nil { - return nil, fmt.Errorf("cannot retrieve best block header: %w", err) - } - - nextChange, err := s.grandpaState.NextGrandpaAuthorityChange(bestBlockHeader.Hash(), bestBlockHeader.Number) + nextChange, err := s.grandpaState.NextGrandpaAuthorityChange(pvb.Hash, uint(pvb.Number)) if errors.Is(err, state.ErrNoNextAuthorityChange) { return &pvb, nil } else if err != nil { @@ -945,7 +818,8 @@ func (s *Service) getPreVotedBlock() (Vote, error) { // if there are multiple, find the one with the highest number and return it highest := Vote{ - Number: uint32(0), + Hash: s.head.Hash(), + Number: uint32(s.head.Number), } for h, n := range blocks { @@ -989,7 +863,8 @@ func (s *Service) getGrandpaGHOST() (Vote, error) { // if there are multiple, find the one with the highest number and return it highest := Vote{ - Number: uint32(0), + Hash: s.head.Hash(), + Number: uint32(s.head.Number), } for h, n := range blocks { @@ -1281,3 +1156,209 @@ func (s *Service) lenVotes(stage Subround) int { return count } + +func (s *Service) handleVoteMessage(from peer.ID, vote *VoteMessage) (err error) { + logger.Debugf("received vote message, (peer: %s, msg: %+v)", from, vote) + s.sendTelemetryVoteMessage(vote) + + grandpaVote, err := s.validateVoteMessage(from, vote) + if err != nil { + return fmt.Errorf("validating vote message: %w", err) + } + + threshold := s.state.threshold() + 1 + logger.Debugf( + "validated vote message %v from %s, round %d, subround %d, "+ + "prevote count %d, precommit count %d, votes needed %d", + grandpaVote, vote.Message.AuthorityID, vote.Round, vote.Message.Stage, + s.lenVotes(prevote), s.lenVotes(precommit), threshold) + return nil +} + +func (s *Service) handleCommitMessage(commitMessage *CommitMessage) error { + logger.Debugf("received commit message: %+v", commitMessage) + + err := verifyBlockHashAgainstBlockNumber(s.blockState, + commitMessage.Vote.Hash, uint(commitMessage.Vote.Number)) + if err != nil { + if errors.Is(err, chaindb.ErrKeyNotFound) { + s.tracker.addCommit(commitMessage) + } + + return fmt.Errorf("verifying block hash against block number: %w", err) + } + + containsPrecommitsSignedBy := make([]string, len(commitMessage.AuthData)) + for i, authData := range commitMessage.AuthData { + containsPrecommitsSignedBy[i] = authData.AuthorityID.String() + } + + s.telemetry.SendMessage( + telemetry.NewAfgReceivedCommit( + commitMessage.Vote.Hash, + fmt.Sprint(commitMessage.Vote.Number), + containsPrecommitsSignedBy, + ), + ) + + has, err := s.blockState.HasFinalisedBlock(commitMessage.Round, s.state.setID) + if err != nil { + return fmt.Errorf("checking for a finalized block in the block state: %w", err) + } + + if has { + return nil + } + + // check justification here + err = verifyCommitMessageJustification(*commitMessage, s.state.setID, + s.state.threshold(), s.authorities(), s.blockState) + if err != nil { + if errors.Is(err, blocktree.ErrStartNodeNotFound) { + // we haven't synced the committed block yet, add this to the tracker for later processing + s.tracker.addCommit(commitMessage) + } + return fmt.Errorf("verifying commit message justification: %w", err) + } + + err = s.blockState.SetFinalisedHash(commitMessage.Vote.Hash, commitMessage.Round, s.state.setID) + if err != nil { + return fmt.Errorf("setting finalised hash: %w", err) + } + + preCommitSigned, err := compactToJustification(commitMessage.Precommits, commitMessage.AuthData) + if err != nil { + return fmt.Errorf("compacting justification: %w", err) + } + + err = s.grandpaState.SetPrecommits(commitMessage.Round, commitMessage.SetID, preCommitSigned) + if err != nil { + return fmt.Errorf("setting precommits: %w", err) + } + + // TODO: re-add catch-up logic (#1531) + return nil +} + +func verifyCommitMessageJustification(commitMessage CommitMessage, setID uint64, threshold uint64, + authorities []*types.Authority, blockState BlockState) error { + if len(commitMessage.Precommits) != len(commitMessage.AuthData) { + return fmt.Errorf("%w: precommits len: %d, authorities len: %d", + ErrPrecommitSignatureMismatch, len(commitMessage.Precommits), len(commitMessage.AuthData)) + } + + if commitMessage.SetID != setID { + return fmt.Errorf("%w: grandpa state set id %d, set id in the commit message %d", + ErrSetIDMismatch, setID, commitMessage.SetID) + } + + highestFinalizedHeader, err := blockState.GetHighestFinalisedHeader() + if err != nil { + return fmt.Errorf("getting highest finalised header: %w", err) + } + + isDescendant, err := blockState.IsDescendantOf(highestFinalizedHeader.Hash(), commitMessage.Vote.Hash) + if err != nil { + return fmt.Errorf("verifying ancestry of highest finalised block: %w", err) + } + + if !isDescendant { + return fmt.Errorf("%w: vote hash %s and highest finalised header %s", + errVoteBlockMismatch, commitMessage.Vote.Hash.Short(), highestFinalizedHeader.Hash()) + } + + eqvVoters := getEquivocatoryVoters(commitMessage.AuthData) + var totalValidPrecommits int + for i, preCommit := range commitMessage.Precommits { + justification := &SignedVote{ + Vote: preCommit, + Signature: commitMessage.AuthData[i].Signature, + AuthorityID: commitMessage.AuthData[i].AuthorityID, + } + + err := verifyJustification(justification, commitMessage.Round, setID, precommit, authorities) + if err != nil { + logger.Errorf("verifying justification: %", err) + continue + } + + isDescendant, err := blockState.IsDescendantOf(commitMessage.Vote.Hash, justification.Vote.Hash) + if err != nil { + logger.Warnf("could not check for descendant: %s", err) + continue + } + + precommitedHeader, err := blockState.GetHeader(preCommit.Hash) + if err != nil { + return fmt.Errorf("getting header: %w", err) + } + + if precommitedHeader.Number != uint(preCommit.Number) { + const errFormat = "%w: pre commit corresponding header has block number %d " + + "and pre commit has block number %d" + + return fmt.Errorf(errFormat, + ErrBlockNumbersMismatch, precommitedHeader.Number, preCommit.Number) + } + + if _, ok := eqvVoters[commitMessage.AuthData[i].AuthorityID]; ok { + continue + } + + if isDescendant { + totalValidPrecommits++ + } + } + + validAndEqv := uint64(totalValidPrecommits) + uint64(len(eqvVoters)) + // confirm total # signatures >= grandpa threshold + if validAndEqv < threshold { + return fmt.Errorf("%w: for finalisation message; need %d votes but received only %d valid votes", + ErrMinVotesNotMet, threshold, validAndEqv) + } + + logger.Debugf("validated commit message: %v", commitMessage) + return nil +} + +func verifyJustification(justification *SignedVote, round, setID uint64, + stage Subround, authorities []*types.Authority) error { + fullVote := FullVote{ + Stage: stage, + Vote: justification.Vote, + Round: round, + SetID: setID, + } + + encodedFullVote, err := scale.Marshal(fullVote) + if err != nil { + return fmt.Errorf("scale encoding full vote: %w", err) + } + + publicKey, err := ed25519.NewPublicKey(justification.AuthorityID[:]) + if err != nil { + return fmt.Errorf("creating ed25519 public key: %w", err) + } + + ok, err := publicKey.Verify(encodedFullVote, justification.Signature[:]) + if err != nil { + return fmt.Errorf("verifying signature: %w", err) + } + + if !ok { + return fmt.Errorf("%w: 0x%x for message {%v}", ErrInvalidSignature, justification.Signature, fullVote) + } + + justificationKey, err := justification.AuthorityID.Encode() + if err != nil { + return fmt.Errorf("encoding authority key: %w", err) + } + + for _, auth := range authorities { + if bytes.Equal(auth.Key.Encode(), justificationKey) { + return nil + } + } + + return fmt.Errorf("%w: authority ID 0x%x", ErrVoterNotFound, justificationKey) +} diff --git a/lib/grandpa/grandpa_test.go b/lib/grandpa/grandpa_test.go index 913d3f7e16..e574f17984 100644 --- a/lib/grandpa/grandpa_test.go +++ b/lib/grandpa/grandpa_test.go @@ -30,11 +30,6 @@ var testGenesisHeader = &types.Header{ Digest: types.NewDigest(), } -var ( - kr, _ = keystore.NewEd25519Keyring() - voters = newTestVoters() -) - //go:generate mockgen -destination=mock_telemetry_test.go -package $GOPACKAGE github.com/ChainSafe/gossamer/dot/telemetry Client func newTestState(t *testing.T) *state.Service { @@ -63,7 +58,7 @@ func newTestState(t *testing.T) *state.Service { require.NoError(t, err) block.StoreRuntime(block.BestBlockHash(), rt) - grandpa, err := state.NewGrandpaStateFromGenesis(db, nil, voters) + grandpa, err := state.NewGrandpaStateFromGenesis(db, nil, newTestVoters(t)) require.NoError(t, err) return &state.Service{ @@ -73,7 +68,12 @@ func newTestState(t *testing.T) *state.Service { } } -func newTestVoters() []Voter { +func newTestVoters(t *testing.T) []Voter { + t.Helper() + + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + vs := []Voter{} for i, k := range kr.Keys { vs = append(vs, Voter{ @@ -85,7 +85,7 @@ func newTestVoters() []Voter { return vs } -func newTestService(t *testing.T) (*Service, *state.Service) { +func newTestService(t *testing.T, keypair *ed25519.Keypair) (*Service, *state.Service) { st := newTestState(t) net := newTestNetwork(t) @@ -96,12 +96,12 @@ func newTestService(t *testing.T) (*Service, *state.Service) { cfg := &Config{ BlockState: st.Block, GrandpaState: st.Grandpa, - Voters: voters, - Keypair: kr.Alice().(*ed25519.Keypair), + Voters: newTestVoters(t), Authority: true, Network: net, Interval: time.Second, Telemetry: telemetryMock, + Keypair: keypair, } gs, err := NewService(cfg) @@ -110,8 +110,13 @@ func newTestService(t *testing.T) (*Service, *state.Service) { } func TestUpdateAuthorities(t *testing.T) { - gs, _ := newTestService(t) - err := gs.updateAuthorities() + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, _ := newTestService(t, aliceKeyPair) + + err = gs.updateAuthorities() require.NoError(t, err) require.Equal(t, uint64(0), gs.state.setID) @@ -133,7 +138,11 @@ func TestUpdateAuthorities(t *testing.T) { } func TestGetDirectVotes(t *testing.T) { - gs, _ := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, _ := newTestService(t, aliceKeyPair) voteA := Vote{ Hash: common.Hash{0xa}, @@ -166,7 +175,11 @@ func TestGetDirectVotes(t *testing.T) { } func TestGetVotesForBlock_NoDescendantVotes(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) branches := map[uint]int{6: 1} state.AddBlocksToStateWithFixedBranches(t, st.Block, 8, branches) @@ -202,7 +215,11 @@ func TestGetVotesForBlock_NoDescendantVotes(t *testing.T) { } func TestGetVotesForBlock_DescendantVotes(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) branches := map[uint]int{6: 1} state.AddBlocksToStateWithFixedBranches(t, st.Block, 8, branches) @@ -252,7 +269,11 @@ func TestGetVotesForBlock_DescendantVotes(t *testing.T) { } func TestGetPossibleSelectedAncestors_SameAncestor(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) // this creates a tree with 3 branches all starting at depth 6 branches := map[uint]int{6: 2} @@ -306,7 +327,11 @@ func TestGetPossibleSelectedAncestors_SameAncestor(t *testing.T) { } func TestGetPossibleSelectedAncestors_VaryingAncestor(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) // this creates a tree with branches starting at depth 6 and another branch starting at depth 7 branches := map[uint]int{6: 1, 7: 1} @@ -360,7 +385,11 @@ func TestGetPossibleSelectedAncestors_VaryingAncestor(t *testing.T) { } func TestGetPossibleSelectedAncestors_VaryingAncestor_MoreBranches(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) // this creates a tree with 2 branches starting at depth 6 and 1 branch starting at depth 7, branches := map[uint]int{6: 2, 7: 1} @@ -420,7 +449,11 @@ func TestGetPossibleSelectedAncestors_VaryingAncestor_MoreBranches(t *testing.T) } func TestGetPossibleSelectedBlocks_OneBlock(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) branches := map[uint]int{6: 1} state.AddBlocksToStateWithFixedBranches(t, st.Block, 8, branches) @@ -452,7 +485,11 @@ func TestGetPossibleSelectedBlocks_OneBlock(t *testing.T) { } func TestGetPossibleSelectedBlocks_EqualVotes_SameAncestor(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) // this creates a tree with 3 branches all starting at depth 6 branches := map[uint]int{6: 2} @@ -499,7 +536,11 @@ func TestGetPossibleSelectedBlocks_EqualVotes_SameAncestor(t *testing.T) { } func TestGetPossibleSelectedBlocks_EqualVotes_VaryingAncestor(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) // this creates a tree with branches starting at depth 6 and another branch starting at depth 7 branches := map[uint]int{6: 1, 7: 1} @@ -547,7 +588,11 @@ func TestGetPossibleSelectedBlocks_EqualVotes_VaryingAncestor(t *testing.T) { } func TestGetPossibleSelectedBlocks_OneThirdEquivocating(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) branches := map[uint]int{6: 1} state.AddBlocksToStateWithFixedBranches(t, st.Block, 8, branches) @@ -588,7 +633,11 @@ func TestGetPossibleSelectedBlocks_OneThirdEquivocating(t *testing.T) { } func TestGetPossibleSelectedBlocks_MoreThanOneThirdEquivocating(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) branches := map[uint]int{6: 1, 7: 1} state.AddBlocksToStateWithFixedBranches(t, st.Block, 8, branches) @@ -635,7 +684,11 @@ func TestGetPossibleSelectedBlocks_MoreThanOneThirdEquivocating(t *testing.T) { } func TestGetPreVotedBlock_OneBlock(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) branches := map[uint]int{6: 1} state.AddBlocksToStateWithFixedBranches(t, st.Block, 8, branches) @@ -666,7 +719,11 @@ func TestGetPreVotedBlock_OneBlock(t *testing.T) { } func TestGetPreVotedBlock_MultipleCandidates(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) // this creates a tree with branches starting at depth 6 and another branch starting at depth 7 branches := map[uint]int{6: 1, 7: 1} @@ -712,7 +769,11 @@ func TestGetPreVotedBlock_MultipleCandidates(t *testing.T) { } func TestGetPreVotedBlock_EvenMoreCandidates(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) // this creates a tree with 6 total branches, one each from depth 3 to 7 branches := map[uint]int{3: 1, 4: 1, 5: 1, 6: 1, 7: 1} @@ -780,7 +841,11 @@ func TestGetPreVotedBlock_EvenMoreCandidates(t *testing.T) { } func TestFindParentWithNumber(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) // no branches needed state.AddBlocksToStateWithFixedBranches(t, st.Block, 8, nil) @@ -799,8 +864,12 @@ func TestFindParentWithNumber(t *testing.T) { } func TestGetBestFinalCandidate_OneBlock(t *testing.T) { + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + // this tests the case when the prevoted block and the precommited block are the same - gs, st := newTestService(t) + gs, st := newTestService(t, aliceKeyPair) branches := map[uint]int{6: 1} state.AddBlocksToStateWithFixedBranches(t, st.Block, 8, branches) @@ -837,8 +906,12 @@ func TestGetBestFinalCandidate_OneBlock(t *testing.T) { } func TestGetBestFinalCandidate_PrecommitAncestor(t *testing.T) { + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + // this tests the case when the highest precommited block is an ancestor of the prevoted block - gs, st := newTestService(t) + gs, st := newTestService(t, aliceKeyPair) branches := map[uint]int{6: 1} state.AddBlocksToStateWithFixedBranches(t, st.Block, 8, branches) @@ -879,9 +952,13 @@ func TestGetBestFinalCandidate_PrecommitAncestor(t *testing.T) { } func TestGetBestFinalCandidate_NoPrecommit(t *testing.T) { + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + // this tests the case when no blocks have >=2/3 precommit votes // it should return the prevoted block - gs, st := newTestService(t) + gs, st := newTestService(t, aliceKeyPair) branches := map[uint]int{6: 1} state.AddBlocksToStateWithFixedBranches(t, st.Block, 8, branches) @@ -915,9 +992,13 @@ func TestGetBestFinalCandidate_NoPrecommit(t *testing.T) { } func TestGetBestFinalCandidate_PrecommitOnAnotherChain(t *testing.T) { + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + // this tests the case when the precommited block is on another chain than the prevoted block // this should return their highest common ancestor - gs, st := newTestService(t) + gs, st := newTestService(t, aliceKeyPair) branches := map[uint]int{6: 1} state.AddBlocksToStateWithFixedBranches(t, st.Block, 8, branches) @@ -957,7 +1038,11 @@ func TestGetBestFinalCandidate_PrecommitOnAnotherChain(t *testing.T) { } func TestDeterminePreVote_NoPrimaryPreVote(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) state.AddBlocksToState(t, st.Block, 3, false) pv, err := gs.determinePreVote() @@ -969,7 +1054,11 @@ func TestDeterminePreVote_NoPrimaryPreVote(t *testing.T) { } func TestDeterminePreVote_WithPrimaryPreVote(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) state.AddBlocksToState(t, st.Block, 3, false) header, err := st.Block.BestBlockHeader() @@ -990,7 +1079,11 @@ func TestDeterminePreVote_WithPrimaryPreVote(t *testing.T) { } func TestDeterminePreVote_WithInvalidPrimaryPreVote(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) state.AddBlocksToState(t, st.Block, 3, false) header, err := st.Block.BestBlockHeader() @@ -1012,7 +1105,11 @@ func TestDeterminePreVote_WithInvalidPrimaryPreVote(t *testing.T) { } func TestGetGrandpaGHOST_CommonAncestor(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) branches := map[uint]int{6: 1} state.AddBlocksToStateWithFixedBranches(t, st.Block, 8, branches) @@ -1046,7 +1143,11 @@ func TestGetGrandpaGHOST_CommonAncestor(t *testing.T) { } func TestGetGrandpaGHOST_MultipleCandidates(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) // this creates a tree with branches starting at depth 3 and another branch starting at depth 7 branches := map[uint]int{3: 1, 7: 1} @@ -1095,8 +1196,12 @@ func TestGetGrandpaGHOST_MultipleCandidates(t *testing.T) { } func TestGrandpaServiceCreateJustification_ShouldCountEquivocatoryVotes(t *testing.T) { + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + // setup granpda service - gs, st := newTestService(t) + gs, st := newTestService(t, aliceKeyPair) now := time.Unix(1000, 0) const previousBlocksToAdd = 9 diff --git a/lib/grandpa/helpers_test.go b/lib/grandpa/helpers_test.go index 990492ef4b..c2c23277d0 100644 --- a/lib/grandpa/helpers_test.go +++ b/lib/grandpa/helpers_test.go @@ -5,27 +5,125 @@ package grandpa import ( "testing" + "time" + "github.com/ChainSafe/gossamer/dot/network" "github.com/ChainSafe/gossamer/dot/types" + "github.com/ChainSafe/gossamer/internal/log" "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/crypto/ed25519" "github.com/ChainSafe/gossamer/lib/genesis" "github.com/ChainSafe/gossamer/lib/runtime/wasmer" "github.com/ChainSafe/gossamer/lib/trie" "github.com/ChainSafe/gossamer/lib/utils" - "github.com/stretchr/testify/require" + "github.com/golang/mock/gomock" + "github.com/libp2p/go-libp2p-core/peer" + protocol "github.com/libp2p/go-libp2p-core/protocol" + "github.com/stretchr/testify/assert" ) +type testJustificationRequest struct { + to peer.ID + num uint32 +} + +type testNetwork struct { + t *testing.T + out chan GrandpaMessage + finalised chan GrandpaMessage + justificationRequest *testJustificationRequest +} + +func newTestNetwork(t *testing.T) *testNetwork { + return &testNetwork{ + t: t, + out: make(chan GrandpaMessage, 128), + finalised: make(chan GrandpaMessage, 128), + } +} + +func (n *testNetwork) GossipMessage(msg NotificationsMessage) { + cm, ok := msg.(*ConsensusMessage) + assert.True(n.t, ok) + + gmsg, err := decodeMessage(cm) + assert.NoError(n.t, err) + + _, ok = gmsg.(*CommitMessage) + if ok { + n.finalised <- gmsg + return + } + n.out <- gmsg +} + +func (n *testNetwork) SendMessage(_ peer.ID, _ NotificationsMessage) error { + return nil +} + +func (n *testNetwork) SendJustificationRequest(to peer.ID, num uint32) { + n.justificationRequest = &testJustificationRequest{ + to: to, + num: num, + } +} + +func (*testNetwork) RegisterNotificationsProtocol( + _ protocol.ID, + _ byte, + _ network.HandshakeGetter, + _ network.HandshakeDecoder, + _ network.HandshakeValidator, + _ network.MessageDecoder, + _ network.NotificationsMessageHandler, + _ network.NotificationsMessageBatchHandler, + _ uint64, +) error { + return nil +} + +func (n *testNetwork) SendBlockReqestByHash(_ common.Hash) {} + +func setupGrandpa(t *testing.T, kp *ed25519.Keypair) *Service { + st := newTestState(t) + net := newTestNetwork(t) + + ctrl := gomock.NewController(t) + telemetryMock := NewMockClient(ctrl) + telemetryMock.EXPECT().SendMessage(gomock.Any()).AnyTimes() + + telemetryMock. + EXPECT(). + SendMessage(gomock.Any()).AnyTimes() + + cfg := &Config{ + BlockState: st.Block, + GrandpaState: st.Grandpa, + Voters: newTestVoters(t), + Keypair: kp, + LogLvl: log.Info, + Authority: true, + Network: net, + Interval: time.Second, + Telemetry: telemetryMock, + } + + gs, err := NewService(cfg) + assert.NoError(t, err) + return gs +} + func newTestGenesisWithTrieAndHeader(t *testing.T) ( gen genesis.Genesis, genesisTrie trie.Trie, genesisHeader types.Header) { t.Helper() genesisPath := utils.GetGssmrV3SubstrateGenesisRawPathTest(t) genesisPtr, err := genesis.NewGenesisFromJSONRaw(genesisPath) - require.NoError(t, err) + assert.NoError(t, err) gen = *genesisPtr genesisTrie, err = wasmer.NewTrieFromGenesis(gen) - require.NoError(t, err) + assert.NoError(t, err) parentHash := common.NewHash([]byte{0}) stateRoot := genesisTrie.MustHash() diff --git a/lib/grandpa/message.go b/lib/grandpa/message.go index 939bd386d8..337cc3c773 100644 --- a/lib/grandpa/message.go +++ b/lib/grandpa/message.go @@ -56,6 +56,10 @@ type VoteMessage struct { Message SignedMessage } +func (v VoteMessage) String() string { + return fmt.Sprintf("round=%d, setID=%d, message={%s}", v.Round, v.SetID, v.Message) +} + // Index returns VDT index func (VoteMessage) Index() uint { return 0 } @@ -157,8 +161,8 @@ type CommitMessage struct { AuthData []AuthData } -func (s *Service) newCommitMessage(header *types.Header, round uint64) (*CommitMessage, error) { - pcs, err := s.grandpaState.GetPrecommits(round, s.state.setID) +func (s *Service) newCommitMessage(header *types.Header, round, setID uint64) (*CommitMessage, error) { + pcs, err := s.grandpaState.GetPrecommits(round, setID) if err != nil { return nil, err } diff --git a/lib/grandpa/message_handler.go b/lib/grandpa/message_handler.go index fef28cccf5..512e52e789 100644 --- a/lib/grandpa/message_handler.go +++ b/lib/grandpa/message_handler.go @@ -7,7 +7,6 @@ import ( "bytes" "errors" "fmt" - "reflect" "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/dot/network" @@ -49,15 +48,18 @@ func (h *MessageHandler) handleMessage(from peer.ID, m GrandpaMessage) (network. switch msg := m.(type) { case *VoteMessage: - // send vote message to grandpa service - h.grandpa.in <- &networkVoteMessage{ - from: from, - msg: msg, + err := h.grandpa.handleVoteMessage(from, msg) + if err != nil { + return nil, fmt.Errorf("handling vote message: %w", err) } - return nil, nil case *CommitMessage: - return nil, h.handleCommitMessage(msg) + err := h.grandpa.handleCommitMessage(msg) + if err != nil { + return nil, fmt.Errorf("handling commit message: %w", err) + } + + return nil, nil case *NeighbourPacketV1: // we can afford to not retry handling neighbour message, if it errors. return nil, h.handleNeighbourMessage(msg) @@ -79,6 +81,27 @@ func (h *MessageHandler) handleMessage(from peer.ID, m GrandpaMessage) (network. } func (h *MessageHandler) handleNeighbourMessage(msg *NeighbourPacketV1) error { + if h.grandpa.authority { + // TODO(#2931): this is a simple hack to ensure that the neighbour messages + // sent by gossamer are being received by substrate nodes + // not intended to be production code + h.grandpa.roundLock.Lock() + neighbourMessage := &NeighbourPacketV1{ + Round: h.grandpa.state.round, + SetID: h.grandpa.state.setID, + Number: uint32(h.grandpa.head.Number), + } + h.grandpa.roundLock.Unlock() + + cm, err := neighbourMessage.ToConsensusMessage() + if err != nil { + return fmt.Errorf("converting neighbour message to network message: %w", err) + } + + logger.Debugf("sending neighbour message: %v", neighbourMessage) + h.grandpa.network.GossipMessage(cm) + } + currFinalized, err := h.blockState.GetFinalisedHeader(0, 0) if err != nil { return err @@ -106,63 +129,6 @@ func (h *MessageHandler) handleNeighbourMessage(msg *NeighbourPacketV1) error { return nil } -func (h *MessageHandler) handleCommitMessage(msg *CommitMessage) error { - logger.Debugf("received commit message, msg: %+v", msg) - - err := verifyBlockHashAgainstBlockNumber(h.blockState, msg.Vote.Hash, uint(msg.Vote.Number)) - if err != nil { - if errors.Is(err, chaindb.ErrKeyNotFound) { - h.grandpa.tracker.addCommit(msg) - logger.Infof("we might not have synced to the given block %s yet: %s", msg.Vote.Hash, err) - return nil - } - return err - } - - containsPrecommitsSignedBy := make([]string, len(msg.AuthData)) - for i, authData := range msg.AuthData { - containsPrecommitsSignedBy[i] = authData.AuthorityID.String() - } - - h.telemetry.SendMessage( - telemetry.NewAfgReceivedCommit( - msg.Vote.Hash, - fmt.Sprint(msg.Vote.Number), - containsPrecommitsSignedBy, - ), - ) - - if has, _ := h.blockState.HasFinalisedBlock(msg.Round, h.grandpa.state.setID); has { - return nil - } - - // check justification here - if err := h.verifyCommitMessageJustification(msg); err != nil { - if errors.Is(err, blocktree.ErrStartNodeNotFound) { - // we haven't synced the committed block yet, add this to the tracker for later processing - h.grandpa.tracker.addCommit(msg) - } - return err - } - - // set finalised head for round in db - if err := h.blockState.SetFinalisedHash(msg.Vote.Hash, msg.Round, h.grandpa.state.setID); err != nil { - return err - } - - pcs, err := compactToJustification(msg.Precommits, msg.AuthData) - if err != nil { - return err - } - - if err = h.grandpa.grandpaState.SetPrecommits(msg.Round, msg.SetID, pcs); err != nil { - return err - } - - // TODO: re-add catch-up logic (#1531) - return nil -} - func (h *MessageHandler) handleCatchUpRequest(msg *CatchUpRequest) (*ConsensusMessage, error) { if !h.grandpa.authority { return nil, nil //nolint:nilnil @@ -313,73 +279,6 @@ func isDescendantOfHighestFinalisedBlock(blockState BlockState, hash common.Hash return blockState.IsDescendantOf(highestHeader.Hash(), hash) } -func (h *MessageHandler) verifyCommitMessageJustification(fm *CommitMessage) error { - if len(fm.Precommits) != len(fm.AuthData) { - return ErrPrecommitSignatureMismatch - } - - if fm.SetID != h.grandpa.state.setID { - return fmt.Errorf("%w: grandpa state set id %d, set id in the commit message %d", - ErrSetIDMismatch, h.grandpa.state.setID, fm.SetID) - } - - isDescendant, err := isDescendantOfHighestFinalisedBlock(h.blockState, fm.Vote.Hash) - if err != nil { - return fmt.Errorf("cannot verify ancestry of highest finalised block: %w", err) - } - if !isDescendant { - return errVoteBlockMismatch - } - - eqvVoters := getEquivocatoryVoters(fm.AuthData) - - var count int - for i, pc := range fm.Precommits { - just := &SignedVote{ - Vote: pc, - Signature: fm.AuthData[i].Signature, - AuthorityID: fm.AuthData[i].AuthorityID, - } - - err := h.verifyJustification(just, fm.Round, h.grandpa.state.setID, precommit) - if err != nil { - logger.Errorf("failed to verify justification for vote from authority id %s, for block hash %s: %s", - just.AuthorityID.String(), just.Vote.Hash, err) - continue - } - - isDescendant, err := h.blockState.IsDescendantOf(fm.Vote.Hash, just.Vote.Hash) - if err != nil { - logger.Warnf("could not check for descendant: %s", err) - continue - } - - err = verifyBlockHashAgainstBlockNumber(h.blockState, pc.Hash, uint(pc.Number)) - if err != nil { - return err - } - - if _, ok := eqvVoters[fm.AuthData[i].AuthorityID]; ok { - continue - } - - if isDescendant { - count++ - } - } - - // confirm total # signatures >= grandpa threshold - if uint64(count)+uint64(len(eqvVoters)) < h.grandpa.state.threshold() { - logger.Debugf( - "minimum votes not met for finalisation message. Need %d votes and received %d votes.", - h.grandpa.state.threshold(), count) - return ErrMinVotesNotMet - } - - logger.Debugf("validated commit message: %v", fm) - return nil -} - func (h *MessageHandler) verifyPreVoteJustification(msg *CatchUpResponse) (common.Hash, error) { voters := make(map[ed25519.PublicKeyBytes]map[common.Hash]int, len(msg.PreVoteJustification)) eqVotesByHash := make(map[common.Hash]map[ed25519.PublicKeyBytes]struct{}) @@ -426,7 +325,7 @@ func (h *MessageHandler) verifyPreVoteJustification(msg *CatchUpResponse) (commo continue } - err := h.verifyJustification(just, msg.Round, msg.SetID, prevote) + err := verifyJustification(just, msg.Round, msg.SetID, prevote, h.grandpa.authorities()) if err != nil { continue } @@ -481,7 +380,7 @@ func (h *MessageHandler) verifyPreCommitJustification(msg *CatchUpResponse) erro return err } - err := h.verifyJustification(just, msg.Round, msg.SetID, precommit) + err := verifyJustification(just, msg.Round, msg.SetID, precommit, h.grandpa.authorities()) if err != nil { logger.Errorf("could not verify precommit justification for block %s from authority %s: %s", just.Vote.Hash.String(), just.AuthorityID.String(), err) @@ -504,51 +403,6 @@ func (h *MessageHandler) verifyPreCommitJustification(msg *CatchUpResponse) erro return nil } -func (h *MessageHandler) verifyJustification(just *SignedVote, round, setID uint64, stage Subround) error { - // verify signature - msg, err := scale.Marshal(FullVote{ - Stage: stage, - Vote: just.Vote, - Round: round, - SetID: setID, - }) - if err != nil { - return err - } - - pk, err := ed25519.NewPublicKey(just.AuthorityID[:]) - if err != nil { - return err - } - - ok, err := pk.Verify(msg, just.Signature[:]) - if err != nil { - return err - } - - if !ok { - return ErrInvalidSignature - } - - // verify authority in justification set - authFound := false - - for _, auth := range h.grandpa.authorities() { - justKey, err := just.AuthorityID.Encode() - if err != nil { - return err - } - if reflect.DeepEqual(auth.Key.Encode(), justKey) { - authFound = true - break - } - } - if !authFound { - return ErrVoterNotFound - } - return nil -} - // VerifyBlockJustification verifies the finality justification for a block, returns scale encoded justification with // any extra bytes removed. func (s *Service) VerifyBlockJustification(hash common.Hash, justification []byte) ([]byte, error) { @@ -623,12 +477,12 @@ func (s *Service) VerifyBlockJustification(hash common.Hash, justification []byt return nil, ErrPrecommitBlockMismatch } - pk, err := ed25519.NewPublicKey(just.AuthorityID[:]) + publicKey, err := ed25519.NewPublicKey(just.AuthorityID[:]) if err != nil { return nil, err } - if !isInAuthSet(pk, auths) { + if !isInAuthSet(publicKey, auths) { return nil, ErrAuthorityNotInSet } @@ -643,7 +497,7 @@ func (s *Service) VerifyBlockJustification(hash common.Hash, justification []byt return nil, err } - ok, err := pk.Verify(msg, just.Signature[:]) + ok, err := publicKey.Verify(msg, just.Signature[:]) if err != nil { return nil, err } diff --git a/lib/grandpa/message_handler_test.go b/lib/grandpa/message_handler_test.go index 0fed2c8a56..b8296a201c 100644 --- a/lib/grandpa/message_handler_test.go +++ b/lib/grandpa/message_handler_test.go @@ -99,6 +99,9 @@ func TestDecodeMessage_VoteMessage(t *testing.T) { } func TestDecodeMessage_CommitMessage(t *testing.T) { + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + expected := &CommitMessage{ Round: 77, SetID: 1, @@ -157,16 +160,25 @@ func TestDecodeMessage_CatchUpRequest(t *testing.T) { } func TestMessageHandler_VoteMessage(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) v, err := NewVoteFromHash(st.Block.BestBlockHash(), st.Block) require.NoError(t, err) - - gs.state.setID = 99 - gs.state.round = 77 v.Number = 0x7777 - _, vm, err := gs.createSignedVoteAndVoteMessage(v, precommit) - require.NoError(t, err) + + charlieAuthority := kr.Charlie().(*ed25519.Keypair) + + const round = 99 + const setID = 77 + + gs.state.round = round + gs.state.setID = setID + + createdSigned, vm := createAndSignVoteMessage(t, charlieAuthority, round, setID, v, precommit) ctrl := gomock.NewController(t) telemetryMock := NewMockClient(ctrl) @@ -177,16 +189,18 @@ func TestMessageHandler_VoteMessage(t *testing.T) { require.NoError(t, err) require.Nil(t, out) - select { - case vote := <-gs.in: - require.Equal(t, vm, vote.msg) - case <-time.After(time.Second): - t.Fatal("did not receive VoteMessage") - } + charlieAuthorityPublicKeyBytes := charlieAuthority.Public().(*ed25519.PublicKey).AsBytes() + gotSignedVote, ok := gs.loadVote(charlieAuthorityPublicKeyBytes, precommit) + require.True(t, ok) + require.Equal(t, createdSigned, gotSignedVote) } func TestMessageHandler_NeighbourMessage(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) ctrl := gomock.NewController(t) telemetryMock := NewMockClient(ctrl) @@ -200,7 +214,7 @@ func TestMessageHandler_NeighbourMessage(t *testing.T) { Number: 1, } - _, err := h.handleMessage("", NeighbourPacketV1) + _, err = h.handleMessage("", NeighbourPacketV1) require.NoError(t, err) digest := types.NewDigest() @@ -230,7 +244,11 @@ func TestMessageHandler_NeighbourMessage(t *testing.T) { } func TestMessageHandler_VerifyJustification_InvalidSig(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) gs.state.round = 77 just := &SignedVote{ @@ -243,21 +261,36 @@ func TestMessageHandler_VerifyJustification_InvalidSig(t *testing.T) { telemetryMock := NewMockClient(ctrl) telemetryMock.EXPECT().SendMessage(gomock.Any()).AnyTimes() + // scale encode the message to assert the wrapped error message + expectedFullVote := FullVote{ + Stage: precommit, + Vote: just.Vote, + Round: gs.state.round, + SetID: gs.state.setID, + } + expectedErr := fmt.Errorf("%w: 0x%x for message {%v}", ErrInvalidSignature, just.Signature, expectedFullVote) + h := NewMessageHandler(gs, st.Block, telemetryMock) - err := h.verifyJustification(just, gs.state.round, gs.state.setID, precommit) - require.Equal(t, err, ErrInvalidSignature) + err = verifyJustification(just, gs.state.round, gs.state.setID, precommit, h.grandpa.authorities()) + + require.ErrorIs(t, err, ErrInvalidSignature) + require.EqualError(t, expectedErr, err.Error()) } func TestMessageHandler_CommitMessage_NoCatchUpRequest_ValidSig(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) round := uint64(1) gs.state.round = round just := buildTestJustification(t, int(gs.state.threshold()), round, gs.state.setID, kr, precommit) - err := st.Grandpa.SetPrecommits(round, gs.state.setID, just) + err = st.Grandpa.SetPrecommits(round, gs.state.setID, just) require.NoError(t, err) - fm, err := gs.newCommitMessage(gs.head, round) + fm, err := gs.newCommitMessage(gs.head, round, gs.state.setID) require.NoError(t, err) fm.Vote = *NewVote(testHash, uint32(round)) @@ -282,8 +315,7 @@ func TestMessageHandler_CommitMessage_NoCatchUpRequest_ValidSig(t *testing.T) { telemetryMock := NewMockClient(ctrl) telemetryMock.EXPECT().SendMessage(gomock.Any()).AnyTimes() - h := NewMessageHandler(gs, st.Block, telemetryMock) - out, err := h.handleMessage("", fm) + out, err := gs.messageHandler.handleMessage("", fm) require.NoError(t, err) require.Nil(t, out) @@ -293,16 +325,20 @@ func TestMessageHandler_CommitMessage_NoCatchUpRequest_ValidSig(t *testing.T) { } func TestMessageHandler_CommitMessage_NoCatchUpRequest_MinVoteError(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) round := uint64(77) gs.state.round = round just := buildTestJustification(t, int(gs.state.threshold()), round, gs.state.setID, kr, precommit) - err := st.Grandpa.SetPrecommits(round, gs.state.setID, just) + err = st.Grandpa.SetPrecommits(round, gs.state.setID, just) require.NoError(t, err) - fm, err := gs.newCommitMessage(testGenesisHeader, round) + fm, err := gs.newCommitMessage(testGenesisHeader, round, gs.state.setID) require.NoError(t, err) ctrl := gomock.NewController(t) @@ -311,12 +347,23 @@ func TestMessageHandler_CommitMessage_NoCatchUpRequest_MinVoteError(t *testing.T h := NewMessageHandler(gs, st.Block, telemetryMock) out, err := h.handleMessage("", fm) - require.EqualError(t, err, ErrMinVotesNotMet.Error()) + + const expectedErrString = "handling commit message: " + + "verifying commit message justification: " + + "minimum number of votes not met in a Justification: " + + "for finalisation message; need 6 votes but received only 0 valid votes" + + require.EqualError(t, err, expectedErrString) + require.ErrorIs(t, err, ErrMinVotesNotMet) require.Nil(t, out) } func TestMessageHandler_CommitMessage_WithCatchUpRequest(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) just := []SignedVote{ { @@ -325,10 +372,10 @@ func TestMessageHandler_CommitMessage_WithCatchUpRequest(t *testing.T) { AuthorityID: gs.publicKeyBytes(), }, } - err := st.Grandpa.SetPrecommits(77, gs.state.setID, just) + err = st.Grandpa.SetPrecommits(77, gs.state.setID, just) require.NoError(t, err) - fm, err := gs.newCommitMessage(gs.head, 77) + fm, err := gs.newCommitMessage(gs.head, 77, gs.state.setID) require.NoError(t, err) gs.state.voters = gs.state.voters[:1] @@ -343,7 +390,11 @@ func TestMessageHandler_CommitMessage_WithCatchUpRequest(t *testing.T) { } func TestMessageHandler_CatchUpRequest_InvalidRound(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) req := newCatchUpRequest(77, 0) ctrl := gomock.NewController(t) @@ -351,12 +402,16 @@ func TestMessageHandler_CatchUpRequest_InvalidRound(t *testing.T) { telemetryMock.EXPECT().SendMessage(gomock.Any()).AnyTimes() h := NewMessageHandler(gs, st.Block, telemetryMock) - _, err := h.handleMessage("", req) + _, err = h.handleMessage("", req) require.Equal(t, ErrInvalidCatchUpRound, err) } func TestMessageHandler_CatchUpRequest_InvalidSetID(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) req := newCatchUpRequest(1, 77) ctrl := gomock.NewController(t) @@ -364,12 +419,16 @@ func TestMessageHandler_CatchUpRequest_InvalidSetID(t *testing.T) { telemetryMock.EXPECT().SendMessage(gomock.Any()).AnyTimes() h := NewMessageHandler(gs, st.Block, telemetryMock) - _, err := h.handleMessage("", req) + _, err = h.handleMessage("", req) require.Equal(t, ErrSetIDMismatch, err) } func TestMessageHandler_CatchUpRequest_WithResponse(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) // set up needed info for response round := uint64(1) @@ -439,11 +498,15 @@ func TestMessageHandler_CatchUpRequest_WithResponse(t *testing.T) { } func TestVerifyJustification(t *testing.T) { + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + ctrl := gomock.NewController(t) telemetryMock := NewMockClient(ctrl) telemetryMock.EXPECT().SendMessage(gomock.Any()).AnyTimes() - gs, st := newTestService(t) + gs, st := newTestService(t, aliceKeyPair) h := NewMessageHandler(gs, st.Block, telemetryMock) vote := NewVote(testHash, 123) @@ -453,36 +516,54 @@ func TestVerifyJustification(t *testing.T) { AuthorityID: kr.Alice().Public().(*ed25519.PublicKey).AsBytes(), } - err := h.verifyJustification(just, 77, gs.state.setID, precommit) + err = verifyJustification(just, 77, gs.state.setID, precommit, h.grandpa.authorities()) require.NoError(t, err) } func TestVerifyJustification_InvalidSignature(t *testing.T) { + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + ctrl := gomock.NewController(t) telemetryMock := NewMockClient(ctrl) telemetryMock.EXPECT().SendMessage(gomock.Any()).AnyTimes() - gs, st := newTestService(t) + gs, st := newTestService(t, aliceKeyPair) h := NewMessageHandler(gs, st.Block, telemetryMock) + const round = 77 vote := NewVote(testHash, 123) just := &SignedVote{ Vote: *vote, // create signed vote with mismatched vote number - Signature: createSignedVoteMsg(t, vote.Number+1, 77, gs.state.setID, kr.Alice().(*ed25519.Keypair), precommit), + Signature: createSignedVoteMsg(t, vote.Number+1, round, gs.state.setID, kr.Alice().(*ed25519.Keypair), precommit), AuthorityID: kr.Alice().Public().(*ed25519.PublicKey).AsBytes(), } - err := h.verifyJustification(just, 77, gs.state.setID, precommit) - require.EqualError(t, err, ErrInvalidSignature.Error()) + expectedFullVote := FullVote{ + Stage: precommit, + Vote: just.Vote, + Round: round, + SetID: gs.state.setID, + } + + expectedErr := fmt.Errorf("%w: 0x%x for message {%v}", ErrInvalidSignature, just.Signature, expectedFullVote) + err = verifyJustification(just, round, gs.state.setID, precommit, h.grandpa.authorities()) + require.ErrorIs(t, err, ErrInvalidSignature) + require.EqualError(t, err, expectedErr.Error()) } func TestVerifyJustification_InvalidAuthority(t *testing.T) { + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + ctrl := gomock.NewController(t) telemetryMock := NewMockClient(ctrl) telemetryMock.EXPECT().SendMessage(gomock.Any()).AnyTimes() - gs, st := newTestService(t) + gs, st := newTestService(t, aliceKeyPair) h := NewMessageHandler(gs, st.Block, telemetryMock) // sign vote with key not in authority set fakeKey, err := ed25519.NewKeypairFromPrivateKeyString( @@ -496,16 +577,25 @@ func TestVerifyJustification_InvalidAuthority(t *testing.T) { AuthorityID: fakeKey.Public().(*ed25519.PublicKey).AsBytes(), } - err = h.verifyJustification(just, 77, gs.state.setID, precommit) - require.EqualError(t, err, ErrVoterNotFound.Error()) + encodedAuthorityID, err := just.AuthorityID.Encode() + require.NoError(t, err) + + expectedErrMessage := fmt.Sprintf("%s: authority ID 0x%x", ErrVoterNotFound, encodedAuthorityID) + err = verifyJustification(just, 77, gs.state.setID, precommit, h.grandpa.authorities()) + require.ErrorIs(t, err, ErrVoterNotFound) + require.EqualError(t, err, expectedErrMessage) } func TestMessageHandler_VerifyPreVoteJustification(t *testing.T) { + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + ctrl := gomock.NewController(t) telemetryMock := NewMockClient(ctrl) telemetryMock.EXPECT().SendMessage(gomock.Any()).AnyTimes() - gs, st := newTestService(t) + gs, st := newTestService(t, aliceKeyPair) body, err := types.NewBodyFromBytes([]byte{0}) require.NoError(t, err) @@ -533,11 +623,15 @@ func TestMessageHandler_VerifyPreVoteJustification(t *testing.T) { } func TestMessageHandler_VerifyPreCommitJustification(t *testing.T) { + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + ctrl := gomock.NewController(t) telemetryMock := NewMockClient(ctrl) telemetryMock.EXPECT().SendMessage(gomock.Any()).AnyTimes() - gs, st := newTestService(t) + gs, st := newTestService(t, aliceKeyPair) body, err := types.NewBodyFromBytes([]byte{0}) require.NoError(t, err) @@ -567,9 +661,13 @@ func TestMessageHandler_VerifyPreCommitJustification(t *testing.T) { } func TestMessageHandler_HandleCatchUpResponse(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) - err := st.Block.SetHeader(testHeader) + err = st.Block.SetHeader(testHeader) require.NoError(t, err) ctrl := gomock.NewController(t) @@ -599,6 +697,10 @@ func TestMessageHandler_HandleCatchUpResponse(t *testing.T) { } func TestMessageHandler_VerifyBlockJustification_WithEquivocatoryVotes(t *testing.T) { + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + auths := []types.GrandpaVoter{ { Key: *kr.Alice().Public().(*ed25519.PublicKey), @@ -629,8 +731,8 @@ func TestMessageHandler_VerifyBlockJustification_WithEquivocatoryVotes(t *testin }, } - gs, st := newTestService(t) - err := st.Grandpa.SetNextChange(auths, 0) + gs, st := newTestService(t, aliceKeyPair) + err = st.Grandpa.SetNextChange(auths, 0) require.NoError(t, err) body, err := types.NewBodyFromBytes([]byte{0}) @@ -660,6 +762,10 @@ func TestMessageHandler_VerifyBlockJustification_WithEquivocatoryVotes(t *testin } func TestMessageHandler_VerifyBlockJustification(t *testing.T) { + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + auths := []types.GrandpaVoter{ { Key: *kr.Alice().Public().(*ed25519.PublicKey), @@ -672,8 +778,8 @@ func TestMessageHandler_VerifyBlockJustification(t *testing.T) { }, } - gs, st := newTestService(t) - err := st.Grandpa.SetNextChange(auths, 0) + gs, st := newTestService(t, aliceKeyPair) + err = st.Grandpa.SetNextChange(auths, 0) require.NoError(t, err) body, err := types.NewBodyFromBytes([]byte{0}) @@ -716,6 +822,10 @@ func TestMessageHandler_VerifyBlockJustification(t *testing.T) { } func TestMessageHandler_VerifyBlockJustification_invalid(t *testing.T) { + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + auths := []types.GrandpaVoter{ { Key: *kr.Alice().Public().(*ed25519.PublicKey), @@ -728,8 +838,8 @@ func TestMessageHandler_VerifyBlockJustification_invalid(t *testing.T) { }, } - gs, st := newTestService(t) - err := st.Grandpa.SetNextChange(auths, 1) + gs, st := newTestService(t, aliceKeyPair) + err = st.Grandpa.SetNextChange(auths, 1) require.NoError(t, err) body, err := types.NewBodyFromBytes([]byte{0}) @@ -1019,9 +1129,13 @@ func Test_getEquivocatoryVoters(t *testing.T) { } func Test_VerifyCommitMessageJustification_ShouldRemoveEquivocatoryVotes(t *testing.T) { + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + const fakeRound = 2 - gs, st := newTestService(t) + gs, st := newTestService(t, aliceKeyPair) ctrl := gomock.NewController(t) telemetryMock := NewMockClient(ctrl) telemetryMock.EXPECT().SendMessage(gomock.Any()).AnyTimes() @@ -1081,16 +1195,22 @@ func Test_VerifyCommitMessageJustification_ShouldRemoveEquivocatoryVotes(t *test AuthData: authData, } - err = h.verifyCommitMessageJustification(testCommitData) + err = verifyCommitMessageJustification(*testCommitData, h.grandpa.state.setID, + h.grandpa.state.threshold(), h.grandpa.authorities(), h.blockState) + require.NoError(t, err) } func Test_VerifyPrevoteJustification_CountEquivocatoryVoters(t *testing.T) { + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + ctrl := gomock.NewController(t) telemetryMock := NewMockClient(ctrl) telemetryMock.EXPECT().SendMessage(gomock.Any()).AnyTimes() - gs, st := newTestService(t) + gs, st := newTestService(t, aliceKeyPair) h := NewMessageHandler(gs, st.Block, telemetryMock) const previousBlocksToAdd = 9 @@ -1160,6 +1280,10 @@ func Test_VerifyPrevoteJustification_CountEquivocatoryVoters(t *testing.T) { } func Test_VerifyPreCommitJustification(t *testing.T) { + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + ctrl := gomock.NewController(t) telemetryMock := NewMockClient(ctrl) telemetryMock.EXPECT().SendMessage(gomock.Any()).AnyTimes() @@ -1169,7 +1293,7 @@ func Test_VerifyPreCommitJustification(t *testing.T) { SendMessage(gomock.Any()). AnyTimes() - gs, st := newTestService(t) + gs, st := newTestService(t, aliceKeyPair) h := NewMessageHandler(gs, st.Block, telemetryMock) const previousBlocksToAdd = 7 @@ -1253,6 +1377,9 @@ func signFakeFullVote( func TestService_VerifyBlockJustification(t *testing.T) { t.Parallel() + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + precommits := buildTestJustification(t, 2, 1, 0, kr, precommit) justification := newJustification(1, testHash, 1, precommits) justificationBytes, err := scale.Marshal(*justification) diff --git a/lib/grandpa/message_test.go b/lib/grandpa/message_test.go index 270ceefd6f..13992702f3 100644 --- a/lib/grandpa/message_test.go +++ b/lib/grandpa/message_test.go @@ -10,6 +10,7 @@ import ( "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/crypto/ed25519" + "github.com/ChainSafe/gossamer/lib/keystore" "github.com/ChainSafe/gossamer/pkg/scale" "github.com/stretchr/testify/require" @@ -29,8 +30,13 @@ var testSignature = [64]byte{1, 2, 3, 4} var testAuthorityID = [32]byte{5, 6, 7, 8} func TestCommitMessageEncode(t *testing.T) { + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + exp := common.MustHexToBytes("0x4d0000000000000000000000000000007db9db5ed9967b80143100189ba69d9e4deab85ac3570e5df25686cabe32964a00000000040a0b0c0d00000000000000000000000000000000000000000000000000000000e7030000040102030400000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000088dc3417d5058ec4b4503e0c12ea1a0a89be200fe98922423d4334014fa6b0ee") //nolint:lll - gs, st := newTestService(t) + + gs, st := newTestService(t, aliceKeyPair) just := []SignedVote{ { Vote: *testVote, @@ -38,10 +44,11 @@ func TestCommitMessageEncode(t *testing.T) { AuthorityID: gs.publicKeyBytes(), }, } - err := st.Grandpa.SetPrecommits(77, gs.state.setID, just) + + err = st.Grandpa.SetPrecommits(77, gs.state.setID, just) require.NoError(t, err) - fm, err := gs.newCommitMessage(gs.head, 77) + fm, err := gs.newCommitMessage(gs.head, 77, 0) require.NoError(t, err) precommits, authData := justificationToCompact(just) @@ -63,7 +70,11 @@ func TestCommitMessageEncode(t *testing.T) { } func TestVoteMessageToConsensusMessage(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) v, err := NewVoteFromHash(st.Block.BestBlockHash(), st.Block) require.NoError(t, err) @@ -110,7 +121,12 @@ func TestVoteMessageToConsensusMessage(t *testing.T) { } func TestCommitMessageToConsensusMessage(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) + just := []SignedVote{ { Vote: *testVote, @@ -118,10 +134,10 @@ func TestCommitMessageToConsensusMessage(t *testing.T) { AuthorityID: gs.publicKeyBytes(), }, } - err := st.Grandpa.SetPrecommits(77, gs.state.setID, just) + err = st.Grandpa.SetPrecommits(77, gs.state.setID, just) require.NoError(t, err) - fm, err := gs.newCommitMessage(gs.head, 77) + fm, err := gs.newCommitMessage(gs.head, 77, 0) require.NoError(t, err) precommits, authData := justificationToCompact(just) @@ -136,7 +152,11 @@ func TestCommitMessageToConsensusMessage(t *testing.T) { } func TestNewCatchUpResponse(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) round := uint64(1) setID := uint64(1) diff --git a/lib/grandpa/message_tracker_integration_test.go b/lib/grandpa/message_tracker_integration_test.go new file mode 100644 index 0000000000..e3bbe0668e --- /dev/null +++ b/lib/grandpa/message_tracker_integration_test.go @@ -0,0 +1,72 @@ +// Copyright 2022 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +//go:build integration + +package grandpa + +import ( + "testing" + "time" + + "github.com/ChainSafe/gossamer/dot/state" + "github.com/ChainSafe/gossamer/dot/types" + "github.com/ChainSafe/gossamer/lib/crypto/ed25519" + "github.com/ChainSafe/gossamer/lib/keystore" + "github.com/stretchr/testify/require" +) + +func TestMessageTracker_SendMessage(t *testing.T) { + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + + gs := setupGrandpa(t, kr.Bob().(*ed25519.Keypair)) + + state.AddBlocksToState(t, gs.blockState.(*state.BlockState), 3, false) + gs.tracker = newTracker(gs.blockState, gs.messageHandler) + gs.tracker.start() + defer gs.tracker.stop() + + parent, err := gs.blockState.BestBlockHeader() + require.NoError(t, err) + + digest := types.NewDigest() + prd, err := types.NewBabeSecondaryPlainPreDigest(0, 1).ToPreRuntimeDigest() + require.NoError(t, err) + err = digest.Add(*prd) + require.NoError(t, err) + + next := &types.Header{ + ParentHash: parent.Hash(), + Number: 4, + Digest: digest, + } + + aliceAuthority := kr.Alice().(*ed25519.Keypair) + aliceSignedVote, aliceVoteMessage := createAndSignVoteMessage(t, aliceAuthority, gs.state.round, + gs.state.setID, NewVoteFromHeader(next), prevote) + + const expectedErr = "validating vote: block does not exist" + _, err = gs.validateVoteMessage("", aliceVoteMessage) + require.ErrorIs(t, err, ErrBlockDoesNotExist) + require.EqualError(t, err, expectedErr) + + authorityID := kr.Alice().Public().(*ed25519.PublicKey).AsBytes() + voteMessage := getMessageFromVotesMapping(gs.tracker.votes.mapping, next.Hash(), authorityID) + require.Equal(t, aliceVoteMessage, voteMessage) + + err = gs.blockState.(*state.BlockState).AddBlock(&types.Block{ + Header: *next, + Body: types.Body{}, + }) + require.NoError(t, err) + + // grandpa tracker check every second if the block + // was included in the block tree + time.Sleep(2 * time.Second) + + aliceAuthorityPublicBytes := aliceAuthority.Public().(*ed25519.PublicKey).AsBytes() + gotSignedVote, ok := gs.loadVote(aliceAuthorityPublicBytes, prevote) + require.True(t, ok) + require.Equal(t, aliceSignedVote, gotSignedVote) +} diff --git a/lib/grandpa/message_tracker_test.go b/lib/grandpa/message_tracker_test.go index 7de9ce4309..68ac3854b0 100644 --- a/lib/grandpa/message_tracker_test.go +++ b/lib/grandpa/message_tracker_test.go @@ -5,14 +5,21 @@ package grandpa import ( "container/list" + "fmt" + "sync" "testing" "time" + "github.com/ChainSafe/chaindb" "github.com/ChainSafe/gossamer/dot/state" + "github.com/ChainSafe/gossamer/dot/telemetry" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" "github.com/ChainSafe/gossamer/lib/crypto/ed25519" "github.com/ChainSafe/gossamer/lib/keystore" + "github.com/ChainSafe/gossamer/pkg/scale" + "github.com/golang/mock/gomock" + "github.com/libp2p/go-libp2p-core/peer" "github.com/stretchr/testify/require" ) @@ -39,7 +46,7 @@ func TestMessageTracker_ValidateMessage(t *testing.T) { kr, err := keystore.NewEd25519Keyring() require.NoError(t, err) - gs, _, _, _ := setupGrandpa(t, kr.Bob().(*ed25519.Keypair)) + gs := setupGrandpa(t, kr.Bob().(*ed25519.Keypair)) state.AddBlocksToState(t, gs.blockState.(*state.BlockState), 3, false) gs.tracker = newTracker(gs.blockState, gs.messageHandler) @@ -52,70 +59,25 @@ func TestMessageTracker_ValidateMessage(t *testing.T) { require.NoError(t, err) gs.keypair = kr.Bob().(*ed25519.Keypair) + const expectedErr = "validating vote: block does not exist" _, err = gs.validateVoteMessage("", msg) - require.Equal(t, err, ErrBlockDoesNotExist) + require.ErrorIs(t, err, ErrBlockDoesNotExist) + require.EqualError(t, err, expectedErr) + authorityID := kr.Alice().Public().(*ed25519.PublicKey).AsBytes() voteMessage := getMessageFromVotesMapping(gs.tracker.votes.mapping, fake.Hash(), authorityID) require.Equal(t, msg, voteMessage) } -func TestMessageTracker_SendMessage(t *testing.T) { - kr, err := keystore.NewEd25519Keyring() - require.NoError(t, err) - - gs, in, _, _ := setupGrandpa(t, kr.Bob().(*ed25519.Keypair)) - state.AddBlocksToState(t, gs.blockState.(*state.BlockState), 3, false) - gs.tracker = newTracker(gs.blockState, gs.messageHandler) - gs.tracker.start() - defer gs.tracker.stop() - - parent, err := gs.blockState.BestBlockHeader() - require.NoError(t, err) - - digest := types.NewDigest() - prd, err := types.NewBabeSecondaryPlainPreDigest(0, 1).ToPreRuntimeDigest() - require.NoError(t, err) - err = digest.Add(*prd) - require.NoError(t, err) - - next := &types.Header{ - ParentHash: parent.Hash(), - Number: 4, - Digest: digest, - } - - gs.keypair = kr.Alice().(*ed25519.Keypair) - _, msg, err := gs.createSignedVoteAndVoteMessage(NewVoteFromHeader(next), prevote) - require.NoError(t, err) - gs.keypair = kr.Bob().(*ed25519.Keypair) - - _, err = gs.validateVoteMessage("", msg) - require.Equal(t, err, ErrBlockDoesNotExist) - authorityID := kr.Alice().Public().(*ed25519.PublicKey).AsBytes() - voteMessage := getMessageFromVotesMapping(gs.tracker.votes.mapping, next.Hash(), authorityID) - require.Equal(t, msg, voteMessage) - - err = gs.blockState.(*state.BlockState).AddBlock(&types.Block{ - Header: *next, - Body: types.Body{}, - }) - require.NoError(t, err) - - const testTimeout = time.Second - select { - case v := <-in: - require.Equal(t, msg, v.msg) - case <-time.After(testTimeout): - t.Errorf("did not receive vote message %v", msg) - } -} - func TestMessageTracker_ProcessMessage(t *testing.T) { kr, err := keystore.NewEd25519Keyring() require.NoError(t, err) - gs, _, _, _ := setupGrandpa(t, kr.Bob().(*ed25519.Keypair)) + gs := setupGrandpa(t, kr.Bob().(*ed25519.Keypair)) + defer gs.cancel() + state.AddBlocksToState(t, gs.blockState.(*state.BlockState), 3, false) + err = gs.Start() require.NoError(t, err) @@ -141,8 +103,11 @@ func TestMessageTracker_ProcessMessage(t *testing.T) { require.NoError(t, err) gs.keypair = kr.Bob().(*ed25519.Keypair) + const expectedErr = "validating vote: block does not exist" _, err = gs.validateVoteMessage("", msg) - require.Equal(t, ErrBlockDoesNotExist, err) + require.ErrorIs(t, err, ErrBlockDoesNotExist) + require.EqualError(t, err, expectedErr) + authorityID := kr.Alice().Public().(*ed25519.PublicKey).AsBytes() voteMessage := getMessageFromVotesMapping(gs.tracker.votes.mapping, next.Hash(), authorityID) require.Equal(t, msg, voteMessage) @@ -167,7 +132,7 @@ func TestMessageTracker_MapInsideMap(t *testing.T) { kr, err := keystore.NewEd25519Keyring() require.NoError(t, err) - gs, _, _, _ := setupGrandpa(t, kr.Bob().(*ed25519.Keypair)) + gs := setupGrandpa(t, kr.Bob().(*ed25519.Keypair)) state.AddBlocksToState(t, gs.blockState.(*state.BlockState), 3, false) gs.tracker = newTracker(gs.blockState, gs.messageHandler) @@ -191,63 +156,292 @@ func TestMessageTracker_MapInsideMap(t *testing.T) { require.NotEmpty(t, voteMessage) } -func TestMessageTracker_handleTick(t *testing.T) { +func TestMessageTracker_handleTick_commitMessage(t *testing.T) { + t.Parallel() + kr, err := keystore.NewEd25519Keyring() require.NoError(t, err) - gs, in, _, _ := setupGrandpa(t, kr.Bob().(*ed25519.Keypair)) - gs.tracker = newTracker(gs.blockState, gs.messageHandler) - - testHash := common.Hash{1, 2, 3} - msg := &VoteMessage{ - Round: 100, - Message: SignedMessage{ - BlockHash: testHash, + testcases := map[string]struct { + expectedCommitMessage bool + newGrandpaService func(ctrl *gomock.Controller) *Service + }{ + "get_header_failed_should_keep_commit": { + expectedCommitMessage: true, + newGrandpaService: func(ctrl *gomock.Controller) *Service { + networkMock := NewMockNetwork(ctrl) + grandpaStateMock := NewMockGrandpaState(ctrl) + + blockStateMock := NewMockBlockState(ctrl) + blockStateMock.EXPECT(). + GetImportedBlockNotifierChannel(). + Return(make(chan *types.Block)) + + blockStateMock.EXPECT(). + GetHeader(testHash). + Return(nil, chaindb.ErrKeyNotFound) + + grandpaService := &Service{ + telemetry: nil, + keypair: kr.Bob().(*ed25519.Keypair), + state: &State{ + voters: newTestVoters(t), + setID: 0, + round: 1, + }, + grandpaState: grandpaStateMock, + blockState: blockStateMock, + network: networkMock, + prevotes: new(sync.Map), + } + messageHandler := NewMessageHandler(grandpaService, blockStateMock, nil) + grandpaService.messageHandler = messageHandler + grandpaService.tracker = newTracker(blockStateMock, messageHandler) + + return grandpaService + }, + }, + "handel_commit_successfully": { + newGrandpaService: func(ctrl *gomock.Controller) *Service { + networkMock := NewMockNetwork(ctrl) + + blockStateMock := NewMockBlockState(ctrl) + blockStateMock.EXPECT(). + GetImportedBlockNotifierChannel(). + Return(make(chan *types.Block)) + blockStateMock.EXPECT(). + GetHeader(testHash). + Return(&types.Header{ + Number: 1, + }, nil) + + highestFinalizedHeader := &types.Header{} + blockStateMock.EXPECT(). + GetHighestFinalisedHeader(). + Return(highestFinalizedHeader, nil) + + blockStateMock.EXPECT(). + IsDescendantOf(highestFinalizedHeader.Hash(), testHash). + Return(true, nil) + + const commitMessageRound = uint64(100) + const serviceStateSetID = uint64(0) + + blockStateMock.EXPECT(). + HasFinalisedBlock(commitMessageRound, serviceStateSetID). + Return(false, nil) + + blockStateMock.EXPECT(). + SetFinalisedHash(testHash, commitMessageRound, serviceStateSetID). + Return(nil) + + grandpaStateMock := NewMockGrandpaState(ctrl) + grandpaStateMock.EXPECT(). + SetPrecommits(commitMessageRound, uint64(0), []types.GrandpaSignedVote{}) + + telemetryMock := NewMockClient(ctrl) + + commitMessageTelemetry := telemetry.NewAfgReceivedCommit( + testHash, "1", []string{}) + telemetryMock.EXPECT().SendMessage(commitMessageTelemetry) + + grandpaService := &Service{ + telemetry: telemetryMock, + keypair: kr.Bob().(*ed25519.Keypair), + state: &State{ + voters: []types.GrandpaVoter{}, + setID: 0, + round: 1, + }, + grandpaState: grandpaStateMock, + blockState: blockStateMock, + network: networkMock, + prevotes: new(sync.Map), + } + messageHandler := NewMessageHandler(grandpaService, blockStateMock, nil) + grandpaService.messageHandler = messageHandler + grandpaService.tracker = newTracker(blockStateMock, messageHandler) + + return grandpaService + }, }, } - gs.tracker.addVote("", msg) - gs.tracker.handleTick() + for tname, tt := range testcases { + tt := tt + + t.Run(tname, func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + grandpaService := tt.newGrandpaService(ctrl) + + commitMessage := &CommitMessage{ + Round: 100, + SetID: 0, + Vote: types.GrandpaVote{ + Hash: testHash, + Number: 1, + }, + } - const testTimeout = time.Second - select { - case v := <-in: - require.Equal(t, msg, v.msg) - case <-time.After(testTimeout): - t.Errorf("did not receive vote message %v", msg) + grandpaService.tracker.addCommit(commitMessage) + grandpaService.tracker.handleTick() + + trackedCommitMessage := grandpaService.tracker.commits.message(testHash) + + if tt.expectedCommitMessage { + require.NotNil(t, trackedCommitMessage) + } else { + require.Nil(t, trackedCommitMessage) + } + }) } - // shouldn't be deleted as round in message >= grandpa round - require.Len(t, gs.tracker.votes.messages(testHash), 1) +} - gs.state.round = 1 - msg = &VoteMessage{ - Round: 0, - Message: SignedMessage{ - BlockHash: testHash, +func TestMessageTracker_handleTick_voteMessage(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + serviceRound uint64 + voteRound uint64 + keepVoting bool + }{ + "vote_round_greater_than_service_round": { + serviceRound: 1, + voteRound: 2, + keepVoting: true, }, - } - gs.tracker.addVote("", msg) - commitMessage := &CommitMessage{ - Round: 100, - SetID: 1, - Vote: types.GrandpaVote{ - Hash: testHash, - Number: 1, + "vote_round_less_than_service_round": { + serviceRound: 2, + voteRound: 1, + keepVoting: false, }, } - gs.tracker.addCommit(commitMessage) - gs.tracker.handleTick() + for tname, tt := range tests { + tt := tt + + t.Run(tname, func(t *testing.T) { + t.Parallel() + + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + + ctrl := gomock.NewController(t) + + telemetryMock := NewMockClient(ctrl) + authority := kr.Charlie().(*ed25519.Keypair) + publicBytes := authority.Public().(*ed25519.PublicKey).AsBytes() + + prevoteTelemetryMessage := telemetry.NewAfgReceivedPrevote( + testGenesisHeader.Hash(), + fmt.Sprint(testGenesisHeader.Number), + publicBytes.String(), + ) + + telemetryMock.EXPECT().SendMessage(prevoteTelemetryMessage) + + const setID uint64 = 0 + grandpaStateMock := NewMockGrandpaState(ctrl) + + blockStateMock := NewMockBlockState(ctrl) + blockStateMock.EXPECT(). + GetImportedBlockNotifierChannel(). + Return(make(chan *types.Block)) + + fakePeerID := peer.ID("charlie-fake-peer-id") + networkMock := NewMockNetwork(ctrl) + + if tt.voteRound < tt.serviceRound { + blockStateMock.EXPECT(). + GetFinalisedHeader(tt.voteRound, setID). + Return(testGenesisHeader, nil) + + grandpaStateMock.EXPECT(). + GetPrecommits(tt.voteRound, setID). + Return([]types.GrandpaSignedVote{}, nil) + + var notificationMessage NotificationsMessage = &ConsensusMessage{} + networkMock.EXPECT(). + SendMessage(fakePeerID, gomock.AssignableToTypeOf(notificationMessage)) + } + + grandpaService := &Service{ + telemetry: telemetryMock, + keypair: kr.Bob().(*ed25519.Keypair), + state: &State{ + voters: newTestVoters(t), + setID: 0, + round: tt.serviceRound, + }, + grandpaState: grandpaStateMock, + blockState: blockStateMock, + network: networkMock, + prevotes: new(sync.Map), + } + + messageHandler := NewMessageHandler(grandpaService, blockStateMock, telemetryMock) + grandpaService.tracker = newTracker(blockStateMock, messageHandler) + grandpaService.messageHandler = messageHandler + + vote := &Vote{ + Hash: testGenesisHeader.Hash(), + Number: uint32(testGenesisHeader.Number), + } + + _, voteMessage := createAndSignVoteMessage(t, authority, + tt.voteRound, setID, vote, prevote) + + grandpaService.tracker.addVote(fakePeerID, voteMessage) + grandpaService.tracker.handleTick() + + expectedLen := 1 + if !tt.keepVoting { + expectedLen = 0 + } + + require.Len(t, grandpaService.tracker.votes.messages(vote.Hash), expectedLen) + }) + } +} + +func createAndSignVoteMessage(t *testing.T, kp *ed25519.Keypair, round, setID uint64, + vote *Vote, stage Subround) (*SignedVote, *VoteMessage) { + t.Helper() + + fullVoteEncoded, err := scale.Marshal(FullVote{ + Stage: stage, + Vote: *vote, + Round: round, + SetID: setID, + }) + require.NoError(t, err) + + signature, err := kp.Sign(fullVoteEncoded) + require.NoError(t, err) + + publicKeyBytes := kp.Public().(*ed25519.PublicKey).AsBytes() + singedVote := &SignedVote{ + Vote: *vote, + Signature: ed25519.NewSignatureBytes(signature), + AuthorityID: publicKeyBytes, + } + + signedMessage := &SignedMessage{ + Stage: stage, + BlockHash: singedVote.Vote.Hash, + Number: singedVote.Vote.Number, + Signature: ed25519.NewSignatureBytes(signature), + AuthorityID: publicKeyBytes, + } - select { - case v := <-in: - require.Equal(t, msg, v.msg) - case <-time.After(testTimeout): - t.Errorf("did not receive vote message %v", msg) + voteMessage := &VoteMessage{ + Round: round, + SetID: setID, + Message: *signedMessage, } - // should be deleted as round in message < grandpa round - require.Empty(t, gs.tracker.votes.messages(testHash)) - require.Empty(t, gs.tracker.commits.message(testHash)) + return singedVote, voteMessage } diff --git a/lib/grandpa/mock_ephemeral_service_test.go b/lib/grandpa/mock_ephemeral_service_test.go new file mode 100644 index 0000000000..a29779c639 --- /dev/null +++ b/lib/grandpa/mock_ephemeral_service_test.go @@ -0,0 +1,62 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: finalisation.go + +// Package grandpa is a generated GoMock package. +package grandpa + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockephemeralService is a mock of ephemeralService interface. +type MockephemeralService struct { + ctrl *gomock.Controller + recorder *MockephemeralServiceMockRecorder +} + +// MockephemeralServiceMockRecorder is the mock recorder for MockephemeralService. +type MockephemeralServiceMockRecorder struct { + mock *MockephemeralService +} + +// NewMockephemeralService creates a new mock instance. +func NewMockephemeralService(ctrl *gomock.Controller) *MockephemeralService { + mock := &MockephemeralService{ctrl: ctrl} + mock.recorder = &MockephemeralServiceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockephemeralService) EXPECT() *MockephemeralServiceMockRecorder { + return m.recorder +} + +// Run mocks base method. +func (m *MockephemeralService) Run() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Run") + ret0, _ := ret[0].(error) + return ret0 +} + +// Run indicates an expected call of Run. +func (mr *MockephemeralServiceMockRecorder) Run() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockephemeralService)(nil).Run)) +} + +// Stop mocks base method. +func (m *MockephemeralService) Stop() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Stop") + ret0, _ := ret[0].(error) + return ret0 +} + +// Stop indicates an expected call of Stop. +func (mr *MockephemeralServiceMockRecorder) Stop() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*MockephemeralService)(nil).Stop)) +} diff --git a/lib/grandpa/mocks_generate_test.go b/lib/grandpa/mocks_generate_test.go index 2d287302b1..2011ad3191 100644 --- a/lib/grandpa/mocks_generate_test.go +++ b/lib/grandpa/mocks_generate_test.go @@ -3,5 +3,6 @@ package grandpa -//go:generate mockgen -destination=mocks_test.go -package $GOPACKAGE . BlockState,GrandpaState +//go:generate mockgen -destination=mocks_test.go -package $GOPACKAGE . BlockState,GrandpaState,Network +//go:generate mockgen -source=finalisation.go -destination=mock_ephemeral_service_test.go -package $GOPACKAGE . ephemeralService //go:generate mockery --name Network --structname Network --case underscore --keeptree diff --git a/lib/grandpa/mocks_test.go b/lib/grandpa/mocks_test.go index d44433c797..52a48c65fb 100644 --- a/lib/grandpa/mocks_test.go +++ b/lib/grandpa/mocks_test.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/ChainSafe/gossamer/lib/grandpa (interfaces: BlockState,GrandpaState) +// Source: github.com/ChainSafe/gossamer/lib/grandpa (interfaces: BlockState,GrandpaState,Network) // Package grandpa is a generated GoMock package. package grandpa @@ -7,9 +7,12 @@ package grandpa import ( reflect "reflect" + network "github.com/ChainSafe/gossamer/dot/network" types "github.com/ChainSafe/gossamer/dot/types" common "github.com/ChainSafe/gossamer/lib/common" gomock "github.com/golang/mock/gomock" + peer "github.com/libp2p/go-libp2p-core/peer" + protocol "github.com/libp2p/go-libp2p-core/protocol" ) // MockBlockState is a mock of BlockState interface. @@ -551,3 +554,66 @@ func (mr *MockGrandpaStateMockRecorder) SetPrevotes(arg0, arg1, arg2 interface{} mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetPrevotes", reflect.TypeOf((*MockGrandpaState)(nil).SetPrevotes), arg0, arg1, arg2) } + +// MockNetwork is a mock of Network interface. +type MockNetwork struct { + ctrl *gomock.Controller + recorder *MockNetworkMockRecorder +} + +// MockNetworkMockRecorder is the mock recorder for MockNetwork. +type MockNetworkMockRecorder struct { + mock *MockNetwork +} + +// NewMockNetwork creates a new mock instance. +func NewMockNetwork(ctrl *gomock.Controller) *MockNetwork { + mock := &MockNetwork{ctrl: ctrl} + mock.recorder = &MockNetworkMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockNetwork) EXPECT() *MockNetworkMockRecorder { + return m.recorder +} + +// GossipMessage mocks base method. +func (m *MockNetwork) GossipMessage(arg0 network.NotificationsMessage) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "GossipMessage", arg0) +} + +// GossipMessage indicates an expected call of GossipMessage. +func (mr *MockNetworkMockRecorder) GossipMessage(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GossipMessage", reflect.TypeOf((*MockNetwork)(nil).GossipMessage), arg0) +} + +// RegisterNotificationsProtocol mocks base method. +func (m *MockNetwork) RegisterNotificationsProtocol(arg0 protocol.ID, arg1 byte, arg2 func() (network.Handshake, error), arg3 func([]byte) (network.Handshake, error), arg4 func(peer.ID, network.Handshake) error, arg5 func([]byte) (network.NotificationsMessage, error), arg6 func(peer.ID, network.NotificationsMessage) (bool, error), arg7 func(peer.ID, network.NotificationsMessage), arg8 uint64) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RegisterNotificationsProtocol", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) + ret0, _ := ret[0].(error) + return ret0 +} + +// RegisterNotificationsProtocol indicates an expected call of RegisterNotificationsProtocol. +func (mr *MockNetworkMockRecorder) RegisterNotificationsProtocol(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterNotificationsProtocol", reflect.TypeOf((*MockNetwork)(nil).RegisterNotificationsProtocol), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) +} + +// SendMessage mocks base method. +func (m *MockNetwork) SendMessage(arg0 peer.ID, arg1 network.NotificationsMessage) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendMessage", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendMessage indicates an expected call of SendMessage. +func (mr *MockNetworkMockRecorder) SendMessage(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessage", reflect.TypeOf((*MockNetwork)(nil).SendMessage), arg0, arg1) +} diff --git a/lib/grandpa/network.go b/lib/grandpa/network.go index 1fd29aa700..7de76846b2 100644 --- a/lib/grandpa/network.go +++ b/lib/grandpa/network.go @@ -6,7 +6,6 @@ package grandpa import ( "fmt" "strings" - "time" "github.com/ChainSafe/gossamer/dot/network" "github.com/ChainSafe/gossamer/lib/common" @@ -16,11 +15,7 @@ import ( "github.com/libp2p/go-libp2p-core/protocol" ) -const ( - grandpaID1 = "grandpa/1" - - neighbourMessageInterval = 5 * time.Minute -) +const grandpaID1 = "grandpa/1" // Handshake is an alias for network.Handshake type Handshake = network.Handshake @@ -156,59 +151,6 @@ func (s *Service) handleNetworkMessage(from peer.ID, msg NotificationsMessage) ( return true, nil } -// sendMessage sends a vote message to be gossiped to the network -func (s *Service) sendMessage(msg GrandpaMessage) error { - cm, err := msg.ToConsensusMessage() - if err != nil { - return err - } - - s.network.GossipMessage(cm) - logger.Tracef("sent message: %v", msg) - return nil -} - -func (s *Service) sendNeighbourMessage(interval time.Duration) { - t := time.NewTicker(interval) - defer t.Stop() - - var neighbourMessage *NeighbourPacketV1 - for { - select { - case <-s.ctx.Done(): - return - - case <-t.C: - s.roundLock.Lock() - neighbourMessage = &NeighbourPacketV1{ - Round: s.state.round, - SetID: s.state.setID, - Number: uint32(s.head.Number), - } - s.roundLock.Unlock() - - case info, ok := <-s.finalisedCh: - if !ok { - return - } - - neighbourMessage = &NeighbourPacketV1{ - Round: info.Round, - SetID: info.SetID, - Number: uint32(info.Header.Number), - } - } - - cm, err := neighbourMessage.ToConsensusMessage() - if err != nil { - logger.Warnf("failed to convert NeighbourMessage to network message: %s", err) - continue - } - - s.network.GossipMessage(cm) - } -} - // decodeMessage decodes a network-level consensus message into a GRANDPA VoteMessage or CommitMessage func decodeMessage(cm *network.ConsensusMessage) (m GrandpaMessage, err error) { msg := newGrandpaMessage() diff --git a/lib/grandpa/network_test.go b/lib/grandpa/network_test.go index 566db1be9f..38302dab27 100644 --- a/lib/grandpa/network_test.go +++ b/lib/grandpa/network_test.go @@ -5,9 +5,9 @@ package grandpa import ( "testing" - "time" - "github.com/ChainSafe/gossamer/dot/types" + "github.com/ChainSafe/gossamer/lib/crypto/ed25519" + "github.com/ChainSafe/gossamer/lib/keystore" "github.com/golang/mock/gomock" "github.com/libp2p/go-libp2p-core/peer" @@ -34,7 +34,11 @@ func TestGrandpaHandshake_Encode(t *testing.T) { } func TestHandleNetworkMessage(t *testing.T) { - gs, st := newTestService(t) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + aliceKeyPair := kr.Alice().(*ed25519.Keypair) + + gs, st := newTestService(t, aliceKeyPair) just := []SignedVote{ { @@ -43,10 +47,10 @@ func TestHandleNetworkMessage(t *testing.T) { AuthorityID: gs.publicKeyBytes(), }, } - err := st.Grandpa.SetPrecommits(77, gs.state.setID, just) + err = st.Grandpa.SetPrecommits(77, gs.state.setID, just) require.NoError(t, err) - fm, err := gs.newCommitMessage(gs.head, 77) + fm, err := gs.newCommitMessage(gs.head, 77, 0) require.NoError(t, err) cm, err := fm.ToConsensusMessage() @@ -77,66 +81,3 @@ func TestHandleNetworkMessage(t *testing.T) { require.NoError(t, err) require.False(t, propagate) } - -func TestSendNeighbourMessage(t *testing.T) { - gs, st := newTestService(t) - go gs.sendNeighbourMessage(time.Second) - - digest := types.NewDigest() - prd, err := types.NewBabeSecondaryPlainPreDigest(0, 1).ToPreRuntimeDigest() - require.NoError(t, err) - err = digest.Add(*prd) - require.NoError(t, err) - block := &types.Block{ - Header: types.Header{ - ParentHash: st.Block.GenesisHash(), - Number: 1, - Digest: digest, - }, - Body: types.Body{}, - } - - err = st.Block.AddBlock(block) - require.NoError(t, err) - - hash := block.Header.Hash() - round := uint64(7) - setID := uint64(33) - - // waits 1.5 seconds and then finalize the block - // we will first send a neighbour message with the initial values - // and send another neighbour message with the finalized block values - time.Sleep(1500 * time.Millisecond) - err = st.Block.SetFinalisedHash(hash, round, setID) - require.NoError(t, err) - - select { - case <-time.After(time.Second): - t.Fatal("did not send message") - case msg := <-gs.network.(*testNetwork).out: - expected := &NeighbourPacketV1{ - SetID: 0, - Round: 0, - Number: 0, - } - - nm, ok := msg.(*NeighbourPacketV1) - require.True(t, ok) - require.Equal(t, expected, nm) - } - - select { - case <-time.After(time.Second): - t.Fatal("did not send message") - case msg := <-gs.network.(*testNetwork).out: - expected := &NeighbourPacketV1{ - SetID: setID, - Round: round, - Number: 1, - } - - nm, ok := msg.(*NeighbourPacketV1) - require.True(t, ok) - require.Equal(t, expected, nm) - } -} diff --git a/lib/grandpa/round_integration_test.go b/lib/grandpa/round_integration_test.go new file mode 100644 index 0000000000..3577efa95c --- /dev/null +++ b/lib/grandpa/round_integration_test.go @@ -0,0 +1,767 @@ +// Copyright 2021 ChainSafe Systems (ON) +// SPDX-License-Identifier: LGPL-3.0-only + +//go:build integration + +package grandpa + +import ( + "context" + "fmt" + "math/rand" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/ChainSafe/gossamer/dot/network" + "github.com/ChainSafe/gossamer/dot/state" + "github.com/ChainSafe/gossamer/dot/telemetry" + "github.com/ChainSafe/gossamer/dot/types" + "github.com/ChainSafe/gossamer/lib/common" + "github.com/ChainSafe/gossamer/lib/crypto/ed25519" + "github.com/ChainSafe/gossamer/lib/keystore" + "github.com/golang/mock/gomock" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGrandpa_DifferentChains(t *testing.T) { + // this asserts that all validators finalise the same block if they all see the + // same pre-votes and pre-commits, even if their chains are different lengths (+/-1 block) + kr, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + + gss := make([]*Service, len(kr.Keys)) + prevotes := new(sync.Map) + precommits := new(sync.Map) + + for i, gs := range gss { + gs = setupGrandpa(t, kr.Keys[i]) + gss[i] = gs + + r := uint(rand.Intn(2)) // 0 or 1 + state.AddBlocksToState(t, gs.blockState.(*state.BlockState), 4+r, false) + pv, err := gs.determinePreVote() + require.NoError(t, err) + prevotes.Store(gs.publicKeyBytes(), &SignedVote{ + Vote: *pv, + }) + } + + // only want to add prevotes for a node that has a block that exists on its chain + for _, gs := range gss { + prevotes.Range(func(key, prevote interface{}) bool { + k := key.(ed25519.PublicKeyBytes) + pv := prevote.(*SignedVote) + err = gs.validateVote(&pv.Vote) + if err == nil { + gs.prevotes.Store(k, pv) + } + return true + }) + } + + for _, gs := range gss { + pc, err := gs.determinePreCommit() + require.NoError(t, err) + precommits.Store(gs.publicKeyBytes(), &SignedVote{ + Vote: *pc, + }) + err = gs.finalise() + require.NoError(t, err) + } + + t.Log(gss[0].blockState.BlocktreeAsString()) + finalised := gss[0].head.Hash() + + for _, gs := range gss[:1] { + require.Equal(t, finalised, gs.head.Hash()) + } +} + +func TestPlayGrandpaRound(t *testing.T) { + t.Parallel() + + ed25519Keyring, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + + tests := map[string]struct { + voters []*ed25519.Keypair + whoEquivocates map[int]struct{} + defineBlockTree func(t *testing.T, blockState BlockState, neighbourServices []*Service) + }{ + // this asserts that all validators finalise the same block if they all see the + // same pre-votes and pre-commits, even if their chains are different lengths + "base_case": { + voters: []*ed25519.Keypair{ + ed25519Keyring.Alice().(*ed25519.Keypair), + ed25519Keyring.Bob().(*ed25519.Keypair), + ed25519Keyring.Charlie().(*ed25519.Keypair), + ed25519Keyring.Ian().(*ed25519.Keypair), + ed25519Keyring.George().(*ed25519.Keypair), + }, + defineBlockTree: func(t *testing.T, blockState BlockState, _ []*Service) { + const withBranches = false + const baseLength = 4 + state.AddBlocksToState(t, blockState.(*state.BlockState), baseLength, withBranches) + }, + }, + + "varying_chain": { + voters: []*ed25519.Keypair{ + ed25519Keyring.Alice().(*ed25519.Keypair), + ed25519Keyring.Bob().(*ed25519.Keypair), + ed25519Keyring.Charlie().(*ed25519.Keypair), + ed25519Keyring.Ian().(*ed25519.Keypair), + }, + defineBlockTree: func(t *testing.T, blockState BlockState, neighbourServices []*Service) { + const diff = 5 + rand := uint(rand.Intn(diff)) + + const withBranches = false + const baseLength = 4 + headers, _ := state.AddBlocksToState(t, blockState.(*state.BlockState), + baseLength+rand, withBranches) + + // sync the created blocks with the neighbour services + // letting them know about those blocks + for _, neighbourService := range neighbourServices { + for _, header := range headers { + block := &types.Block{ + Header: *header, + Body: types.Body{}, + } + neighbourService.blockState.(*state.BlockState).AddBlock(block) + } + } + }, + }, + + "with_equivocations": { + voters: []*ed25519.Keypair{ + ed25519Keyring.Alice().(*ed25519.Keypair), + ed25519Keyring.Bob().(*ed25519.Keypair), + ed25519Keyring.Charlie().(*ed25519.Keypair), + ed25519Keyring.Dave().(*ed25519.Keypair), + ed25519Keyring.Ian().(*ed25519.Keypair), + }, + // alice and charlie equivocates + // it is a map as it is easy to check + whoEquivocates: map[int]struct{}{ + 3: {}, + 4: {}, + }, + defineBlockTree: func(t *testing.T, blockState BlockState, _ []*Service) { + // this creates a tree with 2 branches starting at depth 2 + branches := map[uint]int{2: 1} + const baseLength = 4 + state.AddBlocksToStateWithFixedBranches(t, blockState.(*state.BlockState), baseLength, branches) + }, + }, + } + + for tname, tt := range tests { + tt := tt + t.Run(tname, func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + grandpaServices := make([]*Service, len(tt.voters)) + grandpaVoters := make([]types.GrandpaVoter, len(tt.voters)) + + for idx, kp := range tt.voters { + grandpaVoters[idx] = types.GrandpaVoter{ + Key: *kp.Public().(*ed25519.PublicKey), + } + } + + for idx := range tt.voters { + // gossamer gossips a prevote/precommit message and then waits `subroundInterval` * 4 + // to issue another prevote/precommit message + const subroundInterval = 100 * time.Millisecond + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + st := newTestState(t) + grandpaServices[idx] = &Service{ + ctx: ctx, + cancel: cancel, + paused: atomic.Value{}, + blockState: st.Block, + grandpaState: st.Grandpa, + interval: subroundInterval, + state: &State{ + round: 1, + setID: 0, + voters: grandpaVoters, + }, + head: testGenesisHeader, + authority: true, + keypair: tt.voters[idx], + prevotes: new(sync.Map), + precommits: new(sync.Map), + preVotedBlock: make(map[uint64]*Vote), + bestFinalCandidate: make(map[uint64]*Vote), + pvEquivocations: make(map[ed25519.PublicKeyBytes][]*SignedVote), + pcEquivocations: make(map[ed25519.PublicKeyBytes][]*SignedVote), + } + grandpaServices[idx].paused.Store(false) + } + + neighbourServices := make([][]*Service, len(grandpaServices)) + for idx := range grandpaServices { + neighbours := make([]*Service, len(grandpaServices)-1) + copy(neighbours, grandpaServices[:idx]) + copy(neighbours[idx:], grandpaServices[idx+1:]) + neighbourServices[idx] = neighbours + } + + producedCommitMessages := make([]*CommitMessage, len(grandpaServices)) + for idx, grandpaService := range grandpaServices { + idx := idx + neighbours := neighbourServices[idx] + tt.defineBlockTree(t, grandpaServices[idx].blockState, neighbours) + + // if the service is an equivocator it should send a different vote + // into the same round to all its neighbour peers + serviceNetworkMock := func(serviceIdx int, neighbours []*Service, + equivocateVote *VoteMessage) func(any) { + return func(arg0 any) { + consensusMessage, ok := arg0.(*network.ConsensusMessage) + require.True(t, ok, "expecting *network.ConsensusMessage, got %T", arg0) + + message, err := decodeMessage(consensusMessage) + require.NoError(t, err) + + switch msg := message.(type) { + case *VoteMessage: + for _, neighbour := range neighbours { + neighbour.handleVoteMessage(peer.ID(fmt.Sprint(serviceIdx)), msg) + if equivocateVote != nil { + neighbour.handleVoteMessage(peer.ID(fmt.Sprint(serviceIdx)), equivocateVote) + } + } + case *CommitMessage: + producedCommitMessages[serviceIdx] = msg + } + } + } + + // In this test it is not important to assert the arguments + // to the telemetry SendMessage mocked func + // the TestSendingVotesInRightStage does it properly + telemetryMock := NewMockClient(ctrl) + grandpaService.telemetry = telemetryMock + telemetryMock.EXPECT().SendMessage(gomock.Any()).AnyTimes() + + // if the voter is an equivocator then we issue a vote + // to another block into the same round and set id + var equivocatedVoteMessage *VoteMessage + _, isEquivocator := tt.whoEquivocates[idx] + if isEquivocator { + leaves := grandpaService.blockState.Leaves() + + vote, err := NewVoteFromHash(leaves[1], grandpaService.blockState) + require.NoError(t, err) + + _, vmsg, err := grandpaService.createSignedVoteAndVoteMessage( + vote, prevote) + require.NoError(t, err) + + equivocatedVoteMessage = vmsg + } + + // The network mock works like a wire between the running + // services, and it is not important to assert the arguments + // the TestSendingVotesInRightStage does it properly + mockNet := NewMockNetwork(ctrl) + grandpaService.network = mockNet + mockNet.EXPECT(). + GossipMessage(gomock.Any()). + DoAndReturn(serviceNetworkMock(idx, neighbours, equivocatedVoteMessage)). + AnyTimes() + } + + runfinalisationServices(t, grandpaServices) + + var latestHash common.Hash = grandpaServices[0].head.Hash() + for _, grandpaService := range grandpaServices[1:] { + serviceFinalizedHash := grandpaService.head.Hash() + eql := serviceFinalizedHash.Equal(latestHash) + if !eql { + t.Errorf("miss match service finalized hash\n\texpecting %s\n\tgot%s\n", + latestHash, serviceFinalizedHash) + } + latestHash = serviceFinalizedHash + } + + var latestCommit *CommitMessage = producedCommitMessages[0] + for _, commitMessage := range producedCommitMessages[1:] { + require.NotNil(t, commitMessage) + require.GreaterOrEqual(t, len(commitMessage.Precommits), len(tt.voters)/2) + require.GreaterOrEqual(t, len(commitMessage.AuthData), len(tt.voters)/2) + + require.Equal(t, latestCommit.Round, commitMessage.Round) + require.Equal(t, latestCommit.SetID, commitMessage.SetID) + require.Equal(t, latestCommit.Vote, commitMessage.Vote) + latestCommit = commitMessage + } + + // assert that the services who got an equivocator vote + // stored that information in the map properly + if len(tt.whoEquivocates) > 0 { + for idx, grandpaService := range grandpaServices { + // who equivocates does not take itself in to account + _, isEquivocator := tt.whoEquivocates[idx] + if isEquivocator { + require.LessOrEqual(t, len(grandpaService.pvEquivocations), len(tt.whoEquivocates)-1, + "%s does not have enough equivocations", grandpaService.publicKeyBytes()) + } else { + require.LessOrEqual(t, len(grandpaService.pvEquivocations), len(tt.whoEquivocates), + "%s does not have enough equivocations", grandpaService.publicKeyBytes()) + } + } + } + }) + } +} + +func TestPlayGrandpaRoundMultipleRounds(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + + ed25519Keyring, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + + voters := []*ed25519.Keypair{ + ed25519Keyring.Alice().(*ed25519.Keypair), + ed25519Keyring.Bob().(*ed25519.Keypair), + ed25519Keyring.Charlie().(*ed25519.Keypair), + ed25519Keyring.Dave().(*ed25519.Keypair), + ed25519Keyring.Ian().(*ed25519.Keypair), + } + + grandpaVoters := make([]types.GrandpaVoter, 0, len(voters)) + for _, kp := range voters { + grandpaVoters = append(grandpaVoters, types.GrandpaVoter{ + Key: *kp.Public().(*ed25519.PublicKey), + }) + } + + grandpaServices := make([]*Service, len(voters)) + for idx := range voters { + const subroundInterval = 100 * time.Millisecond + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + st := newTestState(t) + grandpaServices[idx] = &Service{ + ctx: ctx, + cancel: cancel, + paused: atomic.Value{}, + blockState: st.Block, + grandpaState: st.Grandpa, + interval: subroundInterval, + state: &State{ + round: 1, + setID: 0, + voters: grandpaVoters, + }, + head: testGenesisHeader, + authority: true, + keypair: voters[idx], + preVotedBlock: make(map[uint64]*Vote), + bestFinalCandidate: make(map[uint64]*Vote), + } + grandpaServices[idx].paused.Store(false) + + const withBranches = false + const baseLength = 4 + state.AddBlocksToState(t, + grandpaServices[idx].blockState.(*state.BlockState), + baseLength, withBranches) + } + + neighbourServices := make([][]*Service, len(grandpaServices)) + for idx := range grandpaServices { + neighbours := make([]*Service, len(grandpaServices)-1) + copy(neighbours, grandpaServices[:idx]) + copy(neighbours[idx:], grandpaServices[idx+1:]) + neighbourServices[idx] = neighbours + } + + const totalRounds = 10 + for currentRound := 1; currentRound <= totalRounds; currentRound++ { + for _, grandpaService := range grandpaServices { + grandpaService.state.round = uint64(currentRound) + grandpaService.prevotes = new(sync.Map) + grandpaService.precommits = new(sync.Map) + grandpaService.pvEquivocations = make(map[ed25519.PublicKeyBytes][]*SignedVote) + grandpaService.pcEquivocations = make(map[ed25519.PublicKeyBytes][]*SignedVote) + } + + // every grandpa service should produce a commit message + // indicating that it achieved a finalisation in the round + producedCommitMessages := make([]*CommitMessage, len(grandpaServices)) + for idx, grandpaService := range grandpaServices { + idx := idx + neighbours := neighbourServices[idx] + + serviceNetworkMock := func(serviceIdx int, neighbours []*Service) func(any) { + return func(arg0 any) { + consensusMessage, ok := arg0.(*network.ConsensusMessage) + require.True(t, ok, "expecting *network.ConsensusMessage, got %T", arg0) + + message, err := decodeMessage(consensusMessage) + require.NoError(t, err) + + switch msg := message.(type) { + case *VoteMessage: + for _, neighbour := range neighbours { + neighbour.handleVoteMessage(peer.ID(fmt.Sprint(serviceIdx)), msg) + } + case *CommitMessage: + producedCommitMessages[serviceIdx] = msg + } + } + } + + // In this test it is not important to assert the arguments + // to the telemetry SendMessage mocked func + // the TestSendingVotesInRightStage does it properly + telemetryMock := NewMockClient(ctrl) + grandpaService.telemetry = telemetryMock + telemetryMock.EXPECT().SendMessage(gomock.Any()).AnyTimes() + + // The network mock works like a wire between the running + // services, and it is not important to assert the arguments + // the TestSendingVotesInRightStage does it properly + mockNet := NewMockNetwork(ctrl) + grandpaService.network = mockNet + mockNet.EXPECT(). + GossipMessage(gomock.Any()). + Do(serviceNetworkMock(idx, neighbours)). + AnyTimes() + } + + // for each grandpa service we should start the finalisation and voting round + // engines and waits for them to reach finalisation + runfinalisationServices(t, grandpaServices) + + const setID uint64 = 0 + assertSamefinalisationAndChainGrowth(t, grandpaServices, + uint64(currentRound), setID) + + var latestCommit *CommitMessage = producedCommitMessages[0] + for _, commitMessage := range producedCommitMessages[1:] { + require.NotNil(t, commitMessage) + require.GreaterOrEqual(t, len(commitMessage.Precommits), len(voters)/2) + require.GreaterOrEqual(t, len(commitMessage.AuthData), len(voters)/2) + + require.Equal(t, commitMessage.Round, uint64(currentRound)) + require.Equal(t, latestCommit.Round, commitMessage.Round) + require.Equal(t, latestCommit.SetID, commitMessage.SetID) + require.Equal(t, latestCommit.Vote, commitMessage.Vote) + latestCommit = commitMessage + } + } +} + +// runfinalisationServices is designed to handle many grandpa services and starts, for each service, +// the finalisation engine and the voting round engine which will take care of reach finalisation +func runfinalisationServices(t *testing.T, grandpaServices []*Service) { + t.Helper() + + finalisationHandlers := make([]*finalisationHandler, len(grandpaServices)) + for idx, grandpaService := range grandpaServices { + handler := newFinalisationHandler(grandpaService) + handler.firstRun = false + finalisationHandlers[idx] = handler + } + + handlersWg := new(sync.WaitGroup) + handlersWg.Add(len(finalisationHandlers)) + + for _, handler := range finalisationHandlers { + go func(t *testing.T, handler *finalisationHandler) { + defer handlersWg.Done() + + // passing the ready channel as nil since the first run is false + // and we ensure the method fh.newServices() is being called + err := handler.runEphemeralServices(nil) + assert.NoError(t, err) + }(t, handler) + } + + handlersWg.Wait() +} + +// assertChainGrowth ensure that each service reach the same finalisation result +// and that the result belongs to the same chain as the previously finalized block +func assertSamefinalisationAndChainGrowth(t *testing.T, services []*Service, currentRount, setID uint64) { + finalizedHeaderCurrentRound := make([]*types.Header, len(services)) + for idx, grandpaService := range services { + finalizedHeader, err := grandpaService.blockState.GetFinalisedHeader( + currentRount, setID) + require.NoError(t, err) + require.NotNil(t, finalizedHeader, "round %d does not contain an header", currentRount) + finalizedHeaderCurrentRound[idx] = finalizedHeader + } + + var latestFinalized common.Hash = finalizedHeaderCurrentRound[0].Hash() + for _, finalizedHead := range finalizedHeaderCurrentRound[1:] { + eq := finalizedHead.Hash().Equal(latestFinalized) + if !eq { + t.Errorf("miss match finalized hash\n\texpected %s\n\tgot%s\n", + latestFinalized, finalizedHead) + } + latestFinalized = finalizedHead.Hash() + } + + previousRound := currentRount - 1 + // considering that we start from round 1 + // there is nothing to compare before + if previousRound == 0 { + return + } + + for _, grandpaService := range services { + previouslyFinalized, err := grandpaService.blockState. + GetFinalisedHeader(previousRound, setID) + require.NoError(t, err) + + descendant, err := grandpaService.blockState.IsDescendantOf( + previouslyFinalized.Hash(), latestFinalized) + require.NoError(t, err) + require.True(t, descendant) + } +} + +func TestSendingVotesInRightStage(t *testing.T) { + t.Parallel() + + ed25519Keyring, err := keystore.NewEd25519Keyring() + require.NoError(t, err) + + bobAuthority := ed25519Keyring.Bob().(*ed25519.Keypair) + votersPublicKeys := []*ed25519.Keypair{ + ed25519Keyring.Alice().(*ed25519.Keypair), + bobAuthority, + ed25519Keyring.Charlie().(*ed25519.Keypair), + ed25519Keyring.Dave().(*ed25519.Keypair), + } + + grandpaVoters := make([]types.GrandpaVoter, len(votersPublicKeys)) + for idx, pk := range votersPublicKeys { + grandpaVoters[idx] = types.GrandpaVoter{ + Key: *pk.Public().(*ed25519.PublicKey), + } + } + + ctrl := gomock.NewController(t) + mockedGrandpaState := NewMockGrandpaState(ctrl) + mockedGrandpaState.EXPECT(). + NextGrandpaAuthorityChange(testGenesisHeader.Hash(), testGenesisHeader.Number). + Return(uint(0), state.ErrNoNextAuthorityChange). + AnyTimes() + mockedGrandpaState.EXPECT(). + SetPrevotes(uint64(1), uint64(0), gomock.AssignableToTypeOf([]types.GrandpaSignedVote{})). + Return(nil) + mockedGrandpaState.EXPECT(). + SetPrecommits(uint64(1), uint64(0), gomock.AssignableToTypeOf([]types.GrandpaSignedVote{})). + Return(nil) + mockedGrandpaState.EXPECT(). + SetLatestRound(uint64(1)). + Return(nil) + mockedGrandpaState.EXPECT(). + GetPrecommits(uint64(1), uint64(0)). + Return([]types.GrandpaSignedVote{}, nil) + + mockedState := NewMockBlockState(ctrl) + mockedState.EXPECT(). + GenesisHash(). + Return(testGenesisHeader.Hash()). + Times(2) + // since the next 3 function has been called based on the amount of time we wait until we get enough + // prevotes is hard to define a corret amount of times this function shoud be called + mockedState.EXPECT(). + HasFinalisedBlock(uint64(1), uint64(0)). + Return(false, nil). + AnyTimes() + mockedState.EXPECT(). + HasHeader(testGenesisHeader.Hash()). + Return(true, nil). + Times(4) + mockedState.EXPECT(). + GetHighestRoundAndSetID(). + Return(uint64(0), uint64(0), nil). + AnyTimes() + mockedState.EXPECT(). + IsDescendantOf(testGenesisHeader.Hash(), testGenesisHeader.Hash()). + Return(true, nil). + AnyTimes() + + mockedState.EXPECT(). + BestBlockHeader(). + Return(testGenesisHeader, nil). + Times(2) + + // we cannot assert the bytes since some votes is defined while playing grandpa round + mockedState.EXPECT(). + SetJustification(testGenesisHeader.Hash(), gomock.AssignableToTypeOf([]byte{})). + Return(nil) + mockedState.EXPECT(). + GetHeader(testGenesisHeader.Hash()). + Return(testGenesisHeader, nil) + mockedState.EXPECT(). + SetFinalisedHash(testGenesisHeader.Hash(), uint64(1), uint64(0)). + Return(nil) + + expectedFinalizedTelemetryMessage := telemetry.NewAfgFinalizedBlocksUpTo( + testGenesisHeader.Hash(), + fmt.Sprint(testGenesisHeader.Number), + ) + expectedAlicePrevoteTelemetryMessage := telemetry.NewAfgReceivedPrevote( + testGenesisHeader.Hash(), + fmt.Sprint(testGenesisHeader.Number), + grandpaVoters[0].PublicKeyBytes().String(), + ) + expectedCharliePrevoteTelemetryMessage := telemetry.NewAfgReceivedPrevote( + testGenesisHeader.Hash(), + fmt.Sprint(testGenesisHeader.Number), + grandpaVoters[2].PublicKeyBytes().String(), + ) + expectedAlicePrecommitTelemetryMessage := telemetry.NewAfgReceivedPrecommit( + testGenesisHeader.Hash(), + fmt.Sprint(testGenesisHeader.Number), + grandpaVoters[0].PublicKeyBytes().String(), + ) + expectedCharliePrecommitTelemetryMessage := telemetry.NewAfgReceivedPrecommit( + testGenesisHeader.Hash(), + fmt.Sprint(testGenesisHeader.Number), + grandpaVoters[2].PublicKeyBytes().String(), + ) + + mockedTelemetry := NewMockClient(ctrl) + mockedTelemetry.EXPECT(). + SendMessage(expectedAlicePrevoteTelemetryMessage) + mockedTelemetry.EXPECT(). + SendMessage(expectedCharliePrevoteTelemetryMessage) + mockedTelemetry.EXPECT(). + SendMessage(expectedAlicePrecommitTelemetryMessage) + mockedTelemetry.EXPECT(). + SendMessage(expectedCharliePrecommitTelemetryMessage) + mockedTelemetry.EXPECT(). + SendMessage(expectedFinalizedTelemetryMessage) + + mockedNet := NewMockNetwork(ctrl) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // gossamer gossips a prevote/precommit message and then waits `subroundInterval` * 4 + // to issue another prevote/precommit message + const subroundInterval = time.Second + grandpa := &Service{ + ctx: ctx, + cancel: cancel, + paused: atomic.Value{}, + network: mockedNet, + blockState: mockedState, + grandpaState: mockedGrandpaState, + interval: subroundInterval, + state: &State{ + round: 1, + setID: 0, + voters: grandpaVoters, + }, + head: testGenesisHeader, + authority: true, + keypair: bobAuthority, + prevotes: new(sync.Map), + precommits: new(sync.Map), + preVotedBlock: make(map[uint64]*Vote), + bestFinalCandidate: make(map[uint64]*Vote), + telemetry: mockedTelemetry, + } + grandpa.paused.Store(false) + + expectedVote := NewVote(testGenesisHeader.Hash(), uint32(testGenesisHeader.Number)) + _, expectedPrimaryProposal, err := grandpa.createSignedVoteAndVoteMessage(expectedVote, primaryProposal) + require.NoError(t, err) + + primaryProposal, err := expectedPrimaryProposal.ToConsensusMessage() + require.NoError(t, err) + mockedNet.EXPECT(). + GossipMessage(primaryProposal) + + // first of all we should determine our precommit based on our chain view + _, expectedPrevoteMessage, err := grandpa.createSignedVoteAndVoteMessage(expectedVote, prevote) + require.NoError(t, err) + + pv, err := expectedPrevoteMessage.ToConsensusMessage() + require.NoError(t, err) + mockedNet.EXPECT(). + GossipMessage(pv). + AnyTimes() + + // after receive enough prevotes our node should define a precommit message and send it + _, expectedPrecommitMessage, err := grandpa.createSignedVoteAndVoteMessage(expectedVote, precommit) + require.NoError(t, err) + + pc, err := expectedPrecommitMessage.ToConsensusMessage() + require.NoError(t, err) + mockedNet.EXPECT(). + GossipMessage(pc). + AnyTimes() + + wg := sync.WaitGroup{} + wg.Add(1) + + go func() { + defer wg.Done() + finalisationHandler := newFinalisationHandler(grandpa) + finalisationHandler.firstRun = false + + // passing the ready channel as nil since the first run is false + // and we ensure the method fh.newServices() is being called + err := finalisationHandler.runEphemeralServices(nil) + require.NoError(t, err) + }() + + time.Sleep(grandpa.interval * 5) + + // given that we are BOB and we already had predetermined our prevote in a set + // of 4 authorities (ALICE, BOB, CHARLIE and DAVE) then we only need 2 more prevotes + _, aliceVoteMessage := createAndSignVoteMessage(t, votersPublicKeys[0], 1, 0, expectedVote, prevote) + grandpa.handleVoteMessage(peer.ID("alice"), aliceVoteMessage) + + _, charlieVoteMessage := createAndSignVoteMessage(t, votersPublicKeys[2], 1, 0, expectedVote, prevote) + require.NoError(t, err) + grandpa.handleVoteMessage(peer.ID("charlie"), charlieVoteMessage) + + // given that we are BOB and we already had predetermined the precommit given the prevotes + // we only need 2 more precommit messages + _, alicePrecommitMessage := createAndSignVoteMessage(t, votersPublicKeys[0], 1, 0, expectedVote, precommit) + require.NoError(t, err) + grandpa.handleVoteMessage(peer.ID("alice"), alicePrecommitMessage) + + _, charliePrecommitMessage := createAndSignVoteMessage(t, votersPublicKeys[2], 1, 0, expectedVote, precommit) + require.NoError(t, err) + grandpa.handleVoteMessage(peer.ID("charlie"), charliePrecommitMessage) + + commitMessage := &CommitMessage{ + Round: 1, + Vote: *NewVoteFromHeader(testGenesisHeader), + Precommits: []types.GrandpaVote{}, + AuthData: []AuthData{}, + } + expectedGossipCommitMessage, err := commitMessage.ToConsensusMessage() + require.NoError(t, err) + mockedNet.EXPECT(). + GossipMessage(expectedGossipCommitMessage) + + wg.Wait() +} diff --git a/lib/grandpa/round_test.go b/lib/grandpa/round_test.go deleted file mode 100644 index fd9729e602..0000000000 --- a/lib/grandpa/round_test.go +++ /dev/null @@ -1,603 +0,0 @@ -// Copyright 2021 ChainSafe Systems (ON) -// SPDX-License-Identifier: LGPL-3.0-only - -package grandpa - -import ( - //"fmt" - "math/rand" - "sync" - "testing" - "time" - - "github.com/ChainSafe/gossamer/dot/network" - "github.com/ChainSafe/gossamer/dot/state" - "github.com/ChainSafe/gossamer/dot/types" - "github.com/ChainSafe/gossamer/internal/log" - "github.com/ChainSafe/gossamer/lib/common" - "github.com/ChainSafe/gossamer/lib/crypto/ed25519" - "github.com/ChainSafe/gossamer/lib/keystore" - "github.com/golang/mock/gomock" - "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-core/protocol" - "github.com/stretchr/testify/require" -) - -var testTimeout = 20 * time.Second - -type testJustificationRequest struct { - to peer.ID - num uint32 -} - -type testNetwork struct { - t *testing.T - out chan GrandpaMessage - finalised chan GrandpaMessage - justificationRequest *testJustificationRequest -} - -func newTestNetwork(t *testing.T) *testNetwork { - return &testNetwork{ - t: t, - out: make(chan GrandpaMessage, 128), - finalised: make(chan GrandpaMessage, 128), - } -} - -func (n *testNetwork) GossipMessage(msg NotificationsMessage) { - cm, ok := msg.(*ConsensusMessage) - require.True(n.t, ok) - - gmsg, err := decodeMessage(cm) - require.NoError(n.t, err) - - switch gmsg.(type) { - case *CommitMessage: - n.finalised <- gmsg - default: - n.out <- gmsg - } -} - -func (n *testNetwork) SendMessage(_ peer.ID, _ NotificationsMessage) error { - return nil -} - -func (n *testNetwork) SendJustificationRequest(to peer.ID, num uint32) { - n.justificationRequest = &testJustificationRequest{ - to: to, - num: num, - } -} - -func (*testNetwork) RegisterNotificationsProtocol( - _ protocol.ID, - _ byte, - _ network.HandshakeGetter, - _ network.HandshakeDecoder, - _ network.HandshakeValidator, - _ network.MessageDecoder, - _ network.NotificationsMessageHandler, - _ network.NotificationsMessageBatchHandler, - _ uint64, -) error { - return nil -} - -func (n *testNetwork) SendBlockReqestByHash(_ common.Hash) {} - -func setupGrandpa(t *testing.T, kp *ed25519.Keypair) ( - *Service, chan *networkVoteMessage, chan GrandpaMessage, chan GrandpaMessage) { - st := newTestState(t) - net := newTestNetwork(t) - - ctrl := gomock.NewController(t) - telemetryMock := NewMockClient(ctrl) - telemetryMock.EXPECT().SendMessage(gomock.Any()).AnyTimes() - - telemetryMock. - EXPECT(). - SendMessage(gomock.Any()).AnyTimes() - - cfg := &Config{ - BlockState: st.Block, - GrandpaState: st.Grandpa, - Voters: voters, - Keypair: kp, - LogLvl: log.Info, - Authority: true, - Network: net, - Interval: time.Second, - Telemetry: telemetryMock, - } - - gs, err := NewService(cfg) - require.NoError(t, err) - return gs, gs.in, net.out, net.finalised -} - -func TestGrandpa_BaseCase(t *testing.T) { - // this is a base test case that asserts that all validators finalise the same block if they all see the - // same pre-votes and pre-commits, even if their chains are different - kr, err := keystore.NewEd25519Keyring() - require.NoError(t, err) - - gss := make([]*Service, len(kr.Keys)) - prevotes := new(sync.Map) - precommits := new(sync.Map) - - for i, gs := range gss { - gs, _, _, _ = setupGrandpa(t, kr.Keys[i]) - gss[i] = gs - state.AddBlocksToState(t, gs.blockState.(*state.BlockState), 15, false) - pv, err := gs.determinePreVote() - require.NoError(t, err) - prevotes.Store(gs.publicKeyBytes(), &SignedVote{ - Vote: *pv, - }) - } - - for _, gs := range gss { - gs.prevotes = prevotes - gs.precommits = precommits - } - - for _, gs := range gss { - pc, err := gs.determinePreCommit() - require.NoError(t, err) - precommits.Store(gs.publicKeyBytes(), &SignedVote{ - Vote: *pc, - }) - err = gs.finalise() - require.NoError(t, err) - has, err := gs.blockState.HasJustification(gs.head.Hash()) - require.NoError(t, err) - require.True(t, has) - } - - finalised := gss[0].head.Hash() - for _, gs := range gss { - require.Equal(t, finalised, gs.head.Hash()) - } -} - -func TestGrandpa_DifferentChains(t *testing.T) { - // this asserts that all validators finalise the same block if they all see the - // same pre-votes and pre-commits, even if their chains are different lengths (+/-1 block) - kr, err := keystore.NewEd25519Keyring() - require.NoError(t, err) - - gss := make([]*Service, len(kr.Keys)) - prevotes := new(sync.Map) - precommits := new(sync.Map) - - for i, gs := range gss { - gs, _, _, _ = setupGrandpa(t, kr.Keys[i]) - gss[i] = gs - - r := uint(rand.Intn(2)) // 0 or 1 - state.AddBlocksToState(t, gs.blockState.(*state.BlockState), 4+r, false) - pv, err := gs.determinePreVote() - require.NoError(t, err) - prevotes.Store(gs.publicKeyBytes(), &SignedVote{ - Vote: *pv, - }) - } - - // only want to add prevotes for a node that has a block that exists on its chain - for _, gs := range gss { - prevotes.Range(func(key, prevote interface{}) bool { - k := key.(ed25519.PublicKeyBytes) - pv := prevote.(*SignedVote) - err = gs.validateVote(&pv.Vote) - if err == nil { - gs.prevotes.Store(k, pv) - } - return true - }) - } - - for _, gs := range gss { - pc, err := gs.determinePreCommit() - require.NoError(t, err) - precommits.Store(gs.publicKeyBytes(), &SignedVote{ - Vote: *pc, - }) - err = gs.finalise() - require.NoError(t, err) - } - - t.Log(gss[0].blockState.BlocktreeAsString()) - finalised := gss[0].head.Hash() - - for _, gs := range gss[:1] { - require.Equal(t, finalised, gs.head.Hash()) - } -} - -func broadcastVotes(from <-chan GrandpaMessage, to []chan *networkVoteMessage, done *bool) { - for v := range from { - for _, tc := range to { - if *done { - return - } - - tc <- &networkVoteMessage{ - msg: v.(*VoteMessage), - } - } - } -} - -func cleanup(gs *Service, in chan *networkVoteMessage, done *bool) { - *done = true - close(in) - gs.cancel() -} - -func TestPlayGrandpaRound_BaseCase(t *testing.T) { - // this asserts that all validators finalise the same block if they all see the - // same pre-votes and pre-commits, even if their chains are different lengths - kr, err := keystore.NewEd25519Keyring() - require.NoError(t, err) - - gss := make([]*Service, len(kr.Keys)) - ins := make([]chan *networkVoteMessage, len(kr.Keys)) - outs := make([]chan GrandpaMessage, len(kr.Keys)) - fins := make([]chan GrandpaMessage, len(kr.Keys)) - done := false - - for i := range gss { - gs, in, out, fin := setupGrandpa(t, kr.Keys[i]) - defer cleanup(gs, in, &done) - - gss[i] = gs - ins[i] = in - outs[i] = out - fins[i] = fin - - state.AddBlocksToState(t, gs.blockState.(*state.BlockState), 4, false) - } - - for _, out := range outs { - go broadcastVotes(out, ins, &done) - } - - for _, gs := range gss { - time.Sleep(time.Millisecond * 100) - go gs.initiate() - } - - wg := sync.WaitGroup{} - wg.Add(len(kr.Keys)) - - finalised := make([]*CommitMessage, len(kr.Keys)) - - for i, fin := range fins { - go func(i int, fin <-chan GrandpaMessage) { - select { - case f := <-fin: - - // receive first message, which is finalised block from previous round - if f.(*CommitMessage).Round == 0 { - select { - case f = <-fin: - case <-time.After(testTimeout): - t.Errorf("did not receive finalised block from %d", i) - } - } - - finalised[i] = f.(*CommitMessage) - - case <-time.After(testTimeout): - t.Errorf("did not receive finalised block from %d", i) - } - wg.Done() - }(i, fin) - - } - - wg.Wait() - - for _, fb := range finalised { - require.NotNil(t, fb) - require.GreaterOrEqual(t, len(fb.Precommits), len(kr.Keys)/2) - finalised[0].Precommits = []Vote{} - finalised[0].AuthData = []AuthData{} - fb.Precommits = []Vote{} - fb.AuthData = []AuthData{} - require.Equal(t, finalised[0], fb) - } -} - -func TestPlayGrandpaRound_VaryingChain(t *testing.T) { - // this asserts that all validators finalise the same block if they all see the - // same pre-votes and pre-commits, even if their chains are different lengths (+/-1 block) - kr, err := keystore.NewEd25519Keyring() - require.NoError(t, err) - - gss := make([]*Service, len(kr.Keys)) - ins := make([]chan *networkVoteMessage, len(kr.Keys)) - outs := make([]chan GrandpaMessage, len(kr.Keys)) - fins := make([]chan GrandpaMessage, len(kr.Keys)) - done := false - - // this represents the chains that will be slightly ahead of the others - headers := []*types.Header{} - const diff uint = 1 - - for i := range gss { - gs, in, out, fin := setupGrandpa(t, kr.Keys[i]) - defer cleanup(gs, in, &done) - - gss[i] = gs - ins[i] = in - outs[i] = out - fins[i] = fin - - r := uint(rand.Intn(int(diff))) - chain, _ := state.AddBlocksToState(t, gs.blockState.(*state.BlockState), 4+r, false) - if r == diff-1 { - headers = chain - } - } - - for _, out := range outs { - go broadcastVotes(out, ins, &done) - } - - for _, gs := range gss { - time.Sleep(time.Millisecond * 100) - go gs.initiate() - } - - // mimic the chains syncing and catching up - for _, gs := range gss { - for _, h := range headers { - time.Sleep(time.Millisecond * 10) - block := &types.Block{ - Header: *h, - Body: types.Body{}, - } - gs.blockState.(*state.BlockState).AddBlock(block) - } - } - - wg := sync.WaitGroup{} - wg.Add(len(kr.Keys)) - - finalised := make([]*CommitMessage, len(kr.Keys)) - - for i, fin := range fins { - - go func(i int, fin <-chan GrandpaMessage) { - select { - case f := <-fin: - - // receive first message, which is finalised block from previous round - if f.(*CommitMessage).Round == 0 { - select { - case f = <-fin: - case <-time.After(testTimeout): - t.Errorf("did not receive finalised block from %d", i) - } - } - - finalised[i] = f.(*CommitMessage) - - case <-time.After(testTimeout): - t.Errorf("did not receive finalised block from %d", i) - } - wg.Done() - }(i, fin) - - } - - wg.Wait() - - for _, fb := range finalised { - require.NotNil(t, fb) - require.GreaterOrEqual(t, len(fb.Precommits), len(kr.Keys)/2) - require.GreaterOrEqual(t, len(fb.AuthData), len(kr.Keys)/2) - finalised[0].Precommits = []Vote{} - finalised[0].AuthData = []AuthData{} - fb.Precommits = []Vote{} - fb.AuthData = []AuthData{} - require.Equal(t, finalised[0], fb) - } -} - -func TestPlayGrandpaRound_WithEquivocation(t *testing.T) { - // this asserts that all validators finalise the same block even if 2/9 of voters equivocate - kr, err := keystore.NewEd25519Keyring() - require.NoError(t, err) - - gss := make([]*Service, len(kr.Keys)) - ins := make([]chan *networkVoteMessage, len(kr.Keys)) - outs := make([]chan GrandpaMessage, len(kr.Keys)) - fins := make([]chan GrandpaMessage, len(kr.Keys)) - - done := false - - for i := range gss { - gs, in, out, fin := setupGrandpa(t, kr.Keys[i]) - defer cleanup(gs, in, &done) - - gss[i] = gs - ins[i] = in - outs[i] = out - fins[i] = fin - - // this creates a tree with 2 branches starting at depth 2 - branches := map[uint]int{2: 1} - state.AddBlocksToStateWithFixedBranches(t, gs.blockState.(*state.BlockState), 4, branches) - } - - // should have blocktree for all nodes - leaves := gss[0].blockState.Leaves() - - for _, out := range outs { - go broadcastVotes(out, ins, &done) - } - - for _, gs := range gss { - time.Sleep(time.Millisecond * 100) - go gs.initiate() - } - - // nodes 7 and 8 will equivocate - for _, gs := range gss[7:] { - vote, err := NewVoteFromHash(leaves[1], gs.blockState) - require.NoError(t, err) - - _, vmsg, err := gs.createSignedVoteAndVoteMessage(vote, prevote) - require.NoError(t, err) - - for _, in := range ins { - in <- &networkVoteMessage{ - msg: vmsg, - } - } - } - - wg := sync.WaitGroup{} - wg.Add(len(kr.Keys)) - - finalised := make([]*CommitMessage, len(kr.Keys)) - - for i, fin := range fins { - - go func(i int, fin <-chan GrandpaMessage) { - select { - case f := <-fin: - - // receive first message, which is finalised block from previous round - if f.(*CommitMessage).Round == 0 { - - select { - case f = <-fin: - case <-time.After(testTimeout): - t.Errorf("did not receive finalised block from %d", i) - } - } - - finalised[i] = f.(*CommitMessage) - case <-time.After(testTimeout): - t.Errorf("did not receive finalised block from %d", i) - } - wg.Done() - }(i, fin) - - } - - wg.Wait() - - for _, fb := range finalised { - require.NotNil(t, fb) - require.GreaterOrEqual(t, len(fb.Precommits), len(kr.Keys)/2) - require.GreaterOrEqual(t, len(fb.AuthData), len(kr.Keys)/2) - finalised[0].Precommits = []Vote{} - finalised[0].AuthData = []AuthData{} - fb.Precommits = []Vote{} - fb.AuthData = []AuthData{} - require.Equal(t, finalised[0], fb) - } -} - -func TestPlayGrandpaRound_MultipleRounds(t *testing.T) { - // this asserts that all validators finalise the same block in successive rounds - kr, err := keystore.NewEd25519Keyring() - require.NoError(t, err) - - gss := make([]*Service, len(kr.Keys)) - ins := make([]chan *networkVoteMessage, len(kr.Keys)) - outs := make([]chan GrandpaMessage, len(kr.Keys)) - fins := make([]chan GrandpaMessage, len(kr.Keys)) - done := false - - for i := range gss { - gs, in, out, fin := setupGrandpa(t, kr.Keys[i]) - defer cleanup(gs, in, &done) - - gss[i] = gs - ins[i] = in - outs[i] = out - fins[i] = fin - - state.AddBlocksToState(t, gs.blockState.(*state.BlockState), 4, false) - } - - for _, out := range outs { - go broadcastVotes(out, ins, &done) - } - - for _, gs := range gss { - // start rounds at slightly different times to account for real-time node differences - time.Sleep(time.Millisecond * 100) - go gs.initiate() - } - - rounds := 10 - - for j := 0; j < rounds; j++ { - - wg := sync.WaitGroup{} - wg.Add(len(kr.Keys)) - - finalised := make([]*CommitMessage, len(kr.Keys)) - - for i, fin := range fins { - - go func(i int, fin <-chan GrandpaMessage) { - select { - case f := <-fin: - - // receive first message, which is finalised block from previous round - if f.(*CommitMessage).Round == uint64(j) { - select { - case f = <-fin: - case <-time.After(testTimeout): - t.Errorf("did not receive finalised block from %d", i) - } - } - - finalised[i] = f.(*CommitMessage) - case <-time.After(testTimeout): - t.Errorf("did not receive finalised block from %d", i) - } - wg.Done() - }(i, fin) - - } - - wg.Wait() - - for _, fb := range finalised { - require.NotNil(t, fb) - require.Greater(t, len(fb.Precommits), len(kr.Keys)/2) - require.Greater(t, len(fb.AuthData), len(kr.Keys)/2) - finalised[0].Precommits = []Vote{} - finalised[0].AuthData = []AuthData{} - fb.Precommits = []Vote{} - fb.AuthData = []AuthData{} - require.Equal(t, finalised[0], fb) - - if j == rounds-1 { - require.Greater(t, int(fb.Vote.Number), 4) - } - } - - chain, _ := state.AddBlocksToState(t, gss[0].blockState.(*state.BlockState), 1, false) - block := &types.Block{ - Header: *(chain[0]), - Body: types.Body{}, - } - - for _, gs := range gss[1:] { - err := gs.blockState.(*state.BlockState).AddBlock(block) - require.NoError(t, err) - } - - } -} diff --git a/lib/grandpa/types.go b/lib/grandpa/types.go index 418af8837e..27528cb6b0 100644 --- a/lib/grandpa/types.go +++ b/lib/grandpa/types.go @@ -5,6 +5,7 @@ package grandpa import ( "bytes" + "fmt" "github.com/ChainSafe/gossamer/dot/types" "github.com/ChainSafe/gossamer/lib/common" @@ -59,24 +60,13 @@ func NewState(voters []Voter, setID, round uint64) *State { // pubkeyToVoter returns a Voter given a public key func (s *State) pubkeyToVoter(pk *ed25519.PublicKey) (*Voter, error) { - max := uint64(2^64) - 1 - id := max - - for i, v := range s.voters { + for _, v := range s.voters { if bytes.Equal(pk.Encode(), v.Key.Encode()) { - id = uint64(i) - break + return &v, nil } } - if id == max { - return nil, ErrVoterNotFound - } - - return &Voter{ - Key: *pk, - ID: id, - }, nil + return nil, fmt.Errorf("%w", ErrVoterNotFound) } // threshold returns the 2/3 |voters| threshold value diff --git a/lib/grandpa/types_test.go b/lib/grandpa/types_test.go index 8487407e7b..18ae1b7c45 100644 --- a/lib/grandpa/types_test.go +++ b/lib/grandpa/types_test.go @@ -18,6 +18,8 @@ func TestPubkeyToVoter(t *testing.T) { kr, err := keystore.NewEd25519Keyring() require.NoError(t, err) + voters := newTestVoters(t) + state := NewState(voters, 0, 0) voter, err := state.pubkeyToVoter(kr.Alice().Public().(*ed25519.PublicKey)) require.NoError(t, err) diff --git a/lib/grandpa/vote_message.go b/lib/grandpa/vote_message.go index d54e605fa1..3a67d968ce 100644 --- a/lib/grandpa/vote_message.go +++ b/lib/grandpa/vote_message.go @@ -5,7 +5,6 @@ package grandpa import ( "bytes" - "context" "errors" "fmt" @@ -17,63 +16,33 @@ import ( "github.com/libp2p/go-libp2p-core/peer" ) +var errBeforeFinalizedBlock = errors.New("before latest finalized block") + type networkVoteMessage struct { from peer.ID msg *VoteMessage } -// receiveVoteMessages receives messages from the in channel until a grandpa round finishes. -func (s *Service) receiveVoteMessages(ctx context.Context) { - for { - select { - case msg, ok := <-s.in: - if !ok { - return - } - - if msg == nil || msg.msg == nil { - continue - } - - logger.Debugf("received vote message %v from %s", msg.msg, msg.from) - vm := msg.msg - - switch vm.Message.Stage { - case prevote, primaryProposal: - s.telemetry.SendMessage( - telemetry.NewAfgReceivedPrevote( - vm.Message.BlockHash, - fmt.Sprint(vm.Message.Number), - vm.Message.AuthorityID.String(), - ), - ) - case precommit: - s.telemetry.SendMessage( - telemetry.NewAfgReceivedPrecommit( - vm.Message.BlockHash, - fmt.Sprint(vm.Message.Number), - vm.Message.AuthorityID.String(), - ), - ) - default: - logger.Warnf("unsupported stage %s", vm.Message.Stage.String()) - } - - v, err := s.validateVoteMessage(msg.from, vm) - if err != nil { - logger.Debugf("failed to validate vote message %v: %s", vm, err) - continue - } - - logger.Debugf( - "validated vote message %v from %s, round %d, subround %d, "+ - "prevote count %d, precommit count %d, votes needed %d", - v, vm.Message.AuthorityID, vm.Round, vm.Message.Stage, - s.lenVotes(prevote), s.lenVotes(precommit), s.state.threshold()+1) - case <-ctx.Done(): - logger.Trace("returning from receiveMessages") - return - } +func (s *Service) sendTelemetryVoteMessage(vm *VoteMessage) { + switch vm.Message.Stage { + case prevote, primaryProposal: + s.telemetry.SendMessage( + telemetry.NewAfgReceivedPrevote( + vm.Message.BlockHash, + fmt.Sprint(vm.Message.Number), + vm.Message.AuthorityID.String(), + ), + ) + case precommit: + s.telemetry.SendMessage( + telemetry.NewAfgReceivedPrecommit( + vm.Message.BlockHash, + fmt.Sprint(vm.Message.Number), + vm.Message.AuthorityID.String(), + ), + ) + default: + logger.Warnf("unsupported stage %s", vm.Message.Stage) } } @@ -93,10 +62,11 @@ func (s *Service) createSignedVoteAndVoteMessage(vote *Vote, stage Subround) (*S return nil, nil, err } + publicKeyBytes := s.keypair.Public().(*ed25519.PublicKey).AsBytes() pc := &SignedVote{ Vote: *vote, Signature: ed25519.NewSignatureBytes(sig), - AuthorityID: s.keypair.Public().(*ed25519.PublicKey).AsBytes(), + AuthorityID: publicKeyBytes, } sm := &SignedMessage{ @@ -104,7 +74,7 @@ func (s *Service) createSignedVoteAndVoteMessage(vote *Vote, stage Subround) (*S BlockHash: pc.Vote.Hash, Number: pc.Vote.Number, Signature: ed25519.NewSignatureBytes(sig), - AuthorityID: s.keypair.Public().(*ed25519.PublicKey).AsBytes(), + AuthorityID: publicKeyBytes, } vm := &VoteMessage{ @@ -128,14 +98,14 @@ func (s *Service) validateVoteMessage(from peer.ID, m *VoteMessage) (*Vote, erro if err != nil { // TODO Affect peer reputation // https://github.com/ChainSafe/gossamer/issues/2505 - return nil, err + return nil, fmt.Errorf("creating public key: %w", err) } err = validateMessageSignature(pk, m) if err != nil { // TODO Affect peer reputation // https://github.com/ChainSafe/gossamer/issues/2505 - return nil, err + return nil, fmt.Errorf("validating message signature: %w", err) } if m.SetID != s.state.setID { @@ -156,7 +126,8 @@ func (s *Service) validateVoteMessage(from peer.ID, m *VoteMessage) (*Vote, erro // Discard message // TODO: affect peer reputation, this is shameful impolite behaviour // https://github.com/ChainSafe/gossamer/issues/2505 - return nil, nil //nolint:nilnil + return nil, fmt.Errorf("%w: received round: %d, round should be between: <%d, %d>", + errRoundOutOfBounds, m.Round, minRoundAccepted, maxRoundAccepted) } if m.Round < s.state.round { @@ -164,18 +135,19 @@ func (s *Service) validateVoteMessage(from peer.ID, m *VoteMessage) (*Vote, erro // peer doesn't know round was finalised, send out another commit message header, err := s.blockState.GetFinalisedHeader(m.Round, m.SetID) if err != nil { - return nil, err + return nil, fmt.Errorf("getting finalised header: %w", err) } - cm, err := s.newCommitMessage(header, m.Round) + // TODO: should we use `m.SetID` or `s.state.setID`? + cm, err := s.newCommitMessage(header, m.Round, s.state.setID) if err != nil { - return nil, err + return nil, fmt.Errorf("creating commit message: %w", err) } // send finalised block from previous round to network msg, err := cm.ToConsensusMessage() if err != nil { - return nil, err + return nil, fmt.Errorf("converting commit message to consensus message: %w", err) } if err = s.network.SendMessage(from, msg); err != nil { @@ -183,19 +155,21 @@ func (s *Service) validateVoteMessage(from peer.ID, m *VoteMessage) (*Vote, erro } // TODO: get justification if your round is lower, or just do catch-up? (#1815) - return nil, errRoundMismatch(m.Round, s.state.round) + return nil, fmt.Errorf("%w: received round %d but state round is %d", + errRoundsMismatch, m.Round, s.state.round) } else if m.Round > s.state.round { // Message round is higher by 1 than the round of our state, // we may be lagging behind, so store the message in the tracker // for processing later in the coming few milliseconds. s.tracker.addVote(from, m) - return nil, errRoundMismatch(m.Round, s.state.round) + return nil, fmt.Errorf("%w: received round %d but state round is %d", + errRoundsMismatch, m.Round, s.state.round) } // check for equivocation ie. multiple votes within one subround voter, err := s.state.pubkeyToVoter(pk) if err != nil { - return nil, err + return nil, fmt.Errorf("transforming public key into a voter: %w", err) } vote := NewVote(m.Message.BlockHash, m.Message.Number) @@ -214,7 +188,7 @@ func (s *Service) validateVoteMessage(from peer.ID, m *VoteMessage) (*Vote, erro s.tracker.addVote(from, m) } if err != nil { - return nil, err + return nil, fmt.Errorf("validating vote: %w", err) } just := &SignedVote{ diff --git a/lib/grandpa/vote_message_test.go b/lib/grandpa/vote_message_test.go index 0be76de1e8..257efa701e 100644 --- a/lib/grandpa/vote_message_test.go +++ b/lib/grandpa/vote_message_test.go @@ -35,7 +35,7 @@ func TestCheckForEquivocation_NoEquivocation(t *testing.T) { vote := NewVoteFromHeader(h) require.NoError(t, err) - for _, v := range voters { + for _, v := range newTestVoters(t) { equivocated := gs.checkForEquivocation(&v, &SignedVote{ Vote: *vote, }, prevote) @@ -64,6 +64,7 @@ func TestCheckForEquivocation_WithEquivocation(t *testing.T) { vote1, err := NewVoteFromHash(leaves[0], st.Block) require.NoError(t, err) + voters := newTestVoters(t) voter := voters[0] gs.prevotes.Store(voter.Key.AsBytes(), &SignedVote{ @@ -104,6 +105,7 @@ func TestCheckForEquivocation_WithExistingEquivocation(t *testing.T) { vote1, err := NewVoteFromHash(leaves[1], gs.blockState) require.NoError(t, err) + voters := newTestVoters(t) voter := voters[0] gs.prevotes.Store(voter.Key.AsBytes(), &SignedVote{ @@ -143,7 +145,7 @@ func TestValidateMessage_Valid(t *testing.T) { cfg := &Config{ BlockState: st.Block, GrandpaState: st.Grandpa, - Voters: voters, + Voters: newTestVoters(t), Network: net, Interval: time.Second, } @@ -193,8 +195,10 @@ func TestValidateMessage_InvalidSignature(t *testing.T) { msg.Message.Signature[63] = 0 + const expectedErrString = "validating message signature: signature is not valid" _, err = gs.validateVoteMessage("", msg) - require.Equal(t, err, ErrInvalidSignature) + require.ErrorIs(t, err, ErrInvalidSignature) + require.EqualError(t, err, expectedErrString) } func TestValidateMessage_SetIDMismatch(t *testing.T) { @@ -239,7 +243,7 @@ func TestValidateMessage_Equivocation(t *testing.T) { cfg := &Config{ BlockState: st.Block, GrandpaState: st.Grandpa, - Voters: voters, + Voters: newTestVoters(t), Network: net, Interval: time.Second, } @@ -256,6 +260,7 @@ func TestValidateMessage_Equivocation(t *testing.T) { voteB, err := NewVoteFromHash(leaves[1], st.Block) require.NoError(t, err) + voters := newTestVoters(t) voter := voters[0] gs.prevotes.Store(voter.Key.AsBytes(), &SignedVote{ @@ -268,7 +273,8 @@ func TestValidateMessage_Equivocation(t *testing.T) { gs.keypair = kr.Bob().(*ed25519.Keypair) _, err = gs.validateVoteMessage("", msg) - require.Equal(t, ErrEquivocation, err, gs.prevotes) + require.ErrorIs(t, err, ErrEquivocation) + require.EqualError(t, err, ErrEquivocation.Error()) } func TestValidateMessage_BlockDoesNotExist(t *testing.T) { @@ -281,7 +287,7 @@ func TestValidateMessage_BlockDoesNotExist(t *testing.T) { cfg := &Config{ BlockState: st.Block, GrandpaState: st.Grandpa, - Voters: voters, + Voters: newTestVoters(t), Network: net, Interval: time.Second, } @@ -300,8 +306,10 @@ func TestValidateMessage_BlockDoesNotExist(t *testing.T) { require.NoError(t, err) gs.keypair = kr.Bob().(*ed25519.Keypair) + const expectedErrString = "validating vote: block does not exist" _, err = gs.validateVoteMessage("", msg) - require.Equal(t, err, ErrBlockDoesNotExist) + require.ErrorIs(t, err, ErrBlockDoesNotExist) + require.EqualError(t, err, expectedErrString) } func TestValidateMessage_IsNotDescendant(t *testing.T) { @@ -314,7 +322,7 @@ func TestValidateMessage_IsNotDescendant(t *testing.T) { cfg := &Config{ BlockState: st.Block, GrandpaState: st.Grandpa, - Voters: voters, + Voters: newTestVoters(t), Network: net, Interval: time.Second, } @@ -338,6 +346,9 @@ func TestValidateMessage_IsNotDescendant(t *testing.T) { require.NoError(t, err) gs.keypair = kr.Bob().(*ed25519.Keypair) + const expectedErrString = "validating vote: block in vote is not descendant of previously finalised block" _, err = gs.validateVoteMessage("", msg) - require.Equal(t, errVoteBlockMismatch, err, gs.prevotes) + + require.ErrorIs(t, err, errVoteBlockMismatch) + require.EqualError(t, err, expectedErrString) } diff --git a/tests/rpc/rpc_03-chain_test.go b/tests/rpc/rpc_03-chain_test.go index c02b87478b..5aa8f22bcb 100644 --- a/tests/rpc/rpc_03-chain_test.go +++ b/tests/rpc/rpc_03-chain_test.go @@ -5,7 +5,6 @@ package rpc import ( "context" - "errors" "fmt" "math/rand" "testing" @@ -35,38 +34,44 @@ func TestChainRPC(t *testing.T) { tomlConfig := config.Default() tomlConfig.Init.Genesis = genesisPath tomlConfig.Core.BABELead = true + node := node.New(t, tomlConfig) ctx, cancel := context.WithCancel(context.Background()) node.InitAndStartTest(ctx, t, cancel) - // Wait for Gossamer to produce block 2 - errBlockNumberTooHigh := errors.New("block number is too high") + // Wait for Gossamer to produce block 2 or higher and finalize it const retryWaitDuration = 200 * time.Millisecond err := retry.UntilOK(ctx, retryWaitDuration, func() (ok bool, err error) { - var header modules.ChainBlockHeaderResponse - fetchWithTimeout(ctx, t, "chain_getHeader", "[]", &header) - number, err := common.HexToUint(header.Number) + // fetch the latest finalized header hash + var finalizedHead string + fetchWithTimeout(ctx, t, "chain_getFinalizedHead", "[]", &finalizedHead) + assert.Regexp(t, regex32BytesHex, finalizedHead) + + var finalizedBlock modules.ChainBlockResponse + fetchWithTimeout(ctx, t, "chain_getBlock", fmt.Sprintf(`["`+finalizedHead+`"]`), &finalizedBlock) + finalizedNumber, err := common.HexToUint(finalizedBlock.Block.Header.Number) if err != nil { return false, fmt.Errorf("cannot convert header number to uint: %w", err) } - switch number { + switch finalizedNumber { case 0, 1: return false, nil - case 2: - return true, nil default: - return false, fmt.Errorf("%w: %d", errBlockNumberTooHigh, number) + return true, nil } }) require.NoError(t, err) + // fetch the latest finalized header hash var finalizedHead string fetchWithTimeout(ctx, t, "chain_getFinalizedHead", "[]", &finalizedHead) assert.Regexp(t, regex32BytesHex, finalizedHead) - var header modules.ChainBlockHeaderResponse - fetchWithTimeout(ctx, t, "chain_getHeader", "[]", &header) + var finalizedBlock modules.ChainBlockResponse + fetchWithTimeout(ctx, t, "chain_getBlock", fmt.Sprintf(`["`+finalizedHead+`"]`), &finalizedBlock) + + header := finalizedBlock.Block.Header // Check and clear unpredictable fields assert.Regexp(t, regex32BytesHex, header.StateRoot) @@ -79,12 +84,9 @@ func TestChainRPC(t *testing.T) { } header.Digest.Logs = nil - // Assert remaining struct with predictable fields - expectedHeader := modules.ChainBlockHeaderResponse{ - ParentHash: finalizedHead, - Number: "0x02", - } - assert.Equal(t, expectedHeader, header) + blockNumber, err := common.HexToUint(header.Number) + require.NoError(t, err) + assert.GreaterOrEqual(t, blockNumber, uint(2)) var block modules.ChainBlockResponse fetchWithTimeout(ctx, t, "chain_getBlock", fmt.Sprintf(`["`+header.ParentHash+`"]`), &block) @@ -96,7 +98,7 @@ func TestChainRPC(t *testing.T) { block.Block.Header.StateRoot = "" assert.Regexp(t, regex32BytesHex, block.Block.Header.ExtrinsicsRoot) block.Block.Header.ExtrinsicsRoot = "" - assert.Len(t, block.Block.Header.Digest.Logs, 3) + assert.NotEmpty(t, block.Block.Header.Digest.Logs) for _, digestLog := range block.Block.Header.Digest.Logs { assert.Regexp(t, regexBytesHex, digestLog) } @@ -113,15 +115,9 @@ func TestChainRPC(t *testing.T) { assert.Regexp(t, bodyRegex, block.Block.Body[0]) block.Block.Body = nil - // Assert remaining struct with predictable fields - expectedBlock := modules.ChainBlockResponse{ - Block: modules.ChainBlock{ - Header: modules.ChainBlockHeaderResponse{ - Number: "0x01", - }, - }, - } - assert.Equal(t, expectedBlock, block) + blockNumber, err = common.HexToUint(header.Number) + require.NoError(t, err) + assert.GreaterOrEqual(t, blockNumber, uint(1)) var blockHash string fetchWithTimeout(ctx, t, "chain_getBlockHash", "[]", &blockHash) @@ -237,7 +233,13 @@ func TestChainSubscriptionRPC(t *testing.T) { //nolint:tparallel assertResult32BHex(t, result, "parentHash") assertResult32BHex(t, result, "stateRoot") assertResult32BHex(t, result, "extrinsicsRoot") - assertResultDigest(t, result) + + // genesis block does not contain any digest data + if number == uint(0) { + delete(result, "digest") + } else { + assertResultDigest(t, result) + } remainingExpected := subscription.Params{ Result: map[string]interface{}{}, @@ -249,7 +251,6 @@ func TestChainSubscriptionRPC(t *testing.T) { //nolint:tparallel // Check block numbers grow by zero or one in order of responses. for i, blockNumber := range blockNumbers { if i == 0 { - assert.Equal(t, uint(1), blockNumber) continue } assert.GreaterOrEqual(t, blockNumber, blockNumbers[i-1])