raft: add a batch of interaction-driven conf change tests

Verifiy the behavior in various v1 and v2 conf change operations.
This also includes various fixups, notably it adds protection
against transitioning in and out of new configs when this is not
permissible.

There are more threads to pull, but those are left for future commits.

raft/interaction_test.go

@@ -22,6 +22,9 @@ import (
 )
 
 func TestInteraction(t *testing.T) {
+	// NB: if this test fails, run `go test ./raft -rewrite` and inspect the
+	// diff. Only commit the changes if you understand what caused them and if
+	// they are desired.
 	datadriven.Walk(t, "testdata", func(t *testing.T, path string) {
 		env := rafttest.NewInteractionEnv(nil)
 		datadriven.RunTest(t, path, func(d *datadriven.TestData) string {

raft/raft.go

@@ -1036,10 +1036,36 @@ func stepLeader(r *raft, m pb.Message) error {
 
 		for i := range m.Entries {
 			e := &m.Entries[i]
-			if e.Type == pb.EntryConfChange || e.Type == pb.EntryConfChangeV2 {
-				if r.pendingConfIndex > r.raftLog.applied {
-					r.logger.Infof("%x propose conf %s ignored since pending unapplied configuration [index %d, applied %d]",
-						r.id, e, r.pendingConfIndex, r.raftLog.applied)
+			var cc pb.ConfChangeI
+			if e.Type == pb.EntryConfChange {
+				var ccc pb.ConfChange
+				if err := ccc.Unmarshal(e.Data); err != nil {
+					panic(err)
+				}
+				cc = ccc
+			} else if e.Type == pb.EntryConfChangeV2 {
+				var ccc pb.ConfChangeV2
+				if err := ccc.Unmarshal(e.Data); err != nil {
+					panic(err)
+				}
+				cc = ccc
+			}
+			if cc != nil {
+				alreadyPending := r.pendingConfIndex > r.raftLog.applied
+				alreadyJoint := len(r.prs.Config.Voters[1]) > 0
+				wantsLeaveJoint := len(cc.AsV2().Changes) == 0
+
+				var refused string
+				if alreadyPending {
+					refused = fmt.Sprintf("possible unapplied conf change at index %d (applied to %d)", r.pendingConfIndex, r.raftLog.applied)
+				} else if alreadyJoint && !wantsLeaveJoint {
+					refused = "must transition out of joint config first"
+				} else if !alreadyJoint && wantsLeaveJoint {
+					refused = "not in joint state; refusing empty conf change"
+				}
+
+				if refused != "" {
+					r.logger.Infof("%x ignoring conf change %v at config %s: %s", r.id, cc, r.prs.Config, refused)
 					m.Entries[i] = pb.Entry{Type: pb.EntryNormal}
 				} else {
 					r.pendingConfIndex = r.raftLog.lastIndex() + uint64(i) + 1

raft/rafttest/interaction_env_handler_process_ready.go

@@ -19,7 +19,6 @@ import (
 
 	"github.com/cockroachdb/datadriven"
 	"go.etcd.io/etcd/raft"
-	"go.etcd.io/etcd/raft/quorum"
 	"go.etcd.io/etcd/raft/raftpb"
 )
 
@@ -50,20 +49,21 @@ func (env *InteractionEnv) ProcessReady(idx int) error {
 	}
 	for _, ent := range rd.CommittedEntries {
 		var update []byte
+		var cs *raftpb.ConfState
 		switch ent.Type {
 		case raftpb.EntryConfChange:
 			var cc raftpb.ConfChange
 			if err := cc.Unmarshal(ent.Data); err != nil {
 				return err
 			}
 			update = cc.Context
-			rn.ApplyConfChange(cc)
+			cs = rn.ApplyConfChange(cc)
 		case raftpb.EntryConfChangeV2:
 			var cc raftpb.ConfChangeV2
 			if err := cc.Unmarshal(ent.Data); err != nil {
 				return err
 			}
-			rn.ApplyConfChange(cc)
+			cs = rn.ApplyConfChange(cc)
 			update = cc.Context
 		default:
 			update = ent.Data
@@ -78,13 +78,11 @@ func (env *InteractionEnv) ProcessReady(idx int) error {
 		snap.Data = append(snap.Data, update...)
 		snap.Metadata.Index = ent.Index
 		snap.Metadata.Term = ent.Term
-		cfg := rn.Status().Config
-		snap.Metadata.ConfState = raftpb.ConfState{
-			Voters:         cfg.Voters[0].Slice(),
-			VotersOutgoing: cfg.Voters[1].Slice(),
-			Learners:       quorum.MajorityConfig(cfg.Learners).Slice(),
-			LearnersNext:   quorum.MajorityConfig(cfg.LearnersNext).Slice(),
+		if cs == nil {
+			sl := env.Nodes[idx].History
+			cs = &sl[len(sl)-1].Metadata.ConfState
 		}
+		snap.Metadata.ConfState = *cs
 		env.Nodes[idx].History = append(env.Nodes[idx].History, snap)
 	}
 	for _, msg := range rd.Messages {

raft/rafttest/interaction_env_handler_stabilize.go

@@ -67,9 +67,10 @@ func (env *InteractionEnv) Stabilize(idxs ...int) error {
 		}
 		var msgs []raftpb.Message
 		for _, rn := range nodes {
-			msgs, env.Messages = splitMsgs(env.Messages, rn.Status().ID)
+			id := rn.Status().ID
+			msgs, env.Messages = splitMsgs(env.Messages, id)
 			if len(msgs) > 0 {
-				fmt.Fprintf(env.Output, "> delivering messages\n")
+				fmt.Fprintf(env.Output, "> %d receiving messages\n", id)
 				withIndent(func() { env.DeliverMsgs(msgs) })
 				done = false
 			}

raft/testdata/campaign.txt

@@ -31,12 +31,12 @@ stabilize
   Messages:
   1->2 MsgVote Term:1 Log:1/2
   1->3 MsgVote Term:1 Log:1/2
-> delivering messages
+> 2 receiving messages
   1->2 MsgVote Term:1 Log:1/2
   INFO 2 [term: 0] received a MsgVote message with higher term from 1 [term: 1]
   INFO 2 became follower at term 1
   INFO 2 [logterm: 1, index: 2, vote: 0] cast MsgVote for 1 [logterm: 1, index: 2] at term 1
-> delivering messages
+> 3 receiving messages
   1->3 MsgVote Term:1 Log:1/2
   INFO 3 [term: 0] received a MsgVote message with higher term from 1 [term: 1]
   INFO 3 became follower at term 1
@@ -51,7 +51,7 @@ stabilize
   HardState Term:1 Vote:1 Commit:2
   Messages:
   3->1 MsgVoteResp Term:1 Log:0/0
-> delivering messages
+> 1 receiving messages
   2->1 MsgVoteResp Term:1 Log:0/0
   INFO 1 received MsgVoteResp from 2 at term 1
   INFO 1 has received 2 MsgVoteResp votes and 0 vote rejections
@@ -65,9 +65,9 @@ stabilize
   Messages:
   1->2 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""]
   1->3 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""]
-> delivering messages
+> 2 receiving messages
   1->2 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""]
-> delivering messages
+> 3 receiving messages
   1->3 MsgApp Term:1 Log:1/2 Commit:2 Entries:[1/3 EntryNormal ""]
 > 2 handling Ready
   Ready MustSync=true:
@@ -83,7 +83,7 @@ stabilize
   1/3 EntryNormal ""
   Messages:
   3->1 MsgAppResp Term:1 Log:0/3
-> delivering messages
+> 1 receiving messages
   2->1 MsgAppResp Term:1 Log:0/3
   3->1 MsgAppResp Term:1 Log:0/3
 > 1 handling Ready
@@ -94,9 +94,9 @@ stabilize
   Messages:
   1->2 MsgApp Term:1 Log:1/3 Commit:3
   1->3 MsgApp Term:1 Log:1/3 Commit:3
-> delivering messages
+> 2 receiving messages
   1->2 MsgApp Term:1 Log:1/3 Commit:3
-> delivering messages
+> 3 receiving messages
   1->3 MsgApp Term:1 Log:1/3 Commit:3
 > 2 handling Ready
   Ready MustSync=false:
@@ -112,6 +112,6 @@ stabilize
   1/3 EntryNormal ""
   Messages:
   3->1 MsgAppResp Term:1 Log:0/3
-> delivering messages
+> 1 receiving messages
   2->1 MsgAppResp Term:1 Log:0/3
   3->1 MsgAppResp Term:1 Log:0/3

raft/testdata/campaign_learner_must_vote.txt

@@ -0,0 +1,123 @@
+# Regression test that verifies that learners can vote. This holds only in the
+# sense that if a learner is asked to vote, a candidate believes that they are a
+# voter based on its current config, which may be more recent than that of the
+# learner. If learners which are actually voters but don't know it yet don't
+# vote in that situation, the raft group may end up unavailable despite a quorum
+# of voters (as of the latest config) being available.
+#
+# See:
+# https://github.com/etcd-io/etcd/pull/10998
+
+# Turn output off during boilerplate.
+log-level none
+----
+ok
+
+add-nodes 3 voters=(1,2) learners=(3) index=2
+----
+ok
+
+campaign 1
+----
+ok
+
+stabilize
+----
+ok (quiet)
+
+propose-conf-change 1
+v3
+----
+ok
+
+stabilize 1 2
+----
+ok (quiet)
+
+log-level debug
+----
+ok
+
+campaign 2
+----
+INFO 2 is starting a new election at term 1
+INFO 2 became candidate at term 2
+INFO 2 received MsgVoteResp from 2 at term 2
+INFO 2 [logterm: 1, index: 4] sent MsgVote request to 1 at term 2
+INFO 2 [logterm: 1, index: 4] sent MsgVote request to 3 at term 2
+
+# n2 is now campaigning while n1 is down (does not respond). The latest config
+# has n1 as a voter, but n1 doesn't even have the corresponding conf change in
+# its log. Still, it casts a vote for n2 which can in turn become leader and
+# catches up n3.
+stabilize 2 3
+----
+> 2 handling Ready
+  Ready MustSync=true:
+  Lead:0 State:StateCandidate
+  HardState Term:2 Vote:2 Commit:4
+  Messages:
+  2->1 MsgVote Term:2 Log:1/4
+  2->3 MsgVote Term:2 Log:1/4
+> 3 receiving messages
+  1->3 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChangeV2 v3]
+  1->3 MsgApp Term:1 Log:1/4 Commit:4
+  2->3 MsgVote Term:2 Log:1/4
+  INFO 3 [term: 1] received a MsgVote message with higher term from 2 [term: 2]
+  INFO 3 became follower at term 2
+  INFO 3 [logterm: 1, index: 4, vote: 0] cast MsgVote for 2 [logterm: 1, index: 4] at term 2
+> 3 handling Ready
+  INFO 3 switched to configuration voters=(1 2 3)
+  Ready MustSync=true:
+  Lead:0 State:StateFollower
+  HardState Term:2 Vote:2 Commit:4
+  Entries:
+  1/4 EntryConfChangeV2 v3
+  CommittedEntries:
+  1/4 EntryConfChangeV2 v3
+  Messages:
+  3->1 MsgAppResp Term:1 Log:0/4
+  3->1 MsgAppResp Term:1 Log:0/4
+  3->2 MsgVoteResp Term:2 Log:0/0
+> 2 receiving messages
+  3->2 MsgVoteResp Term:2 Log:0/0
+  INFO 2 received MsgVoteResp from 3 at term 2
+  INFO 2 has received 2 MsgVoteResp votes and 0 vote rejections
+  INFO 2 became leader at term 2
+> 2 handling Ready
+  Ready MustSync=true:
+  Lead:2 State:StateLeader
+  Entries:
+  2/5 EntryNormal ""
+  Messages:
+  2->1 MsgApp Term:2 Log:1/4 Commit:4 Entries:[2/5 EntryNormal ""]
+  2->3 MsgApp Term:2 Log:1/4 Commit:4 Entries:[2/5 EntryNormal ""]
+> 3 receiving messages
+  2->3 MsgApp Term:2 Log:1/4 Commit:4 Entries:[2/5 EntryNormal ""]
+> 3 handling Ready
+  Ready MustSync=true:
+  Lead:2 State:StateFollower
+  Entries:
+  2/5 EntryNormal ""
+  Messages:
+  3->2 MsgAppResp Term:2 Log:0/5
+> 2 receiving messages
+  3->2 MsgAppResp Term:2 Log:0/5
+> 2 handling Ready
+  Ready MustSync=false:
+  HardState Term:2 Vote:2 Commit:5
+  CommittedEntries:
+  2/5 EntryNormal ""
+  Messages:
+  2->3 MsgApp Term:2 Log:2/5 Commit:5
+> 3 receiving messages
+  2->3 MsgApp Term:2 Log:2/5 Commit:5
+> 3 handling Ready
+  Ready MustSync=false:
+  HardState Term:2 Vote:2 Commit:5
+  CommittedEntries:
+  2/5 EntryNormal ""
+  Messages:
+  3->2 MsgAppResp Term:2 Log:0/5
+> 2 receiving messages
+  3->2 MsgAppResp Term:2 Log:0/5