diff --git a/raft/raft.go b/raft/raft.go index 02f493a8d89a..54a8192cebe6 100644 --- a/raft/raft.go +++ b/raft/raft.go @@ -1036,10 +1036,36 @@ func stepLeader(r *raft, m pb.Message) error { for i := range m.Entries { e := &m.Entries[i] - if e.Type == pb.EntryConfChange || e.Type == pb.EntryConfChangeV2 { - if r.pendingConfIndex > r.raftLog.applied { - r.logger.Infof("%x propose conf %s ignored since pending unapplied configuration [index %d, applied %d]", - r.id, e, r.pendingConfIndex, r.raftLog.applied) + var cc pb.ConfChangeI + if e.Type == pb.EntryConfChange { + var ccc pb.ConfChange + if err := ccc.Unmarshal(e.Data); err != nil { + panic(err) + } + cc = ccc + } else if e.Type == pb.EntryConfChangeV2 { + var ccc pb.ConfChangeV2 + if err := ccc.Unmarshal(e.Data); err != nil { + panic(err) + } + cc = ccc + } + if cc != nil { + alreadyPending := r.pendingConfIndex > r.raftLog.applied + alreadyJoint := len(r.prs.Config.Voters[1]) > 0 + wantsLeaveJoint := len(cc.AsV2().Changes) == 0 + + var refused string + if alreadyPending { + refused = fmt.Sprintf("possible unapplied conf change at index %d (applied to %d)", r.pendingConfIndex, r.raftLog.applied) + } else if alreadyJoint && !wantsLeaveJoint { + refused = "must transition out of joint config first" + } else if !alreadyJoint && wantsLeaveJoint { + refused = "not in joint state; refusing empty conf change" + } + + if refused != "" { + r.logger.Infof("%x ignoring conf change %v at config %s: %s", r.id, cc, r.prs.Config, refused) m.Entries[i] = pb.Entry{Type: pb.EntryNormal} } else { r.pendingConfIndex = r.raftLog.lastIndex() + uint64(i) + 1 diff --git a/raft/rafttest/interaction_env_handler_process_ready.go b/raft/rafttest/interaction_env_handler_process_ready.go index ff5cf2bd9da6..1006433c493a 100644 --- a/raft/rafttest/interaction_env_handler_process_ready.go +++ b/raft/rafttest/interaction_env_handler_process_ready.go @@ -19,7 +19,6 @@ import ( "github.com/cockroachdb/datadriven" "go.etcd.io/etcd/raft" - "go.etcd.io/etcd/raft/quorum" "go.etcd.io/etcd/raft/raftpb" ) @@ -50,6 +49,7 @@ func (env *InteractionEnv) ProcessReady(idx int) error { } for _, ent := range rd.CommittedEntries { var update []byte + var cs *raftpb.ConfState switch ent.Type { case raftpb.EntryConfChange: var cc raftpb.ConfChange @@ -57,13 +57,13 @@ func (env *InteractionEnv) ProcessReady(idx int) error { return err } update = cc.Context - rn.ApplyConfChange(cc) + cs = rn.ApplyConfChange(cc) case raftpb.EntryConfChangeV2: var cc raftpb.ConfChangeV2 if err := cc.Unmarshal(ent.Data); err != nil { return err } - rn.ApplyConfChange(cc) + cs = rn.ApplyConfChange(cc) update = cc.Context default: update = ent.Data @@ -78,13 +78,11 @@ func (env *InteractionEnv) ProcessReady(idx int) error { snap.Data = append(snap.Data, update...) snap.Metadata.Index = ent.Index snap.Metadata.Term = ent.Term - cfg := rn.Status().Config - snap.Metadata.ConfState = raftpb.ConfState{ - Voters: cfg.Voters[0].Slice(), - VotersOutgoing: cfg.Voters[1].Slice(), - Learners: quorum.MajorityConfig(cfg.Learners).Slice(), - LearnersNext: quorum.MajorityConfig(cfg.LearnersNext).Slice(), + if cs == nil { + sl := env.Nodes[idx].History + cs = &sl[len(sl)-1].Metadata.ConfState } + snap.Metadata.ConfState = *cs env.Nodes[idx].History = append(env.Nodes[idx].History, snap) } for _, msg := range rd.Messages { diff --git a/raft/testdata/confchange_v1_add_single.txt b/raft/testdata/confchange_v1_add_single.txt new file mode 100644 index 000000000000..d38664b02ffe --- /dev/null +++ b/raft/testdata/confchange_v1_add_single.txt @@ -0,0 +1,97 @@ +# Run a V1 membership change that adds a single voter. + +# Bootstrap n1. +add-nodes 1 voters=(1) index=2 +---- +INFO 1 switched to configuration voters=(1) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 +INFO 1 received MsgVoteResp from 1 at term 1 +INFO 1 became leader at term 1 + +# Add v2 (with an auto transition). +propose-conf-change 1 v1=true +v2 +---- +ok + +# Pull n2 out of thin air. +add-nodes 1 +---- +INFO 2 switched to configuration voters=() +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] + +# n1 commits the conf change using itself as commit quorum, immediately transitions into +# the final config, and catches up n2. Note that it's using an EntryConfChange, not an +# EntryConfChangeV2, so this is compatible with nodes that don't know about V2 conf changes. +stabilize +---- +> 1 handling Ready + INFO 1 switched to configuration voters=(1 2) + Ready MustSync=true: + Lead:1 State:StateLeader + HardState Term:1 Vote:1 Commit:4 + Entries: + 1/3 EntryNormal "" + 1/4 EntryConfChange v2 + CommittedEntries: + 1/3 EntryNormal "" + 1/4 EntryConfChange v2 +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChange v2] +> delivering messages + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChange v2] + INFO 2 [term: 0] received a MsgApp message with higher term from 1 [term: 1] + INFO 2 became follower at term 1 + DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 +> 2 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + HardState Term:1 Commit:0 + Messages: + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) + DEBUG 1 received MsgAppResp(MsgApp was rejected, lastindex: 0) from 2 for index 3 + DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=1 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false +> delivering messages + 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + INFO log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 4, term: 1] + INFO 2 switched to configuration voters=(1 2) + INFO 2 [commit: 4, lastindex: 4, lastterm: 1] restored snapshot [index: 4, term: 1] + INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:4 + Snapshot Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/4 + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:1 Log:1/4 Commit:4 +> delivering messages + 1->2 MsgApp Term:1 Log:1/4 Commit:4 +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/4 diff --git a/raft/testdata/confchange_v1_remove_leader.txt b/raft/testdata/confchange_v1_remove_leader.txt new file mode 100644 index 000000000000..1664dec3f340 --- /dev/null +++ b/raft/testdata/confchange_v1_remove_leader.txt @@ -0,0 +1,103 @@ +# We'll turn this back on after the boilerplate. +log-level none +---- +ok + +# Run a V1 membership change that removes the leader. +# Bootstrap n1, n2, n3. +add-nodes 3 voters=(1,2,3) index=2 +---- +ok + +campaign 1 +---- +ok + +stabilize +---- +ok (quiet) + +log-level debug +---- +ok + +# Remove n1. +propose-conf-change 1 v1=true +r1 +---- +ok + +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/4 EntryConfChange r1 + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChange r1] + 1->3 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChange r1] +> delivering messages + 1->2 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChange r1] +> delivering messages + 1->3 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChange r1] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/4 EntryConfChange r1 + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> 3 handling Ready + Ready MustSync=true: + Entries: + 1/4 EntryConfChange r1 + Messages: + 3->1 MsgAppResp Term:1 Log:0/4 +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/4 + 3->1 MsgAppResp Term:1 Log:0/4 +> 1 handling Ready + INFO 1 switched to configuration voters=(2 3) + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:4 + CommittedEntries: + 1/4 EntryConfChange r1 + Messages: + 1->2 MsgApp Term:1 Log:1/4 Commit:4 + 1->3 MsgApp Term:1 Log:1/4 Commit:4 +> delivering messages + 1->2 MsgApp Term:1 Log:1/4 Commit:4 +> delivering messages + 1->3 MsgApp Term:1 Log:1/4 Commit:4 +> 2 handling Ready + INFO 2 switched to configuration voters=(2 3) + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:4 + CommittedEntries: + 1/4 EntryConfChange r1 + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> 3 handling Ready + INFO 3 switched to configuration voters=(2 3) + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:4 + CommittedEntries: + 1/4 EntryConfChange r1 + Messages: + 3->1 MsgAppResp Term:1 Log:0/4 +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/4 + 3->1 MsgAppResp Term:1 Log:0/4 + +status 1 +---- +2: StateReplicate match=4 next=5 +3: StateReplicate match=4 next=5 + +# TODO(tbg): the leader now drops any proposals, but if it has any other +# uncommitted proposals in its log already, it will likely try to distribute +# them which could be buggy. Test that. +propose-conf-change 1 v1=true +v1 +---- +raft proposal dropped + diff --git a/raft/testdata/confchange_upreplicate_1_to_3_v2_auto.txt b/raft/testdata/confchange_v2_add_double_auto.txt similarity index 91% rename from raft/testdata/confchange_upreplicate_1_to_3_v2_auto.txt rename to raft/testdata/confchange_v2_add_double_auto.txt index 4714bfceec13..718d4ece1bb8 100644 --- a/raft/testdata/confchange_upreplicate_1_to_3_v2_auto.txt +++ b/raft/testdata/confchange_v2_add_double_auto.txt @@ -15,7 +15,7 @@ INFO 1 became candidate at term 1 INFO 1 received MsgVoteResp from 1 at term 1 INFO 1 became leader at term 1 -propose-conf-change 1 +propose-conf-change 1 transition=auto v2 v3 ---- ok @@ -32,7 +32,9 @@ INFO newRaft 3 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastter # n1 immediately gets to commit & apply the conf change using only itself. We see that # it starts transitioning out of that joint configuration (though we will only see that -# proposal in the next ready handling loop, when it is emitted). +# proposal in the next ready handling loop, when it is emitted). We also see that this +# is using joint consensus, which it has to since we're carrying out two additions at +# once. process-ready 1 ---- INFO 1 switched to configuration voters=(1 2 3)&&(1) autoleave @@ -85,17 +87,17 @@ stabilize 1 2 > 1 handling Ready Ready MustSync=false: Messages: - 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[1] Learners:[] LearnersNext:[] + 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true > delivering messages - 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[1] Learners:[] LearnersNext:[] + 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true INFO log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 4, term: 1] - INFO 2 switched to configuration voters=(1 2 3)&&(1) + INFO 2 switched to configuration voters=(1 2 3)&&(1) autoleave INFO 2 [commit: 4, lastindex: 4, lastterm: 1] restored snapshot [index: 4, term: 1] INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] > 2 handling Ready Ready MustSync=false: HardState Term:1 Commit:4 - Snapshot Index:4 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[1] Learners:[] LearnersNext:[] + Snapshot Index:4 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true Messages: 2->1 MsgAppResp Term:1 Log:0/4 > delivering messages @@ -159,9 +161,9 @@ stabilize 1 3 > 1 handling Ready Ready MustSync=false: Messages: - 1->3 MsgSnap Term:1 Log:0/0 Snapshot: Index:5 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] + 1->3 MsgSnap Term:1 Log:0/0 Snapshot: Index:5 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false > delivering messages - 1->3 MsgSnap Term:1 Log:0/0 Snapshot: Index:5 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] + 1->3 MsgSnap Term:1 Log:0/0 Snapshot: Index:5 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false INFO log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 5, term: 1] INFO 3 switched to configuration voters=(1 2 3) INFO 3 [commit: 5, lastindex: 5, lastterm: 1] restored snapshot [index: 5, term: 1] @@ -169,7 +171,7 @@ stabilize 1 3 > 3 handling Ready Ready MustSync=false: HardState Term:1 Commit:5 - Snapshot Index:5 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] + Snapshot Index:5 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false Messages: 3->1 MsgAppResp Term:1 Log:0/5 > delivering messages diff --git a/raft/testdata/confchange_v2_add_double_explicit.txt b/raft/testdata/confchange_v2_add_double_explicit.txt new file mode 100644 index 000000000000..23b4e843931d --- /dev/null +++ b/raft/testdata/confchange_v2_add_double_explicit.txt @@ -0,0 +1,206 @@ +# Run a V2 membership change that adds a single voter but explicitly asks for the +# use of joint consensus, including wanting to transition out of the joint config +# manually. + +# Bootstrap n1. +add-nodes 1 voters=(1) index=2 +---- +INFO 1 switched to configuration voters=(1) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 +INFO 1 received MsgVoteResp from 1 at term 1 +INFO 1 became leader at term 1 + +# Add v2 with an explicit transition. +propose-conf-change 1 transition=explicit +v2 +---- +ok + +# Pull n2 out of thin air. +add-nodes 1 +---- +INFO 2 switched to configuration voters=() +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] + +# n1 commits the conf change using itself as commit quorum, then starts catching up n2. +# Everyone remains in the joint config. Note that the snapshot below has AutoLeave unset. +stabilize 1 2 +---- +> 1 handling Ready + INFO 1 switched to configuration voters=(1 2)&&(1) + Ready MustSync=true: + Lead:1 State:StateLeader + HardState Term:1 Vote:1 Commit:4 + Entries: + 1/3 EntryNormal "" + 1/4 EntryConfChangeV2 v2 + CommittedEntries: + 1/3 EntryNormal "" + 1/4 EntryConfChangeV2 v2 +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2] +> delivering messages + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2] + INFO 2 [term: 0] received a MsgApp message with higher term from 1 [term: 1] + INFO 2 became follower at term 1 + DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 +> 2 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + HardState Term:1 Commit:0 + Messages: + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) + DEBUG 1 received MsgAppResp(MsgApp was rejected, lastindex: 0) from 2 for index 3 + DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=1 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:false +> delivering messages + 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:false + INFO log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 4, term: 1] + INFO 2 switched to configuration voters=(1 2)&&(1) + INFO 2 [commit: 4, lastindex: 4, lastterm: 1] restored snapshot [index: 4, term: 1] + INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:4 + Snapshot Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:false + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/4 + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:1 Log:1/4 Commit:4 +> delivering messages + 1->2 MsgApp Term:1 Log:1/4 Commit:4 +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/4 + +# Check that we're not allowed to change membership again while in the joint state. +# This leads to an empty entry being proposed instead (index 5 in the stabilize block +# below). +propose-conf-change 1 +v3 v4 v5 +---- +INFO 1 ignoring conf change {ConfChangeTransitionAuto [{ConfChangeAddNode 3 []} {ConfChangeAddNode 4 []} {ConfChangeAddNode 5 []}] [] []} at config voters=(1 2)&&(1): must transition out of joint config first + +# Propose a transition out of the joint config. We'll see this at index 6 below. +propose-conf-change 1 +---- +ok + +# The group commits the command and everyone switches to the final config. +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryNormal "" + 1/6 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryNormal ""] + 1->2 MsgApp Term:1 Log:1/5 Commit:4 Entries:[1/6 EntryConfChangeV2] +> delivering messages + 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryNormal ""] + 1->2 MsgApp Term:1 Log:1/5 Commit:4 Entries:[1/6 EntryConfChangeV2] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryNormal "" + 1/6 EntryConfChangeV2 + Messages: + 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/6 +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/6 +> 1 handling Ready + INFO 1 switched to configuration voters=(1 2) + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:6 + CommittedEntries: + 1/5 EntryNormal "" + 1/6 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/6 Commit:5 + 1->2 MsgApp Term:1 Log:1/6 Commit:6 +> delivering messages + 1->2 MsgApp Term:1 Log:1/6 Commit:5 + 1->2 MsgApp Term:1 Log:1/6 Commit:6 +> 2 handling Ready + INFO 2 switched to configuration voters=(1 2) + Ready MustSync=false: + HardState Term:1 Commit:6 + CommittedEntries: + 1/5 EntryNormal "" + 1/6 EntryConfChangeV2 + Messages: + 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 + +# Check that trying to transition out again won't do anything. +propose-conf-change 1 +---- +INFO 1 ignoring conf change {ConfChangeTransitionAuto [] [] []} at config voters=(1 2): not in joint state; refusing empty conf change + +# Finishes work for the empty entry we just proposed. +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/7 EntryNormal "" + Messages: + 1->2 MsgApp Term:1 Log:1/6 Commit:6 Entries:[1/7 EntryNormal ""] +> delivering messages + 1->2 MsgApp Term:1 Log:1/6 Commit:6 Entries:[1/7 EntryNormal ""] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/7 EntryNormal "" + Messages: + 2->1 MsgAppResp Term:1 Log:0/7 +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/7 +> 1 handling Ready + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:7 + CommittedEntries: + 1/7 EntryNormal "" + Messages: + 1->2 MsgApp Term:1 Log:1/7 Commit:7 +> delivering messages + 1->2 MsgApp Term:1 Log:1/7 Commit:7 +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:7 + CommittedEntries: + 1/7 EntryNormal "" + Messages: + 2->1 MsgAppResp Term:1 Log:0/7 +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/7 diff --git a/raft/testdata/confchange_v2_add_double_implicit.txt b/raft/testdata/confchange_v2_add_double_implicit.txt new file mode 100644 index 000000000000..6d28dfcb95c1 --- /dev/null +++ b/raft/testdata/confchange_v2_add_double_implicit.txt @@ -0,0 +1,125 @@ +# Run a V2 membership change that adds a single voter but explicitly asks for the +# use of joint consensus (with auto-leaving). + +# TODO(tbg): also verify that if the leader changes while in the joint state, the +# new leader will auto-transition out of the joint state just the same. + +# Bootstrap n1. +add-nodes 1 voters=(1) index=2 +---- +INFO 1 switched to configuration voters=(1) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 +INFO 1 received MsgVoteResp from 1 at term 1 +INFO 1 became leader at term 1 + +propose-conf-change 1 transition=implicit +v2 +---- +ok + +# Add n2. +add-nodes 1 +---- +INFO 2 switched to configuration voters=() +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] + +# n1 commits the conf change using itself as commit quorum, then starts catching up n2. +# When that's done, it starts auto-transitioning out. Note that the snapshots propagating +# the joint config have the AutoLeave flag set in their config. +stabilize 1 2 +---- +> 1 handling Ready + INFO 1 switched to configuration voters=(1 2)&&(1) autoleave + INFO initiating automatic transition out of joint configuration voters=(1 2)&&(1) autoleave + Ready MustSync=true: + Lead:1 State:StateLeader + HardState Term:1 Vote:1 Commit:4 + Entries: + 1/3 EntryNormal "" + 1/4 EntryConfChangeV2 v2 + CommittedEntries: + 1/3 EntryNormal "" + 1/4 EntryConfChangeV2 v2 +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2] +> delivering messages + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2] + INFO 2 [term: 0] received a MsgApp message with higher term from 1 [term: 1] + INFO 2 became follower at term 1 + DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 +> 2 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + HardState Term:1 Commit:0 + Messages: + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) + DEBUG 1 received MsgAppResp(MsgApp was rejected, lastindex: 0) from 2 for index 3 + DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=1 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true +> delivering messages + 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true + INFO log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 4, term: 1] + INFO 2 switched to configuration voters=(1 2)&&(1) autoleave + INFO 2 [commit: 4, lastindex: 4, lastterm: 1] restored snapshot [index: 4, term: 1] + INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:4 + Snapshot Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/4 + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2] +> delivering messages + 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryConfChangeV2 + Messages: + 2->1 MsgAppResp Term:1 Log:0/5 +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/5 +> 1 handling Ready + INFO 1 switched to configuration voters=(1 2) + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:5 + CommittedEntries: + 1/5 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/5 Commit:5 +> delivering messages + 1->2 MsgApp Term:1 Log:1/5 Commit:5 +> 2 handling Ready + INFO 2 switched to configuration voters=(1 2) + Ready MustSync=false: + HardState Term:1 Commit:5 + CommittedEntries: + 1/5 EntryConfChangeV2 + Messages: + 2->1 MsgAppResp Term:1 Log:0/5 +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/5 diff --git a/raft/testdata/confchange_v2_add_single_auto.txt b/raft/testdata/confchange_v2_add_single_auto.txt new file mode 100644 index 000000000000..6c3499590460 --- /dev/null +++ b/raft/testdata/confchange_v2_add_single_auto.txt @@ -0,0 +1,198 @@ +# Run a V2 membership change that adds a single voter in auto mode, which means +# that joint consensus is not used but a direct transition into the new config +# takes place. + +# Bootstrap n1. +add-nodes 1 voters=(1) index=2 +---- +INFO 1 switched to configuration voters=(1) +INFO 1 became follower at term 0 +INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1] + +campaign 1 +---- +INFO 1 is starting a new election at term 0 +INFO 1 became candidate at term 1 +INFO 1 received MsgVoteResp from 1 at term 1 +INFO 1 became leader at term 1 + +# Add v2 (with an auto transition). +propose-conf-change 1 +v2 +---- +ok + +# Pull n2 out of thin air. +add-nodes 1 +---- +INFO 2 switched to configuration voters=() +INFO 2 became follower at term 0 +INFO newRaft 2 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] + +# n1 commits the conf change using itself as commit quorum, immediately transitions into +# the final config, and catches up n2. +stabilize +---- +> 1 handling Ready + INFO 1 switched to configuration voters=(1 2) + Ready MustSync=true: + Lead:1 State:StateLeader + HardState Term:1 Vote:1 Commit:4 + Entries: + 1/3 EntryNormal "" + 1/4 EntryConfChangeV2 v2 + CommittedEntries: + 1/3 EntryNormal "" + 1/4 EntryConfChangeV2 v2 +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2] +> delivering messages + 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2] + INFO 2 [term: 0] received a MsgApp message with higher term from 1 [term: 1] + INFO 2 became follower at term 1 + DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1 +> 2 handling Ready + Ready MustSync=true: + Lead:1 State:StateFollower + HardState Term:1 Commit:0 + Messages: + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0) + DEBUG 1 received MsgAppResp(MsgApp was rejected, lastindex: 0) from 2 for index 3 + DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1] + DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1] + DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=1 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false +> delivering messages + 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + INFO log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 4, term: 1] + INFO 2 switched to configuration voters=(1 2) + INFO 2 [commit: 4, lastindex: 4, lastterm: 1] restored snapshot [index: 4, term: 1] + INFO 2 [commit: 4] restored snapshot [index: 4, term: 1] +> 2 handling Ready + Ready MustSync=false: + HardState Term:1 Commit:4 + Snapshot Index:4 Term:1 ConfState:Voters:[1 2] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/4 + DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4] +> 1 handling Ready + Ready MustSync=false: + Messages: + 1->2 MsgApp Term:1 Log:1/4 Commit:4 +> delivering messages + 1->2 MsgApp Term:1 Log:1/4 Commit:4 +> 2 handling Ready + Ready MustSync=false: + Messages: + 2->1 MsgAppResp Term:1 Log:0/4 +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/4 + +# Check that we're not allowed to change membership again while in the joint state. +# This leads to an empty entry being proposed instead (index 5 in the stabilize block +# below). +propose-conf-change 1 +v3 v4 v5 +---- +ok + +# Propose a transition out of the joint config. We'll see this at index 6 below. +propose-conf-change 1 +---- +INFO 1 ignoring conf change {ConfChangeTransitionAuto [] [] []} at config voters=(1 2): possible unapplied conf change at index 5 (applied to 4) + +# The group commits the command and everyone switches to the final config. +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryConfChangeV2 v3 v4 v5 + 1/6 EntryNormal "" + Messages: + 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2 v3 v4 v5] + 1->2 MsgApp Term:1 Log:1/5 Commit:4 Entries:[1/6 EntryNormal ""] +> delivering messages + 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2 v3 v4 v5] + 1->2 MsgApp Term:1 Log:1/5 Commit:4 Entries:[1/6 EntryNormal ""] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/5 EntryConfChangeV2 v3 v4 v5 + 1/6 EntryNormal "" + Messages: + 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/6 +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/5 + 2->1 MsgAppResp Term:1 Log:0/6 +> 1 handling Ready + INFO 1 switched to configuration voters=(1 2 3 4 5)&&(1 2) autoleave + INFO initiating automatic transition out of joint configuration voters=(1 2 3 4 5)&&(1 2) autoleave + Ready MustSync=false: + HardState Term:1 Vote:1 Commit:6 + CommittedEntries: + 1/5 EntryConfChangeV2 v3 v4 v5 + 1/6 EntryNormal "" + Messages: + 1->2 MsgApp Term:1 Log:1/6 Commit:5 + 1->2 MsgApp Term:1 Log:1/6 Commit:6 +> delivering messages + 1->2 MsgApp Term:1 Log:1/6 Commit:5 + 1->2 MsgApp Term:1 Log:1/6 Commit:6 +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/7 EntryConfChangeV2 + Messages: + 1->3 MsgApp Term:1 Log:1/5 Commit:6 Entries:[1/6 EntryNormal ""] + 1->4 MsgApp Term:1 Log:1/5 Commit:6 Entries:[1/6 EntryNormal ""] + 1->5 MsgApp Term:1 Log:1/5 Commit:6 Entries:[1/6 EntryNormal ""] +> 2 handling Ready + INFO 2 switched to configuration voters=(1 2 3 4 5)&&(1 2) autoleave + Ready MustSync=false: + HardState Term:1 Commit:6 + CommittedEntries: + 1/5 EntryConfChangeV2 v3 v4 v5 + 1/6 EntryNormal "" + Messages: + 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/6 + 2->1 MsgAppResp Term:1 Log:0/6 + +# Check that trying to transition out again won't do anything. +propose-conf-change 1 +---- +ok + +# Finishes work for the empty entry we just proposed. +stabilize +---- +> 1 handling Ready + Ready MustSync=true: + Entries: + 1/8 EntryConfChangeV2 + Messages: + 1->2 MsgApp Term:1 Log:1/6 Commit:6 Entries:[1/7 EntryConfChangeV2, 1/8 EntryConfChangeV2] +> delivering messages + 1->2 MsgApp Term:1 Log:1/6 Commit:6 Entries:[1/7 EntryConfChangeV2, 1/8 EntryConfChangeV2] +> 2 handling Ready + Ready MustSync=true: + Entries: + 1/7 EntryConfChangeV2 + 1/8 EntryConfChangeV2 + Messages: + 2->1 MsgAppResp Term:1 Log:0/8 +> delivering messages + 2->1 MsgAppResp Term:1 Log:0/8 diff --git a/raft/testdata/snapshot_succeed_via_app_resp.txt b/raft/testdata/snapshot_succeed_via_app_resp.txt index c7ddfec41cde..e5ddc469dcd3 100644 --- a/raft/testdata/snapshot_succeed_via_app_resp.txt +++ b/raft/testdata/snapshot_succeed_via_app_resp.txt @@ -91,7 +91,7 @@ stabilize 1 > 1 handling Ready Ready MustSync=false: Messages: - 1->3 MsgSnap Term:1 Log:0/0 Snapshot: Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] + 1->3 MsgSnap Term:1 Log:0/0 Snapshot: Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false status 1 ---- @@ -106,7 +106,7 @@ status 1 stabilize 3 ---- > delivering messages - 1->3 MsgSnap Term:1 Log:0/0 Snapshot: Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] + 1->3 MsgSnap Term:1 Log:0/0 Snapshot: Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false INFO log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 11, term: 1] INFO 3 switched to configuration voters=(1 2 3) INFO 3 [commit: 11, lastindex: 11, lastterm: 1] restored snapshot [index: 11, term: 1] @@ -114,7 +114,7 @@ stabilize 3 > 3 handling Ready Ready MustSync=false: HardState Term:1 Commit:11 - Snapshot Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] + Snapshot Index:11 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false Messages: 3->1 MsgAppResp Term:1 Log:0/11 diff --git a/raft/util.go b/raft/util.go index 881a6e14e241..785cf735d5db 100644 --- a/raft/util.go +++ b/raft/util.go @@ -77,8 +77,8 @@ func DescribeSoftState(ss SoftState) string { func DescribeConfState(state pb.ConfState) string { return fmt.Sprintf( - "Voters:%v VotersOutgoing:%v Learners:%v LearnersNext:%v", - state.Voters, state.VotersOutgoing, state.Learners, state.LearnersNext, + "Voters:%v VotersOutgoing:%v Learners:%v LearnersNext:%v AutoLeave:%v", + state.Voters, state.VotersOutgoing, state.Learners, state.LearnersNext, state.AutoLeave, ) }