member replace e2e test

Signed-off-by: ZhouJianMS <zhoujian@microsoft.com>

tests/e2e/ctl_v3_member_no_proxy_test.go

@@ -0,0 +1,106 @@
+// Copyright 2023 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !cluster_proxy
+
+package e2e
+
+import (
+	"context"
+	"math/rand"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"go.etcd.io/etcd/etcdserver"
+	"go.etcd.io/etcd/pkg/testutil"
+)
+
+func TestMemberReplace(t *testing.T) {
+	defer testutil.AfterTest(t)
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+
+	epc, err := newEtcdProcessCluster(t, &etcdProcessClusterConfig{
+		clusterSize:      3,
+		keepDataDir:      true,
+		corruptCheckTime: time.Second,
+	})
+	require.NoError(t, err)
+	defer epc.Close()
+
+	memberIdx := rand.Int() % len(epc.procs)
+	member := epc.procs[memberIdx]
+	memberName := member.Config().name
+	var endpoints []string
+	for i := 1; i < len(epc.procs); i++ {
+		endpoints = append(endpoints, epc.procs[(memberIdx+i)%len(epc.procs)].EndpointsGRPC()...)
+	}
+	cc := NewEtcdctl(endpoints, clientNonTLS, false, false)
+
+	memberID, found, err := getMemberIdByName(ctx, cc, memberName)
+	require.NoError(t, err)
+	require.Equal(t, found, true, "Member not found")
+
+	// Need to wait health interval for cluster to accept member changes
+	time.Sleep(etcdserver.HealthInterval)
+
+	t.Logf("Removing member %s", memberName)
+	_, err = cc.MemberRemove(memberID)
+	require.NoError(t, err)
+	_, found, err = getMemberIdByName(ctx, cc, memberName)
+	require.NoError(t, err)
+	require.Equal(t, found, false, "Expected member to be removed")
+	for member.IsRunning() {
+		member.Close()
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	t.Logf("Removing member %s data", memberName)
+	err = os.RemoveAll(member.Config().dataDirPath)
+	require.NoError(t, err)
+
+	t.Logf("Adding member %s back", memberName)
+	removedMemberPeerUrl := member.Config().purl.String()
+	// 3.4 has probability that the leader election is still in progress.
+	// So we need to retry until the member is added back.
+	for {
+		_, err = cc.MemberAdd(memberName, []string{removedMemberPeerUrl})
+		if err == nil {
+			break
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	require.NoError(t, err)
+	member.Config().args = patchArgs(member.Config().args, "initial-cluster-state", "existing")
+	require.NoError(t, err)
+
+	// Sleep 100ms to bypass the known issue https://github.com/etcd-io/etcd/issues/16687.
+	time.Sleep(100 * time.Millisecond)
+	t.Logf("Starting member %s", memberName)
+	err = member.Start()
+	require.NoError(t, err)
+	executeUntil(ctx, t, func() {
+		for {
+			_, found, err := getMemberIdByName(ctx, cc, memberName)
+			if err != nil || !found {
+				time.Sleep(10 * time.Millisecond)
+				continue
+			}
+			break
+		}
+	})
+}

tests/e2e/etcdctl.go

@@ -115,6 +115,24 @@ func (ctl *Etcdctl) MemberList() (*clientv3.MemberListResponse, error) {
 	return &resp, err
 }
 
+func (ctl *Etcdctl) MemberAdd(name string, peerURLs []string) (*clientv3.MemberAddResponse, error) {
+	if ctl.v2 {
+		panic("Unsupported method for v2")
+	}
+	var resp clientv3.MemberAddResponse
+	err := ctl.spawnJsonCmd(&resp, "", "member", "add", name, "--peer-urls", strings.Join(peerURLs, ","))
+	return &resp, err
+}
+
+func (ctl *Etcdctl) MemberRemove(id uint64) (*clientv3.MemberRemoveResponse, error) {
+	if ctl.v2 {
+		panic("Unsupported method for v2")
+	}
+	var resp clientv3.MemberRemoveResponse
+	err := ctl.spawnJsonCmd(&resp, "", "member", "remove", fmt.Sprintf("%x", id))
+	return &resp, err
+}
+
 func (ctl *Etcdctl) Compact(rev int64) (*clientv3.CompactResponse, error) {
 	if ctl.v2 {
 		panic("Unsupported method for v2")

tests/e2e/utils.go

@@ -17,6 +17,7 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 
@@ -129,3 +130,29 @@ func fillEtcdWithData(ctx context.Context, c *clientv3.Client, dbSize int) error
 	}
 	return g.Wait()
 }
+
+func getMemberIdByName(ctx context.Context, c *Etcdctl, name string) (id uint64, found bool, err error) {
+	resp, err := c.MemberList()
+	if err != nil {
+		return 0, false, err
+	}
+	for _, member := range resp.Members {
+		if name == member.Name {
+			return member.ID, true, nil
+		}
+	}
+	return 0, false, nil
+}
+
+// Different implementations here since 3.4 e2e test framework does not have "initial-cluster-state" as a default argument
+// Append new flag if not exist, otherwise replace the value
+func patchArgs(args []string, flag, newValue string) []string {
+	for i, arg := range args {
+		if strings.Contains(arg, flag) {
+			args[i] = fmt.Sprintf("--%s=%s", flag, newValue)
+			return args
+		}
+	}
+	args = append(args, fmt.Sprintf("--%s=%s", flag, newValue))
+	return args
+}