From ce9db35ae124a773a3b623524812c7443e3f98e7 Mon Sep 17 00:00:00 2001 From: ahrtr Date: Mon, 21 Feb 2022 13:11:39 +0800 Subject: [PATCH] add failover test cases --- tests/framework/integration/cluster.go | 8 ++ tests/integration/v3_failover_test.go | 173 +++++++++++++++++++++++++ tests/integration/v3_kv_test.go | 14 ++ 3 files changed, 195 insertions(+) create mode 100644 tests/integration/v3_failover_test.go diff --git a/tests/framework/integration/cluster.go b/tests/framework/integration/cluster.go index 49723b50c139..3b5be315f3cd 100644 --- a/tests/framework/integration/cluster.go +++ b/tests/framework/integration/cluster.go @@ -1372,6 +1372,14 @@ func (c *Cluster) Client(i int) *clientv3.Client { return c.Members[i].Client } +func (c *Cluster) Endpoints() []string { + var endpoints []string + for _, m := range c.Members { + endpoints = append(endpoints, m.GrpcURL) + } + return endpoints +} + func (c *Cluster) ClusterClient() (client *clientv3.Client, err error) { if c.clusterClient == nil { endpoints := []string{} diff --git a/tests/integration/v3_failover_test.go b/tests/integration/v3_failover_test.go new file mode 100644 index 000000000000..8fa8dfa16cce --- /dev/null +++ b/tests/integration/v3_failover_test.go @@ -0,0 +1,173 @@ +// Copyright 2022 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package integration + +import ( + "bytes" + "context" + "crypto/tls" + "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" + clientv3test "go.etcd.io/etcd/tests/v3/integration/clientv3" + "testing" + "time" + + "go.etcd.io/etcd/client/v3" + integration2 "go.etcd.io/etcd/tests/v3/framework/integration" + "google.golang.org/grpc" +) + +func TestFailover(t *testing.T) { + cases := []struct { + name string + testFunc func(*testing.T, *tls.Config, *integration2.Cluster) (*clientv3.Client, error) + }{ + { + name: "create client before the first server down", + testFunc: createClientBeforeServerDown, + }, + { + name: "create client after the first server down", + testFunc: createClientAfterServerDown, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Logf("Starting test [%s]", tc.name) + integration2.BeforeTest(t) + + // Launch an etcd cluster with 3 members + t.Logf("Launching an etcd cluster with 3 members [%s]", tc.name) + clus := integration2.NewCluster(t, &integration2.ClusterConfig{Size: 3, ClientTLS: &integration2.TestTLSInfo}) + defer clus.Terminate(t) + + cc, err := integration2.TestTLSInfo.ClientConfig() + if err != nil { + t.Fatal(err) + } + // Create an etcd client before or after first server down + t.Logf("Creating an etcd client [%s]", tc.name) + cli, err := tc.testFunc(t, cc, clus) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + defer cli.Close() + + // Sanity test + t.Logf("Running sanity test [%s]", tc.name) + key, val := "key1", "val1" + putWithRetries(t, cli, key, val, 10) + getWithRetries(t, cli, key, val, 10) + + t.Logf("Test done [%s]", tc.name) + }) + } +} + +func createClientBeforeServerDown(t *testing.T, cc *tls.Config, clus *integration2.Cluster) (*clientv3.Client, error) { + cli, err := createClient(t, cc, clus) + if err != nil { + return nil, err + } + clus.Members[0].Close() + return cli, nil +} + +func createClientAfterServerDown(t *testing.T, cc *tls.Config, clus *integration2.Cluster) (*clientv3.Client, error) { + clus.Members[0].Close() + return createClient(t, cc, clus) +} + +func createClient(t *testing.T, cc *tls.Config, clus *integration2.Cluster) (*clientv3.Client, error) { + cli, err := integration2.NewClient(t, clientv3.Config{ + Endpoints: clus.Endpoints(), + DialTimeout: 5 * time.Second, + DialOptions: []grpc.DialOption{grpc.WithBlock()}, + TLS: cc, + }) + if err != nil { + return nil, err + } + + return cli, nil +} + +func putWithRetries(t *testing.T, cli *clientv3.Client, key, val string, retryCount int) { + for retryCount > 0 { + // put data test + err := func() error { + t.Log("Sanity test, putting data") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + if _, putErr := cli.Put(ctx, key, val); putErr != nil { + t.Logf("Failed to put data (%v)", putErr) + return putErr + } + return nil + }() + + if err != nil { + retryCount-- + if shouldRetry(err) { + continue + } else { + t.Fatal(err) + } + } + break + } +} + +func getWithRetries(t *testing.T, cli *clientv3.Client, key, val string, retryCount int) { + for retryCount > 0 { + // get data test + err := func() error { + t.Log("Sanity test, getting data") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + resp, getErr := cli.Get(ctx, key) + if getErr != nil { + t.Logf("Failed to get key (%v)", getErr) + return getErr + } + if len(resp.Kvs) != 1 { + t.Fatalf("Expected 1 key, got %d", len(resp.Kvs)) + } + if !bytes.Equal([]byte(val), resp.Kvs[0].Value) { + t.Fatalf("Unexpected value, expected: %s, got: %s", val, string(resp.Kvs[0].Value)) + } + return nil + }() + + if err != nil { + retryCount-- + if shouldRetry(err) { + continue + } else { + t.Fatal(err) + } + } + break + } +} + +func shouldRetry(err error) bool { + if clientv3test.IsClientTimeout(err) || clientv3test.IsServerCtxTimeout(err) || + err == rpctypes.ErrTimeout || err == rpctypes.ErrTimeoutDueToLeaderFail { + return true + } + return false +} diff --git a/tests/integration/v3_kv_test.go b/tests/integration/v3_kv_test.go index 01a4570f51ef..d47cc75a20d1 100644 --- a/tests/integration/v3_kv_test.go +++ b/tests/integration/v3_kv_test.go @@ -1,3 +1,17 @@ +// Copyright 2022 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package integration import (