Merge pull request #14611 from serathius/issue14370

tests: Add linearizability tests scenario for #14370

.github/workflows/linearizability.yaml

@@ -9,7 +9,10 @@ jobs:
       with:
         go-version: "1.19.1"
     - run: |
+        pushd tools/mod; go install go.etcd.io/gofail; popd
         mkdir -p /tmp/linearizability
+        FAILPOINTS=true make build
+        cat server/etcdserver/raft.fail.go
         EXPECT_DEBUG=true GO_TEST_FLAGS=-v RESULTS_DIR=/tmp/linearizability make test-linearizability
     - uses: actions/upload-artifact@v2
       if: always()

Makefile

@@ -30,8 +30,8 @@ test-e2e-release: build
 	PASSES="release e2e" ./scripts/test.sh $(GO_TEST_FLAGS)
 
 .PHONY: test-linearizability
-test-linearizability: build
-	PASSES="linearizability" ./scripts/test.sh $(GO_TEST_FLAGS)
+test-linearizability:
+	FAILPOINTS=true PASSES="linearizability" ./scripts/test.sh $(GO_TEST_FLAGS)
 
 .PHONY: fuzz
 fuzz: 

scripts/build_lib.sh

@@ -19,6 +19,22 @@ toggle_failpoints() {
   mode="$1"
   if command -v gofail >/dev/null 2>&1; then
     run gofail "$mode" server/etcdserver/ server/storage/backend/
+    (
+      cd ./server
+      run go get go.etcd.io/gofail/runtime
+    ) || exit 2
+    (
+      cd ./etcdutl
+      run go get go.etcd.io/gofail/runtime
+    ) || exit 2
+    (
+      cd ./etcdctl
+      run go get go.etcd.io/gofail/runtime
+    ) || exit 2
+    (
+      cd ./tests
+      run go get go.etcd.io/gofail/runtime
+    ) || exit 2
   elif [[ "$mode" != "disable" ]]; then
     log_error "FAILPOINTS set but gofail not found"
     exit 1

tests/framework/e2e/cluster.go

@@ -182,6 +182,7 @@ type EtcdProcessClusterConfig struct {
 	CorruptCheckTime        time.Duration
 	CompactHashCheckEnabled bool
 	CompactHashCheckTime    time.Duration
+	GoFailEnabled           bool
 }
 
 // NewEtcdProcessCluster launches a new cluster from etcd processes, returning
@@ -388,12 +389,21 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in
 	if cfg.CompactHashCheckTime != 0 {
 		args = append(args, "--experimental-compact-hash-check-time", cfg.CompactHashCheckTime.String())
 	}
+	envVars := map[string]string{}
+	for key, value := range cfg.EnvVars {
+		envVars[key] = value
+	}
+	var gofailPort int
+	if cfg.GoFailEnabled {
+		gofailPort = (i+1)*10000 + 2381
+		envVars["GOFAIL_HTTP"] = fmt.Sprintf("127.0.0.1:%d", gofailPort)
+	}
 
 	return &EtcdServerProcessConfig{
 		lg:           cfg.Logger,
 		ExecPath:     cfg.ExecPath,
 		Args:         args,
-		EnvVars:      cfg.EnvVars,
+		EnvVars:      envVars,
 		TlsArgs:      cfg.TlsArgs(),
 		DataDirPath:  dataDirPath,
 		KeepDataDir:  cfg.KeepDataDir,
@@ -402,6 +412,7 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in
 		Acurl:        curl,
 		Murl:         murl,
 		InitialToken: cfg.InitialToken,
+		GoFailPort:   gofailPort,
 	}
 }
 

tests/framework/e2e/etcd_process.go

@@ -80,6 +80,7 @@ type EtcdServerProcessConfig struct {
 
 	InitialToken   string
 	InitialCluster string
+	GoFailPort     int
 }
 
 func NewEtcdServerProcess(cfg *EtcdServerProcessConfig) (*EtcdServerProcess, error) {

tests/linearizability/failpoints.go

@@ -15,14 +15,20 @@
 package linearizability
 
 import (
+	"bytes"
 	"context"
+	"fmt"
 	"math/rand"
+	"net/http"
+	"net/url"
+	"time"
 
 	"go.etcd.io/etcd/tests/v3/framework/e2e"
 )
 
 var (
-	KillFailpoint Failpoint = killFailpoint{}
+	KillFailpoint       Failpoint = killFailpoint{}
+	RaftBeforeSavePanic Failpoint = goFailpoint{"etcdserver/raftBeforeSave", "panic"}
 )
 
 type Failpoint interface {
@@ -47,3 +53,51 @@ func (f killFailpoint) Trigger(ctx context.Context, clus *e2e.EtcdProcessCluster
 	}
 	return nil
 }
+
+type goFailpoint struct {
+	failpoint string
+	payload   string
+}
+
+func (f goFailpoint) Trigger(ctx context.Context, clus *e2e.EtcdProcessCluster) error {
+	member := clus.Procs[rand.Int()%len(clus.Procs)]
+	address := fmt.Sprintf("127.0.0.1:%d", member.Config().GoFailPort)
+	err := triggerGoFailpoint(address, f.failpoint, f.payload)
+	if err != nil {
+		return fmt.Errorf("failed to trigger failpoint %q, err: %v", f.failpoint, err)
+	}
+	err = clus.Procs[0].Wait()
+	if err != nil {
+		return err
+	}
+	err = clus.Procs[0].Start(ctx)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func triggerGoFailpoint(host, failpoint, payload string) error {
+	failpointUrl := url.URL{
+		Scheme: "http",
+		Host:   host,
+		Path:   failpoint,
+	}
+	r, err := http.NewRequest("PUT", failpointUrl.String(), bytes.NewBuffer([]byte(payload)))
+	if err != nil {
+		return err
+	}
+	resp, err := httpClient.Do(r)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusNoContent {
+		return fmt.Errorf("bad status code: %d", resp.StatusCode)
+	}
+	return nil
+}
+
+var httpClient = http.Client{
+	Timeout: 10 * time.Millisecond,
+}

tests/linearizability/linearizability_test.go

@@ -60,6 +60,14 @@ func TestLinearizability(t *testing.T) {
 				ClusterSize: 3,
 			},
 		},
+		{
+			name:      "Issue14370",
+			failpoint: RaftBeforeSavePanic,
+			config: e2e.EtcdProcessClusterConfig{
+				ClusterSize:   1,
+				GoFailEnabled: true,
+			},
+		},
 	}
 	for _, tc := range tcs {
 		t.Run(tc.name, func(t *testing.T) {

tools/mod/go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/mdempsky/unconvert v0.0.0-20200228143138-95ecdbfc0b5f
 	github.com/mgechev/revive v1.2.1
 	github.com/mikefarah/yq/v4 v4.24.2
+	go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0
 	go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116
 	gotest.tools/gotestsum v1.7.0
 	gotest.tools/v3 v3.1.0

tools/mod/go.sum

@@ -238,6 +238,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0 h1:TcXBU/YdVROXQ7FUowVK1ih9gu2yi3YMLE+tQb9q964=
+go.etcd.io/gofail v0.0.0-20220826035847-d0d2a96a6ef0/go.mod h1:bOzzUWJ5bNHifkNkoIN6Ydf/z/UPT0bYuPghFYVC8+4=
 go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116 h1:QQiUXlqz+d96jyNG71NE+IGTgOK6Xlhdx+PzvfbLHlQ=
 go.etcd.io/protodoc v0.0.0-20180829002748-484ab544e116/go.mod h1:F9kog+iVAuvPJucb1dkYcDcbV0g4uyGEHllTP5NrXiw=
 go.mongodb.org/mongo-driver v1.7.3 h1:G4l/eYY9VrQAK/AUgkV0koQKzQnyddnWxrd/Etf0jIs=

tools/mod/tools.go

@@ -34,6 +34,7 @@ import (
 	_ "github.com/mdempsky/unconvert"
 	_ "github.com/mgechev/revive"
 	_ "github.com/mikefarah/yq/v4"
+	_ "go.etcd.io/gofail"
 	_ "go.etcd.io/protodoc"
 	_ "gotest.tools/gotestsum"
 	_ "gotest.tools/v3"