Linearizability issue14890

.github/workflows/linearizability.yaml

@@ -13,7 +13,7 @@ jobs:
         make build
         mkdir -p /tmp/linearizability
         cat server/etcdserver/raft.fail.go
-        EXPECT_DEBUG=true GO_TEST_FLAGS='-v --count 60 --failfast --run TestLinearizability' RESULTS_DIR=/tmp/linearizability make test-linearizability
+        EXPECT_DEBUG=true GO_TEST_FLAGS='-v --count 1 --failfast --run TestLinearizability' RESULTS_DIR=/tmp/linearizability make test-linearizability
     - uses: actions/upload-artifact@v2
       if: always()
       with:

client/v2/keys.go

@@ -147,7 +147,7 @@ type WatcherOptions struct {
 
 type CreateInOrderOptions struct {
 	// TTL defines a period of time after-which the Node should
-	// expire and no longer exist. Values <= 0 are ignored. Given
+	// expire and no longer exist. Elements <= 0 are ignored. Given
 	// that the zero-value is ignored, TTL cannot be used to set
 	// a TTL of 0.
 	TTL time.Duration
@@ -177,7 +177,7 @@ type SetOptions struct {
 	PrevExist PrevExistType
 
 	// TTL defines a period of time after-which the Node should
-	// expire and no longer exist. Values <= 0 are ignored. Given
+	// expire and no longer exist. Elements <= 0 are ignored. Given
 	// that the zero-value is ignored, TTL cannot be used to set
 	// a TTL of 0.
 	TTL time.Duration

tests/linearizability/append_model.go

@@ -0,0 +1,138 @@
+// Copyright 2022 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package linearizability
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/anishathalye/porcupine"
+)
+
+const (
+	Append Operation = "append"
+)
+
+type AppendRequest struct {
+	Op         Operation
+	Key        string
+	AppendData string
+}
+
+type AppendResponse struct {
+	GetData string
+}
+
+type AppendState struct {
+	Key      string
+	Elements []string
+}
+
+var appendModel = porcupine.Model{
+	Init: func() interface{} { return "{}" },
+	Step: func(st interface{}, in interface{}, out interface{}) (bool, interface{}) {
+		var state AppendState
+		err := json.Unmarshal([]byte(st.(string)), &state)
+		if err != nil {
+			panic(err)
+		}
+		ok, state := appendModelStep(state, in.(AppendRequest), out.(AppendResponse))
+		data, err := json.Marshal(state)
+		if err != nil {
+			panic(err)
+		}
+		return ok, string(data)
+	},
+	DescribeOperation: func(in, out interface{}) string {
+		request := in.(AppendRequest)
+		response := out.(AppendResponse)
+		switch request.Op {
+		case Get:
+			elements := strings.Split(response.GetData, ",")
+			return fmt.Sprintf("get(%q) -> %q", request.Key, elements[len(elements)-1])
+		case Append:
+			return fmt.Sprintf("append(%q, %q)", request.Key, request.AppendData)
+		default:
+			return "<invalid>"
+		}
+	},
+}
+
+func appendModelStep(state AppendState, request AppendRequest, response AppendResponse) (bool, AppendState) {
+	if request.Key == "" {
+		panic("invalid request")
+	}
+	if state.Key == "" {
+		return true, initAppendState(request, response)
+	}
+	if state.Key != request.Key {
+		panic("Multiple keys not supported")
+	}
+	switch request.Op {
+	case Get:
+		return stepAppendGet(state, request, response)
+	case Append:
+		return stepAppend(state, request, response)
+	default:
+		panic("Unknown operation")
+	}
+}
+
+func initAppendState(request AppendRequest, response AppendResponse) AppendState {
+	state := AppendState{
+		Key: request.Key,
+	}
+	switch request.Op {
+	case Get:
+		state.Elements = elements(response)
+	case Append:
+		state.Elements = []string{request.AppendData}
+	default:
+		panic("Unknown operation")
+	}
+	return state
+}
+
+func stepAppendGet(state AppendState, request AppendRequest, response AppendResponse) (bool, AppendState) {
+	newElements := elements(response)
+	if len(newElements) < len(state.Elements) {
+		return false, state
+	}
+
+	for i := 0; i < len(state.Elements); i++ {
+		if state.Elements[i] != newElements[i] {
+			return false, state
+		}
+	}
+	state.Elements = newElements
+	return true, state
+}
+
+func stepAppend(state AppendState, request AppendRequest, response AppendResponse) (bool, AppendState) {
+	if request.AppendData == "" {
+		panic("unsupported empty appendData")
+	}
+	state.Elements = append(state.Elements, request.AppendData)
+	return true, state
+}
+
+func elements(response AppendResponse) []string {
+	elements := strings.Split(response.GetData, ",")
+	if len(elements) == 1 && elements[0] == "" {
+		elements = []string{}
+	}
+	return elements
+}

tests/linearizability/append_model_test.go

@@ -0,0 +1,68 @@
+// Copyright 2022 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package linearizability
+
+import (
+	"testing"
+)
+
+func TestAppendModel(t *testing.T) {
+	tcs := []struct {
+		name       string
+		operations []testAppendOperation
+	}{
+		{
+			name: "Append appends",
+			operations: []testAppendOperation{
+				{req: AppendRequest{Key: "key", Op: Append, AppendData: "1"}, resp: AppendResponse{}},
+				{req: AppendRequest{Key: "key", Op: Get}, resp: AppendResponse{GetData: "1"}},
+				{req: AppendRequest{Key: "key", Op: Append, AppendData: "2"}, resp: AppendResponse{}},
+				{req: AppendRequest{Key: "key", Op: Get}, resp: AppendResponse{GetData: "1,3"}, failure: true},
+				{req: AppendRequest{Key: "key", Op: Get}, resp: AppendResponse{GetData: "1,2"}},
+			},
+		},
+		{
+			name: "Get validates prefix matches",
+			operations: []testAppendOperation{
+				{req: AppendRequest{Key: "key", Op: Get}, resp: AppendResponse{GetData: ""}},
+				{req: AppendRequest{Key: "key", Op: Get}, resp: AppendResponse{GetData: "1"}},
+				{req: AppendRequest{Key: "key", Op: Get}, resp: AppendResponse{GetData: "2"}, failure: true},
+				{req: AppendRequest{Key: "key", Op: Append, AppendData: "2"}, resp: AppendResponse{}},
+				{req: AppendRequest{Key: "key", Op: Get}, resp: AppendResponse{GetData: "1,3"}, failure: true},
+				{req: AppendRequest{Key: "key", Op: Get}, resp: AppendResponse{GetData: "1,2,3"}},
+				{req: AppendRequest{Key: "key", Op: Get}, resp: AppendResponse{GetData: "2,3"}, failure: true},
+			},
+		},
+	}
+	for _, tc := range tcs {
+		var ok bool
+		t.Run(tc.name, func(t *testing.T) {
+			state := appendModel.Init()
+			for _, op := range tc.operations {
+				t.Logf("state: %v", state)
+				ok, state = appendModel.Step(state, op.req, op.resp)
+				if ok != !op.failure {
+					t.Errorf("Unexpected operation result, expect: %v, got: %v, operation: %s", !op.failure, ok, appendModel.DescribeOperation(op.req, op.resp))
+				}
+			}
+		})
+	}
+}
+
+type testAppendOperation struct {
+	req     AppendRequest
+	resp    AppendResponse
+	failure bool
+}

tests/linearizability/client.go

@@ -47,15 +47,19 @@ func (c *recordingClient) Close() error {
 	return c.client.Close()
 }
 
-func (c *recordingClient) Get(ctx context.Context, key string) error {
+func (c *recordingClient) Get(ctx context.Context, key string) (string, error) {
 	callTime := time.Now()
 	resp, err := c.client.Get(ctx, key)
 	returnTime := time.Now()
 	if err != nil {
-		return err
+		return "", err
 	}
 	c.history.AppendGet(key, callTime, returnTime, resp)
-	return nil
+	var value string
+	if len(resp.Kvs) > 0 {
+		value = string(resp.Kvs[0].Value)
+	}
+	return value, nil
 }
 
 func (c *recordingClient) Put(ctx context.Context, key, value string) error {
@@ -73,3 +77,22 @@ func (c *recordingClient) Delete(ctx context.Context, key string) error {
 	c.history.AppendDelete(key, callTime, returnTime, resp, err)
 	return nil
 }
+
+func (c *recordingClient) Txn(ctx context.Context, key, expectedValue, newValue string) error {
+	callTime := time.Now()
+	txn := c.client.Txn(ctx)
+	var cmp clientv3.Cmp
+	if expectedValue == "" {
+		cmp = clientv3.Compare(clientv3.CreateRevision(key), "=", 0)
+	} else {
+		cmp = clientv3.Compare(clientv3.Value(key), "=", expectedValue)
+	}
+	resp, err := txn.If(
+		cmp,
+	).Then(
+		clientv3.OpPut(key, newValue),
+	).Commit()
+	returnTime := time.Now()
+	c.history.AppendTxn(key, expectedValue, newValue, callTime, returnTime, resp, err)
+	return err
+}

tests/linearizability/history.go

@@ -94,6 +94,25 @@ func (h *appendableHistory) AppendDelete(key string, start, end time.Time, resp
 	})
 }
 
+func (h *appendableHistory) AppendTxn(key, expectValue, newValue string, start, end time.Time, resp *clientv3.TxnResponse, err error) {
+	request := EtcdRequest{Op: Txn, Key: key, TxnExpectData: expectValue, TxnNewData: newValue}
+	if err != nil {
+		h.appendFailed(request, start, err)
+		return
+	}
+	var revision int64
+	if resp != nil && resp.Header != nil {
+		revision = resp.Header.Revision
+	}
+	h.successful = append(h.successful, porcupine.Operation{
+		ClientId: h.id,
+		Input:    request,
+		Call:     start.UnixNano(),
+		Output:   EtcdResponse{Err: err, Revision: revision, TxnSucceeded: resp.Succeeded},
+		Return:   end.UnixNano(),
+	})
+}
+
 func (h *appendableHistory) appendFailed(request EtcdRequest, start time.Time, err error) {
 	h.failed = append(h.failed, porcupine.Operation{
 		ClientId: h.id,
@@ -104,7 +123,7 @@ func (h *appendableHistory) appendFailed(request EtcdRequest, start time.Time, e
 	})
 	// Operations of single client needs to be sequential.
 	// As we don't know return time of failed operations, all new writes need to be done with new client id.
-	h.id = h.idProvider.ClientId()
+	//h.id = h.idProvider.ClientId()
 }
 
 type history struct {