etcd-dump-logs: add entry-type flag to list entries of specific types…

… and add test

tools/etcd-dump-logs/etcd-dump-log_test.go

@@ -0,0 +1,217 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/coreos/etcd/etcdserver/etcdserverpb"
+	"github.com/coreos/etcd/pkg/pbutil"
+	"github.com/coreos/etcd/raft/raftpb"
+	"github.com/coreos/etcd/wal"
+	"go.uber.org/zap"
+)
+
+func TestEtcdDumpLogEntryType(t *testing.T) {
+	// directory where the command is
+	dir, err := os.Getwd()
+	if err != nil {
+		t.Fatal("%v", err)
+	}
+
+	p, err := ioutil.TempDir(os.TempDir(), "etcddumplogstest")
+	if err != nil {
+		t.Fatal("%v", err)
+	}
+	defer os.RemoveAll(p)
+	memberdir := filepath.Join(p, "member")
+	err = os.Mkdir(memberdir, 0744)
+	if err != nil {
+		t.Fatal("%v", err)
+	}
+	waldir := walDir(p)
+	snapdir := snapDir(p)
+
+	w, err := wal.Create(zap.NewExample(), waldir, nil)
+	if err != nil {
+		t.Fatalf("err = %v, want nil", err)
+	}
+
+	err = os.Mkdir(snapdir, 0744)
+	if err != nil {
+		t.Fatal("%v", err)
+	}
+
+	ents := make([]raftpb.Entry, 0)
+
+	// append entries into wal log
+	appendConfigChangeEnts(&ents)
+	appendNormalRequestEnts(&ents)
+	appendNormalIRREnts(&ents)
+
+	// force commit newly appended entries
+	err = w.Save(raftpb.HardState{}, ents)
+	if err != nil {
+		t.Fatal("%v", err)
+	}
+	w.Close()
+
+	argtests := []struct {
+		name         string
+		args         []string
+		fileExpected string
+	}{
+		{"no entry-type", []string{p}, "expectedoutput/listAll.output"},
+		{"confchange entry-type", []string{"-entry-type", "ConfigChange", p}, "expectedoutput/listConfChange.output"},
+		{"normal entry-type", []string{"-entry-type", "Normal", p}, "expectedoutput/listNormal.output"},
+		{"request entry-type", []string{"-entry-type", "Request", p}, "expectedoutput/listRequest.output"},
+		{"internalRaftRequest entry-type", []string{"-entry-type", "InternalRaftRequest", p}, "expectedoutput/listInternalRaftRequest.output"},
+		{"range entry-type", []string{"-entry-type", "InternalRaftRequestRange", p}, "expectedoutput/listRange.output"},
+		{"put entry-type", []string{"-entry-type", "InternalRaftRequestPut", p}, "expectedoutput/listPut.output"},
+		{"del entry-type", []string{"-entry-type", "InternalRaftRequestDeleteRange", p}, "expectedoutput/listDeleterange.output"},
+		{"txn entry-type", []string{"-entry-type", "InternalRaftRequestTxn", p}, "expectedoutput/listTxn.output"},
+		{"confchange and txn entry-type", []string{"-entry-type", "ConfigChange,InternalRaftRequestTxn", p}, "expectedoutput/listConfigChangeInternalRaftRequestTxn.output"},
+	}
+
+	for _, argtest := range argtests {
+		t.Run(argtest.name, func(t *testing.T) {
+			cmd := exec.Command(path.Join(dir, "etcd-dump-logs"), argtest.args...)
+			actual, err := cmd.CombinedOutput()
+			if err != nil {
+				t.Fatal("%v", err)
+			}
+			expected, err := ioutil.ReadFile(path.Join(dir, argtest.fileExpected))
+			if err != nil {
+				t.Fatal("%v", err)
+			}
+			if !bytes.Equal(actual, expected) {
+				t.Errorf(`Got input of length %d, wanted input of length %d
+==== BEGIN RECIEVED FILE ====
+%s
+==== END RECIEVED FILE ====
+==== BEGIN EXPECTED FILE ====
+%s
+==== END EXPECTED FILE ==== 
+`, len(actual), len(expected), actual, expected)
+			}
+		})
+	}
+
+}
+
+func appendConfigChangeEnts(ents *[]raftpb.Entry) {
+	configChangeData := []raftpb.ConfChange{
+		{1, raftpb.ConfChangeAddNode, 2, []byte(""), []byte("")},
+		{2, raftpb.ConfChangeRemoveNode, 2, []byte(""), []byte("")},
+		{3, raftpb.ConfChangeUpdateNode, 2, []byte(""), []byte("")},
+		{4, raftpb.ConfChangeAddLearnerNode, 3, []byte(""), []byte("")},
+	}
+	configChangeEntries := []raftpb.Entry{
+		{1, 1, raftpb.EntryConfChange, pbutil.MustMarshal(&configChangeData[0]), []byte("")},
+		{2, 2, raftpb.EntryConfChange, pbutil.MustMarshal(&configChangeData[1]), []byte("")},
+		{2, 3, raftpb.EntryConfChange, pbutil.MustMarshal(&configChangeData[2]), []byte("")},
+		{2, 4, raftpb.EntryConfChange, pbutil.MustMarshal(&configChangeData[3]), []byte("")},
+	}
+	*ents = append(*ents, configChangeEntries...)
+}
+
+func appendNormalRequestEnts(ents *[]raftpb.Entry) {
+	a := true
+	b := false
+
+	requestdata := []etcdserverpb.Request{
+		{0, "", "/path0", "{\"hey\":\"ho\",\"hi\":[\"yo\"]}", true, "", 0, &b, 9, false, 1, false, false, false, 1, false, &b, []byte("")},
+		{1, "QGET", "/path1", "{\"0\":\"1\",\"2\":[\"3\"]}", false, "", 0, &b, 9, false, 1, false, false, false, 1, false, &b, []byte("")},
+		{2, "SYNC", "/path2", "{\"0\":\"1\",\"2\":[\"3\"]}", false, "", 0, &b, 2, false, 1, false, false, false, 1, false, &b, []byte("")},
+		{3, "DELETE", "/path3", "{\"hey\":\"ho\",\"hi\":[\"yo\"]}", false, "", 0, &a, 2, false, 1, false, false, false, 1, false, &b, []byte("")},
+		{4, "RANDOM", "/path4/superlong" + strings.Repeat("/path", 30), "{\"hey\":\"ho\",\"hi\":[\"yo\"]}", false, "", 0, &b, 2, false, 1, false, false, false, 1, false, &b, []byte("")},
+	}
+	e := []raftpb.Entry{
+		{3, 5, raftpb.EntryNormal, pbutil.MustMarshal(&requestdata[0]), []byte("")},
+		{3, 6, raftpb.EntryNormal, pbutil.MustMarshal(&requestdata[1]), []byte("")},
+		{3, 7, raftpb.EntryNormal, pbutil.MustMarshal(&requestdata[2]), []byte("")},
+		{3, 8, raftpb.EntryNormal, pbutil.MustMarshal(&requestdata[3]), []byte("")},
+		{4, 9, raftpb.EntryNormal, pbutil.MustMarshal(&requestdata[4]), []byte("")},
+	}
+	*ents = append(*ents, e...)
+}
+
+func appendNormalIRREnts(ents *[]raftpb.Entry) {
+	r := &etcdserverpb.RangeRequest{
+		Key:               []byte("1"),
+		RangeEnd:          []byte("hi"),
+		Limit:             6,
+		Revision:          1,
+		SortOrder:         1,
+		SortTarget:        0,
+		Serializable:      false,
+		KeysOnly:          false,
+		CountOnly:         false,
+		MinModRevision:    0,
+		MaxModRevision:    20000,
+		MinCreateRevision: 0,
+		MaxCreateRevision: 20000,
+	}
+
+	p := &etcdserverpb.PutRequest{
+		Key:         []byte("foo1"),
+		Value:       []byte("bar1"),
+		Lease:       1,
+		PrevKv:      false,
+		IgnoreValue: false,
+		IgnoreLease: true,
+	}
+
+	d := &etcdserverpb.DeleteRangeRequest{
+		Key:      []byte("0"),
+		RangeEnd: []byte("9"),
+		PrevKv:   true,
+	}
+
+	delInRangeReq := &etcdserverpb.RequestOp{Request: &etcdserverpb.RequestOp_RequestDeleteRange{
+		RequestDeleteRange: &etcdserverpb.DeleteRangeRequest{
+			Key: []byte("a"), RangeEnd: []byte("b"),
+		},
+	},
+	}
+
+	t := &etcdserverpb.TxnRequest{
+		Success: []*etcdserverpb.RequestOp{delInRangeReq},
+		Failure: []*etcdserverpb.RequestOp{delInRangeReq},
+	}
+
+	irrdata := []etcdserverpb.InternalRaftRequest{
+		{ID: 5, Range: r},
+		{ID: 6, Put: p},
+		{ID: 7, DeleteRange: d},
+		{ID: 8, Txn: t},
+	}
+
+	e := []raftpb.Entry{
+		{4, 10, raftpb.EntryNormal, pbutil.MustMarshal(&irrdata[0]), []byte("")},
+		{5, 11, raftpb.EntryNormal, pbutil.MustMarshal(&irrdata[1]), []byte("")},
+		{6, 12, raftpb.EntryNormal, pbutil.MustMarshal(&irrdata[2]), []byte("")},
+		{7, 13, raftpb.EntryNormal, pbutil.MustMarshal(&irrdata[3]), []byte("")},
+	}
+
+	*ents = append(*ents, e...)
+}

tools/etcd-dump-logs/expectedoutput/listAll.output

@@ -0,0 +1,23 @@
+Snapshot:
+empty
+Start dupmping log entries from snapshot.
+WAL metadata:
+nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
+WAL entries:
+lastIndex=13
+term	     index	type	data
+   1	         1	conf	method=ConfChangeAddNode id=2
+   2	         2	conf	method=ConfChangeRemoveNode id=2
+   2	         3	conf	method=ConfChangeUpdateNode id=2
+   2	         4	conf	method=ConfChangeAddLearnerNode id=3
+   3	         5	norm	noop
+   3	         6	norm	method=QGET path="/path1"
+   3	         7	norm	method=SYNC time="1969-12-31 16:00:00.000000001 -0800 PST"
+   3	         8	norm	method=DELETE path="/path3"
+   4	         9	norm	method=RANDOM path="/path4/superlong/path/path/path/path/path/path/path/path/path/pa"..."path/path/path/path/path/path/path/path/path/path/path/path/path" val="{\"hey\":\"ho\",\"hi\":[\"yo\"]}"
+   4	        10	norm	ID:5 range:<key:"1" range_end:"hi" limit:6 revision:1 sort_order:ASCEND max_mod_revision:20000 max_create_revision:20000 > 
+   5	        11	norm	ID:6 put:<key:"foo1" value:"bar1" lease:1 ignore_lease:true > 
+   6	        12	norm	ID:7 delete_range:<key:"0" range_end:"9" prev_kv:true > 
+   7	        13	norm	ID:8 txn:<success:<request_delete_range:<key:"a" range_end:"b" > > failure:<request_delete_range:<key:"a" range_end:"b" > > > 
+
+Entry types () count is : 13

tools/etcd-dump-logs/expectedoutput/listConfChange.output

@@ -0,0 +1,14 @@
+Snapshot:
+empty
+Start dupmping log entries from snapshot.
+WAL metadata:
+nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
+WAL entries:
+lastIndex=13
+term	     index	type	data
+   1	         1	conf	method=ConfChangeAddNode id=2
+   2	         2	conf	method=ConfChangeRemoveNode id=2
+   2	         3	conf	method=ConfChangeUpdateNode id=2
+   2	         4	conf	method=ConfChangeAddLearnerNode id=3
+
+Entry types (ConfigChange) count is : 4

tools/etcd-dump-logs/expectedoutput/listConfigChangeInternalRaftRequestTxn.output

@@ -0,0 +1,15 @@
+Snapshot:
+empty
+Start dupmping log entries from snapshot.
+WAL metadata:
+nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
+WAL entries:
+lastIndex=13
+term	     index	type	data
+   1	         1	conf	method=ConfChangeAddNode id=2
+   2	         2	conf	method=ConfChangeRemoveNode id=2
+   2	         3	conf	method=ConfChangeUpdateNode id=2
+   2	         4	conf	method=ConfChangeAddLearnerNode id=3
+   7	        13	norm	ID:8 txn:<success:<request_delete_range:<key:"a" range_end:"b" > > failure:<request_delete_range:<key:"a" range_end:"b" > > > 
+
+Entry types (ConfigChange,InternalRaftRequestTxn) count is : 5

tools/etcd-dump-logs/expectedoutput/listDeleterange.output

@@ -0,0 +1,11 @@
+Snapshot:
+empty
+Start dupmping log entries from snapshot.
+WAL metadata:
+nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
+WAL entries:
+lastIndex=13
+term	     index	type	data
+   6	        12	norm	ID:7 delete_range:<key:"0" range_end:"9" prev_kv:true > 
+
+Entry types (InternalRaftRequestDeleteRange) count is : 1

tools/etcd-dump-logs/expectedoutput/listInternalRaftRequest.output

@@ -0,0 +1,14 @@
+Snapshot:
+empty
+Start dupmping log entries from snapshot.
+WAL metadata:
+nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
+WAL entries:
+lastIndex=13
+term	     index	type	data
+   4	        10	norm	ID:5 range:<key:"1" range_end:"hi" limit:6 revision:1 sort_order:ASCEND max_mod_revision:20000 max_create_revision:20000 > 
+   5	        11	norm	ID:6 put:<key:"foo1" value:"bar1" lease:1 ignore_lease:true > 
+   6	        12	norm	ID:7 delete_range:<key:"0" range_end:"9" prev_kv:true > 
+   7	        13	norm	ID:8 txn:<success:<request_delete_range:<key:"a" range_end:"b" > > failure:<request_delete_range:<key:"a" range_end:"b" > > > 
+
+Entry types (InternalRaftRequest) count is : 4

tools/etcd-dump-logs/expectedoutput/listNormal.output

@@ -0,0 +1,19 @@
+Snapshot:
+empty
+Start dupmping log entries from snapshot.
+WAL metadata:
+nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
+WAL entries:
+lastIndex=13
+term	     index	type	data
+   3	         5	norm	noop
+   3	         6	norm	method=QGET path="/path1"
+   3	         7	norm	method=SYNC time="1969-12-31 16:00:00.000000001 -0800 PST"
+   3	         8	norm	method=DELETE path="/path3"
+   4	         9	norm	method=RANDOM path="/path4/superlong/path/path/path/path/path/path/path/path/path/pa"..."path/path/path/path/path/path/path/path/path/path/path/path/path" val="{\"hey\":\"ho\",\"hi\":[\"yo\"]}"
+   4	        10	norm	ID:5 range:<key:"1" range_end:"hi" limit:6 revision:1 sort_order:ASCEND max_mod_revision:20000 max_create_revision:20000 > 
+   5	        11	norm	ID:6 put:<key:"foo1" value:"bar1" lease:1 ignore_lease:true > 
+   6	        12	norm	ID:7 delete_range:<key:"0" range_end:"9" prev_kv:true > 
+   7	        13	norm	ID:8 txn:<success:<request_delete_range:<key:"a" range_end:"b" > > failure:<request_delete_range:<key:"a" range_end:"b" > > > 
+
+Entry types (Normal) count is : 9

tools/etcd-dump-logs/expectedoutput/listPut.output

@@ -0,0 +1,11 @@
+Snapshot:
+empty
+Start dupmping log entries from snapshot.
+WAL metadata:
+nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
+WAL entries:
+lastIndex=13
+term	     index	type	data
+   5	        11	norm	ID:6 put:<key:"foo1" value:"bar1" lease:1 ignore_lease:true > 
+
+Entry types (InternalRaftRequestPut) count is : 1

tools/etcd-dump-logs/expectedoutput/listRange.output

@@ -0,0 +1,11 @@
+Snapshot:
+empty
+Start dupmping log entries from snapshot.
+WAL metadata:
+nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
+WAL entries:
+lastIndex=13
+term	     index	type	data
+   4	        10	norm	ID:5 range:<key:"1" range_end:"hi" limit:6 revision:1 sort_order:ASCEND max_mod_revision:20000 max_create_revision:20000 > 
+
+Entry types (InternalRaftRequestRange) count is : 1

tools/etcd-dump-logs/expectedoutput/listRequest.output

@@ -0,0 +1,18 @@
+Snapshot:
+empty
+Start dupmping log entries from snapshot.
+WAL metadata:
+nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
+WAL entries:
+lastIndex=13
+term	     index	type	data
+   3	         5	norm	noop
+   3	         6	norm	method=QGET path="/path1"
+   3	         7	norm	method=SYNC time="1969-12-31 16:00:00.000000001 -0800 PST"
+   3	         8	norm	method=DELETE path="/path3"
+   4	         9	norm	method=RANDOM path="/path4/superlong/path/path/path/path/path/path/path/path/path/pa"..."path/path/path/path/path/path/path/path/path/path/path/path/path" val="{\"hey\":\"ho\",\"hi\":[\"yo\"]}"
+   4	        10	norm	noop
+   5	        11	norm	noop
+   7	        13	norm	noop
+
+Entry types (Request) count is : 8

tools/etcd-dump-logs/expectedoutput/listTxn.output

@@ -0,0 +1,11 @@
+Snapshot:
+empty
+Start dupmping log entries from snapshot.
+WAL metadata:
+nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
+WAL entries:
+lastIndex=13
+term	     index	type	data
+   7	        13	norm	ID:8 txn:<success:<request_delete_range:<key:"a" range_end:"b" > > failure:<request_delete_range:<key:"a" range_end:"b" > > > 
+
+Entry types (InternalRaftRequestTxn) count is : 1