Skip to content

Commit

Permalink
Merge pull request #9628 from wenjiaswe/new1
Browse files Browse the repository at this point in the history
etcd-dump-logs: add entry-type flag to list entries of specific types…
  • Loading branch information
gyuho authored Apr 27, 2018
2 parents 4a09318 + 5533257 commit 7582a28
Show file tree
Hide file tree
Showing 16 changed files with 707 additions and 41 deletions.
277 changes: 277 additions & 0 deletions tools/etcd-dump-logs/etcd-dump-log_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,277 @@
// Copyright 2018 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
"bytes"
"io/ioutil"
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"testing"

"github.com/coreos/etcd/auth/authpb"
"github.com/coreos/etcd/etcdserver/etcdserverpb"
"github.com/coreos/etcd/pkg/fileutil"
"github.com/coreos/etcd/pkg/pbutil"
"github.com/coreos/etcd/raft/raftpb"
"github.com/coreos/etcd/wal"
"go.uber.org/zap"
)

func TestEtcdDumpLogEntryType(t *testing.T) {
// directory where the command is
binDir, err := os.Getwd()
if err != nil {
t.Fatal(err)
}

dumpLogsBinary := path.Join(binDir + "/etcd-dump-logs")
if !fileutil.Exist(dumpLogsBinary) {
t.Skipf("%q does not exist", dumpLogsBinary)
}

p, err := ioutil.TempDir(os.TempDir(), "etcddumplogstest")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(p)

memberdir := filepath.Join(p, "member")
err = os.Mkdir(memberdir, 0744)
if err != nil {
t.Fatal(err)
}
waldir := walDir(p)
snapdir := snapDir(p)

w, err := wal.Create(zap.NewExample(), waldir, nil)
if err != nil {
t.Fatal(err)
}

err = os.Mkdir(snapdir, 0744)
if err != nil {
t.Fatal(err)
}

ents := make([]raftpb.Entry, 0)

// append entries into wal log
appendConfigChangeEnts(&ents)
appendNormalRequestEnts(&ents)
appendNormalIRREnts(&ents)
appendUnknownNormalEnts(&ents)

// force commit newly appended entries
err = w.Save(raftpb.HardState{}, ents)
if err != nil {
t.Fatal(err)
}
w.Close()

argtests := []struct {
name string
args []string
fileExpected string
}{
{"no entry-type", []string{p}, "expectedoutput/listAll.output"},
{"confchange entry-type", []string{"-entry-type", "ConfigChange", p}, "expectedoutput/listConfigChange.output"},
{"normal entry-type", []string{"-entry-type", "Normal", p}, "expectedoutput/listNormal.output"},
{"request entry-type", []string{"-entry-type", "Request", p}, "expectedoutput/listRequest.output"},
{"internalRaftRequest entry-type", []string{"-entry-type", "InternalRaftRequest", p}, "expectedoutput/listInternalRaftRequest.output"},
{"range entry-type", []string{"-entry-type", "IRRRange", p}, "expectedoutput/listIRRRange.output"},
{"put entry-type", []string{"-entry-type", "IRRPut", p}, "expectedoutput/listIRRPut.output"},
{"del entry-type", []string{"-entry-type", "IRRDeleteRange", p}, "expectedoutput/listIRRDeleteRange.output"},
{"txn entry-type", []string{"-entry-type", "IRRTxn", p}, "expectedoutput/listIRRTxn.output"},
{"compaction entry-type", []string{"-entry-type", "IRRCompaction", p}, "expectedoutput/listIRRCompaction.output"},
{"lease grant entry-type", []string{"-entry-type", "IRRLeaseGrant", p}, "expectedoutput/listIRRLeaseGrant.output"},
{"lease revoke entry-type", []string{"-entry-type", "IRRLeaseRevoke", p}, "expectedoutput/listIRRLeaseRevoke.output"},
{"confchange and txn entry-type", []string{"-entry-type", "ConfigChange,IRRCompaction", p}, "expectedoutput/listConfigChangeIRRCompaction.output"},
}

for _, argtest := range argtests {
t.Run(argtest.name, func(t *testing.T) {
cmd := exec.Command(dumpLogsBinary, argtest.args...)
actual, err := cmd.CombinedOutput()
if err != nil {
t.Fatal(err)
}
expected, err := ioutil.ReadFile(path.Join(binDir, argtest.fileExpected))
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(actual, expected) {
t.Errorf(`Got input of length %d, wanted input of length %d
==== BEGIN RECIEVED FILE ====
%s
==== END RECIEVED FILE ====
==== BEGIN EXPECTED FILE ====
%s
==== END EXPECTED FILE ====
`, len(actual), len(expected), actual, expected)
}
})
}

}

func appendConfigChangeEnts(ents *[]raftpb.Entry) {
configChangeData := []raftpb.ConfChange{
{ID: 1, Type: raftpb.ConfChangeAddNode, NodeID: 2, Context: []byte("")},
{ID: 2, Type: raftpb.ConfChangeRemoveNode, NodeID: 2, Context: []byte("")},
{ID: 3, Type: raftpb.ConfChangeUpdateNode, NodeID: 2, Context: []byte("")},
{ID: 4, Type: raftpb.ConfChangeAddLearnerNode, NodeID: 3, Context: []byte("")},
}
configChangeEntries := []raftpb.Entry{
{Term: 1, Index: 1, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(&configChangeData[0])},
{Term: 2, Index: 2, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(&configChangeData[1])},
{Term: 2, Index: 3, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(&configChangeData[2])},
{Term: 2, Index: 4, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(&configChangeData[3])},
}
*ents = append(*ents, configChangeEntries...)
}

func appendNormalRequestEnts(ents *[]raftpb.Entry) {
a := true
b := false

requests := []etcdserverpb.Request{
{ID: 0, Method: "", Path: "/path0", Val: "{\"hey\":\"ho\",\"hi\":[\"yo\"]}", Dir: true, PrevValue: "", PrevIndex: 0, PrevExist: &b, Expiration: 9, Wait: false, Since: 1, Recursive: false, Sorted: false, Quorum: false, Time: 1, Stream: false, Refresh: &b},
{ID: 1, Method: "QGET", Path: "/path1", Val: "{\"0\":\"1\",\"2\":[\"3\"]}", Dir: false, PrevValue: "", PrevIndex: 0, PrevExist: &b, Expiration: 9, Wait: false, Since: 1, Recursive: false, Sorted: false, Quorum: false, Time: 1, Stream: false, Refresh: &b},
{ID: 2, Method: "SYNC", Path: "/path2", Val: "{\"0\":\"1\",\"2\":[\"3\"]}", Dir: false, PrevValue: "", PrevIndex: 0, PrevExist: &b, Expiration: 2, Wait: false, Since: 1, Recursive: false, Sorted: false, Quorum: false, Time: 1, Stream: false, Refresh: &b},
{ID: 3, Method: "DELETE", Path: "/path3", Val: "{\"hey\":\"ho\",\"hi\":[\"yo\"]}", Dir: false, PrevValue: "", PrevIndex: 0, PrevExist: &a, Expiration: 2, Wait: false, Since: 1, Recursive: false, Sorted: false, Quorum: false, Time: 1, Stream: false, Refresh: &b},
{ID: 4, Method: "RANDOM", Path: "/path4/superlong" + strings.Repeat("/path", 30), Val: "{\"hey\":\"ho\",\"hi\":[\"yo\"]}", Dir: false, PrevValue: "", PrevIndex: 0, PrevExist: &b, Expiration: 2, Wait: false, Since: 1, Recursive: false, Sorted: false, Quorum: false, Time: 1, Stream: false, Refresh: &b},
}

for i, request := range requests {
var currentry raftpb.Entry
currentry.Term = 3
currentry.Index = uint64(i + 5)
currentry.Type = raftpb.EntryNormal
currentry.Data = pbutil.MustMarshal(&request)
*ents = append(*ents, currentry)
}
}

func appendNormalIRREnts(ents *[]raftpb.Entry) {
irrrange := &etcdserverpb.RangeRequest{Key: []byte("1"), RangeEnd: []byte("hi"), Limit: 6, Revision: 1, SortOrder: 1, SortTarget: 0, Serializable: false, KeysOnly: false, CountOnly: false, MinModRevision: 0, MaxModRevision: 20000, MinCreateRevision: 0, MaxCreateRevision: 20000}

irrput := &etcdserverpb.PutRequest{Key: []byte("foo1"), Value: []byte("bar1"), Lease: 1, PrevKv: false, IgnoreValue: false, IgnoreLease: true}

irrdeleterange := &etcdserverpb.DeleteRangeRequest{Key: []byte("0"), RangeEnd: []byte("9"), PrevKv: true}

delInRangeReq := &etcdserverpb.RequestOp{Request: &etcdserverpb.RequestOp_RequestDeleteRange{
RequestDeleteRange: &etcdserverpb.DeleteRangeRequest{
Key: []byte("a"), RangeEnd: []byte("b"),
},
},
}

irrtxn := &etcdserverpb.TxnRequest{Success: []*etcdserverpb.RequestOp{delInRangeReq}, Failure: []*etcdserverpb.RequestOp{delInRangeReq}}

irrcompaction := &etcdserverpb.CompactionRequest{Revision: 0, Physical: true}

irrleasegrant := &etcdserverpb.LeaseGrantRequest{TTL: 1, ID: 1}

irrleaserevoke := &etcdserverpb.LeaseRevokeRequest{ID: 2}

irralarm := &etcdserverpb.AlarmRequest{Action: 3, MemberID: 4, Alarm: 5}

irrauthenable := &etcdserverpb.AuthEnableRequest{}

irrauthdisable := &etcdserverpb.AuthDisableRequest{}

irrauthenticate := &etcdserverpb.InternalAuthenticateRequest{Name: "myname", Password: "password", SimpleToken: "token"}

irrauthuseradd := &etcdserverpb.AuthUserAddRequest{Name: "name1", Password: "pass1"}

irrauthuserdelete := &etcdserverpb.AuthUserDeleteRequest{Name: "name1"}

irrauthuserget := &etcdserverpb.AuthUserGetRequest{Name: "name1"}

irrauthuserchangepassword := &etcdserverpb.AuthUserChangePasswordRequest{Name: "name1", Password: "pass2"}

irrauthusergrantrole := &etcdserverpb.AuthUserGrantRoleRequest{User: "user1", Role: "role1"}

irrauthuserrevokerole := &etcdserverpb.AuthUserRevokeRoleRequest{Name: "user2", Role: "role2"}

irrauthuserlist := &etcdserverpb.AuthUserListRequest{}

irrauthrolelist := &etcdserverpb.AuthRoleListRequest{}

irrauthroleadd := &etcdserverpb.AuthRoleAddRequest{Name: "role2"}

irrauthroledelete := &etcdserverpb.AuthRoleDeleteRequest{Role: "role1"}

irrauthroleget := &etcdserverpb.AuthRoleGetRequest{Role: "role3"}

perm := &authpb.Permission{
PermType: authpb.WRITE,
Key: []byte("Keys"),
RangeEnd: []byte("RangeEnd"),
}

irrauthrolegrantpermission := &etcdserverpb.AuthRoleGrantPermissionRequest{Name: "role3", Perm: perm}

irrauthrolerevokepermission := &etcdserverpb.AuthRoleRevokePermissionRequest{Role: "role3", Key: []byte("key"), RangeEnd: []byte("rangeend")}

irrs := []etcdserverpb.InternalRaftRequest{
{ID: 5, Range: irrrange},
{ID: 6, Put: irrput},
{ID: 7, DeleteRange: irrdeleterange},
{ID: 8, Txn: irrtxn},
{ID: 9, Compaction: irrcompaction},
{ID: 10, LeaseGrant: irrleasegrant},
{ID: 11, LeaseRevoke: irrleaserevoke},
{ID: 12, Alarm: irralarm},
{ID: 13, AuthEnable: irrauthenable},
{ID: 14, AuthDisable: irrauthdisable},
{ID: 15, Authenticate: irrauthenticate},
{ID: 16, AuthUserAdd: irrauthuseradd},
{ID: 17, AuthUserDelete: irrauthuserdelete},
{ID: 18, AuthUserGet: irrauthuserget},
{ID: 19, AuthUserChangePassword: irrauthuserchangepassword},
{ID: 20, AuthUserGrantRole: irrauthusergrantrole},
{ID: 21, AuthUserRevokeRole: irrauthuserrevokerole},
{ID: 22, AuthUserList: irrauthuserlist},
{ID: 23, AuthRoleList: irrauthrolelist},
{ID: 24, AuthRoleAdd: irrauthroleadd},
{ID: 25, AuthRoleDelete: irrauthroledelete},
{ID: 26, AuthRoleGet: irrauthroleget},
{ID: 27, AuthRoleGrantPermission: irrauthrolegrantpermission},
{ID: 28, AuthRoleRevokePermission: irrauthrolerevokepermission},
}

for i, irr := range irrs {
var currentry raftpb.Entry
currentry.Term = uint64(i + 4)
currentry.Index = uint64(i + 10)
currentry.Type = raftpb.EntryNormal
currentry.Data = pbutil.MustMarshal(&irr)
*ents = append(*ents, currentry)
}
}

func appendUnknownNormalEnts(ents *[]raftpb.Entry) {
var currentry raftpb.Entry
currentry.Term = 27
currentry.Index = 34
currentry.Type = raftpb.EntryNormal
currentry.Data = []byte("?")
*ents = append(*ents, currentry)
}
44 changes: 44 additions & 0 deletions tools/etcd-dump-logs/expectedoutput/listAll.output
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
Snapshot:
empty
Start dupmping log entries from snapshot.
WAL metadata:
nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
WAL entries:
lastIndex=34
term index type data
1 1 conf method=ConfChangeAddNode id=2
2 2 conf method=ConfChangeRemoveNode id=2
2 3 conf method=ConfChangeUpdateNode id=2
2 4 conf method=ConfChangeAddLearnerNode id=3
3 5 norm noop
3 6 norm method=QGET path="/path1"
3 7 norm method=SYNC time="1969-12-31 16:00:00.000000001 -0800 PST"
3 8 norm method=DELETE path="/path3"
3 9 norm method=RANDOM path="/path4/superlong/path/path/path/path/path/path/path/path/path/pa"..."path/path/path/path/path/path/path/path/path/path/path/path/path" val="{\"hey\":\"ho\",\"hi\":[\"yo\"]}"
4 10 norm ID:5 range:<key:"1" range_end:"hi" limit:6 revision:1 sort_order:ASCEND max_mod_revision:20000 max_create_revision:20000 >
5 11 norm ID:6 put:<key:"foo1" value:"bar1" lease:1 ignore_lease:true >
6 12 norm ID:7 delete_range:<key:"0" range_end:"9" prev_kv:true >
7 13 norm ID:8 txn:<success:<request_delete_range:<key:"a" range_end:"b" > > failure:<request_delete_range:<key:"a" range_end:"b" > > >
8 14 norm ID:9 compaction:<physical:true >
9 15 norm ID:10 lease_grant:<TTL:1 ID:1 >
10 16 norm ID:11 lease_revoke:<ID:2 >
11 17 norm ID:12 alarm:<action:3 memberID:4 alarm:5 >
12 18 norm ID:13 auth_enable:<>
13 19 norm ID:14 auth_disable:<>
14 20 norm ID:15 authenticate:<name:"myname" password:"password" simple_token:"token" >
15 21 norm ID:16 auth_user_add:<name:"name1" password:"pass1" >
16 22 norm ID:17 auth_user_delete:<name:"name1" >
17 23 norm ID:18 auth_user_get:<name:"name1" >
18 24 norm ID:19 auth_user_change_password:<name:"name1" password:"pass2" >
19 25 norm ID:20 auth_user_grant_role:<user:"user1" role:"role1" >
20 26 norm ID:21 auth_user_revoke_role:<name:"user2" role:"role2" >
21 27 norm ID:22 auth_user_list:<>
22 28 norm ID:23 auth_role_list:<>
23 29 norm ID:24 auth_role_add:<name:"role2" >
24 30 norm ID:25 auth_role_delete:<role:"role1" >
25 31 norm ID:26 auth_role_get:<role:"role3" >
26 32 norm ID:27 auth_role_grant_permission:<name:"role3" perm:<permType:WRITE key:"Keys" range_end:"RangeEnd" > >
27 33 norm ID:28 auth_role_revoke_permission:<role:"role3" key:"key" range_end:"rangeend" >
27 34 norm ???

Entry types () count is : 34
14 changes: 14 additions & 0 deletions tools/etcd-dump-logs/expectedoutput/listConfigChange.output
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
Snapshot:
empty
Start dupmping log entries from snapshot.
WAL metadata:
nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
WAL entries:
lastIndex=34
term index type data
1 1 conf method=ConfChangeAddNode id=2
2 2 conf method=ConfChangeRemoveNode id=2
2 3 conf method=ConfChangeUpdateNode id=2
2 4 conf method=ConfChangeAddLearnerNode id=3

Entry types (ConfigChange) count is : 4
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
Snapshot:
empty
Start dupmping log entries from snapshot.
WAL metadata:
nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
WAL entries:
lastIndex=34
term index type data
1 1 conf method=ConfChangeAddNode id=2
2 2 conf method=ConfChangeRemoveNode id=2
2 3 conf method=ConfChangeUpdateNode id=2
2 4 conf method=ConfChangeAddLearnerNode id=3
8 14 norm ID:9 compaction:<physical:true >

Entry types (ConfigChange,IRRCompaction) count is : 5
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
Snapshot:
empty
Start dupmping log entries from snapshot.
WAL metadata:
nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
WAL entries:
lastIndex=34
term index type data
10 16 norm ID:11 lease_revoke:<ID:2 >

Entry types (IRRLeaseRevoke) count is : 1
11 changes: 11 additions & 0 deletions tools/etcd-dump-logs/expectedoutput/listIRRCompaction.output
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
Snapshot:
empty
Start dupmping log entries from snapshot.
WAL metadata:
nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
WAL entries:
lastIndex=34
term index type data
8 14 norm ID:9 compaction:<physical:true >

Entry types (IRRCompaction) count is : 1
11 changes: 11 additions & 0 deletions tools/etcd-dump-logs/expectedoutput/listIRRDeleteRange.output
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
Snapshot:
empty
Start dupmping log entries from snapshot.
WAL metadata:
nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
WAL entries:
lastIndex=34
term index type data
6 12 norm ID:7 delete_range:<key:"0" range_end:"9" prev_kv:true >

Entry types (IRRDeleteRange) count is : 1
Loading

0 comments on commit 7582a28

Please sign in to comment.