Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve logging for etcd IO bytes #338

Merged
merged 1 commit into from
Sep 28, 2022
Merged

Improve logging for etcd IO bytes #338

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 18 additions & 4 deletions poc-cb-net/cmd/admin-web/admin-web.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,8 +325,8 @@ func getExistingNetworkInfo(etcdClient *clientv3.Client) error {
}
CBLogger.Tracef("GetResponse: %#v", getResp)

fields := createFieldsForResponseSizes(*getResp)
CBLogger.WithFields(fields).Tracef("GetResponse size (bytes)")
totalSize, headerSize, kvsSize, kvsCount := extractSizes(*getResp)
CBLogger.Tracef("GetResponse size (bytes): total_size: %v, header_size: %v, kvs_size: %v, kvs_count: %v", totalSize, headerSize, kvsSize, kvsCount)

for _, kv := range getResp.Kvs {
CBLogger.Tracef("CLADNet ID: %v", kv.Key)
Expand Down Expand Up @@ -358,8 +358,8 @@ func getExistingNetworkInfo(etcdClient *clientv3.Client) error {
}
CBLogger.Tracef("GetResponse: %#v", respMultiSpec)

fields = createFieldsForResponseSizes(*respMultiSpec)
CBLogger.WithFields(fields).Tracef("GetResponse size (bytes)")
totalSize, headerSize, kvsSize, kvsCount = extractSizes(*respMultiSpec)
CBLogger.Tracef("GetResponse size (bytes): total_size: %v, header_size: %v, kvs_size: %v, kvs_count: %v", totalSize, headerSize, kvsSize, kvsCount)

if len(respMultiSpec.Kvs) != 0 {
var cladnetSpecificationList []string
Expand Down Expand Up @@ -677,6 +677,20 @@ func main() {
CBLogger.Debug("End.........")
}

func extractSizes(res clientv3.GetResponse) (totalSize, headerSize, kvsSize, kvsCount int) {

headerSize = res.Header.Size()
kvsCount = int(res.Count)
kvsSize = 0
for _, kv := range res.Kvs {
kvsSize += kv.Size()
}

totalSize = headerSize + kvsSize

return totalSize, headerSize, kvsSize, kvsCount
}

func createFieldsForResponseSizes(res clientv3.GetResponse) logrus.Fields {

// lenKvs := res.Count
Expand Down
81 changes: 49 additions & 32 deletions poc-cb-net/cmd/agent/agent.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"os"
"os/signal"
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
Expand Down Expand Up @@ -334,7 +333,7 @@ func checkConnectivity(data string, etcdClient *clientv3.Client) {
CBLogger.Tracef("Value: %#v", networkStatus)

size := binary.Size(networkStatusBytes)
CBLogger.WithField("total size", size).Tracef("PutRequest size (bytes)")
CBLogger.Tracef("PutRequest size (bytes): total_size: %v", size)

putResp, err := etcdClient.Put(context.Background(), keyStatusInformation, networkStatueStr)
if err != nil {
Expand Down Expand Up @@ -398,7 +397,7 @@ func initializeAgent(etcdClient *clientv3.Client) {
CBLogger.Tracef("Value %#v", temp)

size := binary.Size(currentHostNetworkInformationBytes)
CBLogger.WithField("total size", size).Tracef("PutRequest size (bytes)")
CBLogger.Tracef("PutRequest size (bytes): total_size: %v", size)

putResp, err := etcdClient.Put(context.TODO(), keyHostNetworkInformation, currentHostNetworkInformationStr)
if err != nil {
Expand All @@ -424,7 +423,7 @@ func updatePeerState(state string, etcdClient *clientv3.Client) {
CBLogger.Tracef("Value: %#v", tempPeer)

size := binary.Size(peerBytes)
CBLogger.WithField("total size", size).Tracef("PutRequest size (bytes)")
CBLogger.Tracef("PutRequest size (bytes): total_size: %v", size)

putResp, err := etcdClient.Put(context.TODO(), keyPeer, peerStr)
if err != nil {
Expand Down Expand Up @@ -491,8 +490,8 @@ func initializeSecret(etcdClient *clientv3.Client) {
}
CBLogger.Tracef("GetResponse: %#v", getResp)

fields := createFieldsForResponseSizes(*getResp)
CBLogger.WithFields(fields).Tracef("GetResponse size (bytes)")
totalSize, headerSize, kvsSize, kvsCount := extractSizes(*getResp)
CBLogger.Tracef("GetResponse size (bytes): total_size: %v, header_size: %v, kvs_size: %v, kvs_count: %v", totalSize, headerSize, kvsSize, kvsCount)

// Set the other hosts' secrets
for _, kv := range getResp.Kvs {
Expand All @@ -517,6 +516,10 @@ func initializeSecret(etcdClient *clientv3.Client) {
// Transaction (compare-and-swap(CAS)) the secret
keySecretHost := fmt.Sprint(etcdkey.Secret + "/" + cladnetID + "/" + hostID)
CBLogger.Debugf("Transaction (compare-and-swap(CAS)) - %v", keySecretHost)
CBLogger.Tracef("Value: %v", base64PublicKey)

size := binary.Size(base64PublicKey)
CBLogger.Tracef("TransactionRequest size (bytes): total_size: %v", size)

// NOTICE: "!=" doesn't work..... It might be a temporal issue.
txnResp, err := etcdClient.Txn(context.TODO()).
Expand Down Expand Up @@ -647,9 +650,8 @@ func getRuleType(etcdClient *clientv3.Client) (string, error) {
return "", etcdErr
}
CBLogger.Tracef("GetResponse: %#v", respCLADNetSpec)

fields := createFieldsForResponseSizes(*respCLADNetSpec)
CBLogger.WithFields(fields).Tracef("GetResponse size (bytes)")
totalSize, headerSize, kvsSize, kvsCount := extractSizes(*respCLADNetSpec)
CBLogger.Tracef("GetResponse size (bytes): total_size: %v, header_size: %v, kvs_size: %v, kvs_count: %v", totalSize, headerSize, kvsSize, kvsCount)

var cladnetSpec model.CLADNetSpecification
if err := json.Unmarshal(respCLADNetSpec.Kvs[0].Value, &cladnetSpec); err != nil {
Expand Down Expand Up @@ -703,7 +705,8 @@ func updateNetworkingRule(thisPeer model.Peer, otherPeers map[string]model.Peer,
CBLogger.Tracef("Value: %#v", networkingRule)

size := binary.Size(networkingRuleBytes)
CBLogger.WithField("total size", size).Tracef("TransactionRequest size (bytes)")
networkingRuleCount := len(networkingRule.HostID)
CBLogger.Tracef("TransactionRequest size (bytes): total_size: %v, networking_rule_count", size, networkingRuleCount)

// NOTICE: "!=" doesn't work..... It might be a temporal issue.
txnResp, err := etcdClient.Txn(context.TODO()).
Expand Down Expand Up @@ -750,7 +753,8 @@ func updatePeerInNetworkingRule(thisPeer model.Peer, otherPeer model.Peer, ruleT
CBLogger.Tracef("Value: %#v", networkingRule)

size := binary.Size(networkingRuleBytes)
CBLogger.WithField("total size", size).Tracef("TransactionRequest size (bytes)")
networkingRuleCount := len(networkingRule.HostID)
CBLogger.Tracef("TransactionRequest size (bytes): total_size: %v, networking_rule_count", size, networkingRuleCount)

// NOTICE: "!=" doesn't work..... It might be a temporal issue.
txnResp, err := etcdClient.Txn(context.TODO()).
Expand Down Expand Up @@ -862,9 +866,8 @@ func main() {
CBLogger.Error(respErr)
}
CBLogger.Tracef("GetResponse: %#v", getResp)

fields := createFieldsForResponseSizes(*getResp)
CBLogger.WithFields(fields).Tracef("GetResponse size (bytes)")
totalSize, headerSize, kvsSize, kvsCount := extractSizes(*getResp)
CBLogger.Tracef("GetResponse size (bytes): total_size: %v, header_size: %v, kvs_size: %v, kvs_count: %v", totalSize, headerSize, kvsSize, kvsCount)

for _, kv := range getResp.Kvs {
key := string(kv.Key)
Expand Down Expand Up @@ -926,28 +929,42 @@ func main() {
CBLogger.Debug("End.........")
}

func createFieldsForResponseSizes(res clientv3.GetResponse) logrus.Fields {
func extractSizes(res clientv3.GetResponse) (totalSize, headerSize, kvsSize, kvsCount int) {

// lenKvs := res.Count
fields := logrus.Fields{}

headerSize := res.Header.Size()
kvSize := 0
headerSize = res.Header.Size()
kvsCount = int(res.Count)
kvsSize = 0
for _, kv := range res.Kvs {
kvSize += kv.Size()
kvsSize += kv.Size()
}

totalSize := headerSize + kvSize
totalSize = headerSize + kvsSize

fields["total size"] = totalSize
fields["header size"] = headerSize
fields["kvs size"] = kvSize
return totalSize, headerSize, kvsSize, kvsCount
}

for i, kv := range res.Kvs {
tempKey := "kv " + strconv.Itoa(i)
fields[tempKey] = kv.Size()
// kvSize += kv.Size()
}
// func createFieldsForResponseSizes(res clientv3.GetResponse) logrus.Fields {

return fields
}
// // lenKvs := res.Count
// fields := logrus.Fields{}

// headerSize := res.Header.Size()
// kvSize := 0
// for _, kv := range res.Kvs {
// kvSize += kv.Size()
// }

// totalSize := headerSize + kvSize

// fields["total size"] = totalSize
// fields["header size"] = headerSize
// fields["kvs size"] = kvSize

// for i, kv := range res.Kvs {
// tempKey := "kv " + strconv.Itoa(i)
// fields[tempKey] = kv.Size()
// // kvSize += kv.Size()
// }

// return fields
// }
28 changes: 21 additions & 7 deletions poc-cb-net/cmd/controller/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,8 +182,8 @@ func watchHostNetworkInformation(wg *sync.WaitGroup, etcdClient *clientv3.Client
}
CBLogger.Tracef("GetResponse: %#v", respRule)

fields := createFieldsForResponseSizes(*respRule)
CBLogger.WithFields(fields).Tracef("GetResponse size (bytes)")
totalSize, headerSize, kvsSize, kvsCount := extractSizes(*respRule)
CBLogger.Tracef("GetResponse size (bytes): total_size: %v, header_size: %v, kvs_size: %v, kvs_count: %v", totalSize, headerSize, kvsSize, kvsCount)

var peer model.Peer

Expand Down Expand Up @@ -212,7 +212,7 @@ func watchHostNetworkInformation(wg *sync.WaitGroup, etcdClient *clientv3.Client
CBLogger.Tracef("Value: %#v", peer)

size := binary.Size(peerBytes)
CBLogger.WithField("total size", size).Tracef("PutRequest size (bytes)")
CBLogger.Tracef("PutRequest size (bytes): total_size: %v", size)

putResp, err := etcdClient.Put(context.TODO(), keyPeer, peerStr)
if err != nil {
Expand Down Expand Up @@ -303,8 +303,8 @@ func getIpv4AddressSpace(etcdClient *clientv3.Client, key string) (string, error
}
CBLogger.Tracef("GetResponse: %#v", respSpec)

fields := createFieldsForResponseSizes(*respSpec)
CBLogger.WithFields(fields).Tracef("GetResponse size (bytes)")
totalSize, headerSize, kvsSize, kvsCount := extractSizes(*respSpec)
CBLogger.Tracef("GetResponse size (bytes): total_size: %v, header_size: %v, kvs_size: %v, kvs_count: %v", totalSize, headerSize, kvsSize, kvsCount)

var tempSpec model.CLADNetSpecification

Expand Down Expand Up @@ -398,8 +398,8 @@ func allocatePeer(cladnetID string, hostID string, hostName string, hostIPv4CIDR
}
CBLogger.Tracef("GetResponse: %#v", resp)

fields := createFieldsForResponseSizes(*resp)
CBLogger.WithFields(fields).Tracef("GetResponse size (bytes)")
totalSize, headerSize, kvsSize, kvsCount := extractSizes(*resp)
CBLogger.Tracef("GetResponse size (bytes): total_size: %v, header_size: %v, kvs_size: %v, kvs_count: %v", totalSize, headerSize, kvsSize, kvsCount)

state := netstate.Configuring
peerIPv4CIDR, peerIPAddress, err := assignIPAddressToPeer(cladnetIpv4AddressSpace, uint32(resp.Count+2))
Expand Down Expand Up @@ -465,6 +465,20 @@ func main() {
CBLogger.Debug("End.........")
}

func extractSizes(res clientv3.GetResponse) (totalSize, headerSize, kvsSize, kvsCount int) {

headerSize = res.Header.Size()
kvsCount = int(res.Count)
kvsSize = 0
for _, kv := range res.Kvs {
kvsSize += kv.Size()
}

totalSize = headerSize + kvsSize

return totalSize, headerSize, kvsSize, kvsCount
}

func createFieldsForResponseSizes(res clientv3.GetResponse) logrus.Fields {

// lenKvs := res.Count
Expand Down
Loading