From 25728c1fa5bf3fd4c4c5aebff171072f0a11e041 Mon Sep 17 00:00:00 2001 From: Plamen Petrov Date: Thu, 23 Jul 2020 08:24:02 -0400 Subject: [PATCH] Serve benchmarking Signed-off-by: Plamen Petrov --- bench_test.go | 132 ++++++++++++++++++++++++++++++++++++++ ctriface/bench_test.go | 16 +++-- fccd-orchestrator.go | 3 +- fccd-orchestrator_test.go | 18 +++--- functions.go | 35 +++++++--- go.mod | 5 +- go.sum | 29 +++++++++ manual_cleanup_test.go | 12 ++-- metrics/serveStat.go | 20 +++--- 9 files changed, 229 insertions(+), 41 deletions(-) create mode 100644 bench_test.go diff --git a/bench_test.go b/bench_test.go new file mode 100644 index 000000000..79cf6d1fa --- /dev/null +++ b/bench_test.go @@ -0,0 +1,132 @@ +// MIT License +// +// Copyright (c) 2020 Plamen Petrov +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package main + +import ( + "context" + "os" + "os/exec" + "testing" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "github.com/ustiugov/fccd-orchestrator/metrics" +) + +func TestBenchmarkServeWithCache(t *testing.T) { + fID := "1" + imageName := "ustiugov/helloworld:runner_workload" + var ( + servedTh uint64 + pinnedFuncNum int + isSyncOffload bool = true + ) + funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst) + + // Pull image to work around parallel pulling limitation + resp, _, err := funcPool.Serve(context.Background(), "plr_fnc", imageName, "world") + require.NoError(t, err, "Function returned error") + require.Equal(t, resp.Payload, "Hello, world!") + + message, err := funcPool.RemoveInstance("plr_fnc", imageName, isSyncOffload) + require.NoError(t, err, "Function returned error, "+message) + // ----------------------------------------------------- + + resp, _, err = funcPool.Serve(context.Background(), fID, imageName, "world") + require.NoError(t, err, "Function returned error") + require.Equal(t, resp.Payload, "Hello, world!") + + message, err = funcPool.RemoveInstance(fID, imageName, isSyncOffload) + require.NoError(t, err, "Function returned error, "+message) + + benchCount := 10 + serveStats := make([]*metrics.ServeStat, benchCount) + + for i := 0; i < 10; i++ { + resp, stat, err := funcPool.Serve(context.Background(), fID, imageName, "world") + require.NoError(t, err, "Function returned error") + require.Equal(t, resp.Payload, "Hello, world!") + + serveStats[i] = stat + + message, err := funcPool.RemoveInstance(fID, imageName, isSyncOffload) + require.NoError(t, err, "Function returned error, "+message) + } + + metrics.PrintServeStats(serveStats...) +} + +func TestBenchmarkServeNoCache(t *testing.T) { + fID := "1" + imageName := "ustiugov/helloworld:runner_workload" + var ( + servedTh uint64 + pinnedFuncNum int + isSyncOffload bool = true + ) + funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst) + + // Pull image to work around parallel pulling limitation + resp, _, err := funcPool.Serve(context.Background(), "plr_fnc", imageName, "world") + require.NoError(t, err, "Function returned error") + require.Equal(t, resp.Payload, "Hello, world!") + + message, err := funcPool.RemoveInstance("plr_fnc", imageName, isSyncOffload) + require.NoError(t, err, "Function returned error, "+message) + // ----------------------------------------------------- + + resp, _, err = funcPool.Serve(context.Background(), fID, imageName, "world") + require.NoError(t, err, "Function returned error") + require.Equal(t, resp.Payload, "Hello, world!") + + message, err = funcPool.RemoveInstance(fID, imageName, isSyncOffload) + require.NoError(t, err, "Function returned error, "+message) + + benchCount := 10 + serveStats := make([]*metrics.ServeStat, benchCount) + + for i := 0; i < 10; i++ { + dropPageCache() + + resp, stat, err := funcPool.Serve(context.Background(), fID, imageName, "world") + require.NoError(t, err, "Function returned error") + require.Equal(t, resp.Payload, "Hello, world!") + + serveStats[i] = stat + + message, err := funcPool.RemoveInstance(fID, imageName, isSyncOffload) + require.NoError(t, err, "Function returned error, "+message) + } + + metrics.PrintServeStats(serveStats...) +} + +func dropPageCache() { + cmd := exec.Command("sudo", "/bin/sh", "-c", "sync; echo 1 > /proc/sys/vm/drop_caches") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stdout + + if err := cmd.Run(); err != nil { + log.Fatalf("Failed to drop caches: %v", err) + } +} diff --git a/ctriface/bench_test.go b/ctriface/bench_test.go index e34cc2515..be4824c78 100644 --- a/ctriface/bench_test.go +++ b/ctriface/bench_test.go @@ -101,8 +101,8 @@ func TestBenchmarkLoadSnapshotWithCache(t *testing.T) { benchCount := 10 loadStats := make([]*metrics.LoadSnapshotStat, benchCount) - snapshotFile := "/dev/snapshot_file" - memFile := "/dev/mem_file" + snapshotFile := "/snapshot_file" + memFile := "/mem_file" // Pull image and prepare snapshot message, _, err := orch.StartVM(ctx, vmID, "ustiugov/helloworld:runner_workload") @@ -119,6 +119,14 @@ func TestBenchmarkLoadSnapshotWithCache(t *testing.T) { time.Sleep(300 * time.Millisecond) + message, _, err = orch.LoadSnapshot(ctx, vmID, snapshotFile, memFile) + require.NoError(t, err, "Failed to load snapshot of VM, "+message) + + message, err = orch.Offload(ctx, vmID) + require.NoError(t, err, "Failed to offload VM, "+message) + + time.Sleep(300 * time.Millisecond) + for i := 0; i < benchCount; i++ { message, stat, err := orch.LoadSnapshot(ctx, vmID, snapshotFile, memFile) require.NoError(t, err, "Failed to load snapshot of VM, "+message) @@ -158,8 +166,8 @@ func TestBenchmarkLoadSnapshotNoCache(t *testing.T) { benchCount := 10 loadStats := make([]*metrics.LoadSnapshotStat, benchCount) - snapshotFile := "/dev/snapshot_file" - memFile := "/dev/mem_file" + snapshotFile := "/snapshot_file" + memFile := "/mem_file" // Pull image and prepare snapshot message, _, err := orch.StartVM(ctx, vmID, "ustiugov/helloworld:runner_workload") diff --git a/fccd-orchestrator.go b/fccd-orchestrator.go index 839fcade9..34f1c38f2 100644 --- a/fccd-orchestrator.go +++ b/fccd-orchestrator.go @@ -192,5 +192,6 @@ func (s *fwdServer) FwdHello(ctx context.Context, in *hpb.FwdHelloReq) (*hpb.Fwd logger := log.WithFields(log.Fields{"fID": fID, "image": imageName, "payload": payload}) logger.Debug("Received FwdHelloVM") - return funcPool.Serve(ctx, fID, imageName, payload) + resp, _, err := funcPool.Serve(ctx, fID, imageName, payload) + return resp, err } diff --git a/fccd-orchestrator_test.go b/fccd-orchestrator_test.go index a4b3171f2..1e44f5b29 100644 --- a/fccd-orchestrator_test.go +++ b/fccd-orchestrator_test.go @@ -89,7 +89,7 @@ func TestSendToFunctionSerial(t *testing.T) { funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst) for i := 0; i < 2; i++ { - resp, err := funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error") if i == 0 { require.Equal(t, resp.IsColdStart, true) @@ -117,7 +117,7 @@ func TestSendToFunctionParallel(t *testing.T) { go func(i int) { defer vmGroup.Done() - resp, err := funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error") require.Equal(t, resp.Payload, "Hello, world!") }(i) @@ -140,7 +140,7 @@ func TestStartSendStopTwice(t *testing.T) { for i := 0; i < 2; i++ { for k := 0; k < 2; k++ { - resp, err := funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error") require.Equal(t, resp.Payload, "Hello, world!") } @@ -164,7 +164,7 @@ func TestStatsNotNumericFunction(t *testing.T) { ) funcPool = NewFuncPool(isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst) - resp, err := funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error") require.Equal(t, resp.Payload, "Hello, world!") @@ -186,7 +186,7 @@ func TestStatsNotColdFunction(t *testing.T) { ) funcPool = NewFuncPool(isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst) - resp, err := funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error") require.Equal(t, resp.Payload, "Hello, world!") @@ -209,7 +209,7 @@ func TestSaveMemorySerial(t *testing.T) { funcPool = NewFuncPool(isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst) for i := 0; i < 100; i++ { - resp, err := funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error") require.Equal(t, resp.Payload, "Hello, world!") } @@ -237,7 +237,7 @@ func TestSaveMemoryParallel(t *testing.T) { go func(i int) { defer vmGroup.Done() - resp, err := funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error") require.Equal(t, resp.Payload, "Hello, world!") }(i) @@ -264,7 +264,7 @@ func TestDirectStartStopVM(t *testing.T) { message, err := funcPool.AddInstance(fID, imageName) require.NoError(t, err, "This error should never happen (addInstance())"+message) - resp, err := funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error") require.Equal(t, resp.Payload, "Hello, world!") @@ -300,7 +300,7 @@ func TestAllFunctions(t *testing.T) { go func(fID int, imageName, request, response string) { defer vmGroup.Done() - resp, err := funcPool.Serve(context.Background(), strconv.Itoa(8+fID), imageName, request) + resp, _, err := funcPool.Serve(context.Background(), strconv.Itoa(8+fID), imageName, request) require.NoError(t, err, "Function returned error") require.Equal(t, resp.Payload, "Hello, "+response+"!") diff --git a/functions.go b/functions.go index f5ade070d..2889679b4 100644 --- a/functions.go +++ b/functions.go @@ -38,6 +38,7 @@ import ( log "github.com/sirupsen/logrus" hpb "github.com/ustiugov/fccd-orchestrator/helloworld" + "github.com/ustiugov/fccd-orchestrator/metrics" ) var isTestMode bool // set with a call to NewFuncPool @@ -109,7 +110,7 @@ func (p *FuncPool) getFunction(fID, imageName string) *Function { } // Serve Service RPC request by triggering the corresponding function. -func (p *FuncPool) Serve(ctx context.Context, fID, imageName, payload string) (*hpb.FwdHelloResp, error) { +func (p *FuncPool) Serve(ctx context.Context, fID, imageName, payload string) (*hpb.FwdHelloResp, *metrics.ServeStat, error) { f := p.getFunction(fID, imageName) return f.Serve(ctx, fID, imageName, payload) @@ -204,7 +205,12 @@ func NewFunction(fID, imageName string, Stats *Stats, servedTh uint64, isToPin b // b. The last goroutine is determined by the atomic counter: the goroutine wih syncID==0 shuts down // the instance. // c. Instance shutdown is performed asynchronously because all instances have unique IDs. -func (f *Function) Serve(ctx context.Context, fID, imageName, reqPayload string) (*hpb.FwdHelloResp, error) { +func (f *Function) Serve(ctx context.Context, fID, imageName, reqPayload string) (*hpb.FwdHelloResp, *metrics.ServeStat, error) { + var ( + serveStat *metrics.ServeStat = metrics.NewServeStat() + tStart time.Time + ) + syncID := int64(-1) // default is no synchronization logger := log.WithFields(log.Fields{"fID": f.fID}) @@ -221,12 +227,15 @@ func (f *Function) Serve(ctx context.Context, fID, imageName, reqPayload string) f.stats.IncServed(f.fID) + tStart = time.Now() + f.OnceAddInstance.Do( func() { isColdStart = true logger.Debug("Function is inactive, starting the instance...") f.AddInstance() }) + serveStat.AddInstance = time.Since(tStart).Microseconds() f.RLock() @@ -234,22 +243,26 @@ func (f *Function) Serve(ctx context.Context, fID, imageName, reqPayload string) // Eventually, it needs to be RPC-dependent and probably client-defined ctxFwd, cancel := context.WithDeadline(context.Background(), time.Now().Add(20*time.Second)) defer cancel() + + tStart = time.Now() resp, err := f.fwdRPC(ctxFwd, reqPayload) + serveStat.GetResponse = time.Since(tStart).Microseconds() + if err != nil && ctxFwd.Err() == context.Canceled { // context deadline exceeded f.RUnlock() - return &hpb.FwdHelloResp{IsColdStart: isColdStart, Payload: ""}, err + return &hpb.FwdHelloResp{IsColdStart: isColdStart, Payload: ""}, serveStat, err } else if err != nil { if e, ok := status.FromError(err); ok { switch e.Code() { case codes.DeadlineExceeded: // deadline exceeded f.RUnlock() - return &hpb.FwdHelloResp{IsColdStart: isColdStart, Payload: ""}, err + return &hpb.FwdHelloResp{IsColdStart: isColdStart, Payload: ""}, serveStat, err default: logger.Warn("Function returned error: ", err) f.RUnlock() - return &hpb.FwdHelloResp{IsColdStart: isColdStart, Payload: ""}, err + return &hpb.FwdHelloResp{IsColdStart: isColdStart, Payload: ""}, serveStat, err } } else { logger.Panic("Not able to parse error returned ", err) @@ -257,6 +270,8 @@ func (f *Function) Serve(ctx context.Context, fID, imageName, reqPayload string) } f.RUnlock() + tStart = time.Now() + if !f.isPinnedInMem && syncID == 0 { logger.Debugf("Function has to shut down its instance, served %d requests", f.GetStatServed()) if _, err := f.RemoveInstance(false); err != nil { @@ -267,7 +282,9 @@ func (f *Function) Serve(ctx context.Context, fID, imageName, reqPayload string) f.sem.Release(int64(f.servedTh)) } - return &hpb.FwdHelloResp{IsColdStart: isColdStart, Payload: resp.Message}, err + serveStat.RetireOld = time.Since(tStart).Microseconds() + + return &hpb.FwdHelloResp{IsColdStart: isColdStart, Payload: resp.Message}, serveStat, err } // FwdRPC Forward the RPC to an instance, then forwards the response back. @@ -415,7 +432,7 @@ func (f *Function) LoadInstance() { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - message, err := orch.LoadSnapshot(ctx, f.vmID, f.getSnapshotFilePath(), f.getMemFilePath()) + message, _, err := orch.LoadSnapshot(ctx, f.vmID, f.getSnapshotFilePath(), f.getMemFilePath()) if err != nil { log.Panic(message, err) } @@ -443,10 +460,10 @@ func (f *Function) getVMID() string { // getSnapshotFilePath Creates the snapshot file path for the function func (f *Function) getSnapshotFilePath() string { - return fmt.Sprintf(filepath.Join("/dev", "snap_file_%s"), f.vmID) + return fmt.Sprintf(filepath.Join("/root", "snap_file_%s"), f.vmID) } // getMemFilePath Creates the memory file path for the function func (f *Function) getMemFilePath() string { - return fmt.Sprintf(filepath.Join("/dev", "mem_file_%s"), f.vmID) + return fmt.Sprintf(filepath.Join("/root", "mem_file_%s"), f.vmID) } diff --git a/go.mod b/go.mod index 9cea5b1e7..953c5376f 100644 --- a/go.mod +++ b/go.mod @@ -13,9 +13,10 @@ require ( github.com/containerd/containerd v1.3.6 github.com/golang/protobuf v1.3.3 github.com/sirupsen/logrus v1.6.0 - github.com/stretchr/testify v1.5.1 - github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200722073430-90c1342030ac + github.com/stretchr/testify v1.6.1 + github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200723105930-96d5e1cb279b github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200717125634-528c6e9f9cc9 + github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200723105930-96d5e1cb279b github.com/ustiugov/fccd-orchestrator/misc v0.0.0-20200717125634-528c6e9f9cc9 // indirect github.com/ustiugov/fccd-orchestrator/proto v0.0.0-20200717125634-528c6e9f9cc9 github.com/ustiugov/fccd-orchestrator/taps v0.0.0-20200717125634-528c6e9f9cc9 // indirect diff --git a/go.sum b/go.sum index 82063efb1..5d259561d 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,7 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -104,6 +105,7 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/firecracker-microvm/firecracker-containerd v0.0.0-20200331220105-afedbc74f5ee h1:SKPxUz0fLm3Mf/ZWBTUHuXp/2vLHf+UvtxW3W5KNt/s= github.com/firecracker-microvm/firecracker-containerd v0.0.0-20200331220105-afedbc74f5ee/go.mod h1:7B+eRdKxqB3zSSsXsz5Jp+cGXrLgdyQIhEkT1Pfj+Zk= github.com/firecracker-microvm/firecracker-go-sdk v0.21.0/go.mod h1:zyc9BrKGePpNLbQ5y2ZtdzXEfpMJeHPeFNVpyo0S1WQ= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= @@ -174,6 +176,7 @@ github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -212,6 +215,7 @@ github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6t github.com/juju/errors v0.0.0-20180806074554-22422dad46e1/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/testing v0.0.0-20190613124551-e81189438503/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -296,6 +300,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -373,6 +379,10 @@ github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200722072720-7c2beba403d github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200722072720-7c2beba403de/go.mod h1:UcNV4o8382RvEIe3gb19tIA7wRcZ+Q0DCbmSUCl7tPM= github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200722073430-90c1342030ac h1:r+eb9hmonXVp1R3X6Irp9kY3kU0sA71cZFPxSLZAvZ0= github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200722073430-90c1342030ac/go.mod h1:UcNV4o8382RvEIe3gb19tIA7wRcZ+Q0DCbmSUCl7tPM= +github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200723101147-e9f1a1f5bf4c h1:feusW90Da1zUR2jwxvVfIEYK2B4HUv71+luKk8X9Z6E= +github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200723101147-e9f1a1f5bf4c/go.mod h1:XbU2ULnOfHWIdbLhyp4zaieQ0cIrvUzP9aDsgq3GkGQ= +github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200723105930-96d5e1cb279b h1:jOckr1RkPU23LYyLmwXK30TcEIn21RAAr4fLT4i+kMc= +github.com/ustiugov/fccd-orchestrator/ctriface v0.0.0-20200723105930-96d5e1cb279b/go.mod h1:XbU2ULnOfHWIdbLhyp4zaieQ0cIrvUzP9aDsgq3GkGQ= github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200710144657-9fbec6857e48/go.mod h1:5dhCs/XynpQoQcrhd/YgUBjGahhNpTknQUcC1kHRCaA= github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200710145415-bb09d1a68889/go.mod h1:5dhCs/XynpQoQcrhd/YgUBjGahhNpTknQUcC1kHRCaA= github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200710150633-096cac68bd72 h1:r67pqykSYWZHFYfIKuT44PTXFJWu5lJcatkV16d3vKU= @@ -386,6 +396,10 @@ github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200714162243-d6dc0c083 github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200714162243-d6dc0c083e9e/go.mod h1:5dhCs/XynpQoQcrhd/YgUBjGahhNpTknQUcC1kHRCaA= github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200717125634-528c6e9f9cc9 h1:1EQvsISd3xbC1krrXCCuIsingP9PF4M6pfCTXPnhYb4= github.com/ustiugov/fccd-orchestrator/helloworld v0.0.0-20200717125634-528c6e9f9cc9/go.mod h1:5dhCs/XynpQoQcrhd/YgUBjGahhNpTknQUcC1kHRCaA= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200723085111-a9d697412510 h1:2F3Bp/P7C2UMKU+vakjEW/++E2yCntucHKNiRxcxQOE= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200723085111-a9d697412510/go.mod h1:NOrYF6usVkdr6gpXAP7pk7jp1MlAXlrZQuUEOVhRuV8= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200723105930-96d5e1cb279b h1:0ukOv2nUQqj14TLHf5MkVyvbyiV6AxwUSI7PQQnVdKQ= +github.com/ustiugov/fccd-orchestrator/metrics v0.0.0-20200723105930-96d5e1cb279b/go.mod h1:NOrYF6usVkdr6gpXAP7pk7jp1MlAXlrZQuUEOVhRuV8= github.com/ustiugov/fccd-orchestrator/misc v0.0.0-20200710145415-bb09d1a68889/go.mod h1:Fz+jqGMI/infxs9IhgBZbmW/dtZ7odhhD8cg6W+gAaA= github.com/ustiugov/fccd-orchestrator/misc v0.0.0-20200710150633-096cac68bd72 h1:yDwPSYKCtJaqI2NqVD0ugI7cnCg+ynnEOiVoLXHcWOg= github.com/ustiugov/fccd-orchestrator/misc v0.0.0-20200710150633-096cac68bd72/go.mod h1:Fz+jqGMI/infxs9IhgBZbmW/dtZ7odhhD8cg6W+gAaA= @@ -444,7 +458,11 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -512,11 +530,13 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -531,9 +551,15 @@ golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6 h1:nULzSsKgihxFGLnQFv2T7lE golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200721032237-77f530d86f9a/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200721223218-6123e77877b2/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200723000907-a7c6fd066f6d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.7.0 h1:Hdks0L0hgznZLG9nzXb8vZ0rRvqNvAcgAp84y7Mwkgw= +gonum.org/v1/gonum v0.7.0/go.mod h1:L02bwd0sqlsvRv41G7wGWFCsVNZFv/k1xzGIxeANHGM= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -568,8 +594,11 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/manual_cleanup_test.go b/manual_cleanup_test.go index adae2ec55..688041186 100644 --- a/manual_cleanup_test.go +++ b/manual_cleanup_test.go @@ -41,7 +41,7 @@ func TestParallelServe(t *testing.T) { funcPool = NewFuncPool(isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst) // Pull image to work around parallel pulling limitation - resp, err := funcPool.Serve(context.Background(), "plr_fnc", imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), "plr_fnc", imageName, "world") require.NoError(t, err, "Function returned error") require.Equal(t, resp.Payload, "Hello, world!") // ----------------------------------------------------- @@ -55,11 +55,11 @@ func TestParallelServe(t *testing.T) { defer vmGroup.Done() fID := strconv.Itoa(15 + i) - resp, err := funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error on 1st run") require.Equal(t, resp.Payload, "Hello, world!") - resp, err = funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err = funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error on 2nd run") require.Equal(t, resp.Payload, "Hello, world!") }(i) @@ -76,16 +76,16 @@ func TestServeThree(t *testing.T) { ) funcPool = NewFuncPool(isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst) - resp, err := funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err := funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error on 1st run") require.Equal(t, resp.IsColdStart, true) require.Equal(t, resp.Payload, "Hello, world!") - resp, err = funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err = funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error on 2nd run") require.Equal(t, resp.Payload, "Hello, world!") - resp, err = funcPool.Serve(context.Background(), fID, imageName, "world") + resp, _, err = funcPool.Serve(context.Background(), fID, imageName, "world") require.NoError(t, err, "Function returned error on 3rd run") require.Equal(t, resp.Payload, "Hello, world!") } diff --git a/metrics/serveStat.go b/metrics/serveStat.go index 2a26be571..c3b438083 100644 --- a/metrics/serveStat.go +++ b/metrics/serveStat.go @@ -46,21 +46,21 @@ func (s *ServeStat) PrintTotal() { // PrintAll Prints a breakdown of the time it took to Serve func (s *ServeStat) PrintAll() { - fmt.Printf("Serve Stats \tus\n") - fmt.Printf("GetImage \t%d\n", s.AddInstance) - fmt.Printf("FcCreateVM \t%d\n", s.GetResponse) - fmt.Printf("NewContainer \t%d\n", s.RetireOld) - fmt.Printf("Total \t%d\n", s.Total()) + fmt.Printf("Serve Stats \tus\n") + fmt.Printf("AddInstance \t%d\n", s.AddInstance) + fmt.Printf("GetResponse \t%d\n", s.GetResponse) + fmt.Printf("RetireOld \t%d\n", s.RetireOld) + fmt.Printf("Total \t%d\n", s.Total()) } // PrintServeStats prints the mean and // standard deviation of each component of // ServeStat statistics -func PrintServeStats(startVMstats ...*ServeStat) { - addInstances := make([]float64, len(startVMstats)) - getResponses := make([]float64, len(startVMstats)) - retireOlds := make([]float64, len(startVMstats)) - totals := make([]float64, len(startVMstats)) +func PrintServeStats(serveStats ...*ServeStat) { + addInstances := make([]float64, len(serveStats)) + getResponses := make([]float64, len(serveStats)) + retireOlds := make([]float64, len(serveStats)) + totals := make([]float64, len(serveStats)) for i, s := range startVMstats { addInstances[i] = float64(s.AddInstance)