Added a benchmark for stressing multiple tenants

Also, added a test to travis (in Makefile)

Co-authored-by: NiuJ1ao <NiuJ1ao@github.com>

Makefile

@@ -79,6 +79,10 @@ bench:
 	sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestBenchParallelServe -args $(WITHSNAPSHOTS) $(WITHLAZY) -benchDirTest configLazy -metricsTest
 	./scripts/clean_fcctr.sh
 
+	sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.out 2>$(CTRDLOGDIR)/fccd_orch_noupf_log_bench.err &
+	sudo env "PATH=$(PATH)" go test $(EXTRAGOARGS) -run TestBenchRequestPerSecond -args $(WITHSNAPSHOTS) -benchDirTest configBase -metricsTest
+	./scripts/clean_fcctr.sh
+
 test-man-bench:
 	$(MAKE) test-man
 	$(MAKE) bench

perf_bench_test.go

@@ -0,0 +1,150 @@
+// MIT License
+//
+// Copyright (c) 2020 Yuchen Niu
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package main
+
+import (
+	"context"
+	"flag"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	isColdStart     = flag.Bool("coldStart", false, "Profile cold starts (default is false)")
+	vmNum           = flag.Int("vm", 2, "The number of VMs")
+	targetReqPerSec = flag.Int("requestPerSec", 2, "The target number of requests per second")
+	executionTime   = flag.Int("executionTime", 2, "The execution time of the benchmark in seconds")
+)
+
+func TestBenchRequestPerSecond(t *testing.T) {
+	var (
+		servedTh      uint64
+		pinnedFuncNum int
+		vmID          int
+		isSyncOffload bool = true
+		images             = getImages()
+		funcs              = []string{}
+		timeInterval       = time.Duration(time.Second.Nanoseconds() / int64(*targetReqPerSec))
+		totalRequests      = *executionTime * *targetReqPerSec
+	)
+
+	log.SetLevel(log.InfoLevel)
+
+	funcPool = NewFuncPool(!isSaveMemoryConst, servedTh, pinnedFuncNum, isTestModeConst)
+
+	createResultsDir()
+
+	// Pull images
+	for funcName, imageName := range images {
+		resp, _, err := funcPool.Serve(context.Background(), "plr_fnc", imageName, "record")
+
+		require.NoError(t, err, "Function returned error")
+		require.Equal(t, resp.Payload, "Hello, record_response!")
+
+		// For future use
+		// createSnapshots(t, concurrency, vmID, imageName, isSyncOffload)
+		// log.Info("Snapshot created")
+		// createRecords(t, concurrency, vmID, imageName, isSyncOffload)
+		// log.Info("Record done")
+
+		funcs = append(funcs, funcName)
+	}
+
+	// Boot VMs
+	for i := 0; i < *vmNum; i++ {
+		vmIDString := strconv.Itoa(i)
+		_, err := funcPool.AddInstance(vmIDString, images[funcs[i%len(funcs)]])
+		require.NoError(t, err, "Function returned error")
+	}
+
+	if !*isWithCache && *isColdStart {
+		log.Info("Profile cold start")
+		dropPageCache()
+	}
+
+	ticker := time.NewTicker(timeInterval)
+	var vmGroup sync.WaitGroup
+	done := make(chan bool, 1)
+
+	for totalRequests > 0 {
+		select {
+		case <-ticker.C:
+			totalRequests--
+			vmGroup.Add(1)
+
+			funcName := funcs[vmID%len(funcs)]
+			imageName := images[funcName]
+			vmIDString := strconv.Itoa(vmID)
+
+			tStart := time.Now()
+			go serveVM(t, tStart, vmIDString, imageName, &vmGroup, isSyncOffload)
+
+			vmID = (vmID + 1) % *vmNum
+		case <-done:
+			ticker.Stop()
+		}
+	}
+
+	done <- true
+	vmGroup.Wait()
+}
+
+///////////////////////////////////////////////////////////////////////////////
+////////////////////////// Auxialiary functions below /////////////////////////
+///////////////////////////////////////////////////////////////////////////////
+func serveVM(t *testing.T, start time.Time, vmIDString, imageName string, vmGroup *sync.WaitGroup, isSyncOffload bool) {
+	defer vmGroup.Done()
+
+	resp, _, err := funcPool.Serve(context.Background(), vmIDString, imageName, "replay")
+	require.NoError(t, err, "Function returned error")
+	require.Equal(t, resp.Payload, "Hello, replay_response!")
+
+	if *isColdStart {
+		require.Equal(t, resp.IsColdStart, true)
+		message, err := funcPool.RemoveInstance(vmIDString, imageName, isSyncOffload)
+		require.NoError(t, err, "Function returned error, "+message)
+	} else {
+		require.Equal(t, resp.IsColdStart, false)
+	}
+
+	log.Debugf("vmID %s: returned in %f seconds", vmIDString, time.Since(start).Seconds())
+}
+
+func getImages() map[string]string {
+	return map[string]string{
+		"helloworld": "ustiugov/helloworld:var_workload",
+		// "chameleon":    "ustiugov/chameleon:var_workload",
+		// "pyaes":        "ustiugov/pyaes:var_workload",
+		// "image_rotate": "ustiugov/image_rotate:var_workload",
+		// "json_serdes":  "ustiugov/json_serdes:var_workload",
+		// "lr_serving":   "ustiugov/lr_serving:var_workload",
+		// "cnn_serving":  "ustiugov/cnn_serving:var_workload",
+		// "rnn_serving":  "ustiugov/rnn_serving:var_workload",
+		// "lr_training":  "ustiugov/lr_training:var_workload",
+	}
+}