diff --git a/ctriface/bench_test.go b/ctriface/bench_test.go index c5180ef38..5eda0e888 100644 --- a/ctriface/bench_test.go +++ b/ctriface/bench_test.go @@ -56,7 +56,7 @@ func TestBenchmarkStart(t *testing.T) { ctx, cancel := context.WithTimeout(namespaces.WithNamespace(context.Background(), namespaceName), testTimeout) defer cancel() - orch := NewOrchestrator("devmapper", WithTestModeOn(true), WithUPF(*isUPFEnabled)) + orch := NewOrchestrator("devmapper", "", WithTestModeOn(true), WithUPF(*isUPFEnabled)) images := getAllImages() benchCount := 10 diff --git a/ctriface/failing_test.go b/ctriface/failing_test.go index 8edd3c0c9..0f07caecb 100644 --- a/ctriface/failing_test.go +++ b/ctriface/failing_test.go @@ -51,7 +51,7 @@ func TestStartSnapStop(t *testing.T) { ctx, cancel := context.WithTimeout(namespaces.WithNamespace(context.Background(), namespaceName), testTimeout) defer cancel() - orch := NewOrchestrator("devmapper", WithTestModeOn(true)) + orch := NewOrchestrator("devmapper", "", WithTestModeOn(true)) vmID := "2" diff --git a/ctriface/iface.go b/ctriface/iface.go index faa125cf3..794ef1d10 100644 --- a/ctriface/iface.go +++ b/ctriface/iface.go @@ -26,6 +26,8 @@ import ( "context" "fmt" "os" + "os/exec" + "strings" "sync" "syscall" "time" @@ -73,7 +75,7 @@ func (o *Orchestrator) StartVM(ctx context.Context, vmID, imageName string) (_ * logger := log.WithFields(log.Fields{"vmID": vmID, "image": imageName}) logger.Debug("StartVM: Received StartVM") - vm, err := o.vmPool.Allocate(vmID) + vm, err := o.vmPool.Allocate(vmID, o.hostIface) if err != nil { logger.Error("failed to allocate VM in VM pool") return nil, nil, err @@ -288,6 +290,24 @@ func (o *Orchestrator) getImage(ctx context.Context, imageName string) (*contain return &image, nil } +func getK8sDNS() []string { + //using googleDNS as a backup + dnsIPs := []string{"8.8.8.8"} + //get k8s DNS clusterIP + cmd := exec.Command( + "kubectl", "get", "service", "-n", "kube-system", "kube-dns", "-o=custom-columns=:.spec.clusterIP", "--no-headers", + ) + stdoutStderr, err := cmd.CombinedOutput() + if err != nil { + log.Warnf("Failed to Fetch k8s dns clusterIP %v\n%s\n", err, stdoutStderr) + log.Warnf("Using google dns %s\n", dnsIPs[0]) + } else { + //adding k8s DNS clusterIP to the list + dnsIPs = []string{strings.TrimSpace(string(stdoutStderr)), dnsIPs[0]} + } + return dnsIPs +} + func (o *Orchestrator) getVMConfig(vm *misc.VM) *proto.CreateVMRequest { kernelArgs := "ro noapic reboot=k panic=1 pci=off nomodules systemd.log_color=false systemd.unit=firecracker.target init=/sbin/overlay-init tsc=reliable quiet 8250.nr_uarts=0 ipv6.disable=1" @@ -306,6 +326,7 @@ func (o *Orchestrator) getVMConfig(vm *misc.VM) *proto.CreateVMRequest { IPConfig: &proto.IPConfiguration{ PrimaryAddr: vm.Ni.PrimaryAddress + vm.Ni.Subnet, GatewayAddr: vm.Ni.GatewayAddress, + Nameservers: getK8sDNS(), }, }, }}, @@ -478,7 +499,7 @@ func (o *Orchestrator) Offload(ctx context.Context, vmID string) error { return err } - if err := o.vmPool.RecreateTap(vmID); err != nil { + if err := o.vmPool.RecreateTap(vmID, o.hostIface); err != nil { logger.Error("Failed to recreate tap upon offloading") return err } diff --git a/ctriface/iface_test.go b/ctriface/iface_test.go index cbc455374..e3f7ac5f5 100644 --- a/ctriface/iface_test.go +++ b/ctriface/iface_test.go @@ -60,6 +60,7 @@ func TestPauseSnapResume(t *testing.T) { orch := NewOrchestrator( "devmapper", + "", WithTestModeOn(true), WithUPF(*isUPFEnabled), WithLazyMode(*isLazyMode), @@ -102,6 +103,7 @@ func TestStartStopSerial(t *testing.T) { orch := NewOrchestrator( "devmapper", + "", WithTestModeOn(true), WithUPF(*isUPFEnabled), WithLazyMode(*isLazyMode), @@ -135,6 +137,7 @@ func TestPauseResumeSerial(t *testing.T) { orch := NewOrchestrator( "devmapper", + "", WithTestModeOn(true), WithUPF(*isUPFEnabled), WithLazyMode(*isLazyMode), @@ -175,6 +178,7 @@ func TestStartStopParallel(t *testing.T) { vmNum := 10 orch := NewOrchestrator( "devmapper", + "", WithTestModeOn(true), WithUPF(*isUPFEnabled), WithLazyMode(*isLazyMode), @@ -233,6 +237,7 @@ func TestPauseResumeParallel(t *testing.T) { vmNum := 10 orch := NewOrchestrator( "devmapper", + "", WithTestModeOn(true), WithUPF(*isUPFEnabled), WithLazyMode(*isLazyMode), diff --git a/ctriface/manual_cleanup_test.go b/ctriface/manual_cleanup_test.go index d60d40824..c4c4fd33a 100644 --- a/ctriface/manual_cleanup_test.go +++ b/ctriface/manual_cleanup_test.go @@ -55,6 +55,7 @@ func TestSnapLoad(t *testing.T) { orch := NewOrchestrator( "devmapper", + "", WithTestModeOn(true), WithUPF(*isUPFEnabled), WithLazyMode(*isLazyMode), @@ -104,6 +105,7 @@ func TestSnapLoadMultiple(t *testing.T) { orch := NewOrchestrator( "devmapper", + "", WithTestModeOn(true), WithUPF(*isUPFEnabled), WithLazyMode(*isLazyMode), @@ -165,6 +167,7 @@ func TestParallelSnapLoad(t *testing.T) { orch := NewOrchestrator( "devmapper", + "", WithTestModeOn(true), WithUPF(*isUPFEnabled), WithLazyMode(*isLazyMode), @@ -226,6 +229,7 @@ func TestParallelPhasedSnapLoad(t *testing.T) { orch := NewOrchestrator( "devmapper", + "", WithTestModeOn(true), WithUPF(*isUPFEnabled), WithLazyMode(*isLazyMode), diff --git a/ctriface/orch.go b/ctriface/orch.go index b853aead3..bf1203398 100644 --- a/ctriface/orch.go +++ b/ctriface/orch.go @@ -65,12 +65,13 @@ type Orchestrator struct { isLazyMode bool snapshotsDir string isMetricsMode bool + hostIface string memoryManager *manager.MemoryManager } // NewOrchestrator Initializes a new orchestrator -func NewOrchestrator(snapshotter string, opts ...OrchestratorOption) *Orchestrator { +func NewOrchestrator(snapshotter, hostIface string, opts ...OrchestratorOption) *Orchestrator { var err error o := new(Orchestrator) @@ -78,6 +79,7 @@ func NewOrchestrator(snapshotter string, opts ...OrchestratorOption) *Orchestrat o.cachedImages = make(map[string]containerd.Image) o.snapshotter = snapshotter o.snapshotsDir = "/fccd/snapshots" + o.hostIface = hostIface for _, opt := range opts { opt(o) diff --git a/ctriface/orch_options.go b/ctriface/orch_options.go index 83fb17be9..8b941896f 100644 --- a/ctriface/orch_options.go +++ b/ctriface/orch_options.go @@ -72,3 +72,11 @@ func WithMetricsMode(isMetricsMode bool) OrchestratorOption { o.isMetricsMode = isMetricsMode } } + +// WithCustomHostIface Sets the custom host net interface +// for the VMs to link to +func WithCustomHostIface(hostIface string) OrchestratorOption { + return func(o *Orchestrator) { + o.hostIface = hostIface + } +} diff --git a/misc/misc_test.go b/misc/misc_test.go index 9600355c3..db512c257 100644 --- a/misc/misc_test.go +++ b/misc/misc_test.go @@ -53,7 +53,7 @@ func TestAllocateFreeVMs(t *testing.T) { vmIDs := [2]string{"test1", "test2"} for _, vmID := range vmIDs { - _, err := vmPool.Allocate(vmID) + _, err := vmPool.Allocate(vmID, "") require.NoError(t, err, "Failed to allocate VM") } @@ -76,7 +76,7 @@ func TestAllocateFreeVMsParallel(t *testing.T) { go func(i int) { defer vmGroup.Done() vmID := fmt.Sprintf("test_%d", i) - _, err := vmPool.Allocate(vmID) + _, err := vmPool.Allocate(vmID, "") require.NoError(t, err, "Failed to allocate VM") }(i) } @@ -108,7 +108,7 @@ func TestRecreateParallel(t *testing.T) { go func(i int) { defer vmGroup.Done() vmID := fmt.Sprintf("test_%d", i) - _, err := vmPool.Allocate(vmID) + _, err := vmPool.Allocate(vmID, "") require.NoError(t, err, "Failed to allocate VM") }(i) } @@ -123,7 +123,7 @@ func TestRecreateParallel(t *testing.T) { go func(i int) { defer vmGroupRecreate.Done() vmID := fmt.Sprintf("test_%d", i) - err := vmPool.RecreateTap(vmID) + err := vmPool.RecreateTap(vmID, "") require.NoError(t, err, "Failed to recreate tap") }(i) } diff --git a/misc/vm_pool.go b/misc/vm_pool.go index 740d461fd..33a04b7af 100644 --- a/misc/vm_pool.go +++ b/misc/vm_pool.go @@ -37,7 +37,7 @@ func NewVMPool() *VMPool { } // Allocate Initializes a VM, activates it and then adds it to VM map -func (p *VMPool) Allocate(vmID string) (*VM, error) { +func (p *VMPool) Allocate(vmID, hostIface string) (*VM, error) { logger := log.WithFields(log.Fields{"vmID": vmID}) @@ -50,7 +50,7 @@ func (p *VMPool) Allocate(vmID string) (*VM, error) { vm := NewVM(vmID) var err error - vm.Ni, err = p.tapManager.AddTap(vmID + "_tap") + vm.Ni, err = p.tapManager.AddTap(vmID+"_tap", hostIface) if err != nil { logger.Warn("Ni allocation failed") return nil, err @@ -84,7 +84,7 @@ func (p *VMPool) Free(vmID string) error { } // RecreateTap Deletes and creates the tap for a VM -func (p *VMPool) RecreateTap(vmID string) error { +func (p *VMPool) RecreateTap(vmID, hostIface string) error { logger := log.WithFields(log.Fields{"vmID": vmID}) logger.Debug("Recreating tap") @@ -100,7 +100,7 @@ func (p *VMPool) RecreateTap(vmID string) error { return err } - _, err := p.tapManager.AddTap(vmID + "_tap") + _, err := p.tapManager.AddTap(vmID+"_tap", hostIface) if err != nil { logger.Error("Failed to add tap") return err diff --git a/taps/tapManager.go b/taps/tapManager.go index ee3ddfe1b..2f97e1bf0 100644 --- a/taps/tapManager.go +++ b/taps/tapManager.go @@ -23,8 +23,12 @@ package taps import ( + "bufio" + "bytes" "errors" "fmt" + "os/exec" + "strings" "sync/atomic" log "github.com/sirupsen/logrus" @@ -100,8 +104,62 @@ func createBridge(bridgeName, gatewayAddr string) { } } +//ConfigIPtables Configures IP tables for internet access inside VM +func ConfigIPtables(tapName, hostIface string) error { + + if hostIface == "" { + out, err := exec.Command( + "route", + ).Output() + if err != nil { + log.Warnf("Failed to fetch host net interfaces %v\n%s\n", err, out) + return err + } + scanner := bufio.NewScanner(bytes.NewReader(out)) + for scanner.Scan() { + line := scanner.Text() + if strings.Contains(line, "default") { + hostIface = line[strings.LastIndex(line, " ")+1:] + } + } + } + cmd := exec.Command( + "sudo", "iptables", "-t", "nat", "-A", "POSTROUTING", "-o", hostIface, "-j", "MASQUERADE", + ) + stdoutStderr, err := cmd.CombinedOutput() + if err != nil { + log.Warnf("Failed to configure NAT %v\n%s\n", err, stdoutStderr) + return err + } + cmd = exec.Command( + "sudo", "iptables", "-A", "FORWARD", "-i", tapName, "-o", hostIface, "-j", "ACCEPT", + ) + stdoutStderr, err = cmd.CombinedOutput() + if err != nil { + log.Warnf("Failed to setup forwarding into tap %v\n%s\n", err, stdoutStderr) + return err + } + cmd = exec.Command( + "sudo", "iptables", "-A", "FORWARD", "-o", tapName, "-i", hostIface, "-j", "ACCEPT", + ) + stdoutStderr, err = cmd.CombinedOutput() + if err != nil { + log.Warnf("Failed to setup forwarding out from tap %v\n%s\n", err, stdoutStderr) + return err + } + cmd = exec.Command( + "sudo", "iptables", "-A", "FORWARD", "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT", + ) + stdoutStderr, err = cmd.CombinedOutput() + if err != nil { + log.Warnf("Failed to configure conntrack %v\n%s\n", err, stdoutStderr) + return err + } + return nil +} + // AddTap Creates a new tap and returns the corresponding network interface -func (tm *TapManager) AddTap(tapName string) (*NetworkInterface, error) { +func (tm *TapManager) AddTap(tapName, hostIface string) (*NetworkInterface, error) { tm.Lock() if ni, ok := tm.createdTaps[tapName]; ok { @@ -120,6 +178,10 @@ func (tm *TapManager) AddTap(tapName string) (*NetworkInterface, error) { tm.Lock() tm.createdTaps[tapName] = ni tm.Unlock() + err := ConfigIPtables(tapName, hostIface) + if err != nil { + return nil, err + } } return ni, err diff --git a/taps/taps_test.go b/taps/taps_test.go index dca66d73a..668969e6b 100644 --- a/taps/taps_test.go +++ b/taps/taps_test.go @@ -66,7 +66,7 @@ func TestCreateRemoveTaps(t *testing.T) { wg.Add(1) go func(i int) { defer wg.Done() - tm.AddTap(fmt.Sprintf("tap_%d", i)) + tm.AddTap(fmt.Sprintf("tap_%d", i), "") }(i) } wg.Wait() @@ -91,7 +91,7 @@ func TestCreateRemoveExtra(t *testing.T) { defer tm.RemoveBridges() for i := 0; i < tapsNum; i++ { - _, err := tm.AddTap(fmt.Sprintf("tap_%d", i)) + _, err := tm.AddTap(fmt.Sprintf("tap_%d", i), "") if i < tm.numBridges*TapsPerBridge { require.NoError(t, err, "Failed to create tap") } else { diff --git a/vhive.go b/vhive.go index b791355d7..14e4dd2ac 100644 --- a/vhive.go +++ b/vhive.go @@ -60,6 +60,7 @@ var ( servedThreshold *uint64 pinnedFuncNum *int criSock *string + hostIface *string ) func main() { @@ -78,6 +79,7 @@ func main() { pinnedFuncNum = flag.Int("hn", 0, "Number of functions pinned in memory (IDs from 0 to X)") isLazyMode = flag.Bool("lazy", false, "Enable lazy serving mode when UPFs are enabled") criSock = flag.String("criSock", "/etc/firecracker-containerd/fccd-cri.sock", "Socket address for CRI service") + hostIface = flag.String("hostIface", "", "Host net-interface for the VMs to bind to for internet access") flag.Parse() @@ -119,6 +121,7 @@ func main() { orch = ctriface.NewOrchestrator( *snapshotter, + *hostIface, ctriface.WithTestModeOn(testModeOn), ctriface.WithSnapshots(*isSnapshotsEnabled), ctriface.WithUPF(*isUPFEnabled), diff --git a/vhive_test.go b/vhive_test.go index b21a782af..50346e5f2 100644 --- a/vhive_test.go +++ b/vhive_test.go @@ -74,6 +74,7 @@ func TestMain(m *testing.M) { orch = ctriface.NewOrchestrator( "devmapper", + "", ctriface.WithTestModeOn(true), ctriface.WithSnapshots(*isSnapshotsEnabledTest), ctriface.WithUPF(*isUPFEnabledTest),