Merge pull request openyurtio#147 from openyurtio/revert-133-support-…

…nat-traversal

Revert "feat: support autoconfig Raven gateway topology across multi-edge network"

go.mod

@@ -4,11 +4,10 @@ go 1.18
 
 require (
 	github.com/EvilSuperstars/go-cidrman v0.0.0-20190607145828-28e79e32899a
-	github.com/ccding/go-stun/stun v0.0.0-20200514191101-4dc67bcdb029
 	github.com/coreos/go-iptables v0.6.0
 	github.com/gorilla/mux v1.8.0
 	github.com/lorenzosaino/go-sysctl v0.3.1
-	github.com/openyurtio/openyurt v1.3.1-0.20231031135146-ea8392b1c68c
+	github.com/openyurtio/openyurt v1.3.1-0.20230920090905-424dcc283167
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.15.1
 	github.com/spf13/cobra v1.7.0

go.sum

@@ -96,8 +96,6 @@ github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqO
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/ccding/go-stun/stun v0.0.0-20200514191101-4dc67bcdb029 h1:POmUHfxXdeyM8Aomg4tKDcwATCFuW+cYLkj6pwsw9pc=
-github.com/ccding/go-stun/stun v0.0.0-20200514191101-4dc67bcdb029/go.mod h1:Rpr5n9cGHYdM3S3IK8ROSUUUYjQOu+MSUCZDcJbYWi8=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
@@ -443,8 +441,8 @@ github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/openyurtio/apiserver-network-proxy v0.1.1-0.20231007082056-cecf4c454651 h1:pBjrj95WZ7NkaIQvvb7hveI5fd8fXxlX7OtKRcciamY=
 github.com/openyurtio/apiserver-network-proxy v0.1.1-0.20231007082056-cecf4c454651/go.mod h1:yPrw5zKs7BVVCuQUaY4MzSmsYPKXWnZsrjKUWQrZX3w=
-github.com/openyurtio/openyurt v1.3.1-0.20231031135146-ea8392b1c68c h1:L10aR3duZFrJC7ZUk8BhD0kpSVfEfIP4MYI/zE4WkDI=
-github.com/openyurtio/openyurt v1.3.1-0.20231031135146-ea8392b1c68c/go.mod h1:9TGp6V/7ogHGPoqfXCKlU5FbdjdQ8LECTNHtDfDIFdU=
+github.com/openyurtio/openyurt v1.3.1-0.20230920090905-424dcc283167 h1:gbjh0njlthM0mM9Z6LHURdKpovLXReleT5GUwaksyFE=
+github.com/openyurtio/openyurt v1.3.1-0.20230920090905-424dcc283167/go.mod h1:c77rs9s+ztXyyHNf2EB+lMHuDfSWwel6dhP2ZVi5cvM=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=

pkg/networkengine/vpndriver/driver.go

@@ -12,7 +12,6 @@ import (
 	"github.com/openyurtio/raven/cmd/agent/app/config"
 	netlinkutil "github.com/openyurtio/raven/pkg/networkengine/util/netlink"
 	"github.com/openyurtio/raven/pkg/types"
-	"github.com/openyurtio/raven/pkg/utils"
 )
 
 const (
@@ -95,19 +94,6 @@ func FindCentralGwFn(network *types.Network) *types.Endpoint {
 	return central
 }
 
-// EnableCreateEdgeConnection determine whether VPN tunnels can be established between edges.
-func EnableCreateEdgeConnection(localEndpoint *types.Endpoint, remoteEndpoint *types.Endpoint) bool {
-	if localEndpoint.NATType == utils.NATUndefined || remoteEndpoint.NATType == utils.NATUndefined {
-		return false
-	}
-	if !localEndpoint.UnderNAT || !remoteEndpoint.UnderNAT {
-		return false
-	}
-	return !((localEndpoint.NATType == utils.NATSymmetric && remoteEndpoint.NATType == utils.NATSymmetric) ||
-		(localEndpoint.NATType == utils.NATSymmetric && remoteEndpoint.NATType == utils.NATPortRestricted) ||
-		(localEndpoint.NATType == utils.NATPortRestricted && remoteEndpoint.NATType == utils.NATSymmetric))
-}
-
 func DefaultMTU() (int, error) {
 	routes, err := netlinkutil.RouteListFiltered(
 		netlink.FAMILY_V4,

pkg/networkengine/vpndriver/libreswan/libreswan.go

@@ -46,7 +46,6 @@ var _ vpndriver.Driver = (*libreswan)(nil)
 // can be modified for testing.
 var whackCmd = whackCmdFn
 var findCentralGw = vpndriver.FindCentralGwFn
-var enableCreateEdgeConnection = vpndriver.EnableCreateEdgeConnection
 
 func init() {
 	vpndriver.RegisterDriver(DriverName, New)
@@ -98,39 +97,24 @@ func New(cfg *config.Config) (vpndriver.Driver, error) {
 }
 
 func (l *libreswan) Apply(network *types.Network, routeDriverMTUFn func(*types.Network) (int, error)) (err error) {
+	errList := errorlist.List{}
 	if network.LocalEndpoint == nil || len(network.RemoteEndpoints) == 0 {
-		klog.Info("no local gateway or remote gateway is found, cleaning vpn connections")
+		klog.Info(utils.FormatTunnel("no local gateway or remote gateway is found, cleaning vpn connections"))
 		return l.Cleanup()
 	}
 	if network.LocalEndpoint.NodeName != l.nodeName {
-		klog.Infof(utils.FormatTunnel("the current node is not gateway node, cleaning vpn connections"))
+		klog.Info(utils.FormatTunnel("the current node is not gateway node, cleaning vpn connections"))
 		return l.Cleanup()
 	}
 
-	if err := l.createConnections(network); err != nil {
-		return fmt.Errorf("error create VPN tunnels: %v", err)
-	}
-
-	return nil
-}
-
-func (l *libreswan) MTU() (int, error) {
-	mtu, err := vpndriver.DefaultMTU()
-	if err != nil {
-		return 0, err
-	}
-	return mtu - IPSecEncapLen, nil
-}
-
-func (l *libreswan) createConnections(network *types.Network) error {
-	errList := errorlist.List{}
 	desiredConnections := l.computeDesiredConnections(network)
 	if len(desiredConnections) == 0 {
-		klog.Infof(utils.FormatTunnel("no desired connections, cleaning vpn connections"))
-		l.Cleanup()
-		return nil
+		klog.Info(utils.FormatTunnel("no desired connections, cleaning vpn connections"))
+		return l.Cleanup()
 	}
 
+	centralGw := findCentralGw(network)
+
 	// remove unwanted connections
 	for connName := range l.connections {
 		if _, ok := desiredConnections[connName]; !ok {
@@ -141,78 +125,74 @@ func (l *libreswan) createConnections(network *types.Network) error {
 				continue
 			}
 			delete(l.connections, connName)
+			if centralGw.NodeName == l.nodeName {
+				errList = errList.Append(l.deleteRavenSkipNAT(centralGw, l.connections[connName]))
+			}
 		}
 	}
 
 	// add new connections
 	for name, connection := range desiredConnections {
 		err := l.connectToEndpoint(name, connection)
 		errList = errList.Append(err)
+		if centralGw.NodeName == l.nodeName {
+			err = l.ensureRavenSkipNAT(centralGw, connection)
+			errList = errList.Append(err)
+		}
 	}
 
 	return errList.AsError()
 }
 
-func (l *libreswan) computeDesiredConnections(network *types.Network) map[string]*vpndriver.Connection {
-	centralGw := findCentralGw(network)
-	// This is the desired connection calculated from given *types.Network
-	desiredConns := make(map[string]*vpndriver.Connection)
-
-	leftEndpoint := network.LocalEndpoint
-	for _, remote := range network.RemoteEndpoints {
-		leftSubnets, connectTo := l.resolveEndpoint(network, centralGw, remote)
-		for _, leftSubnet := range leftSubnets {
-			for _, rightSubnet := range remote.Subnets {
-				name := connectionName(leftEndpoint.PrivateIP, remote.PrivateIP, leftSubnet, rightSubnet)
-				desiredConns[name] = &vpndriver.Connection{
-					LocalEndpoint:  leftEndpoint.Copy(),
-					RemoteEndpoint: connectTo.Copy(),
-					LocalSubnet:    leftSubnet,
-					RemoteSubnet:   rightSubnet,
-				}
-			}
-		}
+func (l *libreswan) MTU() (int, error) {
+	mtu, err := vpndriver.DefaultMTU()
+	if err != nil {
+		return 0, err
 	}
-
-	return desiredConns
+	return mtu - IPSecEncapLen, nil
 }
 
-func (l *libreswan) resolveEndpoint(network *types.Network, centralGw, remoteGw *types.Endpoint) (leftSubnets []string, connectTo *types.Endpoint) {
+// getEndpointResolver returns a function that resolve the left subnets and the Endpoint that should connect to.
+func (l *libreswan) getEndpointResolver(network *types.Network) func(centralGw, remoteGw *types.Endpoint) (leftSubnets []string, connectTo *types.Endpoint) {
 	snUnderNAT := make(map[types.GatewayName][]string)
 	for _, v := range network.RemoteEndpoints {
-		if v.UnderNAT && !enableCreateEdgeConnection(v, remoteGw) {
+		if v.UnderNAT {
 			snUnderNAT[v.GatewayName] = v.Subnets
 		}
 	}
-	leftSubnets = network.LocalEndpoint.Subnets
-	if centralGw == nil {
-		// If both local and remote gateway are NATed but no central gateway found,
-		// we cannot set up vpn connections between the local and remote gateway.
-		if network.LocalEndpoint.UnderNAT && remoteGw.UnderNAT {
-			return nil, nil
+	return func(centralGw, remoteGw *types.Endpoint) (leftSubnets []string, connectTo *types.Endpoint) {
+		leftSubnets = network.LocalEndpoint.Subnets
+		if centralGw == nil {
+			// If both local and remote gateway are NATed but no central gateway found,
+			// we cannot set up vpn connections between the local and remote gateway.
+			if network.LocalEndpoint.UnderNAT && remoteGw.UnderNAT {
+				return nil, nil
+			}
+			return leftSubnets, remoteGw
 		}
-		return leftSubnets, remoteGw
-	}
 
-	if centralGw.NodeName == l.nodeName {
-		if remoteGw.UnderNAT {
-			// If the local gateway is the central gateway,
-			// in order to forward traffic from other NATed gateway to the NATed remoteGw,
-			// append all subnets of other NATed gateways into left subnets.
-			for gwName, v := range snUnderNAT {
-				if gwName != remoteGw.GatewayName {
-					leftSubnets = append(leftSubnets, v...)
+		if centralGw.NodeName == l.nodeName {
+			if remoteGw.UnderNAT {
+				// If the local gateway is the central gateway,
+				// in order to forward traffic from other NATed gateway to the NATed remoteGw,
+				// append all subnets of other NATed gateways into left subnets.
+				for gwName, v := range snUnderNAT {
+					if gwName != remoteGw.GatewayName {
+						leftSubnets = append(leftSubnets, v...)
+					}
 				}
 			}
+			return leftSubnets, remoteGw
 		}
-		return leftSubnets, remoteGw
-	}
 
-	if !enableCreateEdgeConnection(network.LocalEndpoint, remoteGw) {
-		return leftSubnets, centralGw
-	}
+		// If both local and remote are NATed, and the local gateway is not the central gateway,
+		// connects to central gateway to forward traffic.
+		if network.LocalEndpoint.UnderNAT && remoteGw.UnderNAT {
+			return leftSubnets, centralGw
+		}
 
-	return leftSubnets, remoteGw
+		return leftSubnets, remoteGw
+	}
 }
 
 func (l *libreswan) whackConnectToEndpoint(connectionName string, connection *vpndriver.Connection) error {
@@ -264,6 +244,68 @@ func (l *libreswan) whackConnectToEndpoint(connectionName string, connection *vp
 	return nil
 }
 
+func (l *libreswan) ensureRavenSkipNAT(centralGw *types.Endpoint, connection *vpndriver.Connection) errorlist.List {
+	errList := errorlist.List{}
+	// for raven skip nat
+	if err := l.iptables.NewChainIfNotExist(iptablesutil.NatTable, iptablesutil.RavenPostRoutingChain); err != nil {
+		errList = errList.Append(fmt.Errorf("error create %s chain: %s", iptablesutil.RavenPostRoutingChain, err))
+	}
+	if err := l.iptables.InsertIfNotExists(iptablesutil.NatTable, iptablesutil.PostRoutingChain, 1, "-m", "comment", "--comment", "raven traffic should skip NAT", "-j", iptablesutil.RavenPostRoutingChain); err != nil {
+		errList = errList.Append(fmt.Errorf("error adding chain %s rule: %s", iptablesutil.PostRoutingChain, err))
+	}
+	for _, subnet := range centralGw.Subnets {
+		if connection.LocalSubnet == subnet || connection.RemoteSubnet == subnet {
+			return errList
+		}
+	}
+	if err := l.iptables.AppendIfNotExists(iptablesutil.NatTable, iptablesutil.RavenPostRoutingChain, "-s", connection.LocalSubnet, "-d", connection.RemoteSubnet, "-j", "ACCEPT"); err != nil {
+		errList = errList.Append(fmt.Errorf("error adding chain %s rule: %s", iptablesutil.RavenPostRoutingChain, err))
+	}
+	return errList
+}
+
+func (l *libreswan) deleteRavenSkipNAT(centralGw *types.Endpoint, connection *vpndriver.Connection) errorlist.List {
+	errList := errorlist.List{}
+	err := l.iptables.NewChainIfNotExist(iptablesutil.NatTable, iptablesutil.RavenPostRoutingChain)
+	if err != nil {
+		errList = errList.Append(fmt.Errorf("error create %s chain: %s", iptablesutil.PostRoutingChain, err))
+	}
+	for _, subnet := range centralGw.Subnets {
+		if connection.LocalSubnet == subnet || connection.RemoteSubnet == subnet {
+			return errList
+		}
+	}
+	err = l.iptables.DeleteIfExists(iptablesutil.NatTable, iptablesutil.RavenPostRoutingChain, "-s", connection.LocalSubnet, "-d", connection.RemoteSubnet, "-j", "ACCEPT")
+	if err != nil {
+		errList = errList.Append(fmt.Errorf("error deleting %s chain rule: %s", iptablesutil.RavenPostRoutingChain, err))
+	}
+	return errList
+}
+
+func (l *libreswan) computeDesiredConnections(network *types.Network) map[string]*vpndriver.Connection {
+	centralGw := findCentralGw(network)
+	resolveEndpoint := l.getEndpointResolver(network)
+	// This is the desired connection calculated from given *types.Network
+	desiredConns := make(map[string]*vpndriver.Connection)
+
+	leftEndpoint := network.LocalEndpoint
+	for _, remoteGw := range network.RemoteEndpoints {
+		leftSubnets, connectTo := resolveEndpoint(centralGw, remoteGw)
+		for _, leftSubnet := range leftSubnets {
+			for _, rightSubnet := range remoteGw.Subnets {
+				name := connectionName(leftEndpoint.PrivateIP, connectTo.PrivateIP, leftSubnet, rightSubnet)
+				desiredConns[name] = &vpndriver.Connection{
+					LocalEndpoint:  leftEndpoint.Copy(),
+					RemoteEndpoint: connectTo.Copy(),
+					LocalSubnet:    leftSubnet,
+					RemoteSubnet:   rightSubnet,
+				}
+			}
+		}
+	}
+	return desiredConns
+}
+
 func whackCmdFn(args ...string) error {
 	var err error
 	var output []byte