feat: AlibabaCloud-SLB support muti-slbIds

Signed-off-by: ChrisLiu <chrisliu1995@163.com>

cloudprovider/alibabacloud/slb.go

@@ -29,6 +29,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	log "k8s.io/klog/v2"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"strconv"
@@ -46,6 +47,7 @@ const (
 	SlbIdLabelKey           = "service.k8s.alibaba/loadbalancer-id"
 	SvcSelectorKey          = "statefulset.kubernetes.io/pod-name"
 	allocatedPortsKey       = "game.kruise.io/AlibabaCloud-SLB-ports-allocated"
+	SlbConfigHashKey        = "game.kruise.io/network-config-hash"
 )
 
 type portAllocated map[int32]bool
@@ -57,6 +59,13 @@ type SlbPlugin struct {
 	mutex   sync.RWMutex
 }
 
+type slbConfig struct {
+	lbIds       []string
+	targetPorts []int
+	protocols   []corev1.Protocol
+	isFixed     bool
+}
+
 func (s *SlbPlugin) Name() string {
 	return SlbNetwork
 }
@@ -105,14 +114,18 @@ func initLbCache(svcList []corev1.Service, minPort, maxPort int32) map[string]po
 
 func (s *SlbPlugin) OnPodAdded(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) {
 	networkManager := utils.NewNetworkManager(pod, c)
-	err := c.Create(ctx, s.createSvc(networkManager.GetNetworkConfig(), pod, c, ctx))
+	networkConfig := networkManager.GetNetworkConfig()
+	sc := parseLbConfig(networkConfig)
+	err := c.Create(ctx, s.consSvc(sc, pod, c, ctx))
 	return pod, cperrors.ToPluginError(err, cperrors.ApiCallError)
 }
 
 func (s *SlbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) {
 	networkManager := utils.NewNetworkManager(pod, c)
 
 	networkStatus, _ := networkManager.GetNetworkStatus()
+	networkConfig := networkManager.GetNetworkConfig()
+	sc := parseLbConfig(networkConfig)
 	if networkStatus == nil {
 		pod, err := networkManager.UpdateNetworkStatus(gamekruiseiov1alpha1.NetworkStatus{
 			CurrentNetworkState: gamekruiseiov1alpha1.NetworkNotReady,
@@ -128,11 +141,21 @@ func (s *SlbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.C
 	}, svc)
 	if err != nil {
 		if errors.IsNotFound(err) {
-			return pod, cperrors.ToPluginError(c.Create(ctx, s.createSvc(networkManager.GetNetworkConfig(), pod, c, ctx)), cperrors.ApiCallError)
+			return pod, cperrors.ToPluginError(c.Create(ctx, s.consSvc(sc, pod, c, ctx)), cperrors.ApiCallError)
 		}
 		return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
 	}
 
+	// update svc
+	if util.GetHash(sc) != svc.GetAnnotations()[SlbConfigHashKey] {
+		networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkNotReady
+		pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
+		if err != nil {
+			return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
+		}
+		return pod, cperrors.ToPluginError(c.Update(ctx, s.consSvc(sc, pod, c, ctx)), cperrors.ApiCallError)
+	}
+
 	// disable network
 	if networkManager.GetNetworkDisabled() && svc.Spec.Type == corev1.ServiceTypeLoadBalancer {
 		svc.Spec.Type = corev1.ServiceTypeClusterIP
@@ -198,18 +221,37 @@ func (s *SlbPlugin) OnPodDeleted(c client.Client, pod *corev1.Pod, ctx context.C
 		return cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
 	}
 
-	for _, port := range getPorts(svc.Spec.Ports) {
+	ports := getPorts(svc.Spec.Ports)
+	for _, port := range ports {
 		s.deAllocate(svc.Annotations[SlbIdAnnotationKey], port)
 	}
 
+	log.Infof("pod %s/%s deallocate slb %s ports %v", pod.GetNamespace(), pod.GetName(), svc.Annotations[SlbIdAnnotationKey], ports)
 	return nil
 }
 
-func (s *SlbPlugin) allocate(lbId string, num int) []int32 {
+func (s *SlbPlugin) allocate(lbIds []string, num int) (string, []int32) {
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
 
 	var ports []int32
+	var lbId string
+
+	// find lb with adequate ports
+	for _, slbId := range lbIds {
+		sum := 0
+		for i := s.minPort; i < s.maxPort; i++ {
+			if !s.cache[slbId][i] {
+				sum++
+			}
+			if sum >= num {
+				lbId = slbId
+				break
+			}
+		}
+	}
+
+	// select ports
 	for i := 0; i < num; i++ {
 		var port int32
 		if s.cache[lbId] == nil {
@@ -228,7 +270,7 @@ func (s *SlbPlugin) allocate(lbId string, num int) []int32 {
 		s.cache[lbId][port] = true
 		ports = append(ports, port)
 	}
-	return ports
+	return lbId, ports
 }
 
 func (s *SlbPlugin) deAllocate(lbId string, port int32) {
@@ -245,15 +287,19 @@ func init() {
 	alibabaCloudProvider.registerPlugin(&slbPlugin)
 }
 
-func parseLbConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) (string, []int, []corev1.Protocol, bool) {
-	var lbId string
+func parseLbConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) *slbConfig {
+	var lbIds []string
 	ports := make([]int, 0)
 	protocols := make([]corev1.Protocol, 0)
 	isFixed := false
 	for _, c := range conf {
 		switch c.Name {
 		case SlbIdsConfigName:
-			lbId = c.Value
+			for _, slbId := range strings.Split(c.Value, ",") {
+				if slbId != "" {
+					lbIds = append(lbIds, slbId)
+				}
+			}
 		case PortProtocolsConfigName:
 			for _, pp := range strings.Split(c.Value, ",") {
 				ppSlice := strings.Split(pp, "/")
@@ -276,7 +322,12 @@ func parseLbConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) (string, []int
 			isFixed = v
 		}
 	}
-	return lbId, ports, protocols, isFixed
+	return &slbConfig{
+		lbIds:       lbIds,
+		protocols:   protocols,
+		targetPorts: ports,
+		isFixed:     isFixed,
+	}
 }
 
 func getPorts(ports []corev1.ServicePort) []int32 {
@@ -287,25 +338,27 @@ func getPorts(ports []corev1.ServicePort) []int32 {
 	return ret
 }
 
-func (s *SlbPlugin) createSvc(conf []gamekruiseiov1alpha1.NetworkConfParams, pod *corev1.Pod, c client.Client, ctx context.Context) *corev1.Service {
-	lbId, targetPorts, protocol, isFixed := parseLbConfig(conf)
-
+func (s *SlbPlugin) consSvc(sc *slbConfig, pod *corev1.Pod, c client.Client, ctx context.Context) *corev1.Service {
 	var ports []int32
+	var lbId string
 	allocatedPorts := pod.Annotations[allocatedPortsKey]
 	if allocatedPorts != "" {
-		ports = util.StringToInt32Slice(allocatedPorts, ",")
+		slbPorts := strings.Split(allocatedPorts, ":")
+		lbId = slbPorts[0]
+		ports = util.StringToInt32Slice(slbPorts[1], ",")
 	} else {
-		ports = s.allocate(lbId, len(targetPorts))
-		pod.Annotations[allocatedPortsKey] = util.Int32SliceToString(ports, ",")
+		lbId, ports = s.allocate(sc.lbIds, len(sc.targetPorts))
+		log.Infof("pod %s/%s allocate slb %s ports %v", pod.GetNamespace(), pod.GetName(), lbId, ports)
+		pod.Annotations[allocatedPortsKey] = lbId + ":" + util.Int32SliceToString(ports, ",")
 	}
 
 	svcPorts := make([]corev1.ServicePort, 0)
-	for i := 0; i < len(targetPorts); i++ {
+	for i := 0; i < len(sc.targetPorts); i++ {
 		svcPorts = append(svcPorts, corev1.ServicePort{
-			Name:       strconv.Itoa(targetPorts[i]),
+			Name:       strconv.Itoa(sc.targetPorts[i]),
 			Port:       ports[i],
-			Protocol:   protocol[i],
-			TargetPort: intstr.FromInt(targetPorts[i]),
+			Protocol:   sc.protocols[i],
+			TargetPort: intstr.FromInt(sc.targetPorts[i]),
 		})
 	}
 
@@ -316,8 +369,9 @@ func (s *SlbPlugin) createSvc(conf []gamekruiseiov1alpha1.NetworkConfParams, pod
 			Annotations: map[string]string{
 				SlbListenerOverrideKey: "true",
 				SlbIdAnnotationKey:     lbId,
+				SlbConfigHashKey:       util.GetHash(sc),
 			},
-			OwnerReferences: getSvcOwnerReference(c, ctx, pod, isFixed),
+			OwnerReferences: getSvcOwnerReference(c, ctx, pod, sc.isFixed),
 		},
 		Spec: corev1.ServiceSpec{
 			Type: corev1.ServiceTypeLoadBalancer,

cloudprovider/alibabacloud/slb_test.go

@@ -22,17 +22,18 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"reflect"
 	"sync"
 	"testing"
 )
 
 func TestAllocate(t *testing.T) {
 	test := struct {
-		lbId string
-		slb  *SlbPlugin
-		num  int
+		lbIds []string
+		slb   *SlbPlugin
+		num   int
 	}{
-		lbId: "xxx-A",
+		lbIds: []string{"xxx-A"},
 		slb: &SlbPlugin{
 			maxPort: int32(712),
 			minPort: int32(512),
@@ -42,26 +43,26 @@ func TestAllocate(t *testing.T) {
 		num: 3,
 	}
 
-	ports := test.slb.allocate(test.lbId, test.num)
+	lbId, ports := test.slb.allocate(test.lbIds, test.num)
 	for _, port := range ports {
 		if port > test.slb.maxPort || port < test.slb.minPort {
 			t.Errorf("allocate port %d, unexpected", port)
 		}
 
-		test.slb.deAllocate(test.lbId, port)
-		if test.slb.cache[test.lbId][port] == true {
+		test.slb.deAllocate(lbId, port)
+		if test.slb.cache[lbId][port] == true {
 			t.Errorf("deAllocate port %d failed", port)
 		}
 	}
 }
 
 func TestParseLbConfig(t *testing.T) {
 	tests := []struct {
-		conf     []gamekruiseiov1alpha1.NetworkConfParams
-		lbId     string
-		ports    []int
-		protocol []corev1.Protocol
-		isFixed  bool
+		conf      []gamekruiseiov1alpha1.NetworkConfParams
+		lbIds     []string
+		ports     []int
+		protocols []corev1.Protocol
+		isFixed   bool
 	}{
 		{
 			conf: []gamekruiseiov1alpha1.NetworkConfParams{
@@ -74,16 +75,16 @@ func TestParseLbConfig(t *testing.T) {
 					Value: "80",
 				},
 			},
-			lbId:     "xxx-A",
-			ports:    []int{80},
-			protocol: []corev1.Protocol{corev1.ProtocolTCP},
-			isFixed:  false,
+			lbIds:     []string{"xxx-A"},
+			ports:     []int{80},
+			protocols: []corev1.Protocol{corev1.ProtocolTCP},
+			isFixed:   false,
 		},
 		{
 			conf: []gamekruiseiov1alpha1.NetworkConfParams{
 				{
 					Name:  SlbIdsConfigName,
-					Value: "xxx-A",
+					Value: "xxx-A,xxx-B,",
 				},
 				{
 					Name:  PortProtocolsConfigName,
@@ -94,31 +95,26 @@ func TestParseLbConfig(t *testing.T) {
 					Value: "true",
 				},
 			},
-			lbId:     "xxx-A",
-			ports:    []int{81, 82, 83},
-			protocol: []corev1.Protocol{corev1.ProtocolUDP, corev1.ProtocolTCP, corev1.ProtocolTCP},
-			isFixed:  true,
+			lbIds:     []string{"xxx-A", "xxx-B"},
+			ports:     []int{81, 82, 83},
+			protocols: []corev1.Protocol{corev1.ProtocolUDP, corev1.ProtocolTCP, corev1.ProtocolTCP},
+			isFixed:   true,
 		},
 	}
 
 	for _, test := range tests {
-		lbId, ports, protocol, isFixed := parseLbConfig(test.conf)
-		if lbId != test.lbId {
-			t.Errorf("lbId expect: %s, actual: %s", test.lbId, lbId)
-		}
-		if !util.IsSliceEqual(ports, test.ports) {
-			t.Errorf("ports expect: %v, actual: %v", test.ports, ports)
+		sc := parseLbConfig(test.conf)
+		if !reflect.DeepEqual(test.lbIds, sc.lbIds) {
+			t.Errorf("lbId expect: %v, actual: %v", test.lbIds, sc.lbIds)
 		}
-		if len(test.protocol) != len(protocol) {
-			t.Errorf("protocol expect: %v, actual: %v", test.protocol, protocol)
+		if !util.IsSliceEqual(test.ports, sc.targetPorts) {
+			t.Errorf("ports expect: %v, actual: %v", test.ports, sc.targetPorts)
 		}
-		for i := 0; i < len(test.protocol); i++ {
-			if protocol[i] != test.protocol[i] {
-				t.Errorf("protocol expect: %v, actual: %v", test.protocol, protocol)
-			}
+		if !reflect.DeepEqual(test.protocols, sc.protocols) {
+			t.Errorf("protocols expect: %v, actual: %v", test.protocols, sc.protocols)
 		}
-		if isFixed != test.isFixed {
-			t.Errorf("protocol expect: %v, actual: %v", test.isFixed, isFixed)
+		if test.isFixed != sc.isFixed {
+			t.Errorf("isFixed expect: %v, actual: %v", test.isFixed, sc.isFixed)
 		}
 	}
 }

docs/en/user_manuals/network.md

@@ -414,21 +414,21 @@ Related design: https://github.com/openkruise/kruise-game/issues/20
 
 SlbIds
 
-- Meaning: the CLB instance ID. You can specify only one CLB instance ID. Multiple CLB instance IDs will be supported in the future.
-- Value: an example value can be lb-9zeo7prq1m25ctpfrw1m7.
-- Configuration change supported or not: no. The configuration change can be supported in future.
+- Meaning: the CLB instance ID. You can fill in multiple ids.
+- Value: in the format of slbId-0,slbId-1,... An example value can be "lb-9zeo7prq1m25ctpfrw1m7,lb-bp1qz7h50yd3w58h2f8je"
+- Configuration change supported or not: yes. You can add new slbIds at the end. However, it is recommended not to change existing slbId that is in use.
 
 PortProtocols
 
 - Meaning: the ports in the pod to be exposed and the protocols. You can specify multiple ports and protocols.
 - Value: in the format of port1/protocol1,port2/protocol2,... The protocol names must be in uppercase letters.
-- Configuration change supported or not: no. The configuration change can be supported in future.
+- Configuration change supported or not: yes.
 
 Fixed
 
 - Meaning: whether the mapping relationship is fixed. If the mapping relationship is fixed, the mapping relationship remains unchanged even if the pod is deleted and recreated.
 - Value: false or true.
-- Configuration change supported or not: no.
+- Configuration change supported or not: yes.
 
 #### Plugin configuration
 ```

docs/中文/用户手册/网络模型.md

@@ -411,21 +411,21 @@ AlibabaCloud
 
 SlbIds
 
-- 含义：填写slb的id。暂只支持填写一例，未来将支持填写多例
-- 填写格式：例如：lb-9zeo7prq1m25ctpfrw1m7
-- 是否支持变更：暂不支持。未来将支持
+- 含义：填写slb的id。可填写多个。
+- 填写格式：各个slbId用,分割。例如：lb-9zeo7prq1m25ctpfrw1m7,lb-bp1qz7h50yd3w58h2f8je,...
+- 是否支持变更：支持。可追加填写SLB实例id。建议不要更换正在被使用的实例id。
 
 PortProtocols
 
 - 含义：pod暴露的端口及协议，支持填写多个端口/协议
 - 格式：port1/protocol1,port2/protocol2,...（协议需大写）
-- 是否支持变更：暂不支持。未来将支持
+- 是否支持变更：支持
 
 Fixed
 
 - 含义：是否固定访问IP/端口。若是，即使pod删除重建，网络内外映射关系不会改变
 - 填写格式：false / true
-- 是否支持变更：不支持
+- 是否支持变更：支持
 
 #### 插件配置
 ```