diff --git a/main.go b/main.go index ce68f0e24..8d3aa639e 100644 --- a/main.go +++ b/main.go @@ -111,7 +111,7 @@ func main() { mgr.GetEventRecorderFor("targetGroupBinding"), ctrl.Log) backendSGProvider := networking.NewBackendSGProvider(controllerCFG.ClusterName, controllerCFG.BackendSecurityGroup, cloud.VpcID(), cloud.EC2(), mgr.GetClient(), controllerCFG.DefaultTags, ctrl.Log.WithName("backend-sg-provider")) - sgResolver := networking.NewDefaultSecurityGroupResolver(cloud.EC2(), cloud.VpcID()) + sgResolver := networking.NewDefaultSecurityGroupResolver(cloud.EC2(), cloud.VpcID(), controllerCFG.ClusterName) ingGroupReconciler := ingress.NewGroupReconciler(cloud, mgr.GetClient(), mgr.GetEventRecorderFor("ingress"), finalizerManager, sgManager, sgReconciler, subnetResolver, controllerCFG, backendSGProvider, sgResolver, ctrl.Log.WithName("controllers").WithName("ingress")) diff --git a/pkg/ingress/model_build_load_balancer.go b/pkg/ingress/model_build_load_balancer.go index fc592572d..da9d06c54 100644 --- a/pkg/ingress/model_build_load_balancer.go +++ b/pkg/ingress/model_build_load_balancer.go @@ -203,12 +203,12 @@ func (t *defaultModelBuildTask) buildLoadBalancerSubnetMappings(ctx context.Cont if len(explicitSubnetSelectorList) != 0 { if len(explicitSubnetNameOrIDsList) != 0 { - return nil, errors.Errorf("conflicting subnet specifications: IngressClassParams versus annotation") + return nil, errors.New("conflicting subnet specifications: IngressClassParams versus annotation") } chosenSubnetSelector := explicitSubnetSelectorList[0] for _, subnetSelector := range explicitSubnetSelectorList[1:] { if !cmp.Equal(*chosenSubnetSelector, *subnetSelector) { - return nil, errors.Errorf("conflicting IngressClassParams subnet specifications") + return nil, errors.New("conflicting IngressClassParams subnet specifications") } } chosenSubnets, err := t.subnetsResolver.ResolveViaSelector(ctx, chosenSubnetSelector, @@ -227,7 +227,7 @@ func (t *defaultModelBuildTask) buildLoadBalancerSubnetMappings(ctx context.Cont for _, subnetNameOrIDs := range explicitSubnetNameOrIDsList[1:] { // subnetNameOrIDs order doesn't matter if !cmp.Equal(chosenSubnetNameOrIDs, subnetNameOrIDs, equality.IgnoreStringSliceOrder()) { - return nil, errors.Errorf("conflicting subnets: %v | %v", chosenSubnetNameOrIDs, subnetNameOrIDs) + return nil, fmt.Errorf("conflicting subnets: %v | %v", chosenSubnetNameOrIDs, subnetNameOrIDs) } } chosenSubnets, err := t.subnetsResolver.ResolveViaNameOrIDSlice(ctx, chosenSubnetNameOrIDs, @@ -269,78 +269,120 @@ func (t *defaultModelBuildTask) buildLoadBalancerSubnetMappings(ctx context.Cont } func (t *defaultModelBuildTask) buildLoadBalancerSecurityGroups(ctx context.Context, listenPortConfigByPort map[int64]listenPortConfig, ipAddressType elbv2model.IPAddressType) ([]core.StringToken, error) { - sgNameOrIDsViaAnnotation, err := t.buildFrontendSGNameOrIDsFromAnnotation(ctx) - if err != nil { - return nil, err - } + var explicitSGSelectorList []*v1beta1.SecurityGroupSelector + var explicitSGNameOrIDsList [][]string var lbSGTokens []core.StringToken - if len(sgNameOrIDsViaAnnotation) == 0 { - managedSG, err := t.buildManagedSecurityGroup(ctx, listenPortConfigByPort, ipAddressType) - if err != nil { - return nil, err + manageBackendSG := t.enableBackendSG + + for _, member := range t.ingGroup.Members { + if member.IngClassConfig.IngClassParams != nil { + if member.IngClassConfig.IngClassParams.Spec.SecurityGroups != nil { + explicitSGSelectorList = append(explicitSGSelectorList, member.IngClassConfig.IngClassParams.Spec.SecurityGroups) + continue + } + if len(member.IngClassConfig.IngClassParams.Spec.InboundCIDRs) > 0 { + explicitSGSelectorList = append(explicitSGSelectorList, &v1beta1.SecurityGroupSelector{ManagedInbound: true}) + continue + } + } + + var rawSGNameOrIDs []string + if exists := t.annotationParser.ParseStringSliceAnnotation(annotations.IngressSuffixSecurityGroups, &rawSGNameOrIDs, member.Ing.Annotations); exists { + explicitSGNameOrIDsList = append(explicitSGNameOrIDsList, rawSGNameOrIDs) } - lbSGTokens = append(lbSGTokens, managedSG.GroupID()) - if !t.enableBackendSG { - t.backendSGIDToken = managedSG.GroupID() + } + + if len(explicitSGSelectorList) != 0 { + if len(explicitSGNameOrIDsList) != 0 { + return nil, errors.New("conflicting security group specifications: IngressClassParams versus annotation") + } + chosenSGSelector := explicitSGSelectorList[0] + for _, sgSelector := range explicitSGSelectorList[1:] { + if !cmp.Equal(*chosenSGSelector, *sgSelector) { + return nil, errors.New("conflicting IngressClassParams security group specifications") + } + } + if chosenSGSelector.ManagedInbound { + if chosenSGSelector.ManagedBackend != nil { + manageBackendSG = *chosenSGSelector.ManagedBackend + } } else { - backendSGID, err := t.backendSGProvider.Get(ctx, networking.ResourceTypeIngress, k8s.ToSliceOfNamespacedNames(t.ingGroup.Members)) + frontendSGIDs, err := t.sgResolver.ResolveViaSelector(ctx, chosenSGSelector) if err != nil { return nil, err } - t.backendSGIDToken = core.LiteralStringToken((backendSGID)) - t.backendSGAllocated = true - lbSGTokens = append(lbSGTokens, t.backendSGIDToken) - } - t.logger.Info("Auto Create SG", "LB SGs", lbSGTokens, "backend SG", t.backendSGIDToken) - } else { - manageBackendSGRules, err := t.buildManageSecurityGroupRulesFlag(ctx) - if err != nil { - return nil, err - } - frontendSGIDs, err := t.sgResolver.ResolveViaNameOrID(ctx, sgNameOrIDsViaAnnotation) - if err != nil { - return nil, err + for _, sgID := range frontendSGIDs { + lbSGTokens = append(lbSGTokens, core.LiteralStringToken(sgID)) + } + if chosenSGSelector.ManagedBackend != nil && *chosenSGSelector.ManagedBackend { + backendSGID, err := t.backendSGProvider.Get(ctx, networking.ResourceTypeIngress, k8s.ToSliceOfNamespacedNames(t.ingGroup.Members)) + if err != nil { + return nil, err + } + t.backendSGIDToken = core.LiteralStringToken(backendSGID) + t.backendSGAllocated = true + lbSGTokens = append(lbSGTokens, t.backendSGIDToken) + } + return lbSGTokens, nil } - for _, sgID := range frontendSGIDs { - lbSGTokens = append(lbSGTokens, core.LiteralStringToken(sgID)) + } + + if len(explicitSGNameOrIDsList) > 0 { + sgNameOrIDsViaAnnotation := explicitSGNameOrIDsList[0] + for _, sgNameOrIDs := range explicitSGNameOrIDsList[1:] { + if !cmp.Equal(sgNameOrIDsViaAnnotation, sgNameOrIDs) { + return nil, fmt.Errorf("conflicting securityGroups: %v | %v", sgNameOrIDsViaAnnotation, sgNameOrIDs) + } } - if manageBackendSGRules { - if !t.enableBackendSG { - return nil, errors.New("backendSG feature is required to manage worker node SG rules when frontendSG manually specified") + if len(sgNameOrIDsViaAnnotation) > 0 { + manageBackendSGRules, err := t.buildManageSecurityGroupRulesFlag(ctx) + if err != nil { + return nil, err } - backendSGID, err := t.backendSGProvider.Get(ctx, networking.ResourceTypeIngress, k8s.ToSliceOfNamespacedNames(t.ingGroup.Members)) + frontendSGIDs, err := t.sgResolver.ResolveViaNameOrID(ctx, sgNameOrIDsViaAnnotation) if err != nil { return nil, err } - t.backendSGIDToken = core.LiteralStringToken(backendSGID) - t.backendSGAllocated = true - lbSGTokens = append(lbSGTokens, t.backendSGIDToken) - } - t.logger.Info("SG configured via annotation", "LB SGs", lbSGTokens, "backend SG", t.backendSGIDToken) - } - return lbSGTokens, nil -} + for _, sgID := range frontendSGIDs { + lbSGTokens = append(lbSGTokens, core.LiteralStringToken(sgID)) + } -func (t *defaultModelBuildTask) buildFrontendSGNameOrIDsFromAnnotation(ctx context.Context) ([]string, error) { - var explicitSGNameOrIDsList [][]string - for _, member := range t.ingGroup.Members { - var rawSGNameOrIDs []string - if exists := t.annotationParser.ParseStringSliceAnnotation(annotations.IngressSuffixSecurityGroups, &rawSGNameOrIDs, member.Ing.Annotations); !exists { - continue + if manageBackendSGRules { + if !t.enableBackendSG { + return nil, errors.New("backendSG feature is required to manage worker node SG rules when frontendSG manually specified") + } + backendSGID, err := t.backendSGProvider.Get(ctx, networking.ResourceTypeIngress, k8s.ToSliceOfNamespacedNames(t.ingGroup.Members)) + if err != nil { + return nil, err + } + t.backendSGIDToken = core.LiteralStringToken(backendSGID) + t.backendSGAllocated = true + lbSGTokens = append(lbSGTokens, t.backendSGIDToken) + } + t.logger.Info("SG configured via annotation", "LB SGs", lbSGTokens, "backend SG", t.backendSGIDToken) + return lbSGTokens, nil } - explicitSGNameOrIDsList = append(explicitSGNameOrIDsList, rawSGNameOrIDs) } - if len(explicitSGNameOrIDsList) == 0 { - return nil, nil + + managedSG, err := t.buildManagedSecurityGroup(ctx, listenPortConfigByPort, ipAddressType) + if err != nil { + return nil, err } - chosenSGNameOrIDs := explicitSGNameOrIDsList[0] - for _, sgNameOrIDs := range explicitSGNameOrIDsList[1:] { - if !cmp.Equal(chosenSGNameOrIDs, sgNameOrIDs) { - return nil, errors.Errorf("conflicting securityGroups: %v | %v", chosenSGNameOrIDs, sgNameOrIDs) + lbSGTokens = append(lbSGTokens, managedSG.GroupID()) + if !manageBackendSG { + t.backendSGIDToken = managedSG.GroupID() + } else { + backendSGID, err := t.backendSGProvider.Get(ctx, networking.ResourceTypeIngress, k8s.ToSliceOfNamespacedNames(t.ingGroup.Members)) + if err != nil { + return nil, err } + t.backendSGIDToken = core.LiteralStringToken(backendSGID) + t.backendSGAllocated = true + lbSGTokens = append(lbSGTokens, t.backendSGIDToken) } - return chosenSGNameOrIDs, nil + t.logger.Info("Auto Create SG", "LB SGs", lbSGTokens, "backend SG", t.backendSGIDToken) + return lbSGTokens, nil } func (t *defaultModelBuildTask) buildLoadBalancerCOIPv4Pool(_ context.Context) (*string, error) { diff --git a/pkg/ingress/model_build_load_balancer_test.go b/pkg/ingress/model_build_load_balancer_test.go index 87df34b55..4be18a511 100644 --- a/pkg/ingress/model_build_load_balancer_test.go +++ b/pkg/ingress/model_build_load_balancer_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/assert" networking "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/annotations" @@ -682,6 +683,924 @@ func Test_defaultModelBuildTask_buildLoadBalancerName(t *testing.T) { } } +var ( + sg1 = &ec2.SecurityGroup{ + GroupId: awssdk.String("sg-1"), + VpcId: awssdk.String("vpc-1"), + } + sg2 = &ec2.SecurityGroup{ + GroupId: awssdk.String("sg-2"), + Tags: []*ec2.Tag{ + { + Key: awssdk.String("Name"), + Value: awssdk.String("namedsg"), + }, + }, + VpcId: awssdk.String("vpc-1"), + } + sg3 = &ec2.SecurityGroup{ + GroupId: awssdk.String("sg-3"), + VpcId: awssdk.String("vpc-1"), + } + sg4 = &ec2.SecurityGroup{ + GroupId: awssdk.String("sg-4"), + Tags: []*ec2.Tag{ + { + Key: awssdk.String("Name"), + Value: awssdk.String("othername"), + }, + }, + VpcId: awssdk.String("vpc-1"), + } + sg5 = &ec2.SecurityGroup{ + GroupId: awssdk.String("sg-5"), + Tags: []*ec2.Tag{ + { + Key: awssdk.String("Name"), + Value: awssdk.String("namedsg"), + }, + }, + VpcId: awssdk.String("vpc-2"), + } + sg6 = &ec2.SecurityGroup{ + GroupId: awssdk.String("sg-6"), + Tags: []*ec2.Tag{ + { + Key: awssdk.String("Name"), + Value: awssdk.String("sg-1"), + }, + }, + VpcId: awssdk.String("vpc-1"), + } + sg7 = &ec2.SecurityGroup{ + GroupId: awssdk.String("sg-7"), + Tags: []*ec2.Tag{ + { + Key: awssdk.String("Name"), + Value: awssdk.String("sg-1"), + }, + { + Key: awssdk.String("class"), + Value: awssdk.String("test"), + }}, + VpcId: awssdk.String("vpc-1"), + } + sg8 = &ec2.SecurityGroup{ + GroupId: awssdk.String("sg-8"), + Tags: []*ec2.Tag{ + { + Key: awssdk.String("alb"), + Value: awssdk.String("test"), + }, + }, + VpcId: awssdk.String("vpc-1"), + } + sg9 = &ec2.SecurityGroup{ + GroupId: awssdk.String("sg-9"), + Tags: []*ec2.Tag{ + { + Key: awssdk.String("Name"), + Value: awssdk.String("sg-12"), + }, + { + Key: awssdk.String("alb"), + Value: awssdk.String("test"), + }, + { + Key: awssdk.String("class"), + Value: awssdk.String("test"), + }}, + VpcId: awssdk.String("vpc-1"), + } + sg10 = &ec2.SecurityGroup{ + GroupId: awssdk.String("sg-10"), + Tags: []*ec2.Tag{ + { + Key: awssdk.String("alb"), + Value: awssdk.String("testing"), + }, + { + Key: awssdk.String("class"), + Value: awssdk.String("test"), + }, + }, + VpcId: awssdk.String("vpc-1"), + } + sg11 = &ec2.SecurityGroup{ + GroupId: awssdk.String("sg-11"), + Tags: []*ec2.Tag{ + { + Key: awssdk.String("tagtest"), + Value: awssdk.String("1"), + }, + { + Key: awssdk.String("tagtest2"), + Value: awssdk.String("1"), + }, + }, + VpcId: awssdk.String("vpc-1"), + } + sg12 = &ec2.SecurityGroup{ + GroupId: awssdk.String("sg-12"), + Tags: []*ec2.Tag{ + { + Key: awssdk.String("tagtest"), + Value: awssdk.String("1"), + }, + }, + VpcId: awssdk.String("vpc-1"), + } + sg13 = &ec2.SecurityGroup{ + GroupId: awssdk.String("sg-13"), + Tags: []*ec2.Tag{ + { + Key: awssdk.String("tagtest"), + Value: awssdk.String("1"), + }, + { + Key: awssdk.String("kubernetes.io/cluster/other-cluster"), + Value: awssdk.String("shared"), + }, + }, + VpcId: awssdk.String("vpc-1"), + } + sg14 = &ec2.SecurityGroup{ + GroupId: awssdk.String("sg-14"), + Tags: []*ec2.Tag{ + { + Key: awssdk.String("tagtest2"), + Value: awssdk.String("1"), + }, + }, + VpcId: awssdk.String("vpc-1"), + } + sg15 = &ec2.SecurityGroup{ + GroupId: awssdk.String("sg-15"), + Tags: []*ec2.Tag{ + { + Key: awssdk.String("tagtest2"), + Value: awssdk.String("1"), + }, + { + Key: awssdk.String("kubernetes.io/cluster/test-cluster"), + Value: awssdk.String("shared"), + }, + }, + VpcId: awssdk.String("vpc-1"), + } +) + +func stubDescribeSecurityGroupsAsList(ctx context.Context, input *ec2.DescribeSecurityGroupsInput) ([]*ec2.SecurityGroup, error) { + sgs := []*ec2.SecurityGroup{ + sg1, + sg2, + sg3, + sg4, + sg5, + sg6, + sg7, + sg8, + sg9, + sg10, + sg11, + sg12, + sg13, + sg14, + sg15, + } + if input.GroupIds != nil { + var filtered []*ec2.SecurityGroup + for _, sg := range sgs { + for _, id := range input.GroupIds { + if awssdk.StringValue(sg.GroupId) == awssdk.StringValue(id) { + filtered = append(filtered, sg) + continue + } + } + } + sgs = filtered + } + if input.Filters != nil { + var filtered []*ec2.SecurityGroup + sgLoop: + for _, sg := range sgs { + for _, filter := range input.Filters { + eligible := false + if awssdk.StringValue(filter.Name) == "vpc-id" { + for _, name := range filter.Values { + if awssdk.StringValue(sg.VpcId) == awssdk.StringValue(name) { + eligible = true + continue + } + } + } else if strings.HasPrefix(awssdk.StringValue(filter.Name), "tag:") { + key := strings.TrimPrefix(awssdk.StringValue(filter.Name), "tag:") + for _, value := range filter.Values { + for _, tag := range sg.Tags { + if awssdk.StringValue(tag.Key) == key && awssdk.StringValue(tag.Value) == awssdk.StringValue(value) { + eligible = true + continue + } + } + } + } else { + return nil, fmt.Errorf("unexpected filter %q", awssdk.StringValue(filter.Name)) + } + if !eligible { + continue sgLoop + } + } + filtered = append(filtered, sg) + } + sgs = filtered + } + return sgs, nil +} + +func Test_defaultModelBuildTask_buildLoadBalancerSecurityGroups(t *testing.T) { + type fields struct { + ingGroup Group + scheme elbv2.LoadBalancerScheme + enableBackendSG bool + } + tests := []struct { + name string + fields fields + want []string + wantErr string + }{ + { + name: "no annotation managed SG", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Namespace: "awesome-ns", Name: "ing-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + }, + }, + }, + }, + }, + scheme: elbv2.LoadBalancerSchemeInternetFacing, + }, + want: []string{"sg-managed"}, + }, + { + name: "no annotation managed SG backend", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Namespace: "awesome-ns", Name: "ing-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + }, + }, + }, + }, + }, + scheme: elbv2.LoadBalancerSchemeInternetFacing, + enableBackendSG: true, + }, + want: []string{"sg-managed", "sg-backend"}, + }, + { + name: "security-groups annotation", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Name: "explicit-group"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/security-groups": "sg-1,namedsg", + }, + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &v1beta1.IngressClassParams{ + Spec: v1beta1.IngressClassParamsSpec{}, + }, + }, + }, + }, + }, + enableBackendSG: true, + }, + want: []string{"sg-1", "sg-2"}, + }, + { + name: "security-groups and backend annotation", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Name: "explicit-group"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/security-groups": "sg-1,namedsg", + "alb.ingress.kubernetes.io/manage-backend-security-group-rules": "true", + }, + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &v1beta1.IngressClassParams{ + Spec: v1beta1.IngressClassParamsSpec{}, + }, + }, + }, + }, + }, + enableBackendSG: true, + }, + want: []string{"sg-1", "sg-2", "sg-backend"}, + }, + { + name: "classparams sg ids", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Namespace: "awesome-ns", Name: "ing-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/security-groups": "sg-4,othername", + "alb.ingress.kubernetes.io/manage-backend-security-group-rules": "true", + }, + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &v1beta1.IngressClassParams{ + Spec: v1beta1.IngressClassParamsSpec{ + SecurityGroups: &v1beta1.SecurityGroupSelector{ + IDs: []v1beta1.SecurityGroupID{"sg-1", "sg-2"}, + }, + }, + }, + }, + }, + }, + }, + enableBackendSG: true, + }, + want: []string{"sg-1", "sg-2"}, + }, + { + name: "classparams sg ids backend", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Namespace: "awesome-ns", Name: "ing-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/security-groups": "sg-4,othername", + "alb.ingress.kubernetes.io/manage-backend-security-group-rules": "false", + }, + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &v1beta1.IngressClassParams{ + Spec: v1beta1.IngressClassParamsSpec{ + SecurityGroups: &v1beta1.SecurityGroupSelector{ + IDs: []v1beta1.SecurityGroupID{"sg-1", "sg-2"}, + ManagedBackend: awssdk.Bool(true), + }, + }, + }, + }, + }, + }, + }, + }, + want: []string{"sg-1", "sg-2", "sg-backend"}, + }, + { + name: "classparams sg managed", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Namespace: "awesome-ns", Name: "ing-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/security-groups": "sg-4,othername", + "alb.ingress.kubernetes.io/manage-backend-security-group-rules": "true", + }, + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &v1beta1.IngressClassParams{ + Spec: v1beta1.IngressClassParamsSpec{ + SecurityGroups: &v1beta1.SecurityGroupSelector{ + ManagedInbound: true, + }, + }, + }, + }, + }, + }, + }, + }, + want: []string{"sg-managed"}, + }, + { + name: "classparams sg managed default backend", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Namespace: "awesome-ns", Name: "ing-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/security-groups": "sg-4,othername", + "alb.ingress.kubernetes.io/manage-backend-security-group-rules": "false", + }, + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &v1beta1.IngressClassParams{ + Spec: v1beta1.IngressClassParamsSpec{ + SecurityGroups: &v1beta1.SecurityGroupSelector{ + ManagedInbound: true, + }, + }, + }, + }, + }, + }, + }, + enableBackendSG: true, + }, + want: []string{"sg-managed", "sg-backend"}, + }, + { + name: "classparams sg managed explicit backend", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Namespace: "awesome-ns", Name: "ing-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/security-groups": "sg-4,othername", + "alb.ingress.kubernetes.io/manage-backend-security-group-rules": "false", + }, + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &v1beta1.IngressClassParams{ + Spec: v1beta1.IngressClassParamsSpec{ + SecurityGroups: &v1beta1.SecurityGroupSelector{ + ManagedBackend: awssdk.Bool(true), + ManagedInbound: true, + }, + }, + }, + }, + }, + }, + }, + }, want: []string{"sg-managed", "sg-backend"}, + }, + { + name: "classparams sg managed disable backend", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Namespace: "awesome-ns", Name: "ing-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/security-groups": "sg-4,othername", + "alb.ingress.kubernetes.io/manage-backend-security-group-rules": "true", + }, + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &v1beta1.IngressClassParams{ + Spec: v1beta1.IngressClassParamsSpec{ + SecurityGroups: &v1beta1.SecurityGroupSelector{ + ManagedBackend: awssdk.Bool(false), + ManagedInbound: true, + }, + }, + }, + }, + }, + }, + }, + enableBackendSG: true, + }, + want: []string{"sg-managed"}, + }, + { + name: "classparams inboundCIDRs", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Namespace: "awesome-ns", Name: "ing-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/security-groups": "sg-4,othername", + "alb.ingress.kubernetes.io/manage-backend-security-group-rules": "true", + }, + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &v1beta1.IngressClassParams{ + Spec: v1beta1.IngressClassParamsSpec{ + InboundCIDRs: []string{"10.0.0.0/8"}, + }, + }, + }, + }, + }, + }, + }, + want: []string{"sg-managed"}, + }, + { + name: "classparams inboundCIDRs backend", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Namespace: "awesome-ns", Name: "ing-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/security-groups": "sg-4,othername", + "alb.ingress.kubernetes.io/manage-backend-security-group-rules": "false", + }, + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &v1beta1.IngressClassParams{ + Spec: v1beta1.IngressClassParamsSpec{ + InboundCIDRs: []string{"10.0.0.0/8"}, + }, + }, + }, + }, + }, + }, + enableBackendSG: true, + }, + want: []string{"sg-managed", "sg-backend"}, + }, + { + name: "classparams sg tag multiple values", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Namespace: "awesome-ns", Name: "ing-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/security-groups": "sg-4,othername", + "alb.ingress.kubernetes.io/manage-backend-security-group-rules": "true", + }, + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &v1beta1.IngressClassParams{ + Spec: v1beta1.IngressClassParamsSpec{ + SecurityGroups: &v1beta1.SecurityGroupSelector{ + Tags: map[string][]string{ + "Name": {"namedsg", "othername"}, + }, + }, + }, + }, + }, + }, + }, + }, + enableBackendSG: true, + }, + want: []string{"sg-2", "sg-4"}, + }, + { + name: "classparams sg tag multiple values backend", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Namespace: "awesome-ns", Name: "ing-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/security-groups": "sg-4,othername", + "alb.ingress.kubernetes.io/manage-backend-security-group-rules": "true", + }, + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &v1beta1.IngressClassParams{ + Spec: v1beta1.IngressClassParamsSpec{ + SecurityGroups: &v1beta1.SecurityGroupSelector{ + ManagedBackend: awssdk.Bool(true), + Tags: map[string][]string{ + "Name": {"namedsg", "othername"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + want: []string{"sg-2", "sg-4", "sg-backend"}, + }, + { + name: "classparams sg tag multiple matches", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Namespace: "awesome-ns", Name: "ing-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/security-groups": "sg-4,othername", + }, + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &v1beta1.IngressClassParams{ + Spec: v1beta1.IngressClassParamsSpec{ + SecurityGroups: &v1beta1.SecurityGroupSelector{ + Tags: map[string][]string{ + "Name": {"sg-1"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + want: []string{"sg-6", "sg-7"}, + }, + { + name: "classparams custom tag", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Namespace: "awesome-ns", Name: "ing-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &v1beta1.IngressClassParams{ + Spec: v1beta1.IngressClassParamsSpec{ + SecurityGroups: &v1beta1.SecurityGroupSelector{ + Tags: map[string][]string{ + "alb": {"test"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + want: []string{"sg-8", "sg-9"}, + }, + { + name: "classparams multiple tags", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Namespace: "awesome-ns", Name: "ing-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &v1beta1.IngressClassParams{ + Spec: v1beta1.IngressClassParamsSpec{ + SecurityGroups: &v1beta1.SecurityGroupSelector{ + Tags: map[string][]string{ + "alb": {"test", "testing"}, + "class": {"test"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + want: []string{"sg-10", "sg-9"}, + }, + { + name: "classparams missing id", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Namespace: "awesome-ns", Name: "ing-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + Annotations: map[string]string{ + "alb.ingress.kubernetes.io/security-groups": "sg-4,othername", + }, + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &v1beta1.IngressClassParams{ + Spec: v1beta1.IngressClassParamsSpec{ + SecurityGroups: &v1beta1.SecurityGroupSelector{ + IDs: []v1beta1.SecurityGroupID{"sg-1234", "sg-1"}, + }, + }, + }, + }, + }, + }, + }, + }, + wantErr: "couldn't find all security groups: IDs: [sg-1234 sg-1], found: 1", + }, + { + name: "classparams ignore tagged other cluster", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Namespace: "awesome-ns", Name: "ing-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &v1beta1.IngressClassParams{ + Spec: v1beta1.IngressClassParamsSpec{ + SecurityGroups: &v1beta1.SecurityGroupSelector{ + Tags: map[string][]string{ + "tagtest": {"1"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + want: []string{"sg-11", "sg-12"}, + }, + { + name: "classparams prefer tagged for cluster", + fields: fields{ + ingGroup: Group{ + ID: GroupID{Namespace: "awesome-ns", Name: "ing-1"}, + Members: []ClassifiedIngress{ + { + Ing: &networking.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "awesome-ns", + Name: "ing-1", + }, + }, + IngClassConfig: ClassConfiguration{ + IngClassParams: &v1beta1.IngressClassParams{ + Spec: v1beta1.IngressClassParamsSpec{ + SecurityGroups: &v1beta1.SecurityGroupSelector{ + Tags: map[string][]string{ + "tagtest2": {"1"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + want: []string{"sg-15"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + backendSGProvider := networking2.NewMockBackendSGProvider(ctrl) + backendSGProvider.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, resourceType networking2.ResourceType, activeResources []types.NamespacedName) (string, error) { + return "sg-backend", nil + }). + MaxTimes(1) + + mockEC2 := services.NewMockEC2(ctrl) + mockEC2.EXPECT().DescribeSecurityGroupsAsList(gomock.Any(), gomock.Any()). + DoAndReturn(stubDescribeSecurityGroupsAsList). + AnyTimes() + + sgResolver := networking2.NewDefaultSecurityGroupResolver( + mockEC2, + "vpc-1", + "test-cluster", + ) + + task := &defaultModelBuildTask{ + featureGates: config.NewFeatureGates(), + ingGroup: tt.fields.ingGroup, + stack: core.NewDefaultStack(core.StackID(tt.fields.ingGroup.ID)), + annotationParser: annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io"), + backendSGProvider: backendSGProvider, + sgResolver: sgResolver, + trackingProvider: tracking.NewDefaultProvider("ingress.k8s.aws", "test-cluster"), + logger: logr.Discard(), + enableBackendSG: tt.fields.enableBackendSG, + } + listenPortConfigByType := map[int64]listenPortConfig{ + 80: { + protocol: elbv2.ProtocolHTTP, + }, + } + got, err := task.buildLoadBalancerSecurityGroups(context.Background(), listenPortConfigByType, elbv2.IPAddressTypeDualStack) + if tt.wantErr != "" { + assert.EqualError(t, err, tt.wantErr) + } else { + assert.NoError(t, err) + var gotSGs []string + for i, mapping := range got { + if _, ok := mapping.(*core.ResourceFieldStringToken); ok { + gotSGs = append(gotSGs, "sg-managed") + continue + } + sg, err := mapping.Resolve(context.Background()) + assert.NoError(t, err, "SG mapping %d", i) + gotSGs = append(gotSGs, sg) + } + assert.Equal(t, tt.want, gotSGs) + } + }) + } +} + var ( subnet1 = &ec2.Subnet{ SubnetId: awssdk.String("subnet-1"), @@ -1283,9 +2202,10 @@ func Test_defaultModelBuildTask_buildLoadBalancerSubnets(t *testing.T) { trackingProvider: tracking.NewDefaultProvider("ingress.k8s.aws", "test-cluster"), } got, err := task.buildLoadBalancerSubnetMappings(context.Background(), elbv2.LoadBalancerSchemeInternetFacing) - if err != nil { + if tt.wantErr != "" { assert.EqualError(t, err, tt.wantErr) } else { + assert.NoError(t, err) var gotSubnets []string for _, mapping := range got { gotSubnets = append(gotSubnets, mapping.SubnetID) diff --git a/pkg/ingress/model_builder_test.go b/pkg/ingress/model_builder_test.go index e8400eef5..cd0f6cfbc 100644 --- a/pkg/ingress/model_builder_test.go +++ b/pkg/ingress/model_builder_test.go @@ -2920,7 +2920,7 @@ func Test_defaultModelBuilder_Build(t *testing.T) { trackingProvider := tracking.NewDefaultProvider("ingress.k8s.aws", clusterName) stackMarshaller := deploy.NewDefaultStackMarshaller() backendSGProvider := networkingpkg.NewMockBackendSGProvider(ctrl) - sgResolver := networkingpkg.NewDefaultSecurityGroupResolver(ec2Client, vpcID) + sgResolver := networkingpkg.NewDefaultSecurityGroupResolver(ec2Client, vpcID, clusterName) if tt.fields.enableBackendSG { if len(tt.fields.backendSecurityGroup) > 0 { backendSGProvider.EXPECT().Get(gomock.Any(), networkingpkg.ResourceType(networkingpkg.ResourceTypeIngress), gomock.Any()).Return(tt.fields.backendSecurityGroup, nil).AnyTimes() diff --git a/pkg/networking/security_group_resolver.go b/pkg/networking/security_group_resolver.go index 402d1795f..a26596fcc 100644 --- a/pkg/networking/security_group_resolver.go +++ b/pkg/networking/security_group_resolver.go @@ -2,11 +2,14 @@ package networking import ( "context" + "fmt" + "sort" "strings" awssdk "github.com/aws/aws-sdk-go/aws" ec2sdk "github.com/aws/aws-sdk-go/service/ec2" "github.com/pkg/errors" + "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" "sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services" ) @@ -14,13 +17,16 @@ import ( type SecurityGroupResolver interface { // ResolveViaNameOrID resolves security groups from the security group names or the IDs ResolveViaNameOrID(ctx context.Context, sgNameOrIDs []string) ([]string, error) + // ResolveViaSelector resolves security groups from a SecurityGroupSelector + ResolveViaSelector(ctx context.Context, sgSelector *v1beta1.SecurityGroupSelector) ([]string, error) } // NewDefaultSecurityGroupResolver constructs new defaultSecurityGroupResolver. -func NewDefaultSecurityGroupResolver(ec2Client services.EC2, vpcID string) *defaultSecurityGroupResolver { +func NewDefaultSecurityGroupResolver(ec2Client services.EC2, vpcID string, clusterName string) *defaultSecurityGroupResolver { return &defaultSecurityGroupResolver{ - ec2Client: ec2Client, - vpcID: vpcID, + ec2Client: ec2Client, + vpcID: vpcID, + clusterName: clusterName, } } @@ -28,8 +34,9 @@ var _ SecurityGroupResolver = &defaultSecurityGroupResolver{} // default implementation for SecurityGroupResolver type defaultSecurityGroupResolver struct { - ec2Client services.EC2 - vpcID string + ec2Client services.EC2 + vpcID string + clusterName string } func (r *defaultSecurityGroupResolver) ResolveViaNameOrID(ctx context.Context, sgNameOrIDs []string) ([]string, error) { @@ -102,3 +109,102 @@ func (r *defaultSecurityGroupResolver) splitIntoSgNameAndIDs(sgNameOrIDs []strin } return sgIDs, sgNames } + +func (r *defaultSecurityGroupResolver) ResolveViaSelector(ctx context.Context, selector *v1beta1.SecurityGroupSelector) ([]string, error) { + var chosenSGs []*ec2sdk.SecurityGroup + var err error + if selector.IDs != nil { + req := &ec2sdk.DescribeSecurityGroupsInput{ + GroupIds: make([]*string, 0, len(selector.IDs)), + } + for _, groupID := range selector.IDs { + id := string(groupID) + req.GroupIds = append(req.GroupIds, &id) + } + chosenSGs, err = r.ec2Client.DescribeSecurityGroupsAsList(ctx, req) + if err != nil { + return nil, err + } + if len(chosenSGs) != len(selector.IDs) { + return nil, fmt.Errorf("couldn't find all security groups: IDs: %v, found: %v", selector.IDs, len(chosenSGs)) + } + } else { + req := &ec2sdk.DescribeSecurityGroupsInput{ + Filters: []*ec2sdk.Filter{ + { + Name: awssdk.String("vpc-id"), + Values: awssdk.StringSlice([]string{r.vpcID}), + }, + }, + } + for key, values := range selector.Tags { + req.Filters = append(req.Filters, &ec2sdk.Filter{ + Name: awssdk.String("tag:" + key), + Values: awssdk.StringSlice(values), + }) + } + + allSGs, err := r.ec2Client.DescribeSecurityGroupsAsList(ctx, req) + if err != nil { + return nil, err + } + var filteredSGs []*ec2sdk.SecurityGroup + for _, sg := range allSGs { + if r.checkSecurityGroupIsNotTaggedForOtherClusters(sg) { + filteredSGs = append(filteredSGs, sg) + } + } + for _, sg := range filteredSGs { + if r.checkSecurityGroupHasClusterTag(sg) { + chosenSGs = append(chosenSGs, sg) + } + } + if len(chosenSGs) == 0 { + chosenSGs = filteredSGs + } + } + if len(chosenSGs) == 0 { + return nil, errors.New("unable to resolve at least one security group") + } + resolvedSGIDs := make([]string, 0, len(chosenSGs)) + for _, sg := range chosenSGs { + resolvedSGIDs = append(resolvedSGIDs, awssdk.StringValue(sg.GroupId)) + } + sort.Strings(resolvedSGIDs) + return resolvedSGIDs, nil +} + +// checkSecurityGroupHasClusterTag checks if the subnet is tagged for the current cluster +func (r *defaultSecurityGroupResolver) checkSecurityGroupHasClusterTag(securityGroup *ec2sdk.SecurityGroup) bool { + clusterResourceTagKey := fmt.Sprintf("kubernetes.io/cluster/%s", r.clusterName) + for _, tag := range securityGroup.Tags { + if clusterResourceTagKey == awssdk.StringValue(tag.Key) { + return true + } + } + return false +} + +// checkSecurityGroupIsNotTaggedForOtherClusters checks whether the security group is tagged for the current cluster +// or it doesn't contain the cluster tag at all. If the security group contains a tag for other clusters, then +// this check returns false so that the security group is not used for the load balancer. +func (r *defaultSecurityGroupResolver) checkSecurityGroupIsNotTaggedForOtherClusters(sg *ec2sdk.SecurityGroup) bool { + clusterResourceTagPrefix := "kubernetes.io/cluster" + clusterResourceTagKey := fmt.Sprintf("kubernetes.io/cluster/%s", r.clusterName) + hasClusterResourceTagPrefix := false + for _, tag := range sg.Tags { + tagKey := awssdk.StringValue(tag.Key) + if tagKey == clusterResourceTagKey { + return true + } + if strings.HasPrefix(tagKey, clusterResourceTagPrefix) { + // If the cluster tag is for a different cluster, keep track of it and exclude + // the security group if no matching tag found for the current cluster. + hasClusterResourceTagPrefix = true + } + } + if hasClusterResourceTagPrefix { + return false + } + return true +} diff --git a/pkg/networking/security_group_resolver_mocks.go b/pkg/networking/security_group_resolver_mocks.go index d294207db..a95eda1d5 100644 --- a/pkg/networking/security_group_resolver_mocks.go +++ b/pkg/networking/security_group_resolver_mocks.go @@ -9,6 +9,7 @@ import ( reflect "reflect" gomock "github.com/golang/mock/gomock" + v1beta1 "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1" ) // MockSecurityGroupResolver is a mock of SecurityGroupResolver interface. @@ -48,3 +49,18 @@ func (mr *MockSecurityGroupResolverMockRecorder) ResolveViaNameOrID(arg0, arg1 i mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveViaNameOrID", reflect.TypeOf((*MockSecurityGroupResolver)(nil).ResolveViaNameOrID), arg0, arg1) } + +// ResolveViaSelector mocks base method. +func (m *MockSecurityGroupResolver) ResolveViaSelector(arg0 context.Context, arg1 *v1beta1.SecurityGroupSelector) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ResolveViaSelector", arg0, arg1) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ResolveViaSelector indicates an expected call of ResolveViaSelector. +func (mr *MockSecurityGroupResolverMockRecorder) ResolveViaSelector(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResolveViaSelector", reflect.TypeOf((*MockSecurityGroupResolver)(nil).ResolveViaSelector), arg0, arg1) +} diff --git a/pkg/networking/subnet_resolver.go b/pkg/networking/subnet_resolver.go index afc663d7d..35e83cac5 100644 --- a/pkg/networking/subnet_resolver.go +++ b/pkg/networking/subnet_resolver.go @@ -396,8 +396,8 @@ func (r *defaultSubnetsResolver) checkSubnetHasClusterTag(subnet *ec2sdk.Subnet) // checkSubnetIsNotTaggedForOtherClusters checks whether the subnet is tagged for the current cluster // or it doesn't contain the cluster tag at all. If the subnet contains a tag for other clusters, then -// this check returns false so that the subnet does not used for the load balancer. -// it returns true if the subnetsClusterTagCheck is disabled +// this check returns false so that the subnet is not used for the load balancer. +// It returns true if the subnetsClusterTagCheck is disabled. func (r *defaultSubnetsResolver) checkSubnetIsNotTaggedForOtherClusters(subnet *ec2sdk.Subnet, subnetsClusterTagCheck bool) bool { if !subnetsClusterTagCheck { return true