Add new machinecontrollermanager Golang component (gardener#7598)

* [component] Add component boilerplate The bootstrapper component is similar to https://github.com/gardener/gardener/blob/master/pkg/operation/botanist/component/clusterautoscaler/bootstrap.go and will be responsible for managing the global resources in the runtime cluster (`ClusterRole`, `ClusterRoleBinding`). * [component] ServiceAccount See https://github.com/gardener/gardener/blob/master/charts/gardener/provider-local/internal/machine-controller-manager/seed/templates/serviceaccount.yaml * [component] ClusterRole See https://github.com/gardener/gardener/blob/master/charts/gardener/provider-local/internal/machine-controller-manager/seed/templates/clusterrole.yaml Earlier, the very same ClusterRole was created for each and every shoot namespace. In the future, there will only be one ClusterRole that will be shared for all instances. * [component] ClusterRoleBinding See https://github.com/gardener/gardener/blob/master/charts/gardener/provider-local/internal/machine-controller-manager/seed/templates/clusterrolebinding.yaml * [component] Service See https://github.com/gardener/gardener/blob/master/charts/gardener/provider-local/internal/machine-controller-manager/seed/templates/service.yaml * [component] Deployment See https://github.com/gardener/gardener/blob/master/charts/gardener/provider-local/internal/machine-controller-manager/seed/templates/deployment.yaml The resource requests are adapted according to https://github.com/gardener/gardener-extension-provider-aws/blob/5f0d082c2fc02ae5fe9a22d7f02af84d05dce111/charts/internal/machine-controller-manager/seed/values.yaml#L36-L42 * [component] PodDisruptionBudget See https://github.com/gardener/gardener/blob/master/charts/gardener/provider-local/internal/machine-controller-manager/seed/templates/poddisruptionbudget.yaml * [component] VerticalPodAutoscaler See https://github.com/gardener/gardener/blob/master/charts/gardener/provider-local/internal/machine-controller-manager/seed/templates/vpa.yaml Adapted `containerPolicies` according to https://github.com/gardener/gardener-extension-provider-aws/blob/5f0d082c2fc02ae5fe9a22d7f02af84d05dce111/charts/internal/machine-controller-manager/seed/templates/vpa.yaml and https://github.com/gardener/gardener-extension-provider-aws/blob/5f0d082c2fc02ae5fe9a22d7f02af84d05dce111/charts/internal/machine-controller-manager/seed/values.yaml#L22-L30 * [component] ClusterRole (target) See https://github.com/gardener/gardener/blob/master/charts/gardener/provider-local/internal/machine-controller-manager/shoot/templates/clusterrole-machine-controller-manager.yaml * [component] ClusterRoleBinding (target) See https://github.com/gardener/gardener/blob/master/charts/gardener/provider-local/internal/machine-controller-manager/shoot/templates/clusterrolebinding-machine-controller-manager.yaml * [component] Role (target) See https://github.com/gardener/gardener/blob/master/charts/gardener/provider-local/internal/machine-controller-manager/shoot/templates/role-machine-controller-manager.yaml * [component] RoleBinding (target) See https://github.com/gardener/gardener/blob/master/charts/gardener/provider-local/internal/machine-controller-manager/shoot/templates/rolebinding-machine-controller-manager.yaml * [component] Logging configuration See https://github.com/gardener/gardener/blob/master/charts/gardener/provider-local/extension/templates/configmap-logging.yaml * [component] Monitoring configuration See https://github.com/gardener/gardener/blob/master/charts/gardener/provider-local/internal/machine-controller-manager/seed/templates/configmap-monitoring.yaml * [component] Allow setting namespace UID after instantiation Similar to https://github.com/gardener/gardener/blob/319a41500affed25cb6808b0e543bbcf0277c3bf/pkg/operation/botanist/component/clusterautoscaler/cluster_autoscaler.go#L69-L70 The namespace UID is typically not known when the component gets instantiated * [component] Implement `Wait*` methods * [component] Mocks for component interface They can be used in the future in the botanist to mock the component behaviour. * Address PR review feedback * Address PR review feedback
andrerun · Jul 6, 2023 · 0cee943 · 0cee943
1 parent 0184594
commit 0cee943
Show file tree

Hide file tree

Showing 12 changed files with 2,024 additions and 3 deletions.
diff --git a/pkg/operation/botanist/component/clusterautoscaler/cluster_autoscaler.go b/pkg/operation/botanist/component/clusterautoscaler/cluster_autoscaler.go
@@ -89,7 +89,7 @@ func New(
 		image:                         image,
 		replicas:                      replicas,
 		config:                        config,
-		runtimeVersionGreaterEqual123: versionutils.ConstraintK8sGreaterEqual123.Check(runtimeKubernetesVersion),
+		runtimeVersionGreaterEqual121: versionutils.ConstraintK8sGreaterEqual121.Check(runtimeKubernetesVersion),
 	}
 }
 
@@ -101,7 +101,7 @@ type clusterAutoscaler struct {
 	replicas       int32
 	config         *gardencorev1beta1.ClusterAutoscaler
 
-	runtimeVersionGreaterEqual123 bool
+	runtimeVersionGreaterEqual121 bool
 	namespaceUID                  types.UID
 	machineDeployments            []extensionsv1alpha1.MachineDeployment
 }
@@ -358,7 +358,7 @@ func (c *clusterAutoscaler) emptyDeployment() *appsv1.Deployment {
 func (c *clusterAutoscaler) emptyPodDisruptionBudget() client.Object {
 	objectMeta := metav1.ObjectMeta{Name: v1beta1constants.DeploymentNameClusterAutoscaler, Namespace: c.namespace}
 
-	if c.runtimeVersionGreaterEqual123 {
+	if c.runtimeVersionGreaterEqual121 {
 		return &policyv1.PodDisruptionBudget{ObjectMeta: objectMeta}
 	}
 	return &policyv1beta1.PodDisruptionBudget{ObjectMeta: objectMeta}

diff --git a/pkg/operation/botanist/component/machinecontrollermanager/bootstrap.go b/pkg/operation/botanist/component/machinecontrollermanager/bootstrap.go
@@ -0,0 +1,113 @@
+// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package machinecontrollermanager
+
+import (
+	"context"
+	"time"
+
+	machinev1alpha1 "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1"
+	coordinationv1 "k8s.io/api/coordination/v1"
+	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/gardener/gardener/pkg/client/kubernetes"
+	"github.com/gardener/gardener/pkg/operation/botanist/component"
+	"github.com/gardener/gardener/pkg/utils/managedresources"
+)
+
+const (
+	managedResourceControlName = "machine-controller-manager"
+	clusterRoleName            = "system:machine-controller-manager-runtime"
+)
+
+// NewBootstrapper creates a new instance of DeployWaiter for the machine-controller-manager bootstrapper.
+func NewBootstrapper(client client.Client, namespace string) component.DeployWaiter {
+	return &bootstrapper{
+		client:    client,
+		namespace: namespace,
+	}
+}
+
+type bootstrapper struct {
+	client    client.Client
+	namespace string
+}
+
+func (b *bootstrapper) Deploy(ctx context.Context) error {
+	var (
+		registry = managedresources.NewRegistry(kubernetes.SeedScheme, kubernetes.SeedCodec, kubernetes.SeedSerializer)
+
+		clusterRole = &rbacv1.ClusterRole{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: clusterRoleName,
+			},
+			Rules: []rbacv1.PolicyRule{
+				{
+					APIGroups: []string{machinev1alpha1.GroupName},
+					Resources: []string{"*"},
+					Verbs:     []string{"*"},
+				},
+				{
+					APIGroups: []string{corev1.GroupName},
+					Resources: []string{"configmaps", "secrets", "endpoints", "events", "pods"},
+					Verbs:     []string{"*"},
+				},
+				{
+					APIGroups: []string{coordinationv1.GroupName},
+					Resources: []string{"leases"},
+					Verbs:     []string{"create"},
+				},
+				{
+					APIGroups:     []string{coordinationv1.GroupName},
+					Resources:     []string{"leases"},
+					Verbs:         []string{"get", "watch", "update"},
+					ResourceNames: []string{"machine-controller", "machine-controller-manager"},
+				},
+			},
+		}
+	)
+
+	resources, err := registry.AddAllAndSerialize(clusterRole)
+	if err != nil {
+		return err
+	}
+
+	return managedresources.CreateForSeed(ctx, b.client, b.namespace, managedResourceControlName, false, resources)
+}
+
+func (b *bootstrapper) Destroy(ctx context.Context) error {
+	return managedresources.DeleteForSeed(ctx, b.client, b.namespace, managedResourceControlName)
+}
+
+// TimeoutWaitForManagedResource is the timeout used while waiting for the ManagedResources to become healthy
+// or deleted.
+var TimeoutWaitForManagedResource = 2 * time.Minute
+
+func (b *bootstrapper) Wait(ctx context.Context) error {
+	timeoutCtx, cancel := context.WithTimeout(ctx, TimeoutWaitForManagedResource)
+	defer cancel()
+
+	return managedresources.WaitUntilHealthy(timeoutCtx, b.client, b.namespace, managedResourceControlName)
+}
+
+func (b *bootstrapper) WaitCleanup(ctx context.Context) error {
+	timeoutCtx, cancel := context.WithTimeout(ctx, TimeoutWaitForManagedResource)
+	defer cancel()
+
+	return managedresources.WaitUntilDeleted(timeoutCtx, b.client, b.namespace, managedResourceControlName)
+}
diff --git a/pkg/operation/botanist/component/machinecontrollermanager/bootstrap_test.go b/pkg/operation/botanist/component/machinecontrollermanager/bootstrap_test.go
@@ -0,0 +1,245 @@
+// Copyright (c) 2023 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package machinecontrollermanager_test
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/pointer"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
+	resourcesv1alpha1 "github.com/gardener/gardener/pkg/apis/resources/v1alpha1"
+	"github.com/gardener/gardener/pkg/client/kubernetes"
+	"github.com/gardener/gardener/pkg/operation/botanist/component"
+	. "github.com/gardener/gardener/pkg/operation/botanist/component/machinecontrollermanager"
+	"github.com/gardener/gardener/pkg/utils/retry"
+	retryfake "github.com/gardener/gardener/pkg/utils/retry/fake"
+	"github.com/gardener/gardener/pkg/utils/test"
+	. "github.com/gardener/gardener/pkg/utils/test/matchers"
+)
+
+var _ = Describe("Bootstrap", func() {
+	var (
+		ctx = context.TODO()
+
+		managedResourceName = "machine-controller-manager"
+		namespace           = "some-namespace"
+
+		fakeClient client.Client
+		mcm        component.DeployWaiter
+
+		managedResource       *resourcesv1alpha1.ManagedResource
+		managedResourceSecret *corev1.Secret
+
+		clusterRoleYAML = `apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: system:machine-controller-manager-runtime
+rules:
+- apiGroups:
+  - machine.sapcloud.io
+  resources:
+  - '*'
+  verbs:
+  - '*'
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - secrets
+  - endpoints
+  - events
+  - pods
+  verbs:
+  - '*'
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+- apiGroups:
+  - coordination.k8s.io
+  resourceNames:
+  - machine-controller
+  - machine-controller-manager
+  resources:
+  - leases
+  verbs:
+  - get
+  - watch
+  - update
+`
+	)
+
+	BeforeEach(func() {
+		fakeClient = fakeclient.NewClientBuilder().WithScheme(kubernetes.SeedScheme).Build()
+		mcm = NewBootstrapper(fakeClient, namespace)
+
+		managedResource = &resourcesv1alpha1.ManagedResource{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      managedResourceName,
+				Namespace: namespace,
+			},
+		}
+		managedResourceSecret = &corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "managedresource-" + managedResource.Name,
+				Namespace: namespace,
+			},
+		}
+	})
+
+	Describe("#Deploy", func() {
+		It("should successfully deploy all resources", func() {
+			Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResource), managedResource)).To(BeNotFoundError())
+			Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceSecret), managedResourceSecret)).To(BeNotFoundError())
+
+			Expect(mcm.Deploy(ctx)).To(Succeed())
+
+			Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResource), managedResource)).To(Succeed())
+			Expect(managedResource).To(Equal(&resourcesv1alpha1.ManagedResource{
+				TypeMeta: metav1.TypeMeta{
+					APIVersion: resourcesv1alpha1.SchemeGroupVersion.String(),
+					Kind:       "ManagedResource",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Name:            managedResource.Name,
+					Namespace:       managedResource.Namespace,
+					ResourceVersion: "1",
+					Labels:          map[string]string{"gardener.cloud/role": "seed-system-component"},
+				},
+				Spec: resourcesv1alpha1.ManagedResourceSpec{
+					Class:       pointer.String("seed"),
+					SecretRefs:  []corev1.LocalObjectReference{{Name: managedResourceSecret.Name}},
+					KeepObjects: pointer.Bool(false),
+				},
+			}))
+
+			Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceSecret), managedResourceSecret)).To(Succeed())
+			Expect(managedResourceSecret.Type).To(Equal(corev1.SecretTypeOpaque))
+			Expect(managedResourceSecret.Data).To(HaveLen(1))
+			Expect(string(managedResourceSecret.Data["clusterrole____system_machine-controller-manager-runtime.yaml"])).To(Equal(clusterRoleYAML))
+		})
+	})
+
+	Describe("#Destroy", func() {
+		It("should successfully destroy all resources", func() {
+			Expect(fakeClient.Create(ctx, managedResource)).To(Succeed())
+			Expect(fakeClient.Create(ctx, managedResourceSecret)).To(Succeed())
+
+			Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResource), managedResource)).To(Succeed())
+			Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceSecret), managedResourceSecret)).To(Succeed())
+
+			Expect(mcm.Destroy(ctx)).To(Succeed())
+
+			Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResource), managedResource)).To(BeNotFoundError())
+			Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(managedResourceSecret), managedResourceSecret)).To(BeNotFoundError())
+		})
+	})
+
+	Context("waiting functions", func() {
+		var fakeOps *retryfake.Ops
+
+		BeforeEach(func() {
+			fakeOps = &retryfake.Ops{MaxAttempts: 1}
+			DeferCleanup(test.WithVars(
+				&retry.Until, fakeOps.Until,
+				&retry.UntilTimeout, fakeOps.UntilTimeout,
+			))
+		})
+
+		Describe("#Wait", func() {
+			It("should fail because reading the ManagedResource fails", func() {
+				Expect(mcm.Wait(ctx)).To(MatchError(ContainSubstring("not found")))
+			})
+
+			It("should fail because the ManagedResource doesn't become healthy", func() {
+				fakeOps.MaxAttempts = 2
+
+				Expect(fakeClient.Create(ctx, &resourcesv1alpha1.ManagedResource{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:       managedResourceName,
+						Namespace:  namespace,
+						Generation: 1,
+					},
+					Status: resourcesv1alpha1.ManagedResourceStatus{
+						ObservedGeneration: 1,
+						Conditions: []gardencorev1beta1.Condition{
+							{
+								Type:   resourcesv1alpha1.ResourcesApplied,
+								Status: gardencorev1beta1.ConditionFalse,
+							},
+							{
+								Type:   resourcesv1alpha1.ResourcesHealthy,
+								Status: gardencorev1beta1.ConditionFalse,
+							},
+						},
+					},
+				})).To(Succeed())
+
+				Expect(mcm.Wait(ctx)).To(MatchError(ContainSubstring("is not healthy")))
+			})
+
+			It("should successfully wait for the managed resource to become healthy", func() {
+				fakeOps.MaxAttempts = 2
+
+				Expect(fakeClient.Create(ctx, &resourcesv1alpha1.ManagedResource{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:       managedResourceName,
+						Namespace:  namespace,
+						Generation: 1,
+					},
+					Status: resourcesv1alpha1.ManagedResourceStatus{
+						ObservedGeneration: 1,
+						Conditions: []gardencorev1beta1.Condition{
+							{
+								Type:   resourcesv1alpha1.ResourcesApplied,
+								Status: gardencorev1beta1.ConditionTrue,
+							},
+							{
+								Type:   resourcesv1alpha1.ResourcesHealthy,
+								Status: gardencorev1beta1.ConditionTrue,
+							},
+						},
+					},
+				})).To(Succeed())
+
+				Expect(mcm.Wait(ctx)).To(Succeed())
+			})
+		})
+
+		Describe("#WaitCleanup", func() {
+			It("should fail when the wait for the managed resource deletion times out", func() {
+				fakeOps.MaxAttempts = 2
+
+				Expect(fakeClient.Create(ctx, managedResource)).To(Succeed())
+
+				Expect(mcm.WaitCleanup(ctx)).To(MatchError(ContainSubstring("still exists")))
+			})
+
+			It("should not return an error when it's already removed", func() {
+				Expect(mcm.WaitCleanup(ctx)).To(Succeed())
+			})
+		})
+	})
+})