diff --git a/pkg/controller/tidbcluster/tidb_cluster_control.go b/pkg/controller/tidbcluster/tidb_cluster_control.go index 3682a26cbe1..895a9d45d0e 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_control.go +++ b/pkg/controller/tidbcluster/tidb_cluster_control.go @@ -46,6 +46,7 @@ func NewDefaultTidbClusterControl( metaManager manager.Manager, orphanPodsCleaner member.OrphanPodsCleaner, pvcCleaner member.PVCCleanerInterface, + pvcResizer member.PVCResizerInterface, pumpMemberManager manager.Manager, tiflashMemberManager manager.Manager, ticdcMemberManager manager.Manager, @@ -63,6 +64,7 @@ func NewDefaultTidbClusterControl( metaManager, orphanPodsCleaner, pvcCleaner, + pvcResizer, pumpMemberManager, tiflashMemberManager, ticdcMemberManager, @@ -83,6 +85,7 @@ type defaultTidbClusterControl struct { metaManager manager.Manager orphanPodsCleaner member.OrphanPodsCleaner pvcCleaner member.PVCCleanerInterface + pvcResizer member.PVCResizerInterface pumpMemberManager manager.Manager tiflashMemberManager manager.Manager ticdcMemberManager manager.Manager @@ -247,6 +250,11 @@ func (tcc *defaultTidbClusterControl) updateTidbCluster(tc *v1alpha1.TidbCluster } } + // resize PVC if necessary + if err := tcc.pvcResizer.Resize(tc); err != nil { + return err + } + // syncing the some tidbcluster status attributes // - sync tidbmonitor reference return tcc.tidbClusterStatusManager.Sync(tc) diff --git a/pkg/controller/tidbcluster/tidb_cluster_control_test.go b/pkg/controller/tidbcluster/tidb_cluster_control_test.go index 6c28cb8ec6e..d5d1f70132c 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_control_test.go +++ b/pkg/controller/tidbcluster/tidb_cluster_control_test.go @@ -322,6 +322,7 @@ func newFakeTidbClusterControl() ( discoveryManager := mm.NewFakeDiscoveryManger() podRestarter := mm.NewFakePodRestarter() statusManager := mm.NewFakeTidbClusterStatusManager() + pvcResizer := mm.NewFakePVCResizer() control := NewDefaultTidbClusterControl( tcUpdater, pdMemberManager, @@ -331,6 +332,7 @@ func newFakeTidbClusterControl() ( metaManager, orphanPodCleaner, pvcCleaner, + pvcResizer, pumpMemberManager, tiflashMemberManager, ticdcMemberManager, diff --git a/pkg/controller/tidbcluster/tidb_cluster_controller.go b/pkg/controller/tidbcluster/tidb_cluster_controller.go index 94039e3af64..b7c2f9e28ce 100644 --- a/pkg/controller/tidbcluster/tidb_cluster_controller.go +++ b/pkg/controller/tidbcluster/tidb_cluster_controller.go @@ -89,6 +89,7 @@ func NewController( epsInformer := kubeInformerFactory.Core().V1().Endpoints() pvcInformer := kubeInformerFactory.Core().V1().PersistentVolumeClaims() pvInformer := kubeInformerFactory.Core().V1().PersistentVolumes() + scInformer := kubeInformerFactory.Storage().V1().StorageClasses() podInformer := kubeInformerFactory.Core().V1().Pods() nodeInformer := kubeInformerFactory.Core().V1().Nodes() secretInformer := kubeInformerFactory.Core().V1().Secrets() @@ -195,6 +196,11 @@ func NewController( pvInformer.Lister(), pvControl, ), + mm.NewPVCResizer( + kubeCli, + pvcInformer, + scInformer, + ), mm.NewPumpMemberManager( setControl, svcControl, diff --git a/pkg/manager/member/pvc_resizer.go b/pkg/manager/member/pvc_resizer.go new file mode 100644 index 00000000000..da0dfa805dd --- /dev/null +++ b/pkg/manager/member/pvc_resizer.go @@ -0,0 +1,203 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package member + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/label" + "github.com/pingcap/tidb-operator/pkg/util" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + coreinformers "k8s.io/client-go/informers/core/v1" + storageinformers "k8s.io/client-go/informers/storage/v1" + "k8s.io/client-go/kubernetes" + corelisters "k8s.io/client-go/listers/core/v1" + storagelisters "k8s.io/client-go/listers/storage/v1" + "k8s.io/klog" +) + +// PVCResizerInterface represents the interface of PVC Resizer. +// It patches the PVCs owned by tidb cluster according to the latest +// storage request specified by the user. See +// https://github.com/pingcap/tidb-operator/issues/3004 for more details. +// +// Implementation: +// +// for every unmatched PVC (desiredCapacity != actualCapacity) +// if storageClass does not support VolumeExpansion, skip and continue +// if not patched, patch +// +// We patch all PVCs at the same time. For many cloud storage plugins (e.g. +// AWS-EBS, GCE-PD), they support online file system expansion in latest +// Kubernetes (1.15+). +// +// Limitations: +// +// - Note that the current statfulset implementation does not allow +// `volumeClaimTemplates` to be changed, so new PVCs created by statefulset +// controller will use the old storage request. +// - This is best effort, before statefulset volume resize feature (e.g. +// https://github.com/kubernetes/enhancements/pull/1848) to be implemented. +// - If the feature `ExpandInUsePersistentVolumes` is not enabled or the volume +// plugin does not support, the pod referencing the volume must be deleted and +// recreted after the `FileSystemResizePending` condition becomes true. +// +type PVCResizerInterface interface { + Resize(*v1alpha1.TidbCluster) error +} + +var ( + pdRequirement = util.MustNewRequirement(label.ComponentLabelKey, selection.Equals, []string{label.PDLabelVal}) + tikvRequirement = util.MustNewRequirement(label.ComponentLabelKey, selection.Equals, []string{label.TiKVLabelVal}) + tiflashRequirement = util.MustNewRequirement(label.ComponentLabelKey, selection.Equals, []string{label.TiFlashLabelVal}) + pumpRequirement = util.MustNewRequirement(label.ComponentLabelKey, selection.Equals, []string{label.PumpLabelVal}) +) + +type pvcResizer struct { + kubeCli kubernetes.Interface + pvcLister corelisters.PersistentVolumeClaimLister + scLister storagelisters.StorageClassLister +} + +func (p *pvcResizer) Resize(tc *v1alpha1.TidbCluster) error { + selector, err := label.New().Instance(tc.GetInstanceName()).Selector() + if err != nil { + return err + } + // patch PD PVCs + if tc.Spec.PD != nil { + if storageRequest, ok := tc.Spec.PD.Requests[corev1.ResourceStorage]; ok { + err = p.patchPVCs(tc.GetNamespace(), selector.Add(*pdRequirement), storageRequest, "") + if err != nil { + return err + } + } + } + // patch TiKV PVCs + if tc.Spec.TiKV != nil { + if storageRequest, ok := tc.Spec.TiKV.Requests[corev1.ResourceStorage]; ok { + err = p.patchPVCs(tc.GetNamespace(), selector.Add(*tikvRequirement), storageRequest, "") + if err != nil { + return err + } + } + } + // patch TiFlash PVCs + if tc.Spec.TiFlash != nil { + for i, claim := range tc.Spec.TiFlash.StorageClaims { + if storageRequest, ok := claim.Resources.Requests[corev1.ResourceStorage]; ok { + prefix := fmt.Sprintf("data%d", i) + err = p.patchPVCs(tc.GetNamespace(), selector.Add(*tiflashRequirement), storageRequest, prefix) + if err != nil { + return err + } + } + } + } + // patch Pump PVCs + if tc.Spec.Pump != nil { + if storageRequest, ok := tc.Spec.Pump.Requests[corev1.ResourceStorage]; ok { + err = p.patchPVCs(tc.GetNamespace(), selector.Add(*pumpRequirement), storageRequest, "") + if err != nil { + return err + } + } + } + return nil +} + +func (p *pvcResizer) isVolumeExpansionSupported(storageClassName string) (bool, error) { + sc, err := p.scLister.Get(storageClassName) + if err != nil { + return false, err + } + if sc.AllowVolumeExpansion == nil { + return false, nil + } + return *sc.AllowVolumeExpansion, nil +} + +// patchPVCs patches PVCs filtered by selector and prefix. +func (p *pvcResizer) patchPVCs(ns string, selector labels.Selector, storageRequest resource.Quantity, prefix string) error { + pvcs, err := p.pvcLister.PersistentVolumeClaims(ns).List(selector) + if err != nil { + return err + } + mergePatch, err := json.Marshal(map[string]interface{}{ + "spec": map[string]interface{}{ + "resources": corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: storageRequest, + }, + }, + }, + }) + if err != nil { + return err + } + for _, pvc := range pvcs { + if !strings.HasPrefix(pvc.Name, prefix) { + continue + } + if pvc.Spec.StorageClassName == nil { + klog.Warningf("PVC %s/%s has no storage class, skipped", pvc.Namespace, pvc.Name) + continue + } + volumeExpansionSupported, err := p.isVolumeExpansionSupported(*pvc.Spec.StorageClassName) + fmt.Printf("volumeExpansionSupported: %v, err: %v\n", volumeExpansionSupported, err) + if err != nil { + return err + } + if !volumeExpansionSupported { + klog.Warningf("Storage Class %q used by PVC %s/%s does not support volume expansion, skipped", *pvc.Spec.StorageClassName, pvc.Namespace, pvc.Name) + continue + } + if currentRequest, ok := pvc.Spec.Resources.Requests[corev1.ResourceStorage]; !ok || currentRequest != storageRequest { + _, err = p.kubeCli.CoreV1().PersistentVolumeClaims(pvc.Namespace).Patch(pvc.Name, types.MergePatchType, mergePatch) + if err != nil { + return err + } + klog.V(2).Infof("PVC %s/%s storage request is updated from %s to %s", pvc.Namespace, pvc.Name, currentRequest.String(), storageRequest.String()) + } else { + klog.V(4).Infof("PVC %s/%s storage request is already %s, skipped", pvc.Namespace, pvc.Name, storageRequest.String()) + } + } + return nil +} + +func NewPVCResizer(kubeCli kubernetes.Interface, pvcInformer coreinformers.PersistentVolumeClaimInformer, storageClassInformer storageinformers.StorageClassInformer) PVCResizerInterface { + return &pvcResizer{ + kubeCli: kubeCli, + pvcLister: pvcInformer.Lister(), + scLister: storageClassInformer.Lister(), + } +} + +type fakePVCResizer struct { +} + +func (f *fakePVCResizer) Resize(_ *v1alpha1.TidbCluster) error { + return nil +} + +func NewFakePVCResizer() PVCResizerInterface { + return &fakePVCResizer{} +} diff --git a/pkg/manager/member/pvc_resizer_test.go b/pkg/manager/member/pvc_resizer_test.go new file mode 100644 index 00000000000..e0ac555393b --- /dev/null +++ b/pkg/manager/member/pvc_resizer_test.go @@ -0,0 +1,298 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package member + +import ( + "context" + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/label" + v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/klog" + "k8s.io/utils/pointer" +) + +func init() { + klog.InitFlags(nil) +} + +func newPVCWithStorage(name string, component string, storaegClass, storageRequest string) *v1.PersistentVolumeClaim { + return &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: v1.NamespaceDefault, + Name: name, + Labels: map[string]string{ + label.NameLabelKey: "tidb-cluster", + label.ManagedByLabelKey: label.TiDBOperator, + label.InstanceLabelKey: "tc", + label.ComponentLabelKey: component, + }, + }, + Spec: v1.PersistentVolumeClaimSpec{ + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse(storageRequest), + }, + }, + StorageClassName: pointer.StringPtr(storaegClass), + }, + } +} + +func newStorageClass(name string, volumeExpansion bool) *storagev1.StorageClass { + return &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + AllowVolumeExpansion: pointer.BoolPtr(volumeExpansion), + } +} + +func TestPVCResizer(t *testing.T) { + tests := []struct { + name string + tc *v1alpha1.TidbCluster + sc *storagev1.StorageClass + pvcs []*v1.PersistentVolumeClaim + wantPVCs []*v1.PersistentVolumeClaim + wantErr error + }{ + { + name: "no PVCs", + tc: &v1alpha1.TidbCluster{ + Spec: v1alpha1.TidbClusterSpec{}, + }, + }, + { + name: "resize PD PVCs", + tc: &v1alpha1.TidbCluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: v1.NamespaceDefault, + Name: "tc", + }, + Spec: v1alpha1.TidbClusterSpec{ + PD: &v1alpha1.PDSpec{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("2Gi"), + }, + }, + }, + }, + }, + sc: newStorageClass("sc", true), + pvcs: []*v1.PersistentVolumeClaim{ + newPVCWithStorage("pd-0", label.PDLabelVal, "sc", "1Gi"), + newPVCWithStorage("pd-1", label.PDLabelVal, "sc", "1Gi"), + newPVCWithStorage("pd-2", label.PDLabelVal, "sc", "1Gi"), + }, + wantPVCs: []*v1.PersistentVolumeClaim{ + newPVCWithStorage("pd-0", label.PDLabelVal, "sc", "2Gi"), + newPVCWithStorage("pd-1", label.PDLabelVal, "sc", "2Gi"), + newPVCWithStorage("pd-2", label.PDLabelVal, "sc", "2Gi"), + }, + }, + { + name: "resize TiKV PVCs", + tc: &v1alpha1.TidbCluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: v1.NamespaceDefault, + Name: "tc", + }, + Spec: v1alpha1.TidbClusterSpec{ + TiKV: &v1alpha1.TiKVSpec{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("2Gi"), + }, + }, + }, + }, + }, + sc: newStorageClass("sc", true), + pvcs: []*v1.PersistentVolumeClaim{ + newPVCWithStorage("tikv-0", label.TiKVLabelVal, "sc", "1Gi"), + newPVCWithStorage("tikv-1", label.TiKVLabelVal, "sc", "1Gi"), + newPVCWithStorage("tikv-2", label.TiKVLabelVal, "sc", "1Gi"), + }, + wantPVCs: []*v1.PersistentVolumeClaim{ + newPVCWithStorage("tikv-0", label.TiKVLabelVal, "sc", "2Gi"), + newPVCWithStorage("tikv-1", label.TiKVLabelVal, "sc", "2Gi"), + newPVCWithStorage("tikv-2", label.TiKVLabelVal, "sc", "2Gi"), + }, + }, + { + name: "resize TiFlash PVCs", + tc: &v1alpha1.TidbCluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: v1.NamespaceDefault, + Name: "tc", + }, + Spec: v1alpha1.TidbClusterSpec{ + TiFlash: &v1alpha1.TiFlashSpec{ + StorageClaims: []v1alpha1.StorageClaim{ + { + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("2Gi"), + }, + }, + StorageClassName: pointer.StringPtr("sc"), + }, + { + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("3Gi"), + }, + }, + StorageClassName: pointer.StringPtr("sc"), + }, + }, + }, + }, + }, + sc: newStorageClass("sc", true), + pvcs: []*v1.PersistentVolumeClaim{ + newPVCWithStorage("data0-tc-tiflash-0", label.TiFlashLabelVal, "sc", "1Gi"), + newPVCWithStorage("data1-tc-tiflash-0", label.TiFlashLabelVal, "sc", "1Gi"), + }, + wantPVCs: []*v1.PersistentVolumeClaim{ + newPVCWithStorage("data0-tc-tiflash-0", label.TiFlashLabelVal, "sc", "2Gi"), + newPVCWithStorage("data1-tc-tiflash-0", label.TiFlashLabelVal, "sc", "3Gi"), + }, + }, + { + name: "resize Pump PVCs", + tc: &v1alpha1.TidbCluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: v1.NamespaceDefault, + Name: "tc", + }, + Spec: v1alpha1.TidbClusterSpec{ + Pump: &v1alpha1.PumpSpec{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("2Gi"), + }, + }, + }, + }, + }, + sc: newStorageClass("sc", true), + pvcs: []*v1.PersistentVolumeClaim{ + newPVCWithStorage("pump-0", label.PumpLabelVal, "sc", "1Gi"), + }, + wantPVCs: []*v1.PersistentVolumeClaim{ + newPVCWithStorage("pump-0", label.PumpLabelVal, "sc", "2Gi"), + }, + }, + { + name: "storage class is missing", + tc: &v1alpha1.TidbCluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: v1.NamespaceDefault, + Name: "tc", + }, + Spec: v1alpha1.TidbClusterSpec{ + PD: &v1alpha1.PDSpec{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("2Gi"), + }, + }, + }, + }, + }, + pvcs: []*v1.PersistentVolumeClaim{ + newPVCWithStorage("pd-0", label.PDLabelVal, "sc", "1Gi"), + }, + wantPVCs: []*v1.PersistentVolumeClaim{ + newPVCWithStorage("pd-0", label.PDLabelVal, "sc", "1Gi"), + }, + wantErr: apierrors.NewNotFound(storagev1.Resource("storageclass"), "sc"), + }, + { + name: "storage class does not support volume expansion", + tc: &v1alpha1.TidbCluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: v1.NamespaceDefault, + Name: "tc", + }, + Spec: v1alpha1.TidbClusterSpec{ + PD: &v1alpha1.PDSpec{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("2Gi"), + }, + }, + }, + }, + }, + sc: newStorageClass("sc", false), + pvcs: []*v1.PersistentVolumeClaim{ + newPVCWithStorage("pd-0", label.PDLabelVal, "sc", "1Gi"), + }, + wantPVCs: []*v1.PersistentVolumeClaim{ + newPVCWithStorage("pd-0", label.PDLabelVal, "sc", "1Gi"), + }, + wantErr: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + kubeCli := fake.NewSimpleClientset() + for _, pvc := range tt.pvcs { + kubeCli.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(pvc) + } + if tt.sc != nil { + kubeCli.StorageV1().StorageClasses().Create(tt.sc) + } + + informerFactory := informers.NewSharedInformerFactory(kubeCli, 0) + resizer := NewPVCResizer(kubeCli, informerFactory.Core().V1().PersistentVolumeClaims(), informerFactory.Storage().V1().StorageClasses()) + + informerFactory.Start(ctx.Done()) + informerFactory.WaitForCacheSync(ctx.Done()) + + err := resizer.Resize(tt.tc) + if !reflect.DeepEqual(tt.wantErr, err) { + t.Errorf("want %v, got %v", tt.wantErr, err) + } + + for i, pvc := range tt.pvcs { + wantPVC := tt.wantPVCs[i] + got, err := kubeCli.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(pvc.Name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(wantPVC, got); diff != "" { + t.Errorf("unexpected (-want, +got): %s", diff) + } + } + }) + } +} diff --git a/pkg/util/util.go b/pkg/util/util.go index 8a445789b22..2da02a45ae5 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -27,7 +27,9 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/selection" "k8s.io/apimachinery/pkg/util/sets" ) @@ -269,3 +271,12 @@ func AppendEnvIfPresent(envs []corev1.EnvVar, name string) []corev1.EnvVar { } return envs } + +// MustNewRequirement calls NewRequirement and panics on failure. +func MustNewRequirement(key string, op selection.Operator, vals []string) *labels.Requirement { + r, err := labels.NewRequirement(key, op, vals) + if err != nil { + panic(err) + } + return r +}