scaleOutThreshold
-
-int32
-
-ScaleOutThreshold describe the consecutive threshold for the auto-scaling, -if the consecutive counts of the scale-out result in auto-scaling reach this number, -the auto-scaling would be performed. -If not set, the default value is 3.
-scaleInThreshold
-
-int32
-
-ScaleInThreshold describe the consecutive threshold for the auto-scaling, -if the consecutive counts of the scale-in result in auto-scaling reach this number, -the auto-scaling would be performed. -If not set, the default value is 5.
-externalEndpoint
@@ -15113,6 +15083,44 @@ BasicAutoScalerStatus
+(Appears on: +TidbClusterStatus) +
++
TidbClusterAutoScalerRef indicates to the target auto-scaler ref
+ +| Field | +Description | +
|---|---|
+name
+
+string
+
+ |
++ | +
+namespace
+
+string
+
+ |
++ | +
(Appears on: @@ -15870,6 +15878,18 @@ TidbMonitorRef
auto-scaler,omitempyt
+
+
+TidbClusterAutoScalerRef
+
+
+conditions
diff --git a/manifests/crd.yaml b/manifests/crd.yaml
index 6ef3cfc7cc..16bfbd7749 100644
--- a/manifests/crd.yaml
+++ b/manifests/crd.yaml
@@ -12791,15 +12791,9 @@ spec:
scaleInIntervalSeconds:
format: int32
type: integer
- scaleInThreshold:
- format: int32
- type: integer
scaleOutIntervalSeconds:
format: int32
type: integer
- scaleOutThreshold:
- format: int32
- type: integer
required:
- maxReplicas
type: object
@@ -12846,15 +12840,9 @@ spec:
scaleInIntervalSeconds:
format: int32
type: integer
- scaleInThreshold:
- format: int32
- type: integer
scaleOutIntervalSeconds:
format: int32
type: integer
- scaleOutThreshold:
- format: int32
- type: integer
required:
- maxReplicas
type: object
diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go
index 9d29a1623b..d5cdbf1323 100644
--- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go
+++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go
@@ -136,6 +136,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA
"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TidbCluster": schema_pkg_apis_pingcap_v1alpha1_TidbCluster(ref),
"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TidbClusterAutoScaler": schema_pkg_apis_pingcap_v1alpha1_TidbClusterAutoScaler(ref),
"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TidbClusterAutoScalerList": schema_pkg_apis_pingcap_v1alpha1_TidbClusterAutoScalerList(ref),
+ "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TidbClusterAutoScalerRef": schema_pkg_apis_pingcap_v1alpha1_TidbClusterAutoScalerRef(ref),
"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TidbClusterAutoScalerSpec": schema_pkg_apis_pingcap_v1alpha1_TidbClusterAutoScalerSpec(ref),
"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TidbClusterAutoSclaerStatus": schema_pkg_apis_pingcap_v1alpha1_TidbClusterAutoSclaerStatus(ref),
"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TidbClusterList": schema_pkg_apis_pingcap_v1alpha1_TidbClusterList(ref),
@@ -916,20 +917,6 @@ func schema_pkg_apis_pingcap_v1alpha1_BasicAutoScalerSpec(ref common.ReferenceCa
Format: "",
},
},
- "scaleOutThreshold": {
- SchemaProps: spec.SchemaProps{
- Description: "ScaleOutThreshold describe the consecutive threshold for the auto-scaling, if the consecutive counts of the scale-out result in auto-scaling reach this number, the auto-scaling would be performed. If not set, the default value is 3.",
- Type: []string{"integer"},
- Format: "int32",
- },
- },
- "scaleInThreshold": {
- SchemaProps: spec.SchemaProps{
- Description: "ScaleInThreshold describe the consecutive threshold for the auto-scaling, if the consecutive counts of the scale-in result in auto-scaling reach this number, the auto-scaling would be performed. If not set, the default value is 5.",
- Type: []string{"integer"},
- Format: "int32",
- },
- },
"externalEndpoint": {
SchemaProps: spec.SchemaProps{
Description: "ExternalEndpoint makes the auto-scaler controller able to query the external service to fetch the recommended replicas for TiKV/TiDB",
@@ -8711,20 +8698,6 @@ func schema_pkg_apis_pingcap_v1alpha1_TidbAutoScalerSpec(ref common.ReferenceCal
Format: "",
},
},
- "scaleOutThreshold": {
- SchemaProps: spec.SchemaProps{
- Description: "ScaleOutThreshold describe the consecutive threshold for the auto-scaling, if the consecutive counts of the scale-out result in auto-scaling reach this number, the auto-scaling would be performed. If not set, the default value is 3.",
- Type: []string{"integer"},
- Format: "int32",
- },
- },
- "scaleInThreshold": {
- SchemaProps: spec.SchemaProps{
- Description: "ScaleInThreshold describe the consecutive threshold for the auto-scaling, if the consecutive counts of the scale-in result in auto-scaling reach this number, the auto-scaling would be performed. If not set, the default value is 5.",
- Type: []string{"integer"},
- Format: "int32",
- },
- },
"externalEndpoint": {
SchemaProps: spec.SchemaProps{
Description: "ExternalEndpoint makes the auto-scaler controller able to query the external service to fetch the recommended replicas for TiKV/TiDB",
@@ -8916,6 +8889,32 @@ func schema_pkg_apis_pingcap_v1alpha1_TidbClusterAutoScalerList(ref common.Refer
}
}
+func schema_pkg_apis_pingcap_v1alpha1_TidbClusterAutoScalerRef(ref common.ReferenceCallback) common.OpenAPIDefinition {
+ return common.OpenAPIDefinition{
+ Schema: spec.Schema{
+ SchemaProps: spec.SchemaProps{
+ Description: "TidbClusterAutoScalerRef indicates to the target auto-scaler ref",
+ Type: []string{"object"},
+ Properties: map[string]spec.Schema{
+ "name": {
+ SchemaProps: spec.SchemaProps{
+ Type: []string{"string"},
+ Format: "",
+ },
+ },
+ "namespace": {
+ SchemaProps: spec.SchemaProps{
+ Type: []string{"string"},
+ Format: "",
+ },
+ },
+ },
+ Required: []string{"name", "namespace"},
+ },
+ },
+ }
+}
+
func schema_pkg_apis_pingcap_v1alpha1_TidbClusterAutoScalerSpec(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
@@ -9832,20 +9831,6 @@ func schema_pkg_apis_pingcap_v1alpha1_TikvAutoScalerSpec(ref common.ReferenceCal
Format: "",
},
},
- "scaleOutThreshold": {
- SchemaProps: spec.SchemaProps{
- Description: "ScaleOutThreshold describe the consecutive threshold for the auto-scaling, if the consecutive counts of the scale-out result in auto-scaling reach this number, the auto-scaling would be performed. If not set, the default value is 3.",
- Type: []string{"integer"},
- Format: "int32",
- },
- },
- "scaleInThreshold": {
- SchemaProps: spec.SchemaProps{
- Description: "ScaleInThreshold describe the consecutive threshold for the auto-scaling, if the consecutive counts of the scale-in result in auto-scaling reach this number, the auto-scaling would be performed. If not set, the default value is 5.",
- Type: []string{"integer"},
- Format: "int32",
- },
- },
"externalEndpoint": {
SchemaProps: spec.SchemaProps{
Description: "ExternalEndpoint makes the auto-scaler controller able to query the external service to fetch the recommended replicas for TiKV/TiDB",
diff --git a/pkg/apis/pingcap/v1alpha1/tidbclusterautoscaler_types.go b/pkg/apis/pingcap/v1alpha1/tidbclusterautoscaler_types.go
index 2ff6d4afbe..fe5f1d6b56 100644
--- a/pkg/apis/pingcap/v1alpha1/tidbclusterautoscaler_types.go
+++ b/pkg/apis/pingcap/v1alpha1/tidbclusterautoscaler_types.go
@@ -135,21 +135,6 @@ type BasicAutoScalerSpec struct {
// MetricsTimeDuration describe the Time duration to be queried in the Prometheus
// +optional
MetricsTimeDuration *string `json:"metricsTimeDuration,omitempty"`
-
- // ScaleOutThreshold describe the consecutive threshold for the auto-scaling,
- // if the consecutive counts of the scale-out result in auto-scaling reach this number,
- // the auto-scaling would be performed.
- // If not set, the default value is 3.
- // +optional
- ScaleOutThreshold *int32 `json:"scaleOutThreshold,omitempty"`
-
- // ScaleInThreshold describe the consecutive threshold for the auto-scaling,
- // if the consecutive counts of the scale-in result in auto-scaling reach this number,
- // the auto-scaling would be performed.
- // If not set, the default value is 5.
- // +optional
- ScaleInThreshold *int32 `json:"scaleInThreshold,omitempty"`
-
// ExternalEndpoint makes the auto-scaler controller able to query the external service
// to fetch the recommended replicas for TiKV/TiDB
// +optional
@@ -246,3 +231,10 @@ type SecretRef struct {
Name string `json:"name"`
Namespace string `json:"namespace"`
}
+
+// +k8s:openapi-gen=true
+// TidbClusterAutoScalerRef indicates to the target auto-scaler ref
+type TidbClusterAutoScalerRef struct {
+ Name string `json:"name"`
+ Namespace string `json:"namespace"`
+}
diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go
index b4eeca181a..13cbfcac9d 100644
--- a/pkg/apis/pingcap/v1alpha1/types.go
+++ b/pkg/apis/pingcap/v1alpha1/types.go
@@ -225,14 +225,15 @@ type TidbClusterSpec struct {
// TidbClusterStatus represents the current status of a tidb cluster.
type TidbClusterStatus struct {
- ClusterID string `json:"clusterID,omitempty"`
- PD PDStatus `json:"pd,omitempty"`
- TiKV TiKVStatus `json:"tikv,omitempty"`
- TiDB TiDBStatus `json:"tidb,omitempty"`
- Pump PumpStatus `josn:"pump,omitempty"`
- TiFlash TiFlashStatus `json:"tiflash,omitempty"`
- TiCDC TiCDCStatus `json:"ticdc,omitempty"`
- Monitor *TidbMonitorRef `json:"monitor,omitempty"`
+ ClusterID string `json:"clusterID,omitempty"`
+ PD PDStatus `json:"pd,omitempty"`
+ TiKV TiKVStatus `json:"tikv,omitempty"`
+ TiDB TiDBStatus `json:"tidb,omitempty"`
+ Pump PumpStatus `josn:"pump,omitempty"`
+ TiFlash TiFlashStatus `json:"tiflash,omitempty"`
+ TiCDC TiCDCStatus `json:"ticdc,omitempty"`
+ Monitor *TidbMonitorRef `json:"monitor,omitempty"`
+ AutoScaler *TidbClusterAutoScalerRef `json:"auto-scaler,omitempyt"`
// Represents the latest available observations of a tidb cluster's state.
// +optional
Conditions []TidbClusterCondition `json:"conditions,omitempty"`
diff --git a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go
index afe590ba91..05687a05dc 100644
--- a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go
+++ b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go
@@ -380,16 +380,6 @@ func (in *BasicAutoScalerSpec) DeepCopyInto(out *BasicAutoScalerSpec) {
*out = new(string)
**out = **in
}
- if in.ScaleOutThreshold != nil {
- in, out := &in.ScaleOutThreshold, &out.ScaleOutThreshold
- *out = new(int32)
- **out = **in
- }
- if in.ScaleInThreshold != nil {
- in, out := &in.ScaleInThreshold, &out.ScaleInThreshold
- *out = new(int32)
- **out = **in
- }
if in.ExternalEndpoint != nil {
in, out := &in.ExternalEndpoint, &out.ExternalEndpoint
*out = new(ExternalEndpoint)
@@ -6651,6 +6641,22 @@ func (in *TidbClusterAutoScalerList) DeepCopyObject() runtime.Object {
return nil
}
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TidbClusterAutoScalerRef) DeepCopyInto(out *TidbClusterAutoScalerRef) {
+ *out = *in
+ return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TidbClusterAutoScalerRef.
+func (in *TidbClusterAutoScalerRef) DeepCopy() *TidbClusterAutoScalerRef {
+ if in == nil {
+ return nil
+ }
+ out := new(TidbClusterAutoScalerRef)
+ in.DeepCopyInto(out)
+ return out
+}
+
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TidbClusterAutoScalerSpec) DeepCopyInto(out *TidbClusterAutoScalerSpec) {
*out = *in
@@ -6901,6 +6907,11 @@ func (in *TidbClusterStatus) DeepCopyInto(out *TidbClusterStatus) {
*out = new(TidbMonitorRef)
**out = **in
}
+ if in.AutoScaler != nil {
+ in, out := &in.AutoScaler, &out.AutoScaler
+ *out = new(TidbClusterAutoScalerRef)
+ **out = **in
+ }
if in.Conditions != nil {
in, out := &in.Conditions, &out.Conditions
*out = make([]TidbClusterCondition, len(*in))
diff --git a/pkg/autoscaler/autoscaler/autoscaler_manager.go b/pkg/autoscaler/autoscaler/autoscaler_manager.go
index 67bc83bf4f..26d50906e0 100644
--- a/pkg/autoscaler/autoscaler/autoscaler_manager.go
+++ b/pkg/autoscaler/autoscaler/autoscaler_manager.go
@@ -14,6 +14,7 @@
package autoscaler
import (
+ "encoding/json"
"fmt"
"strconv"
"strings"
@@ -28,6 +29,7 @@ import (
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/types"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
kubeinformers "k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
@@ -79,12 +81,20 @@ func (am *autoScalerManager) Sync(tac *v1alpha1.TidbClusterAutoScaler) error {
if err != nil {
if errors.IsNotFound(err) {
// Target TidbCluster Ref is deleted, empty the auto-scaling status
- resetAutoScalingAnn(tac)
return nil
}
return err
}
- checkAndUpdateTacAnn(tac)
+ permitted, err := am.checkAutoScalerRef(tc, tac)
+ if err != nil {
+ klog.Error(err)
+ return err
+ }
+ if !permitted {
+ klog.Infof("tac[%s/%s]'s auto-scaling is no permitted", tac.Namespace, tac.Name)
+ return nil
+ }
+
oldTc := tc.DeepCopy()
if err := am.syncAutoScaling(tc, tac); err != nil {
return err
@@ -209,3 +219,45 @@ func (am *autoScalerManager) updateTidbClusterAutoScaler(tac *v1alpha1.TidbClust
return updateErr
})
}
+
+// checkAutoScalerRef will first check whether the target tidbcluster's auto-scaler reference have been occupied.
+// If it has been, and the reference scaler is the current auto-scaler itself, the auto-scaler would be permitted,
+// otherwise the auto-scaling would be forbidden.
+// If the target tidbcluster's auto-scaler reference is empty, then the auto-scaler will try to patch itself to the
+// references, and if the patching is success, the auto-scaling would discard the current syncing and wait for the next
+// syncing round.
+func (am *autoScalerManager) checkAutoScalerRef(tc *v1alpha1.TidbCluster, tac *v1alpha1.TidbClusterAutoScaler) (bool, error) {
+ if tc.Status.AutoScaler != nil {
+ if tc.Status.AutoScaler.Name == tac.Name && tc.Status.AutoScaler.Namespace == tac.Namespace {
+ return true, nil
+ }
+ msg := fmt.Sprintf("tac[%s/%s]'s target tc[%s/%s] already controlled by another auto-scaler", tac.Namespace, tac.Name, tc.Namespace, tc.Name)
+ klog.Info(msg)
+ return false, nil
+ }
+ klog.Infof("tac[%s/%s]'s tc[%s/%s] start to occupy the auto-scaler ref", tac.Namespace, tac.Name, tc.Namespace, tc.Name)
+ err := am.patchAutoScalerRef(tc, tac)
+ return true, err
+}
+
+func (am *autoScalerManager) patchAutoScalerRef(tc *v1alpha1.TidbCluster, tac *v1alpha1.TidbClusterAutoScaler) error {
+ mergePatch, err := json.Marshal(map[string]interface{}{
+ "status": map[string]interface{}{
+ "auto-scaler": map[string]interface{}{
+ "name": tac.Name,
+ "namespace": tac.Namespace,
+ },
+ },
+ })
+ if err != nil {
+ klog.Error(err)
+ return err
+ }
+ _, err = am.cli.PingcapV1alpha1().TidbClusters(tc.Namespace).Patch(tc.Name, types.MergePatchType, mergePatch)
+ if err != nil {
+ klog.Error(err)
+ return err
+ }
+ msg := fmt.Sprintf("tac[%s/%s] patch itself to tc[%s/%s] auto-scaler ref success, do auto-scaling in next round", tac.Namespace, tac.Name, tc.Namespace, tc.Name)
+ return controller.RequeueErrorf(msg)
+}
diff --git a/pkg/autoscaler/autoscaler/tikv_autoscaler.go b/pkg/autoscaler/autoscaler/tikv_autoscaler.go
index 28d7ced390..9c8157710e 100644
--- a/pkg/autoscaler/autoscaler/tikv_autoscaler.go
+++ b/pkg/autoscaler/autoscaler/tikv_autoscaler.go
@@ -17,10 +17,12 @@ import (
"fmt"
"time"
+ "github.com/jonboulle/clockwork"
"github.com/pingcap/advanced-statefulset/client/apis/apps/v1/helper"
"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
"github.com/pingcap/tidb-operator/pkg/autoscaler/autoscaler/calculate"
"github.com/pingcap/tidb-operator/pkg/autoscaler/autoscaler/query"
+ "github.com/pingcap/tidb-operator/pkg/controller"
"github.com/pingcap/tidb-operator/pkg/label"
operatorUtils "github.com/pingcap/tidb-operator/pkg/util"
promClient "github.com/prometheus/client_golang/api"
@@ -87,7 +89,7 @@ func syncTiKVAfterCalculated(tc *v1alpha1.TidbCluster, tac *v1alpha1.TidbCluster
}
}
- ableToScale, err := checkStsAutoScaling(tac, *tac.Spec.TiKV.ReadyToScaleThresholdSeconds, *intervalSeconds, v1alpha1.TiKVMemberType)
+ ableToScale, err := checkTiKVStsAutoScaling(tac, *tac.Spec.TiKV.ReadyToScaleThresholdSeconds, *intervalSeconds)
if err != nil {
return err
}
@@ -166,3 +168,33 @@ func calculateTikvMetrics(tac *v1alpha1.TidbClusterAutoScaler, sts *appsv1.State
return -1, fmt.Errorf(calculate.InvalidTacMetricConfigureMsg, tac.Namespace, tac.Name)
}
}
+
+func checkTiKVStsAutoScaling(tac *v1alpha1.TidbClusterAutoScaler, thresholdSeconds, intervalSeconds int32) (bool, error) {
+ realClock := clockwork.NewRealClock()
+ if tac.Annotations == nil {
+ tac.Annotations = map[string]string{}
+ }
+ // 3*controller.ResyncDuration is maximum time allowed before reset phase status
+ ableToScale, err := checkLastSyncingTimestamp(tac, 3*controller.ResyncDuration, realClock)
+ if err != nil {
+ return false, err
+ }
+ if !ableToScale {
+ return false, nil
+ }
+ ableToScale, err = checkStsReadyAutoScalingTimestamp(tac, thresholdSeconds, realClock)
+ if err != nil {
+ return false, err
+ }
+ if !ableToScale {
+ return false, nil
+ }
+ ableToScale, err = checkStsAutoScalingInterval(tac, intervalSeconds, v1alpha1.TiKVMemberType)
+ if err != nil {
+ return false, err
+ }
+ if !ableToScale {
+ return false, nil
+ }
+ return true, nil
+}
diff --git a/pkg/autoscaler/autoscaler/util.go b/pkg/autoscaler/autoscaler/util.go
index 6a17dd5185..1ebc003551 100644
--- a/pkg/autoscaler/autoscaler/util.go
+++ b/pkg/autoscaler/autoscaler/util.go
@@ -20,7 +20,6 @@ import (
"github.com/jonboulle/clockwork"
"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
- "github.com/pingcap/tidb-operator/pkg/controller"
"github.com/pingcap/tidb-operator/pkg/label"
operatorUtils "github.com/pingcap/tidb-operator/pkg/util"
appsv1 "k8s.io/api/apps/v1"
@@ -29,14 +28,6 @@ import (
"k8s.io/utils/pointer"
)
-const (
- annScaleOutSuffix = "tidb.pingcap.com/consecutive-scale-out-count"
- annScaleInSuffix = "tidb.pingcap.com/consecutive-scale-in-count"
-
- invalidMemberTypeErrorMsg = "tac[%s/%s] invalid set MemberType:%s"
- invalidTacAnnotationErrorMsg = "tac[%s/%s] annotation invalid set,err:%v"
-)
-
var defaultMetricSpec = autoscalingv2beta2.MetricSpec{
Type: autoscalingv2beta2.ResourceMetricSourceType,
Resource: &autoscalingv2beta2.ResourceMetricSource{
@@ -59,36 +50,6 @@ func checkStsAutoScalingPrerequisites(set *appsv1.StatefulSet) bool {
return true
}
-func checkStsAutoScaling(tac *v1alpha1.TidbClusterAutoScaler, thresholdSeconds, intervalSeconds int32, memberType v1alpha1.MemberType) (bool, error) {
- realClock := clockwork.NewRealClock()
- if tac.Annotations == nil {
- tac.Annotations = map[string]string{}
- }
- // 3*controller.ResyncDuration is maximum time allowed before reset phase status
- ableToScale, err := checkLastSyncingTimestamp(tac, 3*controller.ResyncDuration, realClock)
- if err != nil {
- return false, err
- }
- if !ableToScale {
- return false, nil
- }
- ableToScale, err = checkStsReadyAutoScalingTimestamp(tac, thresholdSeconds, realClock)
- if err != nil {
- return false, err
- }
- if !ableToScale {
- return false, nil
- }
- ableToScale, err = checkStsAutoScalingInterval(tac, intervalSeconds, memberType)
- if err != nil {
- return false, err
- }
- if !ableToScale {
- return false, nil
- }
- return true, nil
-}
-
// checkLastSyncingTimestamp reset TiKV phase if last auto scaling timestamp is longer than thresholdSec
func checkLastSyncingTimestamp(tac *v1alpha1.TidbClusterAutoScaler, thresholdSec time.Duration, clock clockwork.Clock) (bool, error) {
if tac.Annotations == nil {
@@ -200,6 +161,9 @@ func limitTargetReplicas(targetReplicas int32, tac *v1alpha1.TidbClusterAutoScal
// If the Metrics not set, the default metric will be set to 80% average CPU utilization.
// defaultTAC would default the omitted value
func defaultTAC(tac *v1alpha1.TidbClusterAutoScaler) {
+ if tac.Annotations == nil {
+ tac.Annotations = map[string]string{}
+ }
if tac.Spec.TiKV != nil {
if tac.Spec.TiKV.MinReplicas == nil {
tac.Spec.TiKV.MinReplicas = pointer.Int32Ptr(1)
@@ -251,29 +215,6 @@ func defaultTAC(tac *v1alpha1.TidbClusterAutoScaler) {
}
}
-func resetAutoScalingAnn(tac *v1alpha1.TidbClusterAutoScaler) {
- tac.Annotations[label.AnnAutoScalingTargetNamespace] = tac.Spec.Cluster.Namespace
- tac.Annotations[label.AnnAutoScalingTargetName] = tac.Spec.Cluster.Name
-}
-
-// checkAndUpdateTacRef would compare the target tidbcluster ref stored in the annotations
-// and in the Spec. It not equal, the previous stored status would be empty and the stored Ref
-// would be updated.
-func checkAndUpdateTacAnn(tac *v1alpha1.TidbClusterAutoScaler) {
- if tac.Annotations == nil {
- tac.Annotations = map[string]string{}
- resetAutoScalingAnn(tac)
- return
- }
- name := tac.Annotations[label.AnnAutoScalingTargetName]
- namespace := tac.Annotations[label.AnnAutoScalingTargetNamespace]
- if name == tac.Spec.Cluster.Name && namespace == tac.Spec.Cluster.Namespace {
- return
- }
- // If not satisfied, reset tac Ann
- resetAutoScalingAnn(tac)
-}
-
func genMetricsEndpoint(tac *v1alpha1.TidbClusterAutoScaler) (string, error) {
if tac.Spec.MetricsUrl == nil && tac.Spec.Monitor == nil {
return "", fmt.Errorf("tac[%s/%s] metrics url or monitor should be defined explicitly", tac.Namespace, tac.Name)
diff --git a/pkg/autoscaler/autoscaler/util_test.go b/pkg/autoscaler/autoscaler/util_test.go
index 5b5fd15b6c..66ca73c61e 100644
--- a/pkg/autoscaler/autoscaler/util_test.go
+++ b/pkg/autoscaler/autoscaler/util_test.go
@@ -383,64 +383,6 @@ func TestDefaultTac(t *testing.T) {
}
-func TestCheckAndUpdateTacAnn(t *testing.T) {
- g := NewGomegaWithT(t)
- tests := []struct {
- name string
- haveScaling bool
- targetNamespace string
- targetName string
- markedNamespace string
- markedName string
- }{
- {
- name: "first syncing",
- haveScaling: false,
- markedName: "",
- markedNamespace: "",
- targetName: "foo",
- targetNamespace: "bar",
- },
- {
- name: "second syncing",
- haveScaling: true,
- markedName: "foo",
- markedNamespace: "bar",
- targetName: "foo",
- targetNamespace: "bar",
- },
- {
- name: "change target",
- haveScaling: true,
- markedName: "foo",
- markedNamespace: "bar",
- targetName: "foo2",
- targetNamespace: "bar2",
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- tac := newTidbClusterAutoScaler()
- tac.Annotations = nil
- tac.Spec.Cluster.Name = tt.targetName
- tac.Spec.Cluster.Namespace = tt.targetNamespace
- if tt.haveScaling {
- tac.Annotations = map[string]string{}
- tac.Annotations[label.AnnAutoScalingTargetNamespace] = tt.targetNamespace
- tac.Annotations[label.AnnAutoScalingTargetName] = tt.targetName
- }
- checkAndUpdateTacAnn(tac)
- v, ok := tac.Annotations[label.AnnAutoScalingTargetNamespace]
- g.Expect(ok).Should(Equal(ok))
- g.Expect(v).Should(Equal(tt.targetNamespace))
- v, ok = tac.Annotations[label.AnnAutoScalingTargetName]
- g.Expect(ok).Should(Equal(ok))
- g.Expect(v).Should(Equal(tt.targetName))
- })
- }
-}
-
func TestGenMetricsEndpoint(t *testing.T) {
g := NewGomegaWithT(t)
tac := newTidbClusterAutoScaler()
@@ -480,10 +422,6 @@ func newTidbClusterAutoScaler() *v1alpha1.TidbClusterAutoScaler {
}
tac.Spec.TiKV = &v1alpha1.TikvAutoScalerSpec{}
tac.Spec.TiDB = &v1alpha1.TidbAutoScalerSpec{}
- tac.Spec.TiKV.ScaleOutThreshold = pointer.Int32Ptr(2)
- tac.Spec.TiKV.ScaleInThreshold = pointer.Int32Ptr(2)
- tac.Spec.TiDB.ScaleOutThreshold = pointer.Int32Ptr(2)
- tac.Spec.TiDB.ScaleInThreshold = pointer.Int32Ptr(2)
return tac
}
diff --git a/pkg/controller/autoscaler/tidbcluster_autoscaler_controller.go b/pkg/controller/autoscaler/tidbcluster_autoscaler_controller.go
index e1539a3ceb..f4cf6db5f5 100644
--- a/pkg/controller/autoscaler/tidbcluster_autoscaler_controller.go
+++ b/pkg/controller/autoscaler/tidbcluster_autoscaler_controller.go
@@ -82,12 +82,12 @@ func (tac *Controller) Run(workers int, stopCh <-chan struct{}) {
}
func (tac *Controller) worker() {
- for tac.processNestWorkItem() {
+ for tac.processNextWorkItem() {
// revive:disable:empty-block
}
}
-func (tac *Controller) processNestWorkItem() bool {
+func (tac *Controller) processNextWorkItem() bool {
key, quit := tac.queue.Get()
if quit {
return false
diff --git a/pkg/controller/tidbcluster/tidb_cluster_controller.go b/pkg/controller/tidbcluster/tidb_cluster_controller.go
index e8d8cce238..d1b13e68b3 100644
--- a/pkg/controller/tidbcluster/tidb_cluster_controller.go
+++ b/pkg/controller/tidbcluster/tidb_cluster_controller.go
@@ -92,6 +92,7 @@ func NewController(
podInformer := kubeInformerFactory.Core().V1().Pods()
nodeInformer := kubeInformerFactory.Core().V1().Nodes()
secretInformer := kubeInformerFactory.Core().V1().Secrets()
+ scalerInformer := informerFactory.Pingcap().V1alpha1().TidbClusterAutoScalers()
tcControl := controller.NewRealTidbClusterControl(cli, tcInformer.Lister(), recorder)
pdControl := pdapi.NewDefaultPDControl(kubeCli)
@@ -227,7 +228,7 @@ func NewController(
setControl,
),
mm.NewTidbDiscoveryManager(typedControl),
- mm.NewTidbClusterStatusManager(kubeCli, cli),
+ mm.NewTidbClusterStatusManager(kubeCli, cli, scalerInformer.Lister()),
podRestarter,
&tidbClusterConditionUpdater{},
recorder,
diff --git a/pkg/label/label.go b/pkg/label/label.go
index 697c6874cf..248ba11b35 100644
--- a/pkg/label/label.go
+++ b/pkg/label/label.go
@@ -117,18 +117,6 @@ const (
// AnnLastSyncingTimestamp records last sync timestamp
AnnLastSyncingTimestamp = "tidb.pingcap.com/last-syncing-timestamp"
- // AnnTiDBConsecutiveScaleOutCount describes the least consecutive count to scale-out for tidb
- AnnTiDBConsecutiveScaleOutCount = "tidb.tidb.pingcap.com/consecutive-scale-out-count"
- // AnnTiDBConsecutiveScaleInCount describes the least consecutive count to scale-in for tidb
- AnnTiDBConsecutiveScaleInCount = "tidb.tidb.pingcap.com/consecutive-scale-in-count"
- // AnnTiKVConsecutiveScaleOutCount describes the least consecutive count to scale-out for tikv
- AnnTiKVConsecutiveScaleOutCount = "tikv.tidb.pingcap.com/consecutive-scale-out-count"
- // AnnTiKVConsecutiveScaleInCount describes the least consecutive count to scale-in for tikv
- AnnTiKVConsecutiveScaleInCount = "tikv.tidb.pingcap.com/consecutive-scale-in-count"
- // AnnAutoScalingTargetName describes the target TidbCluster Ref Name for the TidbCluserAutoScaler
- AnnAutoScalingTargetName = "auto-scaling.tidb.pingcap.com/target-name"
- // AnnAutoScalingTargetNamespace describes the target TidbCluster Ref Namespace for the TidbCluserAutoScaler
- AnnAutoScalingTargetNamespace = "auto-scaling.tidb.pingcap.com/target-namespace"
// AnnTiKVAutoScalingOutOrdinals describe the tikv pods' ordinal list which is created by auto-scaling out
AnnTiKVAutoScalingOutOrdinals = "tikv.tidb.pingcap.com/scale-out-ordinals"
// AnnTiDBAutoScalingOutOrdinals describe the tidb pods' ordinal list which is created by auto-scaling out
diff --git a/pkg/manager/member/tidbcluster_status_manager.go b/pkg/manager/member/tidbcluster_status_manager.go
index d59a6c195b..15ddfe535d 100644
--- a/pkg/manager/member/tidbcluster_status_manager.go
+++ b/pkg/manager/member/tidbcluster_status_manager.go
@@ -19,6 +19,7 @@ import (
"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
+ listers "github.com/pingcap/tidb-operator/pkg/client/listers/pingcap/v1alpha1"
"github.com/pingcap/tidb-operator/pkg/pdapi"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -35,14 +36,16 @@ const (
)
type TidbClusterStatusManager struct {
- cli versioned.Interface
- pdControl pdapi.PDControlInterface
+ cli versioned.Interface
+ pdControl pdapi.PDControlInterface
+ scalerLister listers.TidbClusterAutoScalerLister
}
-func NewTidbClusterStatusManager(kubeCli kubernetes.Interface, cli versioned.Interface) *TidbClusterStatusManager {
+func NewTidbClusterStatusManager(kubeCli kubernetes.Interface, cli versioned.Interface, scalerLister listers.TidbClusterAutoScalerLister) *TidbClusterStatusManager {
return &TidbClusterStatusManager{
- cli: cli,
- pdControl: pdapi.NewDefaultPDControl(kubeCli),
+ cli: cli,
+ pdControl: pdapi.NewDefaultPDControl(kubeCli),
+ scalerLister: scalerLister,
}
}
@@ -55,7 +58,11 @@ func (tcsm *TidbClusterStatusManager) syncTidbMonitorRefAndKey(tc *v1alpha1.Tidb
if err != nil {
return err
}
- return tcsm.syncDashboardMetricStorage(tc, tm)
+ err = tcsm.syncDashboardMetricStorage(tc, tm)
+ if err != nil {
+ return err
+ }
+ return tcsm.syncAutoScalerRef(tc)
}
func (tcsm *TidbClusterStatusManager) syncTidbMonitorRef(tc *v1alpha1.TidbCluster) (*v1alpha1.TidbMonitor, error) {
@@ -120,6 +127,38 @@ func (tcsm *TidbClusterStatusManager) syncDashboardMetricStorage(tc *v1alpha1.Ti
return nil
}
+func (tcsm *TidbClusterStatusManager) syncAutoScalerRef(tc *v1alpha1.TidbCluster) error {
+ if tc.Status.AutoScaler == nil {
+ klog.V(4).Infof("tc[%s/%s] autoscaler is empty", tc.Namespace, tc.Name)
+ return nil
+ }
+ tacNamespace := tc.Status.AutoScaler.Namespace
+ tacName := tc.Status.AutoScaler.Name
+ tac, err := tcsm.scalerLister.TidbClusterAutoScalers(tacNamespace).Get(tacName)
+ if err != nil {
+ if errors.IsNotFound(err) {
+ klog.Infof("tc[%s/%s] failed to find tac[%s/%s]", tc.Namespace, tc.Name, tacNamespace, tacName)
+ tc.Status.AutoScaler = nil
+ err = nil
+ }
+ return err
+ }
+ if tac.Spec.Cluster.Name != tc.Name {
+ klog.Infof("tc[%s/%s]'s target tac[%s/%s]'s cluster have been changed", tc.Namespace, tc.Name, tac.Namespace, tac.Name)
+ tc.Status.AutoScaler = nil
+ return nil
+ }
+ if len(tac.Spec.Cluster.Namespace) < 1 {
+ return nil
+ }
+ if tac.Spec.Cluster.Namespace != tc.Namespace {
+ klog.Infof("tc[%s/%s]'s target tac[%s/%s]'s cluster namespace have been changed", tc.Namespace, tc.Name, tac.Namespace, tac.Name)
+ tc.Status.AutoScaler = nil
+ return nil
+ }
+ return nil
+}
+
func syncComponent(exist bool, tm *v1alpha1.TidbMonitor, componentName string, port int, etcdClient pdapi.PDEtcdClient) error {
key := buildComponentKey(componentName)
if exist {
diff --git a/pkg/manager/member/tidbcluster_status_manager_test.go b/pkg/manager/member/tidbcluster_status_manager_test.go
new file mode 100644
index 0000000000..8d3dc329ec
--- /dev/null
+++ b/pkg/manager/member/tidbcluster_status_manager_test.go
@@ -0,0 +1,124 @@
+// Copyright 2020 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package member
+
+import (
+ "testing"
+
+ . "github.com/onsi/gomega"
+ "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
+ "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned/fake"
+ informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions"
+ "k8s.io/client-go/kubernetes"
+ kubefake "k8s.io/client-go/kubernetes/fake"
+ "k8s.io/client-go/tools/cache"
+)
+
+func TestSyncAutoScalerRef(t *testing.T) {
+ g := NewGomegaWithT(t)
+ testcases := []struct {
+ name string
+ haveRef bool
+ autoScalerExisted bool
+ correctRef bool
+ expectedStatusRef *v1alpha1.TidbClusterAutoScalerRef
+ }{
+ {
+ name: "empty Reference",
+ haveRef: false,
+ autoScalerExisted: false,
+ correctRef: false,
+ expectedStatusRef: nil,
+ },
+ {
+ name: "normal",
+ haveRef: true,
+ autoScalerExisted: true,
+ correctRef: true,
+ expectedStatusRef: &v1alpha1.TidbClusterAutoScalerRef{
+ Name: "auto-scaler",
+ Namespace: "default",
+ },
+ },
+ {
+ name: "target auto-scaler not existed",
+ haveRef: true,
+ autoScalerExisted: false,
+ correctRef: false,
+ expectedStatusRef: nil,
+ },
+ {
+ name: "target auto-scaler have changed the cluster target",
+ haveRef: true,
+ autoScalerExisted: true,
+ correctRef: false,
+ expectedStatusRef: nil,
+ },
+ }
+ for _, testcase := range testcases {
+ t.Run(testcase.name, func(t *testing.T) {
+ tsm, _, _, scalerInder := newFakeTidbClusterStatusManager()
+ tc := newTidbCluster()
+ tc.Namespace = "default"
+ tac := newTidbClusterAutoScaler(tc)
+ if testcase.haveRef {
+ tc.Status.AutoScaler = &v1alpha1.TidbClusterAutoScalerRef{
+ Name: tac.Name,
+ Namespace: tac.Namespace,
+ }
+ } else {
+ tc.Status.AutoScaler = nil
+ }
+ if !testcase.correctRef {
+ tac.Spec.Cluster.Name = "1234"
+ }
+
+ if testcase.autoScalerExisted {
+ scalerInder.Add(tac)
+ }
+ err := tsm.syncAutoScalerRef(tc)
+ g.Expect(err).ShouldNot(HaveOccurred())
+ if testcase.expectedStatusRef == nil {
+ g.Expect(tc.Status.AutoScaler).Should(BeNil())
+ } else {
+ g.Expect(tc.Status.AutoScaler).ShouldNot(BeNil())
+ g.Expect(tc.Status.AutoScaler.Name).Should(Equal(testcase.expectedStatusRef.Name))
+ g.Expect(tc.Status.AutoScaler.Namespace).Should(Equal(testcase.expectedStatusRef.Namespace))
+ }
+ })
+ }
+}
+
+func newFakeTidbClusterStatusManager() (*TidbClusterStatusManager, kubernetes.Interface, *fake.Clientset, cache.Indexer) {
+ cli := fake.NewSimpleClientset()
+ kubeCli := kubefake.NewSimpleClientset()
+ informerFactory := informers.NewSharedInformerFactoryWithOptions(cli, 0)
+ scalerFactory := informerFactory.Pingcap().V1alpha1().TidbClusterAutoScalers()
+ scalerInder := scalerFactory.Informer().GetIndexer()
+ return NewTidbClusterStatusManager(kubeCli, cli, scalerFactory.Lister()), kubeCli, cli, scalerInder
+}
+
+func newTidbClusterAutoScaler(tc *v1alpha1.TidbCluster) *v1alpha1.TidbClusterAutoScaler {
+ tac := &v1alpha1.TidbClusterAutoScaler{
+ Spec: v1alpha1.TidbClusterAutoScalerSpec{
+ Cluster: v1alpha1.TidbClusterRef{
+ Namespace: tc.Namespace,
+ Name: tc.Name,
+ },
+ },
+ }
+ tac.Name = "auto-scaler"
+ tac.Namespace = "default"
+ return tac
+}
diff --git a/pkg/manager/member/tiflash_util_test.go b/pkg/manager/member/tiflash_util_test.go
index 882dc121e5..20c9f441b2 100644
--- a/pkg/manager/member/tiflash_util_test.go
+++ b/pkg/manager/member/tiflash_util_test.go
@@ -479,6 +479,7 @@ func newTidbCluster() *v1alpha1.TidbCluster {
},
TiFlash: &v1alpha1.TiFlashSpec{},
},
+ Status: v1alpha1.TidbClusterStatus{},
}
}
diff --git a/tests/e2e/tidbcluster/serial.go b/tests/e2e/tidbcluster/serial.go
index afe5ce4c5c..e5f9f1dc54 100644
--- a/tests/e2e/tidbcluster/serial.go
+++ b/tests/e2e/tidbcluster/serial.go
@@ -508,6 +508,24 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
_, err = cli.PingcapV1alpha1().TidbClusterAutoScalers(ns).Create(tac)
framework.ExpectNoError(err, "Create TidbMonitorClusterAutoScaler error")
+
+ framework.Logf("start to check auto-scaler ref")
+ // check TidbCluster Status AutoScaler Ref
+ err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
+ tc, err := cli.PingcapV1alpha1().TidbClusters(tc.Namespace).Get(tc.Name, metav1.GetOptions{})
+ if err != nil {
+ return false, nil
+ }
+ if tc.Status.AutoScaler == nil {
+ return false, nil
+ }
+ if tc.Status.AutoScaler.Name != tac.Name || tc.Status.AutoScaler.Namespace != tac.Namespace {
+ return false, fmt.Errorf("wrong tidbcluster auto-scaler ref[%s/%s]", tc.Status.AutoScaler.Namespace, tc.Status.AutoScaler.Name)
+ }
+ return true, nil
+ })
+ framework.ExpectNoError(err, "Check Auto-Scaler Ref failed")
+
pdClient, cancel, err := proxiedpdclient.NewProxiedPDClient(c, fw, ns, clusterName, false, nil)
framework.ExpectNoError(err, "create pdapi error")
defer cancel()
@@ -540,7 +558,7 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
framework.ExpectNoError(err, "check tikv has ready-to-scale-timestamp")
framework.Logf("tikv has checked ready-to-scale-timestamp")
- err = wait.Poll(10*time.Second, 5*time.Minute, func() (done bool, err error) {
+ err = wait.Poll(10*time.Second, 10*time.Minute, func() (done bool, err error) {
stac, err := cli.PingcapV1alpha1().TidbClusterAutoScalers(ns).Get(tac.Name, metav1.GetOptions{})
if err != nil {
return false, nil
@@ -764,7 +782,7 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
_, err = cli.PingcapV1alpha1().TidbClusterAutoScalers(ns).Update(tac)
framework.ExpectNoError(err, "Update TidbMonitorClusterAutoScaler error")
- err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
+ err = wait.Poll(5*time.Second, 10*time.Minute, func() (done bool, err error) {
stac, err := cli.PingcapV1alpha1().TidbClusterAutoScalers(ns).Get(tac.Name, metav1.GetOptions{})
if err != nil {
return false, nil
@@ -818,7 +836,7 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
framework.ExpectNoError(err, "set tidb mock metrics error")
// Scale Tidb to 2 Replicas and Check
- err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
+ err = wait.Poll(5*time.Second, 10*time.Minute, func() (done bool, err error) {
stac, err := cli.PingcapV1alpha1().TidbClusterAutoScalers(ns).Get(tac.Name, metav1.GetOptions{})
if err != nil {
return false, nil
@@ -864,6 +882,20 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
})
framework.ExpectNoError(err, "check tidb auto-scale to 2 error")
framework.Logf("success to check auto scale-in tidb to 2 replicas")
+
+ err = cli.PingcapV1alpha1().TidbClusterAutoScalers(tac.Namespace).Delete(tac.Name, &metav1.DeleteOptions{})
+ framework.ExpectNoError(err, "Expect to delete auto-scaler ref")
+ err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
+ tc, err := cli.PingcapV1alpha1().TidbClusters(tc.Namespace).Get(tc.Name, metav1.GetOptions{})
+ if err != nil {
+ return false, nil
+ }
+ if tc.Status.AutoScaler != nil {
+ return false, nil
+ }
+ return true, nil
+ })
+ framework.ExpectNoError(err, "expect auto-scaler ref empty after delete auto-scaler")
})
})