fix some corner cases and add more tests

charts/tidb-operator/values.yaml

@@ -150,7 +150,7 @@ apiserver:
 #   kubectl apply -f manifests/advanced-statefulset-crd.v1.yaml      # k8s version >= 1.16.0
 advancedStatefulset:
   create: false
-  image: pingcap/advanced-statefulset:v0.2.2
+  image: pingcap/advanced-statefulset:v0.2.3
   imagePullPolicy: IfNotPresent
   serviceAccount: advanced-statefulset-controller
   logLevel: 2

go.mod

@@ -47,7 +47,7 @@ require (
 	github.com/openshift/generic-admission-server v1.14.0
 	github.com/opentracing/opentracing-go v1.1.0 // indirect
 	github.com/pierrec/lz4 v2.0.5+incompatible // indirect
-	github.com/pingcap/advanced-statefulset v0.2.2
+	github.com/pingcap/advanced-statefulset v0.2.4
 	github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 // indirect
 	github.com/pingcap/errors v0.11.0
 	github.com/pingcap/kvproto v0.0.0-20190516013202-4cf58ad90b6c

go.sum

@@ -593,6 +593,10 @@ github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pingcap/advanced-statefulset v0.2.2 h1:rdNHCmtn6KcUqc0ADq0ViRhgbVtWcKnJvGpBhMWnGIw=
 github.com/pingcap/advanced-statefulset v0.2.2/go.mod h1:rg2p1v6AGsKhvEZi6Sm0YNYJCmdXdZZhQ6Sviei7Ivs=
+github.com/pingcap/advanced-statefulset v0.2.3 h1:ZZLZaTquJyWcr4PDSd6sHtLpfNqXmrm5L0fDH/WcTkY=
+github.com/pingcap/advanced-statefulset v0.2.3/go.mod h1:rg2p1v6AGsKhvEZi6Sm0YNYJCmdXdZZhQ6Sviei7Ivs=
+github.com/pingcap/advanced-statefulset v0.2.4 h1:0PRkKLEiwlWgdFWNtbBOa+Dio7MZLqZ3escd0R3s8No=
+github.com/pingcap/advanced-statefulset v0.2.4/go.mod h1:rg2p1v6AGsKhvEZi6Sm0YNYJCmdXdZZhQ6Sviei7Ivs=
 github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg=
 github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ=
 github.com/pingcap/errors v0.11.0 h1:DCJQB8jrHbQ1VVlMFIrbj2ApScNNotVmkSNplu2yUt4=

pkg/manager/member/scaler.go

@@ -124,34 +124,56 @@ func ordinalPodName(memberType v1alpha1.MemberType, tcName string, ordinal int32
 // - ordinal: pod ordinal to create or delete
 // - replicas/deleteSlots: desired replicas and deleteSlots by allowing only one pod to be deleted or created
 func scaleOne(actual *apps.StatefulSet, desired *apps.StatefulSet) (scaling int, ordinal int32, replicas int32, deleteSlots sets.Int32) {
-	actualDesiredPodOrdinals := helper.GetPodOrdinals(*actual.Spec.Replicas, actual)
-	desiredDesiredPodOrdinals := helper.GetPodOrdinals(*desired.Spec.Replicas, desired)
-	additions := desiredDesiredPodOrdinals.Difference(actualDesiredPodOrdinals)
-	deletions := actualDesiredPodOrdinals.Difference(desiredDesiredPodOrdinals)
+	actualPodOrdinals := helper.GetPodOrdinals(*actual.Spec.Replicas, actual)
+	desiredPodOrdinals := helper.GetPodOrdinals(*desired.Spec.Replicas, desired)
+	additions := desiredPodOrdinals.Difference(actualPodOrdinals)
+	deletions := actualPodOrdinals.Difference(desiredPodOrdinals)
 	scaling = 0
 	ordinal = -1
 	replicas = *actual.Spec.Replicas
 	actualDeleteSlots := helper.GetDeleteSlots(actual)
 	desiredDeleteSlots := helper.GetDeleteSlots(desired)
+	// copy delete slots from desired delete slots if not in actual pod ordinals
+	for i := range desiredDeleteSlots {
+		if !actualPodOrdinals.Has(i) {
+			actualDeleteSlots.Insert(i)
+		}
+	}
 	if additions.Len() > 0 {
 		// we always do scaling out before scaling in to maintain maximum avaiability
 		scaling = 1
 		ordinal = additions.List()[0]
-		replicas = *actual.Spec.Replicas + 1
+		replicas += 1
 		if !desiredDeleteSlots.Has(ordinal) {
-			// not in desired delete slots, remote it from actual delete slots
+			// not in desired delete slots, remove it from actual delete slots
 			actualDeleteSlots.Delete(ordinal)
 		}
+		deleteSlots = normalizeDeleteSlots(replicas, actualDeleteSlots, desiredDeleteSlots)
 	} else if deletions.Len() > 0 {
 		scaling = -1
 		deletionsList := deletions.List()
 		ordinal = deletionsList[len(deletionsList)-1]
-		replicas = *actual.Spec.Replicas - 1
+		replicas -= 1
 		if desiredDeleteSlots.Has(ordinal) {
 			// in desired delete slots, add it in actual delete slots
 			actualDeleteSlots.Insert(ordinal)
 		}
+		actualDeleteSlots = normalizeDeleteSlots(replicas, actualDeleteSlots, desiredDeleteSlots)
 	}
 	deleteSlots = actualDeleteSlots
 	return
 }
+
+// normalizeDeleteSlots
+// - add redundant data if in desired delete slots
+// - remove redundant data if not in desired delete slots
+// this is necessary to reach the desired state
+func normalizeDeleteSlots(replicas int32, deleteSlots sets.Int32, desiredDeleteSlots sets.Int32) sets.Int32 {
+	maxReplicaCount, _ := helper.GetMaxReplicaCountAndDeleteSlots(replicas, deleteSlots)
+	for ordinal := range deleteSlots {
+		if ordinal >= maxReplicaCount && !desiredDeleteSlots.Has(ordinal) {
+			deleteSlots.Delete(ordinal)
+		}
+	}
+	return deleteSlots
+}