Implement CL2_RATE_LIMIT_POD_CREATION that drops artificial rate limi…

…ting
kubernetes · Aug 24, 2022 · fa62c29 · fa62c29
1 parent 91d0966
commit fa62c29
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 5 deletions.
diff --git a/clusterloader2/pkg/measurement/common/wait_for_controlled_pods.go b/clusterloader2/pkg/measurement/common/wait_for_controlled_pods.go
@@ -115,6 +115,8 @@ type waitForControlledPodsRunningMeasurement struct {
 	kind             string
 	selector         *util.ObjectSelector
 	operationTimeout time.Duration
+	// exactOperationTimeout controls whether to use 2x timeout during scale down/deletion.
+	exactOperationTimeout bool
 	// countErrorMargin orders measurement to wait for number of pods to be in
 	// <desired count - countErrorMargin, desired count> range
 	// When using preemptibles on large scale, number of ready nodes is not stable
@@ -170,6 +172,13 @@ func (w *waitForControlledPodsRunningMeasurement) Execute(config *measurement.Co
 		if err != nil {
 			return nil, err
 		}
+		// exactOperationTimeout controls whether we should skip multiplying by two operationTimeout on scale down/deletion.
+		// Defaults to false for backward compatibility.
+		// TODO(mborsz): Change default to true and remove.
+		w.exactOperationTimeout, err = util.GetBoolOrDefault(config.Params, "exactOperationTimeout", false)
+		if err != nil {
+			return nil, err
+		}
 		w.countErrorMargin, err = util.GetIntOrDefault(config.Params, "countErrorMargin", 0)
 		if err != nil {
 			return nil, err
@@ -438,7 +447,7 @@ func (w *waitForControlledPodsRunningMeasurement) handleObjectLocked(oldObj, new
 	}
 
 	operationTimeout := w.operationTimeout
-	if isObjDeleted || isScalingDown {
+	if !w.exactOperationTimeout && (isObjDeleted || isScalingDown) {
 		// In case of deleting pods, twice as much time is required.
 		// The pod deletion throughput equals half of the pod creation throughput.
 		// NOTE: Starting from k8s 1.23 it's not true anymore, at least not in all cases.

diff --git a/clusterloader2/testing/load/config.yaml b/clusterloader2/testing/load/config.yaml
@@ -12,6 +12,7 @@
 {{$PODS_PER_NODE := DefaultParam .PODS_PER_NODE 30}}
 {{$LOAD_TEST_THROUGHPUT := DefaultParam .CL2_LOAD_TEST_THROUGHPUT 10}}
 {{$DELETE_TEST_THROUGHPUT := DefaultParam .CL2_DELETE_TEST_THROUGHPUT $LOAD_TEST_THROUGHPUT}}
+{{$RATE_LIMIT_POD_CREATION := DefaultParam .CL2_RATE_LIMIT_POD_CREATION true}}
 {{$BIG_GROUP_SIZE := DefaultParam .BIG_GROUP_SIZE 250}}
 {{$MEDIUM_GROUP_SIZE := DefaultParam .MEDIUM_GROUP_SIZE 30}}
 {{$SMALL_GROUP_SIZE := DefaultParam .SMALL_GROUP_SIZE 5}}
@@ -168,7 +169,13 @@ steps:
     params:
       actionName: "create"
       namespaces: {{$namespaces}}
+      {{if .RATE_LIMIT_POD_CREATION}}
       tuningSet: RandomizedSaturationTimeLimited
+      operationTimeout: 15m
+      {{else}}
+      tuningSet: Global100qps
+      operationTimeout: {{AddInt $saturationTime 900}}s
+      {{end}}
       testMaxReplicaFactor: {{$RANDOM_SCALE_FACTOR}}
       # We rely on the fact that daemonset is using the same image as the 'pod-startup-latency' module.
       # The goal is to cache the image to all nodes before we start any latency pod,
@@ -277,7 +284,13 @@ steps:
     params:
       actionName: "scale and update"
       namespaces: {{$namespaces}}
+      {{if .RATE_LIMIT_POD_CREATION}}
       tuningSet: RandomizedScalingTimeLimited
+      operationTimeout: 15m
+      {{else}}
+      tuningSet: Global100qps
+      operationTimeout: {{AddInt (DivideInt $saturationTime 4) 900}}s
+      {{end}}
       randomScaleFactor: {{$RANDOM_SCALE_FACTOR}}
       testMaxReplicaFactor: {{$RANDOM_SCALE_FACTOR}}
       daemonSetImage: {{$latencyPodImage}}
@@ -305,7 +318,13 @@ steps:
     params:
       actionName: "delete"
       namespaces: {{$namespaces}}
+      {{if .RATE_LIMIT_POD_CREATION}}
       tuningSet: RandomizedDeletionTimeLimited
+      operationTimeout: 15m
+      {{else}}
+      tuningSet: Global100qps
+      operationTimeout: {{AddInt $deletionTime 900}}s
+      {{end}}
       testMaxReplicaFactor: {{$RANDOM_SCALE_FACTOR}}
       daemonSetReplicas: 0
       bigDeploymentSize: {{$BIG_GROUP_SIZE}}

diff --git a/clusterloader2/testing/load/modules/reconcile-objects.yaml b/clusterloader2/testing/load/modules/reconcile-objects.yaml
@@ -10,6 +10,7 @@
 {{$minReplicaFactor := SubtractFloat 1 $randomScaleFactor}}
 {{$maxReplicaFactor := AddFloat 1 $randomScaleFactor}}
 {{$testMaxReplicaFactor := AddFloat 1 .testMaxReplicaFactor}}
+{{$operationTimeout := .operationTimeout}}
 
 # DaemonSets
 {{$daemonSetImage := DefaultParam .daemonSetImage "k8s.gcr.io/pause:3.0"}}
@@ -75,7 +76,8 @@ steps:
       action: start
       checkIfPodsAreUpdated: {{$CHECK_IF_PODS_ARE_UPDATED}}
       labelSelector: group = load
-      operationTimeout: 15m
+      operationTimeout: {{$operationTimeout}}
+      exactOperationTimeout: true
 
 - name: {{$actionName}}
   phases:
@@ -212,7 +214,7 @@ steps:
       min: 1
       max: {{$namespaces}}
     replicasPerNamespace: 0
-    tuningSet: RandomizedDeletionTimeLimited
+    tuningSet: {{$tuningSet}}
     objectBundle:
   {{range $ssIndex := Loop $pvSmallStatefulSetSize}}
       - basename: pv-small-statefulset-{{$ssIndex}}
@@ -226,7 +228,7 @@ steps:
       min: 1
       max: {{$namespaces}}
     replicasPerNamespace: 0
-    tuningSet: RandomizedDeletionTimeLimited
+    tuningSet: {{$tuningSet}}
     objectBundle:
   {{range $ssIndex := Loop $pvMediumStatefulSetSize}}
       - basename: pv-medium-statefulset-{{$ssIndex}}
@@ -254,5 +256,5 @@ steps:
     Params:
       desiredPVCCount: 0
       labelSelector: group = load
-      timeout: 15m
+      timeout: {{$operationTimeout}}
 {{end}}