Add SLO parameter to API availability measurement

byxorna · Mar 24, 2021 · 60cf1a2 · 60cf1a2
1 parent 81c96c3
commit 60cf1a2
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 4 deletions.
diff --git a/clusterloader2/pkg/measurement/common/api_availability_measurement.go b/clusterloader2/pkg/measurement/common/api_availability_measurement.go
@@ -26,6 +26,7 @@ import (
 
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/klog"
+	"k8s.io/perf-tests/clusterloader2/pkg/errors"
 	"k8s.io/perf-tests/clusterloader2/pkg/execservice"
 	"k8s.io/perf-tests/clusterloader2/pkg/measurement"
 	"k8s.io/perf-tests/clusterloader2/pkg/provider"
@@ -56,6 +57,7 @@ type apiAvailabilityMeasurement struct {
 	hostIPs             []string
 	summaries           []measurement.Summary
 	clusterLevelMetrics *apiAvailabilityMetrics
+	threshold           float64
 	// Metrics per host internal IP.
 	hostLevelMetrics       map[string]*apiAvailabilityMetrics
 	hostPollTimeoutSeconds int
@@ -177,6 +179,12 @@ func (a *apiAvailabilityMeasurement) initFields(config *measurement.Config) erro
 	}
 	a.pollFrequency = frequency
 
+	threshold, err := util.GetFloat64OrDefault(config.Params, "threshold", 0.0)
+	if err != nil {
+		return err
+	}
+	a.threshold = threshold
+
 	a.clusterLevelMetrics = &apiAvailabilityMetrics{}
 	if config.ClusterLoaderConfig.EnableExecService {
 		a.hostIPs = config.ClusterFramework.GetClusterConfig().MasterInternalIPs
@@ -252,7 +260,10 @@ func (a *apiAvailabilityMeasurement) gather() ([]measurement.Summary, error) {
 	}
 	summary := measurement.CreateSummary(apiAvailabilityMeasurementName, "json", content)
 	a.summaries = append(a.summaries, summary)
-	return a.summaries, nil
+	if sli := output.ClusterSummary.AvailabilityPercentage; sli < a.threshold {
+		err = errors.NewMetricViolationError("API availability", fmt.Sprintf("SLO not fulfilled (expected >= %.2f, got: %.2f)", a.threshold, sli))
+	}
+	return a.summaries, err
 }
 
 func (a *apiAvailabilityMeasurement) Dispose() {}

diff --git a/clusterloader2/pkg/measurement/common/api_availability_metrics.go b/clusterloader2/pkg/measurement/common/api_availability_metrics.go
@@ -29,7 +29,7 @@ type apiAvailabilityMetrics struct {
 
 type apiAvailabilitySummary struct {
 	IP                       string  `json:"IP,omitempty"`
-	AvailabilityPercentage   float32 `json:"availabilityPercentage"`
+	AvailabilityPercentage   float64 `json:"availabilityPercentage"`
 	LongestUnavailablePeriod string  `json:"longestUnavailablePeriod"`
 }
 
@@ -51,9 +51,9 @@ func (a *apiAvailabilityMetrics) update(availability bool) {
 }
 
 func (a *apiAvailabilityMetrics) buildSummary(pollFrequency time.Duration, hostIP string) apiAvailabilitySummary {
-	availabilityPercentage := float32(100)
+	availabilityPercentage := float64(100)
 	if a.numSuccesses > 0 || a.numFailures > 0 {
-		availabilityPercentage = (float32(a.numSuccesses) / float32(a.numSuccesses+a.numFailures)) * 100
+		availabilityPercentage = (float64(a.numSuccesses) / float64(a.numSuccesses+a.numFailures)) * 100
 	}
 	longestUnavailablePeriod := time.Duration(a.maxConsecutiveFailedProbes) * pollFrequency
 	return apiAvailabilitySummary{

diff --git a/clusterloader2/testing/load/config.yaml b/clusterloader2/testing/load/config.yaml
@@ -38,6 +38,7 @@
 {{$ENABLE_VIOLATIONS_FOR_API_CALL_PROMETHEUS := DefaultParam .CL2_ENABLE_VIOLATIONS_FOR_API_CALL_PROMETHEUS false}}
 {{$ENABLE_VIOLATIONS_FOR_API_CALL_PROMETHEUS_SIMPLE := DefaultParam .CL2_ENABLE_VIOLATIONS_FOR_API_CALL_PROMETHEUS_SIMPLE true}}
 {{$ENABLE_API_AVAILABILITY_MEASUREMENT := DefaultParam .CL2_ENABLE_API_AVAILABILITY_MEASUREMENT false}}
+{{$API_AVAILABILITY_PERCENTAGE_THRESHOLD := DefaultParam .CL2_API_AVAILABILITY_PERCENTAGE_THRESHOLD 0.0}}
 #Variables
 {{$namespaces := DivideInt .Nodes $NODES_PER_NAMESPACE}}
 {{$totalPods := MultiplyInt $namespaces $NODES_PER_NAMESPACE $PODS_PER_NODE}}
@@ -183,6 +184,7 @@ steps:
       action: start
       pollFrequency: "5s"
       hostPollTimeoutSeconds: 5
+      threshold: {{$API_AVAILABILITY_PERCENTAGE_THRESHOLD}}
   {{end}}
   - Identifier: TestMetrics
     Method: TestMetrics