Use performance metrics from the latest dnsperfgo client

mm4tt · Jul 5, 2022 · 51611af · 51611af
1 parent 33012d8
commit 51611af
Show file tree

Hide file tree

Showing 4 changed files with 73 additions and 2 deletions.
diff --git a/clusterloader2/testing/load/config.yaml b/clusterloader2/testing/load/config.yaml
@@ -207,6 +207,9 @@ steps:
       schedulerThroughputPodsPerDeployment: {{$schedulerThroughputPodsPerDeployment}}
 {{end}}
 
+- module:
+    path: modules/dns-k8s-hostnames.yaml
+
 {{if $EXEC_COMMAND}}
 
 {{if $ENABLE_API_AVAILABILITY_MEASUREMENT}}

diff --git a/clusterloader2/testing/load/deployment.yaml b/clusterloader2/testing/load/deployment.yaml
@@ -3,6 +3,9 @@
 # For now we're setting it to 0.5%.
 {{$CpuRequest := DefaultParam .CpuRequest "5m"}}
 {{$MemoryRequest := DefaultParam .MemoryRequest "20M"}}
+{{$dnsQPSPerClient := DefaultParam .CL2_DNS_QPS_PER_CLIENT 1}}
+# Guard the new DNS tests. Remove it once it's confirmed that it works on a subset of tests.
+{{$USE_ADVANCED_DNSTEST := DefaultParam .CL2_USE_ADVANCED_DNSTEST false}}
 
 apiVersion: apps/v1
 kind: Deployment
@@ -22,12 +25,22 @@ spec:
         group: load
         name: {{.Name}}
         svc: {{.SvcName}}-{{.Index}}
+{{if and .EnableDNSTests $USE_ADVANCED_DNSTEST}}
+        dns-test: dnsperfgo
+{{end}}
     spec:
       hostNetwork: {{$HostNetworkMode}}
       containers:
 {{if .EnableDNSTests}}
-      # TODO need to push this image.
+{{if $USE_ADVANCED_DNSTEST}}
+      - image: gcr.io/k8s-staging-perf-tests/dnsperfgo:v1.3.0
+        ports:
+        - containerPort: 9153
+          name: dnsperfmetrics
+          protocol: TCP
+{{else}}
       - image: gcr.io/k8s-staging-perf-tests/dnsperfgo:v1.2.0
+{{end}}
       # Fetches the dns server from /etc/resolv.conf and
       # sends 1 query per second.
       # With searchpath expansion, this is upto 12 queries per second.
@@ -38,7 +51,7 @@ spec:
         - -c
         - server=$(cat /etc/resolv.conf | grep nameserver | cut -d ' ' -f 2); echo
           "Using nameserver ${server}";
-          ./dnsperfgo -duration 60s -idle-duration 10s -inputfile /var/configmap/all-queries -qps 1;
+          ./dnsperfgo -duration 60s -idle-duration 10s -inputfile /var/configmap/all-queries -qps {{$dnsQPSPerClient}};
         name: {{.Name}}-dnsperf
 {{else}}
       - image: k8s.gcr.io/pause:3.1

diff --git a/clusterloader2/testing/load/modules/dns-k8s-hostnames.yaml b/clusterloader2/testing/load/modules/dns-k8s-hostnames.yaml
@@ -0,0 +1,29 @@
+{{$ENABLE_DNSTESTS := DefaultParam .CL2_ENABLE_DNSTESTS false}}
+# Guard the new DNS tests. Remove it once it's confirmed that it works on a subset of tests.
+{{$USE_ADVANCED_DNSTEST := DefaultParam .CL2_USE_ADVANCED_DNSTEST false}}
+
+# The number of DNS client pods for every 100 nodes.
+{{$dnsClientPodsFactor := DefaultParam .CL2_DNS_K8S_HOSTNAMES_CLIENT_PODS_FACTOR 1}}
+# Create 5 DNS client pods, plus additional based on the number of nodes.
+{{$dnsClientPods := AddInt 5 (MultiplyInt $dnsClientPodsFactor (DivideInt .Nodes 100))}}
+{{$qpsPerClient := DefaultParam .CL2_DNS_K8S_HOSTNAMES_PER_CLIENT_QPS 10}}
+{{$testDurationMinutes := DefaultParam .CL2_DNS_K8S_HOSTNAMES_TEST_MINUTES 10}}
+
+{{if and $ENABLE_DNSTESTS $USE_ADVANCED_DNSTEST}}
+steps:
+- name: Run DNS performance test for k8s hostnames
+  measurements:
+  - Identifier: DNSPerformanceK8sHostnames
+    Method: DNSPerformanceK8sHostnames
+    Params:
+      podReplicas: {{$dnsClientPods}}
+      qpsPerClient: {{$qpsPerClient}}
+      testDurationMinutes: {{$testDurationMinutes}}
+
+- name: Wait 1m for DNS test to complete
+  measurements:
+  - Identifier: Wait
+    Method: Sleep
+    Params:
+      duration: 1m
+{{end}}
diff --git a/clusterloader2/testing/load/modules/measurements.yaml b/clusterloader2/testing/load/modules/measurements.yaml
@@ -11,6 +11,9 @@
 {{$CUSTOM_API_CALL_THRESHOLDS := DefaultParam .CUSTOM_API_CALL_THRESHOLDS ""}}
 {{$ENABLE_API_AVAILABILITY_MEASUREMENT := DefaultParam .CL2_ENABLE_API_AVAILABILITY_MEASUREMENT false}}
 {{$ENABLE_CLUSTER_OOMS_TRACKER := DefaultParam .CL2_ENABLE_CLUSTER_OOMS_TRACKER true}}
+{{$ENABLE_DNSTESTS := DefaultParam .CL2_ENABLE_DNSTESTS false}}
+# Guard the new DNS tests. Remove it once it's confirmed that it works on a subset of tests.
+{{$USE_ADVANCED_DNSTEST := DefaultParam .CL2_USE_ADVANCED_DNSTEST false}}
 {{$ENABLE_NODE_LOCAL_DNS_LATENCY := DefaultParam .CL2_ENABLE_NODE_LOCAL_DNS_LATENCY false}}
 {{$ENABLE_RESTART_COUNT_CHECK := DefaultParam .ENABLE_RESTART_COUNT_CHECK true}}
 {{$ENABLE_SYSTEM_POD_METRICS:= DefaultParam .ENABLE_SYSTEM_POD_METRICS true}}
@@ -169,3 +172,26 @@ steps:
       clusterOOMsTrackerEnabled: {{$ENABLE_CLUSTER_OOMS_TRACKER}}
       restartCountThresholdOverrides: {{YamlQuote $RESTART_COUNT_THRESHOLD_OVERRIDES 4}}
       enableRestartCountCheck: {{$ENABLE_RESTART_COUNT_CHECK}}
+{{if and $ENABLE_DNSTESTS $USE_ADVANCED_DNSTEST}}
+  - Identifier: DNSPerformanceMetrics
+    Method: GenericPrometheusQuery
+    Params:
+      action: {{$action}}
+      metricName: DNS Performance
+      metricVersion: v1
+      unit: s
+      queries:
+      - name: DNS Lookup Count
+        query: sum(increase(dns_lookups_total[%v]))
+      - name: DNS Timeout Count
+        query: sum(increase(dns_timeouts_total[%v]))
+      - name: DNS Error Count
+        query: sum(increase(dns_errors_total[%v]))
+        threshold: 1
+      - name: DNS Lookup Latency - Perc50
+        query: histogram_quantile(0.5, sum(rate(dns_lookup_latency_bucket[%v])) by (le))
+        threshold: 0.01
+      - name: DNS Lookup Latency - Perc99
+        query: histogram_quantile(0.99, sum(rate(dns_lookup_latency_bucket[%v])) by (le))
+        threshold: 0.1
+{{end}}