-
Notifications
You must be signed in to change notification settings - Fork 527
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #2072 from dlapcevic/new-dns-client-2
Add measurement for dnsperfgo K8s hostnames test
- Loading branch information
Showing
6 changed files
with
292 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
210 changes: 210 additions & 0 deletions
210
clusterloader2/pkg/measurement/common/dns/dns_performance-k8s-hostnames.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,210 @@ | ||
/* | ||
Copyright 2022 The Kubernetes Authors. | ||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
http://www.apache.org/licenses/LICENSE-2.0 | ||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package dns | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"path/filepath" | ||
"time" | ||
|
||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
clientset "k8s.io/client-go/kubernetes" | ||
"k8s.io/klog" | ||
"k8s.io/perf-tests/clusterloader2/pkg/framework" | ||
"k8s.io/perf-tests/clusterloader2/pkg/framework/client" | ||
"k8s.io/perf-tests/clusterloader2/pkg/measurement" | ||
"k8s.io/perf-tests/clusterloader2/pkg/util" | ||
) | ||
|
||
/* | ||
The DNS Performance test for K8s Hostnames creates the required permissions for | ||
DNS client pods to get K8s hostnames (from services and endpoints). Then DNS | ||
client deployment is created that uses dnsperfgo image | ||
(https://github.com/kubernetes/perf-tests/tree/master/dns/dnsperfgo) to query | ||
the hostnames within the cluster and generate Prometheus metrics for number of | ||
errors, timeouts and their latencies. Outside this measurement, the metrics are | ||
scraped and results verified to not cross the specified thresholds. | ||
*/ | ||
|
||
const ( | ||
dnsPerfK8sHostnamesMeasureName = "DNSPerformanceK8sHostnames" | ||
dnsPerfTestNamespace = "dns-perf-test" | ||
dnsPerfTestPermissionsName = "dns-test-client" | ||
manifestPathPrefix = "./pkg/measurement/common/dns/manifests" | ||
) | ||
|
||
var ( | ||
serviceAccountFilePath = filepath.Join(manifestPathPrefix, "serviceaccount.yaml") | ||
clusterRoleFilePath = filepath.Join(manifestPathPrefix, "clusterrole.yaml") | ||
clusterRoleBindingFilePath = filepath.Join(manifestPathPrefix, "clusterrolebinding.yaml") | ||
clientDeploymentFilePath = filepath.Join(manifestPathPrefix, "dns-client.yaml") | ||
) | ||
|
||
func init() { | ||
klog.Info("Registering measurement: DNS Performance for K8s Hostnames") | ||
if err := measurement.Register(dnsPerfK8sHostnamesMeasureName, createDNSPerfK8sHostnamesMeasurement); err != nil { | ||
klog.Fatalf("Cannot register %s: %v", dnsPerfK8sHostnamesMeasureName, err) | ||
} | ||
} | ||
|
||
func createDNSPerfK8sHostnamesMeasurement() measurement.Measurement { | ||
return &dnsPerfK8sHostnamesMeasurement{} | ||
} | ||
|
||
type dnsPerfK8sHostnamesMeasurement struct { | ||
k8sClient clientset.Interface | ||
framework *framework.Framework | ||
// testClientNamespace is the new namespace where the dns clients are going to | ||
// be deployed. It's cleaned up when the test finishes. | ||
testClientNamespace string | ||
// podReplicas is the number of DNS client pods to be deployed. | ||
podReplicas int | ||
// qpsPerClient is the number of DNS queries each DNS client should send per | ||
// second. | ||
qpsPerClient int | ||
// testDurationMinutes is the duration in minutes for DNS client pods to run. | ||
testDurationMinutes int | ||
} | ||
|
||
func (m *dnsPerfK8sHostnamesMeasurement) Execute(config *measurement.Config) ([]measurement.Summary, error) { | ||
err := m.initializeMeasurement(config) | ||
if err != nil { | ||
return nil, fmt.Errorf("failed to initialize the measurement: %v", err) | ||
} | ||
|
||
if err = client.CreateNamespace(m.k8sClient, m.testClientNamespace); err != nil { | ||
return nil, fmt.Errorf("error while creating namespace: %v", err) | ||
} | ||
|
||
if err = m.createDNSClientPermissions(); err != nil { | ||
return nil, fmt.Errorf("failed to create dns client permission resources: %v", err) | ||
} | ||
|
||
if err = m.createDNSClientDeployment(); err != nil { | ||
return nil, fmt.Errorf("failed to create DNS client deployment: %v", err) | ||
} | ||
|
||
// Keep running the dns clients for the specified duration. | ||
runDuration := time.Duration(m.testDurationMinutes) * time.Minute | ||
klog.Infof("DNS tests are going to run for %v", runDuration) | ||
time.Sleep(runDuration) | ||
|
||
return nil, m.cleanUp() | ||
} | ||
|
||
func (m *dnsPerfK8sHostnamesMeasurement) initializeMeasurement(config *measurement.Config) error { | ||
if m.framework != nil { | ||
klog.Warningf("The %q was already started, but running it again", dnsPerfK8sHostnamesMeasureName) | ||
} | ||
|
||
var err error | ||
if m.testClientNamespace, err = util.GetStringOrDefault(config.Params, "testNamespace", dnsPerfTestNamespace); err != nil { | ||
return err | ||
} | ||
|
||
if m.podReplicas, err = util.GetIntOrDefault(config.Params, "podReplicas", 10); err != nil { | ||
return err | ||
} | ||
|
||
if m.qpsPerClient, err = util.GetIntOrDefault(config.Params, "qpsPerClient", 10); err != nil { | ||
return err | ||
} | ||
|
||
if m.testDurationMinutes, err = util.GetIntOrDefault(config.Params, "testDurationMinutes", 10); err != nil { | ||
return err | ||
} | ||
|
||
m.framework = config.ClusterFramework | ||
m.k8sClient = config.ClusterFramework.GetClientSets().GetClient() | ||
|
||
return nil | ||
} | ||
|
||
// createDNSClientPermissions creates ServiceAccount, ClusterRole and | ||
// ClusterRoleBinding for the test client pods to access Services and Endpoints. | ||
func (m *dnsPerfK8sHostnamesMeasurement) createDNSClientPermissions() error { | ||
templateMap := map[string]interface{}{ | ||
"Name": dnsPerfTestPermissionsName, | ||
"Namespace": m.testClientNamespace, | ||
} | ||
|
||
if err := m.framework.ApplyTemplatedManifests(serviceAccountFilePath, templateMap); err != nil { | ||
return fmt.Errorf("error while creating serviceaccount: %v", err) | ||
} | ||
|
||
if err := m.framework.ApplyTemplatedManifests(clusterRoleFilePath, templateMap); err != nil { | ||
return fmt.Errorf("error while creating clusterrole: %v", err) | ||
} | ||
|
||
if err := m.framework.ApplyTemplatedManifests(clusterRoleBindingFilePath, templateMap); err != nil { | ||
return fmt.Errorf("error while creating clusterrolebinding: %v", err) | ||
} | ||
|
||
return nil | ||
} | ||
|
||
func (m *dnsPerfK8sHostnamesMeasurement) createDNSClientDeployment() error { | ||
templateMap := map[string]interface{}{ | ||
"Namespace": m.testClientNamespace, | ||
"PodReplicas": m.podReplicas, | ||
"QPSPerClient": m.qpsPerClient, | ||
"ServiceAccountName": dnsPerfTestPermissionsName, | ||
} | ||
|
||
return m.framework.ApplyTemplatedManifests(clientDeploymentFilePath, templateMap) | ||
} | ||
|
||
func (m *dnsPerfK8sHostnamesMeasurement) deleteDNSClientPermissions() error { | ||
klog.Infof("Deleting DNS client permission resources for measurement %q", dnsPerfK8sHostnamesMeasureName) | ||
|
||
if err := m.k8sClient.CoreV1().ServiceAccounts(m.testClientNamespace).Delete(context.TODO(), dnsPerfTestPermissionsName, metav1.DeleteOptions{}); err != nil { | ||
return err | ||
} | ||
|
||
if err := m.k8sClient.RbacV1().ClusterRoles().Delete(context.TODO(), dnsPerfTestPermissionsName, metav1.DeleteOptions{}); err != nil { | ||
return err | ||
} | ||
|
||
return m.k8sClient.RbacV1().ClusterRoleBindings().Delete(context.TODO(), dnsPerfTestPermissionsName, metav1.DeleteOptions{}) | ||
} | ||
|
||
func (m *dnsPerfK8sHostnamesMeasurement) cleanUp() error { | ||
if m.framework == nil { | ||
klog.Warning("Cleanup skipped. The measurement is not running") | ||
return nil | ||
} | ||
|
||
if err := m.deleteDNSClientPermissions(); err != nil { | ||
return err | ||
} | ||
|
||
klog.Infof("Deleting namespace %q for measurement %q", m.testClientNamespace, dnsPerfK8sHostnamesMeasureName) | ||
return m.k8sClient.CoreV1().Namespaces().Delete(context.TODO(), m.testClientNamespace, metav1.DeleteOptions{}) | ||
} | ||
|
||
// String returns a string representation of the measurement. | ||
func (m *dnsPerfK8sHostnamesMeasurement) String() string { | ||
return dnsPerfK8sHostnamesMeasureName | ||
} | ||
|
||
// Dispose cleans up after the measurement. | ||
func (m *dnsPerfK8sHostnamesMeasurement) Dispose() { | ||
if err := m.cleanUp(); err != nil { | ||
klog.Infof("Cleanup failed: %v", err) | ||
} | ||
} |
11 changes: 11 additions & 0 deletions
11
clusterloader2/pkg/measurement/common/dns/manifests/clusterrole.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
kind: ClusterRole | ||
apiVersion: rbac.authorization.k8s.io/v1 | ||
metadata: | ||
name: {{.Name}} | ||
rules: | ||
- apiGroups: [""] | ||
resources: ["services"] | ||
verbs: ["create", "get", "list", "delete"] | ||
- apiGroups: [""] | ||
resources: ["endpoints"] | ||
verbs: ["get", "list"] |
12 changes: 12 additions & 0 deletions
12
clusterloader2/pkg/measurement/common/dns/manifests/clusterrolebinding.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
kind: ClusterRoleBinding | ||
apiVersion: rbac.authorization.k8s.io/v1 | ||
metadata: | ||
name: {{.Name}} | ||
subjects: | ||
- kind: ServiceAccount | ||
name: {{.Name}} | ||
namespace: {{.Namespace}} | ||
roleRef: | ||
kind: ClusterRole | ||
name: {{.Name}} | ||
apiGroup: rbac.authorization.k8s.io |
53 changes: 53 additions & 0 deletions
53
clusterloader2/pkg/measurement/common/dns/manifests/dns-client.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
apiVersion: apps/v1 | ||
kind: Deployment | ||
metadata: | ||
name: dnsperfgo | ||
namespace: {{.Namespace}} | ||
spec: | ||
replicas: {{.PodReplicas}} | ||
selector: | ||
matchLabels: | ||
dns-test: dnsperfgo | ||
template: | ||
metadata: | ||
labels: | ||
dns-test: dnsperfgo | ||
spec: | ||
containers: | ||
- name: dnsperfgo | ||
ports: | ||
- containerPort: 9153 | ||
name: dnsperfmetrics | ||
protocol: TCP | ||
image: gcr.io/k8s-staging-perf-tests/dnsperfgo:v1.3.0 | ||
# Fetches the dns server from /etc/resolv.conf and sends 10 queries per second. | ||
# With searchpath expansion, this is 120 queries per second. | ||
# External names like google.com are expanded to 12 queries. | ||
# FQDN lookups like kubernetes.default.svc.cluster.local are also expanded to 12 queries since they have < 5 dots in the name. | ||
# Names like kubernetes.default will get partial search path expansion, since the query will succeed once "svc.cluster.local" path is applied. | ||
# -query-cluster-names flag will generate FQDNs, in order to exercise searchpath expansion. | ||
# dnsperf has a client timeout of 5s. It sends queries for 60s, | ||
# then sleeps for 10s, to mimic bursts of DNS queries. | ||
command: | ||
- sh | ||
- -c | ||
- server=$(cat /etc/resolv.conf | grep nameserver | cut -d ' ' -f 2); echo | ||
"Using nameserver ${server}"; | ||
./dnsperfgo -duration 60s -idle-duration 10s -query-cluster-names -qps {{.QPSPerClient}}; | ||
resources: | ||
requests: | ||
cpu: 10m | ||
memory: 10M | ||
serviceAccountName: {{.ServiceAccountName}} | ||
terminationGracePeriodSeconds: 1 | ||
# Add not-ready/unreachable tolerations for 15 minutes so that node | ||
# failure doesn't trigger pod deletion. | ||
tolerations: | ||
- key: "node.kubernetes.io/not-ready" | ||
operator: "Exists" | ||
effect: "NoExecute" | ||
tolerationSeconds: 900 | ||
- key: "node.kubernetes.io/unreachable" | ||
operator: "Exists" | ||
effect: "NoExecute" | ||
tolerationSeconds: 900 |
5 changes: 5 additions & 0 deletions
5
clusterloader2/pkg/measurement/common/dns/manifests/serviceaccount.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
kind: ServiceAccount | ||
apiVersion: v1 | ||
metadata: | ||
name: {{.Name}} | ||
namespace: {{.Namespace}} |