diff --git a/clusterloader2/cmd/clusterloader.go b/clusterloader2/cmd/clusterloader.go index aa3dee79cc..8cc56ee52e 100644 --- a/clusterloader2/cmd/clusterloader.go +++ b/clusterloader2/cmd/clusterloader.go @@ -44,6 +44,7 @@ import ( _ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common" _ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/bundle" + _ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/dns" _ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/network" _ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/probes" _ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/slos" diff --git a/clusterloader2/pkg/measurement/common/dns/dns_performance-k8s-hostnames.go b/clusterloader2/pkg/measurement/common/dns/dns_performance-k8s-hostnames.go new file mode 100644 index 0000000000..522aefc520 --- /dev/null +++ b/clusterloader2/pkg/measurement/common/dns/dns_performance-k8s-hostnames.go @@ -0,0 +1,210 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dns + +import ( + "context" + "fmt" + "path/filepath" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog" + "k8s.io/perf-tests/clusterloader2/pkg/framework" + "k8s.io/perf-tests/clusterloader2/pkg/framework/client" + "k8s.io/perf-tests/clusterloader2/pkg/measurement" + "k8s.io/perf-tests/clusterloader2/pkg/util" +) + +/* +The DNS Performance test for K8s Hostnames creates the required permissions for +DNS client pods to get K8s hostnames (from services and endpoints). Then DNS +client deployment is created that uses dnsperfgo image +(https://github.com/kubernetes/perf-tests/tree/master/dns/dnsperfgo) to query +the hostnames within the cluster and generate Prometheus metrics for number of +errors, timeouts and their latencies. Outside this measurement, the metrics are +scraped and results verified to not cross the specified thresholds. +*/ + +const ( + dnsPerfK8sHostnamesMeasureName = "DNSPerformanceK8sHostnames" + dnsPerfTestNamespace = "dns-perf-test" + dnsPerfTestPermissionsName = "dns-test-client" + manifestPathPrefix = "./pkg/measurement/common/dns/manifests" +) + +var ( + serviceAccountFilePath = filepath.Join(manifestPathPrefix, "serviceaccount.yaml") + clusterRoleFilePath = filepath.Join(manifestPathPrefix, "clusterrole.yaml") + clusterRoleBindingFilePath = filepath.Join(manifestPathPrefix, "clusterrolebinding.yaml") + clientDeploymentFilePath = filepath.Join(manifestPathPrefix, "dns-client.yaml") +) + +func init() { + klog.Info("Registering measurement: DNS Performance for K8s Hostnames") + if err := measurement.Register(dnsPerfK8sHostnamesMeasureName, createDNSPerfK8sHostnamesMeasurement); err != nil { + klog.Fatalf("Cannot register %s: %v", dnsPerfK8sHostnamesMeasureName, err) + } +} + +func createDNSPerfK8sHostnamesMeasurement() measurement.Measurement { + return &dnsPerfK8sHostnamesMeasurement{} +} + +type dnsPerfK8sHostnamesMeasurement struct { + k8sClient clientset.Interface + framework *framework.Framework + // testClientNamespace is the new namespace where the dns clients are going to + // be deployed. It's cleaned up when the test finishes. + testClientNamespace string + // podReplicas is the number of DNS client pods to be deployed. + podReplicas int + // qpsPerClient is the number of DNS queries each DNS client should send per + // second. + qpsPerClient int + // testDurationMinutes is the duration in minutes for DNS client pods to run. + testDurationMinutes int +} + +func (m *dnsPerfK8sHostnamesMeasurement) Execute(config *measurement.Config) ([]measurement.Summary, error) { + err := m.initializeMeasurement(config) + if err != nil { + return nil, fmt.Errorf("failed to initialize the measurement: %v", err) + } + + if err = client.CreateNamespace(m.k8sClient, m.testClientNamespace); err != nil { + return nil, fmt.Errorf("error while creating namespace: %v", err) + } + + if err = m.createDNSClientPermissions(); err != nil { + return nil, fmt.Errorf("failed to create dns client permission resources: %v", err) + } + + if err = m.createDNSClientDeployment(); err != nil { + return nil, fmt.Errorf("failed to create DNS client deployment: %v", err) + } + + // Keep running the dns clients for the specified duration. + runDuration := time.Duration(m.testDurationMinutes) * time.Minute + klog.Infof("DNS tests are going to run for %v", runDuration) + time.Sleep(runDuration) + + return nil, m.cleanUp() +} + +func (m *dnsPerfK8sHostnamesMeasurement) initializeMeasurement(config *measurement.Config) error { + if m.framework != nil { + klog.Warningf("The %q was already started, but running it again", dnsPerfK8sHostnamesMeasureName) + } + + var err error + if m.testClientNamespace, err = util.GetStringOrDefault(config.Params, "testNamespace", dnsPerfTestNamespace); err != nil { + return err + } + + if m.podReplicas, err = util.GetIntOrDefault(config.Params, "podReplicas", 10); err != nil { + return err + } + + if m.qpsPerClient, err = util.GetIntOrDefault(config.Params, "qpsPerClient", 10); err != nil { + return err + } + + if m.testDurationMinutes, err = util.GetIntOrDefault(config.Params, "testDurationMinutes", 10); err != nil { + return err + } + + m.framework = config.ClusterFramework + m.k8sClient = config.ClusterFramework.GetClientSets().GetClient() + + return nil +} + +// createDNSClientPermissions creates ServiceAccount, ClusterRole and +// ClusterRoleBinding for the test client pods to access Services and Endpoints. +func (m *dnsPerfK8sHostnamesMeasurement) createDNSClientPermissions() error { + templateMap := map[string]interface{}{ + "Name": dnsPerfTestPermissionsName, + "Namespace": m.testClientNamespace, + } + + if err := m.framework.ApplyTemplatedManifests(serviceAccountFilePath, templateMap); err != nil { + return fmt.Errorf("error while creating serviceaccount: %v", err) + } + + if err := m.framework.ApplyTemplatedManifests(clusterRoleFilePath, templateMap); err != nil { + return fmt.Errorf("error while creating clusterrole: %v", err) + } + + if err := m.framework.ApplyTemplatedManifests(clusterRoleBindingFilePath, templateMap); err != nil { + return fmt.Errorf("error while creating clusterrolebinding: %v", err) + } + + return nil +} + +func (m *dnsPerfK8sHostnamesMeasurement) createDNSClientDeployment() error { + templateMap := map[string]interface{}{ + "Namespace": m.testClientNamespace, + "PodReplicas": m.podReplicas, + "QPSPerClient": m.qpsPerClient, + "ServiceAccountName": dnsPerfTestPermissionsName, + } + + return m.framework.ApplyTemplatedManifests(clientDeploymentFilePath, templateMap) +} + +func (m *dnsPerfK8sHostnamesMeasurement) deleteDNSClientPermissions() error { + klog.Infof("Deleting DNS client permission resources for measurement %q", dnsPerfK8sHostnamesMeasureName) + + if err := m.k8sClient.CoreV1().ServiceAccounts(m.testClientNamespace).Delete(context.TODO(), dnsPerfTestPermissionsName, metav1.DeleteOptions{}); err != nil { + return err + } + + if err := m.k8sClient.RbacV1().ClusterRoles().Delete(context.TODO(), dnsPerfTestPermissionsName, metav1.DeleteOptions{}); err != nil { + return err + } + + return m.k8sClient.RbacV1().ClusterRoleBindings().Delete(context.TODO(), dnsPerfTestPermissionsName, metav1.DeleteOptions{}) +} + +func (m *dnsPerfK8sHostnamesMeasurement) cleanUp() error { + if m.framework == nil { + klog.Warning("Cleanup skipped. The measurement is not running") + return nil + } + + if err := m.deleteDNSClientPermissions(); err != nil { + return err + } + + klog.Infof("Deleting namespace %q for measurement %q", m.testClientNamespace, dnsPerfK8sHostnamesMeasureName) + return m.k8sClient.CoreV1().Namespaces().Delete(context.TODO(), m.testClientNamespace, metav1.DeleteOptions{}) +} + +// String returns a string representation of the measurement. +func (m *dnsPerfK8sHostnamesMeasurement) String() string { + return dnsPerfK8sHostnamesMeasureName +} + +// Dispose cleans up after the measurement. +func (m *dnsPerfK8sHostnamesMeasurement) Dispose() { + if err := m.cleanUp(); err != nil { + klog.Infof("Cleanup failed: %v", err) + } +} diff --git a/clusterloader2/pkg/measurement/common/dns/manifests/clusterrole.yaml b/clusterloader2/pkg/measurement/common/dns/manifests/clusterrole.yaml new file mode 100644 index 0000000000..57f669d8af --- /dev/null +++ b/clusterloader2/pkg/measurement/common/dns/manifests/clusterrole.yaml @@ -0,0 +1,11 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{.Name}} +rules: +- apiGroups: [""] + resources: ["services"] + verbs: ["create", "get", "list", "delete"] +- apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "list"] diff --git a/clusterloader2/pkg/measurement/common/dns/manifests/clusterrolebinding.yaml b/clusterloader2/pkg/measurement/common/dns/manifests/clusterrolebinding.yaml new file mode 100644 index 0000000000..b97b8a0571 --- /dev/null +++ b/clusterloader2/pkg/measurement/common/dns/manifests/clusterrolebinding.yaml @@ -0,0 +1,12 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{.Name}} +subjects: +- kind: ServiceAccount + name: {{.Name}} + namespace: {{.Namespace}} +roleRef: + kind: ClusterRole + name: {{.Name}} + apiGroup: rbac.authorization.k8s.io diff --git a/clusterloader2/pkg/measurement/common/dns/manifests/dns-client.yaml b/clusterloader2/pkg/measurement/common/dns/manifests/dns-client.yaml new file mode 100644 index 0000000000..1da8027b57 --- /dev/null +++ b/clusterloader2/pkg/measurement/common/dns/manifests/dns-client.yaml @@ -0,0 +1,53 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dnsperfgo + namespace: {{.Namespace}} +spec: + replicas: {{.PodReplicas}} + selector: + matchLabels: + dns-test: dnsperfgo + template: + metadata: + labels: + dns-test: dnsperfgo + spec: + containers: + - name: dnsperfgo + ports: + - containerPort: 9153 + name: dnsperfmetrics + protocol: TCP + image: gcr.io/k8s-staging-perf-tests/dnsperfgo:v1.3.0 + # Fetches the dns server from /etc/resolv.conf and sends 10 queries per second. + # With searchpath expansion, this is 120 queries per second. + # External names like google.com are expanded to 12 queries. + # FQDN lookups like kubernetes.default.svc.cluster.local are also expanded to 12 queries since they have < 5 dots in the name. + # Names like kubernetes.default will get partial search path expansion, since the query will succeed once "svc.cluster.local" path is applied. + # -query-cluster-names flag will generate FQDNs, in order to exercise searchpath expansion. + # dnsperf has a client timeout of 5s. It sends queries for 60s, + # then sleeps for 10s, to mimic bursts of DNS queries. + command: + - sh + - -c + - server=$(cat /etc/resolv.conf | grep nameserver | cut -d ' ' -f 2); echo + "Using nameserver ${server}"; + ./dnsperfgo -duration 60s -idle-duration 10s -query-cluster-names -qps {{.QPSPerClient}}; + resources: + requests: + cpu: 10m + memory: 10M + serviceAccountName: {{.ServiceAccountName}} + terminationGracePeriodSeconds: 1 + # Add not-ready/unreachable tolerations for 15 minutes so that node + # failure doesn't trigger pod deletion. + tolerations: + - key: "node.kubernetes.io/not-ready" + operator: "Exists" + effect: "NoExecute" + tolerationSeconds: 900 + - key: "node.kubernetes.io/unreachable" + operator: "Exists" + effect: "NoExecute" + tolerationSeconds: 900 diff --git a/clusterloader2/pkg/measurement/common/dns/manifests/serviceaccount.yaml b/clusterloader2/pkg/measurement/common/dns/manifests/serviceaccount.yaml new file mode 100644 index 0000000000..945171fed2 --- /dev/null +++ b/clusterloader2/pkg/measurement/common/dns/manifests/serviceaccount.yaml @@ -0,0 +1,5 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: {{.Name}} + namespace: {{.Namespace}}