Add measurement for dnsperfgo K8s hostnames test

clusterloader2/cmd/clusterloader.go

@@ -44,6 +44,7 @@ import (
 
 	_ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common"
 	_ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/bundle"
+	_ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/dns"
 	_ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/network"
 	_ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/probes"
 	_ "k8s.io/perf-tests/clusterloader2/pkg/measurement/common/slos"

clusterloader2/pkg/measurement/common/dns/dns_performance-k8s-hostnames.go

@@ -0,0 +1,210 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dns
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/klog"
+	"k8s.io/perf-tests/clusterloader2/pkg/framework"
+	"k8s.io/perf-tests/clusterloader2/pkg/framework/client"
+	"k8s.io/perf-tests/clusterloader2/pkg/measurement"
+	"k8s.io/perf-tests/clusterloader2/pkg/util"
+)
+
+/*
+The DNS Performance test for K8s Hostnames creates the required permissions for
+DNS client pods to get K8s hostnames (from services and endpoints). Then DNS
+client deployment is created that uses dnsperfgo image
+(https://github.com/kubernetes/perf-tests/tree/master/dns/dnsperfgo) to query
+the hostnames within the cluster and generate Prometheus metrics for number of
+errors, timeouts and their latencies. Outside this measurement, the metrics are
+scraped and results verified to not cross the specified thresholds.
+*/
+
+const (
+	dnsPerfK8sHostnamesMeasureName = "DNSPerformanceK8sHostnames"
+	dnsPerfTestNamespace           = "dns-perf-test"
+	dnsPerfTestPermissionsName     = "dns-test-client"
+	manifestPathPrefix             = "./pkg/measurement/common/dns/manifests"
+)
+
+var (
+	serviceAccountFilePath     = filepath.Join(manifestPathPrefix, "serviceaccount.yaml")
+	clusterRoleFilePath        = filepath.Join(manifestPathPrefix, "clusterrole.yaml")
+	clusterRoleBindingFilePath = filepath.Join(manifestPathPrefix, "clusterrolebinding.yaml")
+	clientDeploymentFilePath   = filepath.Join(manifestPathPrefix, "dns-client.yaml")
+)
+
+func init() {
+	klog.Info("Registering measurement: DNS Performance for K8s Hostnames")
+	if err := measurement.Register(dnsPerfK8sHostnamesMeasureName, createDNSPerfK8sHostnamesMeasurement); err != nil {
+		klog.Fatalf("Cannot register %s: %v", dnsPerfK8sHostnamesMeasureName, err)
+	}
+}
+
+func createDNSPerfK8sHostnamesMeasurement() measurement.Measurement {
+	return &dnsPerfK8sHostnamesMeasurement{}
+}
+
+type dnsPerfK8sHostnamesMeasurement struct {
+	k8sClient clientset.Interface
+	framework *framework.Framework
+	// testClientNamespace is the new namespace where the dns clients are going to
+	// be deployed. It's cleaned up when the test finishes.
+	testClientNamespace string
+	// podReplicas is the number of DNS client pods to be deployed.
+	podReplicas int
+	// qpsPerClient is the number of DNS queries each DNS client should send per
+	// second.
+	qpsPerClient int
+	// testDurationMinutes is the duration in minutes for DNS client pods to run.
+	testDurationMinutes int
+}
+
+func (m *dnsPerfK8sHostnamesMeasurement) Execute(config *measurement.Config) ([]measurement.Summary, error) {
+	err := m.initializeMeasurement(config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize the measurement: %v", err)
+	}
+
+	if err = client.CreateNamespace(m.k8sClient, m.testClientNamespace); err != nil {
+		return nil, fmt.Errorf("error while creating namespace: %v", err)
+	}
+
+	if err = m.createDNSClientPermissions(); err != nil {
+		return nil, fmt.Errorf("failed to create dns client permission resources: %v", err)
+	}
+
+	if err = m.createDNSClientDeployment(); err != nil {
+		return nil, fmt.Errorf("failed to create DNS client deployment: %v", err)
+	}
+
+	// Keep running the dns clients for the specified duration.
+	runDuration := time.Duration(m.testDurationMinutes) * time.Minute
+	klog.Infof("DNS tests are going to run for %v", runDuration)
+	time.Sleep(runDuration)
+
+	return nil, m.cleanUp()
+}
+
+func (m *dnsPerfK8sHostnamesMeasurement) initializeMeasurement(config *measurement.Config) error {
+	if m.framework != nil {
+		klog.Warningf("The %q was already started, but running it again", dnsPerfK8sHostnamesMeasureName)
+	}
+
+	var err error
+	if m.testClientNamespace, err = util.GetStringOrDefault(config.Params, "testNamespace", dnsPerfTestNamespace); err != nil {
+		return err
+	}
+
+	if m.podReplicas, err = util.GetIntOrDefault(config.Params, "podReplicas", 10); err != nil {
+		return err
+	}
+
+	if m.qpsPerClient, err = util.GetIntOrDefault(config.Params, "qpsPerClient", 10); err != nil {
+		return err
+	}
+
+	if m.testDurationMinutes, err = util.GetIntOrDefault(config.Params, "testDurationMinutes", 10); err != nil {
+		return err
+	}
+
+	m.framework = config.ClusterFramework
+	m.k8sClient = config.ClusterFramework.GetClientSets().GetClient()
+
+	return nil
+}
+
+// createDNSClientPermissions creates ServiceAccount, ClusterRole and
+// ClusterRoleBinding for the test client pods to access Services and Endpoints.
+func (m *dnsPerfK8sHostnamesMeasurement) createDNSClientPermissions() error {
+	templateMap := map[string]interface{}{
+		"Name":      dnsPerfTestPermissionsName,
+		"Namespace": m.testClientNamespace,
+	}
+
+	if err := m.framework.ApplyTemplatedManifests(serviceAccountFilePath, templateMap); err != nil {
+		return fmt.Errorf("error while creating serviceaccount: %v", err)
+	}
+
+	if err := m.framework.ApplyTemplatedManifests(clusterRoleFilePath, templateMap); err != nil {
+		return fmt.Errorf("error while creating clusterrole: %v", err)
+	}
+
+	if err := m.framework.ApplyTemplatedManifests(clusterRoleBindingFilePath, templateMap); err != nil {
+		return fmt.Errorf("error while creating clusterrolebinding: %v", err)
+	}
+
+	return nil
+}
+
+func (m *dnsPerfK8sHostnamesMeasurement) createDNSClientDeployment() error {
+	templateMap := map[string]interface{}{
+		"Namespace":          m.testClientNamespace,
+		"PodReplicas":        m.podReplicas,
+		"QPSPerClient":       m.qpsPerClient,
+		"ServiceAccountName": dnsPerfTestPermissionsName,
+	}
+
+	return m.framework.ApplyTemplatedManifests(clientDeploymentFilePath, templateMap)
+}
+
+func (m *dnsPerfK8sHostnamesMeasurement) deleteDNSClientPermissions() error {
+	klog.Infof("Deleting DNS client permission resources for measurement %q", dnsPerfK8sHostnamesMeasureName)
+
+	if err := m.k8sClient.CoreV1().ServiceAccounts(m.testClientNamespace).Delete(context.TODO(), dnsPerfTestPermissionsName, metav1.DeleteOptions{}); err != nil {
+		return err
+	}
+
+	if err := m.k8sClient.RbacV1().ClusterRoles().Delete(context.TODO(), dnsPerfTestPermissionsName, metav1.DeleteOptions{}); err != nil {
+		return err
+	}
+
+	return m.k8sClient.RbacV1().ClusterRoleBindings().Delete(context.TODO(), dnsPerfTestPermissionsName, metav1.DeleteOptions{})
+}
+
+func (m *dnsPerfK8sHostnamesMeasurement) cleanUp() error {
+	if m.framework == nil {
+		klog.Warning("Cleanup skipped. The measurement is not running")
+		return nil
+	}
+
+	if err := m.deleteDNSClientPermissions(); err != nil {
+		return err
+	}
+
+	klog.Infof("Deleting namespace %q for measurement %q", m.testClientNamespace, dnsPerfK8sHostnamesMeasureName)
+	return m.k8sClient.CoreV1().Namespaces().Delete(context.TODO(), m.testClientNamespace, metav1.DeleteOptions{})
+}
+
+// String returns a string representation of the measurement.
+func (m *dnsPerfK8sHostnamesMeasurement) String() string {
+	return dnsPerfK8sHostnamesMeasureName
+}
+
+// Dispose cleans up after the measurement.
+func (m *dnsPerfK8sHostnamesMeasurement) Dispose() {
+	if err := m.cleanUp(); err != nil {
+		klog.Infof("Cleanup failed: %v", err)
+	}
+}

clusterloader2/pkg/measurement/common/dns/manifests/clusterrole.yaml

@@ -0,0 +1,11 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{.Name}}
+rules:
+- apiGroups: [""]
+  resources: ["services"]
+  verbs: ["create", "get", "list", "delete"]
+- apiGroups: [""]
+  resources: ["endpoints"]
+  verbs: ["get", "list"]

clusterloader2/pkg/measurement/common/dns/manifests/clusterrolebinding.yaml

@@ -0,0 +1,12 @@
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{.Name}}
+subjects:
+- kind: ServiceAccount
+  name: {{.Name}}
+  namespace: {{.Namespace}}
+roleRef:
+  kind: ClusterRole
+  name: {{.Name}}
+  apiGroup: rbac.authorization.k8s.io

clusterloader2/pkg/measurement/common/dns/manifests/dns-client.yaml

@@ -0,0 +1,53 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dnsperfgo
+  namespace: {{.Namespace}}
+spec:
+  replicas: {{.PodReplicas}}
+  selector:
+    matchLabels:
+      dns-test: dnsperfgo
+  template:
+    metadata:
+      labels:
+        dns-test: dnsperfgo
+    spec:
+      containers:
+      - name: dnsperfgo
+        ports:
+        - containerPort: 9153
+          name: dnsperfmetrics
+          protocol: TCP
+        image: gcr.io/k8s-staging-perf-tests/dnsperfgo:v1.3.0
+        # Fetches the dns server from /etc/resolv.conf and sends 10 queries per second.
+        # With searchpath expansion, this is 120 queries per second.
+        # External names like google.com are expanded to 12 queries.
+        # FQDN lookups like kubernetes.default.svc.cluster.local are also expanded to 12 queries since they have < 5 dots in the name.
+        # Names like kubernetes.default will get partial search path expansion, since the query will succeed once "svc.cluster.local" path is applied.
+        # -query-cluster-names flag will generate FQDNs, in order to exercise searchpath expansion.
+        # dnsperf has a client timeout of 5s. It sends queries for 60s,
+        # then sleeps for 10s, to mimic bursts of DNS queries.
+        command:
+        - sh
+        - -c
+        - server=$(cat /etc/resolv.conf | grep nameserver | cut -d ' ' -f 2); echo
+          "Using nameserver ${server}";
+          ./dnsperfgo -duration 60s -idle-duration 10s -query-cluster-names -qps {{.QPSPerClient}};
+        resources:
+          requests:
+            cpu: 10m
+            memory: 10M
+      serviceAccountName: {{.ServiceAccountName}}
+      terminationGracePeriodSeconds: 1
+      # Add not-ready/unreachable tolerations for 15 minutes so that node
+      # failure doesn't trigger pod deletion.
+      tolerations:
+      - key: "node.kubernetes.io/not-ready"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 900
+      - key: "node.kubernetes.io/unreachable"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 900

clusterloader2/pkg/measurement/common/dns/manifests/serviceaccount.yaml

@@ -0,0 +1,5 @@
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: {{.Name}}
+  namespace: {{.Namespace}}