Merge pull request #1196 from xial-thu/improvement-check-labels-for-m…

…aster-node

Clusterloader2: check labels additionally to decide master node

clusterloader2/pkg/measurement/common/scheduler_latency.go

@@ -86,7 +86,7 @@ func (s *schedulerLatencyMeasurement) Execute(config *measurement.MeasurementCon
 
 	var masterRegistered = false
 	for _, node := range nodes.Items {
-		if util.LegacyIsMasterNode(node.Name) {
+		if util.LegacyIsMasterNode(&node) {
 			masterRegistered = true
 		}
 	}

clusterloader2/pkg/measurement/util/gatherers/container_resource_gatherer.go

@@ -25,6 +25,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/klog"
 	"k8s.io/perf-tests/clusterloader2/pkg/measurement/util"
@@ -103,12 +104,25 @@ func NewResourceUsageGatherer(c clientset.Interface, host string, port int, prov
 				return nil, fmt.Errorf("listing pods error: %v", err)
 			}
 		}
+
+		nodeList, err := c.CoreV1().Nodes().List(metav1.ListOptions{})
+		if err != nil {
+			return nil, fmt.Errorf("listing nodes error: %v", err)
+		}
+
+		masterNodes := sets.NewString()
+		for _, node := range nodeList.Items {
+			if pkgutil.LegacyIsMasterNode(&node) {
+				masterNodes.Insert(node.Name)
+			}
+		}
+
 		dnsNodes := make(map[string]bool)
 		for _, pod := range pods.Items {
-			if (options.Nodes == MasterNodes) && !pkgutil.LegacyIsMasterNode(pod.Spec.NodeName) {
+			if (options.Nodes == MasterNodes) && !masterNodes.Has(pod.Spec.NodeName) {
 				continue
 			}
-			if (options.Nodes == MasterAndDNSNodes) && !pkgutil.LegacyIsMasterNode(pod.Spec.NodeName) && pod.Labels["k8s-app"] != "kube-dns" {
+			if (options.Nodes == MasterAndDNSNodes) && !masterNodes.Has(pod.Spec.NodeName) && pod.Labels["k8s-app"] != "kube-dns" {
 				continue
 			}
 			for _, container := range pod.Status.InitContainerStatuses {
@@ -121,16 +135,12 @@ func NewResourceUsageGatherer(c clientset.Interface, host string, port int, prov
 				dnsNodes[pod.Spec.NodeName] = true
 			}
 		}
-		nodeList, err := c.CoreV1().Nodes().List(metav1.ListOptions{})
-		if err != nil {
-			return nil, fmt.Errorf("listing nodes error: %v", err)
-		}
 
 		for _, node := range nodeList.Items {
-			if options.Nodes == AllNodes || pkgutil.LegacyIsMasterNode(node.Name) || dnsNodes[node.Name] {
+			if options.Nodes == AllNodes || masterNodes.Has(node.Name) || dnsNodes[node.Name] {
 				g.workerWg.Add(1)
 				resourceDataGatheringPeriod := options.ResourceDataGatheringPeriod
-				if pkgutil.LegacyIsMasterNode(node.Name) {
+				if masterNodes.Has(node.Name) {
 					resourceDataGatheringPeriod = options.MasterResourceDataGatheringPeriod
 				}
 				g.workers = append(g.workers, resourceGatherWorker{

clusterloader2/pkg/prometheus/prometheus.go

@@ -277,7 +277,7 @@ func (pc *PrometheusController) runNodeExporter() error {
 	numMasters := 0
 	for _, node := range nodes {
 		node := node
-		if util.LegacyIsMasterNode(node.Name) {
+		if util.LegacyIsMasterNode(&node) {
 			numMasters++
 			g.Go(func() error {
 				f, err := os.Open(os.ExpandEnv(nodeExporterPod))

clusterloader2/pkg/util/cluster.go

@@ -26,6 +26,8 @@ import (
 	"k8s.io/perf-tests/clusterloader2/pkg/framework/client"
 )
 
+const keyMasterNodeLabel = "node-role.kubernetes.io/master"
+
 // GetSchedulableUntainedNodesNumber returns number of nodes in the cluster.
 func GetSchedulableUntainedNodesNumber(c clientset.Interface) (int, error) {
 	nodes, err := GetSchedulableUntainedNodes(c)
@@ -129,7 +131,7 @@ func GetMasterName(c clientset.Interface) (string, error) {
 		return "", err
 	}
 	for i := range nodeList {
-		if LegacyIsMasterNode(nodeList[i].Name) {
+		if LegacyIsMasterNode(&nodeList[i]) {
 			return nodeList[i].Name, nil
 		}
 	}
@@ -144,7 +146,7 @@ func GetMasterIPs(c clientset.Interface, addressType corev1.NodeAddressType) ([]
 	}
 	var ips []string
 	for i := range nodeList {
-		if LegacyIsMasterNode(nodeList[i].Name) {
+		if LegacyIsMasterNode(&nodeList[i]) {
 			for _, address := range nodeList[i].Status.Addresses {
 				if address.Type == addressType && address.Address != "" {
 					ips = append(ips, address.Address)
@@ -165,11 +167,18 @@ func GetMasterIPs(c clientset.Interface, addressType corev1.NodeAddressType) ([]
 // This code will not be allowed to update to use the node-role label, since
 // node-roles may not be used for feature enablement.
 // DEPRECATED: this will be removed in Kubernetes 1.19
-func LegacyIsMasterNode(nodeName string) bool {
+func LegacyIsMasterNode(node *corev1.Node) bool {
+	for key, val := range node.GetLabels() {
+		if key == keyMasterNodeLabel {
+			return strings.ToLower(val) == "true"
+		}
+	}
+
 	// We are trying to capture "master(-...)?$" regexp.
 	// However, using regexp.MatchString() results even in more than 35%
 	// of all space allocations in ControllerManager spent in this function.
 	// That's why we are trying to be a bit smarter.
+	nodeName := node.GetName()
 	if strings.HasSuffix(nodeName, "master") {
 		return true
 	}