Dump instanceIDs for master nodes in clusterloader

clusterloader2/cmd/clusterloader.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/perf-tests/clusterloader2/pkg/flags"
 	"k8s.io/perf-tests/clusterloader2/pkg/framework"
 	"k8s.io/perf-tests/clusterloader2/pkg/imagepreload"
+	"k8s.io/perf-tests/clusterloader2/pkg/metadata"
 	"k8s.io/perf-tests/clusterloader2/pkg/modifier"
 	"k8s.io/perf-tests/clusterloader2/pkg/prometheus"
 	"k8s.io/perf-tests/clusterloader2/pkg/provider"
@@ -300,6 +301,10 @@ func main() {
 		klog.Exitf("Error while preloading images: %v", err)
 	}
 
+	if err := metadata.Dump(f, path.Join(clusterLoaderConfig.ReportDir, "cl2-metadata.json")); err != nil {
+		klog.Errorf("Error while dumping metadata: %v", err)
+	}
+
 	testReporter := test.CreateSimpleReporter(path.Join(clusterLoaderConfig.ReportDir, "junit.xml"), "ClusterLoaderV2")
 	testReporter.BeginTestSuite()
 	if testSuiteConfigPath != "" {

clusterloader2/pkg/metadata/metadata.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metadata
+
+import (
+	"encoding/json"
+	"io/ioutil"
+
+	"k8s.io/perf-tests/clusterloader2/pkg/framework"
+)
+
+// Dump writes file with data useful for debugging.
+// kubetest --metadata-sources should point to that file so that
+// it's merged into `finished.json` and available in prow and
+// bigquery tables.
+func Dump(f *framework.Framework, outputFile string) error {
+	output := make(map[string]string)
+
+	// Write any clusterloader-specific metadata here.
+
+	providerMetadata, err := f.GetClusterConfig().Provider.Metadata(f.GetClientSets().GetClient())
+	if err != nil {
+		return err
+	}
+
+	for k, v := range providerMetadata {
+		output[k] = v
+	}
+
+	return write(output, outputFile)
+}
+
+func write(obj map[string]string, outputFile string) error {
+	out, err := json.Marshal(obj)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(outputFile, out, 0644)
+}

clusterloader2/pkg/provider/aks.go

@@ -16,6 +16,10 @@ limitations under the License.
 
 package provider
 
+import (
+	clientset "k8s.io/client-go/kubernetes"
+)
+
 type AKSProvider struct {
 	features Features
 }
@@ -50,3 +54,7 @@ func (p *AKSProvider) GetConfig() Config {
 func (p *AKSProvider) RunSSHCommand(cmd, host string) (string, string, int, error) {
 	return runSSHCommand(cmd, host)
 }
+
+func (p *AKSProvider) Metadata(client clientset.Interface) (map[string]string, error) {
+	return nil, nil
+}

clusterloader2/pkg/provider/aws.go

@@ -17,6 +17,7 @@ limitations under the License.
 package provider
 
 import (
+	clientset "k8s.io/client-go/kubernetes"
 	sshutil "k8s.io/kubernetes/pkg/ssh"
 )
 
@@ -61,3 +62,7 @@ func (p *AWSProvider) RunSSHCommand(cmd, host string) (string, string, int, erro
 	user := defaultSSHUser()
 	return sshutil.RunSSHCommand(cmd, user, host, signer)
 }
+
+func (p *AWSProvider) Metadata(client clientset.Interface) (map[string]string, error) {
+	return nil, nil
+}

clusterloader2/pkg/provider/eks.go

@@ -17,6 +17,7 @@ limitations under the License.
 package provider
 
 import (
+	clientset "k8s.io/client-go/kubernetes"
 	sshutil "k8s.io/kubernetes/pkg/ssh"
 )
 
@@ -60,3 +61,7 @@ func (p *EKSProvider) RunSSHCommand(cmd, host string) (string, string, int, erro
 	user := defaultSSHUser()
 	return sshutil.RunSSHCommand(cmd, user, host, signer)
 }
+
+func (p *EKSProvider) Metadata(client clientset.Interface) (map[string]string, error) {
+	return nil, nil
+}

clusterloader2/pkg/provider/gce.go

@@ -17,7 +17,15 @@ limitations under the License.
 package provider
 
 import (
+	"fmt"
+	"os/exec"
+	"strings"
+
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/klog"
 	sshutil "k8s.io/kubernetes/pkg/ssh"
+	"k8s.io/perf-tests/clusterloader2/pkg/framework/client"
+	"k8s.io/perf-tests/clusterloader2/pkg/util"
 )
 
 type GCEProvider struct {
@@ -64,3 +72,35 @@ func (p *GCEProvider) RunSSHCommand(cmd, host string) (string, string, int, erro
 	user := defaultSSHUser()
 	return sshutil.RunSSHCommand(cmd, user, host, signer)
 }
+
+func (p *GCEProvider) Metadata(c clientset.Interface) (map[string]string, error) {
+	nodes, err := client.ListNodes(c)
+	if err != nil {
+		return nil, err
+	}
+
+	var masterInstanceIDs []string
+	for _, node := range nodes {
+		if util.LegacyIsMasterNode(&node) {
+			zone, ok := node.Labels["failure-domain.beta.kubernetes.io/zone"]
+			if !ok {
+				return nil, fmt.Errorf("unknown zone for %q node: no failure-domain.beta.kubernetes.io/zone label", node.Name)
+			}
+			cmd := exec.Command("gcloud", "compute", "instances", "describe", "--format", "value(id)", "--zone", zone, node.Name)
+			out, err := cmd.Output()
+			if err != nil {
+				var stderr string
+
+				if ee, ok := err.(*exec.ExitError); ok {
+					stderr = string(ee.Stderr)
+				}
+				return nil, fmt.Errorf("fetching instanceID for %q failed with: %v (stderr: %q)", node.Name, err, stderr)
+			}
+			instanceID := strings.TrimSpace(string(out))
+			klog.Infof("Detected instanceID for %s/%s: %q", zone, node.Name, instanceID)
+			masterInstanceIDs = append(masterInstanceIDs, instanceID)
+		}
+	}
+
+	return map[string]string{"masterInstanceIDs": strings.Join(masterInstanceIDs, ",")}, nil
+}

clusterloader2/pkg/provider/gke.go

@@ -17,6 +17,7 @@ limitations under the License.
 package provider
 
 import (
+	clientset "k8s.io/client-go/kubernetes"
 	sshutil "k8s.io/kubernetes/pkg/ssh"
 )
 
@@ -63,3 +64,7 @@ func (p *GKEProvider) RunSSHCommand(cmd, host string) (string, string, int, erro
 	user := defaultSSHUser()
 	return sshutil.RunSSHCommand(cmd, user, host, signer)
 }
+
+func (p *GKEProvider) Metadata(client clientset.Interface) (map[string]string, error) {
+	return nil, nil
+}

clusterloader2/pkg/provider/kubemark.go

@@ -17,6 +17,7 @@ limitations under the License.
 package provider
 
 import (
+	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/klog"
 	sshutil "k8s.io/kubernetes/pkg/ssh"
 )
@@ -67,3 +68,8 @@ func (p *KubemarkProvider) RunSSHCommand(cmd, host string) (string, string, int,
 	user := defaultSSHUser()
 	return sshutil.RunSSHCommand(cmd, user, host, signer)
 }
+
+// TODO(mborsz): Dump instanceIDs for master nodes (as in gce).
+func (p *KubemarkProvider) Metadata(client clientset.Interface) (map[string]string, error) {
+	return nil, nil
+}

clusterloader2/pkg/provider/local.go

@@ -17,6 +17,7 @@ limitations under the License.
 package provider
 
 import (
+	clientset "k8s.io/client-go/kubernetes"
 	sshutil "k8s.io/kubernetes/pkg/ssh"
 )
 
@@ -61,3 +62,7 @@ func (p *LocalProvider) RunSSHCommand(cmd, host string) (string, string, int, er
 	user := defaultSSHUser()
 	return sshutil.RunSSHCommand(cmd, user, host, signer)
 }
+
+func (p *LocalProvider) Metadata(client clientset.Interface) (map[string]string, error) {
+	return nil, nil
+}

clusterloader2/pkg/provider/provider.go

@@ -19,6 +19,8 @@ package provider
 import (
 	"fmt"
 	"strings"
+
+	clientset "k8s.io/client-go/kubernetes"
 )
 
 // InitOptions encapsulates the fields needed to init provider.
@@ -79,6 +81,9 @@ type Provider interface {
 	GetComponentProtocolAndPort(componentName string) (string, int, error)
 
 	RunSSHCommand(cmd, host string) (string, string, int, error)
+
+	// Metadata returns provider-specific test run metadata.
+	Metadata(client clientset.Interface) (map[string]string, error)
 }
 
 // NewProvider creates a new provider from init options. It will return an error if provider name is not supported.

clusterloader2/pkg/provider/skeleton.go

@@ -17,6 +17,7 @@ limitations under the License.
 package provider
 
 import (
+	clientset "k8s.io/client-go/kubernetes"
 	sshutil "k8s.io/kubernetes/pkg/ssh"
 )
 
@@ -65,3 +66,7 @@ func (p *SkeletonProvider) RunSSHCommand(cmd, host string) (string, string, int,
 	user := defaultSSHUser()
 	return sshutil.RunSSHCommand(cmd, user, host, signer)
 }
+
+func (p *SkeletonProvider) Metadata(client clientset.Interface) (map[string]string, error) {
+	return nil, nil
+}

clusterloader2/pkg/provider/vsphere.go

@@ -17,6 +17,7 @@ limitations under the License.
 package provider
 
 import (
+	clientset "k8s.io/client-go/kubernetes"
 	sshutil "k8s.io/kubernetes/pkg/ssh"
 )
 
@@ -61,3 +62,7 @@ func (p *VsphereProvider) RunSSHCommand(cmd, host string) (string, string, int,
 	user := defaultSSHUser()
 	return sshutil.RunSSHCommand(cmd, user, host, signer)
 }
+
+func (p *VsphereProvider) Metadata(client clientset.Interface) (map[string]string, error) {
+	return nil, nil
+}