Add GCP Compute API quotas usage measurement

clusterloader2/go.mod

@@ -45,7 +45,8 @@ require (
 	github.com/prometheus/prometheus v1.8.2-0.20210331101223-3cafc58827d1
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.7.0
-	golang.org/x/net v0.0.0-20211020060615-d418f374d309 // indirect
+	golang.org/x/net v0.0.0-20211020060615-d418f374d309
+	golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac

clusterloader2/pkg/measurement/common/probes/probes.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/perf-tests/clusterloader2/pkg/measurement"
 	measurementutil "k8s.io/perf-tests/clusterloader2/pkg/measurement/util"
 	"k8s.io/perf-tests/clusterloader2/pkg/prometheus"
+	prom "k8s.io/perf-tests/clusterloader2/pkg/prometheus/clients"
 	"k8s.io/perf-tests/clusterloader2/pkg/util"
 )
 
@@ -203,7 +204,8 @@ func (p *probesMeasurement) gather(params map[string]interface{}) (measurement.S
 	measurementEnd := time.Now()
 
 	query := prepareQuery(p.config.Query, p.startTime, measurementEnd)
-	executor := measurementutil.NewQueryExecutor(p.framework.GetClientSets().GetClient())
+	pc := prom.NewInClusterPrometheusClient(p.framework.GetClientSets().GetClient())
+	executor := measurementutil.NewQueryExecutor(pc)
 	samples, err := executor.Query(query, measurementEnd)
 	if err != nil {
 		return nil, err

clusterloader2/pkg/measurement/common/prometheus_measurement.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/perf-tests/clusterloader2/pkg/errors"
 	"k8s.io/perf-tests/clusterloader2/pkg/measurement"
 	measurementutil "k8s.io/perf-tests/clusterloader2/pkg/measurement/util"
+	prom "k8s.io/perf-tests/clusterloader2/pkg/prometheus/clients"
 	"k8s.io/perf-tests/clusterloader2/pkg/util"
 )
 
@@ -56,7 +57,14 @@ type prometheusMeasurement struct {
 }
 
 func (m *prometheusMeasurement) Execute(config *measurement.Config) ([]measurement.Summary, error) {
-	if config.PrometheusFramework == nil {
+	prometheusClient, err := util.GetStringOrDefault(config.Params, "prometheusClient", "inCluster")
+	if err != nil {
+		return nil, err
+	}
+	if prometheusClient != "inCluster" && prometheusClient != "managed" {
+		return nil, fmt.Errorf("unknown Prometheus client")
+	}
+	if prometheusClient == "inCluster" && config.PrometheusFramework == nil {
 		klog.Warningf("%s: Prometheus is disabled, skipping the measurement!", config.Identifier)
 		return nil, nil
 	}
@@ -86,8 +94,17 @@ func (m *prometheusMeasurement) Execute(config *measurement.Config) ([]measureme
 			return nil, err
 		}
 
-		c := config.PrometheusFramework.GetClientSets().GetClient()
-		executor := measurementutil.NewQueryExecutor(c)
+		var pc prom.Client
+		switch prometheusClient {
+		case "inCluster":
+			pc = prom.NewInClusterPrometheusClient(config.PrometheusFramework.GetClientSets().GetClient())
+		case "managed":
+			pc, err = config.CloudProvider.GetManagedPrometheusClient()
+			if err != nil {
+				return nil, fmt.Errorf("error while creating managed Prometheus client: %w", err)
+			}
+		}
+		executor := measurementutil.NewQueryExecutor(pc)
 
 		summary, err := m.gatherer.Gather(executor, m.startTime, time.Now(), config)
 		if err != nil {

clusterloader2/pkg/measurement/util/prometheus.go

@@ -17,7 +17,6 @@ limitations under the License.
 package util
 
 import (
-	"context"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -28,8 +27,8 @@ import (
 	"github.com/prometheus/common/expfmt"
 	"github.com/prometheus/common/model"
 	"k8s.io/apimachinery/pkg/util/wait"
-	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/klog"
+	prom "k8s.io/perf-tests/clusterloader2/pkg/prometheus/clients"
 )
 
 const (
@@ -85,13 +84,13 @@ type promResponseData struct {
 }
 
 // NewQueryExecutor creates instance of PrometheusQueryExecutor.
-func NewQueryExecutor(c clientset.Interface) *PrometheusQueryExecutor {
-	return &PrometheusQueryExecutor{client: c}
+func NewQueryExecutor(pc prom.Client) *PrometheusQueryExecutor {
+	return &PrometheusQueryExecutor{client: pc}
 }
 
-// PrometheusQueryExecutor executes queries against Prometheus instance running inside test cluster.
+// PrometheusQueryExecutor executes queries against Prometheus.
 type PrometheusQueryExecutor struct {
-	client clientset.Interface
+	client prom.Client
 }
 
 // Query executes given prometheus query at given point in time.
@@ -102,16 +101,10 @@ func (e *PrometheusQueryExecutor) Query(query string, queryTime time.Time) ([]*m
 
 	var body []byte
 	var queryErr error
-	params := map[string]string{
-		"query": query,
-		"time":  queryTime.Format(time.RFC3339),
-	}
+
 	klog.V(2).Infof("Executing %q at %v", query, queryTime.Format(time.RFC3339))
 	if err := wait.PollImmediate(queryInterval, queryTimeout, func() (bool, error) {
-		body, queryErr = e.client.CoreV1().
-			Services("monitoring").
-			ProxyGet("http", "prometheus-k8s", "9090", "api/v1/query", params).
-			DoRaw(context.TODO())
+		body, queryErr = e.client.Query(query, queryTime)
 		if queryErr != nil {
 			return false, nil
 		}
@@ -129,7 +122,7 @@ func (e *PrometheusQueryExecutor) Query(query string, queryTime time.Time) ([]*m
 
 	samples, err := ExtractMetricSamples2(body)
 	if err != nil {
-		return nil, fmt.Errorf("exctracting error: %v", err)
+		return nil, fmt.Errorf("extracting error: %v", err)
 	}
 
 	var resultSamples []*model.Sample

clusterloader2/pkg/prometheus/clients/gcp_managed.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package prom
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"time"
+
+	"golang.org/x/oauth2/google"
+)
+
+// gcpManagedPrometheusClient talks to the Google Cloud Managed Service for Prometheus.
+// This only works if the cluster is hosted in GCP.
+// Details: https://cloud.google.com/stackdriver/docs/managed-prometheus.
+type gcpManagedPrometheusClient struct {
+	client *http.Client
+	uri    string
+}
+
+func (mpc *gcpManagedPrometheusClient) Query(query string, queryTime time.Time) ([]byte, error) {
+	params := url.Values{}
+	params.Add("query", query)
+	params.Add("time", queryTime.Format(time.RFC3339))
+	res, err := mpc.client.Get(mpc.uri + "?" + params.Encode())
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	resBody, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return nil, err
+	}
+	return resBody, nil
+}
+
+// NewGCPManagedPrometheusClient returns an HTTP client for talking to
+// the Google Cloud Managed Service for Prometheus.
+func NewGCPManagedPrometheusClient() (Client, error) {
+	client, err := google.DefaultClient(context.TODO(), "https://www.googleapis.com/auth/monitoring.read")
+	if err != nil {
+		return nil, err
+	}
+	return &gcpManagedPrometheusClient{
+		client: client,
+		uri:    fmt.Sprintf("https://monitoring.googleapis.com/v1/projects/%s/location/global/prometheus/api/v1/query", os.Getenv("PROJECT")),
+	}, nil
+}
+
+var _ Client = &gcpManagedPrometheusClient{}

clusterloader2/pkg/prometheus/clients/in_cluster.go

@@ -0,0 +1,46 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package prom
+
+import (
+	"context"
+	"time"
+
+	clientset "k8s.io/client-go/kubernetes"
+)
+
+// inClusterPrometheusClient talks to the Prometheus instance deployed in the test cluster.
+type inClusterPrometheusClient struct {
+	client clientset.Interface
+}
+
+func (icpc *inClusterPrometheusClient) Query(query string, queryTime time.Time) ([]byte, error) {
+	params := map[string]string{
+		"query": query,
+		"time":  queryTime.Format(time.RFC3339),
+	}
+	return icpc.client.CoreV1().
+		Services("monitoring").
+		ProxyGet("http", "prometheus-k8s", "9090", "api/v1/query", params).
+		DoRaw(context.TODO())
+}
+
+func NewInClusterPrometheusClient(c clientset.Interface) Client {
+	return &inClusterPrometheusClient{client: c}
+}
+
+var _ Client = &inClusterPrometheusClient{}

clusterloader2/pkg/prometheus/clients/interface.go

@@ -0,0 +1,28 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package prom
+
+import (
+	"time"
+)
+
+// Client provides interface for communicating with the Prometheus API.
+type Client interface {
+	// Query sends a GET request to Prometheus with the "query" field
+	// in the URL's query string set using the provided arguments.
+	Query(query string, queryTime time.Time) ([]byte, error)
+}

clusterloader2/pkg/provider/aks.go

@@ -18,6 +18,7 @@ package provider
 
 import (
 	clientset "k8s.io/client-go/kubernetes"
+	prom "k8s.io/perf-tests/clusterloader2/pkg/prometheus/clients"
 )
 
 type AKSProvider struct {
@@ -61,3 +62,7 @@ func (p *AKSProvider) RunSSHCommand(cmd, host string) (string, string, int, erro
 func (p *AKSProvider) Metadata(client clientset.Interface) (map[string]string, error) {
 	return nil, nil
 }
+
+func (p *AKSProvider) GetManagedPrometheusClient() (prom.Client, error) {
+	return nil, ErrNoManagedPrometheus
+}

clusterloader2/pkg/provider/autopilot.go

@@ -19,6 +19,7 @@ package provider
 import (
 	clientset "k8s.io/client-go/kubernetes"
 	sshutil "k8s.io/kubernetes/test/e2e/framework/ssh"
+	prom "k8s.io/perf-tests/clusterloader2/pkg/prometheus/clients"
 )
 
 type AutopilotProvider struct {
@@ -66,3 +67,7 @@ func (p *AutopilotProvider) RunSSHCommand(cmd, host string) (string, string, int
 func (p *AutopilotProvider) Metadata(client clientset.Interface) (map[string]string, error) {
 	return nil, nil
 }
+
+func (p *AutopilotProvider) GetManagedPrometheusClient() (prom.Client, error) {
+	return prom.NewGCPManagedPrometheusClient()
+}

clusterloader2/pkg/provider/aws.go

@@ -19,6 +19,7 @@ package provider
 import (
 	clientset "k8s.io/client-go/kubernetes"
 	sshutil "k8s.io/kubernetes/test/e2e/framework/ssh"
+	prom "k8s.io/perf-tests/clusterloader2/pkg/prometheus/clients"
 )
 
 type AWSProvider struct {
@@ -64,3 +65,7 @@ func (p *AWSProvider) RunSSHCommand(cmd, host string) (string, string, int, erro
 func (p *AWSProvider) Metadata(client clientset.Interface) (map[string]string, error) {
 	return nil, nil
 }
+
+func (p *AWSProvider) GetManagedPrometheusClient() (prom.Client, error) {
+	return nil, ErrNoManagedPrometheus
+}

clusterloader2/pkg/provider/eks.go

@@ -19,6 +19,7 @@ package provider
 import (
 	clientset "k8s.io/client-go/kubernetes"
 	sshutil "k8s.io/kubernetes/test/e2e/framework/ssh"
+	prom "k8s.io/perf-tests/clusterloader2/pkg/prometheus/clients"
 )
 
 type EKSProvider struct {
@@ -63,3 +64,7 @@ func (p *EKSProvider) RunSSHCommand(cmd, host string) (string, string, int, erro
 func (p *EKSProvider) Metadata(client clientset.Interface) (map[string]string, error) {
 	return nil, nil
 }
+
+func (p *EKSProvider) GetManagedPrometheusClient() (prom.Client, error) {
+	return nil, ErrNoManagedPrometheus
+}

clusterloader2/pkg/provider/gce.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/klog"
 	sshutil "k8s.io/kubernetes/test/e2e/framework/ssh"
 	"k8s.io/perf-tests/clusterloader2/pkg/framework/client"
+	prom "k8s.io/perf-tests/clusterloader2/pkg/prometheus/clients"
 	"k8s.io/perf-tests/clusterloader2/pkg/util"
 )
 
@@ -108,3 +109,7 @@ func (p *GCEProvider) Metadata(c clientset.Interface) (map[string]string, error)
 
 	return map[string]string{"masterInstanceIDs": strings.Join(masterInstanceIDs, ",")}, nil
 }
+
+func (p *GCEProvider) GetManagedPrometheusClient() (prom.Client, error) {
+	return prom.NewGCPManagedPrometheusClient()
+}

clusterloader2/pkg/provider/gke.go

@@ -19,6 +19,7 @@ package provider
 import (
 	clientset "k8s.io/client-go/kubernetes"
 	sshutil "k8s.io/kubernetes/test/e2e/framework/ssh"
+	prom "k8s.io/perf-tests/clusterloader2/pkg/prometheus/clients"
 )
 
 type GKEProvider struct {
@@ -66,3 +67,7 @@ func (p *GKEProvider) RunSSHCommand(cmd, host string) (string, string, int, erro
 func (p *GKEProvider) Metadata(client clientset.Interface) (map[string]string, error) {
 	return nil, nil
 }
+
+func (p *GKEProvider) GetManagedPrometheusClient() (prom.Client, error) {
+	return prom.NewGCPManagedPrometheusClient()
+}

clusterloader2/pkg/provider/gke_kubemark.go

@@ -20,6 +20,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/klog"
 	sshutil "k8s.io/kubernetes/test/e2e/framework/ssh"
+	prom "k8s.io/perf-tests/clusterloader2/pkg/prometheus/clients"
 )
 
 type GKEKubemarkProvider struct {
@@ -71,3 +72,7 @@ func (p *GKEKubemarkProvider) RunSSHCommand(cmd, host string) (string, string, i
 func (p *GKEKubemarkProvider) Metadata(client clientset.Interface) (map[string]string, error) {
 	return nil, nil
 }
+
+func (p *GKEKubemarkProvider) GetManagedPrometheusClient() (prom.Client, error) {
+	return prom.NewGCPManagedPrometheusClient()
+}