Adds authorization for sidecar-promclient (#4612)

* pull work from #4104 Signed-off-by: someshkoli <kolisomesh27@gmail.com> * update docs Signed-off-by: someshkoli <kolisomesh27@gmail.com> * fix fs perms Signed-off-by: someshkoli <kolisomesh27@gmail.com> * remove flags from exthttp, make prom http config configurable, Signed-off-by: someshkoli <kolisomesh27@gmail.com> * typo fix Signed-off-by: someshkoli <kolisomesh27@gmail.com> * adds method to set http client to realoder Signed-off-by: someshkoli <kolisomesh27@gmail.com> * Merge branch 'main' into feature/promclient-authorization Signed-off-by: Ben Ye <ben.ye@bytedance.com> * fix format Signed-off-by: Ben Ye <ben.ye@bytedance.com> * fix e2e Signed-off-by: Ben Ye <ben.ye@bytedance.com> * fix prom e2e test Signed-off-by: Ben Ye <ben.ye@bytedance.com> * update changelog Signed-off-by: Ben Ye <ben.ye@bytedance.com> * update dependency Signed-off-by: Ben Ye <ben.ye@bytedance.com> Co-authored-by: Ben Ye <ben.ye@bytedance.com>
thanos-io · Nov 9, 2021 · c98324b · c98324b
1 parent ac2cc26
commit c98324b
Show file tree

Hide file tree

Showing 16 changed files with 287 additions and 84 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -23,6 +23,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 - [#4444](https://github.com/thanos-io/thanos/pull/4444) UI: add mark deletion and no compaction to the Block UI.
 - [#4576](https://github.com/thanos-io/thanos/pull/4576) UI: add filter compaction level to the Block UI.
 - [#4731](https://github.com/thanos-io/thanos/pull/4731) Rule: add stateless mode to ruler according to https://thanos.io/tip/proposals-accepted/202005-scalable-rule-storage.md/. Continue https://github.com/thanos-io/thanos/pull/4250.
+- [#4612](https://github.com/thanos-io/thanos/pull/4612) Sidecar: add `--prometheus.http-client` and `--prometheus.http-client-file` flag for sidecar to connect Prometheus with basic auth or TLS.
 
 ### Fixed
 

diff --git a/cmd/thanos/config.go b/cmd/thanos/config.go
@@ -65,6 +65,7 @@ func (hc *httpConfig) registerFlag(cmd extkingpin.FlagClause) *httpConfig {
 type prometheusConfig struct {
 	url          *url.URL
 	readyTimeout time.Duration
+	httpClient   *extflag.PathOrContent
 }
 
 func (pc *prometheusConfig) registerFlag(cmd extkingpin.FlagClause) *prometheusConfig {
@@ -74,22 +75,13 @@ func (pc *prometheusConfig) registerFlag(cmd extkingpin.FlagClause) *prometheusC
 	cmd.Flag("prometheus.ready_timeout",
 		"Maximum time to wait for the Prometheus instance to start up").
 		Default("10m").DurationVar(&pc.readyTimeout)
-	return pc
-}
+	pc.httpClient = extflag.RegisterPathOrContent(
+		cmd,
+		"prometheus.http-client",
+		"YAML file or string with http client configs. see Format details : ...",
+	)
 
-type connConfig struct {
-	maxIdleConns        int
-	maxIdleConnsPerHost int
-}
-
-func (cc *connConfig) registerFlag(cmd extkingpin.FlagClause) *connConfig {
-	cmd.Flag("receive.connection-pool-size",
-		"Controls the http MaxIdleConns. Default is 0, which is unlimited").
-		IntVar(&cc.maxIdleConns)
-	cmd.Flag("receive.connection-pool-size-per-host",
-		"Controls the http MaxIdleConnsPerHost").
-		Default("100").IntVar(&cc.maxIdleConnsPerHost)
-	return cc
+	return pc
 }
 
 type tsdbConfig struct {

diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go
@@ -6,7 +6,6 @@ package main
 import (
 	"context"
 	"math"
-	"net/http"
 	"net/url"
 	"sync"
 	"time"
@@ -26,7 +25,6 @@ import (
 	"github.com/thanos-io/thanos/pkg/block/metadata"
 	"github.com/thanos-io/thanos/pkg/component"
 	"github.com/thanos-io/thanos/pkg/exemplars"
-	"github.com/thanos-io/thanos/pkg/exthttp"
 	"github.com/thanos-io/thanos/pkg/extkingpin"
 	"github.com/thanos-io/thanos/pkg/extprom"
 	"github.com/thanos-io/thanos/pkg/httpconfig"
@@ -49,7 +47,6 @@ import (
 	"github.com/thanos-io/thanos/pkg/store/labelpb"
 	"github.com/thanos-io/thanos/pkg/targets"
 	"github.com/thanos-io/thanos/pkg/tls"
-	"github.com/thanos-io/thanos/pkg/tracing"
 )
 
 func registerSidecar(app *extkingpin.App) {
@@ -88,6 +85,22 @@ func runSidecar(
 	grpcLogOpts []grpc_logging.Option,
 	tagOpts []tags.Option,
 ) error {
+	httpConfContentYaml, err := conf.prometheus.httpClient.Content()
+	if err != nil {
+		return errors.Wrap(err, "getting http client config")
+	}
+	httpClientConfig, err := httpconfig.NewClientConfigFromYAML(httpConfContentYaml)
+	if err != nil {
+		return errors.Wrap(err, "parsing http config YAML")
+	}
+
+	httpClient, err := httpconfig.NewHTTPClient(*httpClientConfig, "thanos-sidecar")
+	if err != nil {
+		return errors.Wrap(err, "Improper http client config")
+	}
+
+	reloader.SetHttpClient(*httpClient)
+
 	var m = &promMetadata{
 		promURL: conf.prometheus.url,
 
@@ -97,7 +110,7 @@ func runSidecar(
 		maxt: math.MaxInt64,
 
 		limitMinTime: conf.limitMinTime,
-		client:       promclient.NewWithTracingClient(logger, "thanos-sidecar"),
+		client:       promclient.NewWithTracingClient(logger, httpClient, "thanos-sidecar"),
 	}
 
 	confContentYaml, err := conf.objStore.Content()
@@ -229,10 +242,7 @@ func runSidecar(
 		})
 	}
 	{
-		t := exthttp.NewTransport()
-		t.MaxIdleConnsPerHost = conf.connection.maxIdleConnsPerHost
-		t.MaxIdleConns = conf.connection.maxIdleConns
-		c := promclient.NewClient(&http.Client{Transport: tracing.HTTPTripperware(logger, t)}, logger, httpconfig.ThanosUserAgent)
+		c := promclient.NewWithTracingClient(logger, httpClient, httpconfig.ThanosUserAgent)
 
 		promStore, err := store.NewPrometheusStore(logger, reg, c, conf.prometheus.url, component.Sidecar, m.Labels, m.Timestamps, m.Version)
 		if err != nil {
@@ -473,7 +483,6 @@ type sidecarConfig struct {
 	http         httpConfig
 	grpc         grpcConfig
 	prometheus   prometheusConfig
-	connection   connConfig
 	tsdb         tsdbConfig
 	reloader     reloaderConfig
 	reqLogConfig *extflag.PathOrContent
@@ -486,7 +495,6 @@ func (sc *sidecarConfig) registerFlag(cmd extkingpin.FlagClause) {
 	sc.http.registerFlag(cmd)
 	sc.grpc.registerFlag(cmd)
 	sc.prometheus.registerFlag(cmd)
-	sc.connection.registerFlag(cmd)
 	sc.tsdb.registerFlag(cmd)
 	sc.reloader.registerFlag(cmd)
 	sc.reqLogConfig = extkingpin.RegisterRequestLoggingFlags(cmd)

diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md
@@ -124,17 +124,20 @@ Flags:
                                  Path to YAML file that contains object store
                                  configuration. See format details:
                                  https://thanos.io/tip/thanos/storage.md/#configuration
+      --prometheus.http-client=<content>  
+                                 Alternative to 'prometheus.http-client-file'
+                                 flag (mutually exclusive). Content of YAML file
+                                 or string with http client configs. see Format
+                                 details : ...
+      --prometheus.http-client-file=<file-path>  
+                                 Path to YAML file or string with http client
+                                 configs. see Format details : ...
       --prometheus.ready_timeout=10m  
                                  Maximum time to wait for the Prometheus
                                  instance to start up
       --prometheus.url=http://localhost:9090  
                                  URL at which to reach Prometheus's API. For
                                  better performance use local network.
-      --receive.connection-pool-size=RECEIVE.CONNECTION-POOL-SIZE  
-                                 Controls the http MaxIdleConns. Default is 0,
-                                 which is unlimited
-      --receive.connection-pool-size-per-host=100  
-                                 Controls the http MaxIdleConnsPerHost
       --reloader.config-envsubst-file=""  
                                  Output file for environment variable
                                  substituted config file.

diff --git a/go.mod b/go.mod
@@ -70,6 +70,7 @@ require (
 	go.uber.org/automaxprocs v1.4.0
 	go.uber.org/goleak v1.1.12
 	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e
+	golang.org/x/net v0.0.0-20211020060615-d418f374d309
 	golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/text v0.3.6

diff --git a/pkg/httpconfig/http.go b/pkg/httpconfig/http.go
@@ -6,20 +6,24 @@ package httpconfig
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"net/http"
 	"net/url"
 	"path"
 	"sync"
+	"time"
 
 	extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http"
 
 	"github.com/go-kit/kit/log"
+	"github.com/mwitkow/go-conntrack"
 	config_util "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/version"
 	"github.com/prometheus/prometheus/discovery/file"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
+	"golang.org/x/net/http2"
 	"gopkg.in/yaml.v2"
 
 	"github.com/thanos-io/thanos/pkg/discovery/cache"
@@ -37,6 +41,8 @@ type ClientConfig struct {
 	ProxyURL string `yaml:"proxy_url"`
 	// TLSConfig to use to connect to the targets.
 	TLSConfig TLSConfig `yaml:"tls_config"`
+	// TransportConfig for Client transport properties
+	TransportConfig TransportConfig `yaml:"transport_config"`
 	// ClientMetrics contains metrics that will be used to instrument
 	// the client that will be created with this config.
 	ClientMetrics *extpromhttp.ClientMetrics `yaml:"-"`
@@ -68,6 +74,104 @@ func (b BasicAuth) IsZero() bool {
 	return b.Username == "" && b.Password == "" && b.PasswordFile == ""
 }
 
+// Transport configures client's transport properties.
+type TransportConfig struct {
+	MaxIdleConns          int  `yaml:"max_idle_conns"`
+	MaxIdleConnsPerHost   int  `yaml:"max_idle_conns_per_host"`
+	IdleConnTimeout       int  `yaml:"idle_conn_timeout"`
+	ResponseHeaderTimeout int  `yaml:"response_header_timeout"`
+	ExpectContinueTimeout int  `yaml:"expect_continue_timeout"`
+	MaxConnsPerHost       int  `yaml:"max_conns_per_host"`
+	DisableCompression    bool `yaml:"disable_compression"`
+	TLSHandshakeTimeout   int  `yaml:"tls_handshake_timeout"`
+}
+
+var defaultTransportConfig TransportConfig = TransportConfig{
+	MaxIdleConns:          100,
+	MaxIdleConnsPerHost:   2,
+	ResponseHeaderTimeout: 0,
+	MaxConnsPerHost:       0,
+	IdleConnTimeout:       int(90 * time.Second),
+	ExpectContinueTimeout: int(10 * time.Second),
+	DisableCompression:    false,
+	TLSHandshakeTimeout:   int(10 * time.Second),
+}
+
+func NewClientConfigFromYAML(cfg []byte) (*ClientConfig, error) {
+	conf := &ClientConfig{TransportConfig: defaultTransportConfig}
+	if err := yaml.Unmarshal(cfg, conf); err != nil {
+		return nil, err
+	}
+	return conf, nil
+}
+
+// NewRoundTripperFromConfig returns a new HTTP RoundTripper configured for the
+// given http.HTTPClientConfig and http.HTTPClientOption.
+func NewRoundTripperFromConfig(cfg config_util.HTTPClientConfig, transportConfig TransportConfig, name string) (http.RoundTripper, error) {
+	newRT := func(tlsConfig *tls.Config) (http.RoundTripper, error) {
+		var rt http.RoundTripper = &http.Transport{
+			Proxy:                 http.ProxyURL(cfg.ProxyURL.URL),
+			MaxIdleConns:          transportConfig.MaxIdleConns,
+			MaxIdleConnsPerHost:   transportConfig.MaxIdleConnsPerHost,
+			MaxConnsPerHost:       transportConfig.MaxConnsPerHost,
+			TLSClientConfig:       tlsConfig,
+			DisableCompression:    transportConfig.DisableCompression,
+			IdleConnTimeout:       time.Duration(transportConfig.IdleConnTimeout),
+			ResponseHeaderTimeout: time.Duration(transportConfig.ResponseHeaderTimeout),
+			ExpectContinueTimeout: time.Duration(transportConfig.ExpectContinueTimeout),
+			TLSHandshakeTimeout:   time.Duration(transportConfig.TLSHandshakeTimeout),
+			DialContext: conntrack.NewDialContextFunc(
+				conntrack.DialWithTracing(),
+				conntrack.DialWithName(name)),
+		}
+
+		// HTTP/2 support is golang has many problematic cornercases where
+		// dead connections would be kept and used in connection pools.
+		// https://github.com/golang/go/issues/32388
+		// https://github.com/golang/go/issues/39337
+		// https://github.com/golang/go/issues/39750
+		// TODO: Re-Enable HTTP/2 once upstream issue is fixed.
+		// TODO: use ForceAttemptHTTP2 when we move to Go 1.13+.
+		err := http2.ConfigureTransport(rt.(*http.Transport))
+		if err != nil {
+			return nil, err
+		}
+
+		// If a authorization_credentials is provided, create a round tripper that will set the
+		// Authorization header correctly on each request.
+		if cfg.Authorization != nil && len(cfg.Authorization.Credentials) > 0 {
+			rt = config_util.NewAuthorizationCredentialsRoundTripper(cfg.Authorization.Type, cfg.Authorization.Credentials, rt)
+		} else if cfg.Authorization != nil && len(cfg.Authorization.CredentialsFile) > 0 {
+			rt = config_util.NewAuthorizationCredentialsFileRoundTripper(cfg.Authorization.Type, cfg.Authorization.CredentialsFile, rt)
+		}
+		// Backwards compatibility, be nice with importers who would not have
+		// called Validate().
+		if len(cfg.BearerToken) > 0 {
+			rt = config_util.NewAuthorizationCredentialsRoundTripper("Bearer", cfg.BearerToken, rt)
+		} else if len(cfg.BearerTokenFile) > 0 {
+			rt = config_util.NewAuthorizationCredentialsFileRoundTripper("Bearer", cfg.BearerTokenFile, rt)
+		}
+
+		if cfg.BasicAuth != nil {
+			rt = config_util.NewBasicAuthRoundTripper(cfg.BasicAuth.Username, cfg.BasicAuth.Password, cfg.BasicAuth.PasswordFile, rt)
+		}
+		// Return a new configured RoundTripper.
+		return rt, nil
+	}
+
+	tlsConfig, err := config_util.NewTLSConfig(&cfg.TLSConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(cfg.TLSConfig.CAFile) == 0 {
+		// No need for a RoundTripper that reloads the CA file automatically.
+		return newRT(tlsConfig)
+	}
+
+	return config_util.NewTLSRoundTripper(tlsConfig, cfg.TLSConfig.CAFile, newRT)
+}
+
 // NewHTTPClient returns a new HTTP client.
 func NewHTTPClient(cfg ClientConfig, name string) (*http.Client, error) {
 	httpClientConfig := config_util.HTTPClientConfig{
@@ -96,22 +200,34 @@ func NewHTTPClient(cfg ClientConfig, name string) (*http.Client, error) {
 			PasswordFile: cfg.BasicAuth.PasswordFile,
 		}
 	}
+
+	if cfg.BearerToken != "" {
+		httpClientConfig.BearerToken = config_util.Secret(cfg.BearerToken)
+	}
+
+	if cfg.BearerTokenFile != "" {
+		httpClientConfig.BearerTokenFile = cfg.BearerTokenFile
+	}
+
 	if err := httpClientConfig.Validate(); err != nil {
 		return nil, err
 	}
 
-	client, err := config_util.NewClientFromConfig(httpClientConfig, name, config_util.WithHTTP2Disabled())
+	rt, err := NewRoundTripperFromConfig(
+		httpClientConfig,
+		cfg.TransportConfig,
+		name,
+	)
 	if err != nil {
 		return nil, err
 	}
 
-	tripper := client.Transport
-
 	if cfg.ClientMetrics != nil {
-		tripper = extpromhttp.InstrumentedRoundTripper(tripper, cfg.ClientMetrics)
+		rt = extpromhttp.InstrumentedRoundTripper(rt, cfg.ClientMetrics)
 	}
 
-	client.Transport = &userAgentRoundTripper{name: ThanosUserAgent, rt: tripper}
+	rt = &userAgentRoundTripper{name: ThanosUserAgent, rt: rt}
+	client := &http.Client{Transport: rt}
 
 	return client, nil
 }

diff --git a/pkg/promclient/promclient.go b/pkg/promclient/promclient.go
@@ -31,15 +31,17 @@ import (
 	"github.com/prometheus/prometheus/pkg/timestamp"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/promql/parser"
+	"google.golang.org/grpc/codes"
+	"gopkg.in/yaml.v2"
+
 	"github.com/thanos-io/thanos/pkg/exemplars/exemplarspb"
+	"github.com/thanos-io/thanos/pkg/httpconfig"
 	"github.com/thanos-io/thanos/pkg/metadata/metadatapb"
 	"github.com/thanos-io/thanos/pkg/rules/rulespb"
 	"github.com/thanos-io/thanos/pkg/runutil"
 	"github.com/thanos-io/thanos/pkg/store/storepb"
 	"github.com/thanos-io/thanos/pkg/targets/targetspb"
 	"github.com/thanos-io/thanos/pkg/tracing"
-	"google.golang.org/grpc/codes"
-	yaml "gopkg.in/yaml.v2"
 )
 
 var (
@@ -84,18 +86,19 @@ func NewClient(c HTTPClient, logger log.Logger, userAgent string) *Client {
 
 // NewDefaultClient returns Client with tracing tripperware.
 func NewDefaultClient() *Client {
+	client, _ := httpconfig.NewHTTPClient(httpconfig.ClientConfig{}, "")
 	return NewWithTracingClient(
 		log.NewNopLogger(),
+		client,
 		"",
 	)
 }
 
 // NewWithTracingClient returns client with tracing tripperware.
-func NewWithTracingClient(logger log.Logger, userAgent string) *Client {
+func NewWithTracingClient(logger log.Logger, httpClient *http.Client, userAgent string) *Client {
+	httpClient.Transport = tracing.HTTPTripperware(log.NewNopLogger(), httpClient.Transport)
 	return NewClient(
-		&http.Client{
-			Transport: tracing.HTTPTripperware(log.NewNopLogger(), http.DefaultTransport),
-		},
+		httpClient,
 		logger,
 		userAgent,
 	)