diff --git a/CHANGELOG.md b/CHANGELOG.md index a9b8ea52b1..86db448b80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#4444](https://github.com/thanos-io/thanos/pull/4444) UI: add mark deletion and no compaction to the Block UI. - [#4576](https://github.com/thanos-io/thanos/pull/4576) UI: add filter compaction level to the Block UI. - [#4731](https://github.com/thanos-io/thanos/pull/4731) Rule: add stateless mode to ruler according to https://thanos.io/tip/proposals-accepted/202005-scalable-rule-storage.md/. Continue https://github.com/thanos-io/thanos/pull/4250. +- [#4612](https://github.com/thanos-io/thanos/pull/4612) Sidecar: add `--prometheus.http-client` and `--prometheus.http-client-file` flag for sidecar to connect Prometheus with basic auth or TLS. ### Fixed diff --git a/cmd/thanos/config.go b/cmd/thanos/config.go index 3770389a68..8966a346f5 100644 --- a/cmd/thanos/config.go +++ b/cmd/thanos/config.go @@ -65,6 +65,7 @@ func (hc *httpConfig) registerFlag(cmd extkingpin.FlagClause) *httpConfig { type prometheusConfig struct { url *url.URL readyTimeout time.Duration + httpClient *extflag.PathOrContent } func (pc *prometheusConfig) registerFlag(cmd extkingpin.FlagClause) *prometheusConfig { @@ -74,22 +75,13 @@ func (pc *prometheusConfig) registerFlag(cmd extkingpin.FlagClause) *prometheusC cmd.Flag("prometheus.ready_timeout", "Maximum time to wait for the Prometheus instance to start up"). Default("10m").DurationVar(&pc.readyTimeout) - return pc -} + pc.httpClient = extflag.RegisterPathOrContent( + cmd, + "prometheus.http-client", + "YAML file or string with http client configs. see Format details : ...", + ) -type connConfig struct { - maxIdleConns int - maxIdleConnsPerHost int -} - -func (cc *connConfig) registerFlag(cmd extkingpin.FlagClause) *connConfig { - cmd.Flag("receive.connection-pool-size", - "Controls the http MaxIdleConns. Default is 0, which is unlimited"). - IntVar(&cc.maxIdleConns) - cmd.Flag("receive.connection-pool-size-per-host", - "Controls the http MaxIdleConnsPerHost"). - Default("100").IntVar(&cc.maxIdleConnsPerHost) - return cc + return pc } type tsdbConfig struct { diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 8584492b4f..6261738e7e 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -6,7 +6,6 @@ package main import ( "context" "math" - "net/http" "net/url" "sync" "time" @@ -26,7 +25,6 @@ import ( "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/exemplars" - "github.com/thanos-io/thanos/pkg/exthttp" "github.com/thanos-io/thanos/pkg/extkingpin" "github.com/thanos-io/thanos/pkg/extprom" "github.com/thanos-io/thanos/pkg/httpconfig" @@ -45,7 +43,6 @@ import ( "github.com/thanos-io/thanos/pkg/store" "github.com/thanos-io/thanos/pkg/targets" "github.com/thanos-io/thanos/pkg/tls" - "github.com/thanos-io/thanos/pkg/tracing" ) func registerSidecar(app *extkingpin.App) { @@ -84,6 +81,22 @@ func runSidecar( grpcLogOpts []grpc_logging.Option, tagOpts []tags.Option, ) error { + httpConfContentYaml, err := conf.prometheus.httpClient.Content() + if err != nil { + return errors.Wrap(err, "getting http client config") + } + httpClientConfig, err := httpconfig.NewClientConfigFromYAML(httpConfContentYaml) + if err != nil { + return errors.Wrap(err, "parsing http config YAML") + } + + httpClient, err := httpconfig.NewHTTPClient(*httpClientConfig, "thanos-sidecar") + if err != nil { + return errors.Wrap(err, "Improper http client config") + } + + reloader.SetHttpClient(*httpClient) + var m = &promMetadata{ promURL: conf.prometheus.url, @@ -93,7 +106,7 @@ func runSidecar( maxt: math.MaxInt64, limitMinTime: conf.limitMinTime, - client: promclient.NewWithTracingClient(logger, "thanos-sidecar"), + client: promclient.NewWithTracingClient(logger, httpClient, "thanos-sidecar"), } confContentYaml, err := conf.objStore.Content() @@ -225,10 +238,7 @@ func runSidecar( }) } { - t := exthttp.NewTransport() - t.MaxIdleConnsPerHost = conf.connection.maxIdleConnsPerHost - t.MaxIdleConns = conf.connection.maxIdleConns - c := promclient.NewClient(&http.Client{Transport: tracing.HTTPTripperware(logger, t)}, logger, httpconfig.ThanosUserAgent) + c := promclient.NewWithTracingClient(logger, httpClient, httpconfig.ThanosUserAgent) promStore, err := store.NewPrometheusStore(logger, reg, c, conf.prometheus.url, component.Sidecar, m.Labels, m.Timestamps, m.Version) if err != nil { @@ -435,7 +445,6 @@ type sidecarConfig struct { http httpConfig grpc grpcConfig prometheus prometheusConfig - connection connConfig tsdb tsdbConfig reloader reloaderConfig reqLogConfig *extflag.PathOrContent @@ -448,7 +457,6 @@ func (sc *sidecarConfig) registerFlag(cmd extkingpin.FlagClause) { sc.http.registerFlag(cmd) sc.grpc.registerFlag(cmd) sc.prometheus.registerFlag(cmd) - sc.connection.registerFlag(cmd) sc.tsdb.registerFlag(cmd) sc.reloader.registerFlag(cmd) sc.reqLogConfig = extkingpin.RegisterRequestLoggingFlags(cmd) diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index 0c6d6865f6..d15011298f 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -124,17 +124,20 @@ Flags: Path to YAML file that contains object store configuration. See format details: https://thanos.io/tip/thanos/storage.md/#configuration + --prometheus.http-client= + Alternative to 'prometheus.http-client-file' + flag (mutually exclusive). Content of YAML file + or string with http client configs. see Format + details : ... + --prometheus.http-client-file= + Path to YAML file or string with http client + configs. see Format details : ... --prometheus.ready_timeout=10m Maximum time to wait for the Prometheus instance to start up --prometheus.url=http://localhost:9090 URL at which to reach Prometheus's API. For better performance use local network. - --receive.connection-pool-size=RECEIVE.CONNECTION-POOL-SIZE - Controls the http MaxIdleConns. Default is 0, - which is unlimited - --receive.connection-pool-size-per-host=100 - Controls the http MaxIdleConnsPerHost --reloader.config-envsubst-file="" Output file for environment variable substituted config file. diff --git a/go.mod b/go.mod index aab8ce0199..a1123ffc0a 100644 --- a/go.mod +++ b/go.mod @@ -70,6 +70,7 @@ require ( go.uber.org/automaxprocs v1.4.0 go.uber.org/goleak v1.1.12 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e + golang.org/x/net v0.0.0-20211020060615-d418f374d309 golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/text v0.3.6 diff --git a/pkg/httpconfig/http.go b/pkg/httpconfig/http.go index b00204e425..9a2ea21927 100644 --- a/pkg/httpconfig/http.go +++ b/pkg/httpconfig/http.go @@ -6,20 +6,24 @@ package httpconfig import ( "context" + "crypto/tls" "fmt" "net/http" "net/url" "path" "sync" + "time" extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http" "github.com/go-kit/kit/log" + "github.com/mwitkow/go-conntrack" config_util "github.com/prometheus/common/config" "github.com/prometheus/common/model" "github.com/prometheus/common/version" "github.com/prometheus/prometheus/discovery/file" "github.com/prometheus/prometheus/discovery/targetgroup" + "golang.org/x/net/http2" "gopkg.in/yaml.v2" "github.com/thanos-io/thanos/pkg/discovery/cache" @@ -37,6 +41,8 @@ type ClientConfig struct { ProxyURL string `yaml:"proxy_url"` // TLSConfig to use to connect to the targets. TLSConfig TLSConfig `yaml:"tls_config"` + // TransportConfig for Client transport properties + TransportConfig TransportConfig `yaml:"transport_config"` // ClientMetrics contains metrics that will be used to instrument // the client that will be created with this config. ClientMetrics *extpromhttp.ClientMetrics `yaml:"-"` @@ -68,6 +74,104 @@ func (b BasicAuth) IsZero() bool { return b.Username == "" && b.Password == "" && b.PasswordFile == "" } +// Transport configures client's transport properties. +type TransportConfig struct { + MaxIdleConns int `yaml:"max_idle_conns"` + MaxIdleConnsPerHost int `yaml:"max_idle_conns_per_host"` + IdleConnTimeout int `yaml:"idle_conn_timeout"` + ResponseHeaderTimeout int `yaml:"response_header_timeout"` + ExpectContinueTimeout int `yaml:"expect_continue_timeout"` + MaxConnsPerHost int `yaml:"max_conns_per_host"` + DisableCompression bool `yaml:"disable_compression"` + TLSHandshakeTimeout int `yaml:"tls_handshake_timeout"` +} + +var defaultTransportConfig TransportConfig = TransportConfig{ + MaxIdleConns: 100, + MaxIdleConnsPerHost: 2, + ResponseHeaderTimeout: 0, + MaxConnsPerHost: 0, + IdleConnTimeout: int(90 * time.Second), + ExpectContinueTimeout: int(10 * time.Second), + DisableCompression: false, + TLSHandshakeTimeout: int(10 * time.Second), +} + +func NewClientConfigFromYAML(cfg []byte) (*ClientConfig, error) { + conf := &ClientConfig{TransportConfig: defaultTransportConfig} + if err := yaml.Unmarshal(cfg, conf); err != nil { + return nil, err + } + return conf, nil +} + +// NewRoundTripperFromConfig returns a new HTTP RoundTripper configured for the +// given http.HTTPClientConfig and http.HTTPClientOption. +func NewRoundTripperFromConfig(cfg config_util.HTTPClientConfig, transportConfig TransportConfig, name string) (http.RoundTripper, error) { + newRT := func(tlsConfig *tls.Config) (http.RoundTripper, error) { + var rt http.RoundTripper = &http.Transport{ + Proxy: http.ProxyURL(cfg.ProxyURL.URL), + MaxIdleConns: transportConfig.MaxIdleConns, + MaxIdleConnsPerHost: transportConfig.MaxIdleConnsPerHost, + MaxConnsPerHost: transportConfig.MaxConnsPerHost, + TLSClientConfig: tlsConfig, + DisableCompression: transportConfig.DisableCompression, + IdleConnTimeout: time.Duration(transportConfig.IdleConnTimeout), + ResponseHeaderTimeout: time.Duration(transportConfig.ResponseHeaderTimeout), + ExpectContinueTimeout: time.Duration(transportConfig.ExpectContinueTimeout), + TLSHandshakeTimeout: time.Duration(transportConfig.TLSHandshakeTimeout), + DialContext: conntrack.NewDialContextFunc( + conntrack.DialWithTracing(), + conntrack.DialWithName(name)), + } + + // HTTP/2 support is golang has many problematic cornercases where + // dead connections would be kept and used in connection pools. + // https://github.com/golang/go/issues/32388 + // https://github.com/golang/go/issues/39337 + // https://github.com/golang/go/issues/39750 + // TODO: Re-Enable HTTP/2 once upstream issue is fixed. + // TODO: use ForceAttemptHTTP2 when we move to Go 1.13+. + err := http2.ConfigureTransport(rt.(*http.Transport)) + if err != nil { + return nil, err + } + + // If a authorization_credentials is provided, create a round tripper that will set the + // Authorization header correctly on each request. + if cfg.Authorization != nil && len(cfg.Authorization.Credentials) > 0 { + rt = config_util.NewAuthorizationCredentialsRoundTripper(cfg.Authorization.Type, cfg.Authorization.Credentials, rt) + } else if cfg.Authorization != nil && len(cfg.Authorization.CredentialsFile) > 0 { + rt = config_util.NewAuthorizationCredentialsFileRoundTripper(cfg.Authorization.Type, cfg.Authorization.CredentialsFile, rt) + } + // Backwards compatibility, be nice with importers who would not have + // called Validate(). + if len(cfg.BearerToken) > 0 { + rt = config_util.NewAuthorizationCredentialsRoundTripper("Bearer", cfg.BearerToken, rt) + } else if len(cfg.BearerTokenFile) > 0 { + rt = config_util.NewAuthorizationCredentialsFileRoundTripper("Bearer", cfg.BearerTokenFile, rt) + } + + if cfg.BasicAuth != nil { + rt = config_util.NewBasicAuthRoundTripper(cfg.BasicAuth.Username, cfg.BasicAuth.Password, cfg.BasicAuth.PasswordFile, rt) + } + // Return a new configured RoundTripper. + return rt, nil + } + + tlsConfig, err := config_util.NewTLSConfig(&cfg.TLSConfig) + if err != nil { + return nil, err + } + + if len(cfg.TLSConfig.CAFile) == 0 { + // No need for a RoundTripper that reloads the CA file automatically. + return newRT(tlsConfig) + } + + return config_util.NewTLSRoundTripper(tlsConfig, cfg.TLSConfig.CAFile, newRT) +} + // NewHTTPClient returns a new HTTP client. func NewHTTPClient(cfg ClientConfig, name string) (*http.Client, error) { httpClientConfig := config_util.HTTPClientConfig{ @@ -96,22 +200,34 @@ func NewHTTPClient(cfg ClientConfig, name string) (*http.Client, error) { PasswordFile: cfg.BasicAuth.PasswordFile, } } + + if cfg.BearerToken != "" { + httpClientConfig.BearerToken = config_util.Secret(cfg.BearerToken) + } + + if cfg.BearerTokenFile != "" { + httpClientConfig.BearerTokenFile = cfg.BearerTokenFile + } + if err := httpClientConfig.Validate(); err != nil { return nil, err } - client, err := config_util.NewClientFromConfig(httpClientConfig, name, config_util.WithHTTP2Disabled()) + rt, err := NewRoundTripperFromConfig( + httpClientConfig, + cfg.TransportConfig, + name, + ) if err != nil { return nil, err } - tripper := client.Transport - if cfg.ClientMetrics != nil { - tripper = extpromhttp.InstrumentedRoundTripper(tripper, cfg.ClientMetrics) + rt = extpromhttp.InstrumentedRoundTripper(rt, cfg.ClientMetrics) } - client.Transport = &userAgentRoundTripper{name: ThanosUserAgent, rt: tripper} + rt = &userAgentRoundTripper{name: ThanosUserAgent, rt: rt} + client := &http.Client{Transport: rt} return client, nil } diff --git a/pkg/promclient/promclient.go b/pkg/promclient/promclient.go index 73043283b7..4bc6fb7657 100644 --- a/pkg/promclient/promclient.go +++ b/pkg/promclient/promclient.go @@ -31,15 +31,17 @@ import ( "github.com/prometheus/prometheus/pkg/timestamp" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/promql/parser" + "google.golang.org/grpc/codes" + "gopkg.in/yaml.v2" + "github.com/thanos-io/thanos/pkg/exemplars/exemplarspb" + "github.com/thanos-io/thanos/pkg/httpconfig" "github.com/thanos-io/thanos/pkg/metadata/metadatapb" "github.com/thanos-io/thanos/pkg/rules/rulespb" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/targets/targetspb" "github.com/thanos-io/thanos/pkg/tracing" - "google.golang.org/grpc/codes" - yaml "gopkg.in/yaml.v2" ) var ( @@ -84,18 +86,19 @@ func NewClient(c HTTPClient, logger log.Logger, userAgent string) *Client { // NewDefaultClient returns Client with tracing tripperware. func NewDefaultClient() *Client { + client, _ := httpconfig.NewHTTPClient(httpconfig.ClientConfig{}, "") return NewWithTracingClient( log.NewNopLogger(), + client, "", ) } // NewWithTracingClient returns client with tracing tripperware. -func NewWithTracingClient(logger log.Logger, userAgent string) *Client { +func NewWithTracingClient(logger log.Logger, httpClient *http.Client, userAgent string) *Client { + httpClient.Transport = tracing.HTTPTripperware(log.NewNopLogger(), httpClient.Transport) return NewClient( - &http.Client{ - Transport: tracing.HTTPTripperware(log.NewNopLogger(), http.DefaultTransport), - }, + httpClient, logger, userAgent, ) diff --git a/pkg/reloader/reloader.go b/pkg/reloader/reloader.go index 1e6c9fde7d..cd11ce2ab9 100644 --- a/pkg/reloader/reloader.go +++ b/pkg/reloader/reloader.go @@ -85,6 +85,7 @@ import ( type Reloader struct { logger log.Logger reloadURL *url.URL + httpClient http.Client cfgFile string cfgOutputFile string watchInterval time.Duration @@ -413,7 +414,7 @@ func (r *Reloader) triggerReload(ctx context.Context) error { } req = req.WithContext(ctx) - resp, err := http.DefaultClient.Do(req) + resp, err := r.httpClient.Do(req) if err != nil { return errors.Wrap(err, "reload request failed") } @@ -425,6 +426,11 @@ func (r *Reloader) triggerReload(ctx context.Context) error { return nil } +// SetHttpClient sets Http client for reloader. +func (r *Reloader) SetHttpClient(client http.Client) { + r.httpClient = client +} + // ReloadURLFromBase returns the standard Prometheus reload URL from its base URL. func ReloadURLFromBase(u *url.URL) *url.URL { r := *u diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index c6fe42d475..6cc0420865 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -65,17 +65,33 @@ func DefaultImage() string { return "thanos" } -func NewPrometheus(e e2e.Environment, name, config, promImage string, enableFeatures ...string) (*e2e.InstrumentedRunnable, string, error) { +func defaultPromHttpConfig() string { + // username: test, secret: test(bcrypt hash) + return `basic_auth: + username: test + password: test +` +} + +func NewPrometheus(e e2e.Environment, name, promConfig, webConfig, promImage string, enableFeatures ...string) (*e2e.InstrumentedRunnable, string, error) { dir := filepath.Join(e.SharedDir(), "data", "prometheus", name) container := filepath.Join(ContainerSharedDir, "data", "prometheus", name) if err := os.MkdirAll(dir, 0750); err != nil { return nil, "", errors.Wrap(err, "create prometheus dir") } - if err := ioutil.WriteFile(filepath.Join(dir, "prometheus.yml"), []byte(config), 0600); err != nil { + if err := ioutil.WriteFile(filepath.Join(dir, "prometheus.yml"), []byte(promConfig), 0600); err != nil { return nil, "", errors.Wrap(err, "creating prom config failed") } + if len(webConfig) > 0 { + if err := ioutil.WriteFile(filepath.Join(dir, "web-config.yml"), []byte(webConfig), 0600); err != nil { + return nil, "", errors.Wrap(err, "creating web-config failed") + } + } + + probe := e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200) + args := e2e.BuildArgs(map[string]string{ "--config.file": filepath.Join(container, "prometheus.yml"), "--storage.tsdb.path": container, @@ -87,6 +103,11 @@ func NewPrometheus(e e2e.Environment, name, config, promImage string, enableFeat if len(enableFeatures) > 0 { args = append(args, fmt.Sprintf("--enable-feature=%s", strings.Join(enableFeatures, ","))) } + if len(webConfig) > 0 { + args = append(args, fmt.Sprintf("--web.config.file=%s", filepath.Join(container, "web-config.yml"))) + // If auth is enabled then prober would get 401 error. + probe = e2e.NewHTTPReadinessProbe("http", "/-/ready", 401, 401) + } prom := e2e.NewInstrumentedRunnable( e, fmt.Sprintf("prometheus-%s", name), @@ -95,7 +116,7 @@ func NewPrometheus(e e2e.Environment, name, config, promImage string, enableFeat e2e.StartOptions{ Image: promImage, Command: e2e.NewCommandWithoutEntrypoint("prometheus", args...), - Readiness: e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), + Readiness: probe, User: strconv.Itoa(os.Getuid()), WaitReadyBackoff: &defaultBackoffConfig, }, @@ -104,29 +125,33 @@ func NewPrometheus(e e2e.Environment, name, config, promImage string, enableFeat return prom, container, nil } -func NewPrometheusWithSidecar(e e2e.Environment, name, config, promImage string, enableFeatures ...string) (*e2e.InstrumentedRunnable, *e2e.InstrumentedRunnable, error) { - return NewPrometheusWithSidecarCustomImage(e, name, config, promImage, DefaultImage(), enableFeatures...) +func NewPrometheusWithSidecar(e e2e.Environment, name, promConfig, webConfig, promImage string, enableFeatures ...string) (*e2e.InstrumentedRunnable, *e2e.InstrumentedRunnable, error) { + return NewPrometheusWithSidecarCustomImage(e, name, promConfig, webConfig, promImage, DefaultImage(), enableFeatures...) } -func NewPrometheusWithSidecarCustomImage(e e2e.Environment, name, config, promImage string, sidecarImage string, enableFeatures ...string) (*e2e.InstrumentedRunnable, *e2e.InstrumentedRunnable, error) { - prom, dataDir, err := NewPrometheus(e, name, config, promImage, enableFeatures...) +func NewPrometheusWithSidecarCustomImage(e e2e.Environment, name, promConfig, webConfig, promImage string, sidecarImage string, enableFeatures ...string) (*e2e.InstrumentedRunnable, *e2e.InstrumentedRunnable, error) { + prom, dataDir, err := NewPrometheus(e, name, promConfig, webConfig, promImage, enableFeatures...) if err != nil { return nil, nil, err } + args := map[string]string{ + "--debug.name": fmt.Sprintf("sidecar-%v", name), + "--grpc-address": ":9091", + "--grpc-grace-period": "0s", + "--http-address": ":8080", + "--prometheus.url": "http://" + prom.InternalEndpoint("http"), + "--tsdb.path": dataDir, + "--log.level": infoLogLevel, + } + if len(webConfig) > 0 { + args["--prometheus.http-client"] = defaultPromHttpConfig() + } sidecar := NewService( e, fmt.Sprintf("sidecar-%s", name), sidecarImage, - e2e.NewCommand("sidecar", e2e.BuildArgs(map[string]string{ - "--debug.name": fmt.Sprintf("sidecar-%v", name), - "--grpc-address": ":9091", - "--grpc-grace-period": "0s", - "--http-address": ":8080", - "--prometheus.url": "http://" + prom.InternalEndpoint("http"), - "--tsdb.path": dataDir, - "--log.level": infoLogLevel, - })...), + e2e.NewCommand("sidecar", e2e.BuildArgs(args)...), e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), 8080, 9091, diff --git a/test/e2e/exemplars_api_test.go b/test/e2e/exemplars_api_test.go index 13aaffca85..151fefd94e 100644 --- a/test/e2e/exemplars_api_test.go +++ b/test/e2e/exemplars_api_test.go @@ -43,6 +43,7 @@ func TestExemplarsAPI_Fanout(t *testing.T) { e, "prom1", defaultPromConfig("ha", 0, "", "", "localhost:9090", qUnitiated.InternalEndpoint("http")), + "", e2ethanos.DefaultPrometheusImage(), e2ethanos.FeatureExemplarStorage, ) @@ -51,6 +52,7 @@ func TestExemplarsAPI_Fanout(t *testing.T) { e, "prom2", defaultPromConfig("ha", 1, "", "", "localhost:9090", qUnitiated.InternalEndpoint("http")), + "", e2ethanos.DefaultPrometheusImage(), e2ethanos.FeatureExemplarStorage, ) diff --git a/test/e2e/metadata_api_test.go b/test/e2e/metadata_api_test.go index 096560e64e..901183c860 100644 --- a/test/e2e/metadata_api_test.go +++ b/test/e2e/metadata_api_test.go @@ -31,6 +31,7 @@ func TestMetadataAPI_Fanout(t *testing.T) { e, "prom1", defaultPromConfig("ha", 0, "", "", "localhost:9090", "sidecar-prom1:8080"), + "", e2ethanos.DefaultPrometheusImage(), ) testutil.Ok(t, err) @@ -39,6 +40,7 @@ func TestMetadataAPI_Fanout(t *testing.T) { e, "prom2", defaultPromConfig("ha", 1, "", "", "localhost:9090", "sidecar-prom2:8080"), + "", e2ethanos.DefaultPrometheusImage(), ) testutil.Ok(t, err) diff --git a/test/e2e/query_frontend_test.go b/test/e2e/query_frontend_test.go index d320c42327..82afb4a9b2 100644 --- a/test/e2e/query_frontend_test.go +++ b/test/e2e/query_frontend_test.go @@ -32,7 +32,7 @@ func TestQueryFrontend(t *testing.T) { now := time.Now() - prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(e, "1", defaultPromConfig("test", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(e, "1", defaultPromConfig("test", 0, "", ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom, sidecar)) @@ -396,7 +396,7 @@ func TestQueryFrontendMemcachedCache(t *testing.T) { now := time.Now() - prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(e, "1", defaultPromConfig("test", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(e, "1", defaultPromConfig("test", 0, "", ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom, sidecar)) diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index c2196636dd..312e74485b 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -88,6 +88,14 @@ rule_files: return config } +func defaultWebConfig() string { + // username: test, secret: test(bcrypt hash) + return ` +basic_auth_users: + test: $2y$10$IsC9GG9U61sPCuDwwwcnPuMRyzx62cIcdNRs4SIdKwgWihfX4IC.C +` +} + func sortResults(res model.Vector) { sort.Slice(res, func(i, j int) bool { return res[i].String() < res[j].String() @@ -106,13 +114,13 @@ func TestQuery(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(receiverRunnable)) - prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, "alone", defaultPromConfig("prom-alone", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, "alone", defaultPromConfig("prom-alone", 0, "", ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom3, sidecar3, err := e2ethanos.NewPrometheusWithSidecar(e, "ha1", defaultPromConfig("prom-ha", 0, "", filepath.Join(e2ethanos.ContainerSharedDir, "", "*.yaml")), e2ethanos.DefaultPrometheusImage()) + prom3, sidecar3, err := e2ethanos.NewPrometheusWithSidecar(e, "ha1", defaultPromConfig("prom-ha", 0, "", filepath.Join(e2ethanos.ContainerSharedDir, "", "*.yaml")), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom4, sidecar4, err := e2ethanos.NewPrometheusWithSidecar(e, "ha2", defaultPromConfig("prom-ha", 1, "", filepath.Join(e2ethanos.ContainerSharedDir, "", "*.yaml")), e2ethanos.DefaultPrometheusImage()) + prom4, sidecar4, err := e2ethanos.NewPrometheusWithSidecar(e, "ha2", defaultPromConfig("prom-ha", 1, "", filepath.Join(e2ethanos.ContainerSharedDir, "", "*.yaml")), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2, prom3, sidecar3, prom4, sidecar4)) @@ -260,9 +268,9 @@ func TestQueryLabelNames(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(receiverRunnable)) - prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, "alone", defaultPromConfig("prom-alone", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, "alone", defaultPromConfig("prom-alone", 0, "", ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) @@ -312,9 +320,9 @@ func TestQueryLabelValues(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(receiverRunnable)) - prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, "alone", defaultPromConfig("prom-alone", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, "alone", defaultPromConfig("prom-alone", 0, "", ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) @@ -348,6 +356,37 @@ func TestQueryLabelValues(t *testing.T) { ) } +func TestQueryWithAuthorizedSidecar(t *testing.T) { + t.Parallel() + + e, err := e2e.NewDockerEnvironment("e2e_test_query_authorized_sidecar") + testutil.Ok(t, err) + t.Cleanup(e2ethanos.CleanScenario(t, e)) + + prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(e, "alone", defaultPromConfig("prom-alone", 0, "", ""), defaultWebConfig(), e2ethanos.DefaultPrometheusImage()) + testutil.Ok(t, err) + testutil.Ok(t, e2e.StartAndWaitReady(prom, sidecar)) + + q, err := e2ethanos.NewQuerierBuilder(e, "1", []string{sidecar.InternalEndpoint("grpc")}...).Build() + testutil.Ok(t, err) + testutil.Ok(t, e2e.StartAndWaitReady(q)) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + t.Cleanup(cancel) + + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics())) + + queryAndAssertSeries(t, ctx, q.Endpoint("http"), queryUpWithoutInstance, time.Now, promclient.QueryOptions{ + Deduplicate: false, + }, []model.Metric{ + { + "job": "myself", + "prometheus": "prom-alone", + "replica": "0", + }, + }) +} + func TestQueryCompatibilityWithPreInfoAPI(t *testing.T) { t.Parallel() @@ -383,6 +422,7 @@ func TestQueryCompatibilityWithPreInfoAPI(t *testing.T) { e, "p1", defaultPromConfig("p1", 0, "", filepath.Join(e2ethanos.ContainerSharedDir, promRulesSubDir, "*.yaml"), "localhost:9090", qUninit.InternalEndpoint("http")), + "", e2ethanos.DefaultPrometheusImage(), tcase.sidecarImage, e2ethanos.FeatureExemplarStorage, diff --git a/test/e2e/receive_test.go b/test/e2e/receive_test.go index bffd990210..4cc7f68c45 100644 --- a/test/e2e/receive_test.go +++ b/test/e2e/receive_test.go @@ -65,7 +65,7 @@ func TestReceive(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(i)) // Setup Prometheus - prom, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(i.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(i.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom)) @@ -148,11 +148,11 @@ func TestReceive(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(i1, i2, i3, r1)) - prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom3, _, err := e2ethanos.NewPrometheus(e, "3", defaultPromConfig("prom3", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom3, _, err := e2ethanos.NewPrometheus(e, "3", defaultPromConfig("prom3", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, prom2, prom3)) @@ -262,9 +262,9 @@ func TestReceive(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(i1, i2, i3, r1, r2)) //Setup Prometheuses - prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, prom2)) @@ -361,11 +361,11 @@ func TestReceive(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(r1Runnable, r2Runnable, r3Runnable)) - prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r2.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r2.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom3, _, err := e2ethanos.NewPrometheus(e, "3", defaultPromConfig("prom3", 0, e2ethanos.RemoteWriteEndpoint(r3.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom3, _, err := e2ethanos.NewPrometheus(e, "3", defaultPromConfig("prom3", 0, e2ethanos.RemoteWriteEndpoint(r3.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, prom2, prom3)) @@ -434,11 +434,11 @@ func TestReceive(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(r1Runnable, r2Runnable, r3Runnable)) - prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r2.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r2.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom3, _, err := e2ethanos.NewPrometheus(e, "3", defaultPromConfig("prom3", 0, e2ethanos.RemoteWriteEndpoint(r3.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom3, _, err := e2ethanos.NewPrometheus(e, "3", defaultPromConfig("prom3", 0, e2ethanos.RemoteWriteEndpoint(r3.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, prom2, prom3)) @@ -511,7 +511,7 @@ func TestReceive(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(r1Runnable, r2Runnable, r3Runnable)) - prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1)) @@ -581,7 +581,7 @@ func TestReceive(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(r1Runnable, r2Runnable)) - prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1)) @@ -640,9 +640,9 @@ func TestReceive(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(rp1, rp2)) - prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, "http://"+rp1.InternalEndpoint("http")+"/api/v1/receive", ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, "http://"+rp1.InternalEndpoint("http")+"/api/v1/receive", ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, "http://"+rp2.InternalEndpoint("http")+"/api/v1/receive", ""), e2ethanos.DefaultPrometheusImage()) + prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, "http://"+rp2.InternalEndpoint("http")+"/api/v1/receive", ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, prom2)) diff --git a/test/e2e/rules_api_test.go b/test/e2e/rules_api_test.go index cdbe28e6f7..75a088e08b 100644 --- a/test/e2e/rules_api_test.go +++ b/test/e2e/rules_api_test.go @@ -48,6 +48,7 @@ func TestRulesAPI_Fanout(t *testing.T) { e, "prom1", defaultPromConfig("ha", 0, "", filepath.Join(e2ethanos.ContainerSharedDir, promRulesSubDir, "*.yaml")), + "", e2ethanos.DefaultPrometheusImage(), ) testutil.Ok(t, err) @@ -55,6 +56,7 @@ func TestRulesAPI_Fanout(t *testing.T) { e, "prom2", defaultPromConfig("ha", 1, "", filepath.Join(e2ethanos.ContainerSharedDir, promRulesSubDir, "*.yaml")), + "", e2ethanos.DefaultPrometheusImage(), ) testutil.Ok(t, err) diff --git a/test/e2e/targets_api_test.go b/test/e2e/targets_api_test.go index 448e2729f2..719e599a5a 100644 --- a/test/e2e/targets_api_test.go +++ b/test/e2e/targets_api_test.go @@ -36,6 +36,7 @@ func TestTargetsAPI_Fanout(t *testing.T) { e, "prom1", defaultPromConfig("ha", 0, "", "", "localhost:9090", "localhost:80"), + "", e2ethanos.DefaultPrometheusImage(), ) testutil.Ok(t, err) @@ -43,6 +44,7 @@ func TestTargetsAPI_Fanout(t *testing.T) { e, "prom2", defaultPromConfig("ha", 1, "", "", "localhost:9090", "localhost:80"), + "", e2ethanos.DefaultPrometheusImage(), ) testutil.Ok(t, err)