From 2076b8bcd39f041106d37de7e4a829f666de9813 Mon Sep 17 00:00:00 2001 From: someshkoli Date: Mon, 30 Aug 2021 20:40:01 +0530 Subject: [PATCH 01/12] pull work from #4104 Signed-off-by: someshkoli --- cmd/thanos/config.go | 18 +---- cmd/thanos/sidecar.go | 28 +++++--- go.mod | 3 +- go.sum | 3 +- pkg/extkingpin/flags.go | 8 +++ pkg/http/http.go | 125 ++++++++++++++++++++++++++++++++- pkg/promclient/promclient.go | 10 +-- pkg/reloader/reloader.go | 3 +- test/e2e/e2ethanos/services.go | 61 ++++++++++++++++ test/e2e/query_test.go | 39 ++++++++++ 10 files changed, 264 insertions(+), 34 deletions(-) diff --git a/cmd/thanos/config.go b/cmd/thanos/config.go index 3770389a68..46be3b70d3 100644 --- a/cmd/thanos/config.go +++ b/cmd/thanos/config.go @@ -65,6 +65,7 @@ func (hc *httpConfig) registerFlag(cmd extkingpin.FlagClause) *httpConfig { type prometheusConfig struct { url *url.URL readyTimeout time.Duration + httpClient *extflag.PathOrContent } func (pc *prometheusConfig) registerFlag(cmd extkingpin.FlagClause) *prometheusConfig { @@ -74,22 +75,9 @@ func (pc *prometheusConfig) registerFlag(cmd extkingpin.FlagClause) *prometheusC cmd.Flag("prometheus.ready_timeout", "Maximum time to wait for the Prometheus instance to start up"). Default("10m").DurationVar(&pc.readyTimeout) - return pc -} + pc.httpClient = extkingpin.RegisterHTTPConfigFlags(cmd) -type connConfig struct { - maxIdleConns int - maxIdleConnsPerHost int -} - -func (cc *connConfig) registerFlag(cmd extkingpin.FlagClause) *connConfig { - cmd.Flag("receive.connection-pool-size", - "Controls the http MaxIdleConns. Default is 0, which is unlimited"). - IntVar(&cc.maxIdleConns) - cmd.Flag("receive.connection-pool-size-per-host", - "Controls the http MaxIdleConnsPerHost"). - Default("100").IntVar(&cc.maxIdleConnsPerHost) - return cc + return pc } type tsdbConfig struct { diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index fee36d6537..d69ce79c80 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -6,7 +6,6 @@ package main import ( "context" "math" - "net/http" "net/url" "sync" "time" @@ -26,7 +25,6 @@ import ( "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/component" "github.com/thanos-io/thanos/pkg/exemplars" - "github.com/thanos-io/thanos/pkg/exthttp" "github.com/thanos-io/thanos/pkg/extkingpin" "github.com/thanos-io/thanos/pkg/extprom" thanoshttp "github.com/thanos-io/thanos/pkg/http" @@ -45,7 +43,6 @@ import ( "github.com/thanos-io/thanos/pkg/store" "github.com/thanos-io/thanos/pkg/targets" "github.com/thanos-io/thanos/pkg/tls" - "github.com/thanos-io/thanos/pkg/tracing" ) func registerSidecar(app *extkingpin.App) { @@ -84,6 +81,22 @@ func runSidecar( grpcLogOpts []grpc_logging.Option, tagOpts []tags.Option, ) error { + httpConfContentYaml, err := conf.prometheus.httpClient.Content() + if err != nil { + return errors.Wrap(err, "getting http client config") + } + httpClientConfig, err := thanoshttp.NewClientConfigFromYAML(httpConfContentYaml) + if err != nil { + return errors.Wrap(err, "parsing http config YAML") + } + + httpClient, err := thanoshttp.NewHTTPClient(*httpClientConfig, "thanos-sidecar") + if err != nil { + return errors.Wrap(err, "Improper http client config") + } + + reloader.HTTPClient = *httpClient + var m = &promMetadata{ promURL: conf.prometheus.url, @@ -93,7 +106,7 @@ func runSidecar( maxt: math.MaxInt64, limitMinTime: conf.limitMinTime, - client: promclient.NewWithTracingClient(logger, "thanos-sidecar"), + client: promclient.NewWithTracingClient(logger, httpClient, "thanos-sidecar"), } confContentYaml, err := conf.objStore.Content() @@ -231,10 +244,7 @@ func runSidecar( }) } { - t := exthttp.NewTransport() - t.MaxIdleConnsPerHost = conf.connection.maxIdleConnsPerHost - t.MaxIdleConns = conf.connection.maxIdleConns - c := promclient.NewClient(&http.Client{Transport: tracing.HTTPTripperware(logger, t)}, logger, thanoshttp.ThanosUserAgent) + c := promclient.NewWithTracingClient(logger, httpClient, thanoshttp.ThanosUserAgent) promStore, err := store.NewPrometheusStore(logger, reg, c, conf.prometheus.url, component.Sidecar, m.Labels, m.Timestamps, m.Version) if err != nil { @@ -441,7 +451,6 @@ type sidecarConfig struct { http httpConfig grpc grpcConfig prometheus prometheusConfig - connection connConfig tsdb tsdbConfig reloader reloaderConfig reqLogConfig *extflag.PathOrContent @@ -454,7 +463,6 @@ func (sc *sidecarConfig) registerFlag(cmd extkingpin.FlagClause) { sc.http.registerFlag(cmd) sc.grpc.registerFlag(cmd) sc.prometheus.registerFlag(cmd) - sc.connection.registerFlag(cmd) sc.tsdb.registerFlag(cmd) sc.reloader.registerFlag(cmd) sc.reqLogConfig = extkingpin.RegisterRequestLoggingFlags(cmd) diff --git a/go.mod b/go.mod index 0586501dd0..e44ac67dd6 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( github.com/prometheus/alertmanager v0.21.1-0.20210422101724-8176f78a70e1 github.com/prometheus/client_golang v1.10.0 github.com/prometheus/client_model v0.2.0 - github.com/prometheus/common v0.23.0 + github.com/prometheus/common v0.25.0 github.com/prometheus/exporter-toolkit v0.5.1 github.com/prometheus/prometheus v1.8.2-0.20210519120135-d95b0972505f github.com/uber/jaeger-client-go v2.28.0+incompatible @@ -68,6 +68,7 @@ require ( go.uber.org/automaxprocs v1.4.0 go.uber.org/goleak v1.1.10 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a + golang.org/x/net v0.0.0-20210505214959-0714010a04ed golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/text v0.3.6 diff --git a/go.sum b/go.sum index 89e4cc817d..23ebe9b8a1 100644 --- a/go.sum +++ b/go.sum @@ -1129,8 +1129,9 @@ github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16 github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.20.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.21.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/common v0.23.0 h1:GXWvPYuTUenIa+BhOq/x+L/QZzCqASkVRny5KTlPDGM= github.com/prometheus/common v0.23.0/go.mod h1:H6QK/N6XVT42whUeIdI3dp36w49c+/iMDk7UAI2qm7Q= +github.com/prometheus/common v0.25.0 h1:IjJYZJCI8HZYtqA3xYwGyDzSCy1r4CA2GRh+4vdOmtE= +github.com/prometheus/common v0.25.0/go.mod h1:H6QK/N6XVT42whUeIdI3dp36w49c+/iMDk7UAI2qm7Q= github.com/prometheus/exporter-toolkit v0.5.0/go.mod h1:OCkM4805mmisBhLmVFw858QYi3v0wKdY6/UxrT0pZVg= github.com/prometheus/exporter-toolkit v0.5.1 h1:9eqgis5er9xN613ZSADjypCJaDGj9ZlcWBvsIHa8/3c= github.com/prometheus/exporter-toolkit v0.5.1/go.mod h1:OCkM4805mmisBhLmVFw858QYi3v0wKdY6/UxrT0pZVg= diff --git a/pkg/extkingpin/flags.go b/pkg/extkingpin/flags.go index a7468b83a9..4d94029794 100644 --- a/pkg/extkingpin/flags.go +++ b/pkg/extkingpin/flags.go @@ -98,3 +98,11 @@ func RegisterSelectorRelabelFlags(cmd FlagClause) *extflag.PathOrContent { extflag.WithEnvSubstitution(), ) } + +func RegisterHTTPConfigFlags(cmd FlagClause) *extflag.PathOrContent { + return extflag.RegisterPathOrContent( + cmd, + "prometheus.http-client", + "YAML file or string with http client configs. see Format details : ...", + ) +} diff --git a/pkg/http/http.go b/pkg/http/http.go index be4233cdfb..a076180c08 100644 --- a/pkg/http/http.go +++ b/pkg/http/http.go @@ -6,18 +6,22 @@ package http import ( "context" + "crypto/tls" "fmt" "net/http" "net/url" "path" "sync" + "time" "github.com/go-kit/kit/log" + "github.com/mwitkow/go-conntrack" config_util "github.com/prometheus/common/config" "github.com/prometheus/common/model" "github.com/prometheus/common/version" "github.com/prometheus/prometheus/discovery/file" "github.com/prometheus/prometheus/discovery/targetgroup" + "golang.org/x/net/http2" "gopkg.in/yaml.v2" "github.com/thanos-io/thanos/pkg/discovery/cache" @@ -35,6 +39,9 @@ type ClientConfig struct { ProxyURL string `yaml:"proxy_url"` // TLSConfig to use to connect to the targets. TLSConfig TLSConfig `yaml:"tls_config"` + + // TransportConfig for Client transport properties + TransportConfig TransportConfig `yaml:"transport_config"` } // TLSConfig configures TLS connections. @@ -63,6 +70,104 @@ func (b BasicAuth) IsZero() bool { return b.Username == "" && b.Password == "" && b.PasswordFile == "" } +// Transport configures client's transport properties. +type TransportConfig struct { + MaxIdleConns int `yaml:"max_idle_conns"` + MaxIdleConnsPerHost int `yaml:"max_idle_conns_per_host"` + IdleConnTimeout int `yaml:"idle_conn_timeout"` + ResponseHeaderTimeout int `yaml:"response_header_timeout"` + ExpectContinueTimeout int `yaml:"expect_continue_timeout"` + MaxConnsPerHost int `yaml:"max_conns_per_host"` + DisableCompression bool `yaml:"disable_compression"` + TLSHandshakeTimeout int `yaml:"tls_handshake_timeout"` +} + +var defaultTransportConfig TransportConfig = TransportConfig{ + MaxIdleConns: 100, + MaxIdleConnsPerHost: 2, + ResponseHeaderTimeout: 0, + MaxConnsPerHost: 0, + IdleConnTimeout: int(90 * time.Second), + ExpectContinueTimeout: int(10 * time.Second), + DisableCompression: false, + TLSHandshakeTimeout: int(10 * time.Second), +} + +func NewClientConfigFromYAML(cfg []byte) (*ClientConfig, error) { + conf := &ClientConfig{TransportConfig: defaultTransportConfig} + if err := yaml.Unmarshal(cfg, conf); err != nil { + return nil, err + } + return conf, nil +} + +// NewRoundTripperFromConfig returns a new HTTP RoundTripper configured for the +// given http.HTTPClientConfig and http.HTTPClientOption. +func NewRoundTripperFromConfig(cfg config_util.HTTPClientConfig, transportConfig TransportConfig, name string) (http.RoundTripper, error) { + newRT := func(tlsConfig *tls.Config) (http.RoundTripper, error) { + var rt http.RoundTripper = &http.Transport{ + Proxy: http.ProxyURL(cfg.ProxyURL.URL), + MaxIdleConns: transportConfig.MaxIdleConns, + MaxIdleConnsPerHost: transportConfig.MaxIdleConnsPerHost, + MaxConnsPerHost: transportConfig.MaxConnsPerHost, + TLSClientConfig: tlsConfig, + DisableCompression: transportConfig.DisableCompression, + IdleConnTimeout: time.Duration(transportConfig.IdleConnTimeout), + ResponseHeaderTimeout: time.Duration(transportConfig.ResponseHeaderTimeout), + ExpectContinueTimeout: time.Duration(transportConfig.ExpectContinueTimeout), + TLSHandshakeTimeout: time.Duration(transportConfig.TLSHandshakeTimeout), + DialContext: conntrack.NewDialContextFunc( + conntrack.DialWithTracing(), + conntrack.DialWithName(name)), + } + + // HTTP/2 support is golang has many problematic cornercases where + // dead connections would be kept and used in connection pools. + // https://github.com/golang/go/issues/32388 + // https://github.com/golang/go/issues/39337 + // https://github.com/golang/go/issues/39750 + // TODO: Re-Enable HTTP/2 once upstream issue is fixed. + // TODO: use ForceAttemptHTTP2 when we move to Go 1.13+. + err := http2.ConfigureTransport(rt.(*http.Transport)) + if err != nil { + return nil, err + } + + // If a authorization_credentials is provided, create a round tripper that will set the + // Authorization header correctly on each request. + if cfg.Authorization != nil && len(cfg.Authorization.Credentials) > 0 { + rt = config_util.NewAuthorizationCredentialsRoundTripper(cfg.Authorization.Type, cfg.Authorization.Credentials, rt) + } else if cfg.Authorization != nil && len(cfg.Authorization.CredentialsFile) > 0 { + rt = config_util.NewAuthorizationCredentialsFileRoundTripper(cfg.Authorization.Type, cfg.Authorization.CredentialsFile, rt) + } + // Backwards compatibility, be nice with importers who would not have + // called Validate(). + if len(cfg.BearerToken) > 0 { + rt = config_util.NewAuthorizationCredentialsRoundTripper("Bearer", cfg.BearerToken, rt) + } else if len(cfg.BearerTokenFile) > 0 { + rt = config_util.NewAuthorizationCredentialsFileRoundTripper("Bearer", cfg.BearerTokenFile, rt) + } + + if cfg.BasicAuth != nil { + rt = config_util.NewBasicAuthRoundTripper(cfg.BasicAuth.Username, cfg.BasicAuth.Password, cfg.BasicAuth.PasswordFile, rt) + } + // Return a new configured RoundTripper. + return rt, nil + } + + tlsConfig, err := config_util.NewTLSConfig(&cfg.TLSConfig) + if err != nil { + return nil, err + } + + if len(cfg.TLSConfig.CAFile) == 0 { + // No need for a RoundTripper that reloads the CA file automatically. + return newRT(tlsConfig) + } + + return config_util.NewTLSRoundTripper(tlsConfig, cfg.TLSConfig.CAFile, newRT) +} + // NewHTTPClient returns a new HTTP client. func NewHTTPClient(cfg ClientConfig, name string) (*http.Client, error) { httpClientConfig := config_util.HTTPClientConfig{ @@ -91,15 +196,31 @@ func NewHTTPClient(cfg ClientConfig, name string) (*http.Client, error) { PasswordFile: cfg.BasicAuth.PasswordFile, } } + + if cfg.BearerToken != "" { + httpClientConfig.BearerToken = config_util.Secret(cfg.BearerToken) + } + + if cfg.BearerTokenFile != "" { + httpClientConfig.BearerTokenFile = cfg.BearerTokenFile + } + if err := httpClientConfig.Validate(); err != nil { return nil, err } - client, err := config_util.NewClientFromConfig(httpClientConfig, name, config_util.WithHTTP2Disabled()) + rt, err := NewRoundTripperFromConfig( + httpClientConfig, + cfg.TransportConfig, + name, + ) if err != nil { return nil, err } - client.Transport = &userAgentRoundTripper{name: ThanosUserAgent, rt: client.Transport} + + rt = &userAgentRoundTripper{name: ThanosUserAgent, rt: rt} + client := &http.Client{Transport: rt} + return client, nil } diff --git a/pkg/promclient/promclient.go b/pkg/promclient/promclient.go index 614bf9df68..83c156e600 100644 --- a/pkg/promclient/promclient.go +++ b/pkg/promclient/promclient.go @@ -32,6 +32,7 @@ import ( "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/promql/parser" "github.com/thanos-io/thanos/pkg/exemplars/exemplarspb" + thanoshttp "github.com/thanos-io/thanos/pkg/http" "github.com/thanos-io/thanos/pkg/metadata/metadatapb" "github.com/thanos-io/thanos/pkg/rules/rulespb" "github.com/thanos-io/thanos/pkg/runutil" @@ -84,18 +85,19 @@ func NewClient(c HTTPClient, logger log.Logger, userAgent string) *Client { // NewDefaultClient returns Client with tracing tripperware. func NewDefaultClient() *Client { + client, _ := thanoshttp.NewHTTPClient(thanoshttp.ClientConfig{}, "") return NewWithTracingClient( log.NewNopLogger(), + client, "", ) } // NewWithTracingClient returns client with tracing tripperware. -func NewWithTracingClient(logger log.Logger, userAgent string) *Client { +func NewWithTracingClient(logger log.Logger, httpClient *http.Client, userAgent string) *Client { + httpClient.Transport = tracing.HTTPTripperware(log.NewNopLogger(), httpClient.Transport) return NewClient( - &http.Client{ - Transport: tracing.HTTPTripperware(log.NewNopLogger(), http.DefaultTransport), - }, + httpClient, logger, userAgent, ) diff --git a/pkg/reloader/reloader.go b/pkg/reloader/reloader.go index 1e6c9fde7d..9d8150f1cb 100644 --- a/pkg/reloader/reloader.go +++ b/pkg/reloader/reloader.go @@ -85,6 +85,7 @@ import ( type Reloader struct { logger log.Logger reloadURL *url.URL + HTTPClient http.Client cfgFile string cfgOutputFile string watchInterval time.Duration @@ -413,7 +414,7 @@ func (r *Reloader) triggerReload(ctx context.Context) error { } req = req.WithContext(ctx) - resp, err := http.DefaultClient.Do(req) + resp, err := r.HTTPClient.Do(req) if err != nil { return errors.Wrap(err, "reload request failed") } diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index 3d83325774..681f15b5e1 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -716,3 +716,64 @@ func NewToolsBucketWeb( return toolsBucketWeb, nil } + +func NewPrometheusAndSidecarWithBasicAuth(sharedDir string, netName string, name string, promConfig, webConfig, promImage string) (*e2e.HTTPService, *Service, error) { + dir := filepath.Join(sharedDir, "data", "prometheus", name) + container := filepath.Join(e2e.ContainerSharedDir, "data", "prometheus", name) + if err := os.MkdirAll(dir, 0777); err != nil { + return nil, nil, errors.Wrap(err, "create prometheus dir") + } + + if err := ioutil.WriteFile(filepath.Join(dir, "prometheus.yml"), []byte(promConfig), 0666); err != nil { + return nil, nil, errors.Wrap(err, "creating prom config failed") + } + + if err := ioutil.WriteFile(filepath.Join(dir, "web-config.yml"), []byte(webConfig), 0666); err != nil { + return nil, nil, errors.Wrap(err, "creating web-config failed") + } + + args := e2e.BuildArgs(map[string]string{ + "--config.file": filepath.Join(container, "prometheus.yml"), + "--storage.tsdb.path": container, + "--storage.tsdb.max-block-duration": "2h", + "--log.level": infoLogLevel, + "--web.listen-address": ":9090", + "--web.config.file": filepath.Join(container, "web-config.yml"), + }) + prom := e2e.NewHTTPService( + fmt.Sprintf("prometheus-%s", name), + promImage, + e2e.NewCommandWithoutEntrypoint("prometheus", args...), + e2e.NewHTTPReadinessProbe(9090, "/-/ready", 200, 200), + 9090, + ) + prom.SetUser(strconv.Itoa(os.Getuid())) + prom.SetBackoff(defaultBackoffConfig) + + args = e2e.BuildArgs(map[string]string{ + "--debug.name": fmt.Sprintf("sidecar-%v", name), + "--grpc-address": ":9091", + "--grpc-grace-period": "0s", + "--http-address": ":8080", + "--prometheus.url": "http://" + prom.NetworkEndpointFor(netName, 9090), + "--tsdb.path": container, + "--log.level": infoLogLevel, + "--prometheus.http-client": ` +basic_auth: + username: test + password: test +`, + }) + sidecar := NewService( + fmt.Sprintf("sidecar-%s", name), + DefaultImage(), + e2e.NewCommand("sidecar", args...), + e2e.NewHTTPReadinessProbe(8080, "/-/ready", 200, 200), + 8080, + 9091, + ) + sidecar.SetUser(strconv.Itoa(os.Getuid())) + sidecar.SetBackoff(defaultBackoffConfig) + + return prom, sidecar, nil +} diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index cbae749af5..56c2a40888 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -85,6 +85,14 @@ rule_files: return config } +func defaultWebConfig() string { + // username: test, secret: test(bcrypt hash) + return ` +basic_auth_users: + test: $2y$10$IsC9GG9U61sPCuDwwwcnPuMRyzx62cIcdNRs4SIdKwgWihfX4IC.C +` +} + func sortResults(res model.Vector) { sort.Slice(res, func(i, j int) bool { return res[i].String() < res[j].String() @@ -340,6 +348,37 @@ func TestQueryLabelValues(t *testing.T) { ) } +func TestQueryWithAuthorizedSidecar(t *testing.T) { + t.Parallel() + + s, err := e2e.NewScenario("e2e_test_query_authorized_sidecar") + testutil.Ok(t, err) + t.Cleanup(e2ethanos.CleanScenario(t, s)) + + prom, sidecar, err := e2ethanos.NewPrometheusAndSidecarWithBasicAuth(s.SharedDir(), s.NetworkName(), "alone", defaultPromConfig("prom-alone", 0, "", ""), defaultWebConfig(), e2ethanos.DefaultPrometheusImage()) + testutil.Ok(t, err) + _ = s.StartAndWaitReady(prom, sidecar) + + q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{sidecar.GRPCNetworkEndpoint()}).Build() + testutil.Ok(t, err) + testutil.Ok(t, s.StartAndWaitReady(q)) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + t.Cleanup(cancel) + + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics)) + + queryAndAssertSeries(t, ctx, q.HTTPEndpoint(), queryUpWithoutInstance, promclient.QueryOptions{ + Deduplicate: false, + }, []model.Metric{ + { + "job": "myself", + "prometheus": "prom-alone", + "replica": "0", + }, + }) +} + func checkNetworkRequests(t *testing.T, addr string) { ctx, cancel := chromedp.NewContext(context.Background()) t.Cleanup(cancel) From fed5d55080c4aea1d1c821bab55245030fcce110 Mon Sep 17 00:00:00 2001 From: someshkoli Date: Mon, 30 Aug 2021 21:00:39 +0530 Subject: [PATCH 02/12] update docs Signed-off-by: someshkoli --- docs/components/sidecar.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/components/sidecar.md b/docs/components/sidecar.md index 0c6d6865f6..d15011298f 100644 --- a/docs/components/sidecar.md +++ b/docs/components/sidecar.md @@ -124,17 +124,20 @@ Flags: Path to YAML file that contains object store configuration. See format details: https://thanos.io/tip/thanos/storage.md/#configuration + --prometheus.http-client= + Alternative to 'prometheus.http-client-file' + flag (mutually exclusive). Content of YAML file + or string with http client configs. see Format + details : ... + --prometheus.http-client-file= + Path to YAML file or string with http client + configs. see Format details : ... --prometheus.ready_timeout=10m Maximum time to wait for the Prometheus instance to start up --prometheus.url=http://localhost:9090 URL at which to reach Prometheus's API. For better performance use local network. - --receive.connection-pool-size=RECEIVE.CONNECTION-POOL-SIZE - Controls the http MaxIdleConns. Default is 0, - which is unlimited - --receive.connection-pool-size-per-host=100 - Controls the http MaxIdleConnsPerHost --reloader.config-envsubst-file="" Output file for environment variable substituted config file. From 1cdddb5f4d91a910a8b14e3c1cbbc220105da096 Mon Sep 17 00:00:00 2001 From: someshkoli Date: Mon, 30 Aug 2021 21:23:29 +0530 Subject: [PATCH 03/12] fix fs perms Signed-off-by: someshkoli --- test/e2e/e2ethanos/services.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index 681f15b5e1..c2cf0df24b 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -720,11 +720,11 @@ func NewToolsBucketWeb( func NewPrometheusAndSidecarWithBasicAuth(sharedDir string, netName string, name string, promConfig, webConfig, promImage string) (*e2e.HTTPService, *Service, error) { dir := filepath.Join(sharedDir, "data", "prometheus", name) container := filepath.Join(e2e.ContainerSharedDir, "data", "prometheus", name) - if err := os.MkdirAll(dir, 0777); err != nil { + if err := os.MkdirAll(dir, 0750); err != nil { return nil, nil, errors.Wrap(err, "create prometheus dir") } - if err := ioutil.WriteFile(filepath.Join(dir, "prometheus.yml"), []byte(promConfig), 0666); err != nil { + if err := ioutil.WriteFile(filepath.Join(dir, "prometheus.yml"), []byte(promConfig), 0600); err != nil { return nil, nil, errors.Wrap(err, "creating prom config failed") } From 4f5aa83bff1f11cbd2326260d717411b535b4df8 Mon Sep 17 00:00:00 2001 From: someshkoli Date: Tue, 21 Sep 2021 00:49:41 +0530 Subject: [PATCH 04/12] remove flags from exthttp, make prom http config configurable, Signed-off-by: someshkoli --- cmd/thanos/config.go | 6 +++++- pkg/extkingpin/flags.go | 8 -------- test/e2e/e2ethanos/services.go | 29 +++++++++++++++++------------ 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/cmd/thanos/config.go b/cmd/thanos/config.go index 46be3b70d3..8966a346f5 100644 --- a/cmd/thanos/config.go +++ b/cmd/thanos/config.go @@ -75,7 +75,11 @@ func (pc *prometheusConfig) registerFlag(cmd extkingpin.FlagClause) *prometheusC cmd.Flag("prometheus.ready_timeout", "Maximum time to wait for the Prometheus instance to start up"). Default("10m").DurationVar(&pc.readyTimeout) - pc.httpClient = extkingpin.RegisterHTTPConfigFlags(cmd) + pc.httpClient = extflag.RegisterPathOrContent( + cmd, + "prometheus.http-client", + "YAML file or string with http client configs. see Format details : ...", + ) return pc } diff --git a/pkg/extkingpin/flags.go b/pkg/extkingpin/flags.go index 4d94029794..a7468b83a9 100644 --- a/pkg/extkingpin/flags.go +++ b/pkg/extkingpin/flags.go @@ -98,11 +98,3 @@ func RegisterSelectorRelabelFlags(cmd FlagClause) *extflag.PathOrContent { extflag.WithEnvSubstitution(), ) } - -func RegisterHTTPConfigFlags(cmd FlagClause) *extflag.PathOrContent { - return extflag.RegisterPathOrContent( - cmd, - "prometheus.http-client", - "YAML file or string with http client configs. see Format details : ...", - ) -} diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index c2cf0df24b..d2d14a2ff3 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -60,6 +60,15 @@ func DefaultImage() string { return "thanos" } +func defaultPromHttpConfig() string { + // username: test, secret: test(bcrypt hash) + return ` +basic_auth: + username: test + password: test +` +} + func NewPrometheus(sharedDir, name, config, promImage string, enableFeatures ...string) (*e2e.HTTPService, string, error) { dir := filepath.Join(sharedDir, "data", "prometheus", name) container := filepath.Join(e2e.ContainerSharedDir, "data", "prometheus", name) @@ -751,18 +760,14 @@ func NewPrometheusAndSidecarWithBasicAuth(sharedDir string, netName string, name prom.SetBackoff(defaultBackoffConfig) args = e2e.BuildArgs(map[string]string{ - "--debug.name": fmt.Sprintf("sidecar-%v", name), - "--grpc-address": ":9091", - "--grpc-grace-period": "0s", - "--http-address": ":8080", - "--prometheus.url": "http://" + prom.NetworkEndpointFor(netName, 9090), - "--tsdb.path": container, - "--log.level": infoLogLevel, - "--prometheus.http-client": ` -basic_auth: - username: test - password: test -`, + "--debug.name": fmt.Sprintf("sidecar-%v", name), + "--grpc-address": ":9091", + "--grpc-grace-period": "0s", + "--http-address": ":8080", + "--prometheus.url": "http://" + prom.NetworkEndpointFor(netName, 9090), + "--tsdb.path": container, + "--log.level": infoLogLevel, + "--prometheus.http-client": defaultPromHttpConfig(), }) sidecar := NewService( fmt.Sprintf("sidecar-%s", name), From 9e1b5d11e51db2c0598c0467f13c15044a14a596 Mon Sep 17 00:00:00 2001 From: someshkoli Date: Tue, 21 Sep 2021 01:00:03 +0530 Subject: [PATCH 05/12] typo fix Signed-off-by: someshkoli --- go.mod | 3 +-- pkg/http/http.go | 6 +++--- test/e2e/query_test.go | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 25f8765550..b672382a9b 100644 --- a/go.mod +++ b/go.mod @@ -69,9 +69,8 @@ require ( go.uber.org/atomic v1.9.0 go.uber.org/automaxprocs v1.4.0 go.uber.org/goleak v1.1.10 - golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e - golang.org/x/net v0.0.0-20210505214959-0714010a04ed + golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/text v0.3.6 diff --git a/pkg/http/http.go b/pkg/http/http.go index 2629250868..36b42b976f 100644 --- a/pkg/http/http.go +++ b/pkg/http/http.go @@ -222,10 +222,10 @@ func NewHTTPClient(cfg ClientConfig, name string) (*http.Client, error) { return nil, err } - if cfg.ClientMetrics != nil { - tripper = extpromhttp.InstrumentedRoundTripper(rt, cfg.ClientMetrics) + if cfg.ClientMetrics != nil { + rt = extpromhttp.InstrumentedRoundTripper(rt, cfg.ClientMetrics) } - + rt = &userAgentRoundTripper{name: ThanosUserAgent, rt: rt} client := &http.Client{Transport: rt} diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index 552d96d760..88f8608dc5 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -365,7 +365,7 @@ func TestQueryWithAuthorizedSidecar(t *testing.T) { testutil.Ok(t, err) _ = s.StartAndWaitReady(prom, sidecar) - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{sidecar.GRPCNetworkEndpoint()}).Build() + q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{sidecar.GRPCNetworkEndpoint()}...).Build() testutil.Ok(t, err) testutil.Ok(t, s.StartAndWaitReady(q)) From d2f80dc278e68274ad19632e84fddac83e035518 Mon Sep 17 00:00:00 2001 From: someshkoli Date: Sun, 7 Nov 2021 16:56:29 +0530 Subject: [PATCH 06/12] adds method to set http client to realoder Signed-off-by: someshkoli --- cmd/thanos/sidecar.go | 2 +- pkg/reloader/reloader.go | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index d69ce79c80..e18985deba 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -95,7 +95,7 @@ func runSidecar( return errors.Wrap(err, "Improper http client config") } - reloader.HTTPClient = *httpClient + reloader.SetHttpClient(*httpClient) var m = &promMetadata{ promURL: conf.prometheus.url, diff --git a/pkg/reloader/reloader.go b/pkg/reloader/reloader.go index 9d8150f1cb..f5880054b9 100644 --- a/pkg/reloader/reloader.go +++ b/pkg/reloader/reloader.go @@ -85,7 +85,7 @@ import ( type Reloader struct { logger log.Logger reloadURL *url.URL - HTTPClient http.Client + httpClient http.Client cfgFile string cfgOutputFile string watchInterval time.Duration @@ -414,7 +414,7 @@ func (r *Reloader) triggerReload(ctx context.Context) error { } req = req.WithContext(ctx) - resp, err := r.HTTPClient.Do(req) + resp, err := r.httpClient.Do(req) if err != nil { return errors.Wrap(err, "reload request failed") } @@ -426,6 +426,11 @@ func (r *Reloader) triggerReload(ctx context.Context) error { return nil } +// Sets Http client for reloader +func (r *Reloader) SetHttpClient(client http.Client) { + r.httpClient = client +} + // ReloadURLFromBase returns the standard Prometheus reload URL from its base URL. func ReloadURLFromBase(u *url.URL) *url.URL { r := *u From 619f929f010645f4cf3ea2c8d159761f078fd842 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Sun, 7 Nov 2021 19:26:18 -0800 Subject: [PATCH 07/12] Merge branch 'main' into feature/promclient-authorization Signed-off-by: Ben Ye --- pkg/promclient/promclient.go | 4 +- pkg/reloader/reloader.go | 2 +- pkg/testutil/e2eutil/sysprocattr.go | 1 + test/e2e/e2ethanos/services.go | 73 ++++++++++------------------- test/e2e/exemplars_api_test.go | 2 + test/e2e/metadata_api_test.go | 2 + test/e2e/query_frontend_test.go | 4 +- test/e2e/query_test.go | 33 ++++++------- test/e2e/rules_api_test.go | 2 + test/e2e/targets_api_test.go | 2 + 10 files changed, 56 insertions(+), 69 deletions(-) diff --git a/pkg/promclient/promclient.go b/pkg/promclient/promclient.go index 2ad017f8e3..79fd6a1057 100644 --- a/pkg/promclient/promclient.go +++ b/pkg/promclient/promclient.go @@ -9,6 +9,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/thanos-io/thanos/pkg/httpconfig" "io" "io/ioutil" "net/http" @@ -32,7 +33,6 @@ import ( "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/promql/parser" "github.com/thanos-io/thanos/pkg/exemplars/exemplarspb" - thanoshttp "github.com/thanos-io/thanos/pkg/http" "github.com/thanos-io/thanos/pkg/metadata/metadatapb" "github.com/thanos-io/thanos/pkg/rules/rulespb" "github.com/thanos-io/thanos/pkg/runutil" @@ -85,7 +85,7 @@ func NewClient(c HTTPClient, logger log.Logger, userAgent string) *Client { // NewDefaultClient returns Client with tracing tripperware. func NewDefaultClient() *Client { - client, _ := thanoshttp.NewHTTPClient(thanoshttp.ClientConfig{}, "") + client, _ := httpconfig.NewHTTPClient(httpconfig.ClientConfig{}, "") return NewWithTracingClient( log.NewNopLogger(), client, diff --git a/pkg/reloader/reloader.go b/pkg/reloader/reloader.go index f5880054b9..cd11ce2ab9 100644 --- a/pkg/reloader/reloader.go +++ b/pkg/reloader/reloader.go @@ -426,7 +426,7 @@ func (r *Reloader) triggerReload(ctx context.Context) error { return nil } -// Sets Http client for reloader +// SetHttpClient sets Http client for reloader. func (r *Reloader) SetHttpClient(client http.Client) { r.httpClient = client } diff --git a/pkg/testutil/e2eutil/sysprocattr.go b/pkg/testutil/e2eutil/sysprocattr.go index cc982a5bd3..53aaa7039f 100644 --- a/pkg/testutil/e2eutil/sysprocattr.go +++ b/pkg/testutil/e2eutil/sysprocattr.go @@ -1,6 +1,7 @@ // Copyright (c) The Thanos Authors. // Licensed under the Apache License 2.0. +//go:build !linux // +build !linux package e2eutil diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index b36d0b5ac2..9bf5ce6c62 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -73,17 +73,23 @@ basic_auth: ` } -func NewPrometheus(e e2e.Environment, name, config, promImage string, enableFeatures ...string) (*e2e.InstrumentedRunnable, string, error) { +func NewPrometheus(e e2e.Environment, name, promConfig, webConfig, promImage string, enableFeatures ...string) (*e2e.InstrumentedRunnable, string, error) { dir := filepath.Join(e.SharedDir(), "data", "prometheus", name) container := filepath.Join(ContainerSharedDir, "data", "prometheus", name) if err := os.MkdirAll(dir, 0750); err != nil { return nil, "", errors.Wrap(err, "create prometheus dir") } - if err := ioutil.WriteFile(filepath.Join(dir, "prometheus.yml"), []byte(config), 0600); err != nil { + if err := ioutil.WriteFile(filepath.Join(dir, "prometheus.yml"), []byte(promConfig), 0600); err != nil { return nil, "", errors.Wrap(err, "creating prom config failed") } + if len(webConfig) > 0 { + if err := ioutil.WriteFile(filepath.Join(dir, "web-config.yml"), []byte(webConfig), 0600); err != nil { + return nil, "", errors.Wrap(err, "creating web-config failed") + } + } + args := e2e.BuildArgs(map[string]string{ "--config.file": filepath.Join(container, "prometheus.yml"), "--storage.tsdb.path": container, @@ -112,29 +118,33 @@ func NewPrometheus(e e2e.Environment, name, config, promImage string, enableFeat return prom, container, nil } -func NewPrometheusWithSidecar(e e2e.Environment, name, config, promImage string, enableFeatures ...string) (*e2e.InstrumentedRunnable, *e2e.InstrumentedRunnable, error) { - return NewPrometheusWithSidecarCustomImage(e, name, config, promImage, DefaultImage(), enableFeatures...) +func NewPrometheusWithSidecar(e e2e.Environment, name, promConfig, webConfig, promImage string, enableFeatures ...string) (*e2e.InstrumentedRunnable, *e2e.InstrumentedRunnable, error) { + return NewPrometheusWithSidecarCustomImage(e, name, promConfig, webConfig, promImage, DefaultImage(), enableFeatures...) } -func NewPrometheusWithSidecarCustomImage(e e2e.Environment, name, config, promImage string, sidecarImage string, enableFeatures ...string) (*e2e.InstrumentedRunnable, *e2e.InstrumentedRunnable, error) { - prom, dataDir, err := NewPrometheus(e, name, config, promImage, enableFeatures...) +func NewPrometheusWithSidecarCustomImage(e e2e.Environment, name, promConfig, webConfig, promImage string, sidecarImage string, enableFeatures ...string) (*e2e.InstrumentedRunnable, *e2e.InstrumentedRunnable, error) { + prom, dataDir, err := NewPrometheus(e, name, promConfig, webConfig, promImage, enableFeatures...) if err != nil { return nil, nil, err } + args := map[string]string{ + "--debug.name": fmt.Sprintf("sidecar-%v", name), + "--grpc-address": ":9091", + "--grpc-grace-period": "0s", + "--http-address": ":8080", + "--prometheus.url": "http://" + prom.InternalEndpoint("http"), + "--tsdb.path": dataDir, + "--log.level": infoLogLevel, + } + if len(webConfig) > 0 { + args["--prometheus.http-client"] = defaultPromHttpConfig() + } sidecar := NewService( e, fmt.Sprintf("sidecar-%s", name), sidecarImage, - e2e.NewCommand("sidecar", e2e.BuildArgs(map[string]string{ - "--debug.name": fmt.Sprintf("sidecar-%v", name), - "--grpc-address": ":9091", - "--grpc-grace-period": "0s", - "--http-address": ":8080", - "--prometheus.url": "http://" + prom.InternalEndpoint("http"), - "--tsdb.path": dataDir, - "--log.level": infoLogLevel, - })...), + e2e.NewCommand("sidecar", e2e.BuildArgs(args)...), e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), 8080, 9091, @@ -867,36 +877,3 @@ func NewToolsBucketWeb( return toolsBucketWeb, nil } - -func NewPrometheusAndSidecarWithBasicAuth(e e2e.Environment, name, promConfig, webConfig, promImage, sidecarImage string, enableFeatures ...string) (*e2e.InstrumentedRunnable, *e2e.InstrumentedRunnable, error) { - prom, dataDir, err := NewPrometheus(e, name, promConfig, promImage, enableFeatures...) - if err != nil { - return nil, nil, err - } - - dir := filepath.Join(e.SharedDir(), "data", "prometheus", name) - if err := ioutil.WriteFile(filepath.Join(dir, "web-config.yml"), []byte(webConfig), 0600); err != nil { - return nil, nil, errors.Wrap(err, "creating web-config failed") - } - - sidecar := NewService( - e, - fmt.Sprintf("sidecar-%s", name), - sidecarImage, - e2e.NewCommand("sidecar", e2e.BuildArgs(map[string]string{ - "--debug.name": fmt.Sprintf("sidecar-%v", name), - "--grpc-address": ":9091", - "--grpc-grace-period": "0s", - "--http-address": ":8080", - "--prometheus.url": "http://" + prom.InternalEndpoint("http"), - "--tsdb.path": dataDir, - "--log.level": infoLogLevel, - "--prometheus.http-client": defaultPromHttpConfig(), - })...), - e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), - 8080, - 9091, - ) - - return prom, sidecar, nil -} diff --git a/test/e2e/exemplars_api_test.go b/test/e2e/exemplars_api_test.go index 13aaffca85..151fefd94e 100644 --- a/test/e2e/exemplars_api_test.go +++ b/test/e2e/exemplars_api_test.go @@ -43,6 +43,7 @@ func TestExemplarsAPI_Fanout(t *testing.T) { e, "prom1", defaultPromConfig("ha", 0, "", "", "localhost:9090", qUnitiated.InternalEndpoint("http")), + "", e2ethanos.DefaultPrometheusImage(), e2ethanos.FeatureExemplarStorage, ) @@ -51,6 +52,7 @@ func TestExemplarsAPI_Fanout(t *testing.T) { e, "prom2", defaultPromConfig("ha", 1, "", "", "localhost:9090", qUnitiated.InternalEndpoint("http")), + "", e2ethanos.DefaultPrometheusImage(), e2ethanos.FeatureExemplarStorage, ) diff --git a/test/e2e/metadata_api_test.go b/test/e2e/metadata_api_test.go index 096560e64e..901183c860 100644 --- a/test/e2e/metadata_api_test.go +++ b/test/e2e/metadata_api_test.go @@ -31,6 +31,7 @@ func TestMetadataAPI_Fanout(t *testing.T) { e, "prom1", defaultPromConfig("ha", 0, "", "", "localhost:9090", "sidecar-prom1:8080"), + "", e2ethanos.DefaultPrometheusImage(), ) testutil.Ok(t, err) @@ -39,6 +40,7 @@ func TestMetadataAPI_Fanout(t *testing.T) { e, "prom2", defaultPromConfig("ha", 1, "", "", "localhost:9090", "sidecar-prom2:8080"), + "", e2ethanos.DefaultPrometheusImage(), ) testutil.Ok(t, err) diff --git a/test/e2e/query_frontend_test.go b/test/e2e/query_frontend_test.go index d320c42327..82afb4a9b2 100644 --- a/test/e2e/query_frontend_test.go +++ b/test/e2e/query_frontend_test.go @@ -32,7 +32,7 @@ func TestQueryFrontend(t *testing.T) { now := time.Now() - prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(e, "1", defaultPromConfig("test", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(e, "1", defaultPromConfig("test", 0, "", ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom, sidecar)) @@ -396,7 +396,7 @@ func TestQueryFrontendMemcachedCache(t *testing.T) { now := time.Now() - prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(e, "1", defaultPromConfig("test", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(e, "1", defaultPromConfig("test", 0, "", ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom, sidecar)) diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index 55509a2dd5..312e74485b 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -114,13 +114,13 @@ func TestQuery(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(receiverRunnable)) - prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, "alone", defaultPromConfig("prom-alone", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, "alone", defaultPromConfig("prom-alone", 0, "", ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom3, sidecar3, err := e2ethanos.NewPrometheusWithSidecar(e, "ha1", defaultPromConfig("prom-ha", 0, "", filepath.Join(e2ethanos.ContainerSharedDir, "", "*.yaml")), e2ethanos.DefaultPrometheusImage()) + prom3, sidecar3, err := e2ethanos.NewPrometheusWithSidecar(e, "ha1", defaultPromConfig("prom-ha", 0, "", filepath.Join(e2ethanos.ContainerSharedDir, "", "*.yaml")), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom4, sidecar4, err := e2ethanos.NewPrometheusWithSidecar(e, "ha2", defaultPromConfig("prom-ha", 1, "", filepath.Join(e2ethanos.ContainerSharedDir, "", "*.yaml")), e2ethanos.DefaultPrometheusImage()) + prom4, sidecar4, err := e2ethanos.NewPrometheusWithSidecar(e, "ha2", defaultPromConfig("prom-ha", 1, "", filepath.Join(e2ethanos.ContainerSharedDir, "", "*.yaml")), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2, prom3, sidecar3, prom4, sidecar4)) @@ -268,9 +268,9 @@ func TestQueryLabelNames(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(receiverRunnable)) - prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, "alone", defaultPromConfig("prom-alone", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, "alone", defaultPromConfig("prom-alone", 0, "", ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) @@ -320,9 +320,9 @@ func TestQueryLabelValues(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(receiverRunnable)) - prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, "alone", defaultPromConfig("prom-alone", 0, "", ""), e2ethanos.DefaultPrometheusImage()) + prom1, sidecar1, err := e2ethanos.NewPrometheusWithSidecar(e, "alone", defaultPromConfig("prom-alone", 0, "", ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom2, sidecar2, err := e2ethanos.NewPrometheusWithSidecar(e, "remote-and-sidecar", defaultPromConfig("prom-both-remote-write-and-sidecar", 1234, e2ethanos.RemoteWriteEndpoint(receiver.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, sidecar1, prom2, sidecar2)) @@ -359,24 +359,24 @@ func TestQueryLabelValues(t *testing.T) { func TestQueryWithAuthorizedSidecar(t *testing.T) { t.Parallel() - s, err := e2e.NewScenario("e2e_test_query_authorized_sidecar") + e, err := e2e.NewDockerEnvironment("e2e_test_query_authorized_sidecar") testutil.Ok(t, err) - t.Cleanup(e2ethanos.CleanScenario(t, s)) + t.Cleanup(e2ethanos.CleanScenario(t, e)) - prom, sidecar, err := e2ethanos.NewPrometheusAndSidecarWithBasicAuth(s.SharedDir(), s.NetworkName(), "alone", defaultPromConfig("prom-alone", 0, "", ""), defaultWebConfig(), e2ethanos.DefaultPrometheusImage()) + prom, sidecar, err := e2ethanos.NewPrometheusWithSidecar(e, "alone", defaultPromConfig("prom-alone", 0, "", ""), defaultWebConfig(), e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - _ = s.StartAndWaitReady(prom, sidecar) + testutil.Ok(t, e2e.StartAndWaitReady(prom, sidecar)) - q, err := e2ethanos.NewQuerierBuilder(s.SharedDir(), "1", []string{sidecar.GRPCNetworkEndpoint()}...).Build() + q, err := e2ethanos.NewQuerierBuilder(e, "1", []string{sidecar.InternalEndpoint("grpc")}...).Build() testutil.Ok(t, err) - testutil.Ok(t, s.StartAndWaitReady(q)) + testutil.Ok(t, e2e.StartAndWaitReady(q)) ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) t.Cleanup(cancel) - testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics)) + testutil.Ok(t, q.WaitSumMetricsWithOptions(e2e.Equals(1), []string{"thanos_store_nodes_grpc_connections"}, e2e.WaitMissingMetrics())) - queryAndAssertSeries(t, ctx, q.HTTPEndpoint(), queryUpWithoutInstance, promclient.QueryOptions{ + queryAndAssertSeries(t, ctx, q.Endpoint("http"), queryUpWithoutInstance, time.Now, promclient.QueryOptions{ Deduplicate: false, }, []model.Metric{ { @@ -422,6 +422,7 @@ func TestQueryCompatibilityWithPreInfoAPI(t *testing.T) { e, "p1", defaultPromConfig("p1", 0, "", filepath.Join(e2ethanos.ContainerSharedDir, promRulesSubDir, "*.yaml"), "localhost:9090", qUninit.InternalEndpoint("http")), + "", e2ethanos.DefaultPrometheusImage(), tcase.sidecarImage, e2ethanos.FeatureExemplarStorage, diff --git a/test/e2e/rules_api_test.go b/test/e2e/rules_api_test.go index 0d94317c8b..7b207bc9a6 100644 --- a/test/e2e/rules_api_test.go +++ b/test/e2e/rules_api_test.go @@ -48,6 +48,7 @@ func TestRulesAPI_Fanout(t *testing.T) { e, "prom1", defaultPromConfig("ha", 0, "", filepath.Join(e2ethanos.ContainerSharedDir, promRulesSubDir, "*.yaml")), + "", e2ethanos.DefaultPrometheusImage(), ) testutil.Ok(t, err) @@ -55,6 +56,7 @@ func TestRulesAPI_Fanout(t *testing.T) { e, "prom2", defaultPromConfig("ha", 1, "", filepath.Join(e2ethanos.ContainerSharedDir, promRulesSubDir, "*.yaml")), + "", e2ethanos.DefaultPrometheusImage(), ) testutil.Ok(t, err) diff --git a/test/e2e/targets_api_test.go b/test/e2e/targets_api_test.go index 448e2729f2..719e599a5a 100644 --- a/test/e2e/targets_api_test.go +++ b/test/e2e/targets_api_test.go @@ -36,6 +36,7 @@ func TestTargetsAPI_Fanout(t *testing.T) { e, "prom1", defaultPromConfig("ha", 0, "", "", "localhost:9090", "localhost:80"), + "", e2ethanos.DefaultPrometheusImage(), ) testutil.Ok(t, err) @@ -43,6 +44,7 @@ func TestTargetsAPI_Fanout(t *testing.T) { e, "prom2", defaultPromConfig("ha", 1, "", "", "localhost:9090", "localhost:80"), + "", e2ethanos.DefaultPrometheusImage(), ) testutil.Ok(t, err) From 982eb0e212e5e78f7993e70deb42bebea3840fb4 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Sun, 7 Nov 2021 19:46:29 -0800 Subject: [PATCH 08/12] fix format Signed-off-by: Ben Ye --- pkg/promclient/promclient.go | 7 ++++--- pkg/testutil/e2eutil/sysprocattr.go | 1 - 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/promclient/promclient.go b/pkg/promclient/promclient.go index 79fd6a1057..4bc6fb7657 100644 --- a/pkg/promclient/promclient.go +++ b/pkg/promclient/promclient.go @@ -9,7 +9,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/thanos-io/thanos/pkg/httpconfig" "io" "io/ioutil" "net/http" @@ -32,15 +31,17 @@ import ( "github.com/prometheus/prometheus/pkg/timestamp" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/promql/parser" + "google.golang.org/grpc/codes" + "gopkg.in/yaml.v2" + "github.com/thanos-io/thanos/pkg/exemplars/exemplarspb" + "github.com/thanos-io/thanos/pkg/httpconfig" "github.com/thanos-io/thanos/pkg/metadata/metadatapb" "github.com/thanos-io/thanos/pkg/rules/rulespb" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/store/storepb" "github.com/thanos-io/thanos/pkg/targets/targetspb" "github.com/thanos-io/thanos/pkg/tracing" - "google.golang.org/grpc/codes" - yaml "gopkg.in/yaml.v2" ) var ( diff --git a/pkg/testutil/e2eutil/sysprocattr.go b/pkg/testutil/e2eutil/sysprocattr.go index 53aaa7039f..cc982a5bd3 100644 --- a/pkg/testutil/e2eutil/sysprocattr.go +++ b/pkg/testutil/e2eutil/sysprocattr.go @@ -1,7 +1,6 @@ // Copyright (c) The Thanos Authors. // Licensed under the Apache License 2.0. -//go:build !linux // +build !linux package e2eutil From 2d6359f5690578faa8e71b1e1874b8849b564bad Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Sun, 7 Nov 2021 20:14:26 -0800 Subject: [PATCH 09/12] fix e2e Signed-off-by: Ben Ye --- test/e2e/receive_test.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/test/e2e/receive_test.go b/test/e2e/receive_test.go index bffd990210..4cc7f68c45 100644 --- a/test/e2e/receive_test.go +++ b/test/e2e/receive_test.go @@ -65,7 +65,7 @@ func TestReceive(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(i)) // Setup Prometheus - prom, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(i.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(i.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom)) @@ -148,11 +148,11 @@ func TestReceive(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(i1, i2, i3, r1)) - prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom3, _, err := e2ethanos.NewPrometheus(e, "3", defaultPromConfig("prom3", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom3, _, err := e2ethanos.NewPrometheus(e, "3", defaultPromConfig("prom3", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, prom2, prom3)) @@ -262,9 +262,9 @@ func TestReceive(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(i1, i2, i3, r1, r2)) //Setup Prometheuses - prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, prom2)) @@ -361,11 +361,11 @@ func TestReceive(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(r1Runnable, r2Runnable, r3Runnable)) - prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r2.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r2.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom3, _, err := e2ethanos.NewPrometheus(e, "3", defaultPromConfig("prom3", 0, e2ethanos.RemoteWriteEndpoint(r3.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom3, _, err := e2ethanos.NewPrometheus(e, "3", defaultPromConfig("prom3", 0, e2ethanos.RemoteWriteEndpoint(r3.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, prom2, prom3)) @@ -434,11 +434,11 @@ func TestReceive(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(r1Runnable, r2Runnable, r3Runnable)) - prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r2.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, e2ethanos.RemoteWriteEndpoint(r2.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom3, _, err := e2ethanos.NewPrometheus(e, "3", defaultPromConfig("prom3", 0, e2ethanos.RemoteWriteEndpoint(r3.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom3, _, err := e2ethanos.NewPrometheus(e, "3", defaultPromConfig("prom3", 0, e2ethanos.RemoteWriteEndpoint(r3.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, prom2, prom3)) @@ -511,7 +511,7 @@ func TestReceive(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(r1Runnable, r2Runnable, r3Runnable)) - prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1)) @@ -581,7 +581,7 @@ func TestReceive(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(r1Runnable, r2Runnable)) - prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, e2ethanos.RemoteWriteEndpoint(r1.InternalEndpoint("remote-write")), ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1)) @@ -640,9 +640,9 @@ func TestReceive(t *testing.T) { testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(rp1, rp2)) - prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, "http://"+rp1.InternalEndpoint("http")+"/api/v1/receive", ""), e2ethanos.DefaultPrometheusImage()) + prom1, _, err := e2ethanos.NewPrometheus(e, "1", defaultPromConfig("prom1", 0, "http://"+rp1.InternalEndpoint("http")+"/api/v1/receive", ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) - prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, "http://"+rp2.InternalEndpoint("http")+"/api/v1/receive", ""), e2ethanos.DefaultPrometheusImage()) + prom2, _, err := e2ethanos.NewPrometheus(e, "2", defaultPromConfig("prom2", 0, "http://"+rp2.InternalEndpoint("http")+"/api/v1/receive", ""), "", e2ethanos.DefaultPrometheusImage()) testutil.Ok(t, err) testutil.Ok(t, e2e.StartAndWaitReady(prom1, prom2)) From e229ea4dea6282925fbb194d4eb36985ce6b923a Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Sun, 7 Nov 2021 21:47:08 -0800 Subject: [PATCH 10/12] fix prom e2e test Signed-off-by: Ben Ye --- test/e2e/e2ethanos/services.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index 9bf5ce6c62..ad06447e16 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -101,6 +101,9 @@ func NewPrometheus(e e2e.Environment, name, promConfig, webConfig, promImage str if len(enableFeatures) > 0 { args = append(args, fmt.Sprintf("--enable-feature=%s", strings.Join(enableFeatures, ","))) } + if len(webConfig) > 0 { + args = append(args, fmt.Sprintf("--web.config.file=%s", filepath.Join(container, "web-config.yml"))) + } prom := e2e.NewInstrumentedRunnable( e, fmt.Sprintf("prometheus-%s", name), From dd2206adab0b4eaf1ba1916fd8d64ae2594e1c6c Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Mon, 8 Nov 2021 00:38:18 -0800 Subject: [PATCH 11/12] update changelog Signed-off-by: Ben Ye --- CHANGELOG.md | 1 + test/e2e/e2ethanos/services.go | 13 ++++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 014f974817..9c065834fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#4801](https://github.com/thanos-io/thanos/pull/4801) Compactor: added Prometheus metrics for tracking the progress of compaction and downsampling. - [#4444](https://github.com/thanos-io/thanos/pull/4444) UI: add mark deletion and no compaction to the Block UI. - [#4576](https://github.com/thanos-io/thanos/pull/4576) UI: add filter compaction level to the Block UI. +- [#4612](https://github.com/thanos-io/thanos/pull/4612) Sidecar: add `--prometheus.http-client` and `--prometheus.http-client-file` flag for sidecar to connect Prometheus with basic auth or TLS. ### Fixed diff --git a/test/e2e/e2ethanos/services.go b/test/e2e/e2ethanos/services.go index ad06447e16..7802976d07 100644 --- a/test/e2e/e2ethanos/services.go +++ b/test/e2e/e2ethanos/services.go @@ -66,10 +66,9 @@ func DefaultImage() string { func defaultPromHttpConfig() string { // username: test, secret: test(bcrypt hash) - return ` -basic_auth: - username: test - password: test + return `basic_auth: + username: test + password: test ` } @@ -90,6 +89,8 @@ func NewPrometheus(e e2e.Environment, name, promConfig, webConfig, promImage str } } + probe := e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200) + args := e2e.BuildArgs(map[string]string{ "--config.file": filepath.Join(container, "prometheus.yml"), "--storage.tsdb.path": container, @@ -103,6 +104,8 @@ func NewPrometheus(e e2e.Environment, name, promConfig, webConfig, promImage str } if len(webConfig) > 0 { args = append(args, fmt.Sprintf("--web.config.file=%s", filepath.Join(container, "web-config.yml"))) + // If auth is enabled then prober would get 401 error. + probe = e2e.NewHTTPReadinessProbe("http", "/-/ready", 401, 401) } prom := e2e.NewInstrumentedRunnable( e, @@ -112,7 +115,7 @@ func NewPrometheus(e e2e.Environment, name, promConfig, webConfig, promImage str e2e.StartOptions{ Image: promImage, Command: e2e.NewCommandWithoutEntrypoint("prometheus", args...), - Readiness: e2e.NewHTTPReadinessProbe("http", "/-/ready", 200, 200), + Readiness: probe, User: strconv.Itoa(os.Getuid()), WaitReadyBackoff: &defaultBackoffConfig, }, From dd142b6d471794885a580921face75324fa53b35 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Mon, 8 Nov 2021 10:52:37 -0800 Subject: [PATCH 12/12] update dependency Signed-off-by: Ben Ye --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index d61cc2b406..a1123ffc0a 100644 --- a/go.mod +++ b/go.mod @@ -70,7 +70,7 @@ require ( go.uber.org/automaxprocs v1.4.0 go.uber.org/goleak v1.1.12 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e - golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f + golang.org/x/net v0.0.0-20211020060615-d418f374d309 golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/text v0.3.6