diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f6011990e..da074a3811 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ### Added - [#4453](https://github.com/thanos-io/thanos/pull/4453) Tools: Add flag `--selector.relabel-config-file` / `--selector.relabel-config` / `--max-time` / `--min-time` to filter served blocks. +- [#4482](https://github.com/thanos-io/thanos/pull/4482) COS: Add http_config for cos object store client. ### Fixed diff --git a/docs/storage.md b/docs/storage.md index 1fa0749a8b..cdd4cfaabe 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -389,6 +389,14 @@ config: app_id: "" secret_key: "" secret_id: "" + http_config: + idle_conn_timeout: 1m30s + response_header_timeout: 2m + tls_handshake_timeout: 10s + expect_continue_timeout: 1s + max_idle_conns: 100 + max_idle_conns_per_host: 100 + max_conns_per_host: 0 ``` Set the flags `--objstore.config-file` to reference to the configuration file. diff --git a/pkg/objstore/cos/cos.go b/pkg/objstore/cos/cos.go index 3fb39948a1..f09d85f299 100644 --- a/pkg/objstore/cos/cos.go +++ b/pkg/objstore/cos/cos.go @@ -16,10 +16,13 @@ import ( "github.com/go-kit/kit/log" "github.com/mozillazg/go-cos" "github.com/pkg/errors" + "github.com/prometheus/common/model" + "gopkg.in/yaml.v2" + + "github.com/thanos-io/thanos/pkg/exthttp" "github.com/thanos-io/thanos/pkg/objstore" "github.com/thanos-io/thanos/pkg/objstore/clientutil" "github.com/thanos-io/thanos/pkg/runutil" - "gopkg.in/yaml.v2" ) // DirDelim is the delimiter used to model a directory structure in an object store bucket. @@ -32,13 +35,27 @@ type Bucket struct { name string } +// DefaultConfig is the default config for an cos client. default tune the `MaxIdleConnsPerHost`. +var DefaultConfig = Config{ + HTTPConfig: HTTPConfig{ + IdleConnTimeout: model.Duration(90 * time.Second), + ResponseHeaderTimeout: model.Duration(2 * time.Minute), + TLSHandshakeTimeout: model.Duration(10 * time.Second), + ExpectContinueTimeout: model.Duration(1 * time.Second), + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, + MaxConnsPerHost: 0, + }, +} + // Config encapsulates the necessary config values to instantiate an cos client. type Config struct { - Bucket string `yaml:"bucket"` - Region string `yaml:"region"` - AppId string `yaml:"app_id"` - SecretKey string `yaml:"secret_key"` - SecretId string `yaml:"secret_id"` + Bucket string `yaml:"bucket"` + Region string `yaml:"region"` + AppId string `yaml:"app_id"` + SecretKey string `yaml:"secret_key"` + SecretId string `yaml:"secret_id"` + HTTPConfig HTTPConfig `yaml:"http_config"` } // Validate checks to see if mandatory cos config options are set. @@ -53,14 +70,48 @@ func (conf *Config) validate() error { return nil } +// parseConfig unmarshal a buffer into a Config with default HTTPConfig values. +func parseConfig(conf []byte) (Config, error) { + config := DefaultConfig + if err := yaml.Unmarshal(conf, &config); err != nil { + return Config{}, err + } + + return config, nil +} + +// HTTPConfig stores the http.Transport configuration for the cos client. +type HTTPConfig struct { + IdleConnTimeout model.Duration `yaml:"idle_conn_timeout"` + ResponseHeaderTimeout model.Duration `yaml:"response_header_timeout"` + TLSHandshakeTimeout model.Duration `yaml:"tls_handshake_timeout"` + ExpectContinueTimeout model.Duration `yaml:"expect_continue_timeout"` + MaxIdleConns int `yaml:"max_idle_conns"` + MaxIdleConnsPerHost int `yaml:"max_idle_conns_per_host"` + MaxConnsPerHost int `yaml:"max_conns_per_host"` +} + +// DefaultTransport build http.Transport from config. +func DefaultTransport(c HTTPConfig) *http.Transport { + transport := exthttp.NewTransport() + transport.IdleConnTimeout = time.Duration(c.IdleConnTimeout) + transport.ResponseHeaderTimeout = time.Duration(c.ResponseHeaderTimeout) + transport.TLSHandshakeTimeout = time.Duration(c.TLSHandshakeTimeout) + transport.ExpectContinueTimeout = time.Duration(c.ExpectContinueTimeout) + transport.MaxIdleConns = c.MaxIdleConns + transport.MaxIdleConnsPerHost = c.MaxIdleConnsPerHost + transport.MaxConnsPerHost = c.MaxConnsPerHost + return transport +} + // NewBucket returns a new Bucket using the provided cos configuration. func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) { if logger == nil { logger = log.NewNopLogger() } - var config Config - if err := yaml.Unmarshal(conf, &config); err != nil { + config, err := parseConfig(conf) + if err != nil { return nil, errors.Wrap(err, "parsing cos configuration") } if err := config.validate(); err != nil { @@ -78,6 +129,7 @@ func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error Transport: &cos.AuthorizationTransport{ SecretID: config.SecretId, SecretKey: config.SecretKey, + Transport: DefaultTransport(config.HTTPConfig), }, }) diff --git a/pkg/objstore/cos/cos_test.go b/pkg/objstore/cos/cos_test.go new file mode 100644 index 0000000000..4779213293 --- /dev/null +++ b/pkg/objstore/cos/cos_test.go @@ -0,0 +1,64 @@ +// Copyright (c) The Thanos Authors. +// Licensed under the Apache License 2.0. + +package cos + +import ( + "testing" + "time" + + "github.com/prometheus/common/model" + "github.com/thanos-io/thanos/pkg/testutil" +) + +func Test_parseConfig(t *testing.T) { + type args struct { + conf []byte + } + tests := []struct { + name string + args args + want Config + wantErr bool + }{ + { + name: "empty", + args: args{ + conf: []byte(""), + }, + want: DefaultConfig, + wantErr: false, + }, + { + name: "max_idle_conns", + args: args{ + conf: []byte(` +http_config: + max_idle_conns: 200 +`), + }, + want: Config{ + HTTPConfig: HTTPConfig{ + IdleConnTimeout: model.Duration(90 * time.Second), + ResponseHeaderTimeout: model.Duration(2 * time.Minute), + TLSHandshakeTimeout: model.Duration(10 * time.Second), + ExpectContinueTimeout: model.Duration(1 * time.Second), + MaxIdleConns: 200, + MaxIdleConnsPerHost: 100, + MaxConnsPerHost: 0, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseConfig(tt.args.conf) + if (err != nil) != tt.wantErr { + t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + testutil.Equals(t, tt.want, got) + }) + } +} diff --git a/scripts/cfggen/main.go b/scripts/cfggen/main.go index 93780b221f..4388802c92 100644 --- a/scripts/cfggen/main.go +++ b/scripts/cfggen/main.go @@ -49,7 +49,7 @@ var ( client.GCS: gcs.Config{}, client.S3: s3.DefaultConfig, client.SWIFT: swift.DefaultConfig, - client.COS: cos.Config{}, + client.COS: cos.DefaultConfig, client.ALIYUNOSS: oss.Config{}, client.FILESYSTEM: filesystem.Config{}, }