Fix cos client MaxIdleConnsPerHost too low, too many TIME_WAIT (thano…

…s-io#4479)

The http.Transport is not auto-tuning and one size does not seem to fit all cases.
Add `HTTPConfig` Allow to tune most of the http.Transport parameters and `MaxIdleConnsPerHost` tune to 100.

Signed-off-by: hanjm <hanjinming@outlook.com>

pkg/objstore/cos/cos.go

@@ -16,10 +16,13 @@ import (
 	"github.com/go-kit/kit/log"
 	"github.com/mozillazg/go-cos"
 	"github.com/pkg/errors"
+	"github.com/prometheus/common/model"
+	"gopkg.in/yaml.v2"
+
+	"github.com/thanos-io/thanos/pkg/exthttp"
 	"github.com/thanos-io/thanos/pkg/objstore"
 	"github.com/thanos-io/thanos/pkg/objstore/clientutil"
 	"github.com/thanos-io/thanos/pkg/runutil"
-	"gopkg.in/yaml.v2"
 )
 
 // DirDelim is the delimiter used to model a directory structure in an object store bucket.
@@ -32,13 +35,27 @@ type Bucket struct {
 	name   string
 }
 
+// DefaultConfig is the default config for an cos client. default tune the `MaxIdleConnsPerHost`.
+var DefaultConfig = Config{
+	HTTPConfig: HTTPConfig{
+		IdleConnTimeout:       model.Duration(90 * time.Second),
+		ResponseHeaderTimeout: model.Duration(2 * time.Minute),
+		TLSHandshakeTimeout:   model.Duration(10 * time.Second),
+		ExpectContinueTimeout: model.Duration(1 * time.Second),
+		MaxIdleConns:          100,
+		MaxIdleConnsPerHost:   100,
+		MaxConnsPerHost:       0,
+	},
+}
+
 // Config encapsulates the necessary config values to instantiate an cos client.
 type Config struct {
-	Bucket    string `yaml:"bucket"`
-	Region    string `yaml:"region"`
-	AppId     string `yaml:"app_id"`
-	SecretKey string `yaml:"secret_key"`
-	SecretId  string `yaml:"secret_id"`
+	Bucket     string     `yaml:"bucket"`
+	Region     string     `yaml:"region"`
+	AppId      string     `yaml:"app_id"`
+	SecretKey  string     `yaml:"secret_key"`
+	SecretId   string     `yaml:"secret_id"`
+	HTTPConfig HTTPConfig `yaml:"http_config"`
 }
 
 // Validate checks to see if mandatory cos config options are set.
@@ -53,14 +70,48 @@ func (conf *Config) validate() error {
 	return nil
 }
 
+// parseConfig unmarshal a buffer into a Config with default HTTPConfig values.
+func parseConfig(conf []byte) (Config, error) {
+	config := DefaultConfig
+	if err := yaml.Unmarshal(conf, &config); err != nil {
+		return Config{}, err
+	}
+
+	return config, nil
+}
+
+// HTTPConfig stores the http.Transport configuration for the cos client.
+type HTTPConfig struct {
+	IdleConnTimeout       model.Duration `yaml:"idle_conn_timeout"`
+	ResponseHeaderTimeout model.Duration `yaml:"response_header_timeout"`
+	TLSHandshakeTimeout   model.Duration `yaml:"tls_handshake_timeout"`
+	ExpectContinueTimeout model.Duration `yaml:"expect_continue_timeout"`
+	MaxIdleConns          int            `yaml:"max_idle_conns"`
+	MaxIdleConnsPerHost   int            `yaml:"max_idle_conns_per_host"`
+	MaxConnsPerHost       int            `yaml:"max_conns_per_host"`
+}
+
+// DefaultTransport build http.Transport from config.
+func DefaultTransport(c HTTPConfig) *http.Transport {
+	transport := exthttp.NewTransport()
+	transport.IdleConnTimeout = time.Duration(c.IdleConnTimeout)
+	transport.ResponseHeaderTimeout = time.Duration(c.ResponseHeaderTimeout)
+	transport.TLSHandshakeTimeout = time.Duration(c.TLSHandshakeTimeout)
+	transport.ExpectContinueTimeout = time.Duration(c.ExpectContinueTimeout)
+	transport.MaxIdleConns = c.MaxIdleConns
+	transport.MaxIdleConnsPerHost = c.MaxIdleConnsPerHost
+	transport.MaxConnsPerHost = c.MaxConnsPerHost
+	return transport
+}
+
 // NewBucket returns a new Bucket using the provided cos configuration.
 func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error) {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
 
-	var config Config
-	if err := yaml.Unmarshal(conf, &config); err != nil {
+	config, err := parseConfig(conf)
+	if err != nil {
 		return nil, errors.Wrap(err, "parsing cos configuration")
 	}
 	if err := config.validate(); err != nil {
@@ -78,6 +129,7 @@ func NewBucket(logger log.Logger, conf []byte, component string) (*Bucket, error
 		Transport: &cos.AuthorizationTransport{
 			SecretID:  config.SecretId,
 			SecretKey: config.SecretKey,
+			Transport: DefaultTransport(config.HTTPConfig),
 		},
 	})
 

pkg/objstore/cos/cos_test.go

@@ -0,0 +1,61 @@
+package cos
+
+import (
+	"testing"
+	"time"
+
+	"github.com/prometheus/common/model"
+	"github.com/thanos-io/thanos/pkg/testutil"
+)
+
+func Test_parseConfig(t *testing.T) {
+	type args struct {
+		conf []byte
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    Config
+		wantErr bool
+	}{
+		{
+			name: "empty",
+			args: args{
+				conf: []byte(""),
+			},
+			want:    DefaultConfig,
+			wantErr: false,
+		},
+		{
+			name: "max_idle_conns",
+			args: args{
+				conf: []byte(`
+http_config:
+  max_idle_conns: 200
+`),
+			},
+			want: Config{
+				HTTPConfig: HTTPConfig{
+					IdleConnTimeout:       model.Duration(90 * time.Second),
+					ResponseHeaderTimeout: model.Duration(2 * time.Minute),
+					TLSHandshakeTimeout:   model.Duration(10 * time.Second),
+					ExpectContinueTimeout: model.Duration(1 * time.Second),
+					MaxIdleConns:          200,
+					MaxIdleConnsPerHost:   100,
+					MaxConnsPerHost:       0,
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := parseConfig(tt.args.conf)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			testutil.Equals(t, tt.want, got)
+		})
+	}
+}