Skip to content

Commit

Permalink
mimir: Add TLS config to S3 client (#7959)
Browse files Browse the repository at this point in the history 
Signed-off-by: Levi Harrison <git@leviharrison.dev>
Co-authored-by: Levi Harrison <git@leviharrison.dev>
Co-authored-by: Jack Baldry <jack.baldry@grafana.com>
  • Loading branch information
@Itaykal @LeviHarrison @jdbaldry
3 people authored Apr 25, 2024
1 parent 00a2d1d commit 40d9a8a
Show file tree
Hide file tree
Showing 6 changed files with 254 additions and 0 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
* [ENHANCEMENT] Query-frontend: use zero-allocation experimental decoder for active series queries via `-query-frontend.use-active-series-decoder`. #7665
* [ENHANCEMENT] Go: updated to 1.22.2. #7802
* [ENHANCEMENT] Query-frontend: support `limit` parameter on `/prometheus/api/v1/label/{name}/values` and `/prometheus/api/v1/labels` endpoints. #7722
* [ENHANCEMENT] Expose TLS configuration for the S3 backend client. #2652
* [BUGFIX] Rules: improve error handling when querier is local to the ruler. #7567
* [BUGFIX] Querier, store-gateway: Protect against panics raised during snappy encoding. #7520
* [BUGFIX] Ingester: Prevent timely compaction of empty blocks. #7624
Expand Down
176 changes: 176 additions & 0 deletions cmd/mimir/config-descriptor.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6054,6 +6054,50 @@
"fieldFlag": "blocks-storage.s3.max-connections-per-host",
"fieldType": "int",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_ca_path",
"required": false,
"desc": "Path to the CA certificates to validate server certificate against. If not set, the host's root CA certificates are used.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "blocks-storage.s3.http.tls-ca-path",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_cert_path",
"required": false,
"desc": "Path to the client certificate, which will be used for authenticating with the server. Also requires the key path to be configured.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "blocks-storage.s3.http.tls-cert-path",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_key_path",
"required": false,
"desc": "Path to the key for the client certificate. Also requires the client certificate to be configured.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "blocks-storage.s3.http.tls-key-path",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_server_name",
"required": false,
"desc": "Override the expected name on the server certificate.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "blocks-storage.s3.http.tls-server-name",
"fieldType": "string",
"fieldCategory": "advanced"
}
],
"fieldValue": null,
Expand Down Expand Up @@ -11939,6 +11983,50 @@
"fieldFlag": "ruler-storage.s3.max-connections-per-host",
"fieldType": "int",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_ca_path",
"required": false,
"desc": "Path to the CA certificates to validate server certificate against. If not set, the host's root CA certificates are used.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "ruler-storage.s3.http.tls-ca-path",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_cert_path",
"required": false,
"desc": "Path to the client certificate, which will be used for authenticating with the server. Also requires the key path to be configured.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "ruler-storage.s3.http.tls-cert-path",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_key_path",
"required": false,
"desc": "Path to the key for the client certificate. Also requires the client certificate to be configured.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "ruler-storage.s3.http.tls-key-path",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_server_name",
"required": false,
"desc": "Override the expected name on the server certificate.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "ruler-storage.s3.http.tls-server-name",
"fieldType": "string",
"fieldCategory": "advanced"
}
],
"fieldValue": null,
Expand Down Expand Up @@ -14024,6 +14112,50 @@
"fieldFlag": "alertmanager-storage.s3.max-connections-per-host",
"fieldType": "int",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_ca_path",
"required": false,
"desc": "Path to the CA certificates to validate server certificate against. If not set, the host's root CA certificates are used.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "alertmanager-storage.s3.http.tls-ca-path",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_cert_path",
"required": false,
"desc": "Path to the client certificate, which will be used for authenticating with the server. Also requires the key path to be configured.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "alertmanager-storage.s3.http.tls-cert-path",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_key_path",
"required": false,
"desc": "Path to the key for the client certificate. Also requires the client certificate to be configured.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "alertmanager-storage.s3.http.tls-key-path",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_server_name",
"required": false,
"desc": "Override the expected name on the server certificate.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "alertmanager-storage.s3.http.tls-server-name",
"fieldType": "string",
"fieldCategory": "advanced"
}
],
"fieldValue": null,
Expand Down Expand Up @@ -16342,6 +16474,50 @@
"fieldFlag": "common.storage.s3.max-connections-per-host",
"fieldType": "int",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_ca_path",
"required": false,
"desc": "Path to the CA certificates to validate server certificate against. If not set, the host's root CA certificates are used.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "common.storage.s3.http.tls-ca-path",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_cert_path",
"required": false,
"desc": "Path to the client certificate, which will be used for authenticating with the server. Also requires the key path to be configured.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "common.storage.s3.http.tls-cert-path",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_key_path",
"required": false,
"desc": "Path to the key for the client certificate. Also requires the client certificate to be configured.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "common.storage.s3.http.tls-key-path",
"fieldType": "string",
"fieldCategory": "advanced"
},
{
"kind": "field",
"name": "tls_server_name",
"required": false,
"desc": "Override the expected name on the server certificate.",
"fieldValue": null,
"fieldDefaultValue": "",
"fieldFlag": "common.storage.s3.http.tls-server-name",
"fieldType": "string",
"fieldCategory": "advanced"
}
],
"fieldValue": null,
Expand Down
32 changes: 32 additions & 0 deletions cmd/mimir/help-all.txt.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,14 @@ Usage of ./cmd/mimir/mimir:
If the client connects to S3 via HTTPS and this option is enabled, the client will accept any certificate and hostname.
-alertmanager-storage.s3.http.response-header-timeout duration
The amount of time the client will wait for a servers response headers. (default 2m0s)
-alertmanager-storage.s3.http.tls-ca-path string
Path to the CA certificates to validate server certificate against. If not set, the host's root CA certificates are used.
-alertmanager-storage.s3.http.tls-cert-path string
Path to the client certificate, which will be used for authenticating with the server. Also requires the key path to be configured.
-alertmanager-storage.s3.http.tls-key-path string
Path to the key for the client certificate. Also requires the client certificate to be configured.
-alertmanager-storage.s3.http.tls-server-name string
Override the expected name on the server certificate.
-alertmanager-storage.s3.insecure
If enabled, use http:// for the S3 endpoint instead of https://. This could be useful in local dev/test environments while using an S3-compatible backend storage, like Minio.
-alertmanager-storage.s3.list-objects-version string
Expand Down Expand Up @@ -685,6 +693,14 @@ Usage of ./cmd/mimir/mimir:
If the client connects to S3 via HTTPS and this option is enabled, the client will accept any certificate and hostname.
-blocks-storage.s3.http.response-header-timeout duration
The amount of time the client will wait for a servers response headers. (default 2m0s)
-blocks-storage.s3.http.tls-ca-path string
Path to the CA certificates to validate server certificate against. If not set, the host's root CA certificates are used.
-blocks-storage.s3.http.tls-cert-path string
Path to the client certificate, which will be used for authenticating with the server. Also requires the key path to be configured.
-blocks-storage.s3.http.tls-key-path string
Path to the key for the client certificate. Also requires the client certificate to be configured.
-blocks-storage.s3.http.tls-server-name string
Override the expected name on the server certificate.
-blocks-storage.s3.insecure
If enabled, use http:// for the S3 endpoint instead of https://. This could be useful in local dev/test environments while using an S3-compatible backend storage, like Minio.
-blocks-storage.s3.list-objects-version string
Expand Down Expand Up @@ -855,6 +871,14 @@ Usage of ./cmd/mimir/mimir:
If the client connects to S3 via HTTPS and this option is enabled, the client will accept any certificate and hostname.
-common.storage.s3.http.response-header-timeout duration
The amount of time the client will wait for a servers response headers. (default 2m0s)
-common.storage.s3.http.tls-ca-path string
Path to the CA certificates to validate server certificate against. If not set, the host's root CA certificates are used.
-common.storage.s3.http.tls-cert-path string
Path to the client certificate, which will be used for authenticating with the server. Also requires the key path to be configured.
-common.storage.s3.http.tls-key-path string
Path to the key for the client certificate. Also requires the client certificate to be configured.
-common.storage.s3.http.tls-server-name string
Override the expected name on the server certificate.
-common.storage.s3.insecure
If enabled, use http:// for the S3 endpoint instead of https://. This could be useful in local dev/test environments while using an S3-compatible backend storage, like Minio.
-common.storage.s3.list-objects-version string
Expand Down Expand Up @@ -2277,6 +2301,14 @@ Usage of ./cmd/mimir/mimir:
If the client connects to S3 via HTTPS and this option is enabled, the client will accept any certificate and hostname.
-ruler-storage.s3.http.response-header-timeout duration
The amount of time the client will wait for a servers response headers. (default 2m0s)
-ruler-storage.s3.http.tls-ca-path string
Path to the CA certificates to validate server certificate against. If not set, the host's root CA certificates are used.
-ruler-storage.s3.http.tls-cert-path string
Path to the client certificate, which will be used for authenticating with the server. Also requires the key path to be configured.
-ruler-storage.s3.http.tls-key-path string
Path to the key for the client certificate. Also requires the client certificate to be configured.
-ruler-storage.s3.http.tls-server-name string
Override the expected name on the server certificate.
-ruler-storage.s3.insecure
If enabled, use http:// for the S3 endpoint instead of https://. This could be useful in local dev/test environments while using an S3-compatible backend storage, like Minio.
-ruler-storage.s3.list-objects-version string
Expand Down
19 changes: 19 additions & 0 deletions docs/sources/mimir/configure/configuration-parameters/index.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4723,6 +4723,25 @@ http:
# (advanced) Maximum number of connections per host. 0 means no limit.
# CLI flag: -<prefix>.s3.max-connections-per-host
[max_connections_per_host: <int> | default = 0]
# (advanced) Path to the CA certificates to validate server certificate
# against. If not set, the host's root CA certificates are used.
# CLI flag: -<prefix>.s3.http.tls-ca-path
[tls_ca_path: <string> | default = ""]
# (advanced) Path to the client certificate, which will be used for
# authenticating with the server. Also requires the key path to be configured.
# CLI flag: -<prefix>.s3.http.tls-cert-path
[tls_cert_path: <string> | default = ""]
# (advanced) Path to the key for the client certificate. Also requires the
# client certificate to be configured.
# CLI flag: -<prefix>.s3.http.tls-key-path
[tls_key_path: <string> | default = ""]
# (advanced) Override the expected name on the server certificate.
# CLI flag: -<prefix>.s3.http.tls-server-name
[tls_server_name: <string> | default = ""]
```

### gcs_storage_backend
Expand Down
7 changes: 7 additions & 0 deletions pkg/storage/bucket/s3/bucket_client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/go-kit/log"
"github.com/prometheus/common/model"
"github.com/thanos-io/objstore"
"github.com/thanos-io/objstore/exthttp"
"github.com/thanos-io/objstore/providers/s3"
)

Expand Down Expand Up @@ -73,6 +74,12 @@ func newS3Config(cfg Config) (s3.Config, error) {
MaxIdleConnsPerHost: cfg.HTTP.MaxIdleConnsPerHost,
MaxConnsPerHost: cfg.HTTP.MaxConnsPerHost,
Transport: cfg.HTTP.Transport,
TLSConfig: exthttp.TLSConfig{
CAFile: cfg.HTTP.TLSConfig.CAPath,
CertFile: cfg.HTTP.TLSConfig.CertPath,
KeyFile: cfg.HTTP.TLSConfig.KeyPath,
ServerName: cfg.HTTP.TLSConfig.ServerName,
},
},
// Enforce signature version 2 if CLI flag is set
SignatureV2: cfg.SignatureVersion == SignatureVersionV2,
Expand Down
19 changes: 19 additions & 0 deletions pkg/storage/bucket/s3/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,16 @@ type HTTPConfig struct {

// Allow upstream callers to inject a round tripper
Transport http.RoundTripper `yaml:"-"`

TLSConfig TLSConfig `yaml:",inline"`
}

// TLSConfig configures the options for TLS connections.
type TLSConfig struct {
CAPath string `yaml:"tls_ca_path" category:"advanced"`
CertPath string `yaml:"tls_cert_path" category:"advanced"`
KeyPath string `yaml:"tls_key_path" category:"advanced"`
ServerName string `yaml:"tls_server_name" category:"advanced"`
}

// RegisterFlagsWithPrefix registers the flags for s3 storage with the provided prefix
Expand All @@ -90,6 +100,15 @@ func (cfg *HTTPConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
f.IntVar(&cfg.MaxIdleConns, prefix+"s3.max-idle-connections", 100, "Maximum number of idle (keep-alive) connections across all hosts. 0 means no limit.")
f.IntVar(&cfg.MaxIdleConnsPerHost, prefix+"s3.max-idle-connections-per-host", 100, "Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used.")
f.IntVar(&cfg.MaxConnsPerHost, prefix+"s3.max-connections-per-host", 0, "Maximum number of connections per host. 0 means no limit.")
cfg.TLSConfig.RegisterFlagsWithPrefix(prefix, f)
}

// RegisterFlagsWithPrefix registers the flags for s3 storage with the provided prefix.
func (cfg *TLSConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
f.StringVar(&cfg.CAPath, prefix+"s3.http.tls-ca-path", "", "Path to the CA certificates to validate server certificate against. If not set, the host's root CA certificates are used.")
f.StringVar(&cfg.CertPath, prefix+"s3.http.tls-cert-path", "", "Path to the client certificate, which will be used for authenticating with the server. Also requires the key path to be configured.")
f.StringVar(&cfg.KeyPath, prefix+"s3.http.tls-key-path", "", "Path to the key for the client certificate. Also requires the client certificate to be configured.")
f.StringVar(&cfg.ServerName, prefix+"s3.http.tls-server-name", "", "Override the expected name on the server certificate.")
}

// Config holds the config options for an S3 backend
Expand Down

0 comments on commit 40d9a8a

Please sign in to comment.