Skip to content

Commit

Permalink
query-tee: gracefully stop inflight requests when shutting down (#8985)
Browse files Browse the repository at this point in the history
* query-tee: gracefully stop inflight requests when shutting down

* Add changelog entry
  • Loading branch information
charleskorn authored Aug 13, 2024
1 parent 1dfcdd5 commit 99716c8
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 1 deletion.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,7 @@
* [ENHANCEMENT] Compare native histograms in query results when comparing results between two backends. #8724
* [ENHANCEMENT] Don't consider responses to be different during response comparison if both backends' responses contain different series, but all samples are within the recent sample window. #8749 #8894
* [ENHANCEMENT] When the expected and actual response for a matrix series is different, the full set of samples for that series from both backends will now be logged. #8947
* [ENHANCEMENT] Wait up to `-server.graceful-shutdown-timeout` for inflight requests to finish when shutting down, rather than immediately terminating inflight requests on shutdown. #8985
* [BUGFIX] Ensure any errors encountered while forwarding a request to a backend (eg. DNS resolution failures) are logged. #8419
* [BUGFIX] The comparison of the results should not fail when either side contains extra samples from within SkipRecentSamples duration. #8920

Expand Down
4 changes: 3 additions & 1 deletion tools/querytee/proxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
type ProxyConfig struct {
ServerHTTPServiceAddress string
ServerHTTPServicePort int
ServerGracefulShutdownTimeout time.Duration
ServerGRPCServiceAddress string
ServerGRPCServicePort int
BackendEndpoints string
Expand All @@ -46,6 +47,7 @@ type ProxyConfig struct {
func (cfg *ProxyConfig) RegisterFlags(f *flag.FlagSet) {
f.StringVar(&cfg.ServerHTTPServiceAddress, "server.http-service-address", "", "Bind address for server where query-tee service listens for HTTP requests.")
f.IntVar(&cfg.ServerHTTPServicePort, "server.http-service-port", 80, "The HTTP port where the query-tee service listens for HTTP requests.")
f.DurationVar(&cfg.ServerGracefulShutdownTimeout, "server.graceful-shutdown-timeout", 30*time.Second, "Time to wait for inflight requests to complete when shutting down. Setting this to 0 will terminate all inflight requests immediately when a shutdown signal is received.")
f.StringVar(&cfg.ServerGRPCServiceAddress, "server.grpc-service-address", "", "Bind address for server where query-tee service listens for HTTP over gRPC requests.")
f.IntVar(&cfg.ServerGRPCServicePort, "server.grpc-service-port", 9095, "The GRPC port where the query-tee service listens for HTTP over gRPC messages.")
f.StringVar(&cfg.BackendEndpoints, "backend.endpoints", "",
Expand Down Expand Up @@ -189,7 +191,7 @@ func (p *Proxy) Start() error {
HTTPListenPort: p.cfg.ServerHTTPServicePort,
HTTPServerReadTimeout: 1 * time.Minute,
HTTPServerWriteTimeout: 2 * time.Minute,
ServerGracefulShutdownTimeout: 0,
ServerGracefulShutdownTimeout: p.cfg.ServerGracefulShutdownTimeout,

// gRPC configs
GRPCListenAddress: p.cfg.ServerGRPCServiceAddress,
Expand Down

0 comments on commit 99716c8

Please sign in to comment.