From 832a7deb76b3c97b9c972b7db3eeeee9cb3e2e67 Mon Sep 17 00:00:00 2001 From: Craig Peterson <192540+captncraig@users.noreply.github.com> Date: Mon, 10 Jul 2023 13:48:20 -0400 Subject: [PATCH 1/3] pkg/reloader: use watchInterval timeout for initial apply Signed-off-by: Craig Peterson <192540+captncraig@users.noreply.github.com> --- pkg/reloader/reloader.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/reloader/reloader.go b/pkg/reloader/reloader.go index 1072026c2a..fb8ed418f5 100644 --- a/pkg/reloader/reloader.go +++ b/pkg/reloader/reloader.go @@ -202,13 +202,14 @@ func (r *Reloader) Watch(ctx context.Context) error { } defer runutil.CloseWithLogOnErr(r.logger, r.watcher, "config watcher close") + applyCtx, applyCancel := context.WithTimeout(ctx, r.watchInterval) if r.cfgFile != "" { if err := r.watcher.addFile(r.cfgFile); err != nil { return errors.Wrapf(err, "add config file %s to watcher", r.cfgFile) } - if err := r.apply(ctx); err != nil { + if err := r.apply(applyCtx); err != nil { return err } } @@ -238,7 +239,9 @@ func (r *Reloader) Watch(ctx context.Context) error { "out", r.cfgOutputFile, "dirs", strings.Join(r.watchedDirs, ",")) - applyCtx, applyCancel := context.WithTimeout(ctx, r.watchInterval) + // Reset the watch timeout. + applyCancel() + applyCtx, applyCancel = context.WithTimeout(ctx, r.watchInterval) for { select { From ce4709a65808a6e76c67fb7f1bb74e4756768e30 Mon Sep 17 00:00:00 2001 From: Craig Peterson <192540+captncraig@users.noreply.github.com> Date: Mon, 10 Jul 2023 14:17:58 -0400 Subject: [PATCH 2/3] changelog Signed-off-by: Craig Peterson <192540+captncraig@users.noreply.github.com> --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 133784cd8a..f7fb390c1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re - [#6325](https://github.com/thanos-io/thanos/pull/6325) Store: return gRPC resource exhausted error for byte limiter. - [#6399](https://github.com/thanos-io/thanos/pull/6399) *: Fix double-counting bug in http_request_duration metric - [#6428](https://github.com/thanos-io/thanos/pull/6428) Report gRPC connnection errors in the logs. +- [#6519](https://github.com/thanos-io/thanos/pull/6519) Reloader: Use timeout for initial apply. ### Changed - [#6049](https://github.com/thanos-io/thanos/pull/6049) Compact: *breaking :warning:* Replace group with resolution in compact metrics to avoid cardinality explosion on compact metrics for large numbers of groups. From 69dbe09462d6cd845121e34509060d7980e64678 Mon Sep 17 00:00:00 2001 From: Craig Peterson <192540+captncraig@users.noreply.github.com> Date: Tue, 11 Jul 2023 14:46:34 -0400 Subject: [PATCH 3/3] use distinct context for initial sync Signed-off-by: Craig Peterson <192540+captncraig@users.noreply.github.com> --- pkg/reloader/reloader.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/reloader/reloader.go b/pkg/reloader/reloader.go index fb8ed418f5..2057ea5f75 100644 --- a/pkg/reloader/reloader.go +++ b/pkg/reloader/reloader.go @@ -202,14 +202,15 @@ func (r *Reloader) Watch(ctx context.Context) error { } defer runutil.CloseWithLogOnErr(r.logger, r.watcher, "config watcher close") - applyCtx, applyCancel := context.WithTimeout(ctx, r.watchInterval) if r.cfgFile != "" { if err := r.watcher.addFile(r.cfgFile); err != nil { return errors.Wrapf(err, "add config file %s to watcher", r.cfgFile) } - - if err := r.apply(applyCtx); err != nil { + initialSyncCtx, initialSyncCancel := context.WithTimeout(ctx, r.watchInterval) + err := r.apply(initialSyncCtx) + initialSyncCancel() + if err != nil { return err } } @@ -239,9 +240,7 @@ func (r *Reloader) Watch(ctx context.Context) error { "out", r.cfgOutputFile, "dirs", strings.Join(r.watchedDirs, ",")) - // Reset the watch timeout. - applyCancel() - applyCtx, applyCancel = context.WithTimeout(ctx, r.watchInterval) + applyCtx, applyCancel := context.WithTimeout(ctx, r.watchInterval) for { select {