From 93783be49fa31c8a80a03bb018f872bbccab9c63 Mon Sep 17 00:00:00 2001 From: Charles Korn Date: Fri, 14 Jun 2024 13:53:16 +1000 Subject: [PATCH] Run `Stop` calls in parallel, to mitigate blocking behaviour of `Stop` introduced in https://github.com/prometheus/prometheus/pull/14290/commits/086be262a8483589400828cd7f5f79e7b7b367dd when draining is enabled --- pkg/ruler/manager.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/ruler/manager.go b/pkg/ruler/manager.go index 5b5865810ba..f7cea08434b 100644 --- a/pkg/ruler/manager.go +++ b/pkg/ruler/manager.go @@ -387,11 +387,20 @@ func (r *DefaultMultiTenantManager) Stop() { // Stop notifiers after all rule evaluations have finished, so that we have // a chance to send any notifications generated while shutting down. + // rulerNotifier.stop() may take some time to complete if notifications need to be drained from the queue. + level.Info(r.logger).Log("msg", "stopping user notifiers") + wg = sync.WaitGroup{} r.notifiersMtx.Lock() for _, n := range r.notifiers { - n.stop() + wg.Add(1) + go func(n *rulerNotifier) { + defer wg.Done() + n.stop() + }(n) } + wg.Wait() r.notifiersMtx.Unlock() + level.Info(r.logger).Log("msg", "all user notifiers stopped") // cleanup user rules directories r.mapper.cleanup()