Run Stop calls in parallel, to mitigate blocking behaviour of `Stop…

…` introduced in prometheus/prometheus@086be26 when draining is enabled

pkg/ruler/manager.go

@@ -387,11 +387,20 @@ func (r *DefaultMultiTenantManager) Stop() {
 
 	// Stop notifiers after all rule evaluations have finished, so that we have
 	// a chance to send any notifications generated while shutting down.
+	// rulerNotifier.stop() may take some time to complete if notifications need to be drained from the queue.
+	level.Info(r.logger).Log("msg", "stopping user notifiers")
+	wg = sync.WaitGroup{}
 	r.notifiersMtx.Lock()
 	for _, n := range r.notifiers {
-		n.stop()
+		wg.Add(1)
+		go func(n *rulerNotifier) {
+			defer wg.Done()
+			n.stop()
+		}(n)
 	}
+	wg.Wait()
 	r.notifiersMtx.Unlock()
+	level.Info(r.logger).Log("msg", "all user notifiers stopped")
 
 	// cleanup user rules directories
 	r.mapper.cleanup()