rule: Add liveness and readiness probe (#1538)

* Add prober to rule

Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com>

* Add changelog entry

Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com>

* Update README

Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com>

* Apply suggestions from code review

Co-Authored-By: Bartlomiej Plotka <bwplotka@gmail.com>
Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com>

* Update rule tutorial

Signed-off-by: Kemal Akkoyun <kakkoyun@gmail.com>
Signed-off-by: Giedrius Statkevičius <giedriuswork@gmail.com>

CHANGELOG.md

@@ -12,11 +12,12 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 ## Unreleased
 
 ## Added
+- [#1538](https://github.com/thanos-io/thanos/pull/1538) Added `/-/ready` and `/-/healthy` endpoints to Thanos Rule.
 -[1533](https://github.com/thanos-io/thanos/pull/1533) Thanos inspect now supports the timeout flag.
 
 ### Fixed
 
--[#1525](https://github.com/thanos-io/thanos/pull/1525) Thanos now deletes block's file in correct order allowing to detect partial blocks without problems. 
+-[#1525](https://github.com/thanos-io/thanos/pull/1525) Thanos now deletes block's file in correct order allowing to detect partial blocks without problems.
 -[#1505](https://github.com/thanos-io/thanos/pull/1505) Thanos store now removes invalid local cache blocks.
 
 ## v0.7.0 - 2019.09.02

cmd/thanos/main.go

@@ -76,7 +76,7 @@ func main() {
 	registerSidecar(cmds, app)
 	registerStore(cmds, app, "store")
 	registerQuery(cmds, app)
-	registerRule(cmds, app, "rule")
+	registerRule(cmds, app)
 	registerCompact(cmds, app)
 	registerBucket(cmds, app, "bucket")
 	registerDownsample(cmds, app, "downsample")

cmd/thanos/rule.go

@@ -41,6 +41,7 @@ import (
 	"github.com/thanos-io/thanos/pkg/extprom"
 	extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http"
 	"github.com/thanos-io/thanos/pkg/objstore/client"
+	"github.com/thanos-io/thanos/pkg/prober"
 	"github.com/thanos-io/thanos/pkg/promclient"
 	thanosrule "github.com/thanos-io/thanos/pkg/rule"
 	v1 "github.com/thanos-io/thanos/pkg/rule/api"
@@ -54,8 +55,9 @@ import (
 )
 
 // registerRule registers a rule command.
-func registerRule(m map[string]setupFunc, app *kingpin.Application, name string) {
-	cmd := app.Command(name, "ruler evaluating Prometheus rules against given Query nodes, exposing Store API and storing old blocks in bucket")
+func registerRule(m map[string]setupFunc, app *kingpin.Application) {
+	comp := component.Rule
+	cmd := app.Command(comp.String(), "ruler evaluating Prometheus rules against given Query nodes, exposing Store API and storing old blocks in bucket")
 
 	grpcBindAddr, httpBindAddr, cert, key, clientCA := regCommonServerFlags(cmd)
 
@@ -104,7 +106,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application, name string)
 	dnsSDResolver := cmd.Flag("query.sd-dns-resolver", "Resolver to use. Possible options: [golang, miekgdns]").
 		Default("golang").Hidden().String()
 
-	m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error {
+	m[comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error {
 		lset, err := parseFlagLabels(*labelStrs)
 		if err != nil {
 			return errors.Wrap(err, "parse labels")
@@ -169,6 +171,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application, name string)
 			fileSD,
 			time.Duration(*dnsSDInterval),
 			*dnsSDResolver,
+			comp,
 		)
 	}
 }
@@ -202,6 +205,7 @@ func runRule(
 	fileSD *file.Discovery,
 	dnsSDInterval time.Duration,
 	dnsSDResolver string,
+	comp component.Component,
 ) error {
 	configSuccess := prometheus.NewGauge(prometheus.GaugeOpts{
 		Name: "thanos_rule_config_last_reload_successful",
@@ -482,6 +486,7 @@ func runRule(
 			cancel()
 		})
 	}
+	statusProber := prober.NewProber(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg))
 	// Start gRPC server.
 	{
 		l, err := net.Listen("tcp", grpcBindAddr)
@@ -499,6 +504,7 @@ func runRule(
 		s := newStoreGRPCServer(logger, reg, tracer, store, opts)
 
 		g.Add(func() error {
+			statusProber.SetReady()
 			return errors.Wrap(s.Serve(l), "serve gRPC")
 		}, func(error) {
 			s.Stop()
@@ -532,22 +538,10 @@ func runRule(
 		api := v1.NewAPI(logger, reg, ruleMgrs)
 		api.Register(router.WithPrefix(path.Join(webRoutePrefix, "/api/v1")), tracer, logger, ins)
 
-		mux := http.NewServeMux()
-		registerMetrics(mux, reg)
-		registerProfile(mux)
-		mux.Handle("/", router)
-
-		l, err := net.Listen("tcp", httpBindAddr)
-		if err != nil {
-			return errors.Wrapf(err, "listen HTTP on address %s", httpBindAddr)
+		// Initiate HTTP listener providing metrics endpoint and readiness/liveness probes.
+		if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, router, comp); err != nil {
+			return errors.Wrap(err, "schedule HTTP server with probes")
 		}
-
-		g.Add(func() error {
-			level.Info(logger).Log("msg", "Listening for ui requests", "address", httpBindAddr)
-			return errors.Wrap(http.Serve(l, mux), "serve query")
-		}, func(error) {
-			runutil.CloseWithLogOnErr(logger, l, "query and metric listener")
-		})
 	}
 
 	confContentYaml, err := objStoreConfig.Content()

tutorials/kubernetes-demo/manifests/thanos-ruler.yaml

@@ -76,6 +76,14 @@ spec:
               containerPort: 10902
             - name: grpc
               containerPort: 10901
+          livenessProbe:
+            httpGet:
+              port: http
+              path: /-/healthy
+          readinessProbe:
+            httpGet:
+              port: http
+              path: /-/ready
           resources:
             limits:
               cpu: 500m
@@ -105,4 +113,4 @@ spec:
       name: http
   selector:
     statefulset.kubernetes.io/pod-name: thanos-ruler-0
-  type: NodePort
+  type: NodePort