diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e0c4df3102..25ffcf972c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,11 +12,12 @@ We use *breaking* word for marking changes that are not backward compatible (rel ## Unreleased ## Added +- [#1538](https://github.com/thanos-io/thanos/pull/1538) Added `/-/ready` and `/-/healthy` endpoints to Thanos Rule. -[1533](https://github.com/thanos-io/thanos/pull/1533) Thanos inspect now supports the timeout flag. ### Fixed --[#1525](https://github.com/thanos-io/thanos/pull/1525) Thanos now deletes block's file in correct order allowing to detect partial blocks without problems. +-[#1525](https://github.com/thanos-io/thanos/pull/1525) Thanos now deletes block's file in correct order allowing to detect partial blocks without problems. -[#1505](https://github.com/thanos-io/thanos/pull/1505) Thanos store now removes invalid local cache blocks. ## v0.7.0 - 2019.09.02 diff --git a/cmd/thanos/main.go b/cmd/thanos/main.go index 123b9f6d77c..a5143be3710 100644 --- a/cmd/thanos/main.go +++ b/cmd/thanos/main.go @@ -76,7 +76,7 @@ func main() { registerSidecar(cmds, app) registerStore(cmds, app, "store") registerQuery(cmds, app) - registerRule(cmds, app, "rule") + registerRule(cmds, app) registerCompact(cmds, app) registerBucket(cmds, app, "bucket") registerDownsample(cmds, app, "downsample") diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index 43ce9a0390a..759eb76d1c4 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -41,6 +41,7 @@ import ( "github.com/thanos-io/thanos/pkg/extprom" extpromhttp "github.com/thanos-io/thanos/pkg/extprom/http" "github.com/thanos-io/thanos/pkg/objstore/client" + "github.com/thanos-io/thanos/pkg/prober" "github.com/thanos-io/thanos/pkg/promclient" thanosrule "github.com/thanos-io/thanos/pkg/rule" v1 "github.com/thanos-io/thanos/pkg/rule/api" @@ -54,8 +55,9 @@ import ( ) // registerRule registers a rule command. -func registerRule(m map[string]setupFunc, app *kingpin.Application, name string) { - cmd := app.Command(name, "ruler evaluating Prometheus rules against given Query nodes, exposing Store API and storing old blocks in bucket") +func registerRule(m map[string]setupFunc, app *kingpin.Application) { + comp := component.Rule + cmd := app.Command(comp.String(), "ruler evaluating Prometheus rules against given Query nodes, exposing Store API and storing old blocks in bucket") grpcBindAddr, httpBindAddr, cert, key, clientCA := regCommonServerFlags(cmd) @@ -104,7 +106,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application, name string) dnsSDResolver := cmd.Flag("query.sd-dns-resolver", "Resolver to use. Possible options: [golang, miekgdns]"). Default("golang").Hidden().String() - m[name] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { + m[comp.String()] = func(g *run.Group, logger log.Logger, reg *prometheus.Registry, tracer opentracing.Tracer, _ bool) error { lset, err := parseFlagLabels(*labelStrs) if err != nil { return errors.Wrap(err, "parse labels") @@ -169,6 +171,7 @@ func registerRule(m map[string]setupFunc, app *kingpin.Application, name string) fileSD, time.Duration(*dnsSDInterval), *dnsSDResolver, + comp, ) } } @@ -202,6 +205,7 @@ func runRule( fileSD *file.Discovery, dnsSDInterval time.Duration, dnsSDResolver string, + comp component.Component, ) error { configSuccess := prometheus.NewGauge(prometheus.GaugeOpts{ Name: "thanos_rule_config_last_reload_successful", @@ -482,6 +486,7 @@ func runRule( cancel() }) } + statusProber := prober.NewProber(comp, logger, prometheus.WrapRegistererWithPrefix("thanos_", reg)) // Start gRPC server. { l, err := net.Listen("tcp", grpcBindAddr) @@ -499,6 +504,7 @@ func runRule( s := newStoreGRPCServer(logger, reg, tracer, store, opts) g.Add(func() error { + statusProber.SetReady() return errors.Wrap(s.Serve(l), "serve gRPC") }, func(error) { s.Stop() @@ -532,22 +538,10 @@ func runRule( api := v1.NewAPI(logger, reg, ruleMgrs) api.Register(router.WithPrefix(path.Join(webRoutePrefix, "/api/v1")), tracer, logger, ins) - mux := http.NewServeMux() - registerMetrics(mux, reg) - registerProfile(mux) - mux.Handle("/", router) - - l, err := net.Listen("tcp", httpBindAddr) - if err != nil { - return errors.Wrapf(err, "listen HTTP on address %s", httpBindAddr) + // Initiate HTTP listener providing metrics endpoint and readiness/liveness probes. + if err := scheduleHTTPServer(g, logger, reg, statusProber, httpBindAddr, router, comp); err != nil { + return errors.Wrap(err, "schedule HTTP server with probes") } - - g.Add(func() error { - level.Info(logger).Log("msg", "Listening for ui requests", "address", httpBindAddr) - return errors.Wrap(http.Serve(l, mux), "serve query") - }, func(error) { - runutil.CloseWithLogOnErr(logger, l, "query and metric listener") - }) } confContentYaml, err := objStoreConfig.Content() diff --git a/tutorials/kubernetes-demo/manifests/thanos-ruler.yaml b/tutorials/kubernetes-demo/manifests/thanos-ruler.yaml index 8f90c022944..9adf1589ab0 100644 --- a/tutorials/kubernetes-demo/manifests/thanos-ruler.yaml +++ b/tutorials/kubernetes-demo/manifests/thanos-ruler.yaml @@ -76,6 +76,14 @@ spec: containerPort: 10902 - name: grpc containerPort: 10901 + livenessProbe: + httpGet: + port: http + path: /-/healthy + readinessProbe: + httpGet: + port: http + path: /-/ready resources: limits: cpu: 500m @@ -105,4 +113,4 @@ spec: name: http selector: statefulset.kubernetes.io/pod-name: thanos-ruler-0 - type: NodePort \ No newline at end of file + type: NodePort