Add support for Prometheus metrics

doc/http.md

@@ -107,6 +107,12 @@ Notes:
 - `/recover-host/<hostname>`: recover a previously skipped host.
 - `/skipped-hosts`:  list currently skipped hosts
 
+### Metric requests
+
+- `/aggregated-metrics`: aggregated metrics for MySQL clusters in json format.
+
+- `/metrics`: metrics in Prometheus format. Prefix for these metrics can be modified using `PrometheusNamespace` and `PrometheusSubsystem` in the config.
+
 ### Other requests
 
 - `/help`: show all supported request paths

go.mod

@@ -17,8 +17,15 @@ require (
 	vitess.io/vitess v2.1.1+incompatible
 )
 
+require github.com/prometheus/client_golang v0.9.2
+
 require (
+	github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
 	github.com/golang/protobuf v1.2.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.0.0 // indirect
 	github.com/hashicorp/golang-lru v0.5.0 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 // indirect
+	github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 // indirect
+	github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a // indirect
 )

go.sum

@@ -3,6 +3,7 @@ github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q
 github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
@@ -27,6 +28,7 @@ github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCO
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/outbrain/golib v0.0.0-20180830062331-ab954725f502 h1:oS2s2j8GP70jE0IDMyA4BO4HgvkNcjHfR40h2fyhc7s=
 github.com/outbrain/golib v0.0.0-20180830062331-ab954725f502/go.mod h1:JDhu//MMvcPVPH889Xr7DyamEbTLumgDBALGUyXrz1g=
@@ -36,9 +38,13 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
 github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ=
 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=

pkg/config/config.go

@@ -109,22 +109,25 @@ type ConfigurationSettings struct {
 	BackendMySQLCollation string   // if specified, use this collation instead of charset when connecting to MySQL backend
 	MemcacheServers       []string // if given, freno will report to aggregated values to given memcache
 	MemcachePath          string   // use as prefix to metric path in memcache key, e.g. if `MemcachePath` is "myprefix" the key would be "myprefix/mysql/maincluster". Default: "freno"
-	EnableProfiling       bool     // enable pprof profiling http api
+	PrometheusNamespace   string
+	PrometheusSubsystem   string
+	EnableProfiling       bool // enable pprof profiling http api
 	Stores                StoresSettings
 }
 
 func newConfigurationSettings() *ConfigurationSettings {
 	return &ConfigurationSettings{
-		ListenPort:         8087,
-		RaftBind:           "127.0.0.1:10008",
-		RaftDataDir:        "",
-		DefaultRaftPort:    0,
-		RaftNodes:          []string{},
-		BackendMySQLHost:   "",
-		BackendMySQLSchema: "",
-		BackendMySQLPort:   3306,
-		MemcacheServers:    []string{},
-		MemcachePath:       "freno",
+		ListenPort:          8087,
+		RaftBind:            "127.0.0.1:10008",
+		RaftDataDir:         "",
+		DefaultRaftPort:     0,
+		RaftNodes:           []string{},
+		BackendMySQLHost:    "",
+		BackendMySQLSchema:  "",
+		BackendMySQLPort:    3306,
+		MemcacheServers:     []string{},
+		MemcachePath:        "freno",
+		PrometheusNamespace: "freno",
 		//Debug:                                        false,
 		//ListenSocket:                                 "",
 		//AnExampleListOfStrings:                       []string{"*"},

pkg/http/api.go

@@ -17,6 +17,7 @@ import (
 	"github.com/rcrowley/go-metrics/exp"
 
 	"github.com/julienschmidt/httprouter"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 // API exposes the contract for the throttler's web API
@@ -33,6 +34,7 @@ type API interface {
 	ReadCheckIfExists(w http.ResponseWriter, r *http.Request, _ httprouter.Params)
 	AggregatedMetrics(w http.ResponseWriter, r *http.Request, _ httprouter.Params)
 	MetricsHealth(w http.ResponseWriter, r *http.Request, _ httprouter.Params)
+	PrometheusMetrics(w http.ResponseWriter, r *http.Request, ps httprouter.Params)
 	ThrottleApp(w http.ResponseWriter, r *http.Request, _ httprouter.Params)
 	UnthrottleApp(w http.ResponseWriter, r *http.Request, _ httprouter.Params)
 	ThrottledApps(w http.ResponseWriter, r *http.Request, _ httprouter.Params)
@@ -236,6 +238,11 @@ func (api *APIImpl) MetricsHealth(w http.ResponseWriter, r *http.Request, ps htt
 	json.NewEncoder(w).Encode(metricsHealth)
 }
 
+func (api *APIImpl) PrometheusMetrics(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+	handler := promhttp.Handler()
+	handler.ServeHTTP(w, r)
+}
+
 // ThrottleApp forcibly marks given app as throttled. Future requests by this app may be denied.
 func (api *APIImpl) ThrottleApp(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
 	storeName := r.URL.Query().Get("store_name")
@@ -416,6 +423,7 @@ func ConfigureRoutes(api API) *httprouter.Router {
 
 	register(router, "/aggregated-metrics", api.AggregatedMetrics)
 	register(router, "/metrics-health", api.MetricsHealth)
+	register(router, "/metrics", api.PrometheusMetrics)
 
 	register(router, "/throttle-app/:app", api.ThrottleApp)
 	register(router, "/throttle-app/:app/ratio/:ratio", api.ThrottleApp)

pkg/throttle/throttler.go

@@ -3,6 +3,7 @@ package throttle
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/prometheus/client_golang/prometheus"
 	"io/ioutil"
 	"math/rand"
 	"net/http"
@@ -49,6 +50,12 @@ func init() {
 	rand.Seed(time.Now().UnixNano())
 }
 
+type PrometheusMetrics struct {
+	value             *prometheus.GaugeVec
+	timestamp         *prometheus.GaugeVec
+	throttleThreshold *prometheus.GaugeVec
+}
+
 type Throttler struct {
 	isLeader                 bool
 	isLeaderFunc             func() bool
@@ -69,6 +76,8 @@ type Throttler struct {
 	metricsHealth           *cache.Cache
 	shareDomainMetricHealth *cache.Cache
 
+	prometheusMetrics *PrometheusMetrics
+
 	memcacheClient *memcache.Client
 	memcachePath   string
 
@@ -81,6 +90,36 @@ type Throttler struct {
 	httpClient                         *http.Client
 }
 
+func NewPrometheusMetrics(namespace string, subsystem string) *PrometheusMetrics {
+	promMetrics := &PrometheusMetrics{
+		value: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "value",
+			Help:      "Result value for the cluster",
+		},
+			[]string{"store_type", "store_name"}),
+		timestamp: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "last_update_timestamp",
+			Help:      "Timestamp of the last update",
+		},
+			[]string{"store_type", "store_name"}),
+		throttleThreshold: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "throttle_threshold",
+			Help:      "Throtthe Threshold",
+		}, []string{"store_type", "store_name"}),
+	}
+	prometheus.MustRegister(promMetrics.value)
+	prometheus.MustRegister(promMetrics.timestamp)
+	prometheus.MustRegister(promMetrics.throttleThreshold)
+
+	return promMetrics
+}
+
 func NewThrottler() *Throttler {
 	throttler := &Throttler{
 		isLeader: false,
@@ -114,6 +153,11 @@ func NewThrottler() *Throttler {
 		throttler.proxysqlClient = proxysql.NewClient(mysqlRefreshInterval)
 	}
 
+	throttler.prometheusMetrics = NewPrometheusMetrics(
+		config.Settings().PrometheusNamespace,
+		config.Settings().PrometheusSubsystem,
+	)
+
 	return throttler
 }
 
@@ -313,6 +357,7 @@ func (throttler *Throttler) refreshMySQLInventory() error {
 		// is immutable and can only be _replaced_. Hence, it's safe to read in a goroutine:
 		go func() error {
 			throttler.mysqlClusterThresholds.Set(clusterName, clusterSettings.ThrottleThreshold, cache.DefaultExpiration)
+			throttler.prometheusMetrics.throttleThreshold.WithLabelValues("mysql", clusterName).Set(clusterSettings.ThrottleThreshold)
 			if !clusterSettings.HAProxySettings.IsEmpty() {
 				poolName := clusterSettings.HAProxySettings.PoolName
 				totalHosts := []string{}
@@ -443,6 +488,8 @@ func (throttler *Throttler) aggregateMySQLMetrics() error {
 		ignoreHostsCount := throttler.mysqlInventory.IgnoreHostsCount[clusterName]
 		ignoreHostsThreshold := throttler.mysqlInventory.IgnoreHostsThreshold[clusterName]
 		aggregatedMetric := aggregateMySQLProbes(probes, clusterName, throttler.mysqlInventory.InstanceKeyMetrics, throttler.mysqlInventory.ClusterInstanceHttpChecks, ignoreHostsCount, config.Settings().Stores.MySQL.IgnoreDialTcpErrors, ignoreHostsThreshold)
+		updateMySQLMetrics(throttler.prometheusMetrics, clusterName, aggregatedMetric)
+
 		go throttler.aggregatedMetrics.Set(metricName, aggregatedMetric, cache.DefaultExpiration)
 		if throttler.memcacheClient != nil {
 			go func() {
@@ -461,6 +508,17 @@ func (throttler *Throttler) aggregateMySQLMetrics() error {
 	return nil
 }
 
+func updateMySQLMetrics(prometheusMetrics *PrometheusMetrics, clusterName string, metric base.MetricResult) {
+	const storeType = "mysql"
+	prometheusMetrics.timestamp.WithLabelValues(storeType, clusterName).Set(float64(time.Now().Unix()))
+	value, err := metric.Get()
+	if err != nil {
+		prometheusMetrics.value.DeleteLabelValues(storeType, clusterName)
+	} else {
+		prometheusMetrics.value.WithLabelValues(storeType, clusterName).Set(value)
+	}
+}
+
 func (throttler *Throttler) pushStatusToExpVar() {
 	metrics.DefaultRegistry.Each(func(metricName string, _ interface{}) {
 		if strings.HasPrefix(metricName, "throttled_states.") {

vendor/modules.txt

@@ -4,6 +4,9 @@ github.com/DATA-DOG/go-sqlmock
 # github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878
 ## explicit; go 1.12
 github.com/armon/go-metrics
+# github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973
+## explicit
+github.com/beorn7/perks/quantile
 # github.com/boltdb/bolt v1.3.1
 ## explicit
 github.com/boltdb/bolt
@@ -28,6 +31,9 @@ github.com/hashicorp/golang-lru/simplelru
 # github.com/julienschmidt/httprouter v1.3.0
 ## explicit; go 1.7
 github.com/julienschmidt/httprouter
+# github.com/matttproud/golang_protobuf_extensions v1.0.1
+## explicit
+github.com/matttproud/golang_protobuf_extensions/pbutil
 # github.com/outbrain/golib v0.0.0-20180830062331-ab954725f502
 ## explicit
 github.com/outbrain/golib/log
@@ -36,6 +42,25 @@ github.com/outbrain/golib/tests
 # github.com/patrickmn/go-cache v2.1.0+incompatible
 ## explicit
 github.com/patrickmn/go-cache
+# github.com/prometheus/client_golang v0.9.2
+## explicit
+github.com/prometheus/client_golang/prometheus
+github.com/prometheus/client_golang/prometheus/internal
+github.com/prometheus/client_golang/prometheus/promhttp
+# github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910
+## explicit
+github.com/prometheus/client_model/go
+# github.com/prometheus/common v0.0.0-20181126121408-4724e9255275
+## explicit
+github.com/prometheus/common/expfmt
+github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg
+github.com/prometheus/common/model
+# github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a
+## explicit
+github.com/prometheus/procfs
+github.com/prometheus/procfs/internal/util
+github.com/prometheus/procfs/nfs
+github.com/prometheus/procfs/xfs
 # github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563
 ## explicit
 github.com/rcrowley/go-metrics