Add a metric to track the delay of the downstream

pingcap · May 15, 2019 · 2451962 · 2451962
1 parent aabc1be
commit 2451962
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 0 deletions.
diff --git a/drainer/metrics.go b/drainer/metrics.go
@@ -69,6 +69,14 @@ var (
 			Help:      "save checkpoint tso of drainer.",
 		})
 
+	checkpointDelayGauge = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: "binlog",
+			Subsystem: "drainer",
+			Name:      "checkpoint_delay_seconds",
+			Help:      "How much the downstream checkpoint lag behind",
+		})
+
 	executeHistogram = prometheus.NewHistogram(
 		prometheus.HistogramOpts{
 			Namespace: "binlog",
@@ -123,6 +131,7 @@ func init() {
 	registry.MustRegister(ddlJobsCounter)
 	registry.MustRegister(errorCount)
 	registry.MustRegister(checkpointTSOGauge)
+	registry.MustRegister(checkpointDelayGauge)
 	registry.MustRegister(eventCounter)
 	registry.MustRegister(executeHistogram)
 	registry.MustRegister(binlogReachDurationHistogram)

diff --git a/drainer/syncer.go b/drainer/syncer.go
@@ -221,6 +221,8 @@ func (s *Syncer) handleSuccess(fakeBinlog chan *pb.Binlog, lastTS *int64) {
 				lastSaveTS = ts
 				eventCounter.WithLabelValues("savepoint").Add(1)
 			}
+			delay := oracle.GetPhysical(time.Now()) - oracle.ExtractPhysical(uint64(ts))
+			checkpointDelayGauge.Set(float64(delay) / 1e3)
 		}
 	}