feat: add metrics to watchdog canister (#189)

This PR adds Prometheus metrics to a watchdog canister.

Metrics response example:
```
# HELP bitcoin_canister_height Height of the main chain of the Bitcoin canister.
# TYPE bitcoin_canister_height gauge
bitcoin_canister_height 786116 1681915083861
# HELP explorers_number Number of explorers inspected.
# TYPE explorers_number gauge
explorers_number 6 1681915083861
# HELP target_height Target height calculated from the explorers.
# TYPE target_height gauge
target_height 786116 1681915083861
# HELP height_diff Difference between the source and the target heights.
# TYPE height_diff gauge
height_diff 0 1681915083861
# HELP status Status code of the Bitcoin canister height.
# TYPE status gauge
status{height="not_enough_data"} 0 1681915083861
status{height="ok"} 1 1681915083861
status{height="ahead"} 0 1681915083861
status{height="behind"} 0 1681915083861
```

.github/workflows/workflow.yml

@@ -415,6 +415,38 @@ jobs:
         run: |
           bash watchdog/tests/get_config.sh
 
+  watchdog_metrics:
+    runs-on: ubuntu-20.04
+    needs: cargo-build
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-1
+
+      - name: Install Rust
+        run: |
+          rustup update ${{ matrix.rust }} --no-self-update
+          rustup default ${{ matrix.rust }}
+          rustup target add wasm32-unknown-unknown
+
+      - name: Install DFX
+        run: |
+          wget --output-document install-dfx.sh "https://internetcomputer.org/install.sh"
+          bash install-dfx.sh < <(yes Y)
+          rm install-dfx.sh
+          dfx cache install
+          echo "$HOME/bin" >> $GITHUB_PATH
+
+      - name: Run metrics test
+        run: |
+          bash watchdog/tests/metrics.sh
+
   checks-pass:
     needs: ["cargo-tests", "shell-checks", "cargo-clippy", "rustfmt", "e2e-disable-api-if-not-fully-synced-flag", "e2e-scenario-1", "e2e-scenario-2", "e2e-scenario-3", "charge-cycles-on-reject", "upgradability", "set_config"]
     runs-on: ubuntu-20.04

watchdog/Cargo.toml

@@ -21,6 +21,8 @@ async-trait = "0.1.67"
 regex = "1.7.0"
 futures = "0.3.27"
 ic-http = {path = "../ic-http"}
+serde_bytes = "0.11"
+ic-metrics-encoder = "1.0.0"
 
 [dev-dependencies]
 tokio = { version = "1.15.0", features = [ "full" ] }

watchdog/candid.did

@@ -18,24 +18,22 @@ type status_code = variant {
 
 // The health status of the Bitcoin canister.
 type health_status = record {
-    /// The height of the block from the Bitcoin canister.
+    /// Height of the block from the Bitcoin canister.
     source_height : opt nat64;
 
-    /// The number of explorers inspected.
+    /// Number of explorers inspected.
     other_number : nat64;
 
-    /// The heights of the blocks from the explorers.
+    /// Heights of the blocks from the explorers.
     other_heights : vec nat64;
 
-    /// The target height of the Bitcoin canister calculated
-    /// from the explorers.
+    /// Target height calculated from the explorers.
     target_height : opt nat64;
 
-    /// The difference between the source height
-    /// and the target height.
+    /// Difference between the source and the target heights.
     height_diff : opt int64;
 
-    /// The code of the health status.
+    /// Status code of the Bitcoin canister health.
     status : status_code;
 };
 

watchdog/src/health.rs

@@ -1,3 +1,4 @@
+use crate::bitcoin_block_apis::BitcoinBlockApi;
 use crate::config::Config;
 use crate::fetch::BlockInfo;
 use candid::CandidType;
@@ -23,32 +24,42 @@ pub enum StatusCode {
     Behind,
 }
 
-/// The health status of the Bitcoin canister.
+/// Health status of the Bitcoin canister.
 #[derive(Clone, Debug, CandidType, PartialEq, Eq, Serialize, Deserialize)]
 pub struct HealthStatus {
-    /// The height of the block from the Bitcoin canister.
+    /// Height of the main chain of the Bitcoin canister.
     pub source_height: Option<u64>,
 
-    /// The number of explorers inspected.
+    /// Number of explorers inspected.
     pub other_number: u64,
 
-    /// The heights of the blocks from the explorers.
+    /// Heights of the blocks from the explorers.
     pub other_heights: Vec<u64>,
 
-    /// The target height of the Bitcoin canister calculated
-    /// from the explorers.
+    /// Target height calculated from the explorers.
     pub target_height: Option<u64>,
 
-    /// The difference between the source height
-    /// and the target height.
+    /// Difference between the source and the target heights.
     pub height_diff: Option<i64>,
 
-    /// The code of the health status.
+    /// Status code of the Bitcoin canister health.
     pub status: StatusCode,
 }
 
+/// Calculates the health status of a Bitcoin canister.
+pub fn health_status() -> HealthStatus {
+    compare(
+        crate::storage::get(&BitcoinBlockApi::BitcoinCanister),
+        BitcoinBlockApi::explorers()
+            .iter()
+            .filter_map(crate::storage::get)
+            .collect::<Vec<_>>(),
+        crate::storage::get_config(),
+    )
+}
+
 /// Compares the source with the other explorers.
-pub fn compare(source: Option<BlockInfo>, other: Vec<BlockInfo>, config: Config) -> HealthStatus {
+fn compare(source: Option<BlockInfo>, other: Vec<BlockInfo>, config: Config) -> HealthStatus {
     let source_height = source.and_then(|block| block.height);
     let heights = other
         .iter()

watchdog/src/lib.rs

@@ -4,7 +4,9 @@ mod endpoints;
 mod fetch;
 mod health;
 mod http;
+mod metrics;
 mod storage;
+mod types;
 
 #[cfg(test)]
 mod test_utils;
@@ -14,13 +16,14 @@ use crate::config::Config;
 use crate::endpoints::*;
 use crate::fetch::BlockInfo;
 use crate::health::HealthStatus;
+use crate::types::{CandidHttpRequest, CandidHttpResponse};
 use ic_cdk::api::management_canister::http_request::{HttpResponse, TransformArgs};
+use ic_cdk_macros::{init, post_upgrade, query};
+use serde_bytes::ByteBuf;
 use std::collections::HashMap;
 use std::sync::RwLock;
 use std::time::Duration;
 
-use ic_cdk_macros::{init, post_upgrade, query};
-
 thread_local! {
     /// The local storage for the data fetched from the external APIs.
     static BLOCK_INFO_DATA: RwLock<HashMap<BitcoinBlockApi, BlockInfo>> = RwLock::new(HashMap::new());
@@ -61,14 +64,7 @@ async fn fetch_data() {
 /// Returns the health status of the Bitcoin canister.
 #[query]
 fn health_status() -> HealthStatus {
-    crate::health::compare(
-        crate::storage::get(&BitcoinBlockApi::BitcoinCanister),
-        BitcoinBlockApi::explorers()
-            .iter()
-            .filter_map(crate::storage::get)
-            .collect::<Vec<_>>(),
-        crate::storage::get_config(),
-    )
+    crate::health::health_status()
 }
 
 /// Returns the configuration of the watchdog canister.
@@ -77,6 +73,20 @@ pub fn get_config() -> Config {
     crate::storage::get_config()
 }
 
+/// Processes external HTTP requests.
+#[query]
+pub fn http_request(request: CandidHttpRequest) -> CandidHttpResponse {
+    let parts: Vec<&str> = request.url.split('?').collect();
+    match parts[0] {
+        "/metrics" => crate::metrics::get_metrics(),
+        _ => CandidHttpResponse {
+            status_code: 404,
+            headers: vec![],
+            body: ByteBuf::from(String::from("Not found.")),
+        },
+    }
+}
+
 /// Prints a message to the console.
 pub fn print(msg: &str) {
     #[cfg(target_arch = "wasm32")]

watchdog/src/metrics.rs

@@ -0,0 +1,76 @@
+use crate::health::StatusCode;
+use crate::types::CandidHttpResponse;
+use ic_metrics_encoder::MetricsEncoder;
+use serde_bytes::ByteBuf;
+
+/// Returns the metrics in the Prometheus format.
+pub fn get_metrics() -> CandidHttpResponse {
+    let now = ic_cdk::api::time();
+    let mut writer = MetricsEncoder::new(vec![], (now / 1_000_000) as i64);
+    match encode_metrics(&mut writer) {
+        Ok(()) => {
+            let body = writer.into_inner();
+            CandidHttpResponse {
+                status_code: 200,
+                headers: vec![
+                    (
+                        "Content-Type".to_string(),
+                        "text/plain; version=0.0.4".to_string(),
+                    ),
+                    ("Content-Length".to_string(), body.len().to_string()),
+                ],
+                body: ByteBuf::from(body),
+            }
+        }
+        Err(err) => CandidHttpResponse {
+            status_code: 500,
+            headers: vec![],
+            body: ByteBuf::from(format!("Failed to encode metrics: {}", err)),
+        },
+    }
+}
+
+/// Encodes the metrics in the Prometheus format.
+fn encode_metrics(w: &mut MetricsEncoder<Vec<u8>>) -> std::io::Result<()> {
+    const NO_HEIGHT: f64 = -1.0;
+    const NO_HEIGHT_DIFF: f64 = -1_000.0;
+
+    let health = crate::health::health_status();
+    w.encode_gauge(
+        "bitcoin_canister_height",
+        health.source_height.map(|x| x as f64).unwrap_or(NO_HEIGHT),
+        "Height of the main chain of the Bitcoin canister.",
+    )?;
+    w.encode_gauge(
+        "explorers_number",
+        health.other_number as f64,
+        "Number of explorers inspected.",
+    )?;
+    w.encode_gauge(
+        "target_height",
+        health.target_height.map(|x| x as f64).unwrap_or(NO_HEIGHT),
+        "Target height calculated from the explorers.",
+    )?;
+    w.encode_gauge(
+        "height_diff",
+        health
+            .height_diff
+            .map(|x| x as f64)
+            .unwrap_or(NO_HEIGHT_DIFF),
+        "Difference between the source and the target heights.",
+    )?;
+
+    let (not_enough_data, ok, ahead, behind) = match health.status {
+        StatusCode::NotEnoughData => (1.0, 0.0, 0.0, 0.0),
+        StatusCode::Ok => (0.0, 1.0, 0.0, 0.0),
+        StatusCode::Ahead => (0.0, 0.0, 1.0, 0.0),
+        StatusCode::Behind => (0.0, 0.0, 0.0, 1.0),
+    };
+    w.gauge_vec("status", "Status code of the Bitcoin canister health.")?
+        .value(&[("height", "not_enough_data")], not_enough_data)?
+        .value(&[("height", "ok")], ok)?
+        .value(&[("height", "ahead")], ahead)?
+        .value(&[("height", "behind")], behind)?;
+
+    Ok(())
+}

watchdog/src/types.rs

@@ -0,0 +1,20 @@
+use ic_cdk::export::candid::CandidType;
+use serde::{Deserialize, Serialize};
+use serde_bytes::ByteBuf;
+
+type HeaderField = (String, String);
+
+#[derive(Clone, Debug, CandidType, Serialize, Deserialize)]
+pub struct CandidHttpRequest {
+    pub method: String,
+    pub url: String,
+    pub headers: Vec<HeaderField>,
+    pub body: ByteBuf,
+}
+
+#[derive(Clone, Debug, CandidType, Serialize, Deserialize)]
+pub struct CandidHttpResponse {
+    pub status_code: u16,
+    pub headers: Vec<HeaderField>,
+    pub body: ByteBuf,
+}

watchdog/tests/metrics.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+#
+# A test that verifies that the `/metrics` endpoint works as expected.
+
+# Run dfx stop if we run into errors.
+trap "dfx stop" EXIT SIGINT
+
+dfx start --background --clean
+
+# Deploy the watchdog canister.
+dfx deploy --no-wallet watchdog
+
+# Request canister id.
+CANISTER_ID=$(dfx canister id watchdog)
+METRICS=$(curl "http://127.0.0.1:8000/metrics?canisterId=$CANISTER_ID")
+
+# Check that metrics report contains some information.
+if ! [[ "$METRICS" == *"bitcoin_canister_height"* ]]; then
+  echo "FAIL"
+  exit 1
+fi
+
+echo "SUCCESS"