diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 21c08ff6..dcb9fe5f 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -415,6 +415,38 @@ jobs: run: | bash watchdog/tests/get_config.sh + watchdog_metrics: + runs-on: ubuntu-20.04 + needs: cargo-build + + steps: + - uses: actions/checkout@v3 + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-1 + + - name: Install Rust + run: | + rustup update ${{ matrix.rust }} --no-self-update + rustup default ${{ matrix.rust }} + rustup target add wasm32-unknown-unknown + + - name: Install DFX + run: | + wget --output-document install-dfx.sh "https://internetcomputer.org/install.sh" + bash install-dfx.sh < <(yes Y) + rm install-dfx.sh + dfx cache install + echo "$HOME/bin" >> $GITHUB_PATH + + - name: Run metrics test + run: | + bash watchdog/tests/metrics.sh + checks-pass: needs: ["cargo-tests", "shell-checks", "cargo-clippy", "rustfmt", "e2e-disable-api-if-not-fully-synced-flag", "e2e-scenario-1", "e2e-scenario-2", "e2e-scenario-3", "charge-cycles-on-reject", "upgradability", "set_config"] runs-on: ubuntu-20.04 diff --git a/Cargo.lock b/Cargo.lock index 79b3b9ec..57f27617 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3244,8 +3244,10 @@ dependencies = [ "ic-cdk-macros 0.6.10", "ic-cdk-timers", "ic-http", + "ic-metrics-encoder", "regex", "serde", + "serde_bytes", "serde_json", "tokio", ] diff --git a/watchdog/Cargo.toml b/watchdog/Cargo.toml index cdeebbb9..9eb53420 100644 --- a/watchdog/Cargo.toml +++ b/watchdog/Cargo.toml @@ -21,6 +21,8 @@ async-trait = "0.1.67" regex = "1.7.0" futures = "0.3.27" ic-http = {path = "../ic-http"} +serde_bytes = "0.11" +ic-metrics-encoder = "1.0.0" [dev-dependencies] tokio = { version = "1.15.0", features = [ "full" ] } diff --git a/watchdog/candid.did b/watchdog/candid.did index c7e5e3a4..aeb7d2a2 100644 --- a/watchdog/candid.did +++ b/watchdog/candid.did @@ -18,24 +18,22 @@ type status_code = variant { // The health status of the Bitcoin canister. type health_status = record { - /// The height of the block from the Bitcoin canister. + /// Height of the block from the Bitcoin canister. source_height : opt nat64; - /// The number of explorers inspected. + /// Number of explorers inspected. other_number : nat64; - /// The heights of the blocks from the explorers. + /// Heights of the blocks from the explorers. other_heights : vec nat64; - /// The target height of the Bitcoin canister calculated - /// from the explorers. + /// Target height calculated from the explorers. target_height : opt nat64; - /// The difference between the source height - /// and the target height. + /// Difference between the source and the target heights. height_diff : opt int64; - /// The code of the health status. + /// Status code of the Bitcoin canister health. status : status_code; }; diff --git a/watchdog/src/health.rs b/watchdog/src/health.rs index 1bbaf677..d17a1742 100644 --- a/watchdog/src/health.rs +++ b/watchdog/src/health.rs @@ -1,3 +1,4 @@ +use crate::bitcoin_block_apis::BitcoinBlockApi; use crate::config::Config; use crate::fetch::BlockInfo; use candid::CandidType; @@ -23,32 +24,42 @@ pub enum StatusCode { Behind, } -/// The health status of the Bitcoin canister. +/// Health status of the Bitcoin canister. #[derive(Clone, Debug, CandidType, PartialEq, Eq, Serialize, Deserialize)] pub struct HealthStatus { - /// The height of the block from the Bitcoin canister. + /// Height of the main chain of the Bitcoin canister. pub source_height: Option, - /// The number of explorers inspected. + /// Number of explorers inspected. pub other_number: u64, - /// The heights of the blocks from the explorers. + /// Heights of the blocks from the explorers. pub other_heights: Vec, - /// The target height of the Bitcoin canister calculated - /// from the explorers. + /// Target height calculated from the explorers. pub target_height: Option, - /// The difference between the source height - /// and the target height. + /// Difference between the source and the target heights. pub height_diff: Option, - /// The code of the health status. + /// Status code of the Bitcoin canister health. pub status: StatusCode, } +/// Calculates the health status of a Bitcoin canister. +pub fn health_status() -> HealthStatus { + compare( + crate::storage::get(&BitcoinBlockApi::BitcoinCanister), + BitcoinBlockApi::explorers() + .iter() + .filter_map(crate::storage::get) + .collect::>(), + crate::storage::get_config(), + ) +} + /// Compares the source with the other explorers. -pub fn compare(source: Option, other: Vec, config: Config) -> HealthStatus { +fn compare(source: Option, other: Vec, config: Config) -> HealthStatus { let source_height = source.and_then(|block| block.height); let heights = other .iter() diff --git a/watchdog/src/lib.rs b/watchdog/src/lib.rs index b5d5b7d2..ee4d7b5d 100644 --- a/watchdog/src/lib.rs +++ b/watchdog/src/lib.rs @@ -4,7 +4,9 @@ mod endpoints; mod fetch; mod health; mod http; +mod metrics; mod storage; +mod types; #[cfg(test)] mod test_utils; @@ -14,13 +16,14 @@ use crate::config::Config; use crate::endpoints::*; use crate::fetch::BlockInfo; use crate::health::HealthStatus; +use crate::types::{CandidHttpRequest, CandidHttpResponse}; use ic_cdk::api::management_canister::http_request::{HttpResponse, TransformArgs}; +use ic_cdk_macros::{init, post_upgrade, query}; +use serde_bytes::ByteBuf; use std::collections::HashMap; use std::sync::RwLock; use std::time::Duration; -use ic_cdk_macros::{init, post_upgrade, query}; - thread_local! { /// The local storage for the data fetched from the external APIs. static BLOCK_INFO_DATA: RwLock> = RwLock::new(HashMap::new()); @@ -61,14 +64,7 @@ async fn fetch_data() { /// Returns the health status of the Bitcoin canister. #[query] fn health_status() -> HealthStatus { - crate::health::compare( - crate::storage::get(&BitcoinBlockApi::BitcoinCanister), - BitcoinBlockApi::explorers() - .iter() - .filter_map(crate::storage::get) - .collect::>(), - crate::storage::get_config(), - ) + crate::health::health_status() } /// Returns the configuration of the watchdog canister. @@ -77,6 +73,20 @@ pub fn get_config() -> Config { crate::storage::get_config() } +/// Processes external HTTP requests. +#[query] +pub fn http_request(request: CandidHttpRequest) -> CandidHttpResponse { + let parts: Vec<&str> = request.url.split('?').collect(); + match parts[0] { + "/metrics" => crate::metrics::get_metrics(), + _ => CandidHttpResponse { + status_code: 404, + headers: vec![], + body: ByteBuf::from(String::from("Not found.")), + }, + } +} + /// Prints a message to the console. pub fn print(msg: &str) { #[cfg(target_arch = "wasm32")] diff --git a/watchdog/src/metrics.rs b/watchdog/src/metrics.rs new file mode 100644 index 00000000..284cbb65 --- /dev/null +++ b/watchdog/src/metrics.rs @@ -0,0 +1,76 @@ +use crate::health::StatusCode; +use crate::types::CandidHttpResponse; +use ic_metrics_encoder::MetricsEncoder; +use serde_bytes::ByteBuf; + +/// Returns the metrics in the Prometheus format. +pub fn get_metrics() -> CandidHttpResponse { + let now = ic_cdk::api::time(); + let mut writer = MetricsEncoder::new(vec![], (now / 1_000_000) as i64); + match encode_metrics(&mut writer) { + Ok(()) => { + let body = writer.into_inner(); + CandidHttpResponse { + status_code: 200, + headers: vec![ + ( + "Content-Type".to_string(), + "text/plain; version=0.0.4".to_string(), + ), + ("Content-Length".to_string(), body.len().to_string()), + ], + body: ByteBuf::from(body), + } + } + Err(err) => CandidHttpResponse { + status_code: 500, + headers: vec![], + body: ByteBuf::from(format!("Failed to encode metrics: {}", err)), + }, + } +} + +/// Encodes the metrics in the Prometheus format. +fn encode_metrics(w: &mut MetricsEncoder>) -> std::io::Result<()> { + const NO_HEIGHT: f64 = -1.0; + const NO_HEIGHT_DIFF: f64 = -1_000.0; + + let health = crate::health::health_status(); + w.encode_gauge( + "bitcoin_canister_height", + health.source_height.map(|x| x as f64).unwrap_or(NO_HEIGHT), + "Height of the main chain of the Bitcoin canister.", + )?; + w.encode_gauge( + "explorers_number", + health.other_number as f64, + "Number of explorers inspected.", + )?; + w.encode_gauge( + "target_height", + health.target_height.map(|x| x as f64).unwrap_or(NO_HEIGHT), + "Target height calculated from the explorers.", + )?; + w.encode_gauge( + "height_diff", + health + .height_diff + .map(|x| x as f64) + .unwrap_or(NO_HEIGHT_DIFF), + "Difference between the source and the target heights.", + )?; + + let (not_enough_data, ok, ahead, behind) = match health.status { + StatusCode::NotEnoughData => (1.0, 0.0, 0.0, 0.0), + StatusCode::Ok => (0.0, 1.0, 0.0, 0.0), + StatusCode::Ahead => (0.0, 0.0, 1.0, 0.0), + StatusCode::Behind => (0.0, 0.0, 0.0, 1.0), + }; + w.gauge_vec("status", "Status code of the Bitcoin canister health.")? + .value(&[("height", "not_enough_data")], not_enough_data)? + .value(&[("height", "ok")], ok)? + .value(&[("height", "ahead")], ahead)? + .value(&[("height", "behind")], behind)?; + + Ok(()) +} diff --git a/watchdog/src/types.rs b/watchdog/src/types.rs new file mode 100644 index 00000000..32d9c636 --- /dev/null +++ b/watchdog/src/types.rs @@ -0,0 +1,20 @@ +use ic_cdk::export::candid::CandidType; +use serde::{Deserialize, Serialize}; +use serde_bytes::ByteBuf; + +type HeaderField = (String, String); + +#[derive(Clone, Debug, CandidType, Serialize, Deserialize)] +pub struct CandidHttpRequest { + pub method: String, + pub url: String, + pub headers: Vec, + pub body: ByteBuf, +} + +#[derive(Clone, Debug, CandidType, Serialize, Deserialize)] +pub struct CandidHttpResponse { + pub status_code: u16, + pub headers: Vec, + pub body: ByteBuf, +} diff --git a/watchdog/tests/metrics.sh b/watchdog/tests/metrics.sh new file mode 100755 index 00000000..3ca4e531 --- /dev/null +++ b/watchdog/tests/metrics.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# A test that verifies that the `/metrics` endpoint works as expected. + +# Run dfx stop if we run into errors. +trap "dfx stop" EXIT SIGINT + +dfx start --background --clean + +# Deploy the watchdog canister. +dfx deploy --no-wallet watchdog + +# Request canister id. +CANISTER_ID=$(dfx canister id watchdog) +METRICS=$(curl "http://127.0.0.1:8000/metrics?canisterId=$CANISTER_ID") + +# Check that metrics report contains some information. +if ! [[ "$METRICS" == *"bitcoin_canister_height"* ]]; then + echo "FAIL" + exit 1 +fi + +echo "SUCCESS"