Add support for wasm runtime metrics try #2

Cargo.toml

@@ -116,6 +116,7 @@ panic = "unwind"
 runtime-benchmarks= [ "polkadot-cli/runtime-benchmarks" ]
 try-runtime = [ "polkadot-cli/try-runtime" ]
 disputes = [ "polkadot-cli/disputes" ]
+with-tracing = [ "polkadot-cli/with-tracing" ]
 
 # Configuration for building a .deb package - for use with `cargo-deb`
 [package.metadata.deb]

cli/Cargo.toml

@@ -27,6 +27,8 @@ frame-benchmarking-cli = { git = "https://github.com/paritytech/substrate", bran
 try-runtime-cli = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
 sc-cli = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
 sc-service = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
+sc-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" }
+polkadot-node-metrics = { path = "../node/metrics" }
 
 # this crate is used only to enable `trie-memory-tracker` feature
 # see https://github.com/paritytech/substrate/pull/6745
@@ -62,3 +64,4 @@ rococo-native = [ "service/rococo-native" ]
 
 malus = [ "full-node", "service/malus" ]
 disputes = [ "service/disputes" ]
+with-tracing = ["service/with-tracing"]

cli/src/cli.rs

@@ -121,3 +121,11 @@ pub struct Cli {
 	#[structopt(flatten)]
 	pub run: RunCmd,
 }
+
+impl Cli {
+	/// Updates the prometheus metric prefix.
+	/// Must be called before `CliConfiguration::create_configuration()`.
+	pub fn update_prometheus_metric_prefix(&mut self, prefix: &'static str) {
+		self.run.base.prometheus_metric_prefix = prefix;
+	}
+}

cli/src/command.rs

@@ -223,8 +223,17 @@ pub fn run_node(run: Cli, overseer_gen: impl service::OverseerGen) -> Result<()>
 	run_node_inner(run, overseer_gen)
 }
 
-fn run_node_inner(cli: Cli, overseer_gen: impl service::OverseerGen) -> Result<()> {
-	let runner = cli.create_runner(&cli.run.base).map_err(Error::from)?;
+fn run_node_inner<F>(
+	cli: Cli,
+	overseer_gen: impl service::OverseerGen,
+	logger_hook: F,
+) -> Result<()>
+where
+	F: FnOnce(&mut sc_cli::LoggerBuilder, &sc_service::Configuration),
+{
+	let runner = cli
+		.create_runner_with_logger_hook::<sc_cli::RunCmd, F>(&cli.run.base, logger_hook)
+		.map_err(Error::from)?;
 	let chain_spec = &runner.config().chain_spec;
 
 	set_default_ss58_version(chain_spec);
@@ -265,12 +274,21 @@ fn run_node_inner(cli: Cli, overseer_gen: impl service::OverseerGen) -> Result<(
 	})
 }
 
+/// TODO: Start the node with support for publishing runtime metrics.
+pub fn run_with_wasm_metrics() -> Result<()> {
+	Ok(())
+}
+
 /// Parses polkadot specific CLI arguments and run the service.
 pub fn run() -> Result<()> {
-	let cli = Cli::from_args();
+	let mut cli: Cli = Cli::from_args();
+
+	// Override the default substrate metric prefix.
+	cli.update_prometheus_metric_prefix("polkadot");
 
 	match &cli.subcommand {
-		None => run_node_inner(cli, service::RealOverseerGen),
+		// TODO: gate by feature `runtime_metrics`.
+		None => run_node_inner(cli, service::RealOverseerGen, polkadot_node_metrics::logger_hook()),
 		Some(Subcommand::BuildSpec(cmd)) => {
 			let runner = cli.create_runner(cmd)?;
 			Ok(runner.sync_run(|config| cmd.run(config.chain_spec, config.network))?)

node/client/Cargo.toml

@@ -45,3 +45,9 @@ polkadot = ["polkadot-runtime"]
 kusama = ["kusama-runtime"]
 rococo = ["rococo-runtime"]
 westend = ["westend-runtime"]
+with-tracing = [
+    "rococo-runtime/with-tracing",
+    "kusama-runtime/with-tracing",
+    "westend-runtime/with-tracing",
+    "polkadot-runtime/with-tracing",
+]

node/metrics/Cargo.toml

@@ -8,10 +8,17 @@ description = "Subsystem traits and message definitions"
 [dependencies]
 futures = "0.3.17"
 futures-timer = "3.0.2"
+tracing = "0.1.29"
 
 metered-channel = { path = "../metered-channel" }
 
 substrate-prometheus-endpoint = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sc-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sc-cli = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sc-service = { git = "https://github.com/paritytech/substrate", branch = "master" }
+codec = { package = "parity-scale-codec", version = "2.2.0" }
+sp-tracing = { git = "https://github.com/paritytech/substrate", branch = "master" }
+primitives = { package = "polkadot-primitives", path = "../../primitives/" }
 
 [features]
 default = []

node/metrics/src/lib.rs

@@ -29,6 +29,8 @@ pub use metered_channel as metered;
 /// Cyclic metric collection support.
 pub mod metronome;
 pub use self::metronome::Metronome;
+pub mod runtime;
+pub use self::runtime::logger_hook;
 
 /// This module reexports Prometheus types and defines the [`Metrics`] trait.
 pub mod metrics {

node/metrics/src/runtime.rs

@@ -0,0 +1,152 @@
+// Copyright 2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Runtime Metrics helpers.
+//!
+//! Builds on top of Substrate wasm tracing support.
+
+use codec::Decode;
+use primitives::v0::{RuntimeMetricOp, RuntimeMetricUpdate};
+use std::{
+	collections::hash_map::HashMap,
+	sync::{Arc, Mutex},
+};
+use substrate_prometheus_endpoint::{register, CounterVec, Opts, PrometheusError, Registry, U64};
+
+/// We only support CounterVec for now.
+/// TODO: add more when needed.
+#[derive(Clone, Default)]
+pub struct Metrics {
+	counter_vecs: Arc<Mutex<HashMap<String, CounterVec<U64>>>>,
+}
+
+/// Runtime metrics wrapper.
+#[derive(Clone)]
+pub struct RuntimeMetricsProvider(Registry, Metrics);
+
+/// A set of metric labels.
+#[derive(Clone)]
+pub struct RuntimeMetricLabels(Vec<RuntimeMetricLabel>);
+
+/// A metric label value.
+#[derive(Clone)]
+pub struct RuntimeMetricLabel(&'static str);
+
+impl RuntimeMetricLabel {
+	/// Returns the inner static string.
+	pub fn as_str(&self) -> &'static str {
+		self.0
+	}
+}
+
+impl From<&'static str> for RuntimeMetricLabel {
+	fn from(s: &'static str) -> Self {
+		Self(s)
+	}
+}
+impl RuntimeMetricsProvider {
+	/// Creates new instance.
+	pub fn new(metrics_registry: Registry) -> Self {
+		Self(metrics_registry, Metrics::default())
+	}
+
+	/// Register a counter vec metric.
+	pub fn register_countervec(
+		&self,
+		metric_name: &str,
+		description: &str,
+		label: RuntimeMetricLabel,
+	) -> Result<(), PrometheusError> {
+		if !self.1.counter_vecs.lock().expect("bad lock").contains_key(metric_name) {
+			let counter_vec = register(
+				CounterVec::new(Opts::new(metric_name, description), &[label.as_str()])?,
+				&self.0,
+			)?;
+
+			self.1
+				.counter_vecs
+				.lock()
+				.expect("bad lock")
+				.insert(metric_name.to_owned(), counter_vec);
+		}
+
+		Ok(())
+	}
+
+	/// Increment a counter vec by a value.
+	pub fn inc_counter_by(&self, name: &str, value: u64, label: RuntimeMetricLabel) {
+		match self.register_countervec(name, "default description", label.clone()) {
+			Ok(_) => {
+				// The metric is in the hashmap, unwrap won't panic.
+				self.1
+					.counter_vecs
+					.lock()
+					.expect("bad lock")
+					.get_mut(name)
+					.unwrap()
+					.with_label_values(&[label.as_str()])
+					.inc_by(value);
+			},
+			Err(_) => {},
+		}
+	}
+}
+
+impl sc_tracing::TraceHandler for RuntimeMetricsProvider {
+	fn handle_span(&self, _span: &sc_tracing::SpanDatum) {}
+	fn handle_event(&self, event: &sc_tracing::TraceEvent) {
+		let target = event.values.string_values.get("target");
+		if target.is_none() || target.unwrap().ne("metrics") {
+			return
+		}
+
+		if let Some(update_op_str) = event.values.string_values.get("params").cloned() {
+			// TODO: Fix ugly hack because the payload comes in as a formatted string.
+			const SKIP_CHARS: usize = " { update_op: ".len();
+
+			match RuntimeMetricUpdate::decode(&mut update_op_str[SKIP_CHARS..].as_bytes()) {
+				Ok(update_op) => {
+					self.parse_metric_update(update_op);
+				},
+				Err(e) => {
+					tracing::error!("TraceEvent decode failed: {:?}", e);
+				},
+			}
+		}
+	}
+}
+
+impl RuntimeMetricsProvider {
+	fn parse_metric_update(&self, update: RuntimeMetricUpdate) {
+		match update.op {
+			RuntimeMetricOp::Increment(value) => {
+				self.inc_counter_by(update.metric_name(), value, "test_label".into());
+			},
+		}
+	}
+}
+
+/// Returns the custom profiling closure that we'll apply to the LoggerBuilder.
+pub fn logger_hook() -> impl FnOnce(&mut sc_cli::LoggerBuilder, &sc_service::Configuration) -> () {
+	|logger_builder, config| {
+		if config.prometheus_registry().is_none() {
+			return
+		}
+		let registry = config.prometheus_registry().cloned().unwrap();
+		let metrics_provider = RuntimeMetricsProvider::new(registry);
+		logger_builder.with_custom_profiling(vec![Box::new(metrics_provider)]);
+	}
+}

node/service/Cargo.toml

@@ -173,3 +173,11 @@ try-runtime = [
 ]
 malus = ["full-node"]
 disputes = ["polkadot-node-core-dispute-coordinator/disputes"]
+with-tracing = [
+	"polkadot-client/with-tracing", 
+	"rococo-runtime/with-tracing",
+	"westend-runtime/with-tracing",
+	"kusama-runtime/with-tracing",
+	"polkadot-runtime/with-tracing",
+	"polkadot-runtime-parachains/with-tracing"
+]

node/service/src/lib.rs

@@ -272,15 +272,6 @@ impl IdentifyVariant for Box<dyn ChainSpec> {
 	}
 }
 
-// If we're using prometheus, use a registry with a prefix of `polkadot`.
-fn set_prometheus_registry(config: &mut Configuration) -> Result<(), Error> {
-	if let Some(PrometheusConfig { registry, .. }) = config.prometheus_config.as_mut() {
-		*registry = Registry::new_custom(Some("polkadot".into()), None)?;
-	}
-
-	Ok(())
-}
-
 /// Initialize the `Jeager` collector. The destination must listen
 /// on the given address and port for `UDP` packets.
 #[cfg(any(test, feature = "full-node"))]
@@ -344,8 +335,6 @@ where
 		RuntimeApiCollection<StateBackend = sc_client_api::StateBackendFor<FullBackend, Block>>,
 	ExecutorDispatch: NativeExecutionDispatch + 'static,
 {
-	set_prometheus_registry(config)?;
-
 	let telemetry = config
 		.telemetry_endpoints
 		.clone()

primitives/src/v0.rs

@@ -923,6 +923,30 @@ pub mod fisherman {
 	}
 }
 
+// TODO: add feature gate.
+/// Runtime metric operations.
+#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
+pub enum RuntimeMetricOp {
+	/// Increment metric by value.
+	Increment(u64),
+}
+
+/// Runtime metric update event.
+#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode, TypeInfo)]
+pub struct RuntimeMetricUpdate {
+	/// The name of the metric.
+	pub metric_name: Vec<u8>,
+	/// The operation applied to the metric.
+	pub op: RuntimeMetricOp,
+}
+
+impl RuntimeMetricUpdate {
+	/// Returns the metric name.
+	pub fn metric_name(&self) -> &str {
+		unsafe { sp_std::str::from_utf8_unchecked(&self.metric_name) }
+	}
+}
+
 #[cfg(test)]
 mod tests {
 	use super::*;

runtime/kusama/Cargo.toml

@@ -264,3 +264,4 @@ disable-runtime-api = []
 on-chain-release-build = [
 	"sp-api/disable-logging",
 ]
+with-tracing = ["frame-executive/with-tracing", "sp-io/with-tracing"]

runtime/parachains/Cargo.toml

@@ -23,6 +23,7 @@ sp-session = { git = "https://github.com/paritytech/substrate", branch = "master
 sp-staking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
+sp-tracing = { version = "4.0.0-dev", branch = "master", git = "https://github.com/paritytech/substrate", default-features = false }
 
 pallet-authority-discovery = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-authorship = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
@@ -98,3 +99,4 @@ try-runtime = [
 	"pallet-timestamp/try-runtime",
 	"pallet-vesting/try-runtime",
 ]
+with-tracing = ["sp-tracing/with-tracing"]