collect better memory stats (#3612)

* add jemalloc memory statistics tracking

* chore: move Metronome in a separate file

* add meta flag spellcheck

* adjust metrics names

* account for new metrics in test

Cargo.toml

@@ -142,3 +142,7 @@ polkadot = { path = "/usr/bin/polkadot" }
 
 [package.metadata.rpm.files]
 "../scripts/packaging/polkadot.service" = { path = "/usr/lib/systemd/system/polkadot.service", mode = "644" }
+
+
+[package.metadata.spellcheck]
+config = "./scripts/gitlab/spellcheck.toml"

cli/Cargo.toml

@@ -49,6 +49,13 @@ cli = [
 	"frame-benchmarking-cli",
 	"try-runtime-cli",
 	"polkadot-node-core-pvf",
+# memory stats require jemalloc, which we know is enabled for linux
+# but not present on wasm or windows
+# https://github.com/paritytech/parity-common/blob/master/parity-util-mem/src/allocators.rs#L9-L34
+# Once
+# https://github.com/rust-lang/cargo/issues/1197
+# is resolved.
+	"service/memory-stats",
 ]
 browser = [
 	"wasm-bindgen",

core-primitives/src/lib.rs

@@ -120,7 +120,7 @@ pub type DownwardMessage = sp_std::vec::Vec<u8>;
 #[derive(Encode, Decode, Clone, sp_runtime::RuntimeDebug, PartialEq)]
 #[cfg_attr(feature = "std", derive(MallocSizeOf))]
 pub struct InboundDownwardMessage<BlockNumber = crate::BlockNumber> {
-	/// The block number at which this messages was put into the downward message queue.
+	/// The block number at which these messages were put into the downward message queue.
 	pub sent_at: BlockNumber,
 	/// The actual downward message to processes.
 	pub msg: DownwardMessage,

node/core/pvf/src/worker_common.rs

@@ -148,7 +148,7 @@ where
 /// A struct that represents an idle worker.
 ///
 /// This struct is supposed to be used as a token that is passed by move into a subroutine that
-/// initiates a job. If the worker dies on the duty, then the token is not returned back.
+/// initiates a job. If the worker dies on the duty, then the token is not returned.
 #[derive(Debug)]
 pub struct IdleWorker {
 	/// The stream to which the child process is connected.

node/metrics/Cargo.toml

@@ -17,3 +17,9 @@ sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-application-crypto = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-keystore = { git = "https://github.com/paritytech/substrate", branch = "master" }
 substrate-prometheus-endpoint = { git = "https://github.com/paritytech/substrate", branch = "master" }
+
+jemalloc-ctl = { version = "0.3.3", optional = true }
+
+[features]
+default = []
+memory-stats = ["jemalloc-ctl"]

node/metrics/src/lib.rs

@@ -14,26 +14,29 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-//! Utility module for subsystems
+//! Metrics helpers
 //!
-//! Many subsystems have common interests such as canceling a bunch of spawned jobs,
-//! or determining what their validator ID is. These common interests are factored into
-//! this module.
+//! Collects a bunch of metrics providers and related features such as
+//! `Metronome` for usage with metrics collections.
 //!
 //! This crate also reexports Prometheus metric types which are expected to be implemented by subsystems.
 
-#![warn(missing_docs)]
-
-use futures::prelude::*;
-use futures_timer::Delay;
-use std::{
-	pin::Pin,
-	task::{Context, Poll},
-	time::Duration,
-};
+#![deny(missing_docs)]
+#![deny(unused_imports)]
 
 pub use metered_channel as metered;
 
+/// Memory allocation stats tracking.
+#[cfg(feature = "memory-stats")]
+pub mod memory_stats;
+
+#[cfg(feature = "memory-stats")]
+pub use self::memory_stats::{MemoryAllocationSnapshot, MemoryAllocationTracker};
+
+/// Cyclic metric collection support.
+pub mod metronome;
+pub use self::metronome::Metronome;
+
 /// This module reexports Prometheus types and defines the [`Metrics`] trait.
 pub mod metrics {
 	/// Reexport Substrate Prometheus types.
@@ -73,47 +76,3 @@ pub mod metrics {
 		}
 	}
 }
-
-#[derive(Copy, Clone)]
-enum MetronomeState {
-	Snooze,
-	SetAlarm,
-}
-
-/// Create a stream of ticks with a defined cycle duration.
-pub struct Metronome {
-	delay: Delay,
-	period: Duration,
-	state: MetronomeState,
-}
-
-impl Metronome {
-	/// Create a new metronome source with a defined cycle duration.
-	pub fn new(cycle: Duration) -> Self {
-		let period = cycle.into();
-		Self { period, delay: Delay::new(period), state: MetronomeState::Snooze }
-	}
-}
-
-impl futures::Stream for Metronome {
-	type Item = ();
-	fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
-		loop {
-			match self.state {
-				MetronomeState::SetAlarm => {
-					let val = self.period.clone();
-					self.delay.reset(val);
-					self.state = MetronomeState::Snooze;
-				},
-				MetronomeState::Snooze => {
-					if !Pin::new(&mut self.delay).poll(cx).is_ready() {
-						break
-					}
-					self.state = MetronomeState::SetAlarm;
-					return Poll::Ready(Some(()))
-				},
-			}
-		}
-		Poll::Pending
-	}
-}

node/metrics/src/memory_stats.rs

@@ -0,0 +1,66 @@
+// Copyright 2021 Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! Memory tracking statistics.
+//!
+//! Many subsystems have common interests such as canceling a bunch of spawned jobs,
+//! or determining what their validator ID is. These common interests are factored into
+//! this module.
+//!
+//! This crate also reexports Prometheus metric types which are expected to be implemented by subsystems.
+
+// #[cfg(not(feature = "memory-stats"))]
+// use std::convert::Infallible;
+
+use jemalloc_ctl::{epoch, stats, Result};
+
+/// Accessor to the allocator internals.
+#[derive(Clone)]
+pub struct MemoryAllocationTracker {
+	epoch: jemalloc_ctl::epoch_mib,
+	allocated: stats::allocated_mib,
+	resident: stats::resident_mib,
+}
+
+impl MemoryAllocationTracker {
+	/// Create an instance of an allocation tracker.
+	pub fn new() -> Result<Self> {
+		Ok(Self {
+			epoch: epoch::mib()?,
+			allocated: stats::allocated::mib()?,
+			resident: stats::resident::mib()?,
+		})
+	}
+
+	/// Create an allocation snapshot.
+	pub fn snapshot(&self) -> Result<MemoryAllocationSnapshot> {
+		// update stats by advancing the allocation epoch
+		self.epoch.advance()?;
+
+		let allocated: u64 = self.allocated.read()? as _;
+		let resident: u64 = self.resident.read()? as _;
+		Ok(MemoryAllocationSnapshot { allocated, resident })
+	}
+}
+
+/// Snapshot of collected memory metrics.
+#[derive(Debug, Clone)]
+pub struct MemoryAllocationSnapshot {
+	/// Total resident memory, in bytes.
+	pub resident: u64,
+	/// Total allocated memory, in bytes.
+	pub allocated: u64,
+}