From d82062007d955d2bc87ba2bc4d37979377abf1b7 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Wed, 15 Sep 2021 04:27:51 +0000 Subject: [PATCH 1/3] Move memory stats gathering from `polkadot` to `parity-util-mem` --- parity-util-mem/Cargo.toml | 6 ++- parity-util-mem/src/lib.rs | 56 ++++++++++++++++++++ parity-util-mem/src/memory_stats_jemalloc.rs | 32 +++++++++++ parity-util-mem/src/memory_stats_noop.rs | 31 +++++++++++ 4 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 parity-util-mem/src/memory_stats_jemalloc.rs create mode 100644 parity-util-mem/src/memory_stats_noop.rs diff --git a/parity-util-mem/Cargo.toml b/parity-util-mem/Cargo.toml index ec90c9565..f27a5395d 100644 --- a/parity-util-mem/Cargo.toml +++ b/parity-util-mem/Cargo.toml @@ -37,6 +37,10 @@ winapi = { version = "0.3.8", features = ["heapapi"] } version = "0.3.2" optional = true +[target.'cfg(not(target_os = "windows"))'.dependencies.jemalloc-ctl] +version = "0.3.3" +optional = true + [features] default = ["std", "ethereum-impls", "lru", "hashbrown", "smallvec", "primitive-types"] std = ["parking_lot"] @@ -45,7 +49,7 @@ dlmalloc-global = ["dlmalloc", "estimate-heapsize"] # use wee_alloc as global allocator weealloc-global = ["wee_alloc", "estimate-heapsize"] # use jemalloc as global allocator -jemalloc-global = ["jemallocator"] +jemalloc-global = ["jemallocator", "jemalloc-ctl"] # use mimalloc as global allocator mimalloc-global = ["mimalloc", "libmimalloc-sys"] # implement additional types diff --git a/parity-util-mem/src/lib.rs b/parity-util-mem/src/lib.rs index bf6334e0e..5b3300069 100644 --- a/parity-util-mem/src/lib.rs +++ b/parity-util-mem/src/lib.rs @@ -24,14 +24,23 @@ cfg_if::cfg_if! { /// Global allocator #[global_allocator] pub static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; + + mod memory_stats_jemalloc; + use memory_stats_jemalloc as memory_stats; } else if #[cfg(feature = "dlmalloc-global")] { /// Global allocator #[global_allocator] pub static ALLOC: dlmalloc::GlobalDlmalloc = dlmalloc::GlobalDlmalloc; + + mod memory_stats_noop; + use memory_stats_noop as memory_stats; } else if #[cfg(feature = "weealloc-global")] { /// Global allocator #[global_allocator] pub static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT; + + mod memory_stats_noop; + use memory_stats_noop as memory_stats; } else if #[cfg(all( feature = "mimalloc-global", not(target_arch = "wasm32") @@ -39,8 +48,13 @@ cfg_if::cfg_if! { /// Global allocator #[global_allocator] pub static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; + + mod memory_stats_noop; + use memory_stats_noop as memory_stats; } else { // default allocator used + mod memory_stats_noop; + use memory_stats_noop as memory_stats; } } @@ -78,6 +92,48 @@ pub fn malloc_size(t: &T) -> usize { MallocSizeOf::size_of(t, &mut allocators::new_malloc_size_ops()) } +/// An error related to the memory stats gathering. +#[derive(Clone, Debug)] +pub struct MemoryStatsError(memory_stats::Error); + +#[cfg(feature = "std")] +impl std::fmt::Display for MemoryStatsError { + fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { + self.0.fmt(fmt) + } +} + +#[cfg(feature = "std")] +impl std::error::Error for MemoryStatsError {} + +/// Snapshot of collected memory metrics. +#[non_exhaustive] +#[derive(Debug, Clone)] +pub struct MemoryAllocationSnapshot { + /// Total resident memory, in bytes. + pub resident: u64, + /// Total allocated memory, in bytes. + pub allocated: u64, +} + +/// Accessor to the allocator internals. +#[derive(Clone)] +pub struct MemoryAllocationTracker(self::memory_stats::MemoryAllocationTracker); + +impl MemoryAllocationTracker { + /// Create an instance of an allocation tracker. + pub fn new() -> Result { + self::memory_stats::MemoryAllocationTracker::new() + .map(MemoryAllocationTracker) + .map_err(MemoryStatsError) + } + + /// Create an allocation snapshot. + pub fn snapshot(&self) -> Result { + self.0.snapshot().map_err(MemoryStatsError) + } +} + #[cfg(feature = "std")] #[cfg(test)] mod test { diff --git a/parity-util-mem/src/memory_stats_jemalloc.rs b/parity-util-mem/src/memory_stats_jemalloc.rs new file mode 100644 index 000000000..854e932cf --- /dev/null +++ b/parity-util-mem/src/memory_stats_jemalloc.rs @@ -0,0 +1,32 @@ +// Copyright 2021 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +pub use jemalloc_ctl::Error; +use jemalloc_ctl::{epoch, stats}; + +#[derive(Clone)] +pub struct MemoryAllocationTracker { + epoch: jemalloc_ctl::epoch_mib, + allocated: stats::allocated_mib, + resident: stats::resident_mib, +} + +impl MemoryAllocationTracker { + pub fn new() -> Result { + Ok(Self { epoch: epoch::mib()?, allocated: stats::allocated::mib()?, resident: stats::resident::mib()? }) + } + + pub fn snapshot(&self) -> Result { + // update stats by advancing the allocation epoch + self.epoch.advance()?; + + let allocated: u64 = self.allocated.read()? as _; + let resident: u64 = self.resident.read()? as _; + Ok(crate::MemoryAllocationSnapshot { allocated, resident }) + } +} diff --git a/parity-util-mem/src/memory_stats_noop.rs b/parity-util-mem/src/memory_stats_noop.rs new file mode 100644 index 000000000..cf077c6f8 --- /dev/null +++ b/parity-util-mem/src/memory_stats_noop.rs @@ -0,0 +1,31 @@ +// Copyright 2021 Parity Technologies +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#[derive(Clone, Debug)] +pub struct Unimplemented; +pub use Unimplemented as Error; + +#[cfg(feature = "std")] +impl std::fmt::Display for Unimplemented { + fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { + fmt.write_str("unimplemented") + } +} + +#[derive(Clone)] +pub struct MemoryAllocationTracker {} + +impl MemoryAllocationTracker { + pub fn new() -> Result { + Err(Error) + } + + pub fn snapshot(&self) -> Result { + unimplemented!(); + } +} From 3c2798f409b40100b36e3c3ef301cac46696fc0b Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Wed, 15 Sep 2021 04:31:16 +0000 Subject: [PATCH 2/3] Bump version to 0.10.1 --- parity-util-mem/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parity-util-mem/Cargo.toml b/parity-util-mem/Cargo.toml index f27a5395d..fac33b42c 100644 --- a/parity-util-mem/Cargo.toml +++ b/parity-util-mem/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "parity-util-mem" -version = "0.10.0" +version = "0.10.1" authors = ["Parity Technologies "] repository = "https://github.com/paritytech/parity-common" description = "Collection of memory related utilities" From 4ca57226c5114fa699eb71ff30d502cba98f5011 Mon Sep 17 00:00:00 2001 From: Jan Bujak Date: Wed, 15 Sep 2021 09:31:09 +0000 Subject: [PATCH 3/3] Update the CHANGELOG --- parity-util-mem/CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/parity-util-mem/CHANGELOG.md b/parity-util-mem/CHANGELOG.md index c5e13a77c..53f0c449c 100644 --- a/parity-util-mem/CHANGELOG.md +++ b/parity-util-mem/CHANGELOG.md @@ -4,7 +4,10 @@ The format is based on [Keep a Changelog]. [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/ -## [Unreleased] +## [0.10.1] - 2021-09-15 +- Added support for memory stats gathering, ported over from `polkadot`. [#588](https://github.com/paritytech/parity-common/pull/588) + +## [0.10.0] - 2021-07-02 - Fixed `malloc_usable_size` for FreeBSD. [#553](https://github.com/paritytech/parity-common/pull/553) ### Breaking