diff --git a/Cargo.lock b/Cargo.lock index 0a1a1e0b17bec..1ecb25b7b4b1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5089,8 +5089,10 @@ dependencies = [ "clap 3.1.6", "frame-benchmarking", "frame-benchmarking-cli", + "frame-system", "jsonrpc-core", "node-template-runtime", + "pallet-transaction-payment", "pallet-transaction-payment-rpc", "sc-basic-authorship", "sc-cli", @@ -5113,6 +5115,8 @@ dependencies = [ "sp-consensus-aura", "sp-core", "sp-finality-grandpa", + "sp-inherents", + "sp-keyring", "sp-runtime", "sp-timestamp", "substrate-build-script-utils", diff --git a/bin/node-template/node/Cargo.toml b/bin/node-template/node/Cargo.toml index 4549a5b613da2..e642ce3c0411e 100644 --- a/bin/node-template/node/Cargo.toml +++ b/bin/node-template/node/Cargo.toml @@ -36,6 +36,10 @@ sp-finality-grandpa = { version = "4.0.0-dev", path = "../../../primitives/final sc-client-api = { version = "4.0.0-dev", path = "../../../client/api" } sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } sp-timestamp = { version = "4.0.0-dev", path = "../../../primitives/timestamp" } +sp-inherents = { version = "4.0.0-dev", path = "../../../primitives/inherents" } +sp-keyring = { version = "6.0.0", path = "../../../primitives/keyring" } +frame-system = { version = "4.0.0-dev", path = "../../../frame/system" } +pallet-transaction-payment = { version = "4.0.0-dev", default-features = false, path = "../../../frame/transaction-payment" } # These dependencies are used for the node template's RPCs jsonrpc-core = "18.0.0" diff --git a/bin/node-template/node/src/cli.rs b/bin/node-template/node/src/cli.rs index c4d27b71e4994..710c7f3f9e145 100644 --- a/bin/node-template/node/src/cli.rs +++ b/bin/node-template/node/src/cli.rs @@ -36,8 +36,8 @@ pub enum Subcommand { /// Revert the chain to a previous state. Revert(sc_cli::RevertCmd), - /// The custom benchmark subcommand benchmarking runtime pallets. - #[clap(name = "benchmark", about = "Benchmark runtime pallets.")] + /// Sub-commands concerned with benchmarking. + #[clap(subcommand)] Benchmark(frame_benchmarking_cli::BenchmarkCmd), /// Try some command against runtime state. diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index f033e779543a0..ede969b3572c8 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -1,11 +1,14 @@ use crate::{ chain_spec, cli::{Cli, Subcommand}, + command_helper::{inherent_benchmark_data, BenchmarkExtrinsicBuilder}, service, }; +use frame_benchmarking_cli::BenchmarkCmd; use node_template_runtime::Block; use sc_cli::{ChainSpec, RuntimeVersion, SubstrateCli}; use sc_service::PartialComponents; +use std::sync::Arc; impl SubstrateCli for Cli { fn impl_name() -> String { @@ -102,16 +105,41 @@ pub fn run() -> sc_cli::Result<()> { Ok((cmd.run(client, backend, Some(aux_revert)), task_manager)) }) }, - Some(Subcommand::Benchmark(cmd)) => - if cfg!(feature = "runtime-benchmarks") { - let runner = cli.create_runner(cmd)?; + Some(Subcommand::Benchmark(cmd)) => { + let runner = cli.create_runner(cmd)?; + + runner.sync_run(|config| { + let PartialComponents { client, backend, .. } = service::new_partial(&config)?; + + // This switch needs to be in the client, since the client decides + // which sub-commands it wants to support. + match cmd { + BenchmarkCmd::Pallet(cmd) => { + if !cfg!(feature = "runtime-benchmarks") { + return Err( + "Runtime benchmarking wasn't enabled when building the node. \ + You can enable it with `--features runtime-benchmarks`." + .into(), + ) + } + + cmd.run::(config) + }, + BenchmarkCmd::Block(cmd) => cmd.run(client), + BenchmarkCmd::Storage(cmd) => { + let db = backend.expose_db(); + let storage = backend.expose_storage(); - runner.sync_run(|config| cmd.run::(config)) - } else { - Err("Benchmarking wasn't enabled when building the node. You can enable it with \ - `--features runtime-benchmarks`." - .into()) - }, + cmd.run(config, client, db, storage) + }, + BenchmarkCmd::Overhead(cmd) => { + let ext_builder = BenchmarkExtrinsicBuilder::new(client.clone()); + + cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder)) + }, + } + }) + }, #[cfg(feature = "try-runtime")] Some(Subcommand::TryRuntime(cmd)) => { let runner = cli.create_runner(cmd)?; diff --git a/bin/node-template/node/src/command_helper.rs b/bin/node-template/node/src/command_helper.rs new file mode 100644 index 0000000000000..287e81b1e96bd --- /dev/null +++ b/bin/node-template/node/src/command_helper.rs @@ -0,0 +1,131 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Contains code to setup the command invocations in [`super::command`] which would +//! otherwise bloat that module. + +use crate::service::FullClient; + +use node_template_runtime as runtime; +use runtime::SystemCall; +use sc_cli::Result; +use sc_client_api::BlockBackend; +use sp_core::{Encode, Pair}; +use sp_inherents::{InherentData, InherentDataProvider}; +use sp_keyring::Sr25519Keyring; +use sp_runtime::{OpaqueExtrinsic, SaturatedConversion}; + +use std::{sync::Arc, time::Duration}; + +/// Generates extrinsics for the `benchmark overhead` command. +/// +/// Note: Should only be used for benchmarking. +pub struct BenchmarkExtrinsicBuilder { + client: Arc, +} + +impl BenchmarkExtrinsicBuilder { + /// Creates a new [`Self`] from the given client. + pub fn new(client: Arc) -> Self { + Self { client } + } +} + +impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder { + fn remark(&self, nonce: u32) -> std::result::Result { + let acc = Sr25519Keyring::Bob.pair(); + let extrinsic: OpaqueExtrinsic = create_benchmark_extrinsic( + self.client.as_ref(), + acc, + SystemCall::remark { remark: vec![] }.into(), + nonce, + ) + .into(); + + Ok(extrinsic) + } +} + +/// Create a transaction using the given `call`. +/// +/// Note: Should only be used for benchmarking. +pub fn create_benchmark_extrinsic( + client: &FullClient, + sender: sp_core::sr25519::Pair, + call: runtime::Call, + nonce: u32, +) -> runtime::UncheckedExtrinsic { + let genesis_hash = client.block_hash(0).ok().flatten().expect("Genesis block exists; qed"); + let best_hash = client.chain_info().best_hash; + let best_block = client.chain_info().best_number; + + let period = runtime::BlockHashCount::get() + .checked_next_power_of_two() + .map(|c| c / 2) + .unwrap_or(2) as u64; + let extra: runtime::SignedExtra = ( + frame_system::CheckNonZeroSender::::new(), + frame_system::CheckSpecVersion::::new(), + frame_system::CheckTxVersion::::new(), + frame_system::CheckGenesis::::new(), + frame_system::CheckEra::::from(sp_runtime::generic::Era::mortal( + period, + best_block.saturated_into(), + )), + frame_system::CheckNonce::::from(nonce), + frame_system::CheckWeight::::new(), + pallet_transaction_payment::ChargeTransactionPayment::::from(0), + ); + + let raw_payload = runtime::SignedPayload::from_raw( + call.clone(), + extra.clone(), + ( + (), + runtime::VERSION.spec_version, + runtime::VERSION.transaction_version, + genesis_hash, + best_hash, + (), + (), + (), + ), + ); + let signature = raw_payload.using_encoded(|e| sender.sign(e)); + + runtime::UncheckedExtrinsic::new_signed( + call.clone(), + sp_runtime::AccountId32::from(sender.public()).into(), + runtime::Signature::Sr25519(signature.clone()), + extra.clone(), + ) +} + +/// Generates inherent data for the `benchmark overhead` command. +/// +/// Note: Should only be used for benchmarking. +pub fn inherent_benchmark_data() -> Result { + let mut inherent_data = InherentData::new(); + let d = Duration::from_millis(0); + let timestamp = sp_timestamp::InherentDataProvider::new(d.into()); + + timestamp + .provide_inherent_data(&mut inherent_data) + .map_err(|e| format!("creating inherent data: {:?}", e))?; + Ok(inherent_data) +} diff --git a/bin/node-template/node/src/main.rs b/bin/node-template/node/src/main.rs index 4449d28b9fa41..0f2fbd5a909c6 100644 --- a/bin/node-template/node/src/main.rs +++ b/bin/node-template/node/src/main.rs @@ -6,6 +6,7 @@ mod chain_spec; mod service; mod cli; mod command; +mod command_helper; mod rpc; fn main() -> sc_cli::Result<()> { diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index fc7dc9b978df3..e2a8cb4ed834b 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -31,7 +31,7 @@ impl sc_executor::NativeExecutionDispatch for ExecutorDispatch { } } -type FullClient = +pub(crate) type FullClient = sc_service::TFullClient>; type FullBackend = sc_service::TFullBackend; type FullSelectChain = sc_consensus::LongestChain; diff --git a/bin/node-template/runtime/src/lib.rs b/bin/node-template/runtime/src/lib.rs index 40adbb0388111..780a84572d582 100644 --- a/bin/node-template/runtime/src/lib.rs +++ b/bin/node-template/runtime/src/lib.rs @@ -33,6 +33,7 @@ pub use frame_support::{ }, StorageValue, }; +pub use frame_system::Call as SystemCall; pub use pallet_balances::Call as BalancesCall; pub use pallet_timestamp::Call as TimestampCall; use pallet_transaction_payment::CurrencyAdapter; @@ -306,6 +307,8 @@ pub type SignedExtra = ( ); /// Unchecked extrinsic type as expected by this runtime. pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; +/// The payload being signed in transactions. +pub type SignedPayload = generic::SignedPayload; /// Executive: handles dispatch to the various modules. pub type Executive = frame_executive::Executive< Runtime, diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index 952c6311f8b38..7430cc46f4cc7 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -38,28 +38,11 @@ pub enum Subcommand { )] Inspect(node_inspect::cli::InspectCmd), - /// The custom benchmark subcommmand benchmarking runtime pallets. - #[clap(name = "benchmark", about = "Benchmark runtime pallets.")] + /// Sub-commands concerned with benchmarking. + /// The pallet benchmarking moved to the `pallet` sub-command. + #[clap(subcommand)] Benchmark(frame_benchmarking_cli::BenchmarkCmd), - /// Benchmark the execution time of historic blocks and compare it to their consumed weight. - #[clap( - name = "benchmark-block", - about = "Benchmark the execution time of historic blocks and compare it to their consumed weight." - )] - BenchmarkBlock(frame_benchmarking_cli::BlockCmd), - - /// Sub command for benchmarking the per-block and per-extrinsic execution overhead. - #[clap( - name = "benchmark-overhead", - about = "Benchmark the per-block and per-extrinsic execution overhead." - )] - BenchmarkOverhead(frame_benchmarking_cli::OverheadCmd), - - /// Sub command for benchmarking the storage speed. - #[clap(name = "benchmark-storage", about = "Benchmark storage speed.")] - BenchmarkStorage(frame_benchmarking_cli::StorageCmd), - /// Try some command against runtime state. #[cfg(feature = "try-runtime")] TryRuntime(try_runtime_cli::TryRuntimeCmd), diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index db243ff6f597b..c752b1c30ae26 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -16,11 +16,13 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . +use super::command_helper::{inherent_benchmark_data, BenchmarkExtrinsicBuilder}; use crate::{ chain_spec, service, service::{new_partial, FullClient}, Cli, Subcommand, }; +use frame_benchmarking_cli::*; use node_executor::ExecutorDispatch; use node_primitives::Block; use node_runtime::RuntimeApi; @@ -92,45 +94,39 @@ pub fn run() -> Result<()> { runner.sync_run(|config| cmd.run::(config)) }, - Some(Subcommand::Benchmark(cmd)) => - if cfg!(feature = "runtime-benchmarks") { - let runner = cli.create_runner(cmd)?; - - runner.sync_run(|config| cmd.run::(config)) - } else { - Err("Benchmarking wasn't enabled when building the node. \ - You can enable it with `--features runtime-benchmarks`." - .into()) - }, - Some(Subcommand::BenchmarkBlock(cmd)) => { + Some(Subcommand::Benchmark(cmd)) => { let runner = cli.create_runner(cmd)?; - runner.async_run(|config| { - let PartialComponents { client, task_manager, .. } = new_partial(&config)?; - Ok((cmd.run(client), task_manager)) - }) - }, - Some(Subcommand::BenchmarkOverhead(cmd)) => { - let runner = cli.create_runner(cmd)?; - runner.async_run(|mut config| { - use super::command_helper::{inherent_data, ExtrinsicBuilder}; - // We don't use the authority role since that would start producing blocks - // in the background which would mess with our benchmark. - config.role = sc_service::Role::Full; - - let PartialComponents { client, task_manager, .. } = new_partial(&config)?; - let ext_builder = ExtrinsicBuilder::new(client.clone()); - - Ok((cmd.run(config, client, inherent_data()?, Arc::new(ext_builder)), task_manager)) - }) - }, - Some(Subcommand::BenchmarkStorage(cmd)) => { - let runner = cli.create_runner(cmd)?; - runner.async_run(|config| { - let PartialComponents { client, task_manager, backend, .. } = new_partial(&config)?; - let db = backend.expose_db(); - let storage = backend.expose_storage(); - Ok((cmd.run(config, client, db, storage), task_manager)) + runner.sync_run(|config| { + let PartialComponents { client, backend, .. } = new_partial(&config)?; + + // This switch needs to be in the client, since the client decides + // which sub-commands it wants to support. + match cmd { + BenchmarkCmd::Pallet(cmd) => { + if !cfg!(feature = "runtime-benchmarks") { + return Err( + "Runtime benchmarking wasn't enabled when building the node. \ + You can enable it with `--features runtime-benchmarks`." + .into(), + ) + } + + cmd.run::(config) + }, + BenchmarkCmd::Block(cmd) => cmd.run(client), + BenchmarkCmd::Storage(cmd) => { + let db = backend.expose_db(); + let storage = backend.expose_storage(); + + cmd.run(config, client, db, storage) + }, + BenchmarkCmd::Overhead(cmd) => { + let ext_builder = BenchmarkExtrinsicBuilder::new(client.clone()); + + cmd.run(config, client, inherent_benchmark_data()?, Arc::new(ext_builder)) + }, + } }) }, Some(Subcommand::Key(cmd)) => cmd.run(&cli), diff --git a/bin/node/cli/src/command_helper.rs b/bin/node/cli/src/command_helper.rs index 51fe7a5c5a7bf..84d85ee367cab 100644 --- a/bin/node/cli/src/command_helper.rs +++ b/bin/node/cli/src/command_helper.rs @@ -29,19 +29,19 @@ use sp_runtime::OpaqueExtrinsic; use std::{sync::Arc, time::Duration}; -/// Generates extrinsics for the `benchmark-overhead` command. -pub struct ExtrinsicBuilder { +/// Generates extrinsics for the `benchmark overhead` command. +pub struct BenchmarkExtrinsicBuilder { client: Arc, } -impl ExtrinsicBuilder { +impl BenchmarkExtrinsicBuilder { /// Creates a new [`Self`] from the given client. pub fn new(client: Arc) -> Self { Self { client } } } -impl frame_benchmarking_cli::ExtrinsicBuilder for ExtrinsicBuilder { +impl frame_benchmarking_cli::ExtrinsicBuilder for BenchmarkExtrinsicBuilder { fn remark(&self, nonce: u32) -> std::result::Result { let acc = Sr25519Keyring::Bob.pair(); let extrinsic: OpaqueExtrinsic = create_extrinsic( @@ -56,8 +56,8 @@ impl frame_benchmarking_cli::ExtrinsicBuilder for ExtrinsicBuilder { } } -/// Generates inherent data for the `benchmark-overhead` command. -pub fn inherent_data() -> Result { +/// Generates inherent data for the `benchmark overhead` command. +pub fn inherent_benchmark_data() -> Result { let mut inherent_data = InherentData::new(); let d = Duration::from_millis(0); let timestamp = sp_timestamp::InherentDataProvider::new(d.into()); diff --git a/bin/node/cli/tests/benchmark_block_works.rs b/bin/node/cli/tests/benchmark_block_works.rs new file mode 100644 index 0000000000000..37a4db25f363b --- /dev/null +++ b/bin/node/cli/tests/benchmark_block_works.rs @@ -0,0 +1,48 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +// Unix only since it uses signals from [`common::run_node_for_a_while`]. +#![cfg(unix)] + +use assert_cmd::cargo::cargo_bin; +use std::process::Command; +use tempfile::tempdir; + +pub mod common; + +/// `benchmark block` works for the dev runtime using the wasm executor. +#[tokio::test] +async fn benchmark_block_works() { + let base_dir = tempdir().expect("could not create a temp dir"); + + common::run_node_for_a_while(base_dir.path(), &["--dev"]).await; + + // Invoke `benchmark block` with all options to make sure that they are valid. + let status = Command::new(cargo_bin("substrate")) + .args(["benchmark", "block", "--dev"]) + .arg("-d") + .arg(base_dir.path()) + .args(["--pruning", "archive"]) + .args(["--from", "1", "--to", "1"]) + .args(["--repeat", "1"]) + .args(["--execution", "wasm", "--wasm-execution", "compiled"]) + .status() + .unwrap(); + + assert!(status.success()) +} diff --git a/bin/node/cli/tests/benchmark_overhead_works.rs b/bin/node/cli/tests/benchmark_overhead_works.rs index 550221ee2f70f..44dcebfbc0c35 100644 --- a/bin/node/cli/tests/benchmark_overhead_works.rs +++ b/bin/node/cli/tests/benchmark_overhead_works.rs @@ -20,7 +20,7 @@ use assert_cmd::cargo::cargo_bin; use std::process::Command; use tempfile::tempdir; -/// Tests that the `benchmark-overhead` command works for the substrate dev runtime. +/// Tests that the `benchmark overhead` command works for the substrate dev runtime. #[test] fn benchmark_overhead_works() { let tmp_dir = tempdir().expect("could not create a temp dir"); @@ -29,7 +29,7 @@ fn benchmark_overhead_works() { // Only put 10 extrinsics into the block otherwise it takes forever to build it // especially for a non-release build. let status = Command::new(cargo_bin("substrate")) - .args(&["benchmark-overhead", "--dev", "-d"]) + .args(&["benchmark", "overhead", "--dev", "-d"]) .arg(base_path) .arg("--weight-path") .arg(base_path) diff --git a/bin/node/cli/tests/benchmark_storage_works.rs b/bin/node/cli/tests/benchmark_storage_works.rs index 1628f9a7e97ba..30f860e48459f 100644 --- a/bin/node/cli/tests/benchmark_storage_works.rs +++ b/bin/node/cli/tests/benchmark_storage_works.rs @@ -23,7 +23,7 @@ use std::{ }; use tempfile::tempdir; -/// Tests that the `benchmark-storage` command works for the dev runtime. +/// Tests that the `benchmark storage` command works for the dev runtime. #[test] fn benchmark_storage_works() { let tmp_dir = tempdir().expect("could not create a temp dir"); @@ -39,7 +39,7 @@ fn benchmark_storage_works() { fn benchmark_storage(db: &str, base_path: &Path) -> ExitStatus { Command::new(cargo_bin("substrate")) - .args(&["benchmark-storage", "--dev"]) + .args(&["benchmark", "storage", "--dev"]) .arg("--db") .arg(db) .arg("--weight-path") diff --git a/utils/frame/benchmarking-cli/src/block/bench.rs b/utils/frame/benchmarking-cli/src/block/bench.rs index d3c1c97b04ab2..e48a7e8b3c6f5 100644 --- a/utils/frame/benchmarking-cli/src/block/bench.rs +++ b/utils/frame/benchmarking-cli/src/block/bench.rs @@ -34,7 +34,7 @@ use serde::Serialize; use std::{fmt::Debug, marker::PhantomData, sync::Arc, time::Instant}; use thousands::Separable; -use crate::storage::record::{StatSelect, Stats}; +use crate::shared::{StatSelect, Stats}; /// Log target for printing block weight info. const LOG_TARGET: &'static str = "benchmark::block::weight"; diff --git a/utils/frame/benchmarking-cli/src/block/cmd.rs b/utils/frame/benchmarking-cli/src/block/cmd.rs index 4618c1dd894e1..e4e1716b1c5ac 100644 --- a/utils/frame/benchmarking-cli/src/block/cmd.rs +++ b/utils/frame/benchmarking-cli/src/block/cmd.rs @@ -29,7 +29,7 @@ use std::{fmt::Debug, sync::Arc}; use super::bench::{Benchmark, BenchmarkParams}; -/// Benchmark the execution time historic blocks. +/// Benchmark the execution time of historic blocks. /// /// This can be used to verify that blocks do not use more weight than they consumed /// in their `WeightInfo`. Example: @@ -73,7 +73,7 @@ impl BlockCmd { /// Benchmark the execution time of historic blocks and compare it to their consumed weight. /// /// Output will be printed to console. - pub async fn run(&self, client: Arc) -> Result<()> + pub fn run(&self, client: Arc) -> Result<()> where Block: BlockT, BA: ClientBackend, diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index 288e6b4b86156..2543a63b2f159 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -15,144 +15,85 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! Contains the root [`BenchmarkCmd`] command and exports its sub-commands. + mod block; -mod command; -pub mod overhead; -mod post_processing; +mod overhead; +mod pallet; +mod shared; mod storage; -mod writer; - -use sc_cli::{ExecutionStrategy, WasmExecutionMethod, DEFAULT_WASM_EXECUTION_METHOD}; -use std::{fmt::Debug, path::PathBuf}; pub use block::BlockCmd; pub use overhead::{ExtrinsicBuilder, OverheadCmd}; +pub use pallet::PalletCmd; pub use storage::StorageCmd; -// Add a more relaxed parsing for pallet names by allowing pallet directory names with `-` to be -// used like crate names with `_` -fn parse_pallet_name(pallet: &str) -> String { - pallet.replace("-", "_") +use sc_cli::{CliConfiguration, DatabaseParams, ImportParams, PruningParams, Result, SharedParams}; + +/// The root `benchmarking` command. +/// +/// Has no effect itself besides printing a help menu of the sub-commands. +#[derive(Debug, clap::Subcommand)] +pub enum BenchmarkCmd { + Pallet(PalletCmd), + Storage(StorageCmd), + Overhead(OverheadCmd), + Block(BlockCmd), } -/// The `benchmark` command used to benchmark FRAME Pallets. -#[derive(Debug, clap::Parser)] -pub struct BenchmarkCmd { - /// Select a FRAME Pallet to benchmark, or `*` for all (in which case `extrinsic` must be `*`). - #[clap(short, long, parse(from_str = parse_pallet_name), required_unless_present = "list")] - pub pallet: Option, - - /// Select an extrinsic inside the pallet to benchmark, or `*` for all. - #[clap(short, long, required_unless_present = "list")] - pub extrinsic: Option, - - /// Select how many samples we should take across the variable components. - #[clap(short, long, default_value = "1")] - pub steps: u32, - - /// Indicates lowest values for each of the component ranges. - #[clap(long = "low", use_value_delimiter = true)] - pub lowest_range_values: Vec, - - /// Indicates highest values for each of the component ranges. - #[clap(long = "high", use_value_delimiter = true)] - pub highest_range_values: Vec, - - /// Select how many repetitions of this benchmark should run from within the wasm. - #[clap(short, long, default_value = "1")] - pub repeat: u32, - - /// Select how many repetitions of this benchmark should run from the client. - /// - /// NOTE: Using this alone may give slower results, but will afford you maximum Wasm memory. - #[clap(long, default_value = "1")] - pub external_repeat: u32, - - /// Print the raw results in JSON format. - #[clap(long = "json")] - pub json_output: bool, - - /// Write the raw results in JSON format into the given file. - #[clap(long, conflicts_with = "json-output")] - pub json_file: Option, - - /// Don't print the median-slopes linear regression analysis. - #[clap(long)] - pub no_median_slopes: bool, - - /// Don't print the min-squares linear regression analysis. - #[clap(long)] - pub no_min_squares: bool, - - /// Output the benchmarks to a Rust file at the given path. - #[clap(long)] - pub output: Option, - - /// Add a header file to your outputted benchmarks - #[clap(long)] - pub header: Option, - - /// Path to Handlebars template file used for outputting benchmark results. (Optional) - #[clap(long)] - pub template: Option, - - /// Which analysis function to use when outputting benchmarks: - /// * min-squares (default) - /// * median-slopes - /// * max (max of min squares and median slopes for each value) - #[clap(long)] - pub output_analysis: Option, - - /// Set the heap pages while running benchmarks. If not set, the default value from the client - /// is used. - #[clap(long)] - pub heap_pages: Option, - - /// Disable verification logic when running benchmarks. - #[clap(long)] - pub no_verify: bool, - - /// Display and run extra benchmarks that would otherwise not be needed for weight - /// construction. - #[clap(long)] - pub extra: bool, - - /// Estimate PoV size. - #[clap(long)] - pub record_proof: bool, - - #[allow(missing_docs)] - #[clap(flatten)] - pub shared_params: sc_cli::SharedParams, - - /// The execution strategy that should be used for benchmarks - #[clap(long, value_name = "STRATEGY", arg_enum, ignore_case = true)] - pub execution: Option, - - /// Method for executing Wasm runtime code. - #[clap( - long = "wasm-execution", - value_name = "METHOD", - possible_values = WasmExecutionMethod::variants(), - ignore_case = true, - default_value = DEFAULT_WASM_EXECUTION_METHOD, - )] - pub wasm_method: WasmExecutionMethod, - - /// Limit the memory the database cache can use. - #[clap(long = "db-cache", value_name = "MiB", default_value = "1024")] - pub database_cache_size: u32, - - /// List the benchmarks that match your query rather than running them. - /// - /// When nothing is provided, we list all benchmarks. - #[clap(long)] - pub list: bool, +/// Unwraps a [`BenchmarkCmd`] into its concrete sub-command. +macro_rules! unwrap_cmd { + { + $self:expr, + $cmd:ident, + $code:expr + } => { + match $self { + BenchmarkCmd::Pallet($cmd) => $code, + BenchmarkCmd::Storage($cmd) => $code, + BenchmarkCmd::Overhead($cmd) => $code, + BenchmarkCmd::Block($cmd) => $code, + } + } +} - /// If enabled, the storage info is not displayed in the output next to the analysis. - /// - /// This is independent of the storage info appearing in the *output file*. Use a Handlebar - /// template for that purpose. - #[clap(long)] - pub no_storage_info: bool, +/// Forward the [`CliConfiguration`] trait implementation. +/// +/// Each time a sub-command exposes a new config option, it must be added here. +impl CliConfiguration for BenchmarkCmd { + fn shared_params(&self) -> &SharedParams { + unwrap_cmd! { + self, cmd, cmd.shared_params() + } + } + + fn import_params(&self) -> Option<&ImportParams> { + unwrap_cmd! { + self, cmd, cmd.import_params() + } + } + + fn database_params(&self) -> Option<&DatabaseParams> { + unwrap_cmd! { + self, cmd, cmd.database_params() + } + } + + fn pruning_params(&self) -> Option<&PruningParams> { + unwrap_cmd! { + self, cmd, cmd.pruning_params() + } + } + + fn state_cache_size(&self) -> Result { + unwrap_cmd! { + self, cmd, cmd.state_cache_size() + } + } + + fn chain_id(&self, is_dev: bool) -> Result { + unwrap_cmd! { + self, cmd, cmd.chain_id(is_dev) + } + } } diff --git a/utils/frame/benchmarking-cli/src/overhead/bench.rs b/utils/frame/benchmarking-cli/src/overhead/bench.rs index 3e18c6a86db24..68f3f6597b466 100644 --- a/utils/frame/benchmarking-cli/src/overhead/bench.rs +++ b/utils/frame/benchmarking-cli/src/overhead/bench.rs @@ -36,7 +36,8 @@ use log::info; use serde::Serialize; use std::{marker::PhantomData, sync::Arc, time::Instant}; -use crate::{overhead::cmd::ExtrinsicBuilder, storage::record::Stats}; +use super::cmd::ExtrinsicBuilder; +use crate::shared::Stats; /// Parameters to configure an *overhead* benchmark. #[derive(Debug, Default, Serialize, Clone, PartialEq, Args)] diff --git a/utils/frame/benchmarking-cli/src/overhead/cmd.rs b/utils/frame/benchmarking-cli/src/overhead/cmd.rs index f74c32ba72a86..3cf281986861f 100644 --- a/utils/frame/benchmarking-cli/src/overhead/cmd.rs +++ b/utils/frame/benchmarking-cli/src/overhead/cmd.rs @@ -35,10 +35,10 @@ use crate::{ bench::{Benchmark, BenchmarkParams, BenchmarkType}, template::TemplateData, }, - post_processing::WeightParams, + shared::WeightParams, }; -/// Benchmarks the per-block and per-extrinsic execution overhead. +/// Benchmark the execution overhead per-block and per-extrinsic. #[derive(Debug, Parser)] pub struct OverheadCmd { #[allow(missing_docs)] @@ -76,11 +76,11 @@ pub trait ExtrinsicBuilder { } impl OverheadCmd { - /// Measures the per-block and per-extrinsic execution overhead. + /// Measure the per-block and per-extrinsic execution overhead. /// /// Writes the results to console and into two instances of the /// `weights.hbs` template, one for each benchmark. - pub async fn run( + pub fn run( &self, cfg: Configuration, client: Arc, diff --git a/utils/frame/benchmarking-cli/src/overhead/template.rs b/utils/frame/benchmarking-cli/src/overhead/template.rs index f6fb8ed9d929e..d5f90d4873866 100644 --- a/utils/frame/benchmarking-cli/src/overhead/template.rs +++ b/utils/frame/benchmarking-cli/src/overhead/template.rs @@ -28,7 +28,7 @@ use std::{env, fs, path::PathBuf}; use crate::{ overhead::{bench::BenchmarkType, cmd::OverheadParams}, - storage::record::Stats, + shared::{Stats, UnderscoreHelper}, }; static VERSION: &'static str = env!("CARGO_PKG_VERSION"); @@ -85,7 +85,7 @@ impl TemplateData { pub fn write(&self, path: &Option) -> Result<()> { let mut handlebars = Handlebars::new(); // Format large integers with underscores. - handlebars.register_helper("underscore", Box::new(crate::writer::UnderscoreHelper)); + handlebars.register_helper("underscore", Box::new(UnderscoreHelper)); // Don't HTML escape any characters. handlebars.register_escape_fn(|s| -> String { s.to_string() }); diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/pallet/command.rs similarity index 95% rename from utils/frame/benchmarking-cli/src/command.rs rename to utils/frame/benchmarking-cli/src/pallet/command.rs index 0ced8b28ce016..89b8d018bcb5c 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/pallet/command.rs @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::BenchmarkCmd; +use super::{writer, PalletCmd}; use codec::{Decode, Encode}; use frame_benchmarking::{ Analysis, BenchmarkBatch, BenchmarkBatchSplitResults, BenchmarkList, BenchmarkParameter, @@ -87,7 +87,13 @@ fn combine_batches( .collect::>() } -impl BenchmarkCmd { +/// Explains possible reasons why the metadata for the benchmarking could not be found. +const ERROR_METADATA_NOT_FOUND: &'static str = "Did not find the benchmarking metadata. \ +This could mean that you either did not build the node correctly with the \ +`--features runtime-benchmarks` flag, or the chain spec that you are using was \ +not created by a node that was compiled with the flag"; + +impl PalletCmd { /// Runs the command and benchmarks the chain. pub fn run(&self, config: Configuration) -> Result<()> where @@ -165,7 +171,7 @@ impl BenchmarkCmd { sp_core::testing::TaskExecutor::new(), ) .execute(strategy.into()) - .map_err(|e| format!("Error getting benchmark list: {}", e))?; + .map_err(|e| format!("{}: {}", ERROR_METADATA_NOT_FOUND, e))?; let (list, storage_info) = <(Vec, Vec) as Decode>::decode(&mut &result[..]) @@ -359,7 +365,7 @@ impl BenchmarkCmd { // Create the weights.rs file. if let Some(output_path) = &self.output { - crate::writer::write_results(&batches, &storage_info, output_path, self)?; + writer::write_results(&batches, &storage_info, output_path, self)?; } // Jsonify the result and write it to a file or stdout if desired. @@ -414,11 +420,7 @@ impl BenchmarkCmd { if !self.no_storage_info { let mut comments: Vec = Default::default(); - crate::writer::add_storage_comments( - &mut comments, - &batch.db_results, - &storage_info, - ); + writer::add_storage_comments(&mut comments, &batch.db_results, &storage_info); println!("Raw Storage Info\n========"); for comment in comments { println!("{}", comment); @@ -469,7 +471,7 @@ impl BenchmarkCmd { } } -impl CliConfiguration for BenchmarkCmd { +impl CliConfiguration for PalletCmd { fn shared_params(&self) -> &SharedParams { &self.shared_params } diff --git a/utils/frame/benchmarking-cli/src/pallet/mod.rs b/utils/frame/benchmarking-cli/src/pallet/mod.rs new file mode 100644 index 0000000000000..48ddcc7ce8eec --- /dev/null +++ b/utils/frame/benchmarking-cli/src/pallet/mod.rs @@ -0,0 +1,150 @@ +// This file is part of Substrate. + +// Copyright (C) 2020-2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod command; +mod writer; + +use sc_cli::{ExecutionStrategy, WasmExecutionMethod, DEFAULT_WASM_EXECUTION_METHOD}; +use std::{fmt::Debug, path::PathBuf}; + +// Add a more relaxed parsing for pallet names by allowing pallet directory names with `-` to be +// used like crate names with `_` +fn parse_pallet_name(pallet: &str) -> String { + pallet.replace("-", "_") +} + +/// Benchmark the extrinsic weight of FRAME Pallets. +#[derive(Debug, clap::Parser)] +pub struct PalletCmd { + /// Select a FRAME Pallet to benchmark, or `*` for all (in which case `extrinsic` must be `*`). + #[clap(short, long, parse(from_str = parse_pallet_name), required_unless_present = "list")] + pub pallet: Option, + + /// Select an extrinsic inside the pallet to benchmark, or `*` for all. + #[clap(short, long, required_unless_present = "list")] + pub extrinsic: Option, + + /// Select how many samples we should take across the variable components. + #[clap(short, long, default_value = "1")] + pub steps: u32, + + /// Indicates lowest values for each of the component ranges. + #[clap(long = "low", use_value_delimiter = true)] + pub lowest_range_values: Vec, + + /// Indicates highest values for each of the component ranges. + #[clap(long = "high", use_value_delimiter = true)] + pub highest_range_values: Vec, + + /// Select how many repetitions of this benchmark should run from within the wasm. + #[clap(short, long, default_value = "1")] + pub repeat: u32, + + /// Select how many repetitions of this benchmark should run from the client. + /// + /// NOTE: Using this alone may give slower results, but will afford you maximum Wasm memory. + #[clap(long, default_value = "1")] + pub external_repeat: u32, + + /// Print the raw results in JSON format. + #[clap(long = "json")] + pub json_output: bool, + + /// Write the raw results in JSON format into the given file. + #[clap(long, conflicts_with = "json-output")] + pub json_file: Option, + + /// Don't print the median-slopes linear regression analysis. + #[clap(long)] + pub no_median_slopes: bool, + + /// Don't print the min-squares linear regression analysis. + #[clap(long)] + pub no_min_squares: bool, + + /// Output the benchmarks to a Rust file at the given path. + #[clap(long)] + pub output: Option, + + /// Add a header file to your outputted benchmarks. + #[clap(long)] + pub header: Option, + + /// Path to Handlebars template file used for outputting benchmark results. (Optional) + #[clap(long)] + pub template: Option, + + /// Which analysis function to use when outputting benchmarks: + /// * min-squares (default) + /// * median-slopes + /// * max (max of min squares and median slopes for each value) + #[clap(long)] + pub output_analysis: Option, + + /// Set the heap pages while running benchmarks. If not set, the default value from the client + /// is used. + #[clap(long)] + pub heap_pages: Option, + + /// Disable verification logic when running benchmarks. + #[clap(long)] + pub no_verify: bool, + + /// Display and run extra benchmarks that would otherwise not be needed for weight + /// construction. + #[clap(long)] + pub extra: bool, + + /// Estimate PoV size. + #[clap(long)] + pub record_proof: bool, + + #[allow(missing_docs)] + #[clap(flatten)] + pub shared_params: sc_cli::SharedParams, + + /// The execution strategy that should be used for benchmarks. + #[clap(long, value_name = "STRATEGY", arg_enum, ignore_case = true)] + pub execution: Option, + + /// Method for executing Wasm runtime code. + #[clap( + long = "wasm-execution", + value_name = "METHOD", + possible_values = WasmExecutionMethod::variants(), + ignore_case = true, + default_value = DEFAULT_WASM_EXECUTION_METHOD, + )] + pub wasm_method: WasmExecutionMethod, + + /// Limit the memory the database cache can use. + #[clap(long = "db-cache", value_name = "MiB", default_value = "1024")] + pub database_cache_size: u32, + + /// List the benchmarks that match your query rather than running them. + /// + /// When nothing is provided, we list all benchmarks. + #[clap(long)] + pub list: bool, + + /// If enabled, the storage info is not displayed in the output next to the analysis. + /// + /// This is independent of the storage info appearing in the *output file*. Use a Handlebar + /// template for that purpose. + #[clap(long)] + pub no_storage_info: bool, +} diff --git a/utils/frame/benchmarking-cli/src/template.hbs b/utils/frame/benchmarking-cli/src/pallet/template.hbs similarity index 100% rename from utils/frame/benchmarking-cli/src/template.hbs rename to utils/frame/benchmarking-cli/src/pallet/template.hbs diff --git a/utils/frame/benchmarking-cli/src/writer.rs b/utils/frame/benchmarking-cli/src/pallet/writer.rs similarity index 93% rename from utils/frame/benchmarking-cli/src/writer.rs rename to utils/frame/benchmarking-cli/src/pallet/writer.rs index 17f1221e46d8b..cd97b3efbd9db 100644 --- a/utils/frame/benchmarking-cli/src/writer.rs +++ b/utils/frame/benchmarking-cli/src/pallet/writer.rs @@ -26,7 +26,7 @@ use std::{ use inflector::Inflector; use serde::Serialize; -use crate::BenchmarkCmd; +use crate::{shared::UnderscoreHelper, PalletCmd}; use frame_benchmarking::{ Analysis, AnalysisChoice, BenchmarkBatchSplitResults, BenchmarkResult, BenchmarkSelector, RegressionModel, @@ -68,7 +68,7 @@ struct BenchmarkData { comments: Vec, } -// This forwards some specific metadata from the `BenchmarkCmd` +// This forwards some specific metadata from the `PalletCmd` #[derive(Serialize, Default, Debug, Clone)] struct CmdData { steps: u32, @@ -255,7 +255,7 @@ pub fn write_results( batches: &[BenchmarkBatchSplitResults], storage_info: &[StorageInfo], path: &PathBuf, - cmd: &BenchmarkCmd, + cmd: &PalletCmd, ) -> Result<(), std::io::Error> { // Use custom template if provided. let template: String = match &cmd.template { @@ -416,44 +416,6 @@ pub(crate) fn add_storage_comments( } } -// Add an underscore after every 3rd character, i.e. a separator for large numbers. -fn underscore(i: Number) -> String -where - Number: std::string::ToString, -{ - let mut s = String::new(); - let i_str = i.to_string(); - let a = i_str.chars().rev().enumerate(); - for (idx, val) in a { - if idx != 0 && idx % 3 == 0 { - s.insert(0, '_'); - } - s.insert(0, val); - } - s -} - -// A Handlebars helper to add an underscore after every 3rd character, -// i.e. a separator for large numbers. -#[derive(Clone, Copy)] -pub(crate) struct UnderscoreHelper; -impl handlebars::HelperDef for UnderscoreHelper { - fn call<'reg: 'rc, 'rc>( - &self, - h: &handlebars::Helper, - _: &handlebars::Handlebars, - _: &handlebars::Context, - _rc: &mut handlebars::RenderContext, - out: &mut dyn handlebars::Output, - ) -> handlebars::HelperResult { - use handlebars::JsonRender; - let param = h.param(0).unwrap(); - let underscore_param = underscore(param.value().render()); - out.write(&underscore_param)?; - Ok(()) - } -} - // A helper to join a string of vectors. #[derive(Clone, Copy)] struct JoinHelper; diff --git a/utils/frame/benchmarking-cli/src/shared/mod.rs b/utils/frame/benchmarking-cli/src/shared/mod.rs new file mode 100644 index 0000000000000..f08d79b9aafca --- /dev/null +++ b/utils/frame/benchmarking-cli/src/shared/mod.rs @@ -0,0 +1,65 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Code that is shared among all benchmarking sub-commands. + +pub mod record; +pub mod stats; +pub mod weight_params; + +pub use record::BenchRecord; +pub use stats::{StatSelect, Stats}; +pub use weight_params::WeightParams; + +/// A Handlebars helper to add an underscore after every 3rd character, +/// i.e. a separator for large numbers. +#[derive(Clone, Copy)] +pub struct UnderscoreHelper; + +impl handlebars::HelperDef for UnderscoreHelper { + fn call<'reg: 'rc, 'rc>( + &self, + h: &handlebars::Helper, + _: &handlebars::Handlebars, + _: &handlebars::Context, + _rc: &mut handlebars::RenderContext, + out: &mut dyn handlebars::Output, + ) -> handlebars::HelperResult { + use handlebars::JsonRender; + let param = h.param(0).unwrap(); + let underscore_param = underscore(param.value().render()); + out.write(&underscore_param)?; + Ok(()) + } +} + +/// Add an underscore after every 3rd character, i.e. a separator for large numbers. +fn underscore(i: Number) -> String +where + Number: std::string::ToString, +{ + let mut s = String::new(); + let i_str = i.to_string(); + let a = i_str.chars().rev().enumerate(); + for (idx, val) in a { + if idx != 0 && idx % 3 == 0 { + s.insert(0, '_'); + } + s.insert(0, val); + } + s +} diff --git a/utils/frame/benchmarking-cli/src/shared/record.rs b/utils/frame/benchmarking-cli/src/shared/record.rs new file mode 100644 index 0000000000000..79ab37f651528 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/shared/record.rs @@ -0,0 +1,72 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Defines the [`BenchRecord`] and its facilities for computing [`super::Stats`]. + +use sc_cli::Result; +use sc_service::Configuration; + +use log::info; +use serde::Serialize; +use std::{fs, path::PathBuf, time::Duration}; + +use super::Stats; + +/// Raw output of a Storage benchmark. +#[derive(Debug, Default, Clone, Serialize)] +pub struct BenchRecord { + /// Multi-Map of value sizes and the time that it took to access them. + ns_per_size: Vec<(u64, u64)>, +} + +impl BenchRecord { + /// Appends a new record. Uses safe casts. + pub fn append(&mut self, size: usize, d: Duration) -> Result<()> { + let size: u64 = size.try_into().map_err(|e| format!("Size overflow u64: {}", e))?; + let ns: u64 = d + .as_nanos() + .try_into() + .map_err(|e| format!("Nanoseconds overflow u64: {}", e))?; + self.ns_per_size.push((size, ns)); + Ok(()) + } + + /// Returns the statistics for *time* and *value size*. + pub fn calculate_stats(self) -> Result<(Stats, Stats)> { + let (size, time): (Vec<_>, Vec<_>) = self.ns_per_size.into_iter().unzip(); + let size = Stats::new(&size)?; + let time = Stats::new(&time)?; + Ok((time, size)) // The swap of time/size here is intentional. + } + + /// Unless a path is specified, saves the raw results in a json file in the current directory. + /// Prefixes it with the DB name and suffixed with `path_suffix`. + pub fn save_json(&self, cfg: &Configuration, out_path: &PathBuf, suffix: &str) -> Result<()> { + let mut path = PathBuf::from(out_path); + if path.is_dir() || path.as_os_str().is_empty() { + path.push(&format!("{}_{}", cfg.database, suffix).to_lowercase()); + path.set_extension("json"); + } + + let json = serde_json::to_string_pretty(&self) + .map_err(|e| format!("Serializing as JSON: {:?}", e))?; + + fs::write(&path, json)?; + info!("Raw data written to {:?}", fs::canonicalize(&path)?); + Ok(()) + } +} diff --git a/utils/frame/benchmarking-cli/src/storage/record.rs b/utils/frame/benchmarking-cli/src/shared/stats.rs similarity index 70% rename from utils/frame/benchmarking-cli/src/storage/record.rs rename to utils/frame/benchmarking-cli/src/shared/stats.rs index 530fa4cdfe965..7785965fed4a7 100644 --- a/utils/frame/benchmarking-cli/src/storage/record.rs +++ b/utils/frame/benchmarking-cli/src/shared/stats.rs @@ -15,46 +15,38 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Calculates statistics and fills out the `weight.hbs` template. +//! Handles statistics that were generated from benchmarking results and +//! that can be used to fill out weight templates. use sc_cli::Result; -use sc_service::Configuration; -use log::info; use serde::Serialize; -use std::{fmt, fs, path::PathBuf, result, str::FromStr, time::Duration}; - -/// Raw output of a Storage benchmark. -#[derive(Debug, Default, Clone, Serialize)] -pub(crate) struct BenchRecord { - /// Multi-Map of value sizes and the time that it took to access them. - ns_per_size: Vec<(u64, u64)>, -} +use std::{fmt, result, str::FromStr}; /// Various statistics that help to gauge the quality of the produced weights. /// Will be written to the weight file and printed to console. #[derive(Serialize, Default, Clone)] -pub(crate) struct Stats { +pub struct Stats { /// Sum of all values. - pub(crate) sum: u64, + pub sum: u64, /// Minimal observed value. - pub(crate) min: u64, + pub min: u64, /// Maximal observed value. - pub(crate) max: u64, + pub max: u64, /// Average of all values. - pub(crate) avg: u64, + pub avg: u64, /// Median of all values. - pub(crate) median: u64, + pub median: u64, /// Standard derivation of all values. - pub(crate) stddev: f64, + pub stddev: f64, /// 99th percentile. At least 99% of all values are below this threshold. - pub(crate) p99: u64, + pub p99: u64, /// 95th percentile. At least 95% of all values are below this threshold. - pub(crate) p95: u64, + pub p95: u64, /// 75th percentile. At least 75% of all values are below this threshold. - pub(crate) p75: u64, + pub p75: u64, } /// Selects a specific field from a [`Stats`] object. @@ -75,44 +67,6 @@ pub enum StatSelect { P75Percentile, } -impl BenchRecord { - /// Appends a new record. Uses safe casts. - pub fn append(&mut self, size: usize, d: Duration) -> Result<()> { - let size: u64 = size.try_into().map_err(|e| format!("Size overflow u64: {}", e))?; - let ns: u64 = d - .as_nanos() - .try_into() - .map_err(|e| format!("Nanoseconds overflow u64: {}", e))?; - self.ns_per_size.push((size, ns)); - Ok(()) - } - - /// Returns the statistics for *time* and *value size*. - pub(crate) fn calculate_stats(self) -> Result<(Stats, Stats)> { - let (size, time): (Vec<_>, Vec<_>) = self.ns_per_size.into_iter().unzip(); - let size = Stats::new(&size)?; - let time = Stats::new(&time)?; - Ok((time, size)) // The swap of time/size here is intentional. - } - - /// Unless a path is specified, saves the raw results in a json file in the current directory. - /// Prefixes it with the DB name and suffixed with `path_suffix`. - pub fn save_json(&self, cfg: &Configuration, out_path: &PathBuf, suffix: &str) -> Result<()> { - let mut path = PathBuf::from(out_path); - if path.is_dir() || path.as_os_str().is_empty() { - path.push(&format!("{}_{}", cfg.database, suffix).to_lowercase()); - path.set_extension("json"); - } - - let json = serde_json::to_string_pretty(&self) - .map_err(|e| format!("Serializing as JSON: {:?}", e))?; - - fs::write(&path, json)?; - info!("Raw data written to {:?}", fs::canonicalize(&path)?); - Ok(()) - } -} - impl Stats { /// Calculates statistics and returns them. pub fn new(xs: &Vec) -> Result { @@ -137,7 +91,7 @@ impl Stats { } /// Returns the selected stat. - pub(crate) fn select(&self, s: StatSelect) -> u64 { + pub fn select(&self, s: StatSelect) -> u64 { match s { StatSelect::Maximum => self.max, StatSelect::Average => self.avg, diff --git a/utils/frame/benchmarking-cli/src/post_processing/mod.rs b/utils/frame/benchmarking-cli/src/shared/weight_params.rs similarity index 92% rename from utils/frame/benchmarking-cli/src/post_processing/mod.rs rename to utils/frame/benchmarking-cli/src/shared/weight_params.rs index fb20d9bd0c488..4dd80cd41ff3d 100644 --- a/utils/frame/benchmarking-cli/src/post_processing/mod.rs +++ b/utils/frame/benchmarking-cli/src/shared/weight_params.rs @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Calculates a weight from the statistics of a benchmark result. +//! Calculates a weight from the [`super::Stats`] of a benchmark result. use sc_cli::Result; @@ -23,7 +23,7 @@ use clap::Args; use serde::Serialize; use std::path::PathBuf; -use crate::storage::record::{StatSelect, Stats}; +use super::{StatSelect, Stats}; /// Configures the weight generation. #[derive(Debug, Default, Serialize, Clone, PartialEq, Args)] @@ -55,7 +55,7 @@ pub struct WeightParams { /// `weight_mul` and adding `weight_add`. /// Does not use safe casts and can overflow. impl WeightParams { - pub(crate) fn calc_weight(&self, stat: &Stats) -> Result { + pub fn calc_weight(&self, stat: &Stats) -> Result { if self.weight_mul.is_sign_negative() || !self.weight_mul.is_normal() { return Err("invalid floating number for `weight_mul`".into()) } @@ -68,7 +68,7 @@ impl WeightParams { #[cfg(test)] mod test_weight_params { use super::WeightParams; - use crate::storage::record::{StatSelect, Stats}; + use crate::shared::{StatSelect, Stats}; #[test] fn calc_weight_works() { diff --git a/utils/frame/benchmarking-cli/src/storage/cmd.rs b/utils/frame/benchmarking-cli/src/storage/cmd.rs index c38e6636e5a3e..c2cc219ef1528 100644 --- a/utils/frame/benchmarking-cli/src/storage/cmd.rs +++ b/utils/frame/benchmarking-cli/src/storage/cmd.rs @@ -34,8 +34,9 @@ use sp_runtime::generic::BlockId; use std::{fmt::Debug, path::PathBuf, sync::Arc}; use super::template::TemplateData; -use crate::post_processing::WeightParams; -/// Benchmark the storage of a Substrate node with a live chain snapshot. +use crate::shared::WeightParams; + +/// Benchmark the storage speed of a chain snapshot. #[derive(Debug, Parser)] pub struct StorageCmd { #[allow(missing_docs)] @@ -99,7 +100,7 @@ pub struct StorageParams { impl StorageCmd { /// Calls into the Read and Write benchmarking functions. /// Processes the output and writes it into files and stdout. - pub async fn run( + pub fn run( &self, cfg: Configuration, client: Arc, diff --git a/utils/frame/benchmarking-cli/src/storage/mod.rs b/utils/frame/benchmarking-cli/src/storage/mod.rs index 9849cbcb6097b..0c722fdd47029 100644 --- a/utils/frame/benchmarking-cli/src/storage/mod.rs +++ b/utils/frame/benchmarking-cli/src/storage/mod.rs @@ -17,7 +17,6 @@ pub mod cmd; pub mod read; -pub mod record; pub mod template; pub mod write; diff --git a/utils/frame/benchmarking-cli/src/storage/read.rs b/utils/frame/benchmarking-cli/src/storage/read.rs index ca506202e1067..f58f3c3de0c19 100644 --- a/utils/frame/benchmarking-cli/src/storage/read.rs +++ b/utils/frame/benchmarking-cli/src/storage/read.rs @@ -27,7 +27,8 @@ use log::info; use rand::prelude::*; use std::{fmt::Debug, sync::Arc, time::Instant}; -use super::{cmd::StorageCmd, record::BenchRecord}; +use super::cmd::StorageCmd; +use crate::shared::BenchRecord; impl StorageCmd { /// Benchmarks the time it takes to read a single Storage item. diff --git a/utils/frame/benchmarking-cli/src/storage/template.rs b/utils/frame/benchmarking-cli/src/storage/template.rs index 10e6902b934bc..26aa8a962301b 100644 --- a/utils/frame/benchmarking-cli/src/storage/template.rs +++ b/utils/frame/benchmarking-cli/src/storage/template.rs @@ -22,7 +22,8 @@ use log::info; use serde::Serialize; use std::{env, fs, path::PathBuf}; -use super::{cmd::StorageParams, record::Stats}; +use super::cmd::StorageParams; +use crate::shared::{Stats, UnderscoreHelper}; static VERSION: &'static str = env!("CARGO_PKG_VERSION"); static TEMPLATE: &str = include_str!("./weights.hbs"); @@ -97,7 +98,7 @@ impl TemplateData { pub fn write(&self, path: &Option, hbs_template: &Option) -> Result<()> { let mut handlebars = handlebars::Handlebars::new(); // Format large integers with underscore. - handlebars.register_helper("underscore", Box::new(crate::writer::UnderscoreHelper)); + handlebars.register_helper("underscore", Box::new(UnderscoreHelper)); // Don't HTML escape any characters. handlebars.register_escape_fn(|s| -> String { s.to_string() }); // Use custom template if provided. diff --git a/utils/frame/benchmarking-cli/src/storage/write.rs b/utils/frame/benchmarking-cli/src/storage/write.rs index 94a0eea9728ff..d5d5bc2fffa5b 100644 --- a/utils/frame/benchmarking-cli/src/storage/write.rs +++ b/utils/frame/benchmarking-cli/src/storage/write.rs @@ -31,7 +31,8 @@ use log::{info, trace}; use rand::prelude::*; use std::{fmt::Debug, sync::Arc, time::Instant}; -use super::{cmd::StorageCmd, record::BenchRecord}; +use super::cmd::StorageCmd; +use crate::shared::BenchRecord; impl StorageCmd { /// Benchmarks the time it takes to write a single Storage item.