diff --git a/Cargo.lock b/Cargo.lock index 58ea4e1077c98..a77c7b2a40e6d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1450,6 +1450,7 @@ dependencies = [ name = "frame-benchmarking-cli" version = "2.0.0-rc4" dependencies = [ + "Inflector", "frame-benchmarking", "parity-scale-codec", "sc-cli", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 70d001d62c135..baf5f12b2f54e 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1118,7 +1118,6 @@ impl_runtime_apis! { let whitelist: Vec> = vec![ // Block Number - // frame_system::Number::::hashed_key().to_vec(), hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef702a5c1b19ab7a04f536c519aca4983ac").to_vec(), // Total Issuance hex_literal::hex!("c2261276cc9d1f8598ea4b6a74b15c2f57c875e4cff74148e4628f264b974c80").to_vec(), @@ -1137,25 +1136,25 @@ impl_runtime_apis! { let mut batches = Vec::::new(); let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat, &whitelist); - add_benchmark!(params, batches, b"babe", Babe); - add_benchmark!(params, batches, b"balances", Balances); - add_benchmark!(params, batches, b"collective", Council); - add_benchmark!(params, batches, b"democracy", Democracy); - add_benchmark!(params, batches, b"elections", Elections); - add_benchmark!(params, batches, b"identity", Identity); - add_benchmark!(params, batches, b"im-online", ImOnline); - add_benchmark!(params, batches, b"indices", Indices); - add_benchmark!(params, batches, b"multisig", Multisig); - add_benchmark!(params, batches, b"offences", OffencesBench::); - add_benchmark!(params, batches, b"proxy", Proxy); - add_benchmark!(params, batches, b"scheduler", Scheduler); - add_benchmark!(params, batches, b"session", SessionBench::); - add_benchmark!(params, batches, b"staking", Staking); - add_benchmark!(params, batches, b"system", SystemBench::); - add_benchmark!(params, batches, b"timestamp", Timestamp); - add_benchmark!(params, batches, b"treasury", Treasury); - add_benchmark!(params, batches, b"utility", Utility); - add_benchmark!(params, batches, b"vesting", Vesting); + add_benchmark!(params, batches, pallet_babe, Babe); + add_benchmark!(params, batches, pallet_balances, Balances); + add_benchmark!(params, batches, pallet_collective, Council); + add_benchmark!(params, batches, pallet_democracy, Democracy); + add_benchmark!(params, batches, pallet_elections_phragmen, Elections); + add_benchmark!(params, batches, pallet_identity, Identity); + add_benchmark!(params, batches, pallet_im_online, ImOnline); + add_benchmark!(params, batches, pallet_indices, Indices); + add_benchmark!(params, batches, pallet_multisig, Multisig); + add_benchmark!(params, batches, pallet_offences, OffencesBench::); + add_benchmark!(params, batches, pallet_proxy, Proxy); + add_benchmark!(params, batches, pallet_scheduler, Scheduler); + add_benchmark!(params, batches, pallet_session, SessionBench::); + add_benchmark!(params, batches, pallet_staking, Staking); + add_benchmark!(params, batches, frame_system, SystemBench::); + add_benchmark!(params, batches, pallet_timestamp, Timestamp); + add_benchmark!(params, batches, pallet_treasury, Treasury); + add_benchmark!(params, batches, pallet_utility, Utility); + add_benchmark!(params, batches, pallet_vesting, Vesting); if batches.is_empty() { return Err("Benchmark not found for this pallet.".into()) } Ok(batches) diff --git a/frame/benchmarking/src/analysis.rs b/frame/benchmarking/src/analysis.rs index 621f3a2941ff5..c17e206c34c6c 100644 --- a/frame/benchmarking/src/analysis.rs +++ b/frame/benchmarking/src/analysis.rs @@ -22,11 +22,11 @@ use linregress::{FormulaRegressionBuilder, RegressionDataBuilder, RegressionMode use crate::BenchmarkResults; pub struct Analysis { - base: u128, - slopes: Vec, - names: Vec, - value_dists: Option, u128, u128)>>, - model: Option, + pub base: u128, + pub slopes: Vec, + pub names: Vec, + pub value_dists: Option, u128, u128)>>, + pub model: Option, } pub enum BenchmarkSelector { diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index 7a7848305a01c..532cb273c989d 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -1158,31 +1158,46 @@ macro_rules! impl_benchmark_test { /// First create an object that holds in the input parameters for the benchmark: /// /// ```ignore -/// let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat); +/// let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat, &whitelist); /// ``` /// +/// The `whitelist` is a `Vec>` of storage keys that you would like to skip for DB tracking. For example: +/// +/// ```ignore +/// let whitelist: Vec> = vec![ +/// // Block Number +/// hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef702a5c1b19ab7a04f536c519aca4983ac").to_vec(), +/// // Total Issuance +/// hex_literal::hex!("c2261276cc9d1f8598ea4b6a74b15c2f57c875e4cff74148e4628f264b974c80").to_vec(), +/// // Execution Phase +/// hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef7ff553b5a9862a516939d82b3d3d8661a").to_vec(), +/// // Event Count +/// hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef70a98fdbe9ce6c55837576c60c7af3850").to_vec(), +/// ]; +/// /// Then define a mutable local variable to hold your `BenchmarkBatch` object: /// /// ```ignore /// let mut batches = Vec::::new(); /// ```` /// -/// Then add the pallets you want to benchmark to this object, including the string -/// you want to use target a particular pallet: +/// Then add the pallets you want to benchmark to this object, using their crate name and generated +/// module struct: /// /// ```ignore -/// add_benchmark!(params, batches, b"balances", Balances); -/// add_benchmark!(params, batches, b"identity", Identity); -/// add_benchmark!(params, batches, b"session", SessionBench::); +/// add_benchmark!(params, batches, pallet_balances, Balances); +/// add_benchmark!(params, batches, pallet_session, SessionBench::); +/// add_benchmark!(params, batches, frame_system, SystemBench::); /// ... /// ``` /// /// At the end of `dispatch_benchmark`, you should return this batches object. #[macro_export] macro_rules! add_benchmark { - ( $params:ident, $batches:ident, $name:literal, $( $location:tt )* ) => ( + ( $params:ident, $batches:ident, $name:ident, $( $location:tt )* ) => ( + let name_string = stringify!($name).as_bytes(); let (pallet, benchmark, lowest_range_values, highest_range_values, steps, repeat, whitelist) = $params; - if &pallet[..] == &$name[..] || &pallet[..] == &b"*"[..] { + if &pallet[..] == &name_string[..] || &pallet[..] == &b"*"[..] { if &pallet[..] == &b"*"[..] || &benchmark[..] == &b"*"[..] { for benchmark in $( $location )*::benchmarks().into_iter() { $batches.push($crate::BenchmarkBatch { @@ -1194,7 +1209,7 @@ macro_rules! add_benchmark { repeat, whitelist, )?, - pallet: $name.to_vec(), + pallet: name_string.to_vec(), benchmark: benchmark.to_vec(), }); } @@ -1208,7 +1223,7 @@ macro_rules! add_benchmark { repeat, whitelist, )?, - pallet: $name.to_vec(), + pallet: name_string.to_vec(), benchmark: benchmark.clone(), }); } diff --git a/utils/frame/benchmarking-cli/Cargo.toml b/utils/frame/benchmarking-cli/Cargo.toml index 003b4d9c05b3e..db620c86ca98e 100644 --- a/utils/frame/benchmarking-cli/Cargo.toml +++ b/utils/frame/benchmarking-cli/Cargo.toml @@ -12,6 +12,7 @@ description = "CLI for benchmarking FRAME" targets = ["x86_64-unknown-linux-gnu"] [dependencies] +Inflector = "0.11.4" frame-benchmarking = { version = "2.0.0-rc4", path = "../../../frame/benchmarking" } sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } sc-service = { version = "0.8.0-rc4", default-features = false, path = "../../../client/service" } diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs index 7f55672885d46..09b246e4766a3 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -55,7 +55,7 @@ impl BenchmarkCmd { let state = BenchmarkingState::::new(genesis_storage, cache_size)?; let executor = NativeExecutor::::new( wasm_method, - None, // heap pages + self.heap_pages, 2, // The runtime instances cache size. ); @@ -89,6 +89,16 @@ impl BenchmarkCmd { let results = , String> as Decode>::decode(&mut &result[..]) .map_err(|e| format!("Failed to decode benchmark results: {:?}", e))?; + if self.output { + if self.weight_trait { + let mut file = crate::writer::open_file("traits.rs")?; + crate::writer::write_trait(&mut file, results.clone())?; + } else { + let mut file = crate::writer::open_file("benchmarks.rs")?; + crate::writer::write_results(&mut file, results.clone())?; + } + } + match results { Ok(batches) => for batch in batches.into_iter() { // Print benchmark metadata diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index 149b971577fab..8a53c9fd8b16b 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -16,6 +16,7 @@ // limitations under the License. mod command; +mod writer; use sc_cli::{ExecutionStrategy, WasmExecutionMethod}; use std::fmt::Debug; @@ -59,6 +60,18 @@ pub struct BenchmarkCmd { #[structopt(long)] pub no_min_squares: bool, + /// Output the benchmarks to a Rust file. + #[structopt(long)] + pub output: bool, + + /// Output the trait definition to a Rust file. + #[structopt(long)] + pub weight_trait: bool, + + /// Set the heap pages while running benchmarks. + #[structopt(long)] + pub heap_pages: Option, + #[allow(missing_docs)] #[structopt(flatten)] pub shared_params: sc_cli::SharedParams, diff --git a/utils/frame/benchmarking-cli/src/writer.rs b/utils/frame/benchmarking-cli/src/writer.rs new file mode 100644 index 0000000000000..bd411b536a826 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/writer.rs @@ -0,0 +1,191 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Outputs benchmark results to Rust files that can be ingested by the runtime. + +use std::fs::{File, OpenOptions}; +use std::io::prelude::*; +use frame_benchmarking::{BenchmarkBatch, BenchmarkSelector, Analysis}; +use inflector::Inflector; + +pub fn open_file(path: &str) -> Result { + OpenOptions::new() + .create(true) + .write(true) + .append(true) + .open(path) +} + +pub fn write_trait(file: &mut File, batches: Result, String>) -> Result<(), std::io::Error> { + let batches = batches.unwrap(); + + let mut current_pallet = Vec::::new(); + + batches.iter().for_each(|batch| { + + let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap(); + let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap(); + + // only create new trait definitions when we go to a new pallet + if batch.pallet != current_pallet { + if !current_pallet.is_empty() { + // close trait + write!(file, "}}\n").unwrap(); + } + + // trait wrapper + write!(file, "// {}\n", pallet_string).unwrap(); + write!(file, "pub trait WeightInfo {{\n").unwrap(); + + current_pallet = batch.pallet.clone() + } + + // function name + write!(file, " fn {}(", benchmark_string).unwrap(); + + // params + let components = &batch.results[0].components; + for component in components { + write!(file, "{:?}: u32, ", component.0).unwrap(); + } + // return value + write!(file, ") -> Weight;\n").unwrap(); + }); + + // final close trait + write!(file, "}}\n").unwrap(); + + // Reset + current_pallet = Vec::::new(); + + batches.iter().for_each(|batch| { + + let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap(); + + // only create new trait definitions when we go to a new pallet + if batch.pallet != current_pallet { + if !current_pallet.is_empty() { + // close trait + write!(file, "}}\n").unwrap(); + } + + // impl trait + write!(file, "\n").unwrap(); + write!(file, "impl WeightInfo for () {{\n").unwrap(); + + current_pallet = batch.pallet.clone() + } + + // function name + write!(file, " fn {}(", benchmark_string).unwrap(); + + // params + let components = &batch.results[0].components; + for component in components { + write!(file, "_{:?}: u32, ", component.0).unwrap(); + } + // return value + write!(file, ") -> Weight {{ 1_000_000_000 }}\n").unwrap(); + }); + + // final close trait + write!(file, "}}\n").unwrap(); + + Ok(()) +} + +pub fn write_results(file: &mut File, batches: Result, String>) -> Result<(), std::io::Error> { + let batches = batches.unwrap(); + + let mut current_pallet = Vec::::new(); + + // general imports + write!(file, "use frame_support::weights::{{Weight, constants::RocksDbWeight as DbWeight}};\n").unwrap(); + + batches.iter().for_each(|batch| { + + let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap(); + let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap(); + + // only create new trait definitions when we go to a new pallet + if batch.pallet != current_pallet { + if !current_pallet.is_empty() { + // close trait + write!(file, "}}\n").unwrap(); + } + + // struct for weights + write!(file, "pub struct WeightFor{};\n", + pallet_string.to_pascal_case(), + ).unwrap(); + + // trait wrapper + write!(file, "impl {}::WeightInfo for WeightFor{} {{\n", + pallet_string, + pallet_string.to_pascal_case(), + ).unwrap(); + + current_pallet = batch.pallet.clone() + } + + // function name + write!(file, " fn {}(", benchmark_string).unwrap(); + + // params + let components = &batch.results[0].components; + for component in components { + write!(file, "{:?}: u32, ", component.0).unwrap(); + } + // return value + write!(file, ") -> Weight {{\n").unwrap(); + + let extrinsic_time = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime).unwrap(); + write!(file, " ({} as Weight)\n", extrinsic_time.base.saturating_mul(1000)).unwrap(); + extrinsic_time.slopes.iter().zip(extrinsic_time.names.iter()).for_each(|(slope, name)| { + write!(file, " .saturating_add(({} as Weight).saturating_mul({} as Weight))\n", + slope.saturating_mul(1000), + name, + ).unwrap(); + }); + + let reads = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads).unwrap(); + write!(file, " .saturating_add(DbWeight::get().reads({} as Weight))\n", reads.base).unwrap(); + reads.slopes.iter().zip(reads.names.iter()).for_each(|(slope, name)| { + write!(file, " .saturating_add(DbWeight::get().reads(({} as Weight).saturating_mul({} as Weight)))\n", + slope, + name, + ).unwrap(); + }); + + let writes = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes).unwrap(); + write!(file, " .saturating_add(DbWeight::get().writes({} as Weight))\n", writes.base).unwrap(); + writes.slopes.iter().zip(writes.names.iter()).for_each(|(slope, name)| { + write!(file, " .saturating_add(DbWeight::get().writes(({} as Weight).saturating_mul({} as Weight)))\n", + slope, + name, + ).unwrap(); + }); + + // close function + write!(file, " }}\n").unwrap(); + }); + + // final close trait + write!(file, "}}\n").unwrap(); + + Ok(()) +}