diff --git a/.gitlab/pipeline/check.yml b/.gitlab/pipeline/check.yml index 4c39539f1e50..89b2c00db9b2 100644 --- a/.gitlab/pipeline/check.yml +++ b/.gitlab/pipeline/check.yml @@ -7,8 +7,8 @@ cargo-clippy: variables: RUSTFLAGS: "-D warnings" script: - - SKIP_WASM_BUILD=1 cargo clippy --all-targets --locked --workspace - - SKIP_WASM_BUILD=1 cargo clippy --all-targets --all-features --locked --workspace + - SKIP_WASM_BUILD=1 cargo clippy --all-targets --locked --workspace --quiet + - SKIP_WASM_BUILD=1 cargo clippy --all-targets --all-features --locked --workspace --quiet check-try-runtime: stage: check diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index 9db89b923009..af16f5d2de7f 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -322,7 +322,24 @@ quick-benchmarks: WASM_BUILD_NO_COLOR: 1 WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings" script: - - time cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks -- benchmark pallet --execution wasm --wasm-execution compiled --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 + - time cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks --quiet -- benchmark pallet --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 --quiet + +quick-benchmarks-omni: + stage: test + extends: + - .docker-env + - .common-refs + - .run-immediately + variables: + # Enable debug assertions since we are running optimized builds for testing + # but still want to have debug assertions. + RUSTFLAGS: "-C debug-assertions" + RUST_BACKTRACE: "full" + WASM_BUILD_NO_COLOR: 1 + WASM_BUILD_RUSTFLAGS: "-C debug-assertions" + script: + - time cargo build --locked --quiet --release -p asset-hub-westend-runtime --features runtime-benchmarks + - time cargo run --locked --release -p frame-omni-bencher --quiet -- v1 benchmark pallet --runtime target/release/wbuild/asset-hub-westend-runtime/asset_hub_westend_runtime.compact.compressed.wasm --all --steps 2 --repeat 1 --quiet test-frame-examples-compile-to-wasm: # into one job diff --git a/Cargo.lock b/Cargo.lock index eb624d0ffa2d..db3ce694aa29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5503,6 +5503,7 @@ dependencies = [ "rand", "rand_pcg", "sc-block-builder", + "sc-chain-spec", "sc-cli", "sc-client-api", "sc-client-db", @@ -5516,6 +5517,7 @@ dependencies = [ "sp-core", "sp-database", "sp-externalities 0.25.0", + "sp-genesis-builder", "sp-inherents", "sp-io", "sp-keystore", @@ -5628,6 +5630,20 @@ dependencies = [ "serde", ] +[[package]] +name = "frame-omni-bencher" +version = "0.1.0" +dependencies = [ + "clap 4.5.3", + "cumulus-primitives-proof-size-hostfunction", + "env_logger 0.11.3", + "frame-benchmarking-cli", + "log", + "sc-cli", + "sp-runtime", + "sp-statement-store", +] + [[package]] name = "frame-remote-externalities" version = "0.35.0" diff --git a/Cargo.toml b/Cargo.toml index f81bc8d10e55..8e6fdb4b8bbc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -494,6 +494,7 @@ members = [ "substrate/utils/frame/frame-utilities-cli", "substrate/utils/frame/generate-bags", "substrate/utils/frame/generate-bags/node-runtime", + "substrate/utils/frame/omni-bencher", "substrate/utils/frame/remote-externalities", "substrate/utils/frame/rpc/client", "substrate/utils/frame/rpc/state-trie-migration-rpc", diff --git a/cumulus/polkadot-parachain/src/command.rs b/cumulus/polkadot-parachain/src/command.rs index 514968f7fbb7..c0bd7acfebf8 100644 --- a/cumulus/polkadot-parachain/src/command.rs +++ b/cumulus/polkadot-parachain/src/command.rs @@ -568,7 +568,7 @@ pub fn run() -> Result<()> { match cmd { BenchmarkCmd::Pallet(cmd) => if cfg!(feature = "runtime-benchmarks") { - runner.sync_run(|config| cmd.run::, ReclaimHostFunctions>(config)) + runner.sync_run(|config| cmd.run_with_spec::, ReclaimHostFunctions>(Some(config.chain_spec))) } else { Err("Benchmarking wasn't enabled when building the node. \ You can enable it with `--features runtime-benchmarks`." diff --git a/polkadot/cli/src/command.rs b/polkadot/cli/src/command.rs index f71891ecde34..b163a2c1367d 100644 --- a/polkadot/cli/src/command.rs +++ b/polkadot/cli/src/command.rs @@ -451,8 +451,10 @@ pub fn run() -> Result<()> { if cfg!(feature = "runtime-benchmarks") { runner.sync_run(|config| { - cmd.run::, ()>(config) - .map_err(|e| Error::SubstrateCli(e)) + cmd.run_with_spec::, ()>( + Some(config.chain_spec), + ) + .map_err(|e| Error::SubstrateCli(e)) }) } else { Err(sc_cli::Error::Input( diff --git a/polkadot/runtime/common/src/auctions.rs b/polkadot/runtime/common/src/auctions.rs index 6914fef99d52..00c2da3016d7 100644 --- a/polkadot/runtime/common/src/auctions.rs +++ b/polkadot/runtime/common/src/auctions.rs @@ -1899,10 +1899,13 @@ mod benchmarking { // Trigger epoch change for new random number value: { + pallet_babe::EpochStart::::set((Zero::zero(), u32::MAX.into())); pallet_babe::Pallet::::on_initialize(duration + now + T::EndingPeriod::get()); let authorities = pallet_babe::Pallet::::authorities(); - let next_authorities = authorities.clone(); - pallet_babe::Pallet::::enact_epoch_change(authorities, next_authorities, None); + // Check for non empty authority set since it otherwise emits a No-OP warning. + if !authorities.is_empty() { + pallet_babe::Pallet::::enact_epoch_change(authorities.clone(), authorities, None); + } } }: { diff --git a/polkadot/runtime/parachains/src/configuration.rs b/polkadot/runtime/parachains/src/configuration.rs index e1246fb88975..33061b789dc4 100644 --- a/polkadot/runtime/parachains/src/configuration.rs +++ b/polkadot/runtime/parachains/src/configuration.rs @@ -231,7 +231,7 @@ pub struct HostConfiguration { impl> Default for HostConfiguration { fn default() -> Self { - Self { + let ret = Self { async_backing_params: AsyncBackingParams { max_candidate_depth: 0, allowed_ancestry_len: 0, @@ -270,7 +270,30 @@ impl> Default for HostConfiguration> HostConfiguration { + /// Mutate the values of self to be good estimates for benchmarking. + /// + /// The values do not need to be worst-case, since the benchmarking logic extrapolates. They + /// should be a bit more than usually expected. + fn with_benchmarking_default(mut self) -> Self { + self.max_head_data_size = self.max_head_data_size.max(1 << 20); + self.max_downward_message_size = self.max_downward_message_size.max(1 << 16); + self.hrmp_channel_max_capacity = self.hrmp_channel_max_capacity.max(1000); + self.hrmp_channel_max_message_size = self.hrmp_channel_max_message_size.max(1 << 16); + self.hrmp_max_parachain_inbound_channels = + self.hrmp_max_parachain_inbound_channels.max(100); + self.hrmp_max_parachain_outbound_channels = + self.hrmp_max_parachain_outbound_channels.max(100); + self } } diff --git a/polkadot/runtime/parachains/src/hrmp/benchmarking.rs b/polkadot/runtime/parachains/src/hrmp/benchmarking.rs index c6baf2f30cf5..0251811d4960 100644 --- a/polkadot/runtime/parachains/src/hrmp/benchmarking.rs +++ b/polkadot/runtime/parachains/src/hrmp/benchmarking.rs @@ -76,6 +76,8 @@ where let deposit: BalanceOf = config.hrmp_sender_deposit.unique_saturated_into(); let capacity = config.hrmp_channel_max_capacity; let message_size = config.hrmp_channel_max_message_size; + assert!(message_size > 0, "Invalid genesis for benchmarking"); + assert!(capacity > 0, "Invalid genesis for benchmarking"); let sender: ParaId = from.into(); let sender_origin: crate::Origin = from.into(); diff --git a/polkadot/runtime/parachains/src/paras/benchmarking.rs b/polkadot/runtime/parachains/src/paras/benchmarking.rs index 56e6ff153c15..77f6a9edddc8 100644 --- a/polkadot/runtime/parachains/src/paras/benchmarking.rs +++ b/polkadot/runtime/parachains/src/paras/benchmarking.rs @@ -82,7 +82,7 @@ fn generate_disordered_actions_queue() { benchmarks! { force_set_current_code { - let c in 1 .. MAX_CODE_SIZE; + let c in MIN_CODE_SIZE .. MAX_CODE_SIZE; let new_code = ValidationCode(vec![0; c as usize]); let para_id = ParaId::from(c as u32); CurrentCodeHash::::insert(¶_id, new_code.hash()); @@ -92,7 +92,7 @@ benchmarks! { assert_last_event::(Event::CurrentCodeUpdated(para_id).into()); } force_set_current_head { - let s in 1 .. MAX_HEAD_DATA_SIZE; + let s in MIN_CODE_SIZE .. MAX_HEAD_DATA_SIZE; let new_head = HeadData(vec![0; s as usize]); let para_id = ParaId::from(1000); }: _(RawOrigin::Root, para_id, new_head) @@ -104,7 +104,7 @@ benchmarks! { let context = BlockNumberFor::::from(1000u32); }: _(RawOrigin::Root, para_id, context) force_schedule_code_upgrade { - let c in 1 .. MAX_CODE_SIZE; + let c in MIN_CODE_SIZE .. MAX_CODE_SIZE; let new_code = ValidationCode(vec![0; c as usize]); let para_id = ParaId::from(c as u32); let block = BlockNumberFor::::from(c); @@ -114,7 +114,7 @@ benchmarks! { assert_last_event::(Event::CodeUpgradeScheduled(para_id).into()); } force_note_new_head { - let s in 1 .. MAX_HEAD_DATA_SIZE; + let s in MIN_CODE_SIZE .. MAX_HEAD_DATA_SIZE; let para_id = ParaId::from(1000); let new_head = HeadData(vec![0; s as usize]); let old_code_hash = ValidationCode(vec![0]).hash(); @@ -126,7 +126,7 @@ benchmarks! { generate_disordered_pruning::(); Pallet::::schedule_code_upgrade( para_id, - ValidationCode(vec![0]), + ValidationCode(vec![0u8; MIN_CODE_SIZE as usize]), expired, &config, UpgradeStrategy::SetGoAheadSignal, @@ -145,7 +145,7 @@ benchmarks! { } add_trusted_validation_code { - let c in 1 .. MAX_CODE_SIZE; + let c in MIN_CODE_SIZE .. MAX_CODE_SIZE; let new_code = ValidationCode(vec![0; c as usize]); pvf_check::prepare_bypassing_bench::(new_code.clone()); diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index 0187a94796e8..d33c58007843 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -1154,6 +1154,8 @@ impl pallet_nis::Config for Runtime { type MaxIntakeWeight = MaxIntakeWeight; type ThawThrottle = ThawThrottle; type RuntimeHoldReason = RuntimeHoldReason; + #[cfg(feature = "runtime-benchmarks")] + type BenchmarkSetup = (); } parameter_types! { diff --git a/prdoc/pr_3512.prdoc b/prdoc/pr_3512.prdoc new file mode 100644 index 000000000000..d44e1ca1e296 --- /dev/null +++ b/prdoc/pr_3512.prdoc @@ -0,0 +1,47 @@ +title: "[FRAME] Introduce Runtime Omni Bencher" + +doc: + - audience: Node Dev + description: | + Introduces a new freestanding binary; the `frame-omni-bencher`. This can be used as alternative to the node-integrated `benchmark pallet` command. It currently only supports pallet benchmarking. + + The optional change to integrate this MR is in the node. The `run` function is now deprecated in favour or `run_with_spec`. This should be rather easy to integrate: + + ```patch + runner.sync_run(|config| cmd + - .run::, ReclaimHostFunctions>(config) + + .run_with_spec::, ReclaimHostFunctions>(Some(config.chain_spec)) + ) + ``` + + Additionally, a new `--runtime` CLI arg was introduced to the `benchmark pallet` command. It allows to generate the genesis state directly by the runtime, instead of using a spec file. + +crates: + - name: pallet-nis + bump: major + - name: frame-benchmarking-cli + bump: major + - name: polkadot-parachain-bin + bump: patch + - name: polkadot-cli + bump: patch + - name: polkadot-runtime-common + bump: patch + - name: polkadot-runtime-parachains + bump: patch + - name: rococo-runtime + bump: patch + - name: staging-node-cli + bump: patch + - name: kitchensink-runtime + bump: patch + - name: pallet-babe + bump: patch + - name: frame-benchmarking + bump: patch + - name: pallet-election-provider-multi-phase + bump: patch + - name: pallet-fast-unstake + bump: patch + - name: pallet-contracts + bump: patch diff --git a/substrate/bin/node/cli/src/command.rs b/substrate/bin/node/cli/src/command.rs index 964517320286..d37325c7187e 100644 --- a/substrate/bin/node/cli/src/command.rs +++ b/substrate/bin/node/cli/src/command.rs @@ -107,7 +107,7 @@ pub fn run() -> Result<()> { ) } - cmd.run::, sp_statement_store::runtime_api::HostFunctions>(config) + cmd.run_with_spec::, sp_statement_store::runtime_api::HostFunctions>(Some(config.chain_spec)) }, BenchmarkCmd::Block(cmd) => { // ensure that we keep the task manager alive diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs index 76799e95a8f2..e3d93b953fc7 100644 --- a/substrate/bin/node/runtime/src/lib.rs +++ b/substrate/bin/node/runtime/src/lib.rs @@ -1759,6 +1759,25 @@ impl pallet_nis::Config for Runtime { type MaxIntakeWeight = MaxIntakeWeight; type ThawThrottle = ThawThrottle; type RuntimeHoldReason = RuntimeHoldReason; + #[cfg(feature = "runtime-benchmarks")] + type BenchmarkSetup = SetupAsset; +} + +#[cfg(feature = "runtime-benchmarks")] +pub struct SetupAsset; +#[cfg(feature = "runtime-benchmarks")] +impl pallet_nis::BenchmarkSetup for SetupAsset { + fn create_counterpart_asset() { + let owner = AccountId::from([0u8; 32]); + // this may or may not fail depending on if the chain spec or runtime genesis is used. + let _ = Assets::force_create( + RuntimeOrigin::root(), + 9u32.into(), + sp_runtime::MultiAddress::Id(owner), + true, + 1, + ); + } } parameter_types! { diff --git a/substrate/frame/babe/src/lib.rs b/substrate/frame/babe/src/lib.rs index 5fb107dde3ba..686ba6ec2d63 100644 --- a/substrate/frame/babe/src/lib.rs +++ b/substrate/frame/babe/src/lib.rs @@ -283,7 +283,7 @@ pub mod pallet { /// entropy was fixed (i.e. it was known to chain observers). Since epochs are defined in /// slots, which may be skipped, the block numbers may not line up with the slot numbers. #[pallet::storage] - pub(super) type EpochStart = + pub type EpochStart = StorageValue<_, (BlockNumberFor, BlockNumberFor), ValueQuery>; /// How late the current block is compared to its parent. diff --git a/substrate/frame/benchmarking/src/v1.rs b/substrate/frame/benchmarking/src/v1.rs index b2449db3d67d..ecebbfa91052 100644 --- a/substrate/frame/benchmarking/src/v1.rs +++ b/substrate/frame/benchmarking/src/v1.rs @@ -850,8 +850,8 @@ macro_rules! impl_bench_name_tests { if !($extra) { let disabled = $crate::__private::vec![ $( stringify!($names_extra).as_ref() ),* ]; if disabled.contains(&stringify!($name)) { - $crate::__private::log::error!( - "INFO: extra benchmark skipped - {}", + $crate::__private::log::debug!( + "extra benchmark skipped - {}", stringify!($name), ); return (); @@ -874,21 +874,21 @@ macro_rules! impl_bench_name_tests { $crate::BenchmarkError::Override(_) => { // This is still considered a success condition. $crate::__private::log::error!( - "WARNING: benchmark error overridden - {}", + "benchmark error overridden - {}", stringify!($name), ); }, $crate::BenchmarkError::Skip => { // This is considered a success condition. - $crate::__private::log::error!( - "WARNING: benchmark error skipped - {}", + $crate::__private::log::debug!( + "benchmark skipped - {}", stringify!($name), ); }, $crate::BenchmarkError::Weightless => { // This is considered a success condition. - $crate::__private::log::error!( - "WARNING: benchmark weightless skipped - {}", + $crate::__private::log::debug!( + "benchmark weightless skipped - {}", stringify!($name), ); } diff --git a/substrate/frame/contracts/src/benchmarking/mod.rs b/substrate/frame/contracts/src/benchmarking/mod.rs index 9fb107537ba0..523d93de2d47 100644 --- a/substrate/frame/contracts/src/benchmarking/mod.rs +++ b/substrate/frame/contracts/src/benchmarking/mod.rs @@ -332,7 +332,7 @@ mod benchmarks { #[benchmark(pov_mode = Measured)] fn migration_noop() { let version = LATEST_MIGRATION_VERSION; - assert_eq!(StorageVersion::get::>(), version); + StorageVersion::new(version).put::>(); #[block] { Migration::::migrate(Weight::MAX); @@ -358,7 +358,7 @@ mod benchmarks { #[benchmark(pov_mode = Measured)] fn on_runtime_upgrade_noop() { let latest_version = LATEST_MIGRATION_VERSION; - assert_eq!(StorageVersion::get::>(), latest_version); + StorageVersion::new(latest_version).put::>(); #[block] { as frame_support::traits::OnRuntimeUpgrade>::on_runtime_upgrade(); diff --git a/substrate/frame/election-provider-multi-phase/src/lib.rs b/substrate/frame/election-provider-multi-phase/src/lib.rs index 11577cd35262..63b4c49cdfe4 100644 --- a/substrate/frame/election-provider-multi-phase/src/lib.rs +++ b/substrate/frame/election-provider-multi-phase/src/lib.rs @@ -931,7 +931,7 @@ pub mod pallet { .expect(error_message); // Store the newly received solution. - log!(info, "queued unsigned solution with score {:?}", ready.score); + log!(debug, "queued unsigned solution with score {:?}", ready.score); let ejected_a_solution = >::exists(); >::put(ready); Self::deposit_event(Event::SolutionStored { diff --git a/substrate/frame/fast-unstake/src/lib.rs b/substrate/frame/fast-unstake/src/lib.rs index 04a50543bcc9..8ba306201310 100644 --- a/substrate/frame/fast-unstake/src/lib.rs +++ b/substrate/frame/fast-unstake/src/lib.rs @@ -560,7 +560,7 @@ pub mod pallet { if !remaining.is_zero() { Self::halt("not enough balance to unreserve"); } else { - log!(info, "unstaked {:?}, outcome: {:?}", stash, result); + log!(debug, "unstaked {:?}, outcome: {:?}", stash, result); Self::deposit_event(Event::::Unstaked { stash, result }); } }; diff --git a/substrate/frame/nis/src/benchmarking.rs b/substrate/frame/nis/src/benchmarking.rs index 0cc9e7421d0e..f6a83b78d518 100644 --- a/substrate/frame/nis/src/benchmarking.rs +++ b/substrate/frame/nis/src/benchmarking.rs @@ -106,6 +106,7 @@ benchmarks! { } fund_deficit { + T::BenchmarkSetup::create_counterpart_asset(); let origin = T::FundOrigin::try_successful_origin().map_err(|_| BenchmarkError::Weightless)?; let caller: T::AccountId = whitelisted_caller(); @@ -126,6 +127,7 @@ benchmarks! { } communify { + T::BenchmarkSetup::create_counterpart_asset(); let caller: T::AccountId = whitelisted_caller(); let bid = T::MinBid::get().max(One::one()) * 100u32.into(); let ed = T::Currency::minimum_balance(); @@ -139,6 +141,7 @@ benchmarks! { } privatize { + T::BenchmarkSetup::create_counterpart_asset(); let caller: T::AccountId = whitelisted_caller(); let bid = T::MinBid::get().max(One::one()); let ed = T::Currency::minimum_balance(); @@ -153,6 +156,7 @@ benchmarks! { } thaw_private { + T::BenchmarkSetup::create_counterpart_asset(); let whale: T::AccountId = account("whale", 0, SEED); let caller: T::AccountId = whitelisted_caller(); let bid = T::MinBid::get().max(One::one()); @@ -170,6 +174,7 @@ benchmarks! { } thaw_communal { + T::BenchmarkSetup::create_counterpart_asset(); let whale: T::AccountId = account("whale", 0, SEED); let caller: T::AccountId = whitelisted_caller(); let bid = T::MinBid::get().max(One::one()); diff --git a/substrate/frame/nis/src/lib.rs b/substrate/frame/nis/src/lib.rs index 7655cd1a8243..63287f6a1802 100644 --- a/substrate/frame/nis/src/lib.rs +++ b/substrate/frame/nis/src/lib.rs @@ -155,6 +155,18 @@ impl Convert for NoCounterpart { } } +/// Setup the empty genesis state for benchmarking. +pub trait BenchmarkSetup { + /// Create the counterpart asset. Should panic on error. + /// + /// This is called prior to assuming that a counterpart balance exists. + fn create_counterpart_asset(); +} + +impl BenchmarkSetup for () { + fn create_counterpart_asset() {} +} + #[frame_support::pallet] pub mod pallet { use super::{FunInspect, FunMutate}; @@ -297,6 +309,10 @@ pub mod pallet { /// The maximum proportion which may be thawed and the period over which it is reset. #[pallet::constant] type ThawThrottle: Get<(Perquintill, BlockNumberFor)>; + + /// Setup the state for benchmarking. + #[cfg(feature = "runtime-benchmarks")] + type BenchmarkSetup: crate::BenchmarkSetup; } #[pallet::pallet] diff --git a/substrate/frame/nis/src/mock.rs b/substrate/frame/nis/src/mock.rs index 33464db34c30..f3320a306df7 100644 --- a/substrate/frame/nis/src/mock.rs +++ b/substrate/frame/nis/src/mock.rs @@ -121,6 +121,8 @@ impl pallet_nis::Config for Test { type MinReceipt = MinReceipt; type ThawThrottle = ThawThrottle; type RuntimeHoldReason = RuntimeHoldReason; + #[cfg(feature = "runtime-benchmarks")] + type BenchmarkSetup = (); } // This function basically just builds a genesis storage key/value store according to diff --git a/substrate/utils/frame/benchmarking-cli/Cargo.toml b/substrate/utils/frame/benchmarking-cli/Cargo.toml index b7fbb24b1a09..92169484c928 100644 --- a/substrate/utils/frame/benchmarking-cli/Cargo.toml +++ b/substrate/utils/frame/benchmarking-cli/Cargo.toml @@ -37,6 +37,7 @@ frame-benchmarking = { path = "../../../frame/benchmarking" } frame-support = { path = "../../../frame/support" } frame-system = { path = "../../../frame/system" } sc-block-builder = { path = "../../../client/block-builder" } +sc-chain-spec = { path = "../../../client/chain-spec", default-features = false } sc-cli = { path = "../../../client/cli", default-features = false } sc-client-api = { path = "../../../client/api" } sc-client-db = { path = "../../../client/db", default-features = false } @@ -48,6 +49,7 @@ sp-blockchain = { path = "../../../primitives/blockchain" } sp-core = { path = "../../../primitives/core" } sp-database = { path = "../../../primitives/database" } sp-externalities = { path = "../../../primitives/externalities" } +sp-genesis-builder = { path = "../../../primitives/genesis-builder" } sp-inherents = { path = "../../../primitives/inherents" } sp-keystore = { path = "../../../primitives/keystore" } sp-runtime = { path = "../../../primitives/runtime" } diff --git a/substrate/utils/frame/benchmarking-cli/README.md b/substrate/utils/frame/benchmarking-cli/README.md index 27673ea9580d..5deb5098b5bf 100644 --- a/substrate/utils/frame/benchmarking-cli/README.md +++ b/substrate/utils/frame/benchmarking-cli/README.md @@ -1,13 +1,23 @@ -# The Benchmarking CLI +# The FRAME Benchmarking CLI -This crate contains commands to benchmark various aspects of Substrate and the hardware. -All commands are exposed by the Substrate node but can be exposed by any Substrate client. +This crate contains commands to benchmark various aspects of Substrate and the hardware. The goal is to have a comprehensive suite of benchmarks that cover all aspects of Substrate and the hardware that its -running on. +running on. +There exist fundamentally two ways to use this crate. A node-integrated CLI version, and a freestanding CLI. If you are +only interested in pallet benchmarking, then skip ahead to the [Freestanding CLI](#freestanding-cli). + +# Node Integrated CLI + +Mostly all Substrate nodes will expose some commands for benchmarking. You can refer to the `staging-node-cli` crate as +an example on how to integrate those. Note that for solely benchmarking pallets, the freestanding CLI is more suitable. + +## Usage + +Here we invoke the root command on the `staging-node-cli`. Most Substrate nodes should have a similar output, depending +on their integration of these commands. -Invoking the root benchmark command prints a help menu: ```sh -$ cargo run --profile=production -- benchmark +$ cargo run -p staging-node-cli --profile=production --features=runtime-benchmarks -- benchmark Sub-commands concerned with benchmarking. @@ -31,7 +41,46 @@ use `--release`. For the final results the `production` profile and reference hardware should be used, otherwise the results are not comparable. -The sub-commands are explained in depth here: +# Freestanding CLI + +The freestanding is a standalone CLI that does not rely on any node integration. It can be used to benchmark pallets of +any FRAME runtime that does not utilize 3rd party host functions. +It currently only supports pallet benchmarking, since the other commands still rely on a node. + +## Installation + +Installing from local source repository: + +```sh +cargo install --locked --path substrate/utils/frame/omni-bencher --profile=production +``` + +## Usage + +The exposed pallet sub-command is identical as the node-integrated CLI. The only difference is that it needs to be prefixed +with a `v1` to ensure drop-in compatibility. + +First we need to ensure that there is a runtime available. As example we will build the Westend runtime: + +```sh +cargo build -p westend-runtime --profile production --features runtime-benchmarks +``` + +Now the benchmarking can be started with: + +```sh +frame-omni-bencher v1 \ + benchmark pallet \ + --runtime target/release/wbuild/westend-runtime/westend-runtime.compact.compressed.wasm \ + --pallet "pallet_balances" --extrinsic "" +``` + +For the exact arguments of the `pallet` command, please refer to the [pallet] sub-module. + +# Commands + +The sub-commands of both CLIs have the same semantics and are documented in their respective sub-modules: + - [block] Compare the weight of a historic block to its actual resource usage - [machine] Gauges the speed of the hardware - [overhead] Creates weight files for the *Block*- and *Extrinsic*-base weights diff --git a/substrate/utils/frame/benchmarking-cli/src/pallet/command.rs b/substrate/utils/frame/benchmarking-cli/src/pallet/command.rs index 5fbfc0530bb3..305a9b960b98 100644 --- a/substrate/utils/frame/benchmarking-cli/src/pallet/command.rs +++ b/substrate/utils/frame/benchmarking-cli/src/pallet/command.rs @@ -15,7 +15,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{writer, ListOutput, PalletCmd}; +use super::{ + types::{ComponentRange, ComponentRangeMap}, + writer, ListOutput, PalletCmd, +}; +use crate::pallet::{types::FetchedCode, GenesisBuilder}; use codec::{Decode, Encode}; use frame_benchmarking::{ Analysis, BenchmarkBatch, BenchmarkBatchSplitResults, BenchmarkList, BenchmarkParameter, @@ -23,23 +27,25 @@ use frame_benchmarking::{ }; use frame_support::traits::StorageInfo; use linked_hash_map::LinkedHashMap; -use sc_cli::{execution_method_from_cli, CliConfiguration, Result, SharedParams}; +use sc_chain_spec::json_patch::merge as json_merge; +use sc_cli::{execution_method_from_cli, ChainSpec, CliConfiguration, Result, SharedParams}; use sc_client_db::BenchmarkingState; use sc_executor::{HeapAllocStrategy, WasmExecutor, DEFAULT_HEAP_ALLOC_STRATEGY}; -use sc_service::Configuration; -use serde::Serialize; use sp_core::{ offchain::{ testing::{TestOffchainExt, TestTransactionPoolExt}, OffchainDbExt, OffchainWorkerExt, TransactionPoolExt, }, - traits::{CallContext, ReadRuntimeVersionExt}, + traits::{CallContext, CodeExecutor, ReadRuntimeVersionExt, WrappedRuntimeCode}, }; use sp_externalities::Extensions; +use sp_genesis_builder::{PresetId, Result as GenesisBuildResult}; use sp_keystore::{testing::MemoryKeystore, KeystoreExt}; use sp_runtime::traits::Hash; -use sp_state_machine::StateMachine; +use sp_state_machine::{OverlayedChanges, StateMachine}; +use sp_wasm_interface::HostFunctions; use std::{ + borrow::Cow, collections::{BTreeMap, BTreeSet, HashMap}, fmt::Debug, fs, @@ -50,17 +56,6 @@ use std::{ /// Logging target const LOG_TARGET: &'static str = "polkadot_sdk_frame::benchmark::pallet"; -/// The inclusive range of a component. -#[derive(Serialize, Debug, Clone, Eq, PartialEq)] -pub(crate) struct ComponentRange { - /// Name of the component. - name: String, - /// Minimal valid value of the component. - min: u32, - /// Maximal valid value of the component. - max: u32, -} - /// How the PoV size of a storage item should be estimated. #[derive(clap::ValueEnum, Debug, Eq, PartialEq, Clone, Copy)] pub enum PovEstimationMode { @@ -87,7 +82,15 @@ impl FromStr for PovEstimationMode { /// Maps (pallet, benchmark) -> ((pallet, storage) -> PovEstimationMode) pub(crate) type PovModesMap = - HashMap<(Vec, Vec), HashMap<(String, String), PovEstimationMode>>; + HashMap<(String, String), HashMap<(String, String), PovEstimationMode>>; + +#[derive(Debug, Clone)] +struct SelectedBenchmark { + pallet: String, + extrinsic: String, + components: Vec<(BenchmarkParameter, u32, u32)>, + pov_modes: Vec<(String, String)>, +} // This takes multiple benchmark batches and combines all the results where the pallet, instance, // and benchmark are the same. @@ -145,41 +148,56 @@ This could mean that you either did not build the node correctly with the \ `--features runtime-benchmarks` flag, or the chain spec that you are using was \ not created by a node that was compiled with the flag"; +/// When the runtime could not build the genesis storage. +const ERROR_CANNOT_BUILD_GENESIS: &str = "The runtime returned \ +an error when trying to build the genesis storage. Please ensure that all pallets \ +define a genesis config that can be built. This can be tested with: \ +https://github.com/paritytech/polkadot-sdk/pull/3412"; + +/// Warn when using the chain spec to generate the genesis state. +const WARN_SPEC_GENESIS_CTOR: &'static str = "Using the chain spec instead of the runtime to \ +generate the genesis state is deprecated. Please remove the `--chain`/`--dev`/`--local` argument, \ +point `--runtime` to your runtime blob and set `--genesis-builder=runtime`. This warning may \ +become a hard error any time after December 2024."; + +/// The preset that we expect to find in the GenesisBuilder runtime API. +const GENESIS_PRESET: &str = "development"; + impl PalletCmd { /// Runs the command and benchmarks a pallet. - pub fn run(&self, config: Configuration) -> Result<()> + #[deprecated( + note = "`run` will be removed after December 2024. Use `run_with_spec` instead or \ + completely remove the code and use the `frame-benchmarking-cli` instead (see \ + https://github.com/paritytech/polkadot-sdk/pull/3512)." + )] + pub fn run(&self, config: sc_service::Configuration) -> Result<()> where Hasher: Hash, - ExtraHostFunctions: sp_wasm_interface::HostFunctions, + ExtraHostFunctions: HostFunctions, { + self.run_with_spec::(Some(config.chain_spec)) + } + + /// Runs the pallet benchmarking command. + pub fn run_with_spec( + &self, + chain_spec: Option>, + ) -> Result<()> + where + Hasher: Hash, + ExtraHostFunctions: HostFunctions, + { + self.check_args()?; let _d = self.execution.as_ref().map(|exec| { - // We print the warning at the end, since there is often A LOT of output. + // We print the error at the end, since there is often A LOT of output. sp_core::defer::DeferGuard::new(move || { - log::warn!( + log::error!( target: LOG_TARGET, "⚠️ Argument `--execution` is deprecated. Its value of `{exec}` has on effect.", ) }) }); - if let Some(output_path) = &self.output { - if !output_path.is_dir() && output_path.file_name().is_none() { - return Err("Output file or path is invalid!".into()) - } - } - - if let Some(header_file) = &self.header { - if !header_file.is_file() { - return Err("Header file is invalid!".into()) - }; - } - - if let Some(handlebars_template_file) = &self.template { - if !handlebars_template_file.is_file() { - return Err("Handlebars template file is invalid!".into()) - }; - } - if let Some(json_input) = &self.json_input { let raw_data = match std::fs::read(json_input) { Ok(raw_data) => raw_data, @@ -194,16 +212,10 @@ impl PalletCmd { return self.output_from_results(&batches) } - let spec = config.chain_spec; - let pallet = self.pallet.clone().unwrap_or_default(); - let pallet = pallet.as_bytes(); - - let extrinsic = self.extrinsic.clone().unwrap_or_default(); - let extrinsic_split: Vec<&str> = extrinsic.split(',').collect(); - let extrinsics: Vec<_> = extrinsic_split.iter().map(|x| x.trim().as_bytes()).collect(); + let (genesis_storage, genesis_changes) = + self.genesis_storage::(&chain_spec)?; + let mut changes = genesis_changes.clone(); - let genesis_storage = spec.build_storage()?; - let mut changes = Default::default(); let cache_size = Some(self.database_cache_size as usize); let state_with_tracking = BenchmarkingState::::new( genesis_storage.clone(), @@ -225,11 +237,10 @@ impl PalletCmd { let method = execution_method_from_cli(self.wasm_method, self.wasmtime_instantiation_strategy); - let heap_pages = - self.heap_pages - .map_or(DEFAULT_HEAP_ALLOC_STRATEGY, |p| HeapAllocStrategy::Static { - extra_pages: p as _, - }); + let state = &state_without_tracking; + let runtime = self.runtime_blob(&state_without_tracking)?; + let runtime_code = runtime.code()?; + let alloc_strategy = Self::alloc_strategy(runtime_code.heap_pages); let executor = WasmExecutor::<( sp_io::SubstrateHostFunctions, @@ -237,91 +248,30 @@ impl PalletCmd { ExtraHostFunctions, )>::builder() .with_execution_method(method) - .with_onchain_heap_alloc_strategy(heap_pages) - .with_offchain_heap_alloc_strategy(heap_pages) + .with_allow_missing_host_functions(self.allow_missing_host_functions) + .with_onchain_heap_alloc_strategy(alloc_strategy) + .with_offchain_heap_alloc_strategy(alloc_strategy) .with_max_runtime_instances(2) .with_runtime_cache_size(2) .build(); - let extensions = || -> Extensions { - let mut extensions = Extensions::default(); - let (offchain, _) = TestOffchainExt::new(); - let (pool, _) = TestTransactionPoolExt::new(); - let keystore = MemoryKeystore::new(); - extensions.register(KeystoreExt::new(keystore)); - extensions.register(OffchainWorkerExt::new(offchain.clone())); - extensions.register(OffchainDbExt::new(offchain)); - extensions.register(TransactionPoolExt::new(pool)); - extensions.register(ReadRuntimeVersionExt::new(executor.clone())); - extensions - }; - - // Get Benchmark List - let state = &state_without_tracking; - let result = StateMachine::new( - state, - &mut changes, - &executor, - "Benchmark_benchmark_metadata", - &(self.extra).encode(), - &mut extensions(), - &sp_state_machine::backend::BackendRuntimeCode::new(state).runtime_code()?, - CallContext::Offchain, - ) - .execute() - .map_err(|e| format!("{}: {}", ERROR_METADATA_NOT_FOUND, e))?; - - let (list, storage_info) = - <(Vec, Vec) as Decode>::decode(&mut &result[..]) - .map_err(|e| format!("Failed to decode benchmark metadata: {:?}", e))?; + let (list, storage_info): (Vec, Vec) = + Self::exec_state_machine( + StateMachine::new( + state, + &mut changes, + &executor, + "Benchmark_benchmark_metadata", + &(self.extra).encode(), + &mut Self::build_extensions(executor.clone()), + &runtime_code, + CallContext::Offchain, + ), + ERROR_METADATA_NOT_FOUND, + )?; // Use the benchmark list and the user input to determine the set of benchmarks to run. - let mut benchmarks_to_run = Vec::new(); - list.iter() - .filter(|item| pallet.is_empty() || pallet == &b"*"[..] || pallet == &item.pallet[..]) - .for_each(|item| { - for benchmark in &item.benchmarks { - let benchmark_name = &benchmark.name; - if extrinsic.is_empty() || - extrinsic.as_bytes() == &b"*"[..] || - extrinsics.contains(&&benchmark_name[..]) - { - benchmarks_to_run.push(( - item.pallet.clone(), - benchmark.name.clone(), - benchmark.components.clone(), - benchmark.pov_modes.clone(), - )) - } - } - }); - // Convert `Vec` to `String` for better readability. - let benchmarks_to_run: Vec<_> = benchmarks_to_run - .into_iter() - .map(|(pallet, extrinsic, components, pov_modes)| { - let pallet_name = - String::from_utf8(pallet.clone()).expect("Encoded from String; qed"); - let extrinsic_name = - String::from_utf8(extrinsic.clone()).expect("Encoded from String; qed"); - ( - pallet, - extrinsic, - components, - pov_modes - .into_iter() - .map(|(p, s)| { - (String::from_utf8(p).unwrap(), String::from_utf8(s).unwrap()) - }) - .collect(), - pallet_name, - extrinsic_name, - ) - }) - .collect(); - - if benchmarks_to_run.is_empty() { - return Err("No benchmarks found which match your input.".into()) - } + let benchmarks_to_run = self.select_benchmarks_to_run(list)?; if let Some(list_output) = self.list { list_benchmark(benchmarks_to_run, list_output, self.no_csv_header); @@ -333,15 +283,17 @@ impl PalletCmd { let mut batches_db = Vec::new(); let mut timer = time::SystemTime::now(); // Maps (pallet, extrinsic) to its component ranges. - let mut component_ranges = HashMap::<(Vec, Vec), Vec>::new(); + let mut component_ranges = HashMap::<(String, String), Vec>::new(); let pov_modes = Self::parse_pov_modes(&benchmarks_to_run)?; + let mut failed = Vec::<(String, String)>::new(); - for (pallet, extrinsic, components, _, pallet_name, extrinsic_name) in - benchmarks_to_run.clone() + 'outer: for (i, SelectedBenchmark { pallet, extrinsic, components, .. }) in + benchmarks_to_run.clone().into_iter().enumerate() { log::info!( target: LOG_TARGET, - "Starting benchmark: {pallet_name}::{extrinsic_name}" + "[{: >3} % ] Starting benchmark: {pallet}::{extrinsic}", + (i * 100) / benchmarks_to_run.len(), ); let all_components = if components.is_empty() { vec![Default::default()] @@ -392,100 +344,127 @@ impl PalletCmd { for (s, selected_components) in all_components.iter().enumerate() { // First we run a verification if !self.no_verify { + let mut changes = genesis_changes.clone(); let state = &state_without_tracking; - let result = StateMachine::new( - state, - &mut changes, - &executor, - "Benchmark_dispatch_benchmark", - &( - &pallet, - &extrinsic, - &selected_components.clone(), - true, // run verification code - 1, // no need to do internal repeats - ) - .encode(), - &mut extensions(), - &sp_state_machine::backend::BackendRuntimeCode::new(state) - .runtime_code()?, - CallContext::Offchain, - ) - .execute() - .map_err(|e| { - format!("Error executing and verifying runtime benchmark: {}", e) - })?; // Don't use these results since verification code will add overhead. - let _batch = - , String> as Decode>::decode( - &mut &result[..], - ) - .map_err(|e| format!("Failed to decode benchmark results: {:?}", e))? - .map_err(|e| { - format!("Benchmark {pallet_name}::{extrinsic_name} failed: {e}",) - })?; + let _batch: Vec = match Self::exec_state_machine::< + std::result::Result, String>, + _, + _, + >( + StateMachine::new( + state, + &mut changes, + &executor, + "Benchmark_dispatch_benchmark", + &( + pallet.as_bytes(), + extrinsic.as_bytes(), + &selected_components.clone(), + true, // run verification code + 1, // no need to do internal repeats + ) + .encode(), + &mut Self::build_extensions(executor.clone()), + &runtime_code, + CallContext::Offchain, + ), + "dispatch a benchmark", + ) { + Err(e) => { + log::error!("Error executing and verifying runtime benchmark: {}", e); + failed.push((pallet.clone(), extrinsic.clone())); + continue 'outer + }, + Ok(Err(e)) => { + log::error!("Error executing and verifying runtime benchmark: {}", e); + failed.push((pallet.clone(), extrinsic.clone())); + continue 'outer + }, + Ok(Ok(b)) => b, + }; } // Do one loop of DB tracking. { + let mut changes = genesis_changes.clone(); let state = &state_with_tracking; - let result = StateMachine::new( - state, // todo remove tracking - &mut changes, - &executor, - "Benchmark_dispatch_benchmark", - &( - &pallet.clone(), - &extrinsic.clone(), - &selected_components.clone(), - false, // don't run verification code for final values - self.repeat, - ) - .encode(), - &mut extensions(), - &sp_state_machine::backend::BackendRuntimeCode::new(state) - .runtime_code()?, - CallContext::Offchain, - ) - .execute() - .map_err(|e| format!("Error executing runtime benchmark: {}", e))?; - - let batch = - , String> as Decode>::decode( - &mut &result[..], - ) - .map_err(|e| format!("Failed to decode benchmark results: {:?}", e))??; + let batch: Vec = match Self::exec_state_machine::< + std::result::Result, String>, + _, + _, + >( + StateMachine::new( + state, // todo remove tracking + &mut changes, + &executor, + "Benchmark_dispatch_benchmark", + &( + pallet.as_bytes(), + extrinsic.as_bytes(), + &selected_components.clone(), + false, // don't run verification code for final values + self.repeat, + ) + .encode(), + &mut Self::build_extensions(executor.clone()), + &runtime_code, + CallContext::Offchain, + ), + "dispatch a benchmark", + ) { + Err(e) => { + log::error!("Error executing runtime benchmark: {}", e); + failed.push((pallet.clone(), extrinsic.clone())); + continue 'outer + }, + Ok(Err(e)) => { + log::error!("Benchmark {pallet}::{extrinsic} failed: {e}",); + failed.push((pallet.clone(), extrinsic.clone())); + continue 'outer + }, + Ok(Ok(b)) => b, + }; batches_db.extend(batch); } // Finally run a bunch of loops to get extrinsic timing information. for r in 0..self.external_repeat { + let mut changes = genesis_changes.clone(); let state = &state_without_tracking; - let result = StateMachine::new( - state, // todo remove tracking - &mut changes, - &executor, - "Benchmark_dispatch_benchmark", - &( - &pallet.clone(), - &extrinsic.clone(), - &selected_components.clone(), - false, // don't run verification code for final values - self.repeat, - ) - .encode(), - &mut extensions(), - &sp_state_machine::backend::BackendRuntimeCode::new(state) - .runtime_code()?, - CallContext::Offchain, - ) - .execute() - .map_err(|e| format!("Error executing runtime benchmark: {}", e))?; - - let batch = - , String> as Decode>::decode( - &mut &result[..], - ) - .map_err(|e| format!("Failed to decode benchmark results: {:?}", e))??; + let batch = match Self::exec_state_machine::< + std::result::Result, String>, + _, + _, + >( + StateMachine::new( + state, // todo remove tracking + &mut changes, + &executor, + "Benchmark_dispatch_benchmark", + &( + pallet.as_bytes(), + extrinsic.as_bytes(), + &selected_components.clone(), + false, // don't run verification code for final values + self.repeat, + ) + .encode(), + &mut Self::build_extensions(executor.clone()), + &runtime_code, + CallContext::Offchain, + ), + "dispatch a benchmark", + ) { + Err(e) => { + return Err(format!("Error executing runtime benchmark: {e}",).into()); + }, + Ok(Err(e)) => { + return Err( + format!("Benchmark {pallet}::{extrinsic} failed: {e}",).into() + ); + }, + Ok(Ok(b)) => b, + }; batches.extend(batch); @@ -496,7 +475,8 @@ impl PalletCmd { log::info!( target: LOG_TARGET, - "Running benchmark: {pallet_name}::{extrinsic_name}({} args) {}/{} {}/{}", + "[{: >3} % ] Running benchmark: {pallet}::{extrinsic}({} args) {}/{} {}/{}", + (i * 100) / benchmarks_to_run.len(), components.len(), s + 1, // s starts at 0. all_components.len(), @@ -509,21 +489,276 @@ impl PalletCmd { } } + assert!(batches_db.len() == batches.len() / self.external_repeat as usize); + + if !failed.is_empty() { + failed.sort(); + eprintln!( + "The following {} benchmarks failed:\n{}", + failed.len(), + failed.iter().map(|(p, e)| format!("- {p}::{e}")).collect::>().join("\n") + ); + return Err(format!("{} benchmarks failed", failed.len()).into()) + } + // Combine all of the benchmark results, so that benchmarks of the same pallet/function // are together. let batches = combine_batches(batches, batches_db); self.output(&batches, &storage_info, &component_ranges, pov_modes) } + fn select_benchmarks_to_run(&self, list: Vec) -> Result> { + let pallet = self.pallet.clone().unwrap_or_default(); + let pallet = pallet.as_bytes(); + + let extrinsic = self.extrinsic.clone().unwrap_or_default(); + let extrinsic_split: Vec<&str> = extrinsic.split(',').collect(); + let extrinsics: Vec<_> = extrinsic_split.iter().map(|x| x.trim().as_bytes()).collect(); + + // Use the benchmark list and the user input to determine the set of benchmarks to run. + let mut benchmarks_to_run = Vec::new(); + list.iter() + .filter(|item| pallet.is_empty() || pallet == &b"*"[..] || pallet == &item.pallet[..]) + .for_each(|item| { + for benchmark in &item.benchmarks { + let benchmark_name = &benchmark.name; + if extrinsic.is_empty() || + extrinsic.as_bytes() == &b"*"[..] || + extrinsics.contains(&&benchmark_name[..]) + { + benchmarks_to_run.push(( + item.pallet.clone(), + benchmark.name.clone(), + benchmark.components.clone(), + benchmark.pov_modes.clone(), + )) + } + } + }); + // Convert `Vec` to `String` for better readability. + let benchmarks_to_run: Vec<_> = benchmarks_to_run + .into_iter() + .map(|(pallet, extrinsic, components, pov_modes)| { + let pallet = String::from_utf8(pallet.clone()).expect("Encoded from String; qed"); + let extrinsic = + String::from_utf8(extrinsic.clone()).expect("Encoded from String; qed"); + + SelectedBenchmark { + pallet, + extrinsic, + components, + pov_modes: pov_modes + .into_iter() + .map(|(p, s)| { + (String::from_utf8(p).unwrap(), String::from_utf8(s).unwrap()) + }) + .collect(), + } + }) + .collect(); + + if benchmarks_to_run.is_empty() { + return Err("No benchmarks found which match your input.".into()) + } + + Ok(benchmarks_to_run) + } + + /// Produce a genesis storage and genesis changes. + /// + /// It would be easier to only return one type, but there is no easy way to convert them. + // TODO: Re-write `BenchmarkingState` to not be such a clusterfuck and only accept + // `OverlayedChanges` instead of a mix between `OverlayedChanges` and `State`. But this can only + // be done once we deprecated and removed the legacy interface :( + fn genesis_storage( + &self, + chain_spec: &Option>, + ) -> Result<(sp_storage::Storage, OverlayedChanges)> { + Ok(match (self.genesis_builder, self.runtime.is_some()) { + (Some(GenesisBuilder::None), _) => Default::default(), + (Some(GenesisBuilder::Spec), _) | (None, false) => { + log::warn!("{WARN_SPEC_GENESIS_CTOR}"); + let Some(chain_spec) = chain_spec else { + return Err("No chain spec specified to generate the genesis state".into()); + }; + + let storage = chain_spec + .build_storage() + .map_err(|e| format!("{ERROR_CANNOT_BUILD_GENESIS}\nError: {e}"))?; + + (storage, Default::default()) + }, + (Some(GenesisBuilder::Runtime), _) | (None, true) => + (Default::default(), self.genesis_from_runtime::()?), + }) + } + + /// Generate the genesis changeset by the runtime API. + fn genesis_from_runtime(&self) -> Result> { + let state = BenchmarkingState::::new( + Default::default(), + Some(self.database_cache_size as usize), + false, + false, + )?; + + // Create a dummy WasmExecutor just to build the genesis storage. + let method = + execution_method_from_cli(self.wasm_method, self.wasmtime_instantiation_strategy); + let executor = WasmExecutor::<( + sp_io::SubstrateHostFunctions, + frame_benchmarking::benchmarking::HostFunctions, + F, + )>::builder() + .with_execution_method(method) + .with_allow_missing_host_functions(self.allow_missing_host_functions) + .build(); + + let runtime = self.runtime_blob(&state)?; + let runtime_code = runtime.code()?; + + // We cannot use the `GenesisConfigBuilderRuntimeCaller` here since it returns the changes + // as `Storage` item, but we need it as `OverlayedChanges`. + let genesis_json: Option> = Self::exec_state_machine( + StateMachine::new( + &state, + &mut Default::default(), + &executor, + "GenesisBuilder_get_preset", + &None::.encode(), // Use the default preset + &mut Self::build_extensions(executor.clone()), + &runtime_code, + CallContext::Offchain, + ), + "build the genesis spec", + )?; + + let Some(base_genesis_json) = genesis_json else { + return Err("GenesisBuilder::get_preset returned no data".into()) + }; + + let base_genesis_json = serde_json::from_slice::(&base_genesis_json) + .map_err(|e| format!("GenesisBuilder::get_preset returned invalid JSON: {:?}", e))?; + + let dev_genesis_json: Option> = Self::exec_state_machine( + StateMachine::new( + &state, + &mut Default::default(), + &executor, + "GenesisBuilder_get_preset", + &Some::(GENESIS_PRESET.into()).encode(), // Use the default preset + &mut Self::build_extensions(executor.clone()), + &runtime_code, + CallContext::Offchain, + ), + "build the genesis spec", + )?; + + let mut genesis_json = serde_json::Value::default(); + json_merge(&mut genesis_json, base_genesis_json); + + if let Some(dev) = dev_genesis_json { + let dev: serde_json::Value = serde_json::from_slice(&dev).map_err(|e| { + format!("GenesisBuilder::get_preset returned invalid JSON: {:?}", e) + })?; + json_merge(&mut genesis_json, dev); + } else { + log::warn!( + "Could not find genesis preset '{GENESIS_PRESET}'. Falling back to default." + ); + } + + let json_pretty_str = serde_json::to_string_pretty(&genesis_json) + .map_err(|e| format!("json to string failed: {e}"))?; + + let mut changes = Default::default(); + let build_res: GenesisBuildResult = Self::exec_state_machine( + StateMachine::new( + &state, + &mut changes, + &executor, + "GenesisBuilder_build_state", + &json_pretty_str.encode(), + &mut Extensions::default(), + &runtime_code, + CallContext::Offchain, + ), + "populate the genesis state", + )?; + + if let Err(e) = build_res { + return Err(format!("GenesisBuilder::build_state failed: {}", e).into()) + } + + Ok(changes) + } + + /// Execute a state machine and decode its return value as `R`. + fn exec_state_machine( + mut machine: StateMachine, H, Exec>, + hint: &str, + ) -> Result { + let res = machine + .execute() + .map_err(|e| format!("Could not call runtime API to {hint}: {}", e))?; + let res = R::decode(&mut &res[..]) + .map_err(|e| format!("Failed to decode runtime API result to {hint}: {:?}", e))?; + Ok(res) + } + + /// Build the extension that are available for pallet benchmarks. + fn build_extensions(exe: E) -> Extensions { + let mut extensions = Extensions::default(); + let (offchain, _) = TestOffchainExt::new(); + let (pool, _) = TestTransactionPoolExt::new(); + let keystore = MemoryKeystore::new(); + extensions.register(KeystoreExt::new(keystore)); + extensions.register(OffchainWorkerExt::new(offchain.clone())); + extensions.register(OffchainDbExt::new(offchain)); + extensions.register(TransactionPoolExt::new(pool)); + extensions.register(ReadRuntimeVersionExt::new(exe)); + extensions + } + + /// Load the runtime blob for this benchmark. + /// + /// The blob will either be loaded from the `:code` key out of the chain spec, or from a file + /// when specified with `--runtime`. + fn runtime_blob<'a, H: Hash>( + &self, + state: &'a BenchmarkingState, + ) -> Result, H>> { + if let Some(runtime) = &self.runtime { + log::info!("Loading WASM from {}", runtime.display()); + let code = fs::read(runtime)?; + let hash = sp_core::blake2_256(&code).to_vec(); + let wrapped_code = WrappedRuntimeCode(Cow::Owned(code)); + + Ok(FetchedCode::FromFile { wrapped_code, heap_pages: self.heap_pages, hash }) + } else { + log::info!("Loading WASM from genesis state"); + let state = sp_state_machine::backend::BackendRuntimeCode::new(state); + + Ok(FetchedCode::FromGenesis { state }) + } + } + + /// Allocation strategy for pallet benchmarking. + fn alloc_strategy(heap_pages: Option) -> HeapAllocStrategy { + heap_pages.map_or(DEFAULT_HEAP_ALLOC_STRATEGY, |p| HeapAllocStrategy::Static { + extra_pages: p as _, + }) + } + fn output( &self, batches: &[BenchmarkBatchSplitResults], storage_info: &[StorageInfo], - component_ranges: &HashMap<(Vec, Vec), Vec>, + component_ranges: &ComponentRangeMap, pov_modes: PovModesMap, ) -> Result<()> { // Jsonify the result and write it to a file or stdout if desired. - if !self.jsonify(&batches)? { + if !self.jsonify(&batches)? && !self.quiet { // Print the summary only if `jsonify` did not write to stdout. self.print_summary(&batches, &storage_info, pov_modes.clone()) } @@ -546,11 +781,13 @@ impl PalletCmd { /// Re-analyze a batch historic benchmark timing data. Will not take the PoV into account. fn output_from_results(&self, batches: &[BenchmarkBatchSplitResults]) -> Result<()> { - let mut component_ranges = - HashMap::<(Vec, Vec), HashMap>::new(); + let mut component_ranges = HashMap::<(String, String), HashMap>::new(); for batch in batches { let range = component_ranges - .entry((batch.pallet.clone(), batch.benchmark.clone())) + .entry(( + String::from_utf8(batch.pallet.clone()).unwrap(), + String::from_utf8(batch.benchmark.clone()).unwrap(), + )) .or_default(); for result in &batch.time_results { for (param, value) in &result.components { @@ -608,10 +845,13 @@ impl PalletCmd { ) { for batch in batches.iter() { // Print benchmark metadata + let pallet = String::from_utf8(batch.pallet.clone()).expect("Encoded from String; qed"); + let benchmark = + String::from_utf8(batch.benchmark.clone()).expect("Encoded from String; qed"); println!( "Pallet: {:?}, Extrinsic: {:?}, Lowest values: {:?}, Highest values: {:?}, Steps: {:?}, Repeat: {:?}", - String::from_utf8(batch.pallet.clone()).expect("Encoded from String; qed"), - String::from_utf8(batch.benchmark.clone()).expect("Encoded from String; qed"), + pallet, + benchmark, self.lowest_range_values, self.highest_range_values, self.steps, @@ -625,10 +865,7 @@ impl PalletCmd { if !self.no_storage_info { let mut storage_per_prefix = HashMap::, Vec>::new(); - let pov_mode = pov_modes - .get(&(batch.pallet.clone(), batch.benchmark.clone())) - .cloned() - .unwrap_or_default(); + let pov_mode = pov_modes.get(&(pallet, benchmark)).cloned().unwrap_or_default(); let comments = writer::process_storage_results( &mut storage_per_prefix, @@ -699,20 +936,11 @@ impl PalletCmd { } /// Parses the PoV modes per benchmark that were specified by the `#[pov_mode]` attribute. - fn parse_pov_modes( - benchmarks: &Vec<( - Vec, - Vec, - Vec<(BenchmarkParameter, u32, u32)>, - Vec<(String, String)>, - String, - String, - )>, - ) -> Result { + fn parse_pov_modes(benchmarks: &Vec) -> Result { use std::collections::hash_map::Entry; let mut parsed = PovModesMap::new(); - for (pallet, call, _components, pov_modes, _, _) in benchmarks { + for SelectedBenchmark { pallet, extrinsic, pov_modes, .. } in benchmarks { for (pallet_storage, mode) in pov_modes { let mode = PovEstimationMode::from_str(&mode)?; let splits = pallet_storage.split("::").collect::>(); @@ -726,7 +954,7 @@ impl PalletCmd { let (pov_pallet, pov_storage) = (splits[0], splits.get(1).unwrap_or(&"ALL")); match parsed - .entry((pallet.clone(), call.clone())) + .entry((pallet.clone(), extrinsic.clone())) .or_default() .entry((pov_pallet.to_string(), pov_storage.to_string())) { @@ -744,6 +972,33 @@ impl PalletCmd { } Ok(parsed) } + + /// Sanity check the CLI arguments. + fn check_args(&self) -> Result<()> { + if self.runtime.is_some() && self.shared_params.chain.is_some() { + unreachable!("Clap should not allow both `--runtime` and `--chain` to be provided.") + } + + if let Some(output_path) = &self.output { + if !output_path.is_dir() && output_path.file_name().is_none() { + return Err("Output file or path is invalid!".into()) + } + } + + if let Some(header_file) = &self.header { + if !header_file.is_file() { + return Err("Header file is invalid!".into()) + }; + } + + if let Some(handlebars_template_file) = &self.template { + if !handlebars_template_file.is_file() { + return Err("Handlebars template file is invalid!".into()) + }; + } + + Ok(()) + } } impl CliConfiguration for PalletCmd { @@ -761,35 +1016,28 @@ impl CliConfiguration for PalletCmd { /// List the benchmarks available in the runtime, in a CSV friendly format. fn list_benchmark( - benchmarks_to_run: Vec<( - Vec, - Vec, - Vec<(BenchmarkParameter, u32, u32)>, - Vec<(String, String)>, - String, - String, - )>, + benchmarks_to_run: Vec, list_output: ListOutput, no_csv_header: bool, ) { let mut benchmarks = BTreeMap::new(); // Sort and de-dub by pallet and function name. - benchmarks_to_run.iter().for_each(|(_, _, _, _, pallet_name, extrinsic_name)| { + benchmarks_to_run.iter().for_each(|bench| { benchmarks - .entry(pallet_name) + .entry(&bench.pallet) .or_insert_with(BTreeSet::new) - .insert(extrinsic_name); + .insert(&bench.extrinsic); }); match list_output { ListOutput::All => { if !no_csv_header { - println!("pallet,extrinsic"); + println!("pallet, extrinsic"); } for (pallet, extrinsics) in benchmarks { for extrinsic in extrinsics { - println!("{pallet},{extrinsic}"); + println!("{pallet}, {extrinsic}"); } } }, diff --git a/substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs b/substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs index 6dc56c0724ea..d05b52f1ac87 100644 --- a/substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs +++ b/substrate/utils/frame/benchmarking-cli/src/pallet/mod.rs @@ -16,9 +16,10 @@ // limitations under the License. mod command; +mod types; mod writer; -use crate::shared::HostInfoParams; +use crate::{pallet::types::GenesisBuilder, shared::HostInfoParams}; use clap::ValueEnum; use sc_cli::{ WasmExecutionMethod, WasmtimeInstantiationStrategy, DEFAULT_WASMTIME_INSTANTIATION_STRATEGY, @@ -166,6 +167,20 @@ pub struct PalletCmd { )] pub wasmtime_instantiation_strategy: WasmtimeInstantiationStrategy, + /// Optional runtime blob to use instead of the one from the genesis config. + #[arg(long, conflicts_with = "chain")] + pub runtime: Option, + + /// Do not fail if there are unknown but also unused host functions in the runtime. + #[arg(long)] + pub allow_missing_host_functions: bool, + + /// How to construct the genesis state. + /// + /// Uses `GenesisBuilder::Spec` by default and `GenesisBuilder::Runtime` if `runtime` is set. + #[arg(long, value_enum)] + pub genesis_builder: Option, + /// DEPRECATED: This argument has no effect. #[arg(long = "execution")] pub execution: Option, @@ -221,4 +236,11 @@ pub struct PalletCmd { /// This exists only to restore legacy behaviour. It should never actually be needed. #[arg(long)] pub unsafe_overwrite_results: bool, + + /// Do not print a summary at the end of the run. + /// + /// These summaries can be very long when benchmarking multiple pallets at once. For CI + /// use-cases, this option reduces the noise. + #[arg(long)] + quiet: bool, } diff --git a/substrate/utils/frame/benchmarking-cli/src/pallet/types.rs b/substrate/utils/frame/benchmarking-cli/src/pallet/types.rs new file mode 100644 index 000000000000..2bb00d66560f --- /dev/null +++ b/substrate/utils/frame/benchmarking-cli/src/pallet/types.rs @@ -0,0 +1,78 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Various types used by this crate. + +use sc_cli::Result; +use sp_core::traits::{RuntimeCode, WrappedRuntimeCode}; +use sp_runtime::traits::Hash; + +/// How the genesis state for benchmarking should be build. +#[derive(clap::ValueEnum, Debug, Eq, PartialEq, Clone, Copy)] +#[clap(rename_all = "kebab-case")] +pub enum GenesisBuilder { + /// Do not provide any genesis state. + /// + /// Benchmarks are advised to function with this, since they should setup their own required + /// state. However, to keep backwards compatibility, this is not the default. + None, + /// Let the runtime build the genesis state through its `BuildGenesisConfig` runtime API. + Runtime, + /// Use the spec file to build the genesis state. This fails when there is no spec. + Spec, +} + +/// A runtime blob that was either fetched from genesis storage or loaded from a file. +// NOTE: This enum is only needed for the annoying lifetime bounds on `RuntimeCode`. Otherwise we +// could just directly return the blob. +pub enum FetchedCode<'a, B, H> { + FromGenesis { state: sp_state_machine::backend::BackendRuntimeCode<'a, B, H> }, + FromFile { wrapped_code: WrappedRuntimeCode<'a>, heap_pages: Option, hash: Vec }, +} + +impl<'a, B, H> FetchedCode<'a, B, H> +where + H: Hash, + B: sc_client_api::StateBackend, +{ + /// The runtime blob. + pub fn code(&'a self) -> Result> { + match self { + Self::FromGenesis { state } => state.runtime_code().map_err(Into::into), + Self::FromFile { wrapped_code, heap_pages, hash } => Ok(RuntimeCode { + code_fetcher: wrapped_code, + heap_pages: *heap_pages, + hash: hash.clone(), + }), + } + } +} + +/// Maps a (pallet, benchmark) to its component ranges. +pub(crate) type ComponentRangeMap = + std::collections::HashMap<(String, String), Vec>; + +/// The inclusive range of a component. +#[derive(serde::Serialize, Debug, Clone, Eq, PartialEq)] +pub(crate) struct ComponentRange { + /// Name of the component. + pub(crate) name: String, + /// Minimal valid value of the component. + pub(crate) min: u32, + /// Maximal valid value of the component. + pub(crate) max: u32, +} diff --git a/substrate/utils/frame/benchmarking-cli/src/pallet/writer.rs b/substrate/utils/frame/benchmarking-cli/src/pallet/writer.rs index bd4b65d8a2e3..df7d81b2822e 100644 --- a/substrate/utils/frame/benchmarking-cli/src/pallet/writer.rs +++ b/substrate/utils/frame/benchmarking-cli/src/pallet/writer.rs @@ -28,7 +28,10 @@ use itertools::Itertools; use serde::Serialize; use crate::{ - pallet::command::{ComponentRange, PovEstimationMode, PovModesMap}, + pallet::{ + command::{PovEstimationMode, PovModesMap}, + types::{ComponentRange, ComponentRangeMap}, + }, shared::UnderscoreHelper, PalletCmd, }; @@ -132,7 +135,7 @@ fn io_error(s: &str) -> std::io::Error { fn map_results( batches: &[BenchmarkBatchSplitResults], storage_info: &[StorageInfo], - component_ranges: &HashMap<(Vec, Vec), Vec>, + component_ranges: &ComponentRangeMap, pov_modes: PovModesMap, default_pov_mode: PovEstimationMode, analysis_choice: &AnalysisChoice, @@ -188,7 +191,7 @@ fn get_benchmark_data( batch: &BenchmarkBatchSplitResults, storage_info: &[StorageInfo], // Per extrinsic component ranges. - component_ranges: &HashMap<(Vec, Vec), Vec>, + component_ranges: &ComponentRangeMap, pov_modes: PovModesMap, default_pov_mode: PovEstimationMode, analysis_choice: &AnalysisChoice, @@ -207,6 +210,8 @@ fn get_benchmark_data( AnalysisChoice::MedianSlopes => Analysis::median_slopes, AnalysisChoice::Max => Analysis::max, }; + let pallet = String::from_utf8(batch.pallet.clone()).unwrap(); + let benchmark = String::from_utf8(batch.benchmark.clone()).unwrap(); let extrinsic_time = analysis_function(&batch.time_results, BenchmarkSelector::ExtrinsicTime) .expect("analysis function should return an extrinsic time for valid inputs"); @@ -282,10 +287,7 @@ fn get_benchmark_data( // We add additional comments showing which storage items were touched. // We find the worst case proof size, and use that as the final proof size result. let mut storage_per_prefix = HashMap::, Vec>::new(); - let pov_mode = pov_modes - .get(&(batch.pallet.clone(), batch.benchmark.clone())) - .cloned() - .unwrap_or_default(); + let pov_mode = pov_modes.get(&(pallet.clone(), benchmark.clone())).cloned().unwrap_or_default(); let comments = process_storage_results( &mut storage_per_prefix, &batch.db_results, @@ -351,12 +353,12 @@ fn get_benchmark_data( .collect::>(); let component_ranges = component_ranges - .get(&(batch.pallet.clone(), batch.benchmark.clone())) + .get(&(pallet.clone(), benchmark.clone())) .map(|c| c.clone()) .unwrap_or_default(); BenchmarkData { - name: String::from_utf8(batch.benchmark.clone()).unwrap(), + name: benchmark, components, base_weight: extrinsic_time.base, base_reads: reads.base, @@ -378,7 +380,7 @@ fn get_benchmark_data( pub(crate) fn write_results( batches: &[BenchmarkBatchSplitResults], storage_info: &[StorageInfo], - component_ranges: &HashMap<(Vec, Vec), Vec>, + component_ranges: &HashMap<(String, String), Vec>, pov_modes: PovModesMap, default_pov_mode: PovEstimationMode, path: &PathBuf, @@ -871,10 +873,10 @@ mod test { fn test_pov_mode() -> PovModesMap { let mut map = PovModesMap::new(); - map.entry((b"scheduler".to_vec(), b"first_benchmark".to_vec())) + map.entry(("scheduler".into(), "first_benchmark".into())) .or_default() .insert(("scheduler".into(), "mel".into()), PovEstimationMode::MaxEncodedLen); - map.entry((b"scheduler".to_vec(), b"first_benchmark".to_vec())) + map.entry(("scheduler".into(), "first_benchmark".into())) .or_default() .insert(("scheduler".into(), "measured".into()), PovEstimationMode::Measured); map diff --git a/substrate/utils/frame/omni-bencher/Cargo.toml b/substrate/utils/frame/omni-bencher/Cargo.toml new file mode 100644 index 000000000000..0c2d1a1b32b1 --- /dev/null +++ b/substrate/utils/frame/omni-bencher/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "frame-omni-bencher" +version = "0.1.0" +description = "Freestanding benchmark runner for any Polkadot runtime." +authors.workspace = true +edition.workspace = true +repository.workspace = true +license.workspace = true + +[lints] +workspace = true + +[dependencies] +clap = { version = "4.5.2", features = ["derive"] } +cumulus-primitives-proof-size-hostfunction = { path = "../../../../cumulus/primitives/proof-size-hostfunction" } +frame-benchmarking-cli = { path = "../benchmarking-cli", default-features = false } +sc-cli = { path = "../../../client/cli" } +sp-runtime = { path = "../../../primitives/runtime" } +sp-statement-store = { path = "../../../primitives/statement-store" } +env_logger = "0.11.2" +log = { workspace = true } diff --git a/substrate/utils/frame/omni-bencher/src/command.rs b/substrate/utils/frame/omni-bencher/src/command.rs new file mode 100644 index 000000000000..f0159f4307d6 --- /dev/null +++ b/substrate/utils/frame/omni-bencher/src/command.rs @@ -0,0 +1,149 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use clap::Parser; +use frame_benchmarking_cli::BenchmarkCmd; +use sc_cli::Result; +use sp_runtime::traits::BlakeTwo256; + +/// # Polkadot Omni Benchmarking CLI +/// +/// The Polkadot Omni benchmarker allows to benchmark the extrinsics of any Polkadot runtime. It is +/// meant to replace the current manual integration of the `benchmark pallet` into every parachain +/// node. This reduces duplicate code and makes maintenance for builders easier. The CLI is +/// currently only able to benchmark extrinsics. In the future it is planned to extend this to some +/// other areas. +/// +/// General FRAME runtimes could also be used with this benchmarker, as long as they don't utilize +/// any host functions that are not part of the Polkadot host specification. +/// +/// ## Installation +/// +/// Directly via crates.io: +/// +/// ```sh +/// cargo install --locked frame-omni-bencher +/// ``` +/// +/// or when the sources are locally checked out: +/// +/// ```sh +/// cargo install --locked --path substrate/utils/frame/omni-bencher --profile=production +/// ``` +/// +/// Check the installed version and print the docs: +/// +/// ```sh +/// frame-omni-bencher --help +/// ``` +/// +/// ## Usage +/// +/// First we need to ensure that there is a runtime available. As example we will build the Westend +/// runtime: +/// +/// ```sh +/// cargo build -p westend-runtime --profile production --features runtime-benchmarks +/// ``` +/// +/// Now as example we benchmark `pallet_balances`: +/// +/// ```sh +/// frame-omni-bencher v1 benchmark pallet \ +/// --runtime target/release/wbuild/westend-runtime/westend-runtime.compact.compressed.wasm \ +/// --pallet "pallet_balances" --extrinsic "" +/// ``` +/// +/// For the exact arguments of the `pallet` command, please refer to the `pallet` sub-module. +/// +/// ## Backwards Compatibility +/// +/// The exposed pallet sub-command is identical as the node-integrated CLI. The only difference is +/// that it needs to be prefixed with a `v1` to ensure drop-in compatibility. +#[derive(Parser, Debug)] +#[clap(author, version, about, verbatim_doc_comment)] +pub struct Command { + #[command(subcommand)] + sub: SubCommand, +} + +/// Root-level subcommands. +#[derive(Debug, clap::Subcommand)] +pub enum SubCommand { + /// Compatibility syntax with the old benchmark runner. + V1(V1Command), + // NOTE: Here we can add new commands in a forward-compatible way. For example when + // transforming the CLI from a monolithic design to a data driven pipeline, there could be + // commands like `measure`, `analyze` and `render`. +} + +/// A command that conforms to the legacy `benchmark` argument syntax. +#[derive(Parser, Debug)] +pub struct V1Command { + #[command(subcommand)] + sub: V1SubCommand, +} + +/// The `v1 benchmark` subcommand. +#[derive(Debug, clap::Subcommand)] +pub enum V1SubCommand { + Benchmark(V1BenchmarkCommand), +} + +/// Subcommands for `v1 benchmark`. +#[derive(Parser, Debug)] +pub struct V1BenchmarkCommand { + #[command(subcommand)] + sub: BenchmarkCmd, +} + +type HostFunctions = ( + sp_statement_store::runtime_api::HostFunctions, + cumulus_primitives_proof_size_hostfunction::storage_proof_size::HostFunctions, +); + +impl Command { + pub fn run(self) -> Result<()> { + match self.sub { + SubCommand::V1(V1Command { sub }) => sub.run(), + } + } +} + +impl V1SubCommand { + pub fn run(self) -> Result<()> { + let pallet = match self { + V1SubCommand::Benchmark(V1BenchmarkCommand { sub }) => match sub { + BenchmarkCmd::Pallet(pallet) => pallet, + _ => + return Err( + "Only the `v1 benchmark pallet` command is currently supported".into() + ), + }, + }; + + if let Some(spec) = pallet.shared_params.chain { + return Err(format!( + "Chain specs are not supported. Please remove `--chain={spec}` and use \ + `--runtime=` instead" + ) + .into()) + } + + pallet.run_with_spec::(None) + } +} diff --git a/substrate/utils/frame/omni-bencher/src/main.rs b/substrate/utils/frame/omni-bencher/src/main.rs new file mode 100644 index 000000000000..c148403f2970 --- /dev/null +++ b/substrate/utils/frame/omni-bencher/src/main.rs @@ -0,0 +1,29 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod command; + +use clap::Parser; +use env_logger::Env; +use sc_cli::Result; + +fn main() -> Result<()> { + env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); + log::warn!("The FRAME omni-bencher is not yet battle tested - double check the results.",); + + command::Command::parse().run() +} diff --git a/templates/parachain/node/src/command.rs b/templates/parachain/node/src/command.rs index 82624ae0be59..4c36fae30a26 100644 --- a/templates/parachain/node/src/command.rs +++ b/templates/parachain/node/src/command.rs @@ -184,7 +184,7 @@ pub fn run() -> Result<()> { match cmd { BenchmarkCmd::Pallet(cmd) => if cfg!(feature = "runtime-benchmarks") { - runner.sync_run(|config| cmd.run::, ReclaimHostFunctions>(config)) + runner.sync_run(|config| cmd.run_with_spec::, ReclaimHostFunctions>(Some(config.chain_spec))) } else { Err("Benchmarking wasn't enabled when building the node. \ You can enable it with `--features runtime-benchmarks`." diff --git a/templates/solochain/node/src/command.rs b/templates/solochain/node/src/command.rs index 42d1477f22f1..1b831f9cbbfd 100644 --- a/templates/solochain/node/src/command.rs +++ b/templates/solochain/node/src/command.rs @@ -117,7 +117,9 @@ pub fn run() -> sc_cli::Result<()> { ) } - cmd.run::, ()>(config) + cmd.run_with_spec::, ()>(Some( + config.chain_spec, + )) }, BenchmarkCmd::Block(cmd) => { let PartialComponents { client, .. } = service::new_partial(&config)?;