Add execution overhead benchmarking

bin/node/cli/src/cli.rs

@@ -42,6 +42,13 @@ pub enum Subcommand {
 	#[clap(name = "benchmark", about = "Benchmark runtime pallets.")]
 	Benchmark(frame_benchmarking_cli::BenchmarkCmd),
 
+	/// Sub command for benchmarking the per-block and per-extrinsic execution overhead.
+	#[clap(
+		name = "benchmark-overhead",
+		about = "Benchmark the per-block and per-extrinsic execution overhead."
+	)]
+	BenchmarkOverhead(frame_benchmarking_cli::OverheadCmd),
+
 	/// Sub command for benchmarking the storage speed.
 	#[clap(name = "benchmark-storage", about = "Benchmark storage speed.")]
 	BenchmarkStorage(frame_benchmarking_cli::StorageCmd),

bin/node/cli/src/command.rs

@@ -16,11 +16,21 @@
 // You should have received a copy of the GNU General Public License
 // along with this program. If not, see <https://www.gnu.org/licenses/>.
 
-use crate::{chain_spec, service, service::new_partial, Cli, Subcommand};
+use crate::{
+	chain_spec, service,
+	service::{create_extrinsic, new_partial, FullClient},
+	Cli, Subcommand,
+};
 use node_executor::ExecutorDispatch;
-use node_runtime::{Block, RuntimeApi};
+use node_primitives::Block;
+use node_runtime::{RuntimeApi, SystemCall};
 use sc_cli::{ChainSpec, Result, RuntimeVersion, SubstrateCli};
 use sc_service::PartialComponents;
+use sp_inherents::{InherentData, InherentDataProvider};
+use sp_keyring::Sr25519Keyring;
+use sp_runtime::OpaqueExtrinsic;
+
+use std::{sync::Arc, time::Duration};
 
 impl SubstrateCli for Cli {
 	fn impl_name() -> String {
@@ -69,6 +79,39 @@ impl SubstrateCli for Cli {
 	}
 }
 
+/// Generates extrinsics for the `overhead` benchmark.
+/// TODO I will move this into a different file once we agreed on the structure.
+struct ExtrinsicBuilder {
+	client: Arc<FullClient>,
+}
+
+impl frame_benchmarking_cli::ExtrinsicBuilder for ExtrinsicBuilder {
+	fn remark(&self, nonce: u32) -> Option<OpaqueExtrinsic> {
+		let acc = Sr25519Keyring::Bob.pair();
+		let extrinsic: OpaqueExtrinsic = create_extrinsic(
+			self.client.as_ref(),
+			acc,
+			SystemCall::remark { remark: vec![] },
+			Some(nonce),
+		)
+		.into();
+
+		Some(extrinsic)
+	}
+}
+
+/// Create the inherent data needed to build a block.
+fn inherent_data() -> Result<InherentData> {
+	let mut inherent_data = InherentData::new();
+	let d = Duration::from_millis(0);
+	let timestamp = sp_timestamp::InherentDataProvider::new(d.into());
+
+	timestamp
+		.provide_inherent_data(&mut inherent_data)
+		.map_err(|e| format!("creating inherent data: {:?}", e))?;
+	Ok(inherent_data)
+}
+
 /// Parse command line arguments into service configuration.
 pub fn run() -> Result<()> {
 	let cli = Cli::from_args();
@@ -95,6 +138,19 @@ pub fn run() -> Result<()> {
 				You can enable it with `--features runtime-benchmarks`."
 					.into())
 			},
+		Some(Subcommand::BenchmarkOverhead(cmd)) => {
+			let runner = cli.create_runner(cmd)?;
+			runner.async_run(|mut config| {
+				// TODO 1. is this needed?
+				//      2. should other stuff be disabled as well?
+				config.role = sc_service::Role::Full;
+
+				let PartialComponents { client, task_manager, .. } = new_partial(&config)?;
+				let ext_builder = ExtrinsicBuilder { client: client.clone() };
+
+				Ok((cmd.run(config, client, inherent_data()?, Arc::new(ext_builder)), task_manager))
+			})
+		},
 		Some(Subcommand::BenchmarkStorage(cmd)) => {
 			if !cfg!(feature = "runtime-benchmarks") {
 				return Err("Benchmarking wasn't enabled when building the node. \

utils/frame/benchmarking-cli/Cargo.toml

@@ -16,6 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"]
 frame-benchmarking = { version = "4.0.0-dev", path = "../../../frame/benchmarking" }
 frame-support = { version = "4.0.0-dev", path = "../../../frame/support" }
 sp-core = { version = "6.0.0", path = "../../../primitives/core" }
+sc-block-builder = { version = "0.10.0-dev", path = "../../../client/block-builder" }
 sc-service = { version = "0.10.0-dev", default-features = false, path = "../../../client/service" }
 sc-client-api = { version = "4.0.0-dev", path = "../../../client/api" }
 sc-cli = { version = "0.10.0-dev", path = "../../../client/cli" }
@@ -26,6 +27,7 @@ sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" }
 sp-externalities = { version = "0.12.0", path = "../../../primitives/externalities" }
 sp-database = { version = "4.0.0-dev", path = "../../../primitives/database" }
 sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" }
+sp-inherents = { version = "4.0.0-dev", path = "../../../primitives/inherents" }
 sp-keystore = { version = "0.12.0", path = "../../../primitives/keystore" }
 sp-storage = { version = "6.0.0", path = "../../../primitives/storage" }
 sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" }

utils/frame/benchmarking-cli/src/lib.rs

@@ -16,12 +16,15 @@
 // limitations under the License.
 
 mod command;
+pub mod overhead;
+mod post_processing;
 mod storage;
 mod writer;
 
 use sc_cli::{ExecutionStrategy, WasmExecutionMethod, DEFAULT_WASM_EXECUTION_METHOD};
 use std::{fmt::Debug, path::PathBuf};
 
+pub use overhead::{ExtrinsicBuilder, OverheadCmd};
 pub use storage::StorageCmd;
 
 // Add a more relaxed parsing for pallet names by allowing pallet directory names with `-` to be

utils/frame/benchmarking-cli/src/overhead/bench.rs

@@ -0,0 +1,196 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Contains the core benchmarking logic.
+
+use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider};
+use sc_cli::{Error, Result};
+use sc_client_api::Backend as ClientBackend;
+use sp_api::{ApiExt, BlockId, Core, ProvideRuntimeApi};
+use sp_blockchain::{
+	ApplyExtrinsicFailed::Validity,
+	Error::{ApplyExtrinsicFailed, RuntimeApiError},
+};
+use sp_runtime::{
+	traits::{Block as BlockT, Zero},
+	transaction_validity::{InvalidTransaction, TransactionValidityError},
+	OpaqueExtrinsic,
+};
+
+use clap::Args;
+use log::info;
+use serde::Serialize;
+use std::{marker::PhantomData, sync::Arc, time::Instant};
+
+use crate::{overhead::cmd::ExtrinsicBuilder, storage::record::Stats};
+
+/// Parameters to configure a *overhead* benchmark.
+#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)]
+pub struct BenchmarkParams {
+	/// Rounds of warmups before measuring.
+	#[clap(long, default_value = "100")]
+	pub warmup: u32,
+
+	/// How many times the benchmark should be repeated.
+	#[clap(long, default_value = "1000")]
+	pub repeat: u32,
+}
+
+/// The results of multiple runs in ns.
+pub(crate) type BenchRecord = Vec<u64>;
+
+/// Type of a benchmark.
+#[derive(Serialize, Clone, PartialEq)]
+pub(crate) enum BenchmarkType {
+	/// Measure the per-extrinsic execution overhead.
+	Extrinsic,
+	/// Measure the per-block execution overhead.
+	Block,
+}
+
+/// Holds all objects needed to run the *overhead* benchmarks.
+pub(crate) struct Benchmark<Block, BA, C> {
+	client: Arc<C>,
+	params: BenchmarkParams,
+	inherent_data: sp_inherents::InherentData,
+	ext_builder: Arc<dyn ExtrinsicBuilder>,
+	_p: PhantomData<(Block, BA)>,
+}
+
+impl<Block, BA, C> Benchmark<Block, BA, C>
+where
+	Block: BlockT<Extrinsic = OpaqueExtrinsic>,
+	BA: ClientBackend<Block>,
+	C: BlockBuilderProvider<BA, Block, C> + ProvideRuntimeApi<Block>,
+	C::Api: ApiExt<Block, StateBackend = BA::State> + BlockBuilderApi<Block>,
+{
+	/// Create a new [`Self`] from the arguments.
+	pub fn new(
+		client: Arc<C>,
+		params: BenchmarkParams,
+		inherent_data: sp_inherents::InherentData,
+		ext_builder: Arc<dyn ExtrinsicBuilder>,
+	) -> Self {
+		Self { client, params, inherent_data, ext_builder, _p: PhantomData }
+	}
+
+	/// Run the specified benchmark.
+	pub fn bench(&self, bench_type: BenchmarkType) -> Result<Stats> {
+		let (block, num_ext) = self.build_block(bench_type.clone())?;
+		let record = self.measure_block(&block, num_ext, bench_type)?;
+		Stats::new(&record)
+	}
+
+	/// Builds a block for the given benchmark type.
+	///
+	/// Returns the block and the number of extrinsics in the block.
+	fn build_block(&self, bench_type: BenchmarkType) -> Result<(Block, u64)> {
+		let mut builder = self.client.new_block(Default::default())?;
+		// Create and insert the inherents.
+		let inherents = builder.create_inherents(self.inherent_data.clone())?;
+		for inherent in inherents {
+			builder.push(inherent)?;
+		}
+
+		// Return early if we just want an empty block.
+		if bench_type == BenchmarkType::Block {
+			return Ok((builder.build()?.block, 0))
+		}
+
+		// Put as many extrinsics into the block as possible and count them.
+		info!("Building block, this takes some time...");
+		let mut num_ext = 0;
+		for nonce in 0.. {
+			let ext = self.ext_builder.remark(nonce).ok_or("Could not build extrinsic")?;
+			match builder.push(ext.clone()) {
+				Ok(_) => {},
+				Err(ApplyExtrinsicFailed(Validity(TransactionValidityError::Invalid(
+					InvalidTransaction::ExhaustsResources,
+				)))) => break,
+				Err(error) => panic!("{}", error),
+			}
+			num_ext += 1;
+		}
+		assert!(num_ext != 0, "A Block must hold at least one extrinsic");
+		info!("Remarks per block: {}", num_ext);
+		let block = builder.build()?.block;
+
+		Ok((block, num_ext))
+	}
+
+	/// Measures the time that it take to execute a block or an extrinsic.
+	fn measure_block(
+		&self,
+		block: &Block,
+		num_ext: u64,
+		bench_type: BenchmarkType,
+	) -> Result<BenchRecord> {
+		let mut record = BenchRecord::new();
+		if bench_type == BenchmarkType::Extrinsic && num_ext == 0 {
+			return Err("Cannot measure the extrinsic time of an empty block".into())
+		}
+		let genesis = BlockId::Number(Zero::zero());
+
+		info!("Running {} warmups...", self.params.warmup);
+		for _ in 0..self.params.warmup {
+			self.client
+				.runtime_api()
+				.execute_block(&genesis, block.clone())
+				.map_err(|e| Error::Client(RuntimeApiError(e)))?;
+		}
+
+		info!("Executing block {} times", self.params.repeat);
+		// Interesting part here:
+		// Execute a block multiple times and record each execution time.
+		for _ in 0..self.params.repeat {
+			let block = block.clone();
+			let start = Instant::now();
+			self.client
+				.runtime_api()
+				.execute_block(&genesis, block)
+				.map_err(|e| Error::Client(RuntimeApiError(e)))?;
+
+			let elapsed = start.elapsed().as_nanos();
+			if bench_type == BenchmarkType::Extrinsic {
+				// Checked for non-zero div above.
+				record.push(elapsed as u64 / num_ext);
+			} else {
+				record.push(elapsed as u64);
+			}
+		}
+
+		Ok(record)
+	}
+}
+
+impl BenchmarkType {
+	/// Short name of the benchmark type.
+	pub(crate) fn short_name(&self) -> &'static str {
+		match self {
+			Self::Extrinsic => "extrinsic",
+			Self::Block => "block",
+		}
+	}
+
+	/// Long name of the benchmark type.
+	pub(crate) fn long_name(&self) -> &'static str {
+		match self {
+			Self::Extrinsic => "ExtrinsicBase",
+			Self::Block => "BlockExecution",
+		}
+	}
+}