frame-omni-bencher fix for --repeat > 1

.github/workflows/tests.yml

@@ -46,7 +46,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: script
-        run: time forklift cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks -- benchmark pallet --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 --quiet
+        run: time forklift cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks -- benchmark pallet --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 2 --quiet
 
   # cf https://github.com/paritytech/polkadot-sdk/issues/1652
   test-syscalls:

.gitlab/pipeline/short-benchmarks.yml

@@ -24,7 +24,7 @@ short-benchmark-westend: &short-bench
   tags:
     - benchmark
   script:
-    - ./artifacts/polkadot benchmark pallet --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1
+    - ./artifacts/polkadot benchmark pallet --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 2 --repeat 2
 
 # run short-benchmarks for system parachain runtimes from cumulus
 
@@ -47,7 +47,7 @@ short-benchmark-westend: &short-bench
   tags:
     - benchmark
   script:
-    - ./artifacts/polkadot-parachain benchmark pallet --chain $RUNTIME_CHAIN --pallet "*" --extrinsic "*" --steps 2 --repeat 1
+    - ./artifacts/polkadot-parachain benchmark pallet --chain $RUNTIME_CHAIN --pallet "*" --extrinsic "*" --steps 2 --repeat 2
 
 short-benchmark-asset-hub-rococo:
   <<: *short-bench-cumulus

.gitlab/pipeline/test.yml

@@ -418,7 +418,7 @@ quick-benchmarks:
     WASM_BUILD_NO_COLOR: 1
     WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
   script:
-    - time cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks --quiet -- benchmark pallet --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 --quiet
+    - time cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks --quiet -- benchmark pallet --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 2 --quiet
 
 quick-benchmarks-omni:
   stage: test
@@ -435,7 +435,79 @@ quick-benchmarks-omni:
     WASM_BUILD_RUSTFLAGS: "-C debug-assertions"
   script:
     - time cargo build --locked --quiet --release -p asset-hub-westend-runtime --features runtime-benchmarks
-    - time cargo run --locked --release -p frame-omni-bencher --quiet -- v1 benchmark pallet --runtime target/release/wbuild/asset-hub-westend-runtime/asset_hub_westend_runtime.compact.compressed.wasm --all --steps 2 --repeat 1 --quiet
+    - time cargo run --locked --release -p frame-omni-bencher --quiet -- v1 benchmark pallet --runtime target/release/wbuild/asset-hub-westend-runtime/asset_hub_westend_runtime.compact.compressed.wasm --all --steps 2 --repeat 2 --quiet
+
+# FAIL-CI - remove when PR is fixed
+quick-benchmarks-omni-repeat-1:
+  stage: test
+  extends:
+    - .docker-env
+    - .common-refs
+    - .run-immediately
+  variables:
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS: "-C debug-assertions"
+    RUST_BACKTRACE: "full"
+    WASM_BUILD_NO_COLOR: 1
+    WASM_BUILD_RUSTFLAGS: "-C debug-assertions"
+  script:
+    - time cargo build --locked --quiet --release -p asset-hub-rococo-runtime --features runtime-benchmarks
+    - time cargo run --locked --release -p frame-omni-bencher --quiet -- v1 benchmark pallet --runtime target/release/wbuild/asset-hub-rococo-runtime/asset_hub_rococo_runtime.compact.compressed.wasm --all --steps 2 --repeat 1
+
+# FAIL-CI - remove when PR is fixed
+quick-benchmarks-omni-repeat-2:
+  stage: test
+  extends:
+    - .docker-env
+    - .common-refs
+    - .run-immediately
+  variables:
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS: "-C debug-assertions"
+    RUST_BACKTRACE: "full"
+    WASM_BUILD_NO_COLOR: 1
+    WASM_BUILD_RUSTFLAGS: "-C debug-assertions"
+  script:
+    - time cargo build --locked --quiet --release -p asset-hub-rococo-runtime --features runtime-benchmarks
+    - time cargo run --locked --release -p frame-omni-bencher --quiet -- v1 benchmark pallet --runtime target/release/wbuild/asset-hub-rococo-runtime/asset_hub_rococo_runtime.compact.compressed.wasm --all --steps 2 --repeat 2
+
+# FAIL-CI - remove when PR is fixed
+quick-benchmarks-omni-repeat-1-leave-intent:
+  stage: test
+  extends:
+    - .docker-env
+    - .common-refs
+    - .run-immediately
+  variables:
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS: "-C debug-assertions"
+    RUST_BACKTRACE: "full"
+    WASM_BUILD_NO_COLOR: 1
+    WASM_BUILD_RUSTFLAGS: "-C debug-assertions"
+  script:
+    - time cargo build --locked --quiet --release -p asset-hub-rococo-runtime --features runtime-benchmarks
+    - time cargo run --locked --release -p frame-omni-bencher --quiet -- v1 benchmark pallet --runtime target/release/wbuild/asset-hub-rococo-runtime/asset_hub_rococo_runtime.compact.compressed.wasm --pallet=pallet_collator_selection  --extrinsic=leave_intent --steps 2 --repeat 1
+
+# FAIL-CI - remove when PR is fixed
+quick-benchmarks-omni-repeat-2-leave-intent:
+  stage: test
+  extends:
+    - .docker-env
+    - .common-refs
+    - .run-immediately
+  variables:
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS: "-C debug-assertions"
+    RUST_BACKTRACE: "full"
+    WASM_BUILD_NO_COLOR: 1
+    WASM_BUILD_RUSTFLAGS: "-C debug-assertions"
+  script:
+    - time cargo build --locked --quiet --release -p asset-hub-rococo-runtime --features runtime-benchmarks
+    - time cargo run --locked --release -p frame-omni-bencher --quiet -- v1 benchmark pallet --runtime target/release/wbuild/asset-hub-rococo-runtime/asset_hub_rococo_runtime.compact.compressed.wasm --pallet=pallet_collator_selection  --extrinsic=leave_intent --steps 2 --repeat 2
 
 test-frame-examples-compile-to-wasm:
   # into one job

cumulus/pallets/collator-selection/src/benchmarking.rs

@@ -342,6 +342,8 @@ mod benchmarks {
 	// worse case is the last candidate leaving.
 	#[benchmark]
 	fn leave_intent(c: Linear<{ min_candidates::<T>() + 1 }, { T::MaxCandidates::get() }>) {
+		log::error!(target: "bencher::FAIL-CI", "leave_intent - eligible_collators {:?}", <CollatorSelection<T>>::eligible_collators());
+
 		CandidacyBond::<T>::put(T::Currency::minimum_balance());
 		DesiredCandidates::<T>::put(c);
 

cumulus/pallets/collator-selection/src/lib.rs

@@ -243,6 +243,8 @@ pub mod pallet {
 	#[pallet::genesis_build]
 	impl<T: Config> BuildGenesisConfig for GenesisConfig<T> {
 		fn build(&self) {
+			log::error!(target: "bencher::FAIL-CI", "\ninvulnerables: {:?}, \ncandidacy_bond: {:?}, \ndesired_candidates:{:?}", self.invulnerables, self.candidacy_bond, self.desired_candidates);
+
 			let duplicate_invulnerables = self
 				.invulnerables
 				.iter()
@@ -805,7 +807,7 @@ pub mod pallet {
 
 		/// Return the total number of accounts that are eligible collators (candidates and
 		/// invulnerables).
-		fn eligible_collators() -> u32 {
+		pub fn eligible_collators() -> u32 {
 			CandidateList::<T>::decode_len()
 				.unwrap_or_default()
 				.saturating_add(Invulnerables::<T>::decode_len().unwrap_or_default())

cumulus/parachains/runtimes/assets/asset-hub-rococo/Cargo.toml

@@ -14,6 +14,7 @@ codec = { features = ["derive", "max-encoded-len"], workspace = true }
 hex-literal = { workspace = true, default-features = true }
 log = { workspace = true }
 scale-info = { features = ["derive"], workspace = true }
+serde_json = { features = ["alloc"], workspace = true }
 
 # Substrate
 frame-benchmarking = { optional = true, workspace = true }
@@ -53,6 +54,7 @@ sp-genesis-builder = { workspace = true }
 sp-offchain = { workspace = true }
 sp-runtime = { workspace = true }
 sp-session = { workspace = true }
+sp-std = { workspace = true }
 sp-storage = { workspace = true }
 sp-transaction-pool = { workspace = true }
 sp-version = { workspace = true }
@@ -230,6 +232,7 @@ std = [
 	"primitive-types/std",
 	"rococo-runtime-constants/std",
 	"scale-info/std",
+	"serde_json/std",
 	"snowbridge-router-primitives/std",
 	"sp-api/std",
 	"sp-block-builder/std",
@@ -240,6 +243,7 @@ std = [
 	"sp-offchain/std",
 	"sp-runtime/std",
 	"sp-session/std",
+	"sp-std/std",
 	"sp-storage/std",
 	"sp-transaction-pool/std",
 	"sp-version/std",

cumulus/parachains/runtimes/assets/asset-hub-rococo/src/genesis_config_presets.rs

@@ -0,0 +1,89 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Genesis configs presets for the AssetHubRococo runtime
+
+use crate::*;
+use sp_std::vec::Vec;
+use testnet_parachains_constants::{genesis_presets::*, rococo::currency::UNITS as ROC};
+
+const ASSET_HUB_ROCOCO_ED: Balance = ExistentialDeposit::get();
+
+/// Default genesis pallet configurations for AssetHubRococo
+pub fn asset_hub_rococo_genesis(
+	invulnerables: Vec<(AccountId, AuraId)>,
+	endowed_accounts: Vec<AccountId>,
+	endowment: Balance,
+	id: ParaId,
+) -> serde_json::Value {
+	serde_json::json!({
+		"balances": BalancesConfig {
+			balances: endowed_accounts
+				.iter()
+				.cloned()
+				.map(|k| (k, endowment))
+				.collect(),
+		},
+		"parachainInfo": ParachainInfoConfig {
+			parachain_id: id,
+			..Default::default()
+		},
+		"collatorSelection": CollatorSelectionConfig {
+			invulnerables: invulnerables.iter().cloned().map(|(acc, _)| acc).collect(),
+			candidacy_bond: ASSET_HUB_ROCOCO_ED * 16,
+			..Default::default()
+		},
+		"session": SessionConfig {
+			keys: invulnerables
+				.into_iter()
+				.map(|(acc, aura)| {
+					(
+						acc.clone(),                         // account id
+						acc,                                 // validator id
+						SessionKeys { aura }, 			// session keys
+					)
+				})
+				.collect(),
+		},
+		"polkadotXcm": PolkadotXcmConfig {
+			safe_xcm_version: Some(SAFE_XCM_VERSION),
+			..Default::default()
+		}
+	})
+}
+
+/// Default genesis setup for `local_testnet` preset id.
+pub fn asset_hub_rococo_local_testnet_genesis(para_id: ParaId) -> serde_json::Value {
+	asset_hub_rococo_genesis(invulnerables(), testnet_accounts(), ROC * 1_000_000, para_id)
+}
+
+/// Default genesis setup for `development` preset id.
+pub fn asset_hub_rococo_development_genesis(para_id: ParaId) -> serde_json::Value {
+	asset_hub_rococo_local_testnet_genesis(para_id)
+}
+
+/// Provides the JSON representation of predefined genesis config for given `id`.
+pub fn get_preset(id: &sp_genesis_builder::PresetId) -> Option<sp_std::vec::Vec<u8>> {
+	let patch = match id.try_into() {
+		Ok("development") => asset_hub_rococo_development_genesis(1000.into()),
+		Ok("local_testnet") => asset_hub_rococo_local_testnet_genesis(1000.into()),
+		_ => return None,
+	};
+	Some(
+		serde_json::to_string(&patch)
+			.expect("serialization to json is expected to work. qed.")
+			.into_bytes(),
+	)
+}

cumulus/parachains/runtimes/assets/asset-hub-rococo/src/lib.rs

@@ -24,6 +24,8 @@
 #[cfg(feature = "std")]
 include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs"));
 
+// Genesis preset configurations.
+pub mod genesis_config_presets;
 mod weights;
 pub mod xcm_config;
 
@@ -1768,11 +1770,14 @@ impl_runtime_apis! {
 		}
 
 		fn get_preset(id: &Option<sp_genesis_builder::PresetId>) -> Option<Vec<u8>> {
-			get_preset::<RuntimeGenesisConfig>(id, |_| None)
+			get_preset::<RuntimeGenesisConfig>(id, &genesis_config_presets::get_preset)
 		}
 
 		fn preset_names() -> Vec<sp_genesis_builder::PresetId> {
-			vec![]
+			vec![
+				sp_genesis_builder::PresetId::from("local_testnet"),
+				sp_genesis_builder::PresetId::from("development"),
+			]
 		}
 	}
 }

cumulus/parachains/runtimes/constants/Cargo.toml

@@ -18,24 +18,32 @@ smallvec = { workspace = true, default-features = true }
 # Substrate
 frame-support = { workspace = true }
 sp-runtime = { workspace = true }
+sp-std = { workspace = true }
+sp-core = { workspace = true }
 
 # Polkadot
 polkadot-core-primitives = { workspace = true }
 rococo-runtime-constants = { optional = true, workspace = true }
 westend-runtime-constants = { optional = true, workspace = true }
 xcm = { workspace = true }
+polkadot-primitives = { workspace = true }
 
 # Cumulus
+parachains-common = { workspace = true }
 cumulus-primitives-core = { workspace = true }
 
 [features]
 default = ["std"]
 std = [
 	"cumulus-primitives-core/std",
 	"frame-support/std",
+	"parachains-common/std",
 	"polkadot-core-primitives/std",
+	"polkadot-primitives/std",
 	"rococo-runtime-constants?/std",
+	"sp-core/std",
 	"sp-runtime/std",
+	"sp-std/std",
 	"westend-runtime-constants?/std",
 	"xcm/std",
 ]