Runtime State Test + Integration with try-runtime (#10174)

* add missing version to dependencies

* Huh

* add features more

* more fixing

* last touches

* it all finally works

* remove some feature gates

* remove unused

* fix old macro

* make it work again

* fmt

* remove unused import

* ".git/.scripts/fmt.sh" 1

* Cleanup more

* fix and rename everything

* a few clippy fixes

* Add try-runtime feature

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* small fixes

* fmt

* Update bin/node-template/runtime/src/lib.rs

* fix build

* Update utils/frame/try-runtime/cli/src/lib.rs

Co-authored-by: David <dvdplm@gmail.com>

* Update utils/frame/try-runtime/cli/src/commands/execute_block.rs

Co-authored-by: David <dvdplm@gmail.com>

* address all review comments

* fix typos

* revert spec change

* last touches

* update docs

* fmt

* remove some debug_assertions

* fmt

Signed-off-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>
Co-authored-by: command-bot <>
Co-authored-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>
Co-authored-by: David <dvdplm@gmail.com>

bin/node-template/runtime/Cargo.toml

@@ -63,6 +63,7 @@ std = [
 	"frame-support/std",
 	"frame-system-rpc-runtime-api/std",
 	"frame-system/std",
+	"frame-try-runtime/std",
 	"pallet-aura/std",
 	"pallet-balances/std",
 	"pallet-grandpa/std",
@@ -97,9 +98,10 @@ runtime-benchmarks = [
 	"sp-runtime/runtime-benchmarks",
 ]
 try-runtime = [
-	"frame-executive/try-runtime",
 	"frame-try-runtime",
+	"frame-executive/try-runtime",
 	"frame-system/try-runtime",
+	"frame-support/try-runtime",
 	"pallet-aura/try-runtime",
 	"pallet-balances/try-runtime",
 	"pallet-grandpa/try-runtime",

bin/node-template/runtime/src/lib.rs

@@ -545,8 +545,14 @@ impl_runtime_apis! {
 			(weight, BlockWeights::get().max_block)
 		}
 
-		fn execute_block_no_check(block: Block) -> Weight {
-			Executive::execute_block_no_check(block)
+		fn execute_block(
+			block: Block,
+			state_root_check: bool,
+			select: frame_try_runtime::TryStateSelect
+		) -> Weight {
+			// NOTE: intentional unwrap: we don't want to propagate the error backwards, and want to
+			// have a backtrace here.
+			Executive::try_execute_block(block, state_root_check, select).expect("execute-block failed")
 		}
 	}
 }

bin/node/runtime/Cargo.toml

@@ -245,14 +245,16 @@ runtime-benchmarks = [
 	"hex-literal",
 ]
 try-runtime = [
-	"frame-executive/try-runtime",
 	"frame-try-runtime",
+	"frame-executive/try-runtime",
 	"frame-system/try-runtime",
+	"frame-support/try-runtime",
 	"pallet-alliance/try-runtime",
 	"pallet-assets/try-runtime",
 	"pallet-authority-discovery/try-runtime",
 	"pallet-authorship/try-runtime",
 	"pallet-babe/try-runtime",
+	"pallet-bags-list/try-runtime",
 	"pallet-balances/try-runtime",
 	"pallet-bounties/try-runtime",
 	"pallet-child-bounties/try-runtime",
@@ -264,32 +266,36 @@ try-runtime = [
 	"pallet-elections-phragmen/try-runtime",
 	"pallet-gilt/try-runtime",
 	"pallet-grandpa/try-runtime",
-	"pallet-identity/try-runtime",
 	"pallet-im-online/try-runtime",
 	"pallet-indices/try-runtime",
+	"pallet-identity/try-runtime",
 	"pallet-lottery/try-runtime",
 	"pallet-membership/try-runtime",
 	"pallet-mmr/try-runtime",
 	"pallet-multisig/try-runtime",
+	"pallet-nomination-pools/try-runtime",
 	"pallet-offences/try-runtime",
 	"pallet-preimage/try-runtime",
 	"pallet-proxy/try-runtime",
-	"pallet-ranked-collective/try-runtime",
 	"pallet-randomness-collective-flip/try-runtime",
+	"pallet-ranked-collective/try-runtime",
 	"pallet-recovery/try-runtime",
 	"pallet-referenda/try-runtime",
-	"pallet-scheduler/try-runtime",
+	"pallet-remark/try-runtime",
 	"pallet-session/try-runtime",
-	"pallet-society/try-runtime",
 	"pallet-staking/try-runtime",
 	"pallet-state-trie-migration/try-runtime",
+	"pallet-scheduler/try-runtime",
+	"pallet-society/try-runtime",
 	"pallet-sudo/try-runtime",
 	"pallet-timestamp/try-runtime",
 	"pallet-tips/try-runtime",
-	"pallet-transaction-payment/try-runtime",
 	"pallet-treasury/try-runtime",
-	"pallet-uniques/try-runtime",
 	"pallet-utility/try-runtime",
+	"pallet-transaction-payment/try-runtime",
+	"pallet-asset-tx-payment/try-runtime",
+	"pallet-transaction-storage/try-runtime",
+	"pallet-uniques/try-runtime",
 	"pallet-vesting/try-runtime",
 	"pallet-whitelist/try-runtime",
 ]

bin/node/runtime/src/lib.rs

@@ -2079,8 +2079,21 @@ impl_runtime_apis! {
 			(weight, RuntimeBlockWeights::get().max_block)
 		}
 
-		fn execute_block_no_check(block: Block) -> Weight {
-			Executive::execute_block_no_check(block)
+		fn execute_block(
+			block: Block,
+			state_root_check: bool,
+			select: frame_try_runtime::TryStateSelect
+		) -> Weight {
+			log::info!(
+				target: "node-runtime",
+				"try-runtime: executing block {:?} / root checks: {:?} / try-state-select: {:?}",
+				block.header.hash(),
+				state_root_check,
+				select,
+			);
+			// NOTE: intentional unwrap: we don't want to propagate the error backwards, and want to
+			// have a backtrace here.
+			Executive::try_execute_block(block, state_root_check, select).unwrap()
 		}
 	}
 

frame/bags-list/fuzzer/src/main.rs

@@ -88,7 +88,7 @@ fn main() {
 				},
 			}
 
-			assert!(BagsList::sanity_check().is_ok());
+			assert!(BagsList::try_state().is_ok());
 		})
 	});
 }

frame/bags-list/remote-tests/src/lib.rs

@@ -24,8 +24,8 @@ use sp_std::prelude::*;
 pub const LOG_TARGET: &str = "runtime::bags-list::remote-tests";
 
 pub mod migration;
-pub mod sanity_check;
 pub mod snapshot;
+pub mod try_state;
 
 /// A wrapper for a runtime that the functions of this crate expect.
 ///

frame/bags-list/remote-tests/src/sanity_check.rs → frame/bags-list/remote-tests/src/try_state.rs

@@ -44,7 +44,7 @@ pub async fn execute<Runtime: crate::RuntimeT, Block: BlockT + DeserializeOwned>
 
 	ext.execute_with(|| {
 		sp_core::crypto::set_default_ss58_version(Runtime::SS58Prefix::get().try_into().unwrap());
-		pallet_bags_list::Pallet::<Runtime>::sanity_check().unwrap();
+		pallet_bags_list::Pallet::<Runtime>::try_state().unwrap();
 		log::info!(target: crate::LOG_TARGET, "executed bags-list sanity check with no errors.");
 
 		crate::display_and_check_bags::<Runtime>(currency_unit, currency_name);

frame/bags-list/src/lib.rs

@@ -263,6 +263,11 @@ pub mod pallet {
 				"thresholds must strictly increase, and have no duplicates",
 			);
 		}
+
+		#[cfg(feature = "try-runtime")]
+		fn try_state(_: BlockNumberFor<T>) -> Result<(), &'static str> {
+			<Self as SortedListProvider<T::AccountId>>::try_state()
+		}
 	}
 }
 
@@ -340,14 +345,8 @@ impl<T: Config<I>, I: 'static> SortedListProvider<T::AccountId> for Pallet<T, I>
 		List::<T, I>::unsafe_regenerate(all, score_of)
 	}
 
-	#[cfg(feature = "std")]
-	fn sanity_check() -> Result<(), &'static str> {
-		List::<T, I>::sanity_check()
-	}
-
-	#[cfg(not(feature = "std"))]
-	fn sanity_check() -> Result<(), &'static str> {
-		Ok(())
+	fn try_state() -> Result<(), &'static str> {
+		List::<T, I>::try_state()
 	}
 
 	fn unsafe_clear() {

frame/bags-list/src/list/mod.rs

@@ -28,8 +28,8 @@ use crate::Config;
 use codec::{Decode, Encode, MaxEncodedLen};
 use frame_election_provider_support::ScoreProvider;
 use frame_support::{
-	ensure,
-	traits::{Defensive, Get},
+	defensive, ensure,
+	traits::{Defensive, DefensiveOption, Get},
 	DefaultNoBound, PalletError,
 };
 use scale_info::TypeInfo;
@@ -220,7 +220,8 @@ impl<T: Config<I>, I: 'static> List<T, I> {
 			crate::ListBags::<T, I>::remove(removed_bag);
 		}
 
-		debug_assert_eq!(Self::sanity_check(), Ok(()));
+		#[cfg(feature = "std")]
+		debug_assert_eq!(Self::try_state(), Ok(()));
 
 		num_affected
 	}
@@ -325,8 +326,7 @@ impl<T: Config<I>, I: 'static> List<T, I> {
 
 		crate::log!(
 			debug,
-			"inserted {:?} with score {:?
-			} into bag {:?}, new count is {}",
+			"inserted {:?} with score {:?} into bag {:?}, new count is {}",
 			id,
 			score,
 			bag_score,
@@ -457,11 +457,8 @@ impl<T: Config<I>, I: 'static> List<T, I> {
 
 		// re-fetch `lighter_node` from storage since it may have been updated when `heavier_node`
 		// was removed.
-		let lighter_node = Node::<T, I>::get(lighter_id).ok_or_else(|| {
-			debug_assert!(false, "id that should exist cannot be found");
-			crate::log!(warn, "id that should exist cannot be found");
-			ListError::NodeNotFound
-		})?;
+		let lighter_node =
+			Node::<T, I>::get(lighter_id).defensive_ok_or_else(|| ListError::NodeNotFound)?;
 
 		// insert `heavier_node` directly in front of `lighter_node`. This will update both nodes
 		// in storage and update the node counter.
@@ -508,7 +505,7 @@ impl<T: Config<I>, I: 'static> List<T, I> {
 		node.put();
 	}
 
-	/// Sanity check the list.
+	/// Check the internal state of the list.
 	///
 	/// This should be called from the call-site, whenever one of the mutating apis (e.g. `insert`)
 	/// is being used, after all other staking data (such as counter) has been updated. It checks:
@@ -517,8 +514,7 @@ impl<T: Config<I>, I: 'static> List<T, I> {
 	/// * length of this list is in sync with `ListNodes::count()`,
 	/// * and sanity-checks all bags and nodes. This will cascade down all the checks and makes sure
 	/// all bags and nodes are checked per *any* update to `List`.
-	#[cfg(feature = "std")]
-	pub(crate) fn sanity_check() -> Result<(), &'static str> {
+	pub(crate) fn try_state() -> Result<(), &'static str> {
 		let mut seen_in_list = BTreeSet::new();
 		ensure!(
 			Self::iter().map(|node| node.id).all(|id| seen_in_list.insert(id)),
@@ -546,7 +542,7 @@ impl<T: Config<I>, I: 'static> List<T, I> {
 			thresholds.into_iter().filter_map(|t| Bag::<T, I>::get(t))
 		};
 
-		let _ = active_bags.clone().try_for_each(|b| b.sanity_check())?;
+		let _ = active_bags.clone().try_for_each(|b| b.try_state())?;
 
 		let nodes_in_bags_count =
 			active_bags.clone().fold(0u32, |acc, cur| acc + cur.iter().count() as u32);
@@ -557,17 +553,12 @@ impl<T: Config<I>, I: 'static> List<T, I> {
 		// check that all nodes are sane. We check the `ListNodes` storage item directly in case we
 		// have some "stale" nodes that are not in a bag.
 		for (_id, node) in crate::ListNodes::<T, I>::iter() {
-			node.sanity_check()?
+			node.try_state()?
 		}
 
 		Ok(())
 	}
 
-	#[cfg(not(feature = "std"))]
-	pub(crate) fn sanity_check() -> Result<(), &'static str> {
-		Ok(())
-	}
-
 	/// Returns the nodes of all non-empty bags. For testing and benchmarks.
 	#[cfg(any(feature = "std", feature = "runtime-benchmarks"))]
 	#[allow(dead_code)]
@@ -701,8 +692,7 @@ impl<T: Config<I>, I: 'static> Bag<T, I> {
 			if *tail == node.id {
 				// this should never happen, but this check prevents one path to a worst case
 				// infinite loop.
-				debug_assert!(false, "system logic error: inserting a node who has the id of tail");
-				crate::log!(warn, "system logic error: inserting a node who has the id of tail");
+				defensive!("system logic error: inserting a node who has the id of tail");
 				return
 			};
 		}
@@ -753,16 +743,15 @@ impl<T: Config<I>, I: 'static> Bag<T, I> {
 		}
 	}
 
-	/// Sanity check this bag.
+	/// Check the internal state of the bag.
 	///
 	/// Should be called by the call-site, after any mutating operation on a bag. The call site of
 	/// this struct is always `List`.
 	///
 	/// * Ensures head has no prev.
 	/// * Ensures tail has no next.
 	/// * Ensures there are no loops, traversal from head to tail is correct.
-	#[cfg(feature = "std")]
-	fn sanity_check(&self) -> Result<(), &'static str> {
+	fn try_state(&self) -> Result<(), &'static str> {
 		frame_support::ensure!(
 			self.head()
 				.map(|head| head.prev().is_none())
@@ -801,7 +790,6 @@ impl<T: Config<I>, I: 'static> Bag<T, I> {
 	}
 
 	/// Check if the bag contains a node with `id`.
-	#[cfg(feature = "std")]
 	fn contains(&self, id: &T::AccountId) -> bool {
 		self.iter().any(|n| n.id() == id)
 	}
@@ -906,8 +894,7 @@ impl<T: Config<I>, I: 'static> Node<T, I> {
 		self.bag_upper
 	}
 
-	#[cfg(feature = "std")]
-	fn sanity_check(&self) -> Result<(), &'static str> {
+	fn try_state(&self) -> Result<(), &'static str> {
 		let expected_bag = Bag::<T, I>::get(self.bag_upper).ok_or("bag not found for node")?;
 
 		let id = self.id();