[Feature] Part 1: add TargetList for validator ranking (paritytech#12034

)

* [Feature] Part 1: add TargetList for validator ranking

* remove redundant todo

* remove typo

* cleanup

* implement score

* more fixes

* fix thresholds

* fmt

* Remove the stuff that has to come in the next PR, some fixes

* extended balance import

* Change all the references from VoteWeight to Self::Score

* Add a migration for VoterBagsList

* fix score

* add targetList to nomination-pools tests

* fix bench

* address review comments

* change get_npos_targets

* address more comments

* remove thresholds for the time being

* fix instance reference

* VoterBagsListInstance

* reus

* remove params that are not used yet

* Introduced pre/post upgrade try-runtime checks

* fix

* fixes

* fix migration

* fix migration

* fix post_upgrade

* change

* Fix

* eloquent PhantomData

* fix PD

* more fixes

* Update frame/staking/src/pallet/impls.rs

Co-authored-by: Squirrel <gilescope@gmail.com>

* is_nominator now works

* fix test-staking

* build fixes

* fix remote-tests

* Apply suggestions from code review

Co-authored-by: parity-processbot <>
Co-authored-by: kianenigma <kian@parity.io>
Co-authored-by: Squirrel <gilescope@gmail.com>
Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com>

bin/node/runtime/src/lib.rs

@@ -568,7 +568,9 @@ impl pallet_staking::Config for Runtime {
 	type OffendingValidatorsThreshold = OffendingValidatorsThreshold;
 	type ElectionProvider = ElectionProviderMultiPhase;
 	type GenesisElectionProvider = onchain::UnboundedExecution<OnChainSeqPhragmen>;
-	type VoterList = BagsList;
+	type VoterList = VoterBagsList;
+	// This a placeholder, to be introduced in the next PR as an instance of bags-list
+	type TargetList = pallet_staking::UseValidatorsMap<Self>;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type OnStakerSlash = NominationPools;
 	type WeightInfo = pallet_staking::weights::SubstrateWeight<Runtime>;
@@ -724,12 +726,15 @@ parameter_types! {
 	pub const BagThresholds: &'static [u64] = &voter_bags::THRESHOLDS;
 }
 
-impl pallet_bags_list::Config for Runtime {
+type VoterBagsListInstance = pallet_bags_list::Instance1;
+impl pallet_bags_list::Config<VoterBagsListInstance> for Runtime {
 	type RuntimeEvent = RuntimeEvent;
+	/// The voter bags-list is loosely kept up to date, and the real source of truth for the score
+	/// of each node is the staking pallet.
 	type ScoreProvider = Staking;
-	type WeightInfo = pallet_bags_list::weights::SubstrateWeight<Runtime>;
 	type BagThresholds = BagThresholds;
 	type Score = VoteWeight;
+	type WeightInfo = pallet_bags_list::weights::SubstrateWeight<Runtime>;
 }
 
 parameter_types! {
@@ -1637,7 +1642,7 @@ construct_runtime!(
 		Gilt: pallet_gilt,
 		Uniques: pallet_uniques,
 		TransactionStorage: pallet_transaction_storage,
-		BagsList: pallet_bags_list,
+		VoterBagsList: pallet_bags_list::<Instance1>,
 		StateTrieMigration: pallet_state_trie_migration,
 		ChildBounties: pallet_child_bounties,
 		Referenda: pallet_referenda,
@@ -1723,7 +1728,7 @@ mod benches {
 		[pallet_alliance, Alliance]
 		[pallet_assets, Assets]
 		[pallet_babe, Babe]
-		[pallet_bags_list, BagsList]
+		[pallet_bags_list, VoterBagsList]
 		[pallet_balances, Balances]
 		[pallet_bounties, Bounties]
 		[pallet_child_bounties, ChildBounties]

bin/node/runtime/src/voter_bags.rs

@@ -1,6 +1,6 @@
 // This file is part of Substrate.
 
-// Copyright (C) 2021-2022 Parity Technologies (UK) Ltd.
+// Copyright (C) 2022 Parity Technologies (UK) Ltd.
 // SPDX-License-Identifier: Apache-2.0
 
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,9 +15,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-//! Autogenerated voter bag thresholds.
+//! Autogenerated bag thresholds.
 //!
-//! Generated on 2021-07-05T09:17:40.469754927+00:00
+//! Generated on 2022-08-15T19:26:59.939787+00:00
+//! Arguments
+//! Total issuance: 100000000000000
+//! Minimum balance: 100000000000000
 //! for the node runtime.
 
 /// Existential weight for this runtime.

frame/babe/src/mock.rs

@@ -202,6 +202,7 @@ impl pallet_staking::Config for Test {
 	type ElectionProvider = onchain::UnboundedExecution<OnChainSeqPhragmen>;
 	type GenesisElectionProvider = Self::ElectionProvider;
 	type VoterList = pallet_staking::UseNominatorsAndValidatorsMap<Self>;
+	type TargetList = pallet_staking::UseValidatorsMap<Self>;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type OnStakerSlash = ();
 	type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig;

frame/bags-list/remote-tests/src/lib.rs

@@ -18,6 +18,7 @@
 //! Utilities for remote-testing pallet-bags-list.
 
 use frame_election_provider_support::ScoreProvider;
+use pallet_bags_list::Instance1;
 use sp_std::prelude::*;
 
 /// A common log target to use.
@@ -30,18 +31,26 @@ pub mod try_state;
 /// A wrapper for a runtime that the functions of this crate expect.
 ///
 /// For example, this can be the `Runtime` type of the Polkadot runtime.
-pub trait RuntimeT:
-	pallet_staking::Config + pallet_bags_list::Config + frame_system::Config
+pub trait RuntimeT<I: 'static>:
+	pallet_staking::Config + pallet_bags_list::Config<I> + frame_system::Config
+{
+}
+impl<
+		I: 'static,
+		T: pallet_staking::Config + pallet_bags_list::Config<I> + frame_system::Config,
+	> RuntimeT<I> for T
 {
 }
-impl<T: pallet_staking::Config + pallet_bags_list::Config + frame_system::Config> RuntimeT for T {}
 
 fn percent(portion: u32, total: u32) -> f64 {
 	(portion as f64 / total as f64) * 100f64
 }
 
 /// Display the number of nodes in each bag, while identifying those that need a rebag.
-pub fn display_and_check_bags<Runtime: RuntimeT>(currency_unit: u64, currency_name: &'static str) {
+pub fn display_and_check_bags<Runtime: RuntimeT<Instance1>>(
+	currency_unit: u64,
+	currency_name: &'static str,
+) {
 	use frame_election_provider_support::SortedListProvider;
 	use frame_support::traits::Get;
 
@@ -55,7 +64,8 @@ pub fn display_and_check_bags<Runtime: RuntimeT>(currency_unit: u64, currency_na
 	let mut seen_in_bags = 0;
 	let mut rebaggable = 0;
 	let mut active_bags = 0;
-	for vote_weight_thresh in <Runtime as pallet_bags_list::Config>::BagThresholds::get() {
+	for vote_weight_thresh in <Runtime as pallet_bags_list::Config<Instance1>>::BagThresholds::get()
+	{
 		let vote_weight_thresh_u64: u64 = (*vote_weight_thresh)
 			.try_into()
 			.map_err(|_| "runtime must configure score to at most u64 to use this test")
@@ -64,7 +74,9 @@ pub fn display_and_check_bags<Runtime: RuntimeT>(currency_unit: u64, currency_na
 		let vote_weight_thresh_as_unit = vote_weight_thresh_u64 as f64 / currency_unit as f64;
 		let pretty_thresh = format!("Threshold: {}. {}", vote_weight_thresh_as_unit, currency_name);
 
-		let bag = match pallet_bags_list::Pallet::<Runtime>::list_bags_get(*vote_weight_thresh) {
+		let bag = match pallet_bags_list::Pallet::<Runtime, Instance1>::list_bags_get(
+			*vote_weight_thresh,
+		) {
 			Some(bag) => bag,
 			None => {
 				log::info!(target: LOG_TARGET, "{} NO VOTERS.", pretty_thresh);
@@ -75,7 +87,8 @@ pub fn display_and_check_bags<Runtime: RuntimeT>(currency_unit: u64, currency_na
 		active_bags += 1;
 
 		for id in bag.std_iter().map(|node| node.std_id().clone()) {
-			let vote_weight = <Runtime as pallet_bags_list::Config>::ScoreProvider::score(&id);
+			let vote_weight =
+				<Runtime as pallet_bags_list::Config<Instance1>>::ScoreProvider::score(&id);
 			let vote_weight_thresh_u64: u64 = (*vote_weight_thresh)
 				.try_into()
 				.map_err(|_| "runtime must configure score to at most u64 to use this test")
@@ -92,8 +105,8 @@ pub fn display_and_check_bags<Runtime: RuntimeT>(currency_unit: u64, currency_na
 				);
 			}
 
-			let node =
-				pallet_bags_list::Node::<Runtime>::get(&id).expect("node in bag must exist.");
+			let node = pallet_bags_list::Node::<Runtime, Instance1>::get(&id)
+				.expect("node in bag must exist.");
 			if node.is_misplaced(vote_weight) {
 				rebaggable += 1;
 				let notional_bag = pallet_bags_list::notional_bag_for::<Runtime, _>(vote_weight);
@@ -141,7 +154,7 @@ pub fn display_and_check_bags<Runtime: RuntimeT>(currency_unit: u64, currency_na
 		"a total of {} nodes are in {} active bags [{} total bags], {} of which can be rebagged.",
 		voter_list_count,
 		active_bags,
-		<Runtime as pallet_bags_list::Config>::BagThresholds::get().len(),
+		<Runtime as pallet_bags_list::Config<Instance1>>::BagThresholds::get().len(),
 		rebaggable,
 	);
 }

frame/bags-list/remote-tests/src/migration.rs

@@ -24,7 +24,10 @@ use sp_runtime::{traits::Block as BlockT, DeserializeOwned};
 
 /// Test voter bags migration. `currency_unit` is the number of planks per the the runtimes `UNITS`
 /// (i.e. number of decimal places per DOT, KSM etc)
-pub async fn execute<Runtime: RuntimeT, Block: BlockT + DeserializeOwned>(
+pub async fn execute<
+	Runtime: RuntimeT<pallet_bags_list::Instance1>,
+	Block: BlockT + DeserializeOwned,
+>(
 	currency_unit: u64,
 	currency_name: &'static str,
 	ws_url: String,

frame/bags-list/remote-tests/src/snapshot.rs

@@ -22,7 +22,10 @@ use remote_externalities::{Builder, Mode, OnlineConfig};
 use sp_runtime::{traits::Block as BlockT, DeserializeOwned};
 
 /// Execute create a snapshot from pallet-staking.
-pub async fn execute<Runtime: crate::RuntimeT, Block: BlockT + DeserializeOwned>(
+pub async fn execute<
+	Runtime: crate::RuntimeT<pallet_bags_list::Instance1>,
+	Block: BlockT + DeserializeOwned,
+>(
 	voter_limit: Option<usize>,
 	currency_unit: u64,
 	ws_url: String,
@@ -34,7 +37,8 @@ pub async fn execute<Runtime: crate::RuntimeT, Block: BlockT + DeserializeOwned>
 			transport: ws_url.to_string().into(),
 			// NOTE: we don't scrape pallet-staking, this kinda ensures that the source of the data
 			// is bags-list.
-			pallets: vec![pallet_bags_list::Pallet::<Runtime>::name().to_string()],
+			pallets: vec![pallet_bags_list::Pallet::<Runtime, pallet_bags_list::Instance1>::name()
+				.to_string()],
 			at: None,
 			..Default::default()
 		}))

frame/bags-list/remote-tests/src/try_state.rs

@@ -25,15 +25,19 @@ use remote_externalities::{Builder, Mode, OnlineConfig};
 use sp_runtime::{traits::Block as BlockT, DeserializeOwned};
 
 /// Execute the sanity check of the bags-list.
-pub async fn execute<Runtime: crate::RuntimeT, Block: BlockT + DeserializeOwned>(
+pub async fn execute<
+	Runtime: crate::RuntimeT<pallet_bags_list::Instance1>,
+	Block: BlockT + DeserializeOwned,
+>(
 	currency_unit: u64,
 	currency_name: &'static str,
 	ws_url: String,
 ) {
 	let mut ext = Builder::<Block>::new()
 		.mode(Mode::Online(OnlineConfig {
 			transport: ws_url.to_string().into(),
-			pallets: vec![pallet_bags_list::Pallet::<Runtime>::name().to_string()],
+			pallets: vec![pallet_bags_list::Pallet::<Runtime, pallet_bags_list::Instance1>::name()
+				.to_string()],
 			..Default::default()
 		}))
 		.inject_hashed_prefix(&<pallet_staking::Bonded<Runtime>>::prefix_hash())
@@ -44,7 +48,7 @@ pub async fn execute<Runtime: crate::RuntimeT, Block: BlockT + DeserializeOwned>
 
 	ext.execute_with(|| {
 		sp_core::crypto::set_default_ss58_version(Runtime::SS58Prefix::get().try_into().unwrap());
-		pallet_bags_list::Pallet::<Runtime>::try_state().unwrap();
+		pallet_bags_list::Pallet::<Runtime, pallet_bags_list::Instance1>::try_state().unwrap();
 		log::info!(target: crate::LOG_TARGET, "executed bags-list sanity check with no errors.");
 
 		crate::display_and_check_bags::<Runtime>(currency_unit, currency_name);

frame/grandpa/src/mock.rs

@@ -206,6 +206,7 @@ impl pallet_staking::Config for Test {
 	type ElectionProvider = onchain::UnboundedExecution<OnChainSeqPhragmen>;
 	type GenesisElectionProvider = Self::ElectionProvider;
 	type VoterList = pallet_staking::UseNominatorsAndValidatorsMap<Self>;
+	type TargetList = pallet_staking::UseValidatorsMap<Self>;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type OnStakerSlash = ();
 	type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig;

frame/nomination-pools/benchmarking/src/lib.rs

@@ -42,8 +42,11 @@ type CurrencyOf<T> = <T as pallet_nomination_pools::Config>::Currency;
 const USER_SEED: u32 = 0;
 const MAX_SPANS: u32 = 100;
 
+type VoterBagsListInstance = pallet_bags_list::Instance1;
 pub trait Config:
-	pallet_nomination_pools::Config + pallet_staking::Config + pallet_bags_list::Config
+	pallet_nomination_pools::Config
+	+ pallet_staking::Config
+	+ pallet_bags_list::Config<VoterBagsListInstance>
 {
 }
 

frame/nomination-pools/benchmarking/src/mock.rs

@@ -15,6 +15,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::VoterBagsListInstance;
 use frame_election_provider_support::VoteWeight;
 use frame_support::{pallet_prelude::*, parameter_types, traits::ConstU64, PalletId};
 use sp_runtime::{
@@ -111,7 +112,8 @@ impl pallet_staking::Config for Runtime {
 	type ElectionProvider =
 		frame_election_provider_support::NoElection<(AccountId, BlockNumber, Staking)>;
 	type GenesisElectionProvider = Self::ElectionProvider;
-	type VoterList = pallet_bags_list::Pallet<Self>;
+	type VoterList = VoterList;
+	type TargetList = pallet_staking::UseValidatorsMap<Self>;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type OnStakerSlash = Pools;
 	type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig;
@@ -122,7 +124,7 @@ parameter_types! {
 	pub static BagThresholds: &'static [VoteWeight] = &[10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000];
 }
 
-impl pallet_bags_list::Config for Runtime {
+impl pallet_bags_list::Config<VoterBagsListInstance> for Runtime {
 	type RuntimeEvent = RuntimeEvent;
 	type WeightInfo = ();
 	type BagThresholds = BagThresholds;
@@ -180,7 +182,7 @@ frame_support::construct_runtime!(
 		Timestamp: pallet_timestamp::{Pallet, Call, Storage, Inherent},
 		Balances: pallet_balances::{Pallet, Call, Storage, Config<T>, Event<T>},
 		Staking: pallet_staking::{Pallet, Call, Config<T>, Storage, Event<T>},
-		BagsList: pallet_bags_list::{Pallet, Call, Storage, Event<T>},
+		VoterList: pallet_bags_list::<Instance1>::{Pallet, Call, Storage, Event<T>},
 		Pools: pallet_nomination_pools::{Pallet, Call, Storage, Event<T>},
 	}
 );

frame/nomination-pools/test-staking/src/mock.rs

@@ -126,7 +126,8 @@ impl pallet_staking::Config for Runtime {
 	type ElectionProvider =
 		frame_election_provider_support::NoElection<(AccountId, BlockNumber, Staking)>;
 	type GenesisElectionProvider = Self::ElectionProvider;
-	type VoterList = pallet_bags_list::Pallet<Self>;
+	type VoterList = VoterList;
+	type TargetList = pallet_staking::UseValidatorsMap<Self>;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type OnStakerSlash = Pools;
 	type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig;
@@ -137,7 +138,8 @@ parameter_types! {
 	pub static BagThresholds: &'static [VoteWeight] = &[10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000];
 }
 
-impl pallet_bags_list::Config for Runtime {
+type VoterBagsListInstance = pallet_bags_list::Instance1;
+impl pallet_bags_list::Config<VoterBagsListInstance> for Runtime {
 	type RuntimeEvent = RuntimeEvent;
 	type WeightInfo = ();
 	type BagThresholds = BagThresholds;
@@ -193,7 +195,7 @@ frame_support::construct_runtime!(
 		Timestamp: pallet_timestamp::{Pallet, Call, Storage, Inherent},
 		Balances: pallet_balances::{Pallet, Call, Storage, Config<T>, Event<T>},
 		Staking: pallet_staking::{Pallet, Call, Config<T>, Storage, Event<T>},
-		BagsList: pallet_bags_list::{Pallet, Call, Storage, Event<T>},
+		VoterList: pallet_bags_list::<Instance1>::{Pallet, Call, Storage, Event<T>},
 		Pools: pallet_nomination_pools::{Pallet, Call, Storage, Event<T>},
 	}
 );

frame/offences/benchmarking/src/mock.rs

@@ -180,6 +180,7 @@ impl pallet_staking::Config for Test {
 	type ElectionProvider = onchain::UnboundedExecution<OnChainSeqPhragmen>;
 	type GenesisElectionProvider = Self::ElectionProvider;
 	type VoterList = pallet_staking::UseNominatorsAndValidatorsMap<Self>;
+	type TargetList = pallet_staking::UseValidatorsMap<Self>;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type OnStakerSlash = ();
 	type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig;

frame/session/benchmarking/src/mock.rs

@@ -175,6 +175,7 @@ impl pallet_staking::Config for Test {
 	type GenesisElectionProvider = Self::ElectionProvider;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type VoterList = pallet_staking::UseNominatorsAndValidatorsMap<Self>;
+	type TargetList = pallet_staking::UseValidatorsMap<Self>;
 	type OnStakerSlash = ();
 	type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig;
 	type WeightInfo = ();

frame/staking/src/benchmarking.rs

@@ -844,7 +844,7 @@ benchmarks! {
 			v, n, T::MaxNominations::get() as usize, false, None
 		)?;
 	}: {
-		let targets = <Staking<T>>::get_npos_targets();
+		let targets = <Staking<T>>::get_npos_targets(None);
 		assert_eq!(targets.len() as u32, v);
 	}
 

frame/staking/src/lib.rs

@@ -886,11 +886,12 @@ enum Releases {
 	V8_0_0,  // populate `VoterList`.
 	V9_0_0,  // inject validators into `VoterList` as well.
 	V10_0_0, // remove `EarliestUnappliedSlash`.
+	V11_0_0, // Move pallet storage prefix, e.g. BagsList -> VoterBagsList
 }
 
 impl Default for Releases {
 	fn default() -> Self {
-		Releases::V10_0_0
+		Releases::V11_0_0
 	}
 }