Merge branch 'master' into test-branch

.github/actions/set-up-mac/README.md

@@ -0,0 +1,15 @@
+# How to use
+
+```yml
+  set-image:
+    runs-on: macos-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - id: set_image
+        run: cat .github/env >> $GITHUB_OUTPUT
+      - name: Install dependencies
+        uses: ./.github/actions/set-up-mac
+        with:
+          IMAGE: ${{ steps.set-image.outputs.IMAGE }}
+```

.github/actions/set-up-mac/action.yml

@@ -0,0 +1,43 @@
+name: "Set up rust on mac"
+description: "Install the required tools for Mac runners"
+inputs:
+  IMAGE:
+    description: "Rust docker image"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Install with Hombrew
+      shell: bash
+      run: brew install protobuf rustup openssl pkg-config zlib xz zstd llvm jq curl gcc make cmake
+    - name: Set version
+      shell: bash
+      run: |
+        VERSION=$(echo $IMAGE | sed -E 's/.*:bullseye-([^-]+)-.*/\1/')
+        echo $VERSION
+        echo "VERSION=$VERSION" >> $GITHUB_ENV
+        NIGHTLY=$(echo $IMAGE | sed -E 's/.*([0-9]{4}-[0-9]{2}-[0-9]{2}).*/\1/')
+        echo $NIGHTLY
+        echo "NIGHTLY=$NIGHTLY" >> $GITHUB_ENV
+      env:
+        IMAGE: ${{ inputs.IMAGE }}
+
+    - name: Install rustup
+      shell: bash
+      run: |
+        rustup-init -y
+        rustup install $VERSION
+        rustup default $VERSION
+        rustup toolchain install "nightly-${NIGHTLY}"
+
+    - name: MacOS Deps
+      shell: bash
+      run: |
+        rustup target add wasm32-unknown-unknown --toolchain $VERSION
+        rustup component add rust-src rustfmt clippy --toolchain $VERSION
+
+    - name: Check Rust
+      shell: bash
+      run: |
+        rustup show
+        rustup +nightly show

.github/workflows/cmd.yml

@@ -268,9 +268,9 @@ jobs:
             });
             const prBranch = pr.data.head.ref;
             const repo = pr.data.head.repo.full_name;
-            console.log(prBranch, repo, pr, pr.data, pr.data.head)
-            return { pr_branch: prBranch, repo: repo };
-          result-encoding: string
+            console.log(prBranch, repo)
+            core.setOutput('pr_branch', prBranch);
+            core.setOutput('repo', repo);
 
       - name: Use PR Branch Name and Repo
         run: |

polkadot/node/collation-generation/src/lib.rs

@@ -45,7 +45,7 @@ use polkadot_node_subsystem::{
 use polkadot_node_subsystem_util::{
 	request_async_backing_params, request_availability_cores, request_para_backing_state,
 	request_persisted_validation_data, request_validation_code, request_validation_code_hash,
-	request_validators, vstaging::fetch_claim_queue,
+	request_validators, runtime::fetch_claim_queue,
 };
 use polkadot_primitives::{
 	collator_signature_payload, CandidateCommitments, CandidateDescriptor, CandidateReceipt,

polkadot/node/core/backing/src/lib.rs

@@ -100,9 +100,9 @@ use polkadot_node_subsystem_util::{
 	executor_params_at_relay_parent, request_from_runtime, request_session_index_for_child,
 	request_validator_groups, request_validators,
 	runtime::{
-		self, prospective_parachains_mode, request_min_backing_votes, ProspectiveParachainsMode,
+		self, fetch_claim_queue, prospective_parachains_mode, request_min_backing_votes,
+		ClaimQueueSnapshot, ProspectiveParachainsMode,
 	},
-	vstaging::{fetch_claim_queue, ClaimQueueSnapshot},
 	Validator,
 };
 use polkadot_primitives::{
@@ -121,7 +121,7 @@ use polkadot_statement_table::{
 	Config as TableConfig, Context as TableContextTrait, Table,
 };
 use sp_keystore::KeystorePtr;
-use util::{runtime::request_node_features, vstaging::get_disabled_validators_with_fallback};
+use util::runtime::{get_disabled_validators_with_fallback, request_node_features};
 
 mod error;
 

polkadot/node/core/prospective-parachains/src/lib.rs

@@ -46,8 +46,7 @@ use polkadot_node_subsystem_util::{
 	backing_implicit_view::{BlockInfoProspectiveParachains as BlockInfo, View as ImplicitView},
 	inclusion_emulator::{Constraints, RelayChainBlockInfo},
 	request_session_index_for_child,
-	runtime::{prospective_parachains_mode, ProspectiveParachainsMode},
-	vstaging::fetch_claim_queue,
+	runtime::{fetch_claim_queue, prospective_parachains_mode, ProspectiveParachainsMode},
 };
 use polkadot_primitives::{
 	async_backing::CandidatePendingAvailability, BlockNumber, CandidateHash,

polkadot/node/network/collator-protocol/src/collator_side/mod.rs

@@ -48,10 +48,9 @@ use polkadot_node_subsystem_util::{
 	backing_implicit_view::View as ImplicitView,
 	reputation::{ReputationAggregator, REPUTATION_CHANGE_INTERVAL},
 	runtime::{
-		get_availability_cores, get_group_rotation_info, prospective_parachains_mode,
-		ProspectiveParachainsMode, RuntimeInfo,
+		fetch_claim_queue, get_availability_cores, get_group_rotation_info,
+		prospective_parachains_mode, ProspectiveParachainsMode, RuntimeInfo,
 	},
-	vstaging::fetch_claim_queue,
 	TimeoutExt,
 };
 use polkadot_primitives::{

polkadot/node/network/collator-protocol/src/validator_side/mod.rs

@@ -50,8 +50,7 @@ use polkadot_node_subsystem::{
 use polkadot_node_subsystem_util::{
 	backing_implicit_view::View as ImplicitView,
 	reputation::{ReputationAggregator, REPUTATION_CHANGE_INTERVAL},
-	runtime::{prospective_parachains_mode, ProspectiveParachainsMode},
-	vstaging::fetch_claim_queue,
+	runtime::{fetch_claim_queue, prospective_parachains_mode, ProspectiveParachainsMode},
 };
 use polkadot_primitives::{
 	CandidateHash, CollatorId, CoreState, Hash, HeadData, Id as ParaId, OccupiedCoreAssumption,

polkadot/node/network/statement-distribution/src/v2/mod.rs

@@ -45,8 +45,9 @@ use polkadot_node_subsystem::{
 use polkadot_node_subsystem_util::{
 	backing_implicit_view::View as ImplicitView,
 	reputation::ReputationAggregator,
-	runtime::{request_min_backing_votes, ProspectiveParachainsMode},
-	vstaging::{fetch_claim_queue, ClaimQueueSnapshot},
+	runtime::{
+		fetch_claim_queue, request_min_backing_votes, ClaimQueueSnapshot, ProspectiveParachainsMode,
+	},
 };
 use polkadot_primitives::{
 	AuthorityDiscoveryId, CandidateHash, CompactStatement, CoreIndex, CoreState, GroupIndex,
@@ -570,7 +571,7 @@ pub(crate) async fn handle_active_leaves_update<Context>(
 
 	for new_relay_parent in new_relay_parents.iter().cloned() {
 		let disabled_validators: HashSet<_> =
-			polkadot_node_subsystem_util::vstaging::get_disabled_validators_with_fallback(
+			polkadot_node_subsystem_util::runtime::get_disabled_validators_with_fallback(
 				ctx.sender(),
 				new_relay_parent,
 			)

polkadot/node/network/statement-distribution/src/v2/statement_store.rs

@@ -292,7 +292,7 @@ impl GroupStatements {
 mod tests {
 	use super::*;
 
-	use polkadot_primitives::v7::{Hash, SigningContext, ValidatorPair};
+	use polkadot_primitives::{Hash, SigningContext, ValidatorPair};
 	use sp_application_crypto::Pair as PairT;
 
 	#[test]

polkadot/node/service/src/chain_spec.rs

@@ -25,7 +25,7 @@ use sp_consensus_babe::AuthorityId as BabeId;
 use sp_consensus_beefy::ecdsa_crypto::AuthorityId as BeefyId;
 
 #[cfg(feature = "westend-native")]
-use polkadot_primitives::vstaging::SchedulerParams;
+use polkadot_primitives::SchedulerParams;
 #[cfg(feature = "rococo-native")]
 use rococo_runtime as rococo;
 use sc_chain_spec::ChainSpecExtension;

polkadot/node/subsystem-util/src/lib.rs

@@ -49,6 +49,7 @@ use polkadot_primitives::{
 	ValidatorSignature,
 };
 pub use rand;
+use runtime::get_disabled_validators_with_fallback;
 use sp_application_crypto::AppCrypto;
 use sp_core::ByteArray;
 use sp_keystore::{Error as KeystoreError, KeystorePtr};
@@ -57,7 +58,6 @@ use std::{
 	time::Duration,
 };
 use thiserror::Error;
-use vstaging::get_disabled_validators_with_fallback;
 
 pub use determine_new_blocks::determine_new_blocks;
 pub use metered;

polkadot/node/subsystem-util/src/runtime/mod.rs

@@ -32,17 +32,20 @@ use polkadot_node_subsystem_types::UnpinHandle;
 use polkadot_primitives::{
 	node_features::FeatureIndex, slashing, AsyncBackingParams, CandidateEvent, CandidateHash,
 	CoreIndex, CoreState, EncodeAs, ExecutorParams, GroupIndex, GroupRotationInfo, Hash,
-	IndexedVec, NodeFeatures, OccupiedCore, ScrapedOnChainVotes, SessionIndex, SessionInfo, Signed,
-	SigningContext, UncheckedSigned, ValidationCode, ValidationCodeHash, ValidatorId,
-	ValidatorIndex, LEGACY_MIN_BACKING_VOTES,
+	Id as ParaId, IndexedVec, NodeFeatures, OccupiedCore, ScrapedOnChainVotes, SessionIndex,
+	SessionInfo, Signed, SigningContext, UncheckedSigned, ValidationCode, ValidationCodeHash,
+	ValidatorId, ValidatorIndex, LEGACY_MIN_BACKING_VOTES,
 };
 
+use std::collections::{BTreeMap, VecDeque};
+
 use crate::{
-	request_async_backing_params, request_availability_cores, request_candidate_events,
+	has_required_runtime, request_async_backing_params, request_availability_cores,
+	request_candidate_events, request_claim_queue, request_disabled_validators,
 	request_from_runtime, request_key_ownership_proof, request_on_chain_votes,
 	request_session_executor_params, request_session_index_for_child, request_session_info,
 	request_submit_report_dispute_lost, request_unapplied_slashes, request_validation_code_by_hash,
-	request_validator_groups, vstaging::get_disabled_validators_with_fallback,
+	request_validator_groups,
 };
 
 /// Errors that can happen on runtime fetches.
@@ -579,3 +582,98 @@ pub async fn request_node_features(
 		res.map(Some)
 	}
 }
+
+/// A snapshot of the runtime claim queue at an arbitrary relay chain block.
+#[derive(Default)]
+pub struct ClaimQueueSnapshot(pub BTreeMap<CoreIndex, VecDeque<ParaId>>);
+
+impl From<BTreeMap<CoreIndex, VecDeque<ParaId>>> for ClaimQueueSnapshot {
+	fn from(claim_queue_snapshot: BTreeMap<CoreIndex, VecDeque<ParaId>>) -> Self {
+		ClaimQueueSnapshot(claim_queue_snapshot)
+	}
+}
+
+impl ClaimQueueSnapshot {
+	/// Returns the `ParaId` that has a claim for `core_index` at the specified `depth` in the
+	/// claim queue. A depth of `0` means the very next block.
+	pub fn get_claim_for(&self, core_index: CoreIndex, depth: usize) -> Option<ParaId> {
+		self.0.get(&core_index)?.get(depth).copied()
+	}
+
+	/// Returns an iterator over all claimed cores and the claiming `ParaId` at the specified
+	/// `depth` in the claim queue.
+	pub fn iter_claims_at_depth(
+		&self,
+		depth: usize,
+	) -> impl Iterator<Item = (CoreIndex, ParaId)> + '_ {
+		self.0
+			.iter()
+			.filter_map(move |(core_index, paras)| Some((*core_index, *paras.get(depth)?)))
+	}
+
+	/// Returns an iterator over all claims on the given core.
+	pub fn iter_claims_for_core(
+		&self,
+		core_index: &CoreIndex,
+	) -> impl Iterator<Item = &ParaId> + '_ {
+		self.0.get(core_index).map(|c| c.iter()).into_iter().flatten()
+	}
+
+	/// Returns an iterator over the whole claim queue.
+	pub fn iter_all_claims(&self) -> impl Iterator<Item = (&CoreIndex, &VecDeque<ParaId>)> + '_ {
+		self.0.iter()
+	}
+}
+
+// TODO: https://github.com/paritytech/polkadot-sdk/issues/1940
+/// Returns disabled validators list if the runtime supports it. Otherwise logs a debug messages and
+/// returns an empty vec.
+/// Once runtime ver `DISABLED_VALIDATORS_RUNTIME_REQUIREMENT` is released remove this function and
+/// replace all usages with `request_disabled_validators`
+pub async fn get_disabled_validators_with_fallback<Sender: SubsystemSender<RuntimeApiMessage>>(
+	sender: &mut Sender,
+	relay_parent: Hash,
+) -> Result<Vec<ValidatorIndex>> {
+	let disabled_validators = if has_required_runtime(
+		sender,
+		relay_parent,
+		RuntimeApiRequest::DISABLED_VALIDATORS_RUNTIME_REQUIREMENT,
+	)
+	.await
+	{
+		request_disabled_validators(relay_parent, sender)
+			.await
+			.await
+			.map_err(Error::RuntimeRequestCanceled)??
+	} else {
+		gum::debug!(target: LOG_TARGET, "Runtime doesn't support `DisabledValidators` - continuing with an empty disabled validators set");
+		vec![]
+	};
+
+	Ok(disabled_validators)
+}
+
+/// Checks if the runtime supports `request_claim_queue` and attempts to fetch the claim queue.
+/// Returns `ClaimQueueSnapshot` or `None` if claim queue API is not supported by runtime.
+/// Any specific `RuntimeApiError` is bubbled up to the caller.
+pub async fn fetch_claim_queue(
+	sender: &mut impl SubsystemSender<RuntimeApiMessage>,
+	relay_parent: Hash,
+) -> Result<Option<ClaimQueueSnapshot>> {
+	if has_required_runtime(
+		sender,
+		relay_parent,
+		RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT,
+	)
+	.await
+	{
+		let res = request_claim_queue(relay_parent, sender)
+			.await
+			.await
+			.map_err(Error::RuntimeRequestCanceled)??;
+		Ok(Some(res.into()))
+	} else {
+		gum::trace!(target: LOG_TARGET, "Runtime doesn't support `request_claim_queue`");
+		Ok(None)
+	}
+}