diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000000..fdaa0c8628f7 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,75 @@ +# Lists some code owners. +# +# A codeowner just oversees some part of the codebase. If an owned file is changed then the +# corresponding codeowner receives a review request. An approval of the codeowner might be +# required for merging a PR (depends on repository settings). +# +# For details about syntax, see: +# https://help.github.com/en/articles/about-code-owners +# But here are some important notes: +# +# - Glob syntax is git-like, e.g. `/core` means the core directory in the root, unlike `core` +# which can be everywhere. +# - Multiple owners are supported. +# - Either handle (e.g, @github_user or @github/team) or email can be used. Keep in mind, +# that handles might work better because they are more recognizable on GitHub, +# eyou can use them for mentioning unlike an email. +# - The latest matching rule, if multiple, takes precedence. + +# CI +/.github/ @paritytech/ci @paritytech/release-engineering +/.gitlab-ci.yml @paritytech/ci +/.gitlab/ @paritytech/ci + +# XCM +/polkadot/xcm/ @paritytech/xcm + +# WASM executor, low-level client <-> WASM interface and other WASM-related code +/substrate/client/allocator/ @koute +/substrate/client/executor/ @koute +/substrate/primitives/panic-handler/ @koute +/substrate/primitives/runtime-interface/ @koute +/substrate/primitives/wasm-interface/ @koute +/substrate/utils/wasm-builder/ @koute + +# Systems-related bits and bobs on the client side +/substrate/client/sysinfo/ @koute +/substrate/client/tracing/ @koute + +# Documentation audit +/substrate/primitives/runtime @paritytech/docs-audit +/substrate/primitives/arithmetic @paritytech/docs-audit +# /primitives/core (to be added later) +# /primitives/io (to be added later) + +# FRAME +/substrate/frame/ @paritytech/frame-coders @paritytech/docs-audit +/substrate/frame/nfts/ @jsidorenko @paritytech/docs-audit +/substrate/frame/state-trie-migration/ @paritytech/frame-coders @cheme +/substrate/frame/uniques/ @jsidorenko @paritytech/docs-audit + +# GRANDPA, BABE, consensus stuff +/substrate/client/consensus/babe/ @andresilva +/substrate/client/consensus/grandpa/ @andresilva +/substrate/client/consensus/pow/ @sorpaas +/substrate/client/consensus/slots/ @andresilva +/substrate/frame/babe/ @andresilva +/substrate/frame/grandpa/ @andresilva +/substrate/primitives/consensus/pow/ @sorpaas + +# BEEFY, MMR +/substrate/frame/beefy/ @acatangiu +/substrate/frame/beefy-mmr/ @acatangiu +/substrate/frame/merkle-mountain-range/ @acatangiu +/substrate/primitives/merkle-mountain-range/ @acatangiu + +# Contracts +/substrate/frame/contracts/ @athei @paritytech/docs-audit + +# NPoS and election +/substrate/frame/election-provider-multi-phase/ @paritytech/staking-core @paritytech/docs-audit +/substrate/frame/election-provider-support/ @paritytech/staking-core @paritytech/docs-audit +/substrate/frame/elections-phragmen/ @paritytech/staking-core @paritytech/docs-audit +/substrate/frame/nomination-pools/ @paritytech/staking-core @paritytech/docs-audit +/substrate/frame/staking/ @paritytech/staking-core @paritytech/docs-audit +/substrate/primitives/npos-elections/ @paritytech/staking-core @paritytech/docs-audit diff --git a/.github/pr-custom-review.yml b/.github/pr-custom-review.yml index bc9a3cfb8d25..ac13d862a4ac 100644 --- a/.github/pr-custom-review.yml +++ b/.github/pr-custom-review.yml @@ -8,7 +8,7 @@ rules: check_type: changed_files condition: include: ^\.gitlab-ci\.yml|^docker/.*|^\.github/.*|^\.gitlab/.*|^\.config/nextest.toml|^\.cargo/.* - exclude: ^./gitlab/pipeline/zombienet.yml$ + exclude: ^\.gitlab/pipeline/zombienet.* min_approvals: 2 teams: - ci @@ -39,7 +39,7 @@ rules: # if there are any changes in the bridges subtree (in case of backport changes back to bridges repo) - name: Bridges subtree files check_type: changed_files - condition: ^cumulus/bridges/.* + condition: ^bridges/.* min_approvals: 1 teams: - bridges-core diff --git a/.github/workflows/release-50_publish-docker.yml b/.github/workflows/release-50_publish-docker.yml index 512b91aa6e55..04b3ebd3e79c 100644 --- a/.github/workflows/release-50_publish-docker.yml +++ b/.github/workflows/release-50_publish-docker.yml @@ -1,10 +1,7 @@ name: Release - Publish Docker Image -# This workflow listens to pubished releases or can be triggered manually. -# It includes releases and rc candidates. -# It fetches the binaries, checks sha256 and GPG -# signatures, then builds an injected docker -# image and publishes it. +# This workflow listens to published releases or can be triggered manually. +# It builds and published releases and rc candidates. on: #TODO: activate automated run later @@ -13,6 +10,24 @@ on: # - published workflow_dispatch: inputs: + image_type: + description: Type of the image to be published + required: true + default: rc + type: choice + options: + - rc + - release + + binary: + description: Binary to be published + required: true + default: polkadot + type: choice + options: + - polkadot + - polkadot-parachain + release_id: description: | Release ID. @@ -22,33 +37,25 @@ on: jq '.[] | { name: .name, id: .id }' required: true type: string - image_type: - description: Type of the image to be published - required: true - default: rc - type: choice - options: - - rc - - release + registry: description: Container registry required: true type: string default: docker.io + + # The owner is often the same than the Docker Hub username but does ont have to be. + # In our case, it is not. owner: description: Owner of the container image repo required: true type: string default: parity - binary: - description: Binary to be published + + version: + description: version to build/release + default: v0.9.18 required: true - default: polkadot - type: choice - options: - - polkadot - - staking-miner - - polkadot-parachain permissions: contents: write @@ -66,7 +73,8 @@ env: IMAGE_TYPE: ${{ inputs.image_type }} jobs: - fetch-artifacts: + fetch-artifacts: # this job will be triggered for the polkadot-parachain rc and release or polkadot rc image build + if: ${{ inputs.binary == 'polkadot-parachain' || inputs.image_type == 'rc' }} runs-on: ubuntu-latest steps: @@ -102,7 +110,8 @@ jobs: path: | ./release-artifacts/${{ env.BINARY }}/**/* - build-container: + build-container: # this job will be triggered for the polkadot-parachain rc and release or polkadot rc image build + if: ${{ inputs.binary == 'polkadot-parachain' || inputs.image_type == 'rc' }} runs-on: ubuntu-latest needs: fetch-artifacts @@ -158,8 +167,8 @@ jobs: echo "tag=latest" >> $GITHUB_OUTPUT echo "release=${release}" >> $GITHUB_OUTPUT - - name: Build Injected Container image for polkadot/staking-miner - if: ${{ env.BINARY == 'polkadot' || env.BINARY == 'staking-miner' }} + - name: Build Injected Container image for polkadot rc + if: ${{ env.BINARY == 'polkadot' }} env: ARTIFACTS_FOLDER: ./release-artifacts IMAGE_NAME: ${{ env.BINARY }} @@ -204,3 +213,73 @@ jobs: run: | echo "Checking tag ${RELEASE_TAG} for image ${REGISTRY}/${DOCKER_OWNER}/${BINARY}" $ENGINE run -i ${REGISTRY}/${DOCKER_OWNER}/${BINARY}:${RELEASE_TAG} --version + + fetch-latest-debian-package-version: # this job will be triggered for polkadot release build + if: ${{ inputs.binary == 'polkadot' && inputs.image_type == 'release' }} + runs-on: ubuntu-latest + outputs: + polkadot_apt_version: ${{ steps.fetch-latest-apt.outputs.polkadot_apt_version }} + container: + image: paritytech/parity-keyring + options: --user root + steps: + - name: Get version + id: fetch-latest-apt + run: | + apt update + apt show polkadot + version=$(apt show polkadot 2>/dev/null | grep "Version:" | awk '{print $2}') + echo "polkadot_apt_version=v$version" >> $GITHUB_OUTPUT + echo "You passed ${{ inputs.version }} but this is ignored" + echo "We use the version from the Debian Package: $version" + + build-polkadot-release-container: # this job will be triggered for polkadot release build + if: ${{ inputs.binary == 'polkadot' && inputs.image_type == 'release' }} + runs-on: ubuntu-latest + needs: fetch-latest-debian-package-version + steps: + - name: Checkout sources + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@95cb08cb2672c73d4ffd2f422e6d11953d2a9c70 # v2.1.0 + + - name: Cache Docker layers + uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Login to Docker Hub + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Fetch values + id: fetch-data + run: | + date=$(date -u '+%Y-%m-%dT%H:%M:%SZ') + echo "date=$date" >> $GITHUB_OUTPUT + + - name: Build and push + id: docker_build + uses: docker/build-push-action@v4 + with: + push: true + file: docker/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile + # TODO: The owner should be used below but buildx does not resolve the VARs + # TODO: It would be good to get rid of this GHA that we don't really need. + tags: | + parity/polkadot:latest + parity/polkadot:${{ needs.fetch-latest-debian-package-version.outputs.polkadot_apt_version }} + build-args: | + VCS_REF=${{ github.ref }} + POLKADOT_VERSION=${{ needs.fetch-latest-debian-package-version.outputs.polkadot_apt_version }} + BUILD_DATE=${{ steps.fetch-data.outputs.date }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 99381fae9ecd..10dd69f12a77 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,7 +21,7 @@ workflow: - if: $CI_COMMIT_BRANCH variables: - CI_IMAGE: "paritytech/ci-unified:bullseye-1.70.0-2023-05-23-v20230706" + CI_IMAGE: !reference [.ci-unified, variables, CI_IMAGE] # BUILDAH_IMAGE is defined in group variables BUILDAH_COMMAND: "buildah --storage-driver overlay2" RELENG_SCRIPTS_BRANCH: "master" @@ -114,7 +114,7 @@ default: - !reference [.rust-info-script, script] - !reference [.rusty-cachier, before_script] tags: - - linux-docker-vm-c2 + - linux-docker # rusty-cachier's hidden job. Parts of this job are used to instrument the pipeline's other real jobs with rusty-cachier # rusty-cachier's commands are described here: https://gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client#description @@ -213,6 +213,10 @@ include: - project: parity/infrastructure/ci_cd/shared ref: v0.2 file: /common/timestamp.yml + # ci image + - project: parity/infrastructure/ci_cd/shared + ref: main + file: /common/ci-unified.yml # This job cancels the whole pipeline if any of provided jobs fail. # In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests # to fail the pipeline as soon as possible to shorten the feedback loop. diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index c4dfc0dd0931..3085d2230063 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -77,26 +77,6 @@ build-malus: - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - cp -r ./docker/* ./artifacts -build-staking-miner: - stage: build - extends: - - .docker-env - - .common-refs - # - .collect-artifacts - # DAG - needs: - - job: build-malus - artifacts: false - script: - - time cargo build -q --locked --release --package staging-staking-miner - # # pack artifacts - # - mkdir -p ./artifacts - # - mv ./target/release/staking-miner ./artifacts/. - # - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - # - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - # - echo "staking-miner = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - # - cp -r ./scripts/* ./artifacts - build-rustdoc: stage: build extends: @@ -131,6 +111,9 @@ build-rustdoc: sed -i "s||$script_content|" "$file" } export -f process_file + # xargs runs process_file in seperate shells without access to outer variables. + # to make script_content available inside process_file, export it as an env var here. + export script_content # Modify .html files in parallel using xargs, otherwise it can take a long time. find "$path" -name '*.html' | xargs -I {} -P "$(nproc)" bash -c 'process_file "$@"' _ {} @@ -163,7 +146,7 @@ build-short-benchmark: - .run-immediately - .collect-artifacts script: - - cargo build --profile release --locked --features=runtime-benchmarks --bin polkadot --workspace + - cargo build --profile release --locked --features=runtime-benchmarks,on-chain-release-build --bin polkadot --workspace - mkdir -p artifacts - target/release/polkadot --version - cp ./target/release/polkadot ./artifacts/ @@ -290,7 +273,7 @@ build-short-benchmark-cumulus: - .run-immediately - .collect-artifacts script: - - cargo build --profile release --locked --features=runtime-benchmarks -p polkadot-parachain-bin --bin polkadot-parachain + - cargo build --profile release --locked --features=runtime-benchmarks,on-chain-release-build -p polkadot-parachain-bin --bin polkadot-parachain --workspace - mkdir -p artifacts - target/release/polkadot-parachain --version - cp ./target/release/polkadot-parachain ./artifacts/ @@ -358,7 +341,7 @@ build-subkey-linux: extends: .build-subkey # DAG needs: - - job: build-staking-miner + - job: build-malus artifacts: false # tbd # build-subkey-macos: diff --git a/.gitlab/pipeline/publish.yml b/.gitlab/pipeline/publish.yml index 1a513e5970d5..9e24b8606a4d 100644 --- a/.gitlab/pipeline/publish.yml +++ b/.gitlab/pipeline/publish.yml @@ -328,24 +328,6 @@ build-push-image-substrate-pr: # # this artifact is used in zombienet-tests job # dotenv: ./artifacts/malus.env -# publish-staking-miner-image: -# stage: publish -# extends: -# - .kubernetes-env -# - .build-push-image -# - .publish-refs -# variables: -# CI_IMAGE: ${BUILDAH_IMAGE} -# # scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile -# DOCKERFILE: ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile -# IMAGE_NAME: docker.io/paritytech/staking-miner -# GIT_STRATEGY: none -# DOCKER_USER: ${Docker_Hub_User_Parity} -# DOCKER_PASS: ${Docker_Hub_Pass_Parity} -# needs: -# - job: build-staking-miner -# artifacts: true - # substrate # publish-substrate-image-pr: diff --git a/.gitlab/pipeline/short-benchmarks.yml b/.gitlab/pipeline/short-benchmarks.yml index 81601fba32ac..7b7704ee66d8 100644 --- a/.gitlab/pipeline/short-benchmarks.yml +++ b/.gitlab/pipeline/short-benchmarks.yml @@ -18,7 +18,7 @@ short-benchmark-polkadot: &short-bench tags: - benchmark script: - - ./artifacts/polkadot benchmark pallet --execution wasm --wasm-execution compiled --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 + - ./artifacts/polkadot benchmark pallet --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 short-benchmark-kusama: <<: *short-bench @@ -45,7 +45,7 @@ short-benchmark-westend: tags: - benchmark script: - - ./artifacts/polkadot-parachain benchmark pallet --wasm-execution compiled --chain $RUNTIME_CHAIN --pallet "*" --extrinsic "*" --steps 2 --repeat 1 + - ./artifacts/polkadot-parachain benchmark pallet --chain $RUNTIME_CHAIN --pallet "*" --extrinsic "*" --steps 2 --repeat 1 short-benchmark-asset-hub-polkadot: <<: *short-bench-cumulus diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index b29574ad14de..3a4f5e71247c 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -23,13 +23,8 @@ test-linux-stable: - echo "Node index - ${CI_NODE_INDEX}. Total amount - ${CI_NODE_TOTAL}" # add experimental to features after https://github.com/paritytech/substrate/pull/14502 is merged # "upgrade_version_checks_should_work" is currently failing - # "receive_rate_limit_is_enforced"and "benchmark_block_works" can be found in test-linux-stable-additional-tests - # they fail if run with other tests - # "rx::tests::sent_views_include_finalized_number_update", "follow_chain_works", "create_snapshot_works" and "block_execution_works" - # can be found in test-linux-stable-slow - | time cargo nextest run \ - -E 'all() & !test(upgrade_version_checks_should_work) & !test(receive_rate_limit_is_enforced) & !test(benchmark_block_works) & !test(rx::tests::sent_views_include_finalized_number_update) & !test(follow_chain_works) & !test(create_snapshot_works) & !test(block_execution_works)' \ --workspace \ --locked \ --release \ @@ -59,6 +54,22 @@ test-linux-oldkernel-stable: tags: - oldkernel-vm +# https://github.com/paritytech/ci_cd/issues/864 +test-linux-stable-runtime-benchmarks: + stage: test + extends: + - .docker-env + - .common-refs + - .run-immediately + - .pipeline-stopper-artifacts + variables: + RUST_TOOLCHAIN: stable + # Enable debug assertions since we are running optimized builds for testing + # but still want to have debug assertions. + RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + script: + - time cargo nextest run --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet + # can be used to run all tests # test-linux-stable-all: # stage: test @@ -85,7 +96,7 @@ test-linux-oldkernel-stable: # --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL} # # todo: add flacky-test collector -# for some reasons these tests fail if run with all tests +# TODO: remove me test-linux-stable-additional-tests: stage: test extends: @@ -99,31 +110,11 @@ test-linux-stable-additional-tests: # but still want to have debug assertions. RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" script: - - | - time cargo nextest run \ - -E 'test(receive_rate_limit_is_enforced) + test(benchmark_block_works)' \ - --workspace \ - --locked \ - --release \ - --features runtime-benchmarks,try-runtime + # tests were moved to test-linux-stable + # the jobs should be removed + - exit 0 -# https://github.com/paritytech/ci_cd/issues/864 -test-linux-stable-runtime-benchmarks: - stage: test - extends: - - .docker-env - - .common-refs - - .run-immediately - - .pipeline-stopper-artifacts - variables: - RUST_TOOLCHAIN: stable - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - script: - - time cargo nextest run --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet - -# these ones can be really slow so it's better to run them separately +# TODO: remove me test-linux-stable-slow: stage: test # remove after cache is setup @@ -139,14 +130,9 @@ test-linux-stable-slow: # but still want to have debug assertions. RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" script: - - | - time cargo nextest run \ - -E 'test(rx::tests::sent_views_include_finalized_number_update) + test(follow_chain_works) + test(create_snapshot_works) + test(block_execution_works)' \ - --workspace \ - --locked \ - --release \ - --features runtime-benchmarks,try-runtime - allow_failure: true + # tests were moved to test-linux-stable + # the jobs should be removed + - exit 0 # takes about 1,5h without cache # can be used to check that nextest works correctly diff --git a/.gitlab/pipeline/zombienet/polkadot.yml b/.gitlab/pipeline/zombienet/polkadot.yml index 349807a610d4..e420baf486aa 100644 --- a/.gitlab/pipeline/zombienet/polkadot.yml +++ b/.gitlab/pipeline/zombienet/polkadot.yml @@ -7,10 +7,18 @@ - export BUILD_RELEASE_VERSION="$(cat ./artifacts/BUILD_RELEASE_VERSION)" # from build-linux-stable job - export DEBUG=zombie,zombie::network-node - export ZOMBIENET_INTEGRATION_TEST_IMAGE="${POLKADOT_IMAGE}":${PIPELINE_IMAGE_TAG} - - export ZOMBIENET_INTEGRATION_TEST_SECONDARY_IMAGE="docker.io/parity/polkadot:${BUILD_RELEASE_VERSION}" - export COL_IMAGE="${COLANDER_IMAGE}":${PIPELINE_IMAGE_TAG} - export CUMULUS_IMAGE="docker.io/paritypr/polkadot-parachain-debug:${DOCKER_IMAGES_VERSION}" - export MALUS_IMAGE="${MALUS_IMAGE}":${PIPELINE_IMAGE_TAG} + - IMAGE_AVAILABLE=$(curl -o /dev/null -w "%{http_code}" -I -L -s https://registry.hub.docker.com/v2/repositories/parity/polkadot/tags/${BUILD_RELEASE_VERSION}) + - if [ $IMAGE_AVAILABLE -eq 200 ]; then + export ZOMBIENET_INTEGRATION_TEST_SECONDARY_IMAGE="docker.io/parity/polkadot:${BUILD_RELEASE_VERSION}"; + else + echo "Getting the image to use as SECONDARY, using ${BUILD_RELEASE_VERSION} as base"; + VERSIONS=$(curl -L -s 'https://registry.hub.docker.com/v2/repositories/parity/polkadot/tags/' | jq -r '.results[].name'| grep -E "v[0-9]" |grep -vE "[0-9]-"); + VERSION_TO_USE=$(echo "${BUILD_RELEASE_VERSION}\n$VERSIONS"|sort -r|grep -A1 "${BUILD_RELEASE_VERSION}"|tail -1); + export ZOMBIENET_INTEGRATION_TEST_SECONDARY_IMAGE="docker.io/parity/polkadot:${VERSION_TO_USE}"; + fi - echo "Zombienet Tests Config" - echo "gh-dir ${GH_DIR}" - echo "local-dir ${LOCAL_DIR}" diff --git a/cumulus/BRIDGES.md b/BRIDGES.md similarity index 100% rename from cumulus/BRIDGES.md rename to BRIDGES.md diff --git a/Cargo.lock b/Cargo.lock index 3ba7edafbda1..a994d14a322d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -494,7 +494,7 @@ dependencies = [ "ark-ff", "ark-std", "tracing", - "tracing-subscriber 0.2.25", + "tracing-subscriber", ] [[package]] @@ -513,7 +513,7 @@ dependencies = [ [[package]] name = "ark-secret-scalar" version = "0.0.2" -source = "git+https://github.com/w3f/ring-vrf?rev=3119f51#3119f51b54b69308abfb0671f6176cb125ae1bf1" +source = "git+https://github.com/w3f/ring-vrf?rev=f4fe253#f4fe2534ccc6d916cd10d9c16891e673728ec8b4" dependencies = [ "ark-ec", "ark-ff", @@ -561,7 +561,7 @@ dependencies = [ [[package]] name = "ark-transcript" version = "0.0.2" -source = "git+https://github.com/w3f/ring-vrf?rev=3119f51#3119f51b54b69308abfb0671f6176cb125ae1bf1" +source = "git+https://github.com/w3f/ring-vrf?rev=f4fe253#f4fe2534ccc6d916cd10d9c16891e673728ec8b4" dependencies = [ "ark-ff", "ark-serialize", @@ -1138,7 +1138,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -1160,7 +1160,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -1177,7 +1177,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -1240,13 +1240,12 @@ dependencies = [ [[package]] name = "bandersnatch_vrfs" version = "0.0.1" -source = "git+https://github.com/w3f/ring-vrf?rev=3119f51#3119f51b54b69308abfb0671f6176cb125ae1bf1" +source = "git+https://github.com/w3f/ring-vrf?rev=f4fe253#f4fe2534ccc6d916cd10d9c16891e673728ec8b4" dependencies = [ "ark-bls12-381", "ark-ec", "ark-ed-on-bls12-381-bandersnatch", "ark-ff", - "ark-scale", "ark-serialize", "ark-std", "dleq_vrf", @@ -1352,7 +1351,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -2274,9 +2273,9 @@ dependencies = [ [[package]] name = "cfg-expr" -version = "0.15.4" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b40ccee03b5175c18cde8f37e7d2a33bcef6f8ec8f7cc0d81090d1bb380949c9" +checksum = "03915af431787e6ffdcc74c645077518c6b6e01f80b761e0fbbfa288536311b3" dependencies = [ "smallvec", ] @@ -2334,7 +2333,7 @@ name = "chain-spec-builder" version = "2.0.0" dependencies = [ "ansi_term", - "clap 4.4.2", + "clap 4.4.3", "node-cli", "rand 0.8.5", "sc-chain-spec", @@ -2464,9 +2463,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.2" +version = "4.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6" +checksum = "84ed82781cea27b43c9b106a979fe450a13a31aab0500595fb3fc06616de08e6" dependencies = [ "clap_builder", "clap_derive 4.4.2", @@ -2490,7 +2489,7 @@ version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "586a385f7ef2f8b4d86bddaa0c094794e7ccbfe5ffef1f434fe928143fc783a5" dependencies = [ - "clap 4.4.2", + "clap 4.4.3", ] [[package]] @@ -2515,7 +2514,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -2704,7 +2703,7 @@ dependencies = [ [[package]] name = "common" version = "0.1.0" -source = "git+https://github.com/w3f/ring-proof?rev=0e948f3#0e948f3c28cbacecdd3020403c4841c0eb339213" +source = "git+https://github.com/w3f/ring-proof?rev=8657210#86572101f4210647984ab4efedba6b3fcc890895" dependencies = [ "ark-ec", "ark-ff", @@ -2713,6 +2712,7 @@ dependencies = [ "ark-std", "fflonk", "merlin 3.0.0", + "rand_chacha 0.3.1", ] [[package]] @@ -3085,7 +3085,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.4.2", + "clap 4.4.3", "criterion-plot", "futures", "is-terminal", @@ -3250,7 +3250,7 @@ dependencies = [ name = "cumulus-client-cli" version = "0.1.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.3", "parity-scale-codec", "sc-chain-spec", "sc-cli", @@ -3574,7 +3574,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -3899,14 +3899,6 @@ dependencies = [ "sp-trie", ] -[[package]] -name = "cumulus-test-relay-validation-worker-provider" -version = "0.1.0" -dependencies = [ - "polkadot-node-core-pvf", - "toml 0.7.6", -] - [[package]] name = "cumulus-test-runtime" version = "0.1.0" @@ -3943,7 +3935,7 @@ name = "cumulus-test-service" version = "0.1.0" dependencies = [ "async-trait", - "clap 4.4.2", + "clap 4.4.3", "criterion 0.5.1", "cumulus-client-cli", "cumulus-client-consensus-common", @@ -3958,7 +3950,6 @@ dependencies = [ "cumulus-relay-chain-minimal-node", "cumulus-test-client", "cumulus-test-relay-sproof-builder", - "cumulus-test-relay-validation-worker-provider", "cumulus-test-runtime", "frame-system", "frame-system-rpc-runtime-api", @@ -4066,7 +4057,7 @@ checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -4106,7 +4097,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -4123,7 +4114,7 @@ checksum = "50c49547d73ba8dcfd4ad7325d64c6d5391ff4224d498fc39a6f3f49825a530d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -4421,7 +4412,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -4433,7 +4424,7 @@ checksum = "86e3bdc80eee6e16b2b6b0f87fbc98c04bee3455e35174c0de1a125d0688c632" [[package]] name = "dleq_vrf" version = "0.0.2" -source = "git+https://github.com/w3f/ring-vrf?rev=3119f51#3119f51b54b69308abfb0671f6176cb125ae1bf1" +source = "git+https://github.com/w3f/ring-vrf?rev=f4fe253#f4fe2534ccc6d916cd10d9c16891e673728ec8b4" dependencies = [ "ark-ec", "ark-ff", @@ -4464,18 +4455,18 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "docify" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "029de870d175d11969524d91a3fb2cbf6d488b853bff99d41cf65e533ac7d9d2" +checksum = "ff509d6aa8e7ca86b36eb3d593132e64204597de3ccb763ffd8bfd2264d54cf3" dependencies = [ "docify_macros", ] [[package]] name = "docify_macros" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac43324656a1b05eb0186deb51f27d2d891c704c37f34de281ef6297ba193e5" +checksum = "b135598b950330937f3a0ddfbe908106ee2ecd5fa95d8ae7952a3c3863efe8da" dependencies = [ "common-path", "derive-syn-parse", @@ -4483,7 +4474,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.31", + "syn 2.0.33", "termcolor", "toml 0.7.6", "walkdir", @@ -4704,7 +4695,7 @@ checksum = "5e9a1f9f7d83e59740248a6e14ecf93929ade55027844dfcea78beafccc15745" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -4715,7 +4706,7 @@ checksum = "c2ad8cef1d801a4686bfd8919f0b30eac4c8e48968c437a6405ded4fb5272d2b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -4838,12 +4829,6 @@ dependencies = [ "futures", ] -[[package]] -name = "exitcode" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193" - [[package]] name = "expander" version = "0.0.4" @@ -4856,18 +4841,6 @@ dependencies = [ "quote", ] -[[package]] -name = "expander" -version = "0.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3774182a5df13c3d1690311ad32fbe913feef26baba609fa2dd5f72042bd2ab6" -dependencies = [ - "blake2", - "fs-err", - "proc-macro2", - "quote", -] - [[package]] name = "expander" version = "2.0.0" @@ -4878,7 +4851,7 @@ dependencies = [ "fs-err", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -5168,7 +5141,7 @@ dependencies = [ "Inflector", "array-bytes", "chrono", - "clap 4.4.2", + "clap 4.4.3", "comfy-table", "frame-benchmarking", "frame-support", @@ -5234,7 +5207,7 @@ dependencies = [ "quote", "scale-info", "sp-arithmetic", - "syn 2.0.31", + "syn 2.0.33", "trybuild", ] @@ -5260,7 +5233,7 @@ dependencies = [ name = "frame-election-solution-type-fuzzer" version = "2.0.0-alpha.5" dependencies = [ - "clap 4.4.2", + "clap 4.4.3", "frame-election-provider-solution-type", "frame-election-provider-support", "frame-support", @@ -5386,7 +5359,7 @@ dependencies = [ "proc-macro-warning", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -5397,7 +5370,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -5406,7 +5379,7 @@ version = "3.0.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -5629,7 +5602,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -6911,6 +6884,15 @@ dependencies = [ "thiserror", ] +[[package]] +name = "layout-rs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1164ef87cb9607c2d887216eca79f0fc92895affe1789bba805dd38d829584e0" +dependencies = [ + "log", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -7630,7 +7612,7 @@ dependencies = [ "macro_magic_core", "macro_magic_macros", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -7644,7 +7626,7 @@ dependencies = [ "macro_magic_core_macros", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -7655,7 +7637,7 @@ checksum = "c12469fc165526520dff2807c2975310ab47cf7190a45b99b49a7dc8befab17b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -7666,7 +7648,7 @@ checksum = "b8fb85ec1620619edf2984a7693497d4ec88a9665d8b87e942856884c92dbf2a" dependencies = [ "macro_magic_core", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -7690,15 +7672,6 @@ dependencies = [ "regex-automata 0.1.10", ] -[[package]] -name = "matchers" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" -dependencies = [ - "regex-automata 0.1.10", -] - [[package]] name = "matches" version = "0.1.10" @@ -8162,7 +8135,7 @@ name = "node-bench" version = "0.9.0-dev" dependencies = [ "array-bytes", - "clap 4.4.2", + "clap 4.4.3", "derive_more", "fs_extra", "futures", @@ -8199,7 +8172,7 @@ version = "3.0.0-dev" dependencies = [ "array-bytes", "assert_cmd", - "clap 4.4.2", + "clap 4.4.3", "clap_complete", "criterion 0.4.0", "frame-benchmarking-cli", @@ -8325,7 +8298,7 @@ dependencies = [ name = "node-inspect" version = "0.9.0-dev" dependencies = [ - "clap 4.4.2", + "clap 4.4.3", "parity-scale-codec", "sc-cli", "sc-client-api", @@ -8379,7 +8352,7 @@ dependencies = [ name = "node-runtime-generate-bags" version = "3.0.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.3", "generate-bags", "kitchensink-runtime", ] @@ -8388,7 +8361,7 @@ dependencies = [ name = "node-template" version = "4.0.0-dev" dependencies = [ - "clap 4.4.2", + "clap 4.4.3", "frame-benchmarking", "frame-benchmarking-cli", "frame-system", @@ -8431,7 +8404,7 @@ dependencies = [ name = "node-template-release" version = "3.0.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.3", "flate2", "fs_extra", "glob", @@ -8542,16 +8515,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" -[[package]] -name = "nu-ansi-term" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" -dependencies = [ - "overload", - "winapi", -] - [[package]] name = "num" version = "0.4.1" @@ -8726,9 +8689,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "orchestra" -version = "0.0.5" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "227585216d05ba65c7ab0a0450a3cf2cbd81a98862a54c4df8e14d5ac6adb015" +checksum = "46d78e1deb2a8d54fc1f063a544130db4da31dfe4d5d3b493186424910222a76" dependencies = [ "async-trait", "dyn-clonable", @@ -8743,12 +8706,16 @@ dependencies = [ [[package]] name = "orchestra-proc-macro" -version = "0.0.5" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2871aadd82a2c216ee68a69837a526dfe788ecbe74c4c5038a6acdbff6653066" +checksum = "d035b1f968d91a826f2e34a9d6d02cb2af5aa7ca39ebd27922d850ab4b2dd2c6" dependencies = [ - "expander 0.0.6", - "itertools 0.10.5", + "anyhow", + "expander 2.0.0", + "fs-err", + "indexmap 2.0.0", + "itertools 0.11.0", + "layout-rs", "petgraph", "proc-macro-crate", "proc-macro2", @@ -8771,12 +8738,6 @@ version = "6.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac" -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "owo-colors" version = "3.5.0" @@ -9373,7 +9334,7 @@ version = "4.0.0-dev" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -10444,7 +10405,7 @@ dependencies = [ "proc-macro2", "quote", "sp-runtime", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -10510,6 +10471,7 @@ dependencies = [ name = "pallet-sudo" version = "4.0.0-dev" dependencies = [ + "docify", "frame-benchmarking", "frame-support", "frame-system", @@ -10832,7 +10794,7 @@ dependencies = [ name = "parachain-template-node" version = "0.1.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.3", "color-print", "cumulus-client-cli", "cumulus-client-collator", @@ -11309,7 +11271,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -11350,7 +11312,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -11568,7 +11530,7 @@ dependencies = [ name = "polkadot-cli" version = "1.0.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.3", "frame-benchmarking-cli", "futures", "log", @@ -12048,7 +12010,6 @@ dependencies = [ "slotmap", "sp-core", "sp-maybe-compressed-blob", - "sp-tracing", "sp-wasm-interface", "substrate-build-script-utils", "tempfile", @@ -12389,7 +12350,7 @@ dependencies = [ "bridge-hub-kusama-runtime", "bridge-hub-polkadot-runtime", "bridge-hub-rococo-runtime", - "clap 4.4.2", + "clap 4.4.3", "collectives-polkadot-runtime", "color-print", "contracts-rococo-runtime", @@ -13004,7 +12965,7 @@ version = "1.0.0" dependencies = [ "assert_matches", "async-trait", - "clap 4.4.2", + "clap 4.4.3", "color-eyre", "futures", "futures-timer", @@ -13150,7 +13111,7 @@ dependencies = [ name = "polkadot-voter-bags" version = "1.0.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.3", "generate-bags", "polkadot-runtime", "sp-io", @@ -13330,7 +13291,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" dependencies = [ "proc-macro2", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -13350,9 +13311,9 @@ dependencies = [ [[package]] name = "prioritized-metered-channel" -version = "0.2.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "382698e48a268c832d0b181ed438374a6bb708a82a8ca273bb0f61c74cf209c4" +checksum = "e99f0c89bd88f393aab44a4ab949351f7bc7e7e1179d11ecbfe50cbe4c47e342" dependencies = [ "coarsetime", "crossbeam-queue", @@ -13412,14 +13373,14 @@ checksum = "3d1eaa7fa0aa1929ffdf7eeb6eac234dde6268914a14ad44d23521ab6a9b258e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" dependencies = [ "unicode-ident", ] @@ -13458,7 +13419,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -13849,7 +13810,7 @@ checksum = "7f7473c2cfcf90008193dd0e3e16599455cb601a9fce322b5bb55de799664925" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -13912,7 +13873,7 @@ checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" name = "remote-ext-tests-bags-list" version = "1.0.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.3", "frame-system", "kusama-runtime-constants", "log", @@ -14000,13 +13961,14 @@ dependencies = [ [[package]] name = "ring" version = "0.1.0" -source = "git+https://github.com/w3f/ring-proof?rev=0e948f3#0e948f3c28cbacecdd3020403c4841c0eb339213" +source = "git+https://github.com/w3f/ring-proof?rev=8657210#86572101f4210647984ab4efedba6b3fcc890895" dependencies = [ "ark-ec", "ark-ff", "ark-poly", "ark-serialize", "ark-std", + "blake2", "common", "fflonk", "merlin 3.0.0", @@ -14611,7 +14573,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -14620,7 +14582,7 @@ version = "0.10.0-dev" dependencies = [ "array-bytes", "chrono", - "clap 4.4.2", + "clap 4.4.3", "fdlimit", "futures", "futures-timer", @@ -15116,7 +15078,7 @@ dependencies = [ "substrate-test-runtime", "tempfile", "tracing", - "tracing-subscriber 0.2.25", + "tracing-subscriber", "wat", ] @@ -15724,7 +15686,7 @@ dependencies = [ name = "sc-storage-monitor" version = "0.1.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.3", "fs4", "log", "sc-client-db", @@ -15814,7 +15776,7 @@ dependencies = [ "thiserror", "tracing", "tracing-log", - "tracing-subscriber 0.2.25", + "tracing-subscriber", ] [[package]] @@ -15824,7 +15786,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -16178,7 +16140,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -16192,9 +16154,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.105" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ "itoa", "ryu", @@ -16244,7 +16206,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -16393,18 +16355,6 @@ dependencies = [ "libc", ] -[[package]] -name = "signal-hook-tokio" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213241f76fb1e37e27de3b6aa1b068a2c333233b59cca6634f634b80a27ecf1e" -dependencies = [ - "futures-core", - "libc", - "signal-hook", - "tokio", -] - [[package]] name = "signature" version = "1.6.4" @@ -16684,7 +16634,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -17022,7 +16972,6 @@ name = "sp-core" version = "21.0.0" dependencies = [ "array-bytes", - "arrayvec 0.7.4", "bandersnatch_vrfs", "bitflags 1.3.2", "blake2", @@ -17085,7 +17034,7 @@ version = "9.0.0" dependencies = [ "quote", "sp-core-hashing", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -17129,7 +17078,7 @@ version = "8.0.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -17267,7 +17216,7 @@ dependencies = [ name = "sp-npos-elections-fuzzer" version = "2.0.0-alpha.5" dependencies = [ - "clap 4.4.2", + "clap 4.4.3", "honggfuzz", "rand 0.8.5", "sp-npos-elections", @@ -17360,7 +17309,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -17524,7 +17473,7 @@ dependencies = [ "sp-std", "tracing", "tracing-core", - "tracing-subscriber 0.2.25", + "tracing-subscriber", ] [[package]] @@ -17600,7 +17549,7 @@ dependencies = [ "proc-macro2", "quote", "sp-version", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -17806,47 +17755,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "staging-staking-miner" -version = "1.0.0" -dependencies = [ - "assert_cmd", - "clap 4.4.2", - "exitcode", - "frame-election-provider-support", - "frame-remote-externalities", - "frame-support", - "frame-system", - "futures-util", - "jsonrpsee", - "log", - "pallet-balances", - "pallet-election-provider-multi-phase", - "pallet-staking", - "pallet-transaction-payment", - "parity-scale-codec", - "paste", - "polkadot-core-primitives", - "polkadot-runtime", - "polkadot-runtime-common", - "sc-transaction-pool-api", - "serde", - "serde_json", - "signal-hook", - "signal-hook-tokio", - "sp-core", - "sp-npos-elections", - "sp-runtime", - "sp-state-machine", - "sp-version", - "staging-kusama-runtime", - "sub-tokens", - "thiserror", - "tokio", - "tracing-subscriber 0.3.17", - "westend-runtime", -] - [[package]] name = "staging-xcm" version = "1.0.0" @@ -18021,19 +17929,11 @@ dependencies = [ "webrtc-util", ] -[[package]] -name = "sub-tokens" -version = "0.1.0" -source = "git+https://github.com/paritytech/substrate-debug-kit?branch=master#e12503ab781e913735dc389865a3b8b4a6c6399d" -dependencies = [ - "separator", -] - [[package]] name = "subkey" version = "3.0.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.3", "sc-cli", ] @@ -18075,7 +17975,7 @@ dependencies = [ name = "substrate-frame-cli" version = "4.0.0-dev" dependencies = [ - "clap 4.4.2", + "clap 4.4.3", "frame-support", "frame-system", "sc-cli", @@ -18424,9 +18324,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.31" +version = "2.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" +checksum = "9caece70c63bfba29ec2fed841a09851b14a235c60010fa4de58089b6c025668" dependencies = [ "proc-macro2", "quote", @@ -18534,7 +18434,7 @@ dependencies = [ name = "test-parachain-adder-collator" version = "1.0.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.3", "futures", "futures-timer", "log", @@ -18553,7 +18453,6 @@ dependencies = [ "sp-keyring", "substrate-test-utils", "test-parachain-adder", - "test-parachain-adder-collator", "tokio", ] @@ -18583,7 +18482,7 @@ dependencies = [ name = "test-parachain-undying-collator" version = "1.0.0" dependencies = [ - "clap 4.4.2", + "clap 4.4.3", "futures", "futures-timer", "log", @@ -18602,7 +18501,6 @@ dependencies = [ "sp-keyring", "substrate-test-utils", "test-parachain-undying", - "test-parachain-undying-collator", "tokio", ] @@ -18673,7 +18571,7 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -18853,7 +18751,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -19034,7 +18932,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -19077,7 +18975,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -19110,7 +19008,7 @@ dependencies = [ "ansi_term", "chrono", "lazy_static", - "matchers 0.0.1", + "matchers", "parking_lot 0.11.2", "regex", "serde", @@ -19124,29 +19022,11 @@ dependencies = [ "tracing-serde", ] -[[package]] -name = "tracing-subscriber" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" -dependencies = [ - "matchers 0.1.0", - "nu-ansi-term", - "once_cell", - "regex", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", -] - [[package]] name = "trie-bench" -version = "0.37.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f54b4f9d51d368e62cf7e0730c7c1e18fc658cc84333656bab5b328f44aa964" +checksum = "a4680cb226e31d2a096592d0edecdda91cc371743002f80c0f8cf80219819b3b" dependencies = [ "criterion 0.4.0", "hash-db", @@ -19160,9 +19040,9 @@ dependencies = [ [[package]] name = "trie-db" -version = "0.27.1" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "767abe6ffed88a1889671a102c2861ae742726f52e0a5a425b92c9fbfa7e9c85" +checksum = "ff28e0f815c2fea41ebddf148e008b077d2faddb026c9555b29696114d602642" dependencies = [ "hash-db", "hashbrown 0.13.2", @@ -19248,7 +19128,7 @@ version = "0.10.0-dev" dependencies = [ "assert_cmd", "async-trait", - "clap 4.4.2", + "clap 4.4.3", "frame-remote-externalities", "frame-try-runtime", "hex", @@ -19644,7 +19524,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", "wasm-bindgen-shared", ] @@ -19678,7 +19558,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -20814,7 +20694,7 @@ dependencies = [ "Inflector", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -20933,7 +20813,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 8a9c2afe1d1b..d1078e3c86a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,31 +8,31 @@ license = "GPL-3.0-only" resolver = "2" members = [ - "cumulus/bridges/bin/runtime-common", - "cumulus/bridges/modules/grandpa", - "cumulus/bridges/modules/messages", - "cumulus/bridges/modules/parachains", - "cumulus/bridges/modules/relayers", - "cumulus/bridges/modules/xcm-bridge-hub-router", - "cumulus/bridges/primitives/chain-asset-hub-kusama", - "cumulus/bridges/primitives/chain-asset-hub-polkadot", - "cumulus/bridges/primitives/chain-bridge-hub-cumulus", - "cumulus/bridges/primitives/chain-bridge-hub-kusama", - "cumulus/bridges/primitives/chain-bridge-hub-polkadot", - "cumulus/bridges/primitives/chain-bridge-hub-rococo", - "cumulus/bridges/primitives/chain-bridge-hub-wococo", - "cumulus/bridges/primitives/chain-kusama", - "cumulus/bridges/primitives/chain-polkadot", - "cumulus/bridges/primitives/chain-rococo", - "cumulus/bridges/primitives/chain-wococo", - "cumulus/bridges/primitives/header-chain", - "cumulus/bridges/primitives/messages", - "cumulus/bridges/primitives/parachains", - "cumulus/bridges/primitives/polkadot-core", - "cumulus/bridges/primitives/relayers", - "cumulus/bridges/primitives/runtime", - "cumulus/bridges/primitives/test-utils", - "cumulus/bridges/primitives/xcm-bridge-hub-router", + "bridges/bin/runtime-common", + "bridges/modules/grandpa", + "bridges/modules/messages", + "bridges/modules/parachains", + "bridges/modules/relayers", + "bridges/modules/xcm-bridge-hub-router", + "bridges/primitives/chain-asset-hub-kusama", + "bridges/primitives/chain-asset-hub-polkadot", + "bridges/primitives/chain-bridge-hub-cumulus", + "bridges/primitives/chain-bridge-hub-kusama", + "bridges/primitives/chain-bridge-hub-polkadot", + "bridges/primitives/chain-bridge-hub-rococo", + "bridges/primitives/chain-bridge-hub-wococo", + "bridges/primitives/chain-kusama", + "bridges/primitives/chain-polkadot", + "bridges/primitives/chain-rococo", + "bridges/primitives/chain-wococo", + "bridges/primitives/header-chain", + "bridges/primitives/messages", + "bridges/primitives/parachains", + "bridges/primitives/polkadot-core", + "bridges/primitives/relayers", + "bridges/primitives/runtime", + "bridges/primitives/test-utils", + "bridges/primitives/xcm-bridge-hub-router", "cumulus/client/cli", "cumulus/client/collator", "cumulus/client/consensus/aura", @@ -92,7 +92,6 @@ members = [ "cumulus/primitives/utility", "cumulus/test/client", "cumulus/test/relay-sproof-builder", - "cumulus/test/relay-validation-worker-provider", "cumulus/test/runtime", "cumulus/test/service", "cumulus/xcm/xcm-emulator", @@ -172,7 +171,6 @@ members = [ "polkadot/statement-table", "polkadot/utils/generate-bags", "polkadot/utils/remote-ext-tests/bags-list", - "polkadot/utils/staking-miner", "polkadot/xcm", "polkadot/xcm/pallet-xcm", "polkadot/xcm/pallet-xcm-benchmarks", diff --git a/cumulus/bridges/.gitignore b/bridges/.gitignore similarity index 100% rename from cumulus/bridges/.gitignore rename to bridges/.gitignore diff --git a/cumulus/bridges/CODE_OF_CONDUCT.md b/bridges/CODE_OF_CONDUCT.md similarity index 100% rename from cumulus/bridges/CODE_OF_CONDUCT.md rename to bridges/CODE_OF_CONDUCT.md diff --git a/cumulus/bridges/LICENSE b/bridges/LICENSE similarity index 100% rename from cumulus/bridges/LICENSE rename to bridges/LICENSE diff --git a/cumulus/bridges/README.md b/bridges/README.md similarity index 100% rename from cumulus/bridges/README.md rename to bridges/README.md diff --git a/cumulus/bridges/SECURITY.md b/bridges/SECURITY.md similarity index 100% rename from cumulus/bridges/SECURITY.md rename to bridges/SECURITY.md diff --git a/cumulus/bridges/bin/runtime-common/Cargo.toml b/bridges/bin/runtime-common/Cargo.toml similarity index 72% rename from cumulus/bridges/bin/runtime-common/Cargo.toml rename to bridges/bin/runtime-common/Cargo.toml index b139f835c1a0..5c3fefc69ce7 100644 --- a/cumulus/bridges/bin/runtime-common/Cargo.toml +++ b/bridges/bin/runtime-common/Cargo.toml @@ -29,24 +29,24 @@ pallet-bridge-relayers = { path = "../../modules/relayers", default-features = f # Substrate dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -frame-system = { path = "../../../../substrate/frame/system", default-features = false } -pallet-transaction-payment = { path = "../../../../substrate/frame/transaction-payment", default-features = false } -pallet-utility = { path = "../../../../substrate/frame/utility", default-features = false } -sp-api = { path = "../../../../substrate/primitives/api", default-features = false } -sp-core = { path = "../../../../substrate/primitives/core", default-features = false } -sp-io = { path = "../../../../substrate/primitives/io", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } -sp-trie = { path = "../../../../substrate/primitives/trie", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +frame-system = { path = "../../../substrate/frame/system", default-features = false } +pallet-transaction-payment = { path = "../../../substrate/frame/transaction-payment", default-features = false } +pallet-utility = { path = "../../../substrate/frame/utility", default-features = false } +sp-api = { path = "../../../substrate/primitives/api", default-features = false } +sp-core = { path = "../../../substrate/primitives/core", default-features = false } +sp-io = { path = "../../../substrate/primitives/io", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } +sp-trie = { path = "../../../substrate/primitives/trie", default-features = false } # Polkadot dependencies -xcm = { package = "staging-xcm", path = "../../../../polkadot/xcm", default-features = false } -xcm-builder = { package = "staging-xcm-builder", path = "../../../../polkadot/xcm/xcm-builder", default-features = false } +xcm = { package = "staging-xcm", path = "../../../polkadot/xcm", default-features = false } +xcm-builder = { package = "staging-xcm-builder", path = "../../../polkadot/xcm/xcm-builder", default-features = false } [dev-dependencies] bp-test-utils = { path = "../../primitives/test-utils" } -pallet-balances = { path = "../../../../substrate/frame/balances" } +pallet-balances = { path = "../../../substrate/frame/balances" } [features] default = [ "std" ] diff --git a/cumulus/bridges/bin/runtime-common/src/integrity.rs b/bridges/bin/runtime-common/src/integrity.rs similarity index 96% rename from cumulus/bridges/bin/runtime-common/src/integrity.rs rename to bridges/bin/runtime-common/src/integrity.rs index 290c22f835d2..d3827a14dd6c 100644 --- a/cumulus/bridges/bin/runtime-common/src/integrity.rs +++ b/bridges/bin/runtime-common/src/integrity.rs @@ -27,7 +27,6 @@ use codec::Encode; use frame_support::{storage::generator::StorageValue, traits::Get, weights::Weight}; use frame_system::limits; use pallet_bridge_messages::WeightInfoExt as _; -use sp_runtime::traits::SignedExtension; /// Macro that ensures that the runtime configuration and chain primitives crate are sharing /// the same types (nonce, block number, hash, hasher, account id and header). @@ -347,15 +346,3 @@ pub fn check_message_lane_weights< ); } } - -/// Check that the `AdditionalSigned` type of a wrapped runtime is the same as the one of the -/// corresponding actual runtime. -/// -/// This method doesn't perform any `assert`. If the condition is not true it will generate a -/// compile-time error. -pub fn check_additional_signed() -where - SignedExt: SignedExtension, - IndirectSignedExt: SignedExtension, -{ -} diff --git a/cumulus/bridges/bin/runtime-common/src/lib.rs b/bridges/bin/runtime-common/src/lib.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/lib.rs rename to bridges/bin/runtime-common/src/lib.rs diff --git a/cumulus/bridges/bin/runtime-common/src/messages.rs b/bridges/bin/runtime-common/src/messages.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/messages.rs rename to bridges/bin/runtime-common/src/messages.rs diff --git a/cumulus/bridges/bin/runtime-common/src/messages_api.rs b/bridges/bin/runtime-common/src/messages_api.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/messages_api.rs rename to bridges/bin/runtime-common/src/messages_api.rs diff --git a/cumulus/bridges/bin/runtime-common/src/messages_benchmarking.rs b/bridges/bin/runtime-common/src/messages_benchmarking.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/messages_benchmarking.rs rename to bridges/bin/runtime-common/src/messages_benchmarking.rs diff --git a/cumulus/bridges/bin/runtime-common/src/messages_call_ext.rs b/bridges/bin/runtime-common/src/messages_call_ext.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/messages_call_ext.rs rename to bridges/bin/runtime-common/src/messages_call_ext.rs diff --git a/cumulus/bridges/bin/runtime-common/src/messages_generation.rs b/bridges/bin/runtime-common/src/messages_generation.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/messages_generation.rs rename to bridges/bin/runtime-common/src/messages_generation.rs diff --git a/cumulus/bridges/bin/runtime-common/src/messages_xcm_extension.rs b/bridges/bin/runtime-common/src/messages_xcm_extension.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/messages_xcm_extension.rs rename to bridges/bin/runtime-common/src/messages_xcm_extension.rs diff --git a/cumulus/bridges/bin/runtime-common/src/mock.rs b/bridges/bin/runtime-common/src/mock.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/mock.rs rename to bridges/bin/runtime-common/src/mock.rs diff --git a/cumulus/bridges/bin/runtime-common/src/parachains_benchmarking.rs b/bridges/bin/runtime-common/src/parachains_benchmarking.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/parachains_benchmarking.rs rename to bridges/bin/runtime-common/src/parachains_benchmarking.rs diff --git a/cumulus/bridges/bin/runtime-common/src/priority_calculator.rs b/bridges/bin/runtime-common/src/priority_calculator.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/priority_calculator.rs rename to bridges/bin/runtime-common/src/priority_calculator.rs diff --git a/cumulus/bridges/bin/runtime-common/src/refund_relayer_extension.rs b/bridges/bin/runtime-common/src/refund_relayer_extension.rs similarity index 100% rename from cumulus/bridges/bin/runtime-common/src/refund_relayer_extension.rs rename to bridges/bin/runtime-common/src/refund_relayer_extension.rs diff --git a/cumulus/bridges/docs/complex-relay.html b/bridges/docs/complex-relay.html similarity index 100% rename from cumulus/bridges/docs/complex-relay.html rename to bridges/docs/complex-relay.html diff --git a/cumulus/bridges/docs/grandpa-finality-relay.html b/bridges/docs/grandpa-finality-relay.html similarity index 100% rename from cumulus/bridges/docs/grandpa-finality-relay.html rename to bridges/docs/grandpa-finality-relay.html diff --git a/cumulus/bridges/docs/high-level-overview.md b/bridges/docs/high-level-overview.md similarity index 100% rename from cumulus/bridges/docs/high-level-overview.md rename to bridges/docs/high-level-overview.md diff --git a/cumulus/bridges/docs/messages-relay.html b/bridges/docs/messages-relay.html similarity index 100% rename from cumulus/bridges/docs/messages-relay.html rename to bridges/docs/messages-relay.html diff --git a/cumulus/bridges/docs/parachains-finality-relay.html b/bridges/docs/parachains-finality-relay.html similarity index 100% rename from cumulus/bridges/docs/parachains-finality-relay.html rename to bridges/docs/parachains-finality-relay.html diff --git a/cumulus/bridges/docs/polkadot-kusama-bridge-overview.md b/bridges/docs/polkadot-kusama-bridge-overview.md similarity index 100% rename from cumulus/bridges/docs/polkadot-kusama-bridge-overview.md rename to bridges/docs/polkadot-kusama-bridge-overview.md diff --git a/cumulus/bridges/docs/polkadot-kusama-bridge.html b/bridges/docs/polkadot-kusama-bridge.html similarity index 100% rename from cumulus/bridges/docs/polkadot-kusama-bridge.html rename to bridges/docs/polkadot-kusama-bridge.html diff --git a/cumulus/bridges/modules/grandpa/Cargo.toml b/bridges/modules/grandpa/Cargo.toml similarity index 65% rename from cumulus/bridges/modules/grandpa/Cargo.toml rename to bridges/modules/grandpa/Cargo.toml index 87eda6b29a70..05d6a8e5c26b 100644 --- a/cumulus/bridges/modules/grandpa/Cargo.toml +++ b/bridges/modules/grandpa/Cargo.toml @@ -20,20 +20,20 @@ bp-header-chain = { path = "../../primitives/header-chain", default-features = f # Substrate Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -frame-system = { path = "../../../../substrate/frame/system", default-features = false } -sp-consensus-grandpa = { path = "../../../../substrate/primitives/consensus/grandpa", default-features = false, features = ["serde"] } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false, features = ["serde"] } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } -sp-trie = { path = "../../../../substrate/primitives/trie", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +frame-system = { path = "../../../substrate/frame/system", default-features = false } +sp-consensus-grandpa = { path = "../../../substrate/primitives/consensus/grandpa", default-features = false, features = ["serde"] } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false, features = ["serde"] } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } +sp-trie = { path = "../../../substrate/primitives/trie", default-features = false } # Optional Benchmarking Dependencies bp-test-utils = { path = "../../primitives/test-utils", default-features = false, optional = true } -frame-benchmarking = { path = "../../../../substrate/frame/benchmarking", default-features = false, optional = true } +frame-benchmarking = { path = "../../../substrate/frame/benchmarking", default-features = false, optional = true } [dev-dependencies] -sp-core = { path = "../../../../substrate/primitives/core" } -sp-io = { path = "../../../../substrate/primitives/io" } +sp-core = { path = "../../../substrate/primitives/core" } +sp-io = { path = "../../../substrate/primitives/io" } [features] default = [ "std" ] diff --git a/cumulus/bridges/modules/grandpa/README.md b/bridges/modules/grandpa/README.md similarity index 100% rename from cumulus/bridges/modules/grandpa/README.md rename to bridges/modules/grandpa/README.md diff --git a/cumulus/bridges/modules/grandpa/src/benchmarking.rs b/bridges/modules/grandpa/src/benchmarking.rs similarity index 100% rename from cumulus/bridges/modules/grandpa/src/benchmarking.rs rename to bridges/modules/grandpa/src/benchmarking.rs diff --git a/cumulus/bridges/modules/grandpa/src/call_ext.rs b/bridges/modules/grandpa/src/call_ext.rs similarity index 100% rename from cumulus/bridges/modules/grandpa/src/call_ext.rs rename to bridges/modules/grandpa/src/call_ext.rs diff --git a/cumulus/bridges/modules/grandpa/src/lib.rs b/bridges/modules/grandpa/src/lib.rs similarity index 100% rename from cumulus/bridges/modules/grandpa/src/lib.rs rename to bridges/modules/grandpa/src/lib.rs diff --git a/cumulus/bridges/modules/grandpa/src/mock.rs b/bridges/modules/grandpa/src/mock.rs similarity index 100% rename from cumulus/bridges/modules/grandpa/src/mock.rs rename to bridges/modules/grandpa/src/mock.rs diff --git a/cumulus/bridges/modules/grandpa/src/storage_types.rs b/bridges/modules/grandpa/src/storage_types.rs similarity index 100% rename from cumulus/bridges/modules/grandpa/src/storage_types.rs rename to bridges/modules/grandpa/src/storage_types.rs diff --git a/cumulus/bridges/modules/grandpa/src/weights.rs b/bridges/modules/grandpa/src/weights.rs similarity index 100% rename from cumulus/bridges/modules/grandpa/src/weights.rs rename to bridges/modules/grandpa/src/weights.rs diff --git a/cumulus/bridges/modules/messages/Cargo.toml b/bridges/modules/messages/Cargo.toml similarity index 67% rename from cumulus/bridges/modules/messages/Cargo.toml rename to bridges/modules/messages/Cargo.toml index 5eecdb147fad..7b7ea0619810 100644 --- a/cumulus/bridges/modules/messages/Cargo.toml +++ b/bridges/modules/messages/Cargo.toml @@ -19,17 +19,17 @@ bp-runtime = { path = "../../primitives/runtime", default-features = false } # Substrate Dependencies -frame-benchmarking = { path = "../../../../substrate/frame/benchmarking", default-features = false, optional = true } -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -frame-system = { path = "../../../../substrate/frame/system", default-features = false } -sp-core = { path = "../../../../substrate/primitives/core", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-benchmarking = { path = "../../../substrate/frame/benchmarking", default-features = false, optional = true } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +frame-system = { path = "../../../substrate/frame/system", default-features = false } +sp-core = { path = "../../../substrate/primitives/core", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [dev-dependencies] bp-test-utils = { path = "../../primitives/test-utils" } -pallet-balances = { path = "../../../../substrate/frame/balances" } -sp-io = { path = "../../../../substrate/primitives/io" } +pallet-balances = { path = "../../../substrate/frame/balances" } +sp-io = { path = "../../../substrate/primitives/io" } [features] default = [ "std" ] diff --git a/cumulus/bridges/modules/messages/README.md b/bridges/modules/messages/README.md similarity index 100% rename from cumulus/bridges/modules/messages/README.md rename to bridges/modules/messages/README.md diff --git a/cumulus/bridges/modules/messages/src/benchmarking.rs b/bridges/modules/messages/src/benchmarking.rs similarity index 100% rename from cumulus/bridges/modules/messages/src/benchmarking.rs rename to bridges/modules/messages/src/benchmarking.rs diff --git a/cumulus/bridges/modules/messages/src/inbound_lane.rs b/bridges/modules/messages/src/inbound_lane.rs similarity index 100% rename from cumulus/bridges/modules/messages/src/inbound_lane.rs rename to bridges/modules/messages/src/inbound_lane.rs diff --git a/cumulus/bridges/modules/messages/src/lib.rs b/bridges/modules/messages/src/lib.rs similarity index 100% rename from cumulus/bridges/modules/messages/src/lib.rs rename to bridges/modules/messages/src/lib.rs diff --git a/cumulus/bridges/modules/messages/src/mock.rs b/bridges/modules/messages/src/mock.rs similarity index 100% rename from cumulus/bridges/modules/messages/src/mock.rs rename to bridges/modules/messages/src/mock.rs diff --git a/cumulus/bridges/modules/messages/src/outbound_lane.rs b/bridges/modules/messages/src/outbound_lane.rs similarity index 100% rename from cumulus/bridges/modules/messages/src/outbound_lane.rs rename to bridges/modules/messages/src/outbound_lane.rs diff --git a/cumulus/bridges/modules/messages/src/weights.rs b/bridges/modules/messages/src/weights.rs similarity index 100% rename from cumulus/bridges/modules/messages/src/weights.rs rename to bridges/modules/messages/src/weights.rs diff --git a/cumulus/bridges/modules/messages/src/weights_ext.rs b/bridges/modules/messages/src/weights_ext.rs similarity index 100% rename from cumulus/bridges/modules/messages/src/weights_ext.rs rename to bridges/modules/messages/src/weights_ext.rs diff --git a/cumulus/bridges/modules/parachains/Cargo.toml b/bridges/modules/parachains/Cargo.toml similarity index 71% rename from cumulus/bridges/modules/parachains/Cargo.toml rename to bridges/modules/parachains/Cargo.toml index 50a838edf56d..4eb09ed78d17 100644 --- a/cumulus/bridges/modules/parachains/Cargo.toml +++ b/bridges/modules/parachains/Cargo.toml @@ -20,18 +20,18 @@ pallet-bridge-grandpa = { path = "../grandpa", default-features = false } # Substrate Dependencies -frame-benchmarking = { path = "../../../../substrate/frame/benchmarking", default-features = false, optional = true } -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -frame-system = { path = "../../../../substrate/frame/system", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } -sp-trie = { path = "../../../../substrate/primitives/trie", default-features = false } +frame-benchmarking = { path = "../../../substrate/frame/benchmarking", default-features = false, optional = true } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +frame-system = { path = "../../../substrate/frame/system", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } +sp-trie = { path = "../../../substrate/primitives/trie", default-features = false } [dev-dependencies] bp-header-chain = { path = "../../primitives/header-chain" } bp-test-utils = { path = "../../primitives/test-utils" } -sp-core = { path = "../../../../substrate/primitives/core" } -sp-io = { path = "../../../../substrate/primitives/io" } +sp-core = { path = "../../../substrate/primitives/core" } +sp-io = { path = "../../../substrate/primitives/io" } [features] default = [ "std" ] diff --git a/cumulus/bridges/modules/parachains/README.md b/bridges/modules/parachains/README.md similarity index 100% rename from cumulus/bridges/modules/parachains/README.md rename to bridges/modules/parachains/README.md diff --git a/cumulus/bridges/modules/parachains/src/benchmarking.rs b/bridges/modules/parachains/src/benchmarking.rs similarity index 100% rename from cumulus/bridges/modules/parachains/src/benchmarking.rs rename to bridges/modules/parachains/src/benchmarking.rs diff --git a/cumulus/bridges/modules/parachains/src/call_ext.rs b/bridges/modules/parachains/src/call_ext.rs similarity index 100% rename from cumulus/bridges/modules/parachains/src/call_ext.rs rename to bridges/modules/parachains/src/call_ext.rs diff --git a/cumulus/bridges/modules/parachains/src/lib.rs b/bridges/modules/parachains/src/lib.rs similarity index 100% rename from cumulus/bridges/modules/parachains/src/lib.rs rename to bridges/modules/parachains/src/lib.rs diff --git a/cumulus/bridges/modules/parachains/src/mock.rs b/bridges/modules/parachains/src/mock.rs similarity index 100% rename from cumulus/bridges/modules/parachains/src/mock.rs rename to bridges/modules/parachains/src/mock.rs diff --git a/cumulus/bridges/modules/parachains/src/weights.rs b/bridges/modules/parachains/src/weights.rs similarity index 100% rename from cumulus/bridges/modules/parachains/src/weights.rs rename to bridges/modules/parachains/src/weights.rs diff --git a/cumulus/bridges/modules/parachains/src/weights_ext.rs b/bridges/modules/parachains/src/weights_ext.rs similarity index 100% rename from cumulus/bridges/modules/parachains/src/weights_ext.rs rename to bridges/modules/parachains/src/weights_ext.rs diff --git a/cumulus/bridges/modules/relayers/Cargo.toml b/bridges/modules/relayers/Cargo.toml similarity index 66% rename from cumulus/bridges/modules/relayers/Cargo.toml rename to bridges/modules/relayers/Cargo.toml index 3a7a57e18018..46fc3bb43b1d 100644 --- a/cumulus/bridges/modules/relayers/Cargo.toml +++ b/bridges/modules/relayers/Cargo.toml @@ -20,19 +20,19 @@ pallet-bridge-messages = { path = "../messages", default-features = false } # Substrate Dependencies -frame-benchmarking = { path = "../../../../substrate/frame/benchmarking", default-features = false, optional = true } -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -frame-system = { path = "../../../../substrate/frame/system", default-features = false } -sp-arithmetic = { path = "../../../../substrate/primitives/arithmetic", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-benchmarking = { path = "../../../substrate/frame/benchmarking", default-features = false, optional = true } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +frame-system = { path = "../../../substrate/frame/system", default-features = false } +sp-arithmetic = { path = "../../../substrate/primitives/arithmetic", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [dev-dependencies] bp-runtime = { path = "../../primitives/runtime" } -pallet-balances = { path = "../../../../substrate/frame/balances" } -sp-core = { path = "../../../../substrate/primitives/core" } -sp-io = { path = "../../../../substrate/primitives/io" } -sp-runtime = { path = "../../../../substrate/primitives/runtime" } +pallet-balances = { path = "../../../substrate/frame/balances" } +sp-core = { path = "../../../substrate/primitives/core" } +sp-io = { path = "../../../substrate/primitives/io" } +sp-runtime = { path = "../../../substrate/primitives/runtime" } [features] default = [ "std" ] diff --git a/cumulus/bridges/modules/relayers/README.md b/bridges/modules/relayers/README.md similarity index 100% rename from cumulus/bridges/modules/relayers/README.md rename to bridges/modules/relayers/README.md diff --git a/cumulus/bridges/modules/relayers/src/benchmarking.rs b/bridges/modules/relayers/src/benchmarking.rs similarity index 100% rename from cumulus/bridges/modules/relayers/src/benchmarking.rs rename to bridges/modules/relayers/src/benchmarking.rs diff --git a/cumulus/bridges/modules/relayers/src/lib.rs b/bridges/modules/relayers/src/lib.rs similarity index 100% rename from cumulus/bridges/modules/relayers/src/lib.rs rename to bridges/modules/relayers/src/lib.rs diff --git a/cumulus/bridges/modules/relayers/src/mock.rs b/bridges/modules/relayers/src/mock.rs similarity index 100% rename from cumulus/bridges/modules/relayers/src/mock.rs rename to bridges/modules/relayers/src/mock.rs diff --git a/cumulus/bridges/modules/relayers/src/payment_adapter.rs b/bridges/modules/relayers/src/payment_adapter.rs similarity index 100% rename from cumulus/bridges/modules/relayers/src/payment_adapter.rs rename to bridges/modules/relayers/src/payment_adapter.rs diff --git a/cumulus/bridges/modules/relayers/src/stake_adapter.rs b/bridges/modules/relayers/src/stake_adapter.rs similarity index 100% rename from cumulus/bridges/modules/relayers/src/stake_adapter.rs rename to bridges/modules/relayers/src/stake_adapter.rs diff --git a/cumulus/bridges/modules/relayers/src/weights.rs b/bridges/modules/relayers/src/weights.rs similarity index 100% rename from cumulus/bridges/modules/relayers/src/weights.rs rename to bridges/modules/relayers/src/weights.rs diff --git a/cumulus/bridges/modules/relayers/src/weights_ext.rs b/bridges/modules/relayers/src/weights_ext.rs similarity index 100% rename from cumulus/bridges/modules/relayers/src/weights_ext.rs rename to bridges/modules/relayers/src/weights_ext.rs diff --git a/cumulus/bridges/modules/xcm-bridge-hub-router/Cargo.toml b/bridges/modules/xcm-bridge-hub-router/Cargo.toml similarity index 59% rename from cumulus/bridges/modules/xcm-bridge-hub-router/Cargo.toml rename to bridges/modules/xcm-bridge-hub-router/Cargo.toml index 6ad3c57ca73a..c61cab291e14 100644 --- a/cumulus/bridges/modules/xcm-bridge-hub-router/Cargo.toml +++ b/bridges/modules/xcm-bridge-hub-router/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0-or-later WITH Classpath-exception-2.0" [dependencies] codec = { package = "parity-scale-codec", version = "3.1.5", default-features = false } -log = { version = "0.4.19", default-features = false } +log = { version = "0.4.20", default-features = false } scale-info = { version = "2.8.0", default-features = false, features = ["bit-vec", "derive", "serde"] } # Bridge dependencies @@ -17,21 +17,21 @@ bp-xcm-bridge-hub-router = { path = "../../primitives/xcm-bridge-hub-router", de # Substrate Dependencies -frame-benchmarking = { path = "../../../../substrate/frame/benchmarking", default-features = false, optional = true } -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -frame-system = { path = "../../../../substrate/frame/system", default-features = false } -sp-core = { path = "../../../../substrate/primitives/core", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-benchmarking = { path = "../../../substrate/frame/benchmarking", default-features = false, optional = true } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +frame-system = { path = "../../../substrate/frame/system", default-features = false } +sp-core = { path = "../../../substrate/primitives/core", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } # Polkadot Dependencies -xcm = { package = "staging-xcm", path = "../../../../polkadot/xcm", default-features = false } -xcm-builder = { package = "staging-xcm-builder", path = "../../../../polkadot/xcm/xcm-builder", default-features = false } +xcm = { package = "staging-xcm", path = "../../../polkadot/xcm", default-features = false } +xcm-builder = { package = "staging-xcm-builder", path = "../../../polkadot/xcm/xcm-builder", default-features = false } [dev-dependencies] -sp-io = { path = "../../../../substrate/primitives/io" } -sp-std = { path = "../../../../substrate/primitives/std" } +sp-io = { path = "../../../substrate/primitives/io" } +sp-std = { path = "../../../substrate/primitives/std" } [features] default = [ "std" ] diff --git a/cumulus/bridges/modules/xcm-bridge-hub-router/src/benchmarking.rs b/bridges/modules/xcm-bridge-hub-router/src/benchmarking.rs similarity index 100% rename from cumulus/bridges/modules/xcm-bridge-hub-router/src/benchmarking.rs rename to bridges/modules/xcm-bridge-hub-router/src/benchmarking.rs diff --git a/cumulus/bridges/modules/xcm-bridge-hub-router/src/lib.rs b/bridges/modules/xcm-bridge-hub-router/src/lib.rs similarity index 100% rename from cumulus/bridges/modules/xcm-bridge-hub-router/src/lib.rs rename to bridges/modules/xcm-bridge-hub-router/src/lib.rs diff --git a/cumulus/bridges/modules/xcm-bridge-hub-router/src/mock.rs b/bridges/modules/xcm-bridge-hub-router/src/mock.rs similarity index 99% rename from cumulus/bridges/modules/xcm-bridge-hub-router/src/mock.rs rename to bridges/modules/xcm-bridge-hub-router/src/mock.rs index cd50b98a1688..58df21a6d901 100644 --- a/cumulus/bridges/modules/xcm-bridge-hub-router/src/mock.rs +++ b/bridges/modules/xcm-bridge-hub-router/src/mock.rs @@ -144,5 +144,5 @@ pub fn new_test_ext() -> sp_io::TestExternalities { /// Run pallet test. pub fn run_test(test: impl FnOnce() -> T) -> T { - new_test_ext().execute_with(|| test()) + new_test_ext().execute_with(test) } diff --git a/cumulus/bridges/modules/xcm-bridge-hub-router/src/weights.rs b/bridges/modules/xcm-bridge-hub-router/src/weights.rs similarity index 100% rename from cumulus/bridges/modules/xcm-bridge-hub-router/src/weights.rs rename to bridges/modules/xcm-bridge-hub-router/src/weights.rs diff --git a/cumulus/bridges/primitives/chain-asset-hub-kusama/Cargo.toml b/bridges/primitives/chain-asset-hub-kusama/Cargo.toml similarity index 88% rename from cumulus/bridges/primitives/chain-asset-hub-kusama/Cargo.toml rename to bridges/primitives/chain-asset-hub-kusama/Cargo.toml index 557f56bfb62e..adb9a57bc134 100644 --- a/cumulus/bridges/primitives/chain-asset-hub-kusama/Cargo.toml +++ b/bridges/primitives/chain-asset-hub-kusama/Cargo.toml @@ -11,7 +11,7 @@ codec = { package = "parity-scale-codec", version = "3.1.5", default-features = scale-info = { version = "2.9.0", default-features = false, features = ["derive"] } # Substrate Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } # Bridge Dependencies bp-xcm-bridge-hub-router = { path = "../xcm-bridge-hub-router", default-features = false } diff --git a/cumulus/bridges/primitives/chain-asset-hub-kusama/src/lib.rs b/bridges/primitives/chain-asset-hub-kusama/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/chain-asset-hub-kusama/src/lib.rs rename to bridges/primitives/chain-asset-hub-kusama/src/lib.rs diff --git a/cumulus/bridges/primitives/chain-asset-hub-polkadot/Cargo.toml b/bridges/primitives/chain-asset-hub-polkadot/Cargo.toml similarity index 79% rename from cumulus/bridges/primitives/chain-asset-hub-polkadot/Cargo.toml rename to bridges/primitives/chain-asset-hub-polkadot/Cargo.toml index 6fe9ffab44b8..857ead15b0dd 100644 --- a/cumulus/bridges/primitives/chain-asset-hub-polkadot/Cargo.toml +++ b/bridges/primitives/chain-asset-hub-polkadot/Cargo.toml @@ -11,8 +11,8 @@ codec = { package = "parity-scale-codec", version = "3.1.5", default-features = scale-info = { version = "2.9.0", default-features = false, features = ["derive"] } # Substrate Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } # Bridge Dependencies bp-xcm-bridge-hub-router = { path = "../xcm-bridge-hub-router", default-features = false } diff --git a/cumulus/bridges/primitives/chain-asset-hub-polkadot/src/lib.rs b/bridges/primitives/chain-asset-hub-polkadot/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/chain-asset-hub-polkadot/src/lib.rs rename to bridges/primitives/chain-asset-hub-polkadot/src/lib.rs diff --git a/cumulus/bridges/primitives/chain-bridge-hub-cumulus/Cargo.toml b/bridges/primitives/chain-bridge-hub-cumulus/Cargo.toml similarity index 62% rename from cumulus/bridges/primitives/chain-bridge-hub-cumulus/Cargo.toml rename to bridges/primitives/chain-bridge-hub-cumulus/Cargo.toml index 865cead3c878..24cf7236d453 100644 --- a/cumulus/bridges/primitives/chain-bridge-hub-cumulus/Cargo.toml +++ b/bridges/primitives/chain-bridge-hub-cumulus/Cargo.toml @@ -15,13 +15,13 @@ bp-runtime = { path = "../runtime", default-features = false } # Substrate Based Dependencies -frame-system = { path = "../../../../substrate/frame/system", default-features = false } -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-api = { path = "../../../../substrate/primitives/api", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-system = { path = "../../../substrate/frame/system", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-api = { path = "../../../substrate/primitives/api", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } # Polkadot Dependencies -polkadot-primitives = { path = "../../../../polkadot/primitives", default-features = false } +polkadot-primitives = { path = "../../../polkadot/primitives", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/chain-bridge-hub-cumulus/src/lib.rs b/bridges/primitives/chain-bridge-hub-cumulus/src/lib.rs similarity index 72% rename from cumulus/bridges/primitives/chain-bridge-hub-cumulus/src/lib.rs rename to bridges/primitives/chain-bridge-hub-cumulus/src/lib.rs index feef61c7f20a..c1dbc6db36f6 100644 --- a/cumulus/bridges/primitives/chain-bridge-hub-cumulus/src/lib.rs +++ b/bridges/primitives/chain-bridge-hub-cumulus/src/lib.rs @@ -23,10 +23,9 @@ pub use bp_polkadot_core::{ }; use bp_messages::*; +use bp_polkadot_core::SuffixedCommonSignedExtension; use bp_runtime::extensions::{ - BridgeRejectObsoleteHeadersAndMessages, ChargeTransactionPayment, CheckEra, CheckGenesis, - CheckNonZeroSender, CheckNonce, CheckSpecVersion, CheckTxVersion, CheckWeight, - GenericSignedExtension, RefundBridgedParachainMessagesSchema, + BridgeRejectObsoleteHeadersAndMessages, RefundBridgedParachainMessagesSchema, }; use frame_support::{ dispatch::DispatchClass, @@ -133,88 +132,8 @@ pub const MAX_UNREWARDED_RELAYERS_IN_CONFIRMATION_TX: MessageNonce = 1024; /// analysis (like the one above). pub const MAX_UNCONFIRMED_MESSAGES_IN_CONFIRMATION_TX: MessageNonce = 4096; -/// Extra signed extension data that is used by all bridge hubs. -pub type SignedExtra = ( - CheckNonZeroSender, - CheckSpecVersion, - CheckTxVersion, - CheckGenesis, - CheckEra, - CheckNonce, - CheckWeight, - ChargeTransactionPayment, +/// Signed extension that is used by all bridge hubs. +pub type SignedExtension = SuffixedCommonSignedExtension<( BridgeRejectObsoleteHeadersAndMessages, RefundBridgedParachainMessagesSchema, -); - -/// Signed extension that is used by all bridge hubs. -pub type SignedExtension = GenericSignedExtension; - -/// Helper trait to define some extra methods on bridge hubs signed extension (and -/// overcome Rust limitations). -pub trait BridgeHubSignedExtension { - /// Create signed extension from its components. - fn from_params( - spec_version: u32, - transaction_version: u32, - era: bp_runtime::TransactionEra, - genesis_hash: Hash, - nonce: Nonce, - tip: Balance, - ) -> Self; - - /// Return transaction nonce. - fn nonce(&self) -> Nonce; - - /// Return transaction tip. - fn tip(&self) -> Balance; -} - -impl BridgeHubSignedExtension for SignedExtension { - /// Create signed extension from its components. - fn from_params( - spec_version: u32, - transaction_version: u32, - era: bp_runtime::TransactionEra, - genesis_hash: Hash, - nonce: Nonce, - tip: Balance, - ) -> Self { - GenericSignedExtension::new( - ( - (), // non-zero sender - (), // spec version - (), // tx version - (), // genesis - era.frame_era(), // era - nonce.into(), // nonce (compact encoding) - (), // Check weight - tip.into(), // transaction payment / tip (compact encoding) - (), // bridge reject obsolete headers and msgs - (), // bridge reward to relayer for message passing - ), - Some(( - (), - spec_version, - transaction_version, - genesis_hash, - era.signed_payload(genesis_hash), - (), - (), - (), - (), - (), - )), - ) - } - - /// Return transaction nonce. - fn nonce(&self) -> Nonce { - self.payload.5 .0 - } - - /// Return transaction tip. - fn tip(&self) -> Balance { - self.payload.7 .0 - } -} +)>; diff --git a/cumulus/bridges/primitives/chain-bridge-hub-kusama/Cargo.toml b/bridges/primitives/chain-bridge-hub-kusama/Cargo.toml similarity index 66% rename from cumulus/bridges/primitives/chain-bridge-hub-kusama/Cargo.toml rename to bridges/primitives/chain-bridge-hub-kusama/Cargo.toml index 92b0c9bf4475..387f5e8ade6e 100644 --- a/cumulus/bridges/primitives/chain-bridge-hub-kusama/Cargo.toml +++ b/bridges/primitives/chain-bridge-hub-kusama/Cargo.toml @@ -15,10 +15,10 @@ bp-messages = { path = "../messages", default-features = false } # Substrate Based Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-api = { path = "../../../../substrate/primitives/api", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-api = { path = "../../../substrate/primitives/api", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/chain-bridge-hub-kusama/src/lib.rs b/bridges/primitives/chain-bridge-hub-kusama/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/chain-bridge-hub-kusama/src/lib.rs rename to bridges/primitives/chain-bridge-hub-kusama/src/lib.rs diff --git a/cumulus/bridges/primitives/chain-bridge-hub-polkadot/Cargo.toml b/bridges/primitives/chain-bridge-hub-polkadot/Cargo.toml similarity index 66% rename from cumulus/bridges/primitives/chain-bridge-hub-polkadot/Cargo.toml rename to bridges/primitives/chain-bridge-hub-polkadot/Cargo.toml index 7468d5be60d3..40b386e22d22 100644 --- a/cumulus/bridges/primitives/chain-bridge-hub-polkadot/Cargo.toml +++ b/bridges/primitives/chain-bridge-hub-polkadot/Cargo.toml @@ -16,10 +16,10 @@ bp-messages = { path = "../messages", default-features = false } # Substrate Based Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-api = { path = "../../../../substrate/primitives/api", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-api = { path = "../../../substrate/primitives/api", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/chain-bridge-hub-polkadot/src/lib.rs b/bridges/primitives/chain-bridge-hub-polkadot/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/chain-bridge-hub-polkadot/src/lib.rs rename to bridges/primitives/chain-bridge-hub-polkadot/src/lib.rs diff --git a/cumulus/bridges/primitives/chain-bridge-hub-rococo/Cargo.toml b/bridges/primitives/chain-bridge-hub-rococo/Cargo.toml similarity index 66% rename from cumulus/bridges/primitives/chain-bridge-hub-rococo/Cargo.toml rename to bridges/primitives/chain-bridge-hub-rococo/Cargo.toml index 8dde903701c7..05b8163e9fca 100644 --- a/cumulus/bridges/primitives/chain-bridge-hub-rococo/Cargo.toml +++ b/bridges/primitives/chain-bridge-hub-rococo/Cargo.toml @@ -15,10 +15,10 @@ bp-messages = { path = "../messages", default-features = false } # Substrate Based Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-api = { path = "../../../../substrate/primitives/api", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-api = { path = "../../../substrate/primitives/api", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/chain-bridge-hub-rococo/src/lib.rs b/bridges/primitives/chain-bridge-hub-rococo/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/chain-bridge-hub-rococo/src/lib.rs rename to bridges/primitives/chain-bridge-hub-rococo/src/lib.rs diff --git a/cumulus/bridges/primitives/chain-bridge-hub-wococo/Cargo.toml b/bridges/primitives/chain-bridge-hub-wococo/Cargo.toml similarity index 66% rename from cumulus/bridges/primitives/chain-bridge-hub-wococo/Cargo.toml rename to bridges/primitives/chain-bridge-hub-wococo/Cargo.toml index c93cdad5110b..17c134f4412f 100644 --- a/cumulus/bridges/primitives/chain-bridge-hub-wococo/Cargo.toml +++ b/bridges/primitives/chain-bridge-hub-wococo/Cargo.toml @@ -16,10 +16,10 @@ bp-messages = { path = "../messages", default-features = false } # Substrate Based Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-api = { path = "../../../../substrate/primitives/api", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-api = { path = "../../../substrate/primitives/api", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/chain-bridge-hub-wococo/src/lib.rs b/bridges/primitives/chain-bridge-hub-wococo/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/chain-bridge-hub-wococo/src/lib.rs rename to bridges/primitives/chain-bridge-hub-wococo/src/lib.rs diff --git a/cumulus/bridges/primitives/chain-kusama/Cargo.toml b/bridges/primitives/chain-kusama/Cargo.toml similarity index 71% rename from cumulus/bridges/primitives/chain-kusama/Cargo.toml rename to bridges/primitives/chain-kusama/Cargo.toml index e404bdb3d94f..2d63c3f374fb 100644 --- a/cumulus/bridges/primitives/chain-kusama/Cargo.toml +++ b/bridges/primitives/chain-kusama/Cargo.toml @@ -16,9 +16,9 @@ bp-runtime = { path = "../runtime", default-features = false } # Substrate Based Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-api = { path = "../../../../substrate/primitives/api", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-api = { path = "../../../substrate/primitives/api", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/chain-kusama/src/lib.rs b/bridges/primitives/chain-kusama/src/lib.rs similarity index 96% rename from cumulus/bridges/primitives/chain-kusama/src/lib.rs rename to bridges/primitives/chain-kusama/src/lib.rs index e234a87b6cf6..8c3fbd9c203e 100644 --- a/cumulus/bridges/primitives/chain-kusama/src/lib.rs +++ b/bridges/primitives/chain-kusama/src/lib.rs @@ -57,6 +57,9 @@ impl ChainWithGrandpa for Kusama { const AVERAGE_HEADER_SIZE_IN_JUSTIFICATION: u32 = AVERAGE_HEADER_SIZE_IN_JUSTIFICATION; } +// The SignedExtension used by Kusama. +pub use bp_polkadot_core::CommonSignedExtension as SignedExtension; + /// Name of the parachains pallet in the Kusama runtime. pub const PARAS_PALLET_NAME: &str = "Paras"; diff --git a/cumulus/bridges/primitives/chain-polkadot/Cargo.toml b/bridges/primitives/chain-polkadot/Cargo.toml similarity index 71% rename from cumulus/bridges/primitives/chain-polkadot/Cargo.toml rename to bridges/primitives/chain-polkadot/Cargo.toml index 4d9bf8c182f9..539b10ef9c68 100644 --- a/cumulus/bridges/primitives/chain-polkadot/Cargo.toml +++ b/bridges/primitives/chain-polkadot/Cargo.toml @@ -16,9 +16,9 @@ bp-runtime = { path = "../runtime", default-features = false } # Substrate Based Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-api = { path = "../../../../substrate/primitives/api", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-api = { path = "../../../substrate/primitives/api", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/chain-polkadot/src/lib.rs b/bridges/primitives/chain-polkadot/src/lib.rs similarity index 92% rename from cumulus/bridges/primitives/chain-polkadot/src/lib.rs rename to bridges/primitives/chain-polkadot/src/lib.rs index 9585fd4d7163..d1d6f7454312 100644 --- a/cumulus/bridges/primitives/chain-polkadot/src/lib.rs +++ b/bridges/primitives/chain-polkadot/src/lib.rs @@ -21,7 +21,7 @@ pub use bp_polkadot_core::*; use bp_header_chain::ChainWithGrandpa; -use bp_runtime::{decl_bridge_finality_runtime_apis, Chain}; +use bp_runtime::{decl_bridge_finality_runtime_apis, extensions::PrevalidateAttests, Chain}; use frame_support::weights::Weight; use sp_std::prelude::Vec; @@ -57,6 +57,9 @@ impl ChainWithGrandpa for Polkadot { const AVERAGE_HEADER_SIZE_IN_JUSTIFICATION: u32 = AVERAGE_HEADER_SIZE_IN_JUSTIFICATION; } +/// The SignedExtension used by Polkadot. +pub type SignedExtension = SuffixedCommonSignedExtension; + /// Name of the parachains pallet in the Polkadot runtime. pub const PARAS_PALLET_NAME: &str = "Paras"; diff --git a/cumulus/bridges/primitives/chain-rococo/Cargo.toml b/bridges/primitives/chain-rococo/Cargo.toml similarity index 71% rename from cumulus/bridges/primitives/chain-rococo/Cargo.toml rename to bridges/primitives/chain-rococo/Cargo.toml index 72c6d4c03b66..3c4d3917bc21 100644 --- a/cumulus/bridges/primitives/chain-rococo/Cargo.toml +++ b/bridges/primitives/chain-rococo/Cargo.toml @@ -16,9 +16,9 @@ bp-runtime = { path = "../runtime", default-features = false } # Substrate Based Dependencies -sp-api = { path = "../../../../substrate/primitives/api", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } -frame-support = { path = "../../../../substrate/frame/support", default-features = false } +sp-api = { path = "../../../substrate/primitives/api", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/chain-rococo/src/lib.rs b/bridges/primitives/chain-rococo/src/lib.rs similarity index 96% rename from cumulus/bridges/primitives/chain-rococo/src/lib.rs rename to bridges/primitives/chain-rococo/src/lib.rs index cf7cd16990f8..1589d14ea514 100644 --- a/cumulus/bridges/primitives/chain-rococo/src/lib.rs +++ b/bridges/primitives/chain-rococo/src/lib.rs @@ -61,6 +61,9 @@ parameter_types! { pub const SS58Prefix: u8 = 42; } +// The SignedExtension used by Rococo. +pub use bp_polkadot_core::CommonSignedExtension as SignedExtension; + /// Name of the parachains pallet in the Rococo runtime. pub const PARAS_PALLET_NAME: &str = "Paras"; diff --git a/cumulus/bridges/primitives/chain-wococo/Cargo.toml b/bridges/primitives/chain-wococo/Cargo.toml similarity index 73% rename from cumulus/bridges/primitives/chain-wococo/Cargo.toml rename to bridges/primitives/chain-wococo/Cargo.toml index f7feb7f96eb1..05901821b366 100644 --- a/cumulus/bridges/primitives/chain-wococo/Cargo.toml +++ b/bridges/primitives/chain-wococo/Cargo.toml @@ -17,9 +17,9 @@ bp-rococo = { path = "../chain-rococo", default-features = false } # Substrate Based Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-api = { path = "../../../../substrate/primitives/api", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-api = { path = "../../../substrate/primitives/api", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/chain-wococo/src/lib.rs b/bridges/primitives/chain-wococo/src/lib.rs similarity index 96% rename from cumulus/bridges/primitives/chain-wococo/src/lib.rs rename to bridges/primitives/chain-wococo/src/lib.rs index c64451993ee7..5b5bde826904 100644 --- a/cumulus/bridges/primitives/chain-wococo/src/lib.rs +++ b/bridges/primitives/chain-wococo/src/lib.rs @@ -60,6 +60,9 @@ impl ChainWithGrandpa for Wococo { const AVERAGE_HEADER_SIZE_IN_JUSTIFICATION: u32 = AVERAGE_HEADER_SIZE_IN_JUSTIFICATION; } +// The SignedExtension used by Wococo. +pub use bp_rococo::CommonSignedExtension as SignedExtension; + /// Name of the With-Wococo GRANDPA pallet instance that is deployed at bridged chains. pub const WITH_WOCOCO_GRANDPA_PALLET_NAME: &str = "BridgeWococoGrandpa"; diff --git a/cumulus/bridges/primitives/header-chain/Cargo.toml b/bridges/primitives/header-chain/Cargo.toml similarity index 66% rename from cumulus/bridges/primitives/header-chain/Cargo.toml rename to bridges/primitives/header-chain/Cargo.toml index a7b53b0336dc..e3e83235960a 100644 --- a/cumulus/bridges/primitives/header-chain/Cargo.toml +++ b/bridges/primitives/header-chain/Cargo.toml @@ -18,11 +18,11 @@ bp-runtime = { path = "../runtime", default-features = false } # Substrate Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-core = { path = "../../../../substrate/primitives/core", default-features = false, features = ["serde"] } -sp-consensus-grandpa = { path = "../../../../substrate/primitives/consensus/grandpa", default-features = false, features = ["serde"] } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false, features = ["serde"] } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-core = { path = "../../../substrate/primitives/core", default-features = false, features = ["serde"] } +sp-consensus-grandpa = { path = "../../../substrate/primitives/consensus/grandpa", default-features = false, features = ["serde"] } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false, features = ["serde"] } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [dev-dependencies] bp-test-utils = { path = "../test-utils" } diff --git a/cumulus/bridges/primitives/header-chain/src/justification/mod.rs b/bridges/primitives/header-chain/src/justification/mod.rs similarity index 97% rename from cumulus/bridges/primitives/header-chain/src/justification/mod.rs rename to bridges/primitives/header-chain/src/justification/mod.rs index 24c453a0790c..72a5f68918d9 100644 --- a/cumulus/bridges/primitives/header-chain/src/justification/mod.rs +++ b/bridges/primitives/header-chain/src/justification/mod.rs @@ -97,7 +97,11 @@ impl GrandpaJustification { } } -impl crate::FinalityProof for GrandpaJustification { +impl crate::FinalityProof for GrandpaJustification { + fn target_header_hash(&self) -> H::Hash { + self.commit.target_hash + } + fn target_header_number(&self) -> H::Number { self.commit.target_number } diff --git a/cumulus/bridges/primitives/header-chain/src/justification/verification/equivocation.rs b/bridges/primitives/header-chain/src/justification/verification/equivocation.rs similarity index 100% rename from cumulus/bridges/primitives/header-chain/src/justification/verification/equivocation.rs rename to bridges/primitives/header-chain/src/justification/verification/equivocation.rs diff --git a/cumulus/bridges/primitives/header-chain/src/justification/verification/mod.rs b/bridges/primitives/header-chain/src/justification/verification/mod.rs similarity index 100% rename from cumulus/bridges/primitives/header-chain/src/justification/verification/mod.rs rename to bridges/primitives/header-chain/src/justification/verification/mod.rs diff --git a/cumulus/bridges/primitives/header-chain/src/justification/verification/optimizer.rs b/bridges/primitives/header-chain/src/justification/verification/optimizer.rs similarity index 100% rename from cumulus/bridges/primitives/header-chain/src/justification/verification/optimizer.rs rename to bridges/primitives/header-chain/src/justification/verification/optimizer.rs diff --git a/cumulus/bridges/primitives/header-chain/src/justification/verification/strict.rs b/bridges/primitives/header-chain/src/justification/verification/strict.rs similarity index 100% rename from cumulus/bridges/primitives/header-chain/src/justification/verification/strict.rs rename to bridges/primitives/header-chain/src/justification/verification/strict.rs diff --git a/cumulus/bridges/primitives/header-chain/src/lib.rs b/bridges/primitives/header-chain/src/lib.rs similarity index 98% rename from cumulus/bridges/primitives/header-chain/src/lib.rs rename to bridges/primitives/header-chain/src/lib.rs index 7008dfa6063a..d2c7ec0759e8 100644 --- a/cumulus/bridges/primitives/header-chain/src/lib.rs +++ b/bridges/primitives/header-chain/src/lib.rs @@ -127,7 +127,10 @@ pub struct InitializationData { } /// Abstract finality proof that is justifying block finality. -pub trait FinalityProof: Clone + Send + Sync + Debug { +pub trait FinalityProof: Clone + Send + Sync + Debug { + /// Return hash of header that this proof is generated for. + fn target_header_hash(&self) -> Hash; + /// Return number of header that this proof is generated for. fn target_header_number(&self) -> Number; } @@ -209,7 +212,7 @@ impl TryFrom> for HeaderGrandpa /// Helper trait for finding equivocations in finality proofs. pub trait FindEquivocations { /// The type returned when encountering an error while looking for equivocations. - type Error; + type Error: Debug; /// Find equivocations. fn find_equivocations( diff --git a/cumulus/bridges/primitives/header-chain/src/storage_keys.rs b/bridges/primitives/header-chain/src/storage_keys.rs similarity index 100% rename from cumulus/bridges/primitives/header-chain/src/storage_keys.rs rename to bridges/primitives/header-chain/src/storage_keys.rs diff --git a/cumulus/bridges/primitives/header-chain/tests/implementation_match.rs b/bridges/primitives/header-chain/tests/implementation_match.rs similarity index 100% rename from cumulus/bridges/primitives/header-chain/tests/implementation_match.rs rename to bridges/primitives/header-chain/tests/implementation_match.rs diff --git a/cumulus/bridges/primitives/header-chain/tests/justification/equivocation.rs b/bridges/primitives/header-chain/tests/justification/equivocation.rs similarity index 100% rename from cumulus/bridges/primitives/header-chain/tests/justification/equivocation.rs rename to bridges/primitives/header-chain/tests/justification/equivocation.rs diff --git a/cumulus/bridges/primitives/header-chain/tests/justification/optimizer.rs b/bridges/primitives/header-chain/tests/justification/optimizer.rs similarity index 100% rename from cumulus/bridges/primitives/header-chain/tests/justification/optimizer.rs rename to bridges/primitives/header-chain/tests/justification/optimizer.rs diff --git a/cumulus/bridges/primitives/header-chain/tests/justification/strict.rs b/bridges/primitives/header-chain/tests/justification/strict.rs similarity index 100% rename from cumulus/bridges/primitives/header-chain/tests/justification/strict.rs rename to bridges/primitives/header-chain/tests/justification/strict.rs diff --git a/cumulus/bridges/primitives/header-chain/tests/tests.rs b/bridges/primitives/header-chain/tests/tests.rs similarity index 100% rename from cumulus/bridges/primitives/header-chain/tests/tests.rs rename to bridges/primitives/header-chain/tests/tests.rs diff --git a/cumulus/bridges/primitives/messages/Cargo.toml b/bridges/primitives/messages/Cargo.toml similarity index 78% rename from cumulus/bridges/primitives/messages/Cargo.toml rename to bridges/primitives/messages/Cargo.toml index b1fa6d575aeb..b30d6d2559f7 100644 --- a/cumulus/bridges/primitives/messages/Cargo.toml +++ b/bridges/primitives/messages/Cargo.toml @@ -18,9 +18,9 @@ bp-header-chain = { path = "../header-chain", default-features = false } # Substrate Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-core = { path = "../../../../substrate/primitives/core", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-core = { path = "../../../substrate/primitives/core", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [dev-dependencies] hex = "0.4" diff --git a/cumulus/bridges/primitives/messages/src/lib.rs b/bridges/primitives/messages/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/messages/src/lib.rs rename to bridges/primitives/messages/src/lib.rs diff --git a/cumulus/bridges/primitives/messages/src/source_chain.rs b/bridges/primitives/messages/src/source_chain.rs similarity index 100% rename from cumulus/bridges/primitives/messages/src/source_chain.rs rename to bridges/primitives/messages/src/source_chain.rs diff --git a/cumulus/bridges/primitives/messages/src/storage_keys.rs b/bridges/primitives/messages/src/storage_keys.rs similarity index 100% rename from cumulus/bridges/primitives/messages/src/storage_keys.rs rename to bridges/primitives/messages/src/storage_keys.rs diff --git a/cumulus/bridges/primitives/messages/src/target_chain.rs b/bridges/primitives/messages/src/target_chain.rs similarity index 100% rename from cumulus/bridges/primitives/messages/src/target_chain.rs rename to bridges/primitives/messages/src/target_chain.rs diff --git a/cumulus/bridges/primitives/parachains/Cargo.toml b/bridges/primitives/parachains/Cargo.toml similarity index 72% rename from cumulus/bridges/primitives/parachains/Cargo.toml rename to bridges/primitives/parachains/Cargo.toml index 978296954bb7..ca69523dde37 100644 --- a/cumulus/bridges/primitives/parachains/Cargo.toml +++ b/bridges/primitives/parachains/Cargo.toml @@ -19,10 +19,10 @@ bp-runtime = { path = "../runtime", default-features = false } # Substrate dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-core = { path = "../../../../substrate/primitives/core", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-core = { path = "../../../substrate/primitives/core", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/parachains/src/lib.rs b/bridges/primitives/parachains/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/parachains/src/lib.rs rename to bridges/primitives/parachains/src/lib.rs diff --git a/cumulus/bridges/primitives/polkadot-core/Cargo.toml b/bridges/primitives/polkadot-core/Cargo.toml similarity index 69% rename from cumulus/bridges/primitives/polkadot-core/Cargo.toml rename to bridges/primitives/polkadot-core/Cargo.toml index 2563adaf219c..aa7eb8024fbf 100644 --- a/cumulus/bridges/primitives/polkadot-core/Cargo.toml +++ b/bridges/primitives/polkadot-core/Cargo.toml @@ -19,11 +19,11 @@ bp-runtime = { path = "../runtime", default-features = false } # Substrate Based Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -frame-system = { path = "../../../../substrate/frame/system", default-features = false } -sp-core = { path = "../../../../substrate/primitives/core", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +frame-system = { path = "../../../substrate/frame/system", default-features = false } +sp-core = { path = "../../../substrate/primitives/core", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [dev-dependencies] hex = "0.4" diff --git a/cumulus/bridges/primitives/polkadot-core/src/lib.rs b/bridges/primitives/polkadot-core/src/lib.rs similarity index 81% rename from cumulus/bridges/primitives/polkadot-core/src/lib.rs rename to bridges/primitives/polkadot-core/src/lib.rs index b35d97ec79ae..af39b5ab9bab 100644 --- a/cumulus/bridges/primitives/polkadot-core/src/lib.rs +++ b/bridges/primitives/polkadot-core/src/lib.rs @@ -17,7 +17,15 @@ #![cfg_attr(not(feature = "std"), no_std)] use bp_messages::MessageNonce; -use bp_runtime::{Chain, EncodedOrDecodedCall, StorageMapKeyProvider}; +use bp_runtime::{ + self, + extensions::{ + ChargeTransactionPayment, CheckEra, CheckGenesis, CheckNonZeroSender, CheckNonce, + CheckSpecVersion, CheckTxVersion, CheckWeight, GenericSignedExtension, + SignedExtensionSchema, + }, + Chain, EncodedOrDecodedCall, StorageMapKeyProvider, TransactionEra, +}; use frame_support::{ dispatch::DispatchClass, parameter_types, @@ -272,6 +280,99 @@ impl AccountInfoStorageMapKeyProvider { } } +/// Extra signed extension data that is used by most chains. +pub type CommonSignedExtra = ( + CheckNonZeroSender, + CheckSpecVersion, + CheckTxVersion, + CheckGenesis, + CheckEra, + CheckNonce, + CheckWeight, + ChargeTransactionPayment, +); + +/// Extra signed extension data that starts with `CommonSignedExtra`. +pub type SuffixedCommonSignedExtension = + GenericSignedExtension<(CommonSignedExtra, Suffix)>; + +/// Helper trait to define some extra methods on `SuffixedCommonSignedExtension`. +pub trait SuffixedCommonSignedExtensionExt { + /// Create signed extension from its components. + fn from_params( + spec_version: u32, + transaction_version: u32, + era: TransactionEra, + genesis_hash: Hash, + nonce: Nonce, + tip: Balance, + extra: (Suffix::Payload, Suffix::AdditionalSigned), + ) -> Self; + + /// Return transaction nonce. + fn nonce(&self) -> Nonce; + + /// Return transaction tip. + fn tip(&self) -> Balance; +} + +impl SuffixedCommonSignedExtensionExt for SuffixedCommonSignedExtension +where + Suffix: SignedExtensionSchema, +{ + fn from_params( + spec_version: u32, + transaction_version: u32, + era: TransactionEra, + genesis_hash: Hash, + nonce: Nonce, + tip: Balance, + extra: (Suffix::Payload, Suffix::AdditionalSigned), + ) -> Self { + GenericSignedExtension::new( + ( + ( + (), // non-zero sender + (), // spec version + (), // tx version + (), // genesis + era.frame_era(), // era + nonce.into(), // nonce (compact encoding) + (), // Check weight + tip.into(), // transaction payment / tip (compact encoding) + ), + extra.0, + ), + Some(( + ( + (), + spec_version, + transaction_version, + genesis_hash, + era.signed_payload(genesis_hash), + (), + (), + (), + ), + extra.1, + )), + ) + } + + fn nonce(&self) -> Nonce { + let common_payload = self.payload.0; + common_payload.5 .0 + } + + fn tip(&self) -> Balance { + let common_payload = self.payload.0; + common_payload.7 .0 + } +} + +/// Signed extension that is used by most chains. +pub type CommonSignedExtension = SuffixedCommonSignedExtension<()>; + #[cfg(test)] mod tests { use super::*; diff --git a/cumulus/bridges/primitives/polkadot-core/src/parachains.rs b/bridges/primitives/polkadot-core/src/parachains.rs similarity index 100% rename from cumulus/bridges/primitives/polkadot-core/src/parachains.rs rename to bridges/primitives/polkadot-core/src/parachains.rs diff --git a/cumulus/bridges/primitives/relayers/Cargo.toml b/bridges/primitives/relayers/Cargo.toml similarity index 74% rename from cumulus/bridges/primitives/relayers/Cargo.toml rename to bridges/primitives/relayers/Cargo.toml index bc674d4cb773..99cd79c68417 100644 --- a/cumulus/bridges/primitives/relayers/Cargo.toml +++ b/bridges/primitives/relayers/Cargo.toml @@ -17,9 +17,9 @@ bp-runtime = { path = "../runtime", default-features = false } # Substrate Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } [dev-dependencies] hex = "0.4" diff --git a/cumulus/bridges/primitives/relayers/src/lib.rs b/bridges/primitives/relayers/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/relayers/src/lib.rs rename to bridges/primitives/relayers/src/lib.rs diff --git a/cumulus/bridges/primitives/relayers/src/registration.rs b/bridges/primitives/relayers/src/registration.rs similarity index 100% rename from cumulus/bridges/primitives/relayers/src/registration.rs rename to bridges/primitives/relayers/src/registration.rs diff --git a/cumulus/bridges/primitives/runtime/Cargo.toml b/bridges/primitives/runtime/Cargo.toml similarity index 57% rename from cumulus/bridges/primitives/runtime/Cargo.toml rename to bridges/primitives/runtime/Cargo.toml index f6134d6e3328..4454066b59fa 100644 --- a/cumulus/bridges/primitives/runtime/Cargo.toml +++ b/bridges/primitives/runtime/Cargo.toml @@ -17,15 +17,15 @@ serde = { version = "1.0", default-features = false, features = ["alloc", "deriv # Substrate Dependencies -frame-support = { path = "../../../../substrate/frame/support", default-features = false } -frame-system = { path = "../../../../substrate/frame/system", default-features = false } -sp-core = { path = "../../../../substrate/primitives/core", default-features = false } -sp-io = { path = "../../../../substrate/primitives/io", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false, features = ["serde"] } -sp-state-machine = { path = "../../../../substrate/primitives/state-machine", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } -sp-trie = { path = "../../../../substrate/primitives/trie", default-features = false } -trie-db = { version = "0.27.1", default-features = false } +frame-support = { path = "../../../substrate/frame/support", default-features = false } +frame-system = { path = "../../../substrate/frame/system", default-features = false } +sp-core = { path = "../../../substrate/primitives/core", default-features = false } +sp-io = { path = "../../../substrate/primitives/io", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false, features = ["serde"] } +sp-state-machine = { path = "../../../substrate/primitives/state-machine", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } +sp-trie = { path = "../../../substrate/primitives/trie", default-features = false } +trie-db = { version = "0.28.0", default-features = false } [dev-dependencies] hex-literal = "0.4" diff --git a/cumulus/bridges/primitives/runtime/src/chain.rs b/bridges/primitives/runtime/src/chain.rs similarity index 100% rename from cumulus/bridges/primitives/runtime/src/chain.rs rename to bridges/primitives/runtime/src/chain.rs diff --git a/cumulus/bridges/primitives/runtime/src/extensions.rs b/bridges/primitives/runtime/src/extensions.rs similarity index 94% rename from cumulus/bridges/primitives/runtime/src/extensions.rs rename to bridges/primitives/runtime/src/extensions.rs index 253350d17e74..44eeaad93c91 100644 --- a/cumulus/bridges/primitives/runtime/src/extensions.rs +++ b/bridges/primitives/runtime/src/extensions.rs @@ -35,7 +35,12 @@ pub trait SignedExtensionSchema: Encode + Decode + Debug + Eq + Clone + StaticTy type AdditionalSigned: Encode + Debug + Eq + Clone + StaticTypeInfo; } -// An implementation of `SignedExtensionSchema` using generic params. +impl SignedExtensionSchema for () { + type Payload = (); + type AdditionalSigned = (); +} + +/// An implementation of `SignedExtensionSchema` using generic params. #[derive(Encode, Decode, Clone, Debug, PartialEq, Eq, TypeInfo)] pub struct GenericSignedExtensionSchema(PhantomData<(P, S)>); @@ -72,6 +77,9 @@ pub type CheckWeight = GenericSignedExtensionSchema<(), ()>; /// The `SignedExtensionSchema` for `pallet_transaction_payment::ChargeTransactionPayment`. pub type ChargeTransactionPayment = GenericSignedExtensionSchema, ()>; +/// The `SignedExtensionSchema` for `polkadot-runtime-common::PrevalidateAttests`. +pub type PrevalidateAttests = GenericSignedExtensionSchema<(), ()>; + /// The `SignedExtensionSchema` for `BridgeRejectObsoleteHeadersAndMessages`. pub type BridgeRejectObsoleteHeadersAndMessages = GenericSignedExtensionSchema<(), ()>; @@ -99,7 +107,7 @@ pub struct GenericSignedExtension { // It may be set to `None` if extensions are decoded. We are never reconstructing transactions // (and it makes no sense to do that) => decoded version of `SignedExtensions` is only used to // read fields of the `payload`. And when resigning transaction, we're reconstructing - // `SignedExtensions` from the scratch. + // `SignedExtensions` from scratch. additional_signed: Option, } diff --git a/cumulus/bridges/primitives/runtime/src/lib.rs b/bridges/primitives/runtime/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/runtime/src/lib.rs rename to bridges/primitives/runtime/src/lib.rs diff --git a/cumulus/bridges/primitives/runtime/src/messages.rs b/bridges/primitives/runtime/src/messages.rs similarity index 100% rename from cumulus/bridges/primitives/runtime/src/messages.rs rename to bridges/primitives/runtime/src/messages.rs diff --git a/cumulus/bridges/primitives/runtime/src/storage_proof.rs b/bridges/primitives/runtime/src/storage_proof.rs similarity index 100% rename from cumulus/bridges/primitives/runtime/src/storage_proof.rs rename to bridges/primitives/runtime/src/storage_proof.rs diff --git a/cumulus/bridges/primitives/runtime/src/storage_types.rs b/bridges/primitives/runtime/src/storage_types.rs similarity index 100% rename from cumulus/bridges/primitives/runtime/src/storage_types.rs rename to bridges/primitives/runtime/src/storage_types.rs diff --git a/cumulus/bridges/primitives/test-utils/Cargo.toml b/bridges/primitives/test-utils/Cargo.toml similarity index 62% rename from cumulus/bridges/primitives/test-utils/Cargo.toml rename to bridges/primitives/test-utils/Cargo.toml index f6e8533705e0..7081ce90f97b 100644 --- a/cumulus/bridges/primitives/test-utils/Cargo.toml +++ b/bridges/primitives/test-utils/Cargo.toml @@ -4,7 +4,6 @@ version = "0.1.0" authors.workspace = true edition.workspace = true license = "GPL-3.0-or-later WITH Classpath-exception-2.0" -publish = false [dependencies] bp-header-chain = { path = "../header-chain", default-features = false } @@ -14,12 +13,12 @@ bp-runtime = { path = "../runtime", default-features = false } codec = { package = "parity-scale-codec", version = "3.1.5", default-features = false } ed25519-dalek = { version = "2.0", default-features = false } finality-grandpa = { version = "0.16.2", default-features = false } -sp-application-crypto = { path = "../../../../substrate/primitives/application-crypto", default-features = false } -sp-consensus-grandpa = { path = "../../../../substrate/primitives/consensus/grandpa", default-features = false } -sp-core = { path = "../../../../substrate/primitives/core", default-features = false } -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-std = { path = "../../../../substrate/primitives/std", default-features = false } -sp-trie = { path = "../../../../substrate/primitives/trie", default-features = false } +sp-application-crypto = { path = "../../../substrate/primitives/application-crypto", default-features = false } +sp-consensus-grandpa = { path = "../../../substrate/primitives/consensus/grandpa", default-features = false } +sp-core = { path = "../../../substrate/primitives/core", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-std = { path = "../../../substrate/primitives/std", default-features = false } +sp-trie = { path = "../../../substrate/primitives/trie", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/test-utils/src/keyring.rs b/bridges/primitives/test-utils/src/keyring.rs similarity index 100% rename from cumulus/bridges/primitives/test-utils/src/keyring.rs rename to bridges/primitives/test-utils/src/keyring.rs diff --git a/cumulus/bridges/primitives/test-utils/src/lib.rs b/bridges/primitives/test-utils/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/test-utils/src/lib.rs rename to bridges/primitives/test-utils/src/lib.rs diff --git a/cumulus/bridges/primitives/xcm-bridge-hub-router/Cargo.toml b/bridges/primitives/xcm-bridge-hub-router/Cargo.toml similarity index 76% rename from cumulus/bridges/primitives/xcm-bridge-hub-router/Cargo.toml rename to bridges/primitives/xcm-bridge-hub-router/Cargo.toml index df2b00563deb..725a7d94564e 100644 --- a/cumulus/bridges/primitives/xcm-bridge-hub-router/Cargo.toml +++ b/bridges/primitives/xcm-bridge-hub-router/Cargo.toml @@ -11,8 +11,8 @@ codec = { package = "parity-scale-codec", version = "3.1.5", default-features = scale-info = { version = "2.9.0", default-features = false, features = ["bit-vec", "derive"] } # Substrate Dependencies -sp-runtime = { path = "../../../../substrate/primitives/runtime", default-features = false } -sp-core = { path = "../../../../substrate/primitives/core", default-features = false } +sp-runtime = { path = "../../../substrate/primitives/runtime", default-features = false } +sp-core = { path = "../../../substrate/primitives/core", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/bridges/primitives/xcm-bridge-hub-router/src/lib.rs b/bridges/primitives/xcm-bridge-hub-router/src/lib.rs similarity index 100% rename from cumulus/bridges/primitives/xcm-bridge-hub-router/src/lib.rs rename to bridges/primitives/xcm-bridge-hub-router/src/lib.rs diff --git a/cumulus/bridges/scripts/verify-pallets-build.sh b/bridges/scripts/verify-pallets-build.sh similarity index 93% rename from cumulus/bridges/scripts/verify-pallets-build.sh rename to bridges/scripts/verify-pallets-build.sh index 1ea3dbe86260..b8ac09e26b8b 100755 --- a/cumulus/bridges/scripts/verify-pallets-build.sh +++ b/bridges/scripts/verify-pallets-build.sh @@ -89,17 +89,26 @@ rm -rf $BRIDGES_FOLDER/scripts/update-weights-setup.sh rm -rf $BRIDGES_FOLDER/scripts/update_substrate.sh rm -rf $BRIDGES_FOLDER/tools rm -f $BRIDGES_FOLDER/.dockerignore +rm -f $BRIDGES_FOLDER/local.Dockerfile.dockerignore rm -f $BRIDGES_FOLDER/deny.toml rm -f $BRIDGES_FOLDER/.gitlab-ci.yml rm -f $BRIDGES_FOLDER/.editorconfig rm -f $BRIDGES_FOLDER/Cargo.toml rm -f $BRIDGES_FOLDER/ci.Dockerfile +rm -f $BRIDGES_FOLDER/local.Dockerfile rm -f $BRIDGES_FOLDER/CODEOWNERS rm -f $BRIDGES_FOLDER/Dockerfile +rm -f $BRIDGES_FOLDER/rustfmt.toml # let's fix Cargo.toml a bit (it'll be helpful if we are in the bridges repo) if [[ ! -f "Cargo.toml" ]]; then cat > Cargo.toml <<-CARGO_TOML + [workspace.package] + authors = ["Parity Technologies "] + edition = "2021" + repository = "https://github.com/paritytech/parity-bridges-common.git" + license = "GPL-3.0-only" + [workspace] resolver = "2" diff --git a/cumulus/CODEOWNERS b/cumulus/CODEOWNERS deleted file mode 100644 index 3e5e8bd062dd..000000000000 --- a/cumulus/CODEOWNERS +++ /dev/null @@ -1,23 +0,0 @@ -# Lists some code owners. -# -# A codeowner just oversees some part of the codebase. If an owned file is changed then the -# corresponding codeowner receives a review request. An approval of the codeowner might be -# required for merging a PR (depends on repository settings). -# -# For details about syntax, see: -# https://help.github.com/en/articles/about-code-owners -# But here are some important notes: -# -# - Glob syntax is git-like, e.g. `/core` means the core directory in the root, unlike `core` -# which can be everywhere. -# - Multiple owners are supported. -# - Either handle (e.g, @github_user or @github/team) or email can be used. Keep in mind, -# that handles might work better because they are more recognizable on GitHub, -# eyou can use them for mentioning unlike an email. -# - The latest matching rule, if multiple, takes precedence. - -# CI -/.github/ @paritytech/ci @paritytech/release-engineering -/.gitlab-ci.yml @paritytech/ci -/scripts/ci/ @paritytech/ci @paritytech/release-engineering - diff --git a/cumulus/client/cli/Cargo.toml b/cumulus/client/cli/Cargo.toml index 6c44d5f5fd3b..15f28d73b36e 100644 --- a/cumulus/client/cli/Cargo.toml +++ b/cumulus/client/cli/Cargo.toml @@ -5,7 +5,7 @@ authors.workspace = true edition.workspace = true [dependencies] -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } codec = { package = "parity-scale-codec", version = "3.0.0" } url = "2.4.0" diff --git a/cumulus/client/consensus/aura/src/collators/lookahead.rs b/cumulus/client/consensus/aura/src/collators/lookahead.rs index b1a7cde55c2c..57cd646fbcde 100644 --- a/cumulus/client/consensus/aura/src/collators/lookahead.rs +++ b/cumulus/client/consensus/aura/src/collators/lookahead.rs @@ -45,11 +45,13 @@ use cumulus_primitives_core::{ use cumulus_relay_chain_interface::RelayChainInterface; use polkadot_node_primitives::SubmitCollationParams; -use polkadot_node_subsystem::messages::CollationGenerationMessage; +use polkadot_node_subsystem::messages::{ + CollationGenerationMessage, RuntimeApiMessage, RuntimeApiRequest, +}; use polkadot_overseer::Handle as OverseerHandle; use polkadot_primitives::{CollatorPair, Id as ParaId, OccupiedCoreAssumption}; -use futures::prelude::*; +use futures::{channel::oneshot, prelude::*}; use sc_client_api::{backend::AuxStore, BlockBackend, BlockOf}; use sc_consensus::BlockImport; use sc_consensus_aura::standalone as aura_internal; @@ -181,6 +183,17 @@ where while let Some(relay_parent_header) = import_notifications.next().await { let relay_parent = relay_parent_header.hash(); + if !is_para_scheduled(relay_parent, params.para_id, &mut params.overseer_handle).await { + tracing::trace!( + target: crate::LOG_TARGET, + ?relay_parent, + ?params.para_id, + "Para is not scheduled on any core, skipping import notification", + ); + + continue + } + let max_pov_size = match params .relay_client .persisted_validation_data( @@ -444,3 +457,41 @@ async fn max_ancestry_lookback( }, } } + +// Checks if there exists a scheduled core for the para at the provided relay parent. +// +// Falls back to `false` in case of an error. +async fn is_para_scheduled( + relay_parent: PHash, + para_id: ParaId, + overseer_handle: &mut OverseerHandle, +) -> bool { + let (tx, rx) = oneshot::channel(); + let request = RuntimeApiRequest::AvailabilityCores(tx); + overseer_handle + .send_msg(RuntimeApiMessage::Request(relay_parent, request), "LookaheadCollator") + .await; + + let cores = match rx.await { + Ok(Ok(cores)) => cores, + Ok(Err(error)) => { + tracing::error!( + target: crate::LOG_TARGET, + ?error, + ?relay_parent, + "Failed to query availability cores runtime API", + ); + return false + }, + Err(oneshot::Canceled) => { + tracing::error!( + target: crate::LOG_TARGET, + ?relay_parent, + "Sender for availability cores runtime request dropped", + ); + return false + }, + }; + + cores.iter().any(|core| core.para_id() == Some(para_id)) +} diff --git a/cumulus/client/consensus/common/src/lib.rs b/cumulus/client/consensus/common/src/lib.rs index 29ee7356ee24..08bceabb2bd4 100644 --- a/cumulus/client/consensus/common/src/lib.rs +++ b/cumulus/client/consensus/common/src/lib.rs @@ -211,6 +211,7 @@ pub trait ParachainBlockImportMarker {} impl ParachainBlockImportMarker for ParachainBlockImport {} /// Parameters when searching for suitable parents to build on top of. +#[derive(Debug)] pub struct ParentSearchParams { /// The relay-parent that is intended to be used. pub relay_parent: PHash, @@ -228,6 +229,7 @@ pub struct ParentSearchParams { } /// A potential parent block returned from [`find_potential_parents`] +#[derive(Debug, PartialEq)] pub struct PotentialParent { /// The hash of the block. pub hash: B::Hash, diff --git a/cumulus/client/consensus/common/src/tests.rs b/cumulus/client/consensus/common/src/tests.rs index 15586d81d9bf..22d3dd3abd49 100644 --- a/cumulus/client/consensus/common/src/tests.rs +++ b/cumulus/client/consensus/common/src/tests.rs @@ -19,7 +19,10 @@ use crate::*; use async_trait::async_trait; use codec::Encode; use cumulus_client_pov_recovery::RecoveryKind; -use cumulus_primitives_core::{relay_chain::BlockId, InboundDownwardMessage, InboundHrmpMessage}; +use cumulus_primitives_core::{ + relay_chain::{self, BlockId}, + CumulusDigestItem, InboundDownwardMessage, InboundHrmpMessage, +}; use cumulus_relay_chain_interface::{ CommittedCandidateReceipt, OccupiedCoreAssumption, OverseerHandle, PHeader, ParaId, RelayChainInterface, RelayChainResult, SessionIndex, StorageValue, ValidatorId, @@ -42,12 +45,21 @@ use std::{ time::Duration, }; +fn relay_block_num_from_hash(hash: &PHash) -> relay_chain::BlockNumber { + hash.to_low_u64_be() as u32 +} + +fn relay_hash_from_block_num(block_number: relay_chain::BlockNumber) -> PHash { + PHash::from_low_u64_be(block_number as u64) +} + struct RelaychainInner { new_best_heads: Option>, finalized_heads: Option>, new_best_heads_sender: mpsc::UnboundedSender
, finalized_heads_sender: mpsc::UnboundedSender
, relay_chain_hash_to_header: HashMap, + relay_chain_hash_to_header_pending: HashMap, } impl RelaychainInner { @@ -61,6 +73,7 @@ impl RelaychainInner { new_best_heads: Some(new_best_heads), finalized_heads: Some(finalized_heads), relay_chain_hash_to_header: Default::default(), + relay_chain_hash_to_header_pending: Default::default(), } } } @@ -110,20 +123,17 @@ impl RelayChainInterface for Relaychain { &self, hash: PHash, _: ParaId, - _: OccupiedCoreAssumption, + assumption: OccupiedCoreAssumption, ) -> RelayChainResult> { - Ok(Some(PersistedValidationData { - parent_head: self - .inner - .lock() - .unwrap() - .relay_chain_hash_to_header - .get(&hash) - .unwrap() - .encode() - .into(), - ..Default::default() - })) + let inner = self.inner.lock().unwrap(); + let relay_to_header = match assumption { + OccupiedCoreAssumption::Included => &inner.relay_chain_hash_to_header_pending, + _ => &inner.relay_chain_hash_to_header, + }; + let Some(parent_head) = relay_to_header.get(&hash).map(|head| head.encode().into()) else { + return Ok(None) + }; + Ok(Some(PersistedValidationData { parent_head, ..Default::default() })) } async fn candidate_pending_availability( @@ -135,7 +145,7 @@ impl RelayChainInterface for Relaychain { } async fn session_index_for_child(&self, _: PHash) -> RelayChainResult { - unimplemented!("Not needed for test") + Ok(0) } async fn import_notification_stream( @@ -210,8 +220,23 @@ impl RelayChainInterface for Relaychain { .boxed()) } - async fn header(&self, _block_id: BlockId) -> RelayChainResult> { - unimplemented!("Not needed for test") + async fn header(&self, block_id: BlockId) -> RelayChainResult> { + let number = match block_id { + BlockId::Hash(hash) => relay_block_num_from_hash(&hash), + BlockId::Number(block_number) => block_number, + }; + let parent_hash = number + .checked_sub(1) + .map(relay_hash_from_block_num) + .unwrap_or_else(|| PHash::zero()); + + Ok(Some(PHeader { + parent_hash, + number, + digest: sp_runtime::Digest::default(), + state_root: PHash::zero(), + extrinsics_root: PHash::zero(), + })) } } @@ -238,6 +263,7 @@ fn build_block( sproof: RelayStateSproofBuilder, at: Option, timestamp: Option, + relay_parent: Option, ) -> Block { let builder = match at { Some(at) => match timestamp { @@ -249,10 +275,17 @@ fn build_block( let mut block = builder.build().unwrap().block; - // Simulate some form of post activity (like a Seal or Other generic things). - // This is mostly used to exercise the `LevelMonitor` correct behavior. - // (in practice we want that header post-hash != pre-hash) - block.header.digest.push(sp_runtime::DigestItem::Other(vec![1, 2, 3])); + if let Some(relay_parent) = relay_parent { + block + .header + .digest + .push(CumulusDigestItem::RelayParent(relay_parent).to_digest_item()); + } else { + // Simulate some form of post activity (like a Seal or Other generic things). + // This is mostly used to exercise the `LevelMonitor` correct behavior. + // (in practice we want that header post-hash != pre-hash) + block.header.digest.push(sp_runtime::DigestItem::Other(vec![1, 2, 3])); + } block } @@ -292,13 +325,14 @@ fn build_and_import_block_ext>( importer: &mut I, at: Option, timestamp: Option, + relay_parent: Option, ) -> Block { let sproof = match at { None => sproof_with_best_parent(client), Some(at) => sproof_with_parent_by_hash(client, at), }; - let block = build_block(client, sproof, at, timestamp); + let block = build_block(client, sproof, at, timestamp, relay_parent); import_block_sync(importer, block.clone(), origin, import_as_best); block } @@ -311,6 +345,7 @@ fn build_and_import_block(mut client: Arc, import_as_best: bool) -> Bloc &mut client, None, None, + None, ) } @@ -372,7 +407,7 @@ fn follow_new_best_with_dummy_recovery_works() { let header = client.header(best).ok().flatten().expect("No header for best"); sproof_with_parent(HeadData(header.encode())) }; - let block = build_block(&*client, sproof, None, None); + let block = build_block(&*client, sproof, None, None, None); let block_clone = block.clone(); let client_clone = client.clone(); @@ -638,6 +673,7 @@ fn prune_blocks_on_level_overflow() { &mut para_import, None, None, + None, ); let id0 = block0.header.hash(); @@ -650,6 +686,7 @@ fn prune_blocks_on_level_overflow() { &mut para_import, Some(id0), Some(i as u64 * TIMESTAMP_MULTIPLIER), + None, ) }) .collect::>(); @@ -664,6 +701,7 @@ fn prune_blocks_on_level_overflow() { &mut para_import, Some(id10), Some(i as u64 * TIMESTAMP_MULTIPLIER), + None, ) }) .collect::>(); @@ -692,6 +730,7 @@ fn prune_blocks_on_level_overflow() { &mut para_import, Some(id0), Some(LEVEL_LIMIT as u64 * TIMESTAMP_MULTIPLIER), + None, ); // Expected scenario @@ -711,6 +750,7 @@ fn prune_blocks_on_level_overflow() { &mut para_import, Some(id0), Some(2 * LEVEL_LIMIT as u64 * TIMESTAMP_MULTIPLIER), + None, ); // Expected scenario @@ -749,6 +789,7 @@ fn restore_limit_monitor() { &mut para_import, None, None, + None, ); let id00 = block00.header.hash(); @@ -761,6 +802,7 @@ fn restore_limit_monitor() { &mut para_import, Some(id00), Some(i as u64 * TIMESTAMP_MULTIPLIER), + None, ) }) .collect::>(); @@ -775,6 +817,7 @@ fn restore_limit_monitor() { &mut para_import, Some(id10), Some(i as u64 * TIMESTAMP_MULTIPLIER), + None, ) }) .collect::>(); @@ -809,6 +852,7 @@ fn restore_limit_monitor() { &mut para_import, Some(id00), Some(LEVEL_LIMIT as u64 * TIMESTAMP_MULTIPLIER), + None, ); // Expected scenario @@ -830,3 +874,487 @@ fn restore_limit_monitor() { })); assert_eq!(*monitor.freshness.get(&block13.header.hash()).unwrap(), monitor.import_counter); } + +#[test] +fn find_potential_parents_in_allowed_ancestry() { + sp_tracing::try_init_simple(); + + let backend = Arc::new(Backend::new_test(1000, 1)); + let client = Arc::new(TestClientBuilder::with_backend(backend.clone()).build()); + let mut para_import = ParachainBlockImport::new(client.clone(), backend.clone()); + + let relay_parent = relay_hash_from_block_num(10); + let block = build_and_import_block_ext( + &client, + BlockOrigin::Own, + true, + &mut para_import, + None, + None, + Some(relay_parent), + ); + + let relay_chain = Relaychain::new(); + { + let included_map = &mut relay_chain.inner.lock().unwrap().relay_chain_hash_to_header; + included_map.insert(relay_parent, block.header().clone()); + } + + let potential_parents = block_on(find_potential_parents( + ParentSearchParams { + relay_parent, + para_id: ParaId::from(100), + ancestry_lookback: 0, + max_depth: 0, + ignore_alternative_branches: true, + }, + &*backend, + &relay_chain, + )) + .unwrap(); + assert_eq!(potential_parents.len(), 1); + let parent = &potential_parents[0]; + + assert_eq!(parent.hash, block.hash()); + assert_eq!(&parent.header, block.header()); + assert_eq!(parent.depth, 0); + assert!(parent.aligned_with_pending); + + // New block is not pending or included. + let block_relay_parent = relay_hash_from_block_num(11); + let search_relay_parent = relay_hash_from_block_num(13); + { + let included_map = &mut relay_chain.inner.lock().unwrap().relay_chain_hash_to_header; + included_map.insert(search_relay_parent, block.header().clone()); + } + let block = build_and_import_block_ext( + &client, + BlockOrigin::Own, + true, + &mut para_import, + Some(block.header().hash()), + None, + Some(block_relay_parent), + ); + let potential_parents = block_on(find_potential_parents( + ParentSearchParams { + relay_parent: search_relay_parent, + para_id: ParaId::from(100), + ancestry_lookback: 2, + max_depth: 1, + ignore_alternative_branches: true, + }, + &*backend, + &relay_chain, + )) + .unwrap(); + + assert_eq!(potential_parents.len(), 2); + let parent = &potential_parents[1]; + + assert_eq!(parent.hash, block.hash()); + assert_eq!(&parent.header, block.header()); + assert_eq!(parent.depth, 1); + assert!(parent.aligned_with_pending); + + // Reduce allowed ancestry. + let potential_parents = block_on(find_potential_parents( + ParentSearchParams { + relay_parent: search_relay_parent, + para_id: ParaId::from(100), + ancestry_lookback: 1, + max_depth: 1, + ignore_alternative_branches: true, + }, + &*backend, + &relay_chain, + )) + .unwrap(); + assert_eq!(potential_parents.len(), 1); + let parent = &potential_parents[0]; + assert_ne!(parent.hash, block.hash()); +} + +/// Tests that pending availability block is always potential parent. +#[test] +fn find_potential_pending_parent() { + sp_tracing::try_init_simple(); + + let backend = Arc::new(Backend::new_test(1000, 1)); + let client = Arc::new(TestClientBuilder::with_backend(backend.clone()).build()); + let mut para_import = ParachainBlockImport::new(client.clone(), backend.clone()); + + let relay_parent = relay_hash_from_block_num(10); + let included_block = build_and_import_block_ext( + &client, + BlockOrigin::Own, + true, + &mut para_import, + None, + None, + Some(relay_parent), + ); + let relay_parent = relay_hash_from_block_num(12); + let pending_block = build_and_import_block_ext( + &client, + BlockOrigin::Own, + true, + &mut para_import, + Some(included_block.header().hash()), + None, + Some(relay_parent), + ); + + let relay_chain = Relaychain::new(); + let search_relay_parent = relay_hash_from_block_num(15); + { + let relay_inner = &mut relay_chain.inner.lock().unwrap(); + relay_inner + .relay_chain_hash_to_header + .insert(search_relay_parent, included_block.header().clone()); + relay_inner + .relay_chain_hash_to_header_pending + .insert(search_relay_parent, pending_block.header().clone()); + } + + let potential_parents = block_on(find_potential_parents( + ParentSearchParams { + relay_parent: search_relay_parent, + para_id: ParaId::from(100), + ancestry_lookback: 0, + max_depth: 1, + ignore_alternative_branches: true, + }, + &*backend, + &relay_chain, + )) + .unwrap(); + assert_eq!(potential_parents.len(), 2); + let included_parent = &potential_parents[0]; + + assert_eq!(included_parent.hash, included_block.hash()); + assert_eq!(&included_parent.header, included_block.header()); + assert_eq!(included_parent.depth, 0); + assert!(included_parent.aligned_with_pending); + + let pending_parent = &potential_parents[1]; + + assert_eq!(pending_parent.hash, pending_block.hash()); + assert_eq!(&pending_parent.header, pending_block.header()); + assert_eq!(pending_parent.depth, 1); + assert!(pending_parent.aligned_with_pending); +} + +#[test] +fn find_potential_parents_with_max_depth() { + sp_tracing::try_init_simple(); + + const NON_INCLUDED_CHAIN_LEN: usize = 5; + + let backend = Arc::new(Backend::new_test(1000, 1)); + let client = Arc::new(TestClientBuilder::with_backend(backend.clone()).build()); + let mut para_import = ParachainBlockImport::new(client.clone(), backend.clone()); + + let relay_parent = relay_hash_from_block_num(10); + let included_block = build_and_import_block_ext( + &client, + BlockOrigin::Own, + true, + &mut para_import, + None, + None, + Some(relay_parent), + ); + + let relay_chain = Relaychain::new(); + { + let included_map = &mut relay_chain.inner.lock().unwrap().relay_chain_hash_to_header; + included_map.insert(relay_parent, included_block.header().clone()); + } + + let mut blocks = Vec::new(); + let mut parent = included_block.header().hash(); + for _ in 0..NON_INCLUDED_CHAIN_LEN { + let block = build_and_import_block_ext( + &client, + BlockOrigin::Own, + true, + &mut para_import, + Some(parent), + None, + Some(relay_parent), + ); + parent = block.header().hash(); + blocks.push(block); + } + for max_depth in 0..=NON_INCLUDED_CHAIN_LEN { + let potential_parents = block_on(find_potential_parents( + ParentSearchParams { + relay_parent, + para_id: ParaId::from(100), + ancestry_lookback: 0, + max_depth, + ignore_alternative_branches: true, + }, + &*backend, + &relay_chain, + )) + .unwrap(); + assert_eq!(potential_parents.len(), max_depth + 1); + let expected_parents: Vec<_> = + std::iter::once(&included_block).chain(blocks.iter().take(max_depth)).collect(); + + for i in 0..(max_depth + 1) { + let parent = &potential_parents[i]; + let expected = &expected_parents[i]; + + assert_eq!(parent.hash, expected.hash()); + assert_eq!(&parent.header, expected.header()); + assert_eq!(parent.depth, i); + assert!(parent.aligned_with_pending); + } + } +} + +#[test] +fn find_potential_parents_aligned_with_pending() { + sp_tracing::try_init_simple(); + + const NON_INCLUDED_CHAIN_LEN: usize = 5; + + let backend = Arc::new(Backend::new_test(1000, 1)); + let client = Arc::new(TestClientBuilder::with_backend(backend.clone()).build()); + let mut para_import = ParachainBlockImport::new(client.clone(), backend.clone()); + + let relay_parent = relay_hash_from_block_num(10); + // Choose different relay parent for alternative chain to get new hashes. + let search_relay_parent = relay_hash_from_block_num(11); + let included_block = build_and_import_block_ext( + &client, + BlockOrigin::NetworkInitialSync, + true, + &mut para_import, + None, + None, + Some(relay_parent), + ); + let pending_block = build_and_import_block_ext( + &client, + BlockOrigin::Own, + true, + &mut para_import, + Some(included_block.header().hash()), + None, + Some(relay_parent), + ); + + let relay_chain = Relaychain::new(); + { + let relay_inner = &mut relay_chain.inner.lock().unwrap(); + relay_inner + .relay_chain_hash_to_header + .insert(search_relay_parent, included_block.header().clone()); + relay_inner + .relay_chain_hash_to_header_pending + .insert(search_relay_parent, pending_block.header().clone()); + } + + // Build two sibling chains from the included block. + let mut aligned_blocks = Vec::new(); + let mut parent = pending_block.header().hash(); + for _ in 1..NON_INCLUDED_CHAIN_LEN { + let block = build_and_import_block_ext( + &client, + BlockOrigin::Own, + true, + &mut para_import, + Some(parent), + None, + Some(relay_parent), + ); + parent = block.header().hash(); + aligned_blocks.push(block); + } + + let mut alt_blocks = Vec::new(); + let mut parent = included_block.header().hash(); + for _ in 0..NON_INCLUDED_CHAIN_LEN { + let block = build_and_import_block_ext( + &client, + BlockOrigin::NetworkInitialSync, + true, + &mut para_import, + Some(parent), + None, + Some(search_relay_parent), + ); + parent = block.header().hash(); + alt_blocks.push(block); + } + + // Ignore alternative branch: + for max_depth in 0..=NON_INCLUDED_CHAIN_LEN { + let potential_parents = block_on(find_potential_parents( + ParentSearchParams { + relay_parent: search_relay_parent, + para_id: ParaId::from(100), + ancestry_lookback: 1, // aligned chain is in ancestry. + max_depth, + ignore_alternative_branches: true, + }, + &*backend, + &relay_chain, + )) + .unwrap(); + assert_eq!(potential_parents.len(), max_depth + 1); + let expected_parents: Vec<_> = [&included_block, &pending_block] + .into_iter() + .chain(aligned_blocks.iter()) + .take(max_depth + 1) + .collect(); + + for i in 0..(max_depth + 1) { + let parent = &potential_parents[i]; + let expected = &expected_parents[i]; + + assert_eq!(parent.hash, expected.hash()); + assert_eq!(&parent.header, expected.header()); + assert_eq!(parent.depth, i); + assert!(parent.aligned_with_pending); + } + } + + // Do not ignore: + for max_depth in 0..=NON_INCLUDED_CHAIN_LEN { + let potential_parents = block_on(find_potential_parents( + ParentSearchParams { + relay_parent: search_relay_parent, + para_id: ParaId::from(100), + ancestry_lookback: 1, // aligned chain is in ancestry. + max_depth, + ignore_alternative_branches: false, + }, + &*backend, + &relay_chain, + )) + .unwrap(); + + let expected_len = 2 * max_depth + 1; + assert_eq!(potential_parents.len(), expected_len); + let expected_aligned: Vec<_> = [&included_block, &pending_block] + .into_iter() + .chain(aligned_blocks.iter()) + .take(max_depth + 1) + .collect(); + let expected_alt = alt_blocks.iter().take(max_depth); + + let expected_parents: Vec<_> = + expected_aligned.clone().into_iter().chain(expected_alt).collect(); + // Check correctness. + assert_eq!(expected_parents.len(), expected_len); + + for i in 0..expected_len { + let parent = &potential_parents[i]; + let expected = expected_parents + .iter() + .find(|block| block.header().hash() == parent.hash) + .expect("missing parent"); + + let is_aligned = expected_aligned.contains(&expected); + + assert_eq!(parent.hash, expected.hash()); + assert_eq!(&parent.header, expected.header()); + + assert_eq!(parent.aligned_with_pending, is_aligned); + } + } +} + +/// Tests that no potential parent gets discarded if there's no pending availability block. +#[test] +fn find_potential_parents_aligned_no_pending() { + sp_tracing::try_init_simple(); + + const NON_INCLUDED_CHAIN_LEN: usize = 5; + + let backend = Arc::new(Backend::new_test(1000, 1)); + let client = Arc::new(TestClientBuilder::with_backend(backend.clone()).build()); + let mut para_import = ParachainBlockImport::new(client.clone(), backend.clone()); + + let relay_parent = relay_hash_from_block_num(10); + // Choose different relay parent for alternative chain to get new hashes. + let search_relay_parent = relay_hash_from_block_num(11); + let included_block = build_and_import_block_ext( + &client, + BlockOrigin::Own, + true, + &mut para_import, + None, + None, + Some(relay_parent), + ); + + let relay_chain = Relaychain::new(); + { + let included_map = &mut relay_chain.inner.lock().unwrap().relay_chain_hash_to_header; + included_map.insert(search_relay_parent, included_block.header().clone()); + } + + // Build two sibling chains from the included block. + let mut parent = included_block.header().hash(); + for _ in 0..NON_INCLUDED_CHAIN_LEN { + let block = build_and_import_block_ext( + &client, + BlockOrigin::Own, + true, + &mut para_import, + Some(parent), + None, + Some(relay_parent), + ); + parent = block.header().hash(); + } + + let mut parent = included_block.header().hash(); + for _ in 0..NON_INCLUDED_CHAIN_LEN { + let block = build_and_import_block_ext( + &client, + BlockOrigin::NetworkInitialSync, + true, + &mut para_import, + Some(parent), + None, + Some(search_relay_parent), + ); + parent = block.header().hash(); + } + + for max_depth in 0..=NON_INCLUDED_CHAIN_LEN { + let potential_parents_aligned = block_on(find_potential_parents( + ParentSearchParams { + relay_parent: search_relay_parent, + para_id: ParaId::from(100), + ancestry_lookback: 1, // aligned chain is in ancestry. + max_depth, + ignore_alternative_branches: true, + }, + &*backend, + &relay_chain, + )) + .unwrap(); + let potential_parents = block_on(find_potential_parents( + ParentSearchParams { + relay_parent: search_relay_parent, + para_id: ParaId::from(100), + ancestry_lookback: 1, + max_depth, + ignore_alternative_branches: false, + }, + &*backend, + &relay_chain, + )) + .unwrap(); + assert_eq!(potential_parents.len(), 2 * max_depth + 1); + assert_eq!(potential_parents, potential_parents_aligned); + } +} diff --git a/cumulus/client/relay-chain-inprocess-interface/Cargo.toml b/cumulus/client/relay-chain-inprocess-interface/Cargo.toml index c198b22cad4a..39eda5075e29 100644 --- a/cumulus/client/relay-chain-inprocess-interface/Cargo.toml +++ b/cumulus/client/relay-chain-inprocess-interface/Cargo.toml @@ -37,7 +37,7 @@ sp-keyring = { path = "../../../substrate/primitives/keyring" } # Polkadot polkadot-primitives = { path = "../../../polkadot/primitives" } polkadot-test-client = { path = "../../../polkadot/node/test/client" } -metered = { package = "prioritized-metered-channel", version = "0.2.0" } +metered = { package = "prioritized-metered-channel", version = "0.5.1", default-features = false, features=["futures_channel"] } # Cumulus cumulus-test-service = { path = "../../test/service" } diff --git a/cumulus/client/relay-chain-rpc-interface/Cargo.toml b/cumulus/client/relay-chain-rpc-interface/Cargo.toml index 305ab82b064c..0f09377e106c 100644 --- a/cumulus/client/relay-chain-rpc-interface/Cargo.toml +++ b/cumulus/client/relay-chain-rpc-interface/Cargo.toml @@ -32,7 +32,7 @@ jsonrpsee = { version = "0.16.2", features = ["ws-client"] } tracing = "0.1.37" async-trait = "0.1.73" url = "2.4.0" -serde_json = "1.0.105" +serde_json = "1.0.107" serde = "1.0.188" schnellru = "0.2.1" smoldot = { version = "0.11.0", default_features = false, features = ["std"]} diff --git a/cumulus/client/service/src/lib.rs b/cumulus/client/service/src/lib.rs index 211a5cc3b79b..82890666ebae 100644 --- a/cumulus/client/service/src/lib.rs +++ b/cumulus/client/service/src/lib.rs @@ -484,6 +484,7 @@ where import_queue, block_announce_validator_builder: Some(Box::new(move |_| block_announce_validator)), warp_sync_params, + block_relay: None, }) } diff --git a/cumulus/pallets/parachain-system/Cargo.toml b/cumulus/pallets/parachain-system/Cargo.toml index 3063083c0ee4..5470dce47480 100644 --- a/cumulus/pallets/parachain-system/Cargo.toml +++ b/cumulus/pallets/parachain-system/Cargo.toml @@ -11,7 +11,7 @@ codec = { package = "parity-scale-codec", version = "3.0.0", default-features = environmental = { version = "1.1.4", default-features = false } impl-trait-for-tuples = "0.2.1" log = { version = "0.4.20", default-features = false } -trie-db = { version = "0.27.1", default-features = false } +trie-db = { version = "0.28.0", default-features = false } scale-info = { version = "2.9.0", default-features = false, features = ["derive"] } # Substrate diff --git a/cumulus/pallets/parachain-system/proc-macro/Cargo.toml b/cumulus/pallets/parachain-system/proc-macro/Cargo.toml index e5e7da81d2ae..77e298363c85 100644 --- a/cumulus/pallets/parachain-system/proc-macro/Cargo.toml +++ b/cumulus/pallets/parachain-system/proc-macro/Cargo.toml @@ -9,7 +9,7 @@ description = "Proc macros provided by the parachain-system pallet" proc-macro = true [dependencies] -syn = "2.0.31" +syn = "2.0.33" proc-macro2 = "1.0.64" quote = "1.0.33" proc-macro-crate = "1.3.1" diff --git a/cumulus/pallets/parachain-system/src/lib.rs b/cumulus/pallets/parachain-system/src/lib.rs index a8e9a0bf9ae4..a7e59a61c9be 100644 --- a/cumulus/pallets/parachain-system/src/lib.rs +++ b/cumulus/pallets/parachain-system/src/lib.rs @@ -1400,12 +1400,12 @@ impl Pallet { CustomValidationHeadData::::put(head_data); } - /// Open HRMP channel for using it in benchmarks. + /// Open HRMP channel for using it in benchmarks or tests. /// /// The caller assumes that the pallet will accept regular outbound message to the sibling /// `target_parachain` after this call. No other assumptions are made. - #[cfg(feature = "runtime-benchmarks")] - pub fn open_outbound_hrmp_channel_for_benchmarks(target_parachain: ParaId) { + #[cfg(any(feature = "runtime-benchmarks", feature = "std"))] + pub fn open_outbound_hrmp_channel_for_benchmarks_or_tests(target_parachain: ParaId) { RelevantMessagingState::::put(MessagingStateSnapshot { dmq_mqc_head: Default::default(), relay_dispatch_queue_remaining_capacity: Default::default(), diff --git a/cumulus/pallets/xcmp-queue/Cargo.toml b/cumulus/pallets/xcmp-queue/Cargo.toml index de4ad61de9e0..5aab57da91b4 100644 --- a/cumulus/pallets/xcmp-queue/Cargo.toml +++ b/cumulus/pallets/xcmp-queue/Cargo.toml @@ -38,7 +38,7 @@ pallet-balances = { path = "../../../substrate/frame/balances" } xcm-builder = { package = "staging-xcm-builder", path = "../../../polkadot/xcm/xcm-builder" } # Cumulus -cumulus-pallet-parachain-system = { path = "../parachain-system" } +cumulus-pallet-parachain-system = { path = "../parachain-system", features = ["parameterized-consensus-hook"] } [features] default = [ "std" ] diff --git a/cumulus/pallets/xcmp-queue/src/tests.rs b/cumulus/pallets/xcmp-queue/src/tests.rs index 2929e8452bef..ba005a5badfe 100644 --- a/cumulus/pallets/xcmp-queue/src/tests.rs +++ b/cumulus/pallets/xcmp-queue/src/tests.rs @@ -14,9 +14,9 @@ // limitations under the License. use super::*; -use cumulus_primitives_core::XcmpMessageHandler; +use cumulus_primitives_core::{ParaId, XcmpMessageHandler}; use frame_support::{assert_noop, assert_ok}; -use mock::{new_test_ext, RuntimeCall, RuntimeOrigin, Test, XcmpQueue}; +use mock::{new_test_ext, ParachainSystem, RuntimeCall, RuntimeOrigin, Test, XcmpQueue}; use sp_runtime::traits::BadOrigin; #[test] @@ -341,3 +341,32 @@ fn xcmp_queue_consumes_dest_and_msg_on_ok_validate() { ); }); } + +#[test] +fn xcmp_queue_send_xcm_works() { + new_test_ext().execute_with(|| { + let sibling_para_id = ParaId::from(12345); + let dest = (Parent, X1(Parachain(sibling_para_id.into()))).into(); + let msg = Xcm(vec![ClearOrigin]); + + // try to send without opened HRMP channel to the sibling_para_id + assert_eq!( + send_xcm::(dest, msg.clone()), + Err(SendError::Transport("NoChannel")), + ); + + // open HRMP channel to the sibling_para_id + ParachainSystem::open_outbound_hrmp_channel_for_benchmarks_or_tests(sibling_para_id); + + // check empty outbound queue + assert!(XcmpQueue::take_outbound_messages(usize::MAX).is_empty()); + + // now send works + assert_ok!(send_xcm::(dest, msg)); + + // check outbound queue contains message/page for sibling_para_id + assert!(XcmpQueue::take_outbound_messages(usize::MAX) + .iter() + .any(|(para_id, _)| para_id == &sibling_para_id)); + }) +} diff --git a/cumulus/parachain-template/node/Cargo.toml b/cumulus/parachain-template/node/Cargo.toml index b1f1946e3ed4..6de57b185e4a 100644 --- a/cumulus/parachain-template/node/Cargo.toml +++ b/cumulus/parachain-template/node/Cargo.toml @@ -11,7 +11,7 @@ build = "build.rs" publish = false [dependencies] -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } log = "0.4.20" codec = { package = "parity-scale-codec", version = "3.0.0" } serde = { version = "1.0.188", features = ["derive"] } diff --git a/cumulus/parachain-template/runtime/Cargo.toml b/cumulus/parachain-template/runtime/Cargo.toml index f26b7be23533..4e51f16dca16 100644 --- a/cumulus/parachain-template/runtime/Cargo.toml +++ b/cumulus/parachain-template/runtime/Cargo.toml @@ -128,6 +128,7 @@ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-session-benchmarking/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking/runtime-benchmarks", diff --git a/cumulus/parachains/common/Cargo.toml b/cumulus/parachains/common/Cargo.toml index 18cafde0d303..963de03fa16a 100644 --- a/cumulus/parachains/common/Cargo.toml +++ b/cumulus/parachains/common/Cargo.toml @@ -79,3 +79,17 @@ std = [ "xcm-executor/std", "xcm/std", ] + +runtime-benchmarks = [ + "cumulus-primitives-utility/runtime-benchmarks", + "frame-support/runtime-benchmarks", + "frame-system/runtime-benchmarks", + "pallet-asset-tx-payment/runtime-benchmarks", + "pallet-assets/runtime-benchmarks", + "pallet-balances/runtime-benchmarks", + "pallet-collator-selection/runtime-benchmarks", + "polkadot-primitives/runtime-benchmarks", + "sp-runtime/runtime-benchmarks", + "xcm-builder/runtime-benchmarks", + "xcm-executor/runtime-benchmarks", +] diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/Cargo.toml b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/Cargo.toml index 4f17b1ac55b5..788b2482be87 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/Cargo.toml +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-kusama/Cargo.toml @@ -45,6 +45,7 @@ runtime-benchmarks = [ "pallet-assets/runtime-benchmarks", "pallet-balances/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-parachains/runtime-benchmarks", "polkadot-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/Cargo.toml b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/Cargo.toml index ed383207228c..023e8b84f11b 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/Cargo.toml +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-polkadot/Cargo.toml @@ -44,6 +44,7 @@ runtime-benchmarks = [ "pallet-assets/runtime-benchmarks", "pallet-balances/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-parachains/runtime-benchmarks", "polkadot-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/Cargo.toml b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/Cargo.toml index 0c60a30a0b92..80c41c24aa75 100644 --- a/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/Cargo.toml +++ b/cumulus/parachains/integration-tests/emulated/assets/asset-hub-westend/Cargo.toml @@ -45,6 +45,7 @@ runtime-benchmarks = [ "pallet-assets/runtime-benchmarks", "pallet-balances/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-parachains/runtime-benchmarks", "polkadot-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/integration-tests/emulated/bridges/bridge-hub-rococo/Cargo.toml b/cumulus/parachains/integration-tests/emulated/bridges/bridge-hub-rococo/Cargo.toml index ee6896855549..c02c96255e18 100644 --- a/cumulus/parachains/integration-tests/emulated/bridges/bridge-hub-rococo/Cargo.toml +++ b/cumulus/parachains/integration-tests/emulated/bridges/bridge-hub-rococo/Cargo.toml @@ -25,8 +25,8 @@ pallet-xcm = { path = "../../../../../../polkadot/xcm/pallet-xcm", default-featu parachains-common = { path = "../../../../common" } cumulus-pallet-xcmp-queue = { path = "../../../../../pallets/xcmp-queue", default-features = false} cumulus-pallet-dmp-queue = { path = "../../../../../pallets/dmp-queue", default-features = false} -pallet-bridge-messages = { path = "../../../../../bridges/modules/messages", default-features = false} -bp-messages = { path = "../../../../../bridges/primitives/messages", default-features = false} +pallet-bridge-messages = { path = "../../../../../../bridges/modules/messages", default-features = false} +bp-messages = { path = "../../../../../../bridges/primitives/messages", default-features = false} # Local xcm-emulator = { path = "../../../../../xcm/xcm-emulator", default-features = false} diff --git a/cumulus/parachains/integration-tests/emulated/common/Cargo.toml b/cumulus/parachains/integration-tests/emulated/common/Cargo.toml index ac1e650d5de7..f1db26c2a540 100644 --- a/cumulus/parachains/integration-tests/emulated/common/Cargo.toml +++ b/cumulus/parachains/integration-tests/emulated/common/Cargo.toml @@ -56,9 +56,9 @@ xcm-emulator = { path = "../../../../xcm/xcm-emulator", default-features = false cumulus-pallet-dmp-queue = { path = "../../../../pallets/dmp-queue" } cumulus-pallet-xcmp-queue = { path = "../../../../pallets/xcmp-queue", default-features = false} cumulus-pallet-parachain-system = { path = "../../../../pallets/parachain-system" } -bp-messages = { path = "../../../../bridges/primitives/messages" } -pallet-bridge-messages = { path = "../../../../bridges/modules/messages" } -bridge-runtime-common = { path = "../../../../bridges/bin/runtime-common" } +bp-messages = { path = "../../../../../bridges/primitives/messages" } +pallet-bridge-messages = { path = "../../../../../bridges/modules/messages" } +bridge-runtime-common = { path = "../../../../../bridges/bin/runtime-common" } [features] runtime-benchmarks = [ @@ -80,6 +80,7 @@ runtime-benchmarks = [ "pallet-message-queue/runtime-benchmarks", "pallet-staking/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "penpal-runtime/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-primitives/runtime-benchmarks", diff --git a/cumulus/parachains/pallets/parachain-info/Cargo.toml b/cumulus/parachains/pallets/parachain-info/Cargo.toml index 4ed11e394398..931df9d9273d 100644 --- a/cumulus/parachains/pallets/parachain-info/Cargo.toml +++ b/cumulus/parachains/pallets/parachain-info/Cargo.toml @@ -3,7 +3,6 @@ authors.workspace = true edition.workspace = true name = "parachain-info" version = "0.1.0" -publish = false [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] } diff --git a/cumulus/parachains/runtimes/assets/asset-hub-kusama/Cargo.toml b/cumulus/parachains/runtimes/assets/asset-hub-kusama/Cargo.toml index 53555499842f..1d0c7bc98563 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-kusama/Cargo.toml +++ b/cumulus/parachains/runtimes/assets/asset-hub-kusama/Cargo.toml @@ -100,6 +100,7 @@ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-session-benchmarking/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking/runtime-benchmarks", @@ -118,6 +119,7 @@ runtime-benchmarks = [ "pallet-utility/runtime-benchmarks", "pallet-xcm-benchmarks/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", "sp-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/runtimes/assets/asset-hub-polkadot/Cargo.toml b/cumulus/parachains/runtimes/assets/asset-hub-polkadot/Cargo.toml index dfc3d5416cc3..4a6ce4cbf593 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-polkadot/Cargo.toml +++ b/cumulus/parachains/runtimes/assets/asset-hub-polkadot/Cargo.toml @@ -88,6 +88,7 @@ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-session-benchmarking/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking/runtime-benchmarks", @@ -105,6 +106,7 @@ runtime-benchmarks = [ "pallet-utility/runtime-benchmarks", "pallet-xcm-benchmarks/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", "sp-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/runtimes/assets/asset-hub-westend/Cargo.toml b/cumulus/parachains/runtimes/assets/asset-hub-westend/Cargo.toml index 8c25842cbf38..f436ae9537f4 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-westend/Cargo.toml +++ b/cumulus/parachains/runtimes/assets/asset-hub-westend/Cargo.toml @@ -92,6 +92,7 @@ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-session-benchmarking/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking/runtime-benchmarks", @@ -110,6 +111,7 @@ runtime-benchmarks = [ "pallet-utility/runtime-benchmarks", "pallet-xcm-benchmarks/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", "sp-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/runtimes/assets/common/Cargo.toml b/cumulus/parachains/runtimes/assets/common/Cargo.toml index a8b5f3f8c6b9..0cd5de2ddcd4 100644 --- a/cumulus/parachains/runtimes/assets/common/Cargo.toml +++ b/cumulus/parachains/runtimes/assets/common/Cargo.toml @@ -56,6 +56,7 @@ runtime-benchmarks = [ "pallet-asset-conversion/runtime-benchmarks", "pallet-asset-tx-payment/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "sp-runtime/runtime-benchmarks", "xcm-builder/runtime-benchmarks", "xcm-executor/runtime-benchmarks", diff --git a/cumulus/parachains/runtimes/assets/test-utils/Cargo.toml b/cumulus/parachains/runtimes/assets/test-utils/Cargo.toml index 7fee710f000c..0a27535ed22b 100644 --- a/cumulus/parachains/runtimes/assets/test-utils/Cargo.toml +++ b/cumulus/parachains/runtimes/assets/test-utils/Cargo.toml @@ -4,7 +4,6 @@ version = "1.0.0" authors.workspace = true edition.workspace = true description = "Test utils for Asset Hub runtimes." -publish = false [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "max-encoded-len"] } diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/Cargo.toml b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/Cargo.toml index 91eb7adc61fe..67c3fa37df04 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/Cargo.toml +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/Cargo.toml @@ -137,6 +137,7 @@ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-session-benchmarking/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking/runtime-benchmarks", @@ -148,6 +149,7 @@ runtime-benchmarks = [ "pallet-utility/runtime-benchmarks", "pallet-xcm-benchmarks/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", "sp-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/Cargo.toml b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/Cargo.toml index 52a866b70971..77923ee74f89 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/Cargo.toml +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/Cargo.toml @@ -137,6 +137,7 @@ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-session-benchmarking/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking/runtime-benchmarks", @@ -148,6 +149,7 @@ runtime-benchmarks = [ "pallet-utility/runtime-benchmarks", "pallet-xcm-benchmarks/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", "sp-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/Cargo.toml b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/Cargo.toml index d65fb9b72cd1..70c62d1a34fd 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/Cargo.toml +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/Cargo.toml @@ -72,26 +72,26 @@ parachain-info = { path = "../../../pallets/parachain-info", default-features = parachains-common = { path = "../../../common", default-features = false } # Bridges -bp-bridge-hub-rococo = { path = "../../../../bridges/primitives/chain-bridge-hub-rococo", default-features = false } -bp-bridge-hub-wococo = { path = "../../../../bridges/primitives/chain-bridge-hub-wococo", default-features = false } -bp-header-chain = { path = "../../../../bridges/primitives/header-chain", default-features = false } -bp-messages = { path = "../../../../bridges/primitives/messages", default-features = false } -bp-parachains = { path = "../../../../bridges/primitives/parachains", default-features = false } -bp-polkadot-core = { path = "../../../../bridges/primitives/polkadot-core", default-features = false } -bp-relayers = { path = "../../../../bridges/primitives/relayers", default-features = false } -bp-runtime = { path = "../../../../bridges/primitives/runtime", default-features = false } -bp-rococo = { path = "../../../../bridges/primitives/chain-rococo", default-features = false } -bp-wococo = { path = "../../../../bridges/primitives/chain-wococo", default-features = false } -pallet-bridge-grandpa = { path = "../../../../bridges/modules/grandpa", default-features = false } -pallet-bridge-messages = { path = "../../../../bridges/modules/messages", default-features = false } -pallet-bridge-parachains = { path = "../../../../bridges/modules/parachains", default-features = false } -pallet-bridge-relayers = { path = "../../../../bridges/modules/relayers", default-features = false } -bridge-runtime-common = { path = "../../../../bridges/bin/runtime-common", default-features = false } +bp-bridge-hub-rococo = { path = "../../../../../bridges/primitives/chain-bridge-hub-rococo", default-features = false } +bp-bridge-hub-wococo = { path = "../../../../../bridges/primitives/chain-bridge-hub-wococo", default-features = false } +bp-header-chain = { path = "../../../../../bridges/primitives/header-chain", default-features = false } +bp-messages = { path = "../../../../../bridges/primitives/messages", default-features = false } +bp-parachains = { path = "../../../../../bridges/primitives/parachains", default-features = false } +bp-polkadot-core = { path = "../../../../../bridges/primitives/polkadot-core", default-features = false } +bp-relayers = { path = "../../../../../bridges/primitives/relayers", default-features = false } +bp-runtime = { path = "../../../../../bridges/primitives/runtime", default-features = false } +bp-rococo = { path = "../../../../../bridges/primitives/chain-rococo", default-features = false } +bp-wococo = { path = "../../../../../bridges/primitives/chain-wococo", default-features = false } +pallet-bridge-grandpa = { path = "../../../../../bridges/modules/grandpa", default-features = false } +pallet-bridge-messages = { path = "../../../../../bridges/modules/messages", default-features = false } +pallet-bridge-parachains = { path = "../../../../../bridges/modules/parachains", default-features = false } +pallet-bridge-relayers = { path = "../../../../../bridges/modules/relayers", default-features = false } +bridge-runtime-common = { path = "../../../../../bridges/bin/runtime-common", default-features = false } [dev-dependencies] static_assertions = "1.1" bridge-hub-test-utils = { path = "../test-utils" } -bridge-runtime-common = { path = "../../../../bridges/bin/runtime-common", features = ["integrity-test"] } +bridge-runtime-common = { path = "../../../../../bridges/bin/runtime-common", features = ["integrity-test"] } sp-keyring = { path = "../../../../../substrate/primitives/keyring" } [features] @@ -173,6 +173,7 @@ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-session-benchmarking/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking/runtime-benchmarks", @@ -188,6 +189,7 @@ runtime-benchmarks = [ "pallet-utility/runtime-benchmarks", "pallet-xcm-benchmarks/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", "sp-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs index 4311a6a629f9..dbdc2133ec88 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs @@ -1070,7 +1070,7 @@ impl_runtime_apis! { ) -> (bridge_hub_rococo_config::FromWococoBridgeHubMessagesProof, Weight) { use cumulus_primitives_core::XcmpMessageSource; assert!(XcmpQueue::take_outbound_messages(usize::MAX).is_empty()); - ParachainSystem::open_outbound_hrmp_channel_for_benchmarks(42.into()); + ParachainSystem::open_outbound_hrmp_channel_for_benchmarks_or_tests(42.into()); prepare_message_proof_from_parachain::< Runtime, BridgeGrandpaWococoInstance, @@ -1113,7 +1113,7 @@ impl_runtime_apis! { ) -> (bridge_hub_wococo_config::FromRococoBridgeHubMessagesProof, Weight) { use cumulus_primitives_core::XcmpMessageSource; assert!(XcmpQueue::take_outbound_messages(usize::MAX).is_empty()); - ParachainSystem::open_outbound_hrmp_channel_for_benchmarks(42.into()); + ParachainSystem::open_outbound_hrmp_channel_for_benchmarks_or_tests(42.into()); prepare_message_proof_from_parachain::< Runtime, BridgeGrandpaRococoInstance, @@ -1243,55 +1243,67 @@ cumulus_pallet_parachain_system::register_validate_block! { #[cfg(test)] mod tests { use super::*; - use bp_runtime::TransactionEra; - use bridge_hub_test_utils::test_header; use codec::Encode; - - pub type TestBlockHeader = - sp_runtime::generic::Header; + use sp_runtime::{ + generic::Era, + traits::{SignedExtension, Zero}, + }; #[test] fn ensure_signed_extension_definition_is_compatible_with_relay() { - let payload: SignedExtra = ( - frame_system::CheckNonZeroSender::new(), - frame_system::CheckSpecVersion::new(), - frame_system::CheckTxVersion::new(), - frame_system::CheckGenesis::new(), - frame_system::CheckEra::from(sp_runtime::generic::Era::Immortal), - frame_system::CheckNonce::from(10), - frame_system::CheckWeight::new(), - pallet_transaction_payment::ChargeTransactionPayment::from(10), - BridgeRejectObsoleteHeadersAndMessages {}, - ( - BridgeRefundBridgeHubRococoMessages::default(), - BridgeRefundBridgeHubWococoMessages::default(), - ), - ); - - { - use bp_bridge_hub_rococo::BridgeHubSignedExtension; - let bhr_indirect_payload = bp_bridge_hub_rococo::SignedExtension::from_params( - 10, - 10, - TransactionEra::Immortal, - test_header::(1).hash(), - 10, - 10, + use bp_polkadot_core::SuffixedCommonSignedExtensionExt; + + sp_io::TestExternalities::default().execute_with(|| { + frame_system::BlockHash::::insert(BlockNumber::zero(), Hash::default()); + let payload: SignedExtra = ( + frame_system::CheckNonZeroSender::new(), + frame_system::CheckSpecVersion::new(), + frame_system::CheckTxVersion::new(), + frame_system::CheckGenesis::new(), + frame_system::CheckEra::from(Era::Immortal), + frame_system::CheckNonce::from(10), + frame_system::CheckWeight::new(), + pallet_transaction_payment::ChargeTransactionPayment::from(10), + BridgeRejectObsoleteHeadersAndMessages, + ( + BridgeRefundBridgeHubRococoMessages::default(), + BridgeRefundBridgeHubWococoMessages::default(), + ), ); - assert_eq!(payload.encode(), bhr_indirect_payload.encode()); - } - { - use bp_bridge_hub_wococo::BridgeHubSignedExtension; - let bhw_indirect_payload = bp_bridge_hub_wococo::SignedExtension::from_params( - 10, - 10, - TransactionEra::Immortal, - test_header::(1).hash(), - 10, - 10, - ); - assert_eq!(payload.encode(), bhw_indirect_payload.encode()); - } + { + let bhr_indirect_payload = bp_bridge_hub_rococo::SignedExtension::from_params( + VERSION.spec_version, + VERSION.transaction_version, + bp_runtime::TransactionEra::Immortal, + System::block_hash(BlockNumber::zero()), + 10, + 10, + (((), ()), ((), ())), + ); + assert_eq!(payload.encode(), bhr_indirect_payload.encode()); + assert_eq!( + payload.additional_signed().unwrap().encode(), + bhr_indirect_payload.additional_signed().unwrap().encode() + ) + } + + { + let bhw_indirect_payload = bp_bridge_hub_rococo::SignedExtension::from_params( + VERSION.spec_version, + VERSION.transaction_version, + bp_runtime::TransactionEra::Immortal, + System::block_hash(BlockNumber::zero()), + 10, + 10, + (((), ()), ((), ())), + ); + assert_eq!(payload.encode(), bhw_indirect_payload.encode()); + assert_eq!( + payload.additional_signed().unwrap().encode(), + bhw_indirect_payload.additional_signed().unwrap().encode() + ) + } + }); } } diff --git a/cumulus/parachains/runtimes/bridge-hubs/test-utils/Cargo.toml b/cumulus/parachains/runtimes/bridge-hubs/test-utils/Cargo.toml index 73678e8f91a3..d56c32d8afab 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/test-utils/Cargo.toml +++ b/cumulus/parachains/runtimes/bridge-hubs/test-utils/Cargo.toml @@ -4,7 +4,6 @@ version = "0.1.0" authors.workspace = true edition.workspace = true description = "Utils for BridgeHub testing" -publish = false [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "max-encoded-len"] } @@ -42,20 +41,20 @@ xcm-builder = { package = "staging-xcm-builder", path = "../../../../../polkadot xcm-executor = { package = "staging-xcm-executor", path = "../../../../../polkadot/xcm/xcm-executor", default-features = false} # Bridges -bp-bridge-hub-rococo = { path = "../../../../bridges/primitives/chain-bridge-hub-rococo", default-features = false } -bp-bridge-hub-wococo = { path = "../../../../bridges/primitives/chain-bridge-hub-wococo", default-features = false } -bp-header-chain = { path = "../../../../bridges/primitives/header-chain", default-features = false } -bp-messages = { path = "../../../../bridges/primitives/messages", default-features = false } -bp-parachains = { path = "../../../../bridges/primitives/parachains", default-features = false } -bp-polkadot-core = { path = "../../../../bridges/primitives/polkadot-core", default-features = false } -bp-relayers = { path = "../../../../bridges/primitives/relayers", default-features = false } -bp-runtime = { path = "../../../../bridges/primitives/runtime", default-features = false } -bp-test-utils = { path = "../../../../bridges/primitives/test-utils", default-features = false } -pallet-bridge-grandpa = { path = "../../../../bridges/modules/grandpa", default-features = false } -pallet-bridge-parachains = { path = "../../../../bridges/modules/parachains", default-features = false } -pallet-bridge-messages = { path = "../../../../bridges/modules/messages", default-features = false } -pallet-bridge-relayers = { path = "../../../../bridges/modules/relayers", default-features = false } -bridge-runtime-common = { path = "../../../../bridges/bin/runtime-common", default-features = false } +bp-bridge-hub-rococo = { path = "../../../../../bridges/primitives/chain-bridge-hub-rococo", default-features = false } +bp-bridge-hub-wococo = { path = "../../../../../bridges/primitives/chain-bridge-hub-wococo", default-features = false } +bp-header-chain = { path = "../../../../../bridges/primitives/header-chain", default-features = false } +bp-messages = { path = "../../../../../bridges/primitives/messages", default-features = false } +bp-parachains = { path = "../../../../../bridges/primitives/parachains", default-features = false } +bp-polkadot-core = { path = "../../../../../bridges/primitives/polkadot-core", default-features = false } +bp-relayers = { path = "../../../../../bridges/primitives/relayers", default-features = false } +bp-runtime = { path = "../../../../../bridges/primitives/runtime", default-features = false } +bp-test-utils = { path = "../../../../../bridges/primitives/test-utils", default-features = false } +pallet-bridge-grandpa = { path = "../../../../../bridges/modules/grandpa", default-features = false } +pallet-bridge-parachains = { path = "../../../../../bridges/modules/parachains", default-features = false } +pallet-bridge-messages = { path = "../../../../../bridges/modules/messages", default-features = false } +pallet-bridge-relayers = { path = "../../../../../bridges/modules/relayers", default-features = false } +bridge-runtime-common = { path = "../../../../../bridges/bin/runtime-common", default-features = false } [features] default = [ "std" ] diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/Cargo.toml b/cumulus/parachains/runtimes/collectives/collectives-polkadot/Cargo.toml index 1f17a0ef0e69..167f460610d6 100644 --- a/cumulus/parachains/runtimes/collectives/collectives-polkadot/Cargo.toml +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/Cargo.toml @@ -87,6 +87,7 @@ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-session-benchmarking/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking/runtime-benchmarks", @@ -106,6 +107,7 @@ runtime-benchmarks = [ "pallet-timestamp/runtime-benchmarks", "pallet-utility/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", "sp-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/impls.rs b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/impls.rs index c970d82cfe50..9f4c2a6a4c94 100644 --- a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/impls.rs +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/impls.rs @@ -184,7 +184,7 @@ pub mod benchmarks { impl> EnsureSuccessful for OpenHrmpChannel { fn ensure_successful() { if let ChannelStatus::Closed = ParachainSystem::get_channel_status(I::get().into()) { - ParachainSystem::open_outbound_hrmp_channel_for_benchmarks(I::get().into()) + ParachainSystem::open_outbound_hrmp_channel_for_benchmarks_or_tests(I::get().into()) } } } diff --git a/cumulus/parachains/runtimes/contracts/contracts-rococo/Cargo.toml b/cumulus/parachains/runtimes/contracts/contracts-rococo/Cargo.toml index 2e298e05ec76..f6fccc043542 100644 --- a/cumulus/parachains/runtimes/contracts/contracts-rococo/Cargo.toml +++ b/cumulus/parachains/runtimes/contracts/contracts-rococo/Cargo.toml @@ -138,6 +138,7 @@ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-session-benchmarking/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking/runtime-benchmarks", @@ -151,6 +152,7 @@ runtime-benchmarks = [ "pallet-timestamp/runtime-benchmarks", "pallet-utility/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", "sp-runtime/runtime-benchmarks", diff --git a/cumulus/parachains/runtimes/glutton/glutton-kusama/Cargo.toml b/cumulus/parachains/runtimes/glutton/glutton-kusama/Cargo.toml index 0caf00340d3a..2e2975ab87b8 100644 --- a/cumulus/parachains/runtimes/glutton/glutton-kusama/Cargo.toml +++ b/cumulus/parachains/runtimes/glutton/glutton-kusama/Cargo.toml @@ -55,6 +55,7 @@ runtime-benchmarks = [ "frame-system/runtime-benchmarks", "pallet-glutton/runtime-benchmarks", "pallet-sudo?/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "sp-runtime/runtime-benchmarks", "xcm-builder/runtime-benchmarks", "xcm-executor/runtime-benchmarks", diff --git a/cumulus/parachains/runtimes/test-utils/Cargo.toml b/cumulus/parachains/runtimes/test-utils/Cargo.toml index 681e5e64d412..378d9ea4c42c 100644 --- a/cumulus/parachains/runtimes/test-utils/Cargo.toml +++ b/cumulus/parachains/runtimes/test-utils/Cargo.toml @@ -4,7 +4,6 @@ version = "1.0.0" authors.workspace = true edition.workspace = true description = "Utils for Runtimes testing" -publish = false [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "max-encoded-len"] } diff --git a/cumulus/parachains/runtimes/testing/penpal/Cargo.toml b/cumulus/parachains/runtimes/testing/penpal/Cargo.toml index 99caf2f27d03..547695e76210 100644 --- a/cumulus/parachains/runtimes/testing/penpal/Cargo.toml +++ b/cumulus/parachains/runtimes/testing/penpal/Cargo.toml @@ -135,6 +135,7 @@ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-session-benchmarking/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system-benchmarking/runtime-benchmarks", @@ -147,6 +148,7 @@ runtime-benchmarks = [ "pallet-sudo/runtime-benchmarks", "pallet-timestamp/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "polkadot-primitives/runtime-benchmarks", "polkadot-runtime-common/runtime-benchmarks", diff --git a/cumulus/parachains/runtimes/testing/rococo-parachain/Cargo.toml b/cumulus/parachains/runtimes/testing/rococo-parachain/Cargo.toml index 863b9edd7258..54055ca732ba 100644 --- a/cumulus/parachains/runtimes/testing/rococo-parachain/Cargo.toml +++ b/cumulus/parachains/runtimes/testing/rococo-parachain/Cargo.toml @@ -105,6 +105,7 @@ std = [ runtime-benchmarks = [ "cumulus-pallet-parachain-system/runtime-benchmarks", "cumulus-pallet-xcmp-queue/runtime-benchmarks", + "cumulus-primitives-utility/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", @@ -113,6 +114,7 @@ runtime-benchmarks = [ "pallet-sudo/runtime-benchmarks", "pallet-timestamp/runtime-benchmarks", "pallet-xcm/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-parachain-primitives/runtime-benchmarks", "sp-runtime/runtime-benchmarks", "xcm-builder/runtime-benchmarks", diff --git a/cumulus/polkadot-parachain/Cargo.toml b/cumulus/polkadot-parachain/Cargo.toml index dc59b8207528..e6fedb5f1fab 100644 --- a/cumulus/polkadot-parachain/Cargo.toml +++ b/cumulus/polkadot-parachain/Cargo.toml @@ -12,13 +12,13 @@ path = "src/main.rs" [dependencies] async-trait = "0.1.73" -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } codec = { package = "parity-scale-codec", version = "3.0.0" } futures = "0.3.28" hex-literal = "0.4.1" log = "0.4.20" serde = { version = "1.0.188", features = ["derive"] } -serde_json = "1.0.105" +serde_json = "1.0.107" # Local rococo-parachain-runtime = { path = "../parachains/runtimes/testing/rococo-parachain" } @@ -116,6 +116,7 @@ runtime-benchmarks = [ "frame-benchmarking-cli/runtime-benchmarks", "frame-benchmarking/runtime-benchmarks", "glutton-runtime/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "penpal-runtime/runtime-benchmarks", "polkadot-cli/runtime-benchmarks", "polkadot-primitives/runtime-benchmarks", diff --git a/cumulus/primitives/utility/Cargo.toml b/cumulus/primitives/utility/Cargo.toml index 47c5442574fb..9ed1e664ac21 100644 --- a/cumulus/primitives/utility/Cargo.toml +++ b/cumulus/primitives/utility/Cargo.toml @@ -38,3 +38,11 @@ std = [ "xcm-executor/std", "xcm/std", ] + +runtime-benchmarks = [ + "frame-support/runtime-benchmarks", + "polkadot-runtime-common/runtime-benchmarks", + "sp-runtime/runtime-benchmarks", + "xcm-builder/runtime-benchmarks", + "xcm-executor/runtime-benchmarks", +] diff --git a/cumulus/test/relay-sproof-builder/Cargo.toml b/cumulus/test/relay-sproof-builder/Cargo.toml index c987a2b66c90..e044b92f7c4a 100644 --- a/cumulus/test/relay-sproof-builder/Cargo.toml +++ b/cumulus/test/relay-sproof-builder/Cargo.toml @@ -3,7 +3,6 @@ name = "cumulus-test-relay-sproof-builder" version = "0.1.0" authors.workspace = true edition.workspace = true -publish = false [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive" ] } diff --git a/cumulus/test/relay-validation-worker-provider/Cargo.toml b/cumulus/test/relay-validation-worker-provider/Cargo.toml deleted file mode 100644 index b7c59e832995..000000000000 --- a/cumulus/test/relay-validation-worker-provider/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "cumulus-test-relay-validation-worker-provider" -version = "0.1.0" -authors.workspace = true -edition.workspace = true -build = "build.rs" -publish = false - -[dependencies] - -# Polkadot -polkadot-node-core-pvf = { path = "../../../polkadot/node/core/pvf", features = ["test-utils"] } - -[build-dependencies] -toml = "0.7.6" diff --git a/cumulus/test/relay-validation-worker-provider/build.rs b/cumulus/test/relay-validation-worker-provider/build.rs deleted file mode 100644 index 60bb950db1fc..000000000000 --- a/cumulus/test/relay-validation-worker-provider/build.rs +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. - -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . - -use std::{ - env, fs, - path::{Path, PathBuf}, - process::{self, Command}, -}; -use toml::value::Table; - -/// The name of the project we will building. -const PROJECT_NAME: &str = "validation-worker"; -/// The env variable that instructs us to skip the build. -const SKIP_ENV: &str = "SKIP_BUILD"; - -fn main() { - if env::var(SKIP_ENV).is_ok() { - return - } - - let out_dir = PathBuf::from(env::var("OUT_DIR").expect("`OUT_DIR` is set by cargo")); - - let project = create_project(&out_dir); - build_project(&project.join("Cargo.toml")); - - fs::copy(project.join("target/release").join(PROJECT_NAME), out_dir.join(PROJECT_NAME)) - .expect("Copies validation worker"); -} - -fn find_cargo_lock() -> PathBuf { - let mut path = PathBuf::from( - env::var("CARGO_MANIFEST_DIR").expect("`CARGO_MANIFEST_DIR` is set by cargo"), - ); - - loop { - if path.join("Cargo.lock").exists() { - return path.join("Cargo.lock") - } - - if !path.pop() { - panic!("Could not find `Cargo.lock`") - } - } -} - -fn create_project(out_dir: &Path) -> PathBuf { - let project_dir = out_dir.join(format!("{}-project", PROJECT_NAME)); - fs::create_dir_all(project_dir.join("src")).expect("Creates project dir and project src dir"); - - let mut project_toml = Table::new(); - - let mut package = Table::new(); - package.insert("name".into(), PROJECT_NAME.into()); - package.insert("version".into(), "1.0.0".into()); - package.insert("edition".into(), "2021".into()); - - project_toml.insert("package".into(), package.into()); - - project_toml.insert("workspace".into(), Table::new().into()); - - let mut dependencies = Table::new(); - - let mut dependency_project = Table::new(); - dependency_project.insert( - "path".into(), - env::var("CARGO_MANIFEST_DIR") - .expect("`CARGO_MANIFEST_DIR` is set by cargo") - .into(), - ); - - dependencies - .insert("cumulus-test-relay-validation-worker-provider".into(), dependency_project.into()); - - project_toml.insert("dependencies".into(), dependencies.into()); - - add_patches(&mut project_toml); - - fs::write( - project_dir.join("Cargo.toml"), - toml::to_string_pretty(&project_toml).expect("Wasm workspace toml is valid; qed"), - ) - .expect("Writes project `Cargo.toml`"); - - fs::write( - project_dir.join("src").join("main.rs"), - r#" - cumulus_test_relay_validation_worker_provider::polkadot_node_core_pvf::decl_puppet_worker_main!(); - "#, - ) - .expect("Writes `main.rs`"); - - let cargo_lock = find_cargo_lock(); - fs::copy(&cargo_lock, project_dir.join("Cargo.lock")).expect("Copies `Cargo.lock`"); - println!("cargo:rerun-if-changed={}", cargo_lock.display()); - - project_dir -} - -fn add_patches(project_toml: &mut Table) { - let workspace_toml_path = PathBuf::from( - env::var("CARGO_MANIFEST_DIR").expect("`CARGO_MANIFEST_DIR` is set by cargo"), - ) - .join("../../../Cargo.toml"); - - let mut workspace_toml: Table = toml::from_str( - &fs::read_to_string(&workspace_toml_path).expect("Workspace root `Cargo.toml` exists; qed"), - ) - .expect("Workspace root `Cargo.toml` is a valid toml file; qed"); - - let mut workspace_path = workspace_toml_path; - workspace_path.pop(); - - while let Some(mut patch) = - workspace_toml.remove("patch").and_then(|p| p.try_into::().ok()) - { - // Iterate over all patches and make the patch path absolute from the workspace root path. - patch - .iter_mut() - .filter_map(|p| { - p.1.as_table_mut().map(|t| t.iter_mut().filter_map(|t| t.1.as_table_mut())) - }) - .flatten() - .for_each(|p| { - p.iter_mut().filter(|(k, _)| k == &"path").for_each(|(_, v)| { - if let Some(path) = v.as_str().map(PathBuf::from) { - if path.is_relative() { - *v = workspace_path.join(path).display().to_string().into(); - } - } - }) - }); - - project_toml.insert("patch".into(), patch.into()); - } -} - -fn build_project(cargo_toml: &Path) { - let cargo = env::var("CARGO").expect("`CARGO` env variable is always set by cargo"); - - let status = Command::new(cargo) - .arg("build") - .arg("--release") - .arg(format!("--manifest-path={}", cargo_toml.display())) - // Unset the `CARGO_TARGET_DIR` to prevent a cargo deadlock (cargo locks a target dir - // exclusive). - .env_remove("CARGO_TARGET_DIR") - // Do not call us recursively. - .env(SKIP_ENV, "1") - .status(); - - match status.map(|s| s.success()) { - Ok(true) => {}, - // Use `process.exit(1)` to have a clean error output. - _ => process::exit(1), - } -} diff --git a/cumulus/test/relay-validation-worker-provider/src/lib.rs b/cumulus/test/relay-validation-worker-provider/src/lib.rs deleted file mode 100644 index 6c3f4182b03b..000000000000 --- a/cumulus/test/relay-validation-worker-provider/src/lib.rs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Cumulus. - -// Cumulus is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Cumulus is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Cumulus. If not, see . - -//! Provides the [`VALIDATION_WORKER`] for integration tests in Cumulus. -//! -//! The validation worker is used by the relay chain to validate parachains. This worker is placed -//! in an extra process to provide better security and to ensure that a worker can be killed etc. -//! -//! !!This should only be used for tests!! - -pub use polkadot_node_core_pvf; - -/// The path to the validation worker. -pub const VALIDATION_WORKER: &str = concat!(env!("OUT_DIR"), "/validation-worker"); diff --git a/cumulus/test/service/Cargo.toml b/cumulus/test/service/Cargo.toml index 04d53545ead7..b8469cb66a74 100644 --- a/cumulus/test/service/Cargo.toml +++ b/cumulus/test/service/Cargo.toml @@ -11,7 +11,7 @@ path = "src/main.rs" [dependencies] async-trait = "0.1.73" -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } codec = { package = "parity-scale-codec", version = "3.0.0" } criterion = { version = "0.5.1", features = [ "async_tokio" ] } jsonrpsee = { version = "0.16.2", features = ["server"] } @@ -72,7 +72,6 @@ cumulus-primitives-core = { path = "../../primitives/core" } cumulus-primitives-parachain-inherent = { path = "../../primitives/parachain-inherent" } cumulus-relay-chain-inprocess-interface = { path = "../../client/relay-chain-inprocess-interface" } cumulus-relay-chain-interface = { path = "../../client/relay-chain-interface" } -cumulus-test-relay-validation-worker-provider = { path = "../relay-validation-worker-provider" } cumulus-test-runtime = { path = "../runtime" } cumulus-relay-chain-minimal-node = { path = "../../client/relay-chain-minimal-node" } cumulus-client-pov-recovery = { path = "../../client/pov-recovery" } @@ -102,6 +101,7 @@ runtime-benchmarks = [ "frame-system/runtime-benchmarks", "pallet-im-online/runtime-benchmarks", "pallet-timestamp/runtime-benchmarks", + "parachains-common/runtime-benchmarks", "polkadot-cli/runtime-benchmarks", "polkadot-primitives/runtime-benchmarks", "polkadot-service/runtime-benchmarks", diff --git a/cumulus/test/service/src/lib.rs b/cumulus/test/service/src/lib.rs index 3275aabc4d86..a721645546af 100644 --- a/cumulus/test/service/src/lib.rs +++ b/cumulus/test/service/src/lib.rs @@ -903,8 +903,9 @@ pub fn run_relay_chain_validator_node( config.rpc_addr = Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), port)); } - polkadot_test_service::run_validator_node( - config, - Some(cumulus_test_relay_validation_worker_provider::VALIDATION_WORKER.into()), - ) + let mut workers_path = std::env::current_exe().unwrap(); + workers_path.pop(); + workers_path.pop(); + + polkadot_test_service::run_validator_node(config, Some(workers_path)) } diff --git a/docker/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile b/docker/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile index e2c72dcfe2e9..7ad092476fec 100644 --- a/docker/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile +++ b/docker/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/library/ubuntu:20.04 +FROM docker.io/parity/base-bin # metadata ARG VCS_REF @@ -16,38 +16,29 @@ LABEL io.parity.image.authors="devops-team@parity.io" \ io.parity.image.created="${BUILD_DATE}" \ io.parity.image.documentation="https://github.com/paritytech/polkadot/" +USER root + # show backtraces ENV RUST_BACKTRACE 1 -# install tools and dependencies -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - libssl1.1 \ - ca-certificates \ - gnupg && \ - useradd -m -u 1000 -U -s /bin/sh -d /polkadot polkadot && \ -# add repo's gpg keys and install the published polkadot binary - gpg --keyserver ${GPG_KEYSERVER} --recv-keys ${POLKADOT_GPGKEY} && \ - gpg --export ${POLKADOT_GPGKEY} > /usr/share/keyrings/parity.gpg && \ - echo 'deb [signed-by=/usr/share/keyrings/parity.gpg] https://releases.parity.io/deb release main' > /etc/apt/sources.list.d/parity.list && \ +RUN \ apt-get update && \ apt-get install -y --no-install-recommends polkadot=${POLKADOT_VERSION#?} && \ -# apt cleanup apt-get autoremove -y && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* ; \ mkdir -p /data /polkadot/.local/share && \ - chown -R polkadot:polkadot /data && \ + chown -R parity:parity /data && \ ln -s /data /polkadot/.local/share/polkadot -USER polkadot +USER parity # check if executable works in this container RUN /usr/bin/polkadot --version -RUN /usr/bin/polkadot-execute-worker --version -RUN /usr/bin/polkadot-prepare-worker --version +RUN /usr/lib/polkadot/polkadot-execute-worker --version +RUN /usr/lib/polkadot/polkadot-prepare-worker --version -EXPOSE 30333 9933 9944 +EXPOSE 30333 9933 9944 9615 VOLUME ["/polkadot"] ENTRYPOINT ["/usr/bin/polkadot"] diff --git a/docker/dockerfiles/staking-miner/staking-miner_builder.Dockerfile b/docker/dockerfiles/staking-miner/staking-miner_builder.Dockerfile deleted file mode 100644 index a1932095fd4c..000000000000 --- a/docker/dockerfiles/staking-miner/staking-miner_builder.Dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -FROM paritytech/ci-linux:production as builder - -# metadata -ARG VCS_REF -ARG BUILD_DATE -ARG IMAGE_NAME="staking-miner" -ARG PROFILE=release - -LABEL description="This is the build stage. Here we create the binary." - -WORKDIR /app -COPY . /app -RUN cargo build --locked --$PROFILE --package staking-miner - -# ===== SECOND STAGE ====== - -FROM docker.io/library/ubuntu:20.04 -LABEL description="This is the 2nd stage: a very small image where we copy the binary." -LABEL io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${IMAGE_NAME}" \ - io.parity.image.description="${IMAGE_NAME} for substrate based chains" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/${IMAGE_NAME}/${IMAGE_NAME}_builder.Dockerfile" \ - io.parity.image.revision="${VCS_REF}" \ - io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/polkadot/" - -ARG PROFILE=release -COPY --from=builder /app/target/$PROFILE/staking-miner /usr/local/bin - -RUN useradd -u 1000 -U -s /bin/sh miner && \ - rm -rf /usr/bin /usr/sbin - -# show backtraces -ENV RUST_BACKTRACE 1 - -USER miner - -ENV SEED="" -ENV URI="wss://rpc.polkadot.io" -ENV RUST_LOG="info" - -# check if the binary works in this container -RUN /usr/local/bin/staking-miner --version - -ENTRYPOINT [ "/usr/local/bin/staking-miner" ] diff --git a/docker/dockerfiles/staking-miner/staking-miner_injected.Dockerfile b/docker/dockerfiles/staking-miner/staking-miner_injected.Dockerfile deleted file mode 100644 index 4901ab4a3736..000000000000 --- a/docker/dockerfiles/staking-miner/staking-miner_injected.Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -FROM docker.io/library/ubuntu:20.04 - -# metadata -ARG VCS_REF -ARG BUILD_DATE -ARG IMAGE_NAME="staking-miner" - -LABEL io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${IMAGE_NAME}" \ - io.parity.image.description="${IMAGE_NAME} for substrate based chains" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/${IMAGE_NAME}/${IMAGE_NAME}_injected.Dockerfile" \ - io.parity.image.revision="${VCS_REF}" \ - io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/polkadot/" - -# show backtraces -ENV RUST_BACKTRACE 1 - -# install tools and dependencies -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y \ - libssl1.1 \ - ca-certificates && \ -# apt cleanup - apt-get autoremove -y && \ - apt-get clean && \ - find /var/lib/apt/lists/ -type f -not -name lock -delete; \ - useradd -u 1000 -U -s /bin/sh miner - -# add binary to docker image -COPY ./staking-miner /usr/local/bin - -USER miner - -ENV SEED="" -ENV URI="wss://rpc.polkadot.io" -ENV RUST_LOG="info" - -# check if the binary works in this container -RUN /usr/local/bin/staking-miner --version - -ENTRYPOINT [ "/usr/local/bin/staking-miner" ] diff --git a/docker/scripts/staking-miner/README.md b/docker/scripts/staking-miner/README.md deleted file mode 100644 index 3610e1130316..000000000000 --- a/docker/scripts/staking-miner/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# staking-miner container image - -## Build using the Builder - -``` -./build.sh -``` - -## Build the injected Image - -You first need a valid Linux binary to inject. Let's assume this binary is located in `BIN_FOLDER`. - -``` -./build-injected.sh "$BIN_FOLDER" -``` - -## Test - -Here is how to test the image. We can generate a valid seed but the staking-miner will quickly notice that our -account is not funded and "does not exist". - -You may pass any ENV supported by the binary and must provide at least a few such as `SEED` and `URI`: -``` -ENV SEED="" -ENV URI="wss://rpc.polkadot.io:443" -ENV RUST_LOG="info" -``` - -``` -export SEED=$(subkey generate -n polkadot --output-type json | jq -r .secretSeed) -podman run --rm -it \ - -e URI="wss://rpc.polkadot.io:443" \ - -e RUST_LOG="info" \ - -e SEED \ - localhost/parity/staking-miner \ - dry-run seq-phragmen -``` diff --git a/docker/scripts/staking-miner/build-injected.sh b/docker/scripts/staking-miner/build-injected.sh deleted file mode 100755 index efe323b5fed8..000000000000 --- a/docker/scripts/staking-miner/build-injected.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -# Sample call: -# $0 /path/to/folder_with_staking-miner_binary -# This script replace the former dedicated staking-miner "injected" Dockerfile -# and shows how to use the generic binary_injected.dockerfile - -PROJECT_ROOT=`git rev-parse --show-toplevel` - -export BINARY=staking-miner -export ARTIFACTS_FOLDER=$1 - -$PROJECT_ROOT/docker/scripts/build-injected.sh diff --git a/docker/scripts/staking-miner/build.sh b/docker/scripts/staking-miner/build.sh deleted file mode 100755 index c2b6ab77e531..000000000000 --- a/docker/scripts/staking-miner/build.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -# Sample call: -# $0 /path/to/folder_with_staking-miner_binary -# This script replace the former dedicated staking-miner "injected" Dockerfile -# and shows how to use the generic binary_injected.dockerfile - -PROJECT_ROOT=`git rev-parse --show-toplevel` -ENGINE=podman - -echo "Building the staking-miner using the Builder image" -echo "PROJECT_ROOT=$PROJECT_ROOT" -$ENGINE build -t staking-miner -f "${PROJECT_ROOT}/docker/dockerfiles/staking-miner/staking-miner_builder.Dockerfile" "$PROJECT_ROOT" diff --git a/docker/scripts/staking-miner/staking-miner_Dockerfile.README.md b/docker/scripts/staking-miner/staking-miner_Dockerfile.README.md deleted file mode 100644 index ce424c42f479..000000000000 --- a/docker/scripts/staking-miner/staking-miner_Dockerfile.README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Staking-miner Docker image - -## [GitHub](https://github.com/paritytech/polkadot/tree/master/utils/staking-miner) diff --git a/docker/scripts/staking-miner/staking-miner_builder.Dockerfile b/docker/scripts/staking-miner/staking-miner_builder.Dockerfile deleted file mode 100644 index 0ae77f36c79d..000000000000 --- a/docker/scripts/staking-miner/staking-miner_builder.Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -FROM paritytech/ci-linux:production as builder - -# metadata -ARG VCS_REF -ARG BUILD_DATE -ARG IMAGE_NAME="staking-miner" -ARG PROFILE=production - -LABEL description="This is the build stage. Here we create the binary." - -WORKDIR /app -COPY . /app -RUN cargo build --locked --profile $PROFILE --package staking-miner - -# ===== SECOND STAGE ====== - -FROM docker.io/parity/base-bin:latest -LABEL description="This is the 2nd stage: a very small image where we copy the binary." -LABEL io.parity.image.authors="devops-team@parity.io" \ - io.parity.image.vendor="Parity Technologies" \ - io.parity.image.title="${IMAGE_NAME}" \ - io.parity.image.description="${IMAGE_NAME} for substrate based chains" \ - io.parity.image.source="https://github.com/paritytech/polkadot/blob/${VCS_REF}/scripts/ci/dockerfiles/${IMAGE_NAME}/${IMAGE_NAME}_builder.Dockerfile" \ - io.parity.image.revision="${VCS_REF}" \ - io.parity.image.created="${BUILD_DATE}" \ - io.parity.image.documentation="https://github.com/paritytech/polkadot/" - -ARG PROFILE=release -COPY --from=builder /app/target/$PROFILE/staking-miner /usr/local/bin - -# show backtraces -ENV RUST_BACKTRACE 1 - -USER parity - -ENV SEED="" -ENV URI="wss://rpc.polkadot.io" -ENV RUST_LOG="info" - -# check if the binary works in this container -RUN /usr/local/bin/staking-miner --version - -ENTRYPOINT [ "/usr/local/bin/staking-miner" ] diff --git a/docker/scripts/staking-miner/test-build.sh b/docker/scripts/staking-miner/test-build.sh deleted file mode 100755 index 0ce74e2df296..000000000000 --- a/docker/scripts/staking-miner/test-build.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -TMP=$(mktemp -d) -ENGINE=${ENGINE:-podman} - -# You need to build an injected image first - -# Fetch some binaries -$ENGINE run --user root --rm -i \ - -v "$TMP:/export" \ - --entrypoint /bin/bash \ - parity/staking-miner -c \ - 'cp "$(which staking-miner)" /export' - -echo "Checking binaries we got:" -tree $TMP - -./build-injected.sh $TMP diff --git a/polkadot/.github/CODEOWNERS b/polkadot/.github/CODEOWNERS deleted file mode 100644 index a92dc0bb006c..000000000000 --- a/polkadot/.github/CODEOWNERS +++ /dev/null @@ -1,6 +0,0 @@ -# CI -/.github/ @paritytech/ci @chevdor -/scripts/ci/ @paritytech/ci @chevdor -/.gitlab-ci.yml @paritytech/ci -# lingua.dic is not managed by CI team -/scripts/ci/gitlab/lingua.dic diff --git a/polkadot/cli/Cargo.toml b/polkadot/cli/Cargo.toml index c8c3f91f9835..794d8c4714af 100644 --- a/polkadot/cli/Cargo.toml +++ b/polkadot/cli/Cargo.toml @@ -15,7 +15,7 @@ wasm-opt = false crate-type = ["cdylib", "rlib"] [dependencies] -clap = { version = "4.4.2", features = ["derive"], optional = true } +clap = { version = "4.4.3", features = ["derive"], optional = true } log = "0.4.17" thiserror = "1.0.48" futures = "0.3.21" diff --git a/polkadot/cli/src/cli.rs b/polkadot/cli/src/cli.rs index 66205902b79d..aaf8f1705760 100644 --- a/polkadot/cli/src/cli.rs +++ b/polkadot/cli/src/cli.rs @@ -19,8 +19,15 @@ use clap::Parser; use std::path::PathBuf; -/// The version of the node. The passed-in version of the workers should match this. -pub const NODE_VERSION: &'static str = env!("SUBSTRATE_CLI_IMPL_VERSION"); +/// The version of the node. +/// +/// This is the version that is used for versioning this node binary. +/// By default the `minor` version is bumped in every release. `Major` or `patch` releases are only +/// expected in very rare cases. +/// +/// The worker binaries associated to the node binary should ensure that they are using the same +/// version as the main node that started them. +pub const NODE_VERSION: &'static str = "1.1.0"; #[allow(missing_docs)] #[derive(Debug, Parser)] diff --git a/polkadot/cli/src/command.rs b/polkadot/cli/src/command.rs index a2a00d0ebd3f..a081e1b5181b 100644 --- a/polkadot/cli/src/command.rs +++ b/polkadot/cli/src/command.rs @@ -55,7 +55,8 @@ impl SubstrateCli for Cli { } fn impl_version() -> String { - NODE_VERSION.into() + let commit_hash = env!("SUBSTRATE_CLI_COMMIT_HASH"); + format!("{NODE_VERSION}-{commit_hash}") } fn description() -> String { diff --git a/polkadot/node/core/backing/src/tests/mod.rs b/polkadot/node/core/backing/src/tests/mod.rs index a981487db445..4c2fd6becb42 100644 --- a/polkadot/node/core/backing/src/tests/mod.rs +++ b/polkadot/node/core/backing/src/tests/mod.rs @@ -18,7 +18,7 @@ use self::test_helpers::mock::new_leaf; use super::*; use ::test_helpers::{ dummy_candidate_receipt_bad_sig, dummy_collator, dummy_collator_signature, - dummy_committed_candidate_receipt, dummy_hash, + dummy_committed_candidate_receipt, dummy_hash, validator_pubkeys, }; use assert_matches::assert_matches; use futures::{future, Future}; @@ -48,10 +48,6 @@ mod prospective_parachains; const ASYNC_BACKING_DISABLED_ERROR: RuntimeApiError = RuntimeApiError::NotSupported { runtime_api_name: "test-runtime" }; -fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec { - val_ids.iter().map(|v| v.public().into()).collect() -} - fn table_statement_to_primitive(statement: TableStatement) -> Statement { match statement { TableStatement::Seconded(committed_candidate_receipt) => @@ -299,6 +295,81 @@ async fn test_startup(virtual_overseer: &mut VirtualOverseer, test_state: &TestS ); } +async fn assert_validation_requests( + virtual_overseer: &mut VirtualOverseer, + validation_code: ValidationCode, +) { + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) + ) if hash == validation_code.hash() => { + tx.send(Ok(Some(validation_code))).unwrap(); + } + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) + ) => { + tx.send(Ok(1u32.into())).unwrap(); + } + ); + + assert_matches!( + virtual_overseer.recv().await, + AllMessages::RuntimeApi( + RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) + ) if sess_idx == 1 => { + tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); + } + ); +} + +async fn assert_validate_from_exhaustive( + virtual_overseer: &mut VirtualOverseer, + pvd: &PersistedValidationData, + pov: &PoV, + validation_code: &ValidationCode, + candidate: &CommittedCandidateReceipt, + expected_head_data: &HeadData, + result_validation_data: PersistedValidationData, +) { + assert_matches!( + virtual_overseer.recv().await, + AllMessages::CandidateValidation( + CandidateValidationMessage::ValidateFromExhaustive( + _pvd, + _validation_code, + candidate_receipt, + _pov, + _, + timeout, + tx, + ), + ) if _pvd == *pvd && + _validation_code == *validation_code && + *_pov == *pov && &candidate_receipt.descriptor == candidate.descriptor() && + timeout == PvfExecTimeoutKind::Backing && + candidate.commitments.hash() == candidate_receipt.commitments_hash => + { + tx.send(Ok(ValidationResult::Valid( + CandidateCommitments { + head_data: expected_head_data.clone(), + horizontal_messages: Default::default(), + upward_messages: Default::default(), + new_validation_code: None, + processed_downward_messages: 0, + hrmp_watermark: 0, + }, + result_validation_data, + ))) + .unwrap(); + } + ); +} + // Test that a `CandidateBackingMessage::Second` issues validation work // and in case validation is successful issues a `StatementDistributionMessage`. #[test] @@ -334,65 +405,18 @@ fn backing_second_works() { virtual_overseer.send(FromOrchestra::Communication { msg: second }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code.clone()).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromExhaustive( - _pvd, - _validation_code, - candidate_receipt, - _pov, - _, - timeout, - tx, - ), - ) if _pvd == pvd && - _validation_code == validation_code && - *_pov == pov && &candidate_receipt.descriptor == candidate.descriptor() && - timeout == PvfExecTimeoutKind::Backing && - candidate.commitments.hash() == candidate_receipt.commitments_hash => - { - tx.send(Ok(ValidationResult::Valid( - CandidateCommitments { - head_data: expected_head_data.clone(), - horizontal_messages: Default::default(), - upward_messages: Default::default(), - new_validation_code: None, - processed_downward_messages: 0, - hrmp_watermark: 0, - }, - test_state.validation_data.clone(), - ))) - .unwrap(); - } - ); + assert_validate_from_exhaustive( + &mut virtual_overseer, + &pvd, + &pov, + &validation_code, + &candidate, + expected_head_data, + test_state.validation_data.clone(), + ) + .await; assert_matches!( virtual_overseer.recv().await, @@ -499,32 +523,7 @@ fn backing_works() { virtual_overseer.send(FromOrchestra::Communication { msg: statement }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code.clone()).await; // Sending a `Statement::Seconded` for our assignment will start // validation process. The first thing requested is the PoV. @@ -702,32 +701,7 @@ fn backing_works_while_validation_ongoing() { CandidateBackingMessage::Statement(test_state.relay_parent, signed_a.clone()); virtual_overseer.send(FromOrchestra::Communication { msg: statement }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code.clone()).await; // Sending a `Statement::Seconded` for our assignment will start // validation process. The first thing requested is PoV from the @@ -893,32 +867,7 @@ fn backing_misbehavior_works() { virtual_overseer.send(FromOrchestra::Communication { msg: statement }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code.clone()).await; assert_matches!( virtual_overseer.recv().await, @@ -1098,32 +1047,7 @@ fn backing_dont_second_invalid() { virtual_overseer.send(FromOrchestra::Communication { msg: second }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code_a.hash() => { - tx.send(Ok(Some(validation_code_a.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code_a.clone()).await; assert_matches!( virtual_overseer.recv().await, @@ -1163,32 +1087,7 @@ fn backing_dont_second_invalid() { virtual_overseer.send(FromOrchestra::Communication { msg: second }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code_b.hash() => { - tx.send(Ok(Some(validation_code_b.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code_b.clone()).await; assert_matches!( virtual_overseer.recv().await, @@ -1300,32 +1199,7 @@ fn backing_second_after_first_fails_works() { virtual_overseer.send(FromOrchestra::Communication { msg: statement }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code.clone()).await; // Subsystem requests PoV and requests validation. assert_matches!( @@ -1408,32 +1282,7 @@ fn backing_second_after_first_fails_works() { // triggered on the prev step. virtual_overseer.send(FromOrchestra::Communication { msg: second }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code_to_second.hash() => { - tx.send(Ok(Some(validation_code_to_second.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code_to_second.clone()).await; assert_matches!( virtual_overseer.recv().await, @@ -1494,32 +1343,7 @@ fn backing_works_after_failed_validation() { virtual_overseer.send(FromOrchestra::Communication { msg: statement }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code.clone()).await; // Subsystem requests PoV and requests validation. assert_matches!( @@ -1722,32 +1546,7 @@ fn retry_works() { CandidateBackingMessage::Statement(test_state.relay_parent, signed_a.clone()); virtual_overseer.send(FromOrchestra::Communication { msg: statement }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code.clone()).await; // Subsystem requests PoV and requests validation. // We cancel - should mean retry on next backing statement. @@ -1810,32 +1609,7 @@ fn retry_works() { CandidateBackingMessage::Statement(test_state.relay_parent, signed_c.clone()); virtual_overseer.send(FromOrchestra::Communication { msg: statement }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code.clone()).await; assert_matches!( virtual_overseer.recv().await, @@ -2026,65 +1800,18 @@ fn cannot_second_multiple_candidates_per_parent() { virtual_overseer.send(FromOrchestra::Communication { msg: second }).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code.clone()).await; - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromExhaustive( - _pvd, - _validation_code, - candidate_receipt, - _pov, - _, - timeout, - tx, - ), - ) if _pvd == pvd && - _validation_code == validation_code && - *_pov == pov && &candidate_receipt.descriptor == candidate.descriptor() && - timeout == PvfExecTimeoutKind::Backing && - candidate.commitments.hash() == candidate_receipt.commitments_hash => - { - tx.send(Ok(ValidationResult::Valid( - CandidateCommitments { - head_data: expected_head_data.clone(), - horizontal_messages: Default::default(), - upward_messages: Default::default(), - new_validation_code: None, - processed_downward_messages: 0, - hrmp_watermark: 0, - }, - test_state.validation_data.clone(), - ))) - .unwrap(); - } - ); + assert_validate_from_exhaustive( + &mut virtual_overseer, + &pvd, + &pov, + &validation_code, + &candidate, + expected_head_data, + test_state.validation_data.clone(), + ) + .await; assert_matches!( virtual_overseer.recv().await, @@ -2131,32 +1858,7 @@ fn cannot_second_multiple_candidates_per_parent() { virtual_overseer.send(FromOrchestra::Communication { msg: second }).await; // The validation is still requested. - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(&mut virtual_overseer, validation_code.clone()).await; assert_matches!( virtual_overseer.recv().await, diff --git a/polkadot/node/core/backing/src/tests/prospective_parachains.rs b/polkadot/node/core/backing/src/tests/prospective_parachains.rs index d6e93fb04d34..14f720b721f2 100644 --- a/polkadot/node/core/backing/src/tests/prospective_parachains.rs +++ b/polkadot/node/core/backing/src/tests/prospective_parachains.rs @@ -208,32 +208,7 @@ async fn assert_validate_seconded_candidate( expected_head_data: &HeadData, fetch_pov: bool, ) { - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(parent, RuntimeApiRequest::ValidationCodeByHash(hash, tx)) - ) if parent == relay_parent && hash == validation_code.hash() => { - tx.send(Ok(Some(validation_code.clone()))).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionIndexForChild(tx)) - ) => { - tx.send(Ok(1u32.into())).unwrap(); - } - ); - - assert_matches!( - virtual_overseer.recv().await, - AllMessages::RuntimeApi( - RuntimeApiMessage::Request(_, RuntimeApiRequest::SessionExecutorParams(sess_idx, tx)) - ) if sess_idx == 1 => { - tx.send(Ok(Some(ExecutorParams::default()))).unwrap(); - } - ); + assert_validation_requests(virtual_overseer, validation_code.clone()).await; if fetch_pov { assert_matches!( diff --git a/polkadot/node/core/dispute-coordinator/src/initialized.rs b/polkadot/node/core/dispute-coordinator/src/initialized.rs index 9bfca2d81a04..9cd544a8c536 100644 --- a/polkadot/node/core/dispute-coordinator/src/initialized.rs +++ b/polkadot/node/core/dispute-coordinator/src/initialized.rs @@ -301,6 +301,13 @@ impl Initialized { self.participation.process_active_leaves_update(ctx, &update).await?; if let Some(new_leaf) = update.activated { + gum::trace!( + target: LOG_TARGET, + leaf_hash = ?new_leaf.hash, + block_number = new_leaf.number, + "Processing ActivatedLeaf" + ); + let session_idx = self.runtime_info.get_session_index_for_child(ctx.sender(), new_leaf.hash).await; diff --git a/polkadot/node/core/dispute-coordinator/src/participation/mod.rs b/polkadot/node/core/dispute-coordinator/src/participation/mod.rs index 5a3c4be90aa0..35d07b411e8f 100644 --- a/polkadot/node/core/dispute-coordinator/src/participation/mod.rs +++ b/polkadot/node/core/dispute-coordinator/src/participation/mod.rs @@ -260,6 +260,13 @@ impl Participation { req: ParticipationRequest, recent_head: Hash, ) -> FatalResult<()> { + gum::trace!( + target: LOG_TARGET, + candidate_hash = ?req.candidate_hash(), + session = req.session(), + "Forking participation" + ); + let participation_timer = self.metrics.time_participation(); if self.running_participations.insert(*req.candidate_hash()) { let sender = ctx.sender().clone(); @@ -314,12 +321,24 @@ async fn participate( }, Ok(Ok(data)) => data, Ok(Err(RecoveryError::Invalid)) => { + gum::debug!( + target: LOG_TARGET, + candidate_hash = ?req.candidate_hash(), + session = req.session(), + "Invalid availability data during participation" + ); // the available data was recovered but it is invalid, therefore we'll // vote negatively for the candidate dispute send_result(&mut result_sender, req, ParticipationOutcome::Invalid).await; return }, Ok(Err(RecoveryError::Unavailable)) | Ok(Err(RecoveryError::ChannelClosed)) => { + gum::debug!( + target: LOG_TARGET, + candidate_hash = ?req.candidate_hash(), + session = req.session(), + "Can't fetch availability data in participation" + ); send_result(&mut result_sender, req, ParticipationOutcome::Unavailable).await; return }, diff --git a/polkadot/node/core/dispute-coordinator/src/participation/queues/mod.rs b/polkadot/node/core/dispute-coordinator/src/participation/queues/mod.rs index 1105135a747f..d9e86def168c 100644 --- a/polkadot/node/core/dispute-coordinator/src/participation/queues/mod.rs +++ b/polkadot/node/core/dispute-coordinator/src/participation/queues/mod.rs @@ -277,6 +277,11 @@ impl Queues { match self.priority.entry(comparator) { Entry::Occupied(_) => req.discard_timer(), Entry::Vacant(vac) => { + gum::trace!( + target: LOG_TARGET, + candidate_hash = ?req.candidate_hash(), + "Added to priority participation queue" + ); vac.insert(req); }, } @@ -295,6 +300,11 @@ impl Queues { match self.best_effort.entry(comparator) { Entry::Occupied(_) => req.discard_timer(), Entry::Vacant(vac) => { + gum::trace!( + target: LOG_TARGET, + candidate_hash = ?req.candidate_hash(), + "Added to best effort participation queue" + ); vac.insert(req); }, } diff --git a/polkadot/node/core/dispute-coordinator/src/scraping/candidates.rs b/polkadot/node/core/dispute-coordinator/src/scraping/candidates.rs index b1870164cb88..c7c9c519916e 100644 --- a/polkadot/node/core/dispute-coordinator/src/scraping/candidates.rs +++ b/polkadot/node/core/dispute-coordinator/src/scraping/candidates.rs @@ -120,18 +120,13 @@ impl ScrapedCandidates { // Removes all candidates up to a given height. The candidates at the block height are NOT // removed. - pub fn remove_up_to_height(&mut self, height: &BlockNumber) -> HashSet { - let mut candidates_modified: HashSet = HashSet::new(); + pub fn remove_up_to_height(&mut self, height: &BlockNumber) { let not_stale = self.candidates_by_block_number.split_off(&height); let stale = std::mem::take(&mut self.candidates_by_block_number); self.candidates_by_block_number = not_stale; - for candidates in stale.values() { - for c in candidates { - self.candidates.remove(c); - candidates_modified.insert(*c); - } + for candidate in stale.values().flatten() { + self.candidates.remove(candidate); } - candidates_modified } pub fn insert(&mut self, block_number: BlockNumber, candidate_hash: CandidateHash) { diff --git a/polkadot/node/core/dispute-coordinator/src/scraping/mod.rs b/polkadot/node/core/dispute-coordinator/src/scraping/mod.rs index 67434bca85d4..fdf39cff0f2e 100644 --- a/polkadot/node/core/dispute-coordinator/src/scraping/mod.rs +++ b/polkadot/node/core/dispute-coordinator/src/scraping/mod.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use std::collections::{BTreeMap, HashSet}; +use std::collections::{btree_map::Entry, BTreeMap, HashSet}; use futures::channel::oneshot; use schnellru::{ByLength, LruMap}; @@ -78,49 +78,64 @@ impl ScrapedUpdates { /// concluding against a candidate. Each candidate hash maps to a number of /// block heights, which in turn map to vectors of blocks at those heights. pub struct Inclusions { - inclusions_inner: BTreeMap>>, + inclusions_inner: BTreeMap>>, + /// Keeps track at which block number a candidate was inserted. Used in `remove_up_to_height`. + /// Without this tracking we won't be able to remove all candidates before block X. + candidates_by_block_number: BTreeMap>, } impl Inclusions { pub fn new() -> Self { - Self { inclusions_inner: BTreeMap::new() } + Self { inclusions_inner: BTreeMap::new(), candidates_by_block_number: BTreeMap::new() } } - // Add parent block to the vector which has CandidateHash as an outer key and - // BlockNumber as an inner key pub fn insert( &mut self, candidate_hash: CandidateHash, block_number: BlockNumber, block_hash: Hash, ) { - if let Some(blocks_including) = self.inclusions_inner.get_mut(&candidate_hash) { - if let Some(blocks_at_height) = blocks_including.get_mut(&block_number) { - blocks_at_height.push(block_hash); - } else { - blocks_including.insert(block_number, Vec::from([block_hash])); - } - } else { - let mut blocks_including: BTreeMap> = BTreeMap::new(); - blocks_including.insert(block_number, Vec::from([block_hash])); - self.inclusions_inner.insert(candidate_hash, blocks_including); - } + self.inclusions_inner + .entry(candidate_hash) + .or_default() + .entry(block_number) + .or_default() + .insert(block_hash); + + self.candidates_by_block_number + .entry(block_number) + .or_default() + .insert(candidate_hash); } - pub fn remove_up_to_height( - &mut self, - height: &BlockNumber, - candidates_modified: HashSet, - ) { - for candidate in candidates_modified { - if let Some(blocks_including) = self.inclusions_inner.get_mut(&candidate) { - // Returns everything after the given key, including the key. This works because the - // blocks are sorted in ascending order. - *blocks_including = blocks_including.split_off(height); + /// Removes all candidates up to a given height. + /// + /// The candidates at the block height are NOT removed. + pub fn remove_up_to_height(&mut self, height: &BlockNumber) { + let not_stale = self.candidates_by_block_number.split_off(&height); + let stale = std::mem::take(&mut self.candidates_by_block_number); + self.candidates_by_block_number = not_stale; + + for candidate in stale.into_values().flatten() { + match self.inclusions_inner.entry(candidate) { + Entry::Vacant(_) => { + // Rare case where same candidate was present on multiple heights, but all are + // pruned at the same time. This candidate was already pruned in the previous + // occurence so it is skipped now. + }, + Entry::Occupied(mut e) => { + let mut blocks_including = std::mem::take(e.get_mut()); + // Returns everything after the given key, including the key. This works because + // the blocks are sorted in ascending order. + blocks_including = blocks_including.split_off(&height); + if blocks_including.is_empty() { + e.remove_entry(); + } else { + *e.get_mut() = blocks_including; + } + }, } } - self.inclusions_inner - .retain(|_, blocks_including| blocks_including.keys().len() > 0); } pub fn get(&self, candidate: &CandidateHash) -> Vec<(BlockNumber, Hash)> { @@ -134,6 +149,10 @@ impl Inclusions { } inclusions_as_vec } + + pub fn contains(&self, candidate: &CandidateHash) -> bool { + self.inclusions_inner.get(candidate).is_some() + } } /// Chain scraper @@ -159,20 +178,20 @@ impl Inclusions { /// another precaution to have their `CandidateReceipts` available in case a dispute is raised on /// them, pub struct ChainScraper { - /// All candidates we have seen included, which not yet have been finalized. - included_candidates: candidates::ScrapedCandidates, - /// All candidates we have seen backed + /// All candidates we have seen backed. backed_candidates: candidates::ScrapedCandidates, - /// Latest relay blocks observed by the provider. - /// - /// We assume that ancestors of cached blocks are already processed, i.e. we have saved - /// corresponding included candidates. - last_observed_blocks: LruMap, + /// Maps included candidate hashes to one or more relay block heights and hashes. /// These correspond to all the relay blocks which marked a candidate as included, /// and are needed to apply reversions in case a dispute is concluded against the /// candidate. inclusions: Inclusions, + + /// Latest relay blocks observed by the provider. + /// + /// This is used to avoid redundant scraping of ancestry. We assume that ancestors of cached + /// blocks are already processed, i.e. we have saved corresponding included candidates. + last_observed_blocks: LruMap, } impl ChainScraper { @@ -194,10 +213,9 @@ impl ChainScraper { Sender: overseer::DisputeCoordinatorSenderTrait, { let mut s = Self { - included_candidates: candidates::ScrapedCandidates::new(), backed_candidates: candidates::ScrapedCandidates::new(), - last_observed_blocks: LruMap::new(ByLength::new(LRU_OBSERVED_BLOCKS_CAPACITY)), inclusions: Inclusions::new(), + last_observed_blocks: LruMap::new(ByLength::new(LRU_OBSERVED_BLOCKS_CAPACITY)), }; let update = ActiveLeavesUpdate { activated: Some(initial_head), deactivated: Default::default() }; @@ -207,7 +225,7 @@ impl ChainScraper { /// Check whether we have seen a candidate included on any chain. pub fn is_candidate_included(&self, candidate_hash: &CandidateHash) -> bool { - self.included_candidates.contains(candidate_hash) + self.inclusions.contains(candidate_hash) } /// Check whether the candidate is backed @@ -298,9 +316,7 @@ impl ChainScraper { { Some(key_to_prune) => { self.backed_candidates.remove_up_to_height(&key_to_prune); - let candidates_modified = - self.included_candidates.remove_up_to_height(&key_to_prune); - self.inclusions.remove_up_to_height(&key_to_prune, candidates_modified); + self.inclusions.remove_up_to_height(&key_to_prune); }, None => { // Nothing to prune. We are still in the beginning of the chain and there are not @@ -337,7 +353,6 @@ impl ChainScraper { ?block_number, "Processing included event" ); - self.included_candidates.insert(block_number, candidate_hash); self.inclusions.insert(candidate_hash, block_number, block_hash); included_receipts.push(receipt); }, diff --git a/polkadot/node/core/dispute-coordinator/src/scraping/tests.rs b/polkadot/node/core/dispute-coordinator/src/scraping/tests.rs index ae722fcb1b47..748f9a16f493 100644 --- a/polkadot/node/core/dispute-coordinator/src/scraping/tests.rs +++ b/polkadot/node/core/dispute-coordinator/src/scraping/tests.rs @@ -41,7 +41,7 @@ use polkadot_primitives::{ GroupIndex, Hash, HashT, HeadData, Id as ParaId, }; -use crate::LOG_TARGET; +use crate::{scraping::Inclusions, LOG_TARGET}; use super::ChainScraper; @@ -149,6 +149,11 @@ fn get_block_number_hash(n: BlockNumber) -> Hash { BlakeTwo256::hash(&n.encode()) } +// Creates a dummy relay chain block hash with the convention of hash(b). +fn get_relay_block_hash(height: BlockNumber, fork: u32) -> Hash { + BlakeTwo256::hash(&format!("b_{}_{}", height, fork).encode()) +} + /// Get a dummy event that corresponds to candidate inclusion for the given block number. fn get_backed_and_included_candidate_events(block_number: BlockNumber) -> Vec { let candidate_receipt = make_candidate_receipt(get_block_number_hash(block_number)); @@ -662,3 +667,594 @@ fn inclusions_per_candidate_properly_adds_and_prunes() { assert!(scraper.get_blocks_including_candidate(&candidate.hash()).len() == 0); }); } + +// ----- Inclusions tests ----- + +#[test] +fn inclusions_initialization() { + let inclusions = Inclusions::new(); + + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} +#[test] +fn inclusions_insertion() { + let mut inclusions = Inclusions::new(); + let candidate_receipt = make_candidate_receipt(get_magic_candidate_hash()); + let candidate_hash = candidate_receipt.hash(); + let block_number = 0; + let block_hash = get_block_number_hash(block_number); + + inclusions.insert(candidate_hash, block_number, block_hash); + + // Check inclusions_inner + assert!(inclusions.inclusions_inner.len() == 1, "Expected inclusions_inner to have length 1"); + assert!( + inclusions.inclusions_inner.contains_key(&candidate_hash), + "Expected candidate_hash to be present in inclusions_inner" + ); + let inner_map = inclusions.inclusions_inner.get(&candidate_hash).unwrap(); + assert!(inner_map.len() == 1, "Expected inner_map to have length 1"); + assert!( + inner_map.contains_key(&block_number), + "Expected block_number to be present for the candidate_hash in inclusions_inner" + ); + let hash_set = inner_map.get(&block_number).unwrap(); + assert!(hash_set.len() == 1, "Expected hash_map to have length 1"); + assert!( + hash_set.contains(&block_hash), + "Expected block_hash to be present for the block_number in inclusions_inner" + ); + + // Check candidates_by_block_number + assert!( + inclusions.candidates_by_block_number.len() == 1, + "Expected candidates_by_block_number to have length 1" + ); + assert!( + inclusions.candidates_by_block_number.contains_key(&block_number), + "Expected block_number to be present in candidates_by_block_number" + ); + let candidate_set = inclusions.candidates_by_block_number.get(&block_number).unwrap(); + assert!( + candidate_set.len() == 1, + "Expected candidate_set to have length 1 for the block_number in candidates_by_block_number" + ); + assert!( + candidate_set.contains(&candidate_hash), + "Expected candidate_hash to be present for the block_number in candidates_by_block_number" + ); +} + +#[test] +fn inclusions_get() { + let mut inclusions = Inclusions::new(); + let candidate_receipt = make_candidate_receipt(get_magic_candidate_hash()); + let candidate_hash = candidate_receipt.hash(); + + // Insert the candidate with multiple block numbers and block hashes + let block_numbers = [0, 1, 2]; + let block_hashes: Vec<_> = + block_numbers.iter().map(|&num| get_block_number_hash(num)).collect(); + + for (&block_number, &block_hash) in block_numbers.iter().zip(&block_hashes) { + inclusions.insert(candidate_hash, block_number, block_hash); + } + + // Call the get method for that candidate + let result = inclusions.get(&candidate_hash); + + // Verify that the method returns the correct list of block numbers and hashes associated with + // that candidate + assert_eq!( + result.len(), + block_numbers.len(), + "Expected the same number of results as inserted block numbers" + ); + + for (&block_number, &block_hash) in block_numbers.iter().zip(&block_hashes) { + assert!( + result.contains(&(block_number, block_hash)), + "Expected to find ({}, {}) in the result", + block_number, + block_hash + ); + } +} + +#[test] +fn inclusions_duplicate_insertion_same_height_and_block() { + let mut inclusions = Inclusions::new(); + + // Insert a candidate + let candidate1 = make_candidate_receipt(get_magic_candidate_hash()).hash(); + let block_number = 0; + let block_hash = get_block_number_hash(block_number); + + // Insert the candidate once + inclusions.insert(candidate1, block_number, block_hash); + + // Insert the same candidate again at the same height and block + inclusions.insert(candidate1, block_number, block_hash); + + // Check inclusions_inner + assert!( + inclusions.inclusions_inner.contains_key(&candidate1), + "Expected candidate1 to be present in inclusions_inner" + ); + let inner_map = inclusions.inclusions_inner.get(&candidate1).unwrap(); + assert!( + inner_map.contains_key(&block_number), + "Expected block_number to be present for the candidate1 in inclusions_inner" + ); + let hash_set = inner_map.get(&block_number).unwrap(); + assert_eq!( + hash_set.len(), + 1, + "Expected only one block_hash for the block_number in inclusions_inner" + ); + assert!( + hash_set.contains(&block_hash), + "Expected block_hash to be present for the block_number in inclusions_inner" + ); + + // Check candidates_by_block_number + assert!( + inclusions.candidates_by_block_number.contains_key(&block_number), + "Expected block_number to be present in candidates_by_block_number" + ); + let candidate_set = inclusions.candidates_by_block_number.get(&block_number).unwrap(); + assert_eq!( + candidate_set.len(), + 1, + "Expected only one candidate for the block_number in candidates_by_block_number" + ); + assert!( + candidate_set.contains(&candidate1), + "Expected candidate1 to be present for the block_number in candidates_by_block_number" + ); +} + +#[test] +fn test_duplicate_insertion_same_height_different_blocks() { + let mut inclusions = Inclusions::new(); + + // Insert a candidate + let candidate1 = make_candidate_receipt(get_magic_candidate_hash()).hash(); + let block_number = 0; + let block_hash1 = BlakeTwo256::hash(&"b1".encode()); + let block_hash2 = BlakeTwo256::hash(&"b2".encode()); // Different block hash for the same height + inclusions.insert(candidate1, block_number, block_hash1); + inclusions.insert(candidate1, block_number, block_hash2); + + // Check inclusions_inner + assert!( + inclusions.inclusions_inner.contains_key(&candidate1), + "Expected candidate1 to be present in inclusions_inner" + ); + let inner_map = inclusions.inclusions_inner.get(&candidate1).unwrap(); + assert!( + inner_map.contains_key(&block_number), + "Expected block_number to be present for the candidate1 in inclusions_inner" + ); + let hash_set = inner_map.get(&block_number).unwrap(); + assert_eq!( + hash_set.len(), + 2, + "Expected two block_hashes for the block_number in inclusions_inner" + ); + assert!( + hash_set.contains(&block_hash1), + "Expected block_hash1 to be present for the block_number in inclusions_inner" + ); + assert!( + hash_set.contains(&block_hash2), + "Expected block_hash2 to be present for the block_number in inclusions_inner" + ); + + // Check candidates_by_block_number + assert!( + inclusions.candidates_by_block_number.contains_key(&block_number), + "Expected block_number to be present in candidates_by_block_number" + ); + let candidate_set = inclusions.candidates_by_block_number.get(&block_number).unwrap(); + assert_eq!( + candidate_set.len(), + 1, + "Expected only one candidate for the block_number in candidates_by_block_number" + ); + assert!( + candidate_set.contains(&candidate1), + "Expected candidate1 to be present for the block_number in candidates_by_block_number" + ); +} + +// ----- Inclusions removal tests ----- +// inclusions_removal_null_case +// +// inclusions_removal_one_candidate_one_height_one_branch +// +// inclusions_removal_one_candidate_one_height_multi_branch +// inclusions_removal_one_candidate_multi_height_one_branch +// inclusions_removal_multi_candidate_one_height_one_branch +// +// inclusions_removal_multi_candidate_multi_height_one_branch +// inclusions_removal_one_candidate_multi_height_multi_branch +// inclusions_removal_multi_candidate_one_height_multi_branch +// +// inclusions_removal_multi_candidate_multi_height_multi_branch +#[test] +fn inclusions_removal_null_case() { + let mut inclusions = Inclusions::new(); + let height = 5; + + // Ensure both maps are empty before the operation + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); + + inclusions.remove_up_to_height(&height); + + // Ensure both maps remain empty after the operation + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} + +#[test] +fn inclusions_removal_one_candidate_one_height_one_branch() { + let mut inclusions = Inclusions::new(); + + let candidate1 = make_candidate_receipt(BlakeTwo256::hash(&"c1".encode())).hash(); + + // B 0 + // C1 0 + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 0)); + + // No prune case + inclusions.remove_up_to_height(&0); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.inclusions_inner.len() == 1); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 1 + inclusions.remove_up_to_height(&1); + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} + +#[test] +fn inclusions_removal_one_candidate_one_height_multi_branch() { + let mut inclusions = Inclusions::new(); + + let candidate1 = make_candidate_receipt(BlakeTwo256::hash(&"c1".encode())).hash(); + + // B 0 + // C1 0&1 + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 1)); + + // No prune case + inclusions.remove_up_to_height(&0); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.inclusions_inner.len() == 1); + assert!(inclusions.inclusions_inner.get(&candidate1).unwrap().get(&0).unwrap().len() == 2); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 1 + inclusions.remove_up_to_height(&1); + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} + +#[test] +fn inclusions_removal_one_candidate_multi_height_one_branch() { + let mut inclusions = Inclusions::new(); + + let candidate1 = make_candidate_receipt(BlakeTwo256::hash(&"c1".encode())).hash(); + + // B 0 1 2 3 4 + // C1 0 0 + inclusions.insert(candidate1, 1, get_relay_block_hash(1, 0)); + inclusions.insert(candidate1, 3, get_relay_block_hash(3, 0)); + + // No prune case + inclusions.remove_up_to_height(&1); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.inclusions_inner.len() == 1); + assert!(inclusions.candidates_by_block_number.len() == 2); + + // Prune case up to height 2 + inclusions.remove_up_to_height(&2); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.inclusions_inner.len() == 1); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 3 + inclusions.remove_up_to_height(&3); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.inclusions_inner.len() == 1); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 20 (overshot) + inclusions.remove_up_to_height(&20); + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} + +#[test] +fn inclusions_removal_multi_candidate_one_height_one_branch() { + let mut inclusions = Inclusions::new(); + + let candidate1 = make_candidate_receipt(BlakeTwo256::hash(&"c1".encode())).hash(); + let candidate2 = make_candidate_receipt(BlakeTwo256::hash(&"c2".encode())).hash(); + let candidate3 = make_candidate_receipt(BlakeTwo256::hash(&"c3".encode())).hash(); + + // B 0 + // C1 0 + // C2 0 + // C3 0 + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate2, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate3, 0, get_relay_block_hash(0, 0)); + + // No prune case + inclusions.remove_up_to_height(&0); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.contains(&candidate2), "Expected candidate2 to remain"); + assert!(inclusions.contains(&candidate3), "Expected candidate3 to remain"); + assert!(inclusions.inclusions_inner.len() == 3); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 1 + inclusions.remove_up_to_height(&1); + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} + +#[test] +fn inclusions_removal_multi_candidate_multi_height_one_branch() { + let mut inclusions = Inclusions::new(); + + let candidate1 = make_candidate_receipt(BlakeTwo256::hash(&"c1".encode())).hash(); + let candidate2 = make_candidate_receipt(BlakeTwo256::hash(&"c2".encode())).hash(); + let candidate3 = make_candidate_receipt(BlakeTwo256::hash(&"c3".encode())).hash(); + + // B 0 1 2 3 + // C1 0 0 + // C2 0 + // C3 0 + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate1, 2, get_relay_block_hash(2, 0)); + inclusions.insert(candidate2, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate3, 2, get_relay_block_hash(2, 0)); + + // No prune case + inclusions.remove_up_to_height(&0); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.contains(&candidate2), "Expected candidate2 to remain"); + assert!(inclusions.contains(&candidate3), "Expected candidate3 to remain"); + assert!(inclusions.inclusions_inner.len() == 3); + assert!(inclusions.candidates_by_block_number.len() == 2); + + // Prune case up to height 1 + inclusions.remove_up_to_height(&1); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(!inclusions.contains(&candidate2), "Expected candidate2 to be removed"); + assert!(inclusions.contains(&candidate3), "Expected candidate3 to remain"); + assert!(inclusions.inclusions_inner.len() == 2); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 2 + inclusions.remove_up_to_height(&2); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(!inclusions.contains(&candidate2), "Expected candidate2 to be removed"); + assert!(inclusions.contains(&candidate3), "Expected candidate3 to remain"); + assert!(inclusions.inclusions_inner.len() == 2); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 3 + inclusions.remove_up_to_height(&3); + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} + +#[test] +fn inclusions_removal_one_candidate_multi_height_multi_branch() { + let mut inclusions = Inclusions::new(); + + let candidate1 = make_candidate_receipt(BlakeTwo256::hash(&"c1".encode())).hash(); + + // B 0 1 2 + // C1 0 0&1 1 + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate1, 1, get_relay_block_hash(1, 0)); + inclusions.insert(candidate1, 1, get_relay_block_hash(1, 1)); + inclusions.insert(candidate1, 2, get_relay_block_hash(2, 1)); + + // No prune case + inclusions.remove_up_to_height(&0); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.inclusions_inner.len() == 1); + assert!(inclusions.candidates_by_block_number.len() == 3); + + // Prune case up to height 1 + inclusions.remove_up_to_height(&1); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.inclusions_inner.len() == 1); + assert!(inclusions.candidates_by_block_number.len() == 2); + + // Prune case up to height 2 + inclusions.remove_up_to_height(&2); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.inclusions_inner.len() == 1); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 3 + inclusions.remove_up_to_height(&3); + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} + +#[test] +fn inclusions_removal_multi_candidate_one_height_multi_branch() { + let mut inclusions = Inclusions::new(); + + let candidate1 = make_candidate_receipt(BlakeTwo256::hash(&"c1".encode())).hash(); + let candidate2 = make_candidate_receipt(BlakeTwo256::hash(&"c2".encode())).hash(); + let candidate3 = make_candidate_receipt(BlakeTwo256::hash(&"c3".encode())).hash(); + + // B 0 + // C1 0 + // C2 0&1 + // C3 1 + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate2, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate2, 0, get_relay_block_hash(0, 1)); + inclusions.insert(candidate3, 0, get_relay_block_hash(0, 1)); + + // No prune case + inclusions.remove_up_to_height(&0); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.contains(&candidate2), "Expected candidate2 to remain"); + assert!(inclusions.contains(&candidate3), "Expected candidate3 to remain"); + assert!(inclusions.inclusions_inner.len() == 3); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 1 + inclusions.remove_up_to_height(&1); + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} + +#[test] +fn inclusions_removal_multi_candidate_multi_height_multi_branch() { + let mut inclusions = Inclusions::new(); + + let candidate1 = make_candidate_receipt(BlakeTwo256::hash(&"c1".encode())).hash(); + let candidate2 = make_candidate_receipt(BlakeTwo256::hash(&"c2".encode())).hash(); + let candidate3 = make_candidate_receipt(BlakeTwo256::hash(&"c3".encode())).hash(); + let candidate4 = make_candidate_receipt(BlakeTwo256::hash(&"c4".encode())).hash(); + + // B 0 1 2 + // C1 0&1 0 0 //shouldn't get pruned as long as one of the forks need it + // C2 1 1 + // C3 0 1 + // C4 0&1 + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 1)); + inclusions.insert(candidate1, 1, get_relay_block_hash(1, 0)); + inclusions.insert(candidate1, 2, get_relay_block_hash(2, 0)); + inclusions.insert(candidate2, 1, get_relay_block_hash(1, 1)); + inclusions.insert(candidate2, 2, get_relay_block_hash(2, 1)); + inclusions.insert(candidate3, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate3, 1, get_relay_block_hash(1, 1)); + inclusions.insert(candidate4, 1, get_relay_block_hash(1, 0)); + inclusions.insert(candidate4, 1, get_relay_block_hash(1, 1)); + + // No prune case + inclusions.remove_up_to_height(&0); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.contains(&candidate2), "Expected candidate2 to remain"); + assert!(inclusions.contains(&candidate3), "Expected candidate3 to remain"); + assert!(inclusions.contains(&candidate4), "Expected candidate4 to remain"); + assert!(inclusions.inclusions_inner.len() == 4); + assert!(inclusions.candidates_by_block_number.len() == 3); + + // Prune case up to height 1 + inclusions.remove_up_to_height(&1); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.contains(&candidate2), "Expected candidate2 to remain"); + assert!(inclusions.contains(&candidate3), "Expected candidate3 to remain"); + assert!(inclusions.contains(&candidate4), "Expected candidate4 to remain"); + assert!(inclusions.inclusions_inner.len() == 4); + assert!(inclusions.candidates_by_block_number.len() == 2); + + // Prune case up to height 2 + inclusions.remove_up_to_height(&2); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.contains(&candidate2), "Expected candidate2 to remain"); + assert!(!inclusions.contains(&candidate3), "Expected candidate3 to be removed"); + assert!(!inclusions.contains(&candidate4), "Expected candidate4 to be removed"); + assert!(inclusions.inclusions_inner.len() == 2); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 3 + inclusions.remove_up_to_height(&3); + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} + +#[test] +fn inclusions_removal_multi_candidate_multi_height_multi_branch_multi_height_prune() { + let mut inclusions = Inclusions::new(); + + let candidate1 = make_candidate_receipt(BlakeTwo256::hash(&"c1".encode())).hash(); + let candidate2 = make_candidate_receipt(BlakeTwo256::hash(&"c2".encode())).hash(); + let candidate3 = make_candidate_receipt(BlakeTwo256::hash(&"c3".encode())).hash(); + let candidate4 = make_candidate_receipt(BlakeTwo256::hash(&"c4".encode())).hash(); + + // B 0 1 2 + // C1 0&1 0 0 + // C2 1 1 + // C3 0 1 + // C4 0&1 + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate1, 0, get_relay_block_hash(0, 1)); + inclusions.insert(candidate1, 1, get_relay_block_hash(1, 0)); + inclusions.insert(candidate1, 2, get_relay_block_hash(2, 0)); + inclusions.insert(candidate2, 1, get_relay_block_hash(1, 1)); + inclusions.insert(candidate2, 2, get_relay_block_hash(2, 1)); + inclusions.insert(candidate3, 0, get_relay_block_hash(0, 0)); + inclusions.insert(candidate3, 1, get_relay_block_hash(1, 1)); + inclusions.insert(candidate4, 1, get_relay_block_hash(1, 0)); + inclusions.insert(candidate4, 1, get_relay_block_hash(1, 1)); + + // Prune case up to height 2 + inclusions.remove_up_to_height(&2); + assert!(inclusions.contains(&candidate1), "Expected candidate1 to remain"); + assert!(inclusions.contains(&candidate2), "Expected candidate2 to remain"); + assert!(!inclusions.contains(&candidate3), "Expected candidate3 to be removed"); + assert!(!inclusions.contains(&candidate4), "Expected candidate4 to be removed"); + assert!(inclusions.inclusions_inner.len() == 2); + assert!(inclusions.candidates_by_block_number.len() == 1); + + // Prune case up to height 20 + inclusions.remove_up_to_height(&20); + assert!(inclusions.inclusions_inner.is_empty(), "Expected inclusions_inner to be empty"); + assert!( + inclusions.candidates_by_block_number.is_empty(), + "Expected candidates_by_block_number to be empty" + ); +} diff --git a/polkadot/node/core/pvf-checker/src/tests.rs b/polkadot/node/core/pvf-checker/src/tests.rs index bd8817000aeb..4ac6e5eba31d 100644 --- a/polkadot/node/core/pvf-checker/src/tests.rs +++ b/polkadot/node/core/pvf-checker/src/tests.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use ::test_helpers::{dummy_digest, dummy_hash}; +use ::test_helpers::{dummy_digest, dummy_hash, validator_pubkeys}; use futures::{channel::oneshot, future::BoxFuture, prelude::*}; use polkadot_node_subsystem::{ messages::{ @@ -94,10 +94,6 @@ struct TestState { const OUR_VALIDATOR: Sr25519Keyring = Sr25519Keyring::Alice; -fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec { - val_ids.iter().map(|v| v.public().into()).collect() -} - impl TestState { fn new() -> Self { // Initialize the default session 1. No validators are present there. diff --git a/polkadot/node/core/pvf/Cargo.toml b/polkadot/node/core/pvf/Cargo.toml index ad9120295d22..478d1952d9d9 100644 --- a/polkadot/node/core/pvf/Cargo.toml +++ b/polkadot/node/core/pvf/Cargo.toml @@ -6,11 +6,6 @@ authors.workspace = true edition.workspace = true license.workspace = true -[[bin]] -name = "puppet_worker" -path = "bin/puppet_worker.rs" -required-features = ["test-utils"] - [dependencies] always-assert = "0.1" futures = "0.3.21" @@ -35,7 +30,6 @@ polkadot-primitives = { path = "../../../primitives" } sp-core = { path = "../../../../substrate/primitives/core" } sp-wasm-interface = { path = "../../../../substrate/primitives/wasm-interface" } sp-maybe-compressed-blob = { path = "../../../../substrate/primitives/maybe-compressed-blob" } -sp-tracing = { path = "../../../../substrate/primitives/tracing", optional = true } polkadot-node-core-pvf-prepare-worker = { path = "prepare-worker", optional = true } polkadot-node-core-pvf-execute-worker = { path = "execute-worker", optional = true } @@ -56,9 +50,7 @@ halt = { package = "test-parachain-halt", path = "../../../parachain/test-parach ci-only-tests = [] jemalloc-allocator = [ "polkadot-node-core-pvf-common/jemalloc-allocator" ] # This feature is used to export test code to other crates without putting it in the production build. -# This is also used by the `puppet_worker` binary. test-utils = [ "polkadot-node-core-pvf-execute-worker", "polkadot-node-core-pvf-prepare-worker", - "sp-tracing", ] diff --git a/polkadot/node/core/pvf/common/src/worker/mod.rs b/polkadot/node/core/pvf/common/src/worker/mod.rs index 40e540bb3f7e..bcdf882f300c 100644 --- a/polkadot/node/core/pvf/common/src/worker/mod.rs +++ b/polkadot/node/core/pvf/common/src/worker/mod.rs @@ -33,7 +33,7 @@ use tokio::{io, net::UnixStream, runtime::Runtime}; /// spawning the desired worker. #[macro_export] macro_rules! decl_worker_main { - ($expected_command:expr, $entrypoint:expr, $worker_version:expr) => { + ($expected_command:expr, $entrypoint:expr, $worker_version:expr $(,)*) => { fn print_help(expected_command: &str) { println!("{} {}", expected_command, $worker_version); println!(); @@ -60,6 +60,10 @@ macro_rules! decl_worker_main { println!("{}", $worker_version); return }, + "test-sleep" => { + std::thread::sleep(std::time::Duration::from_secs(5)); + return + }, subcommand => { // Must be passed for compatibility with the single-binary test workers. if subcommand != $expected_command { diff --git a/polkadot/node/core/pvf/src/lib.rs b/polkadot/node/core/pvf/src/lib.rs index c3a7a4613139..0e4f2444adf7 100644 --- a/polkadot/node/core/pvf/src/lib.rs +++ b/polkadot/node/core/pvf/src/lib.rs @@ -100,10 +100,6 @@ mod worker_intf; #[cfg(feature = "test-utils")] pub mod testing; -// Used by `decl_puppet_worker_main!`. -#[cfg(feature = "test-utils")] -pub use sp_tracing; - pub use error::{InvalidCandidate, ValidationError}; pub use host::{start, Config, ValidationHost, EXECUTE_BINARY_NAME, PREPARE_BINARY_NAME}; pub use metrics::Metrics; @@ -117,11 +113,5 @@ pub use polkadot_node_core_pvf_common::{ pvf::PvfPrepData, }; -// Re-export worker entrypoints. -#[cfg(feature = "test-utils")] -pub use polkadot_node_core_pvf_execute_worker::worker_entrypoint as execute_worker_entrypoint; -#[cfg(feature = "test-utils")] -pub use polkadot_node_core_pvf_prepare_worker::worker_entrypoint as prepare_worker_entrypoint; - /// The log target for this crate. pub const LOG_TARGET: &str = "parachain::pvf"; diff --git a/polkadot/node/core/pvf/src/testing.rs b/polkadot/node/core/pvf/src/testing.rs index 980a28c01566..4301afc3cc7e 100644 --- a/polkadot/node/core/pvf/src/testing.rs +++ b/polkadot/node/core/pvf/src/testing.rs @@ -47,45 +47,3 @@ pub fn validate_candidate( Ok(result) } - -/// Use this macro to declare a `fn main() {}` that will check the arguments and dispatch them to -/// the appropriate worker, making the executable that can be used for spawning workers. -#[macro_export] -macro_rules! decl_puppet_worker_main { - () => { - fn main() { - $crate::sp_tracing::try_init_simple(); - - let args = std::env::args().collect::>(); - if args.len() == 1 { - panic!("wrong number of arguments"); - } - - let entrypoint = match args[1].as_ref() { - "exit" => { - std::process::exit(1); - }, - "sleep" => { - std::thread::sleep(std::time::Duration::from_secs(5)); - return - }, - "prepare-worker" => $crate::prepare_worker_entrypoint, - "execute-worker" => $crate::execute_worker_entrypoint, - other => panic!("unknown subcommand: {}", other), - }; - - let mut node_version = None; - let mut socket_path: &str = ""; - - for i in (2..args.len()).step_by(2) { - match args[i].as_ref() { - "--socket-path" => socket_path = args[i + 1].as_str(), - "--node-impl-version" => node_version = Some(args[i + 1].as_str()), - arg => panic!("Unexpected argument found: {}", arg), - } - } - - entrypoint(&socket_path, node_version, None); - } - }; -} diff --git a/polkadot/node/core/pvf/tests/README.md b/polkadot/node/core/pvf/tests/README.md new file mode 100644 index 000000000000..27385e190250 --- /dev/null +++ b/polkadot/node/core/pvf/tests/README.md @@ -0,0 +1,9 @@ +# PVF host integration tests + +## Testing + +Before running these tests, make sure the worker binaries are built first. This can be done with: + +```sh +cargo build --bin polkadot-execute-worker --bin polkadot-prepare-worker +``` diff --git a/polkadot/node/core/pvf/tests/it/main.rs b/polkadot/node/core/pvf/tests/it/main.rs index 12d87ee44262..dc8f00098ec5 100644 --- a/polkadot/node/core/pvf/tests/it/main.rs +++ b/polkadot/node/core/pvf/tests/it/main.rs @@ -33,7 +33,6 @@ use tokio::sync::Mutex; mod adder; mod worker_common; -const PUPPET_EXE: &str = env!("CARGO_BIN_EXE_puppet_worker"); const TEST_EXECUTION_TIMEOUT: Duration = Duration::from_secs(3); const TEST_PREPARATION_TIMEOUT: Duration = Duration::from_secs(3); @@ -51,10 +50,20 @@ impl TestHost { where F: FnOnce(&mut Config), { + let mut workers_path = std::env::current_exe().unwrap(); + workers_path.pop(); + workers_path.pop(); + let mut prepare_worker_path = workers_path.clone(); + prepare_worker_path.push("polkadot-prepare-worker"); + let mut execute_worker_path = workers_path.clone(); + execute_worker_path.push("polkadot-execute-worker"); let cache_dir = tempfile::tempdir().unwrap(); - let program_path = std::path::PathBuf::from(PUPPET_EXE); - let mut config = - Config::new(cache_dir.path().to_owned(), None, program_path.clone(), program_path); + let mut config = Config::new( + cache_dir.path().to_owned(), + None, + prepare_worker_path, + execute_worker_path, + ); f(&mut config); let (host, task) = start(config, Metrics::default()); let _ = tokio::task::spawn(task); diff --git a/polkadot/node/core/pvf/tests/it/worker_common.rs b/polkadot/node/core/pvf/tests/it/worker_common.rs index a3bf552e894a..875ae79af097 100644 --- a/polkadot/node/core/pvf/tests/it/worker_common.rs +++ b/polkadot/node/core/pvf/tests/it/worker_common.rs @@ -14,26 +14,41 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use std::time::Duration; - use polkadot_node_core_pvf::testing::{spawn_with_program_path, SpawnErr}; +use std::time::Duration; -use crate::PUPPET_EXE; +fn worker_path(name: &str) -> std::path::PathBuf { + let mut worker_path = std::env::current_exe().unwrap(); + worker_path.pop(); + worker_path.pop(); + worker_path.push(name); + worker_path +} // Test spawning a program that immediately exits with a failure code. #[tokio::test] async fn spawn_immediate_exit() { - let result = - spawn_with_program_path("integration-test", PUPPET_EXE, &["exit"], Duration::from_secs(2)) - .await; + // There's no explicit `exit` subcommand in the worker; it will panic on an unknown + // subcommand anyway + let result = spawn_with_program_path( + "integration-test", + worker_path("polkadot-prepare-worker"), + &["exit"], + Duration::from_secs(2), + ) + .await; assert!(matches!(result, Err(SpawnErr::AcceptTimeout))); } #[tokio::test] async fn spawn_timeout() { - let result = - spawn_with_program_path("integration-test", PUPPET_EXE, &["sleep"], Duration::from_secs(2)) - .await; + let result = spawn_with_program_path( + "integration-test", + worker_path("polkadot-execute-worker"), + &["test-sleep"], + Duration::from_secs(2), + ) + .await; assert!(matches!(result, Err(SpawnErr::AcceptTimeout))); } @@ -41,7 +56,7 @@ async fn spawn_timeout() { async fn should_connect() { let _ = spawn_with_program_path( "integration-test", - PUPPET_EXE, + worker_path("polkadot-prepare-worker"), &["prepare-worker"], Duration::from_secs(2), ) diff --git a/polkadot/node/gum/proc-macro/Cargo.toml b/polkadot/node/gum/proc-macro/Cargo.toml index ec6b817476a9..0d6cee2ccf0a 100644 --- a/polkadot/node/gum/proc-macro/Cargo.toml +++ b/polkadot/node/gum/proc-macro/Cargo.toml @@ -13,7 +13,7 @@ targets = ["x86_64-unknown-linux-gnu"] proc-macro = true [dependencies] -syn = { version = "2.0.31", features = ["full", "extra-traits"] } +syn = { version = "2.0.33", features = ["full", "extra-traits"] } quote = "1.0.28" proc-macro2 = "1.0.56" proc-macro-crate = "1.1.3" diff --git a/polkadot/node/malus/Cargo.toml b/polkadot/node/malus/Cargo.toml index 56053b8814a7..08d203281cff 100644 --- a/polkadot/node/malus/Cargo.toml +++ b/polkadot/node/malus/Cargo.toml @@ -40,7 +40,7 @@ assert_matches = "1.5" async-trait = "0.1.57" sp-keystore = { path = "../../../substrate/primitives/keystore" } sp-core = { path = "../../../substrate/primitives/core" } -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } futures = "0.3.21" futures-timer = "3.0.2" gum = { package = "tracing-gum", path = "../gum" } diff --git a/polkadot/node/malus/src/interceptor.rs b/polkadot/node/malus/src/interceptor.rs index cbf39bccd160..04ee0905deeb 100644 --- a/polkadot/node/malus/src/interceptor.rs +++ b/polkadot/node/malus/src/interceptor.rs @@ -47,12 +47,20 @@ where Some(msg) } - /// Modify outgoing messages. + /// Specifies if we need to replace some outgoing message with another (potentially empty) + /// message + fn need_intercept_outgoing( + &self, + _msg: &::OutgoingMessages, + ) -> bool { + false + } + /// Send modified message instead of the original one fn intercept_outgoing( &self, - msg: ::OutgoingMessages, + _msg: &::OutgoingMessages, ) -> Option<::OutgoingMessages> { - Some(msg) + None } } @@ -66,7 +74,7 @@ pub struct InterceptedSender { #[async_trait::async_trait] impl overseer::SubsystemSender for InterceptedSender where - OutgoingMessage: overseer::AssociateOutgoing + Send + 'static, + OutgoingMessage: overseer::AssociateOutgoing + Send + 'static + TryFrom, Sender: overseer::SubsystemSender + overseer::SubsystemSender< < @@ -78,17 +86,48 @@ where < >::Message as overseer::AssociateOutgoing >::OutgoingMessages: - From, + From + Send + Sync, + >::Error: std::fmt::Debug, { async fn send_message(&mut self, msg: OutgoingMessage) { let msg = < <>::Message as overseer::AssociateOutgoing >::OutgoingMessages as From>::from(msg); - if let Some(msg) = self.message_filter.intercept_outgoing(msg) { + if self.message_filter.need_intercept_outgoing(&msg) { + if let Some(msg) = self.message_filter.intercept_outgoing(&msg) { + self.inner.send_message(msg).await; + } + } + else { self.inner.send_message(msg).await; } } + fn try_send_message(&mut self, msg: OutgoingMessage) -> Result<(), TrySendError> { + let msg = < + <>::Message as overseer::AssociateOutgoing + >::OutgoingMessages as From>::from(msg); + if self.message_filter.need_intercept_outgoing(&msg) { + if let Some(real_msg) = self.message_filter.intercept_outgoing(&msg) { + let orig_msg : OutgoingMessage = msg.into().try_into().expect("must be able to recover the original message"); + self.inner.try_send_message(real_msg).map_err(|e| { + match e { + TrySendError::Full(_) => TrySendError::Full(orig_msg), + TrySendError::Closed(_) => TrySendError::Closed(orig_msg), + } + }) + } + else { + // No message to send after intercepting + Ok(()) + } + } + else { + let orig_msg : OutgoingMessage = msg.into().try_into().expect("must be able to recover the original message"); + self.inner.try_send_message(orig_msg) + } + } + async fn send_messages(&mut self, msgs: T) where T: IntoIterator + Send, @@ -101,9 +140,14 @@ where fn send_unbounded_message(&mut self, msg: OutgoingMessage) { let msg = < - <>::Message as overseer::AssociateOutgoing - >::OutgoingMessages as From>::from(msg); - if let Some(msg) = self.message_filter.intercept_outgoing(msg) { + <>::Message as overseer::AssociateOutgoing + >::OutgoingMessages as From>::from(msg); + if self.message_filter.need_intercept_outgoing(&msg) { + if let Some(msg) = self.message_filter.intercept_outgoing(&msg) { + self.inner.send_unbounded_message(msg); + } + } + else { self.inner.send_unbounded_message(msg); } } diff --git a/polkadot/node/malus/src/variants/common.rs b/polkadot/node/malus/src/variants/common.rs index bc5f6f92aedb..365b2f16ac21 100644 --- a/polkadot/node/malus/src/variants/common.rs +++ b/polkadot/node/malus/src/variants/common.rs @@ -498,11 +498,4 @@ where msg => Some(msg), } } - - fn intercept_outgoing( - &self, - msg: overseer::CandidateValidationOutgoingMessages, - ) -> Option { - Some(msg) - } } diff --git a/polkadot/node/metrics/Cargo.toml b/polkadot/node/metrics/Cargo.toml index d497fa7607a1..e13ae63199ff 100644 --- a/polkadot/node/metrics/Cargo.toml +++ b/polkadot/node/metrics/Cargo.toml @@ -11,8 +11,7 @@ futures = "0.3.21" futures-timer = "3.0.2" gum = { package = "tracing-gum", path = "../gum" } -metered = { package = "prioritized-metered-channel", version = "0.2.0" } - +metered = { package = "prioritized-metered-channel", version = "0.5.1", default-features = false, features=["futures_channel"] } # Both `sc-service` and `sc-cli` are required by runtime metrics `logger_hook()`. sc-service = { path = "../../../substrate/client/service" } sc-cli = { path = "../../../substrate/client/cli" } diff --git a/polkadot/node/network/bridge/src/metrics.rs b/polkadot/node/network/bridge/src/metrics.rs index bb90daad5676..083a2a71aa0f 100644 --- a/polkadot/node/network/bridge/src/metrics.rs +++ b/polkadot/node/network/bridge/src/metrics.rs @@ -105,9 +105,27 @@ impl Metrics { pub fn on_report_event(&self) { if let Some(metrics) = self.0.as_ref() { + self.on_message("report_peer"); metrics.report_events.inc() } } + + pub fn on_message(&self, message_type: &'static str) { + if let Some(metrics) = self.0.as_ref() { + metrics.messages_sent.with_label_values(&[message_type]).inc() + } + } + + pub fn on_delayed_rx_queue(&self, queue_size: usize) { + if let Some(metrics) = self.0.as_ref() { + metrics.rx_delayed_processing.observe(queue_size as f64); + } + } + pub fn time_delayed_rx_events( + &self, + ) -> Option { + self.0.as_ref().map(|metrics| metrics.rx_delayed_processing_time.start_timer()) + } } #[derive(Clone)] @@ -123,6 +141,13 @@ pub(crate) struct MetricsInner { bytes_received: prometheus::CounterVec, bytes_sent: prometheus::CounterVec, + + messages_sent: prometheus::CounterVec, + // The reason why a `Histogram` is used to track a queue size is that + // we need not only an average size of the queue (that will be 0 normally), but + // we also need a dynamics for this queue size in case of messages delays. + rx_delayed_processing: prometheus::Histogram, + rx_delayed_processing_time: prometheus::Histogram, } impl metrics::Metrics for Metrics { @@ -217,6 +242,34 @@ impl metrics::Metrics for Metrics { )?, registry, )?, + messages_sent: prometheus::register( + prometheus::CounterVec::new( + prometheus::Opts::new( + "polkadot_parachain_messages_sent_total", + "The number of messages sent via network bridge", + ), + &["type"] + )?, + registry, + )?, + rx_delayed_processing: prometheus::register( + prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "polkadot_parachain_network_bridge_rx_delayed", + "Number of events being delayed while broadcasting from the network bridge", + ).buckets(vec![0.0, 1.0, 2.0, 8.0, 16.0]), + )?, + registry, + )?, + rx_delayed_processing_time: prometheus::register( + prometheus::Histogram::with_opts( + prometheus::HistogramOpts::new( + "polkadot_parachain_network_bridge_rx_delayed_time", + "Time spent for waiting of the delayed events", + ), + )?, + registry, + )?, }; Ok(Metrics(Some(metrics))) diff --git a/polkadot/node/network/bridge/src/network.rs b/polkadot/node/network/bridge/src/network.rs index 4f21212dcb64..823e1254612f 100644 --- a/polkadot/node/network/bridge/src/network.rs +++ b/polkadot/node/network/bridge/src/network.rs @@ -61,6 +61,7 @@ pub(crate) fn send_message( let message = { let encoded = message.encode(); metrics.on_notification_sent(peer_set, version, encoded.len(), peers.len()); + metrics.on_message(std::any::type_name::()); encoded }; diff --git a/polkadot/node/network/bridge/src/rx/mod.rs b/polkadot/node/network/bridge/src/rx/mod.rs index 51d248ca2d49..e1125ebc904d 100644 --- a/polkadot/node/network/bridge/src/rx/mod.rs +++ b/polkadot/node/network/bridge/src/rx/mod.rs @@ -20,7 +20,10 @@ use super::*; use always_assert::never; use bytes::Bytes; -use futures::stream::BoxStream; +use futures::{ + future::BoxFuture, + stream::{BoxStream, FuturesUnordered, StreamExt}, +}; use parity_scale_codec::{Decode, DecodeAll}; use sc_network::Event as NetworkEvent; @@ -244,6 +247,7 @@ where NetworkBridgeEvent::PeerViewChange(peer, View::default()), ], &mut sender, + &metrics, ) .await; @@ -352,6 +356,7 @@ where dispatch_validation_event_to_all( NetworkBridgeEvent::PeerDisconnected(peer), &mut sender, + &metrics, ) .await, PeerSet::Collation => @@ -490,7 +495,7 @@ where network_service.report_peer(remote, report.into()); } - dispatch_validation_events_to_all(events, &mut sender).await; + dispatch_validation_events_to_all(events, &mut sender, &metrics).await; } if !c_messages.is_empty() { @@ -992,8 +997,9 @@ fn send_collation_message_vstaging( async fn dispatch_validation_event_to_all( event: NetworkBridgeEvent, ctx: &mut impl overseer::NetworkBridgeRxSenderTrait, + metrics: &Metrics, ) { - dispatch_validation_events_to_all(std::iter::once(event), ctx).await + dispatch_validation_events_to_all(std::iter::once(event), ctx, metrics).await } async fn dispatch_collation_event_to_all( @@ -1038,20 +1044,65 @@ fn dispatch_collation_event_to_all_unbounded( } } +fn send_or_queue_validation_event( + event: E, + sender: &mut Sender, + delayed_queue: &FuturesUnordered>, +) where + E: Send + 'static, + Sender: overseer::NetworkBridgeRxSenderTrait + overseer::SubsystemSender, +{ + match sender.try_send_message(event) { + Ok(()) => {}, + Err(overseer::TrySendError::Full(event)) => { + let mut sender = sender.clone(); + delayed_queue.push(Box::pin(async move { + sender.send_message(event).await; + })); + }, + Err(overseer::TrySendError::Closed(_)) => { + panic!( + "NetworkBridgeRxSender is closed when trying to send event of type: {}", + std::any::type_name::() + ); + }, + } +} + async fn dispatch_validation_events_to_all( events: I, sender: &mut impl overseer::NetworkBridgeRxSenderTrait, + metrics: &Metrics, ) where I: IntoIterator>, I::IntoIter: Send, { + let delayed_messages: FuturesUnordered> = FuturesUnordered::new(); + + // Fast path for sending events to subsystems, if any subsystem's queue is full, we hold + // the slow path future in the `delayed_messages` queue. for event in events { - sender - .send_messages(event.focus().map(StatementDistributionMessage::from)) - .await; - sender.send_messages(event.focus().map(BitfieldDistributionMessage::from)).await; - sender.send_messages(event.focus().map(ApprovalDistributionMessage::from)).await; - sender.send_messages(event.focus().map(GossipSupportMessage::from)).await; + if let Ok(msg) = event.focus().map(StatementDistributionMessage::from) { + send_or_queue_validation_event(msg, sender, &delayed_messages); + } + if let Ok(msg) = event.focus().map(BitfieldDistributionMessage::from) { + send_or_queue_validation_event(msg, sender, &delayed_messages); + } + if let Ok(msg) = event.focus().map(ApprovalDistributionMessage::from) { + send_or_queue_validation_event(msg, sender, &delayed_messages); + } + if let Ok(msg) = event.focus().map(GossipSupportMessage::from) { + send_or_queue_validation_event(msg, sender, &delayed_messages); + } + } + + let delayed_messages_count = delayed_messages.len(); + metrics.on_delayed_rx_queue(delayed_messages_count); + + if delayed_messages_count > 0 { + // Here we wait for all the delayed messages to be sent. + let _timer = metrics.time_delayed_rx_events(); // Dropped after `await` is completed + let _: Vec<()> = delayed_messages.collect().await; } } diff --git a/polkadot/node/network/bridge/src/tx/mod.rs b/polkadot/node/network/bridge/src/tx/mod.rs index 1b386ce1239b..7fa1149593ca 100644 --- a/polkadot/node/network/bridge/src/tx/mod.rs +++ b/polkadot/node/network/bridge/src/tx/mod.rs @@ -33,6 +33,7 @@ use polkadot_node_subsystem::{ /// /// To be passed to [`FullNetworkConfiguration::add_notification_protocol`](). pub use polkadot_node_network_protocol::peer_set::{peer_sets_info, IsAuthority}; +use polkadot_node_network_protocol::request_response::Requests; use sc_network::ReputationChange; use crate::validator_discovery; @@ -290,6 +291,20 @@ where ); for req in reqs { + match req { + Requests::ChunkFetchingV1(_) => metrics.on_message("chunk_fetching_v1"), + Requests::AvailableDataFetchingV1(_) => + metrics.on_message("available_data_fetching_v1"), + Requests::CollationFetchingV1(_) => metrics.on_message("collation_fetching_v1"), + Requests::CollationFetchingVStaging(_) => + metrics.on_message("collation_fetching_vstaging"), + Requests::PoVFetchingV1(_) => metrics.on_message("pov_fetching_v1"), + Requests::DisputeSendingV1(_) => metrics.on_message("dispute_sending_v1"), + Requests::StatementFetchingV1(_) => metrics.on_message("statement_fetching_v1"), + Requests::AttestedCandidateVStaging(_) => + metrics.on_message("attested_candidate_vstaging"), + } + network_service .start_request( &mut authority_discovery_service, diff --git a/polkadot/node/overseer/Cargo.toml b/polkadot/node/overseer/Cargo.toml index 0efd4d4c6ca8..5d41407ef83a 100644 --- a/polkadot/node/overseer/Cargo.toml +++ b/polkadot/node/overseer/Cargo.toml @@ -16,7 +16,7 @@ polkadot-node-primitives = { path = "../primitives" } polkadot-node-subsystem-types = { path = "../subsystem-types" } polkadot-node-metrics = { path = "../metrics" } polkadot-primitives = { path = "../../primitives" } -orchestra = "0.0.5" +orchestra = { version = "0.3.3", default-features = false, features=["futures_channel"] } gum = { package = "tracing-gum", path = "../gum" } schnellru = "0.2.1" sp-core = { path = "../../../substrate/primitives/core" } @@ -24,7 +24,7 @@ async-trait = "0.1.57" tikv-jemalloc-ctl = { version = "0.5.0", optional = true } [dev-dependencies] -metered = { package = "prioritized-metered-channel", version = "0.2.0" } +metered = { package = "prioritized-metered-channel", version = "0.5.1", default-features = false, features=["futures_channel"] } sp-core = { path = "../../../substrate/primitives/core" } futures = { version = "0.3.21", features = ["thread-pool"] } femme = "2.2.1" @@ -36,7 +36,8 @@ node-test-helpers = { package = "polkadot-node-subsystem-test-helpers", path = " tikv-jemalloc-ctl = "0.5.0" [features] -default = [] -expand = [ "orchestra/expand" ] +default = [ "futures_channel" ] dotgraph = [ "orchestra/dotgraph" ] +expand = [ "orchestra/expand" ] +futures_channel = [ "metered/futures_channel", "orchestra/futures_channel" ] jemalloc-allocator = [ "dep:tikv-jemalloc-ctl" ] diff --git a/polkadot/node/overseer/src/lib.rs b/polkadot/node/overseer/src/lib.rs index 7337f1e6be7c..84d5d19c3b93 100644 --- a/polkadot/node/overseer/src/lib.rs +++ b/polkadot/node/overseer/src/lib.rs @@ -107,7 +107,7 @@ pub use orchestra::{ contextbounds, orchestra, subsystem, FromOrchestra, MapSubsystem, MessagePacket, OrchestraError as OverseerError, SignalsReceived, Spawner, Subsystem, SubsystemContext, SubsystemIncomingMessages, SubsystemInstance, SubsystemMeterReadouts, SubsystemMeters, - SubsystemSender, TimeoutExt, ToOrchestra, + SubsystemSender, TimeoutExt, ToOrchestra, TrySendError, }; /// Store 2 days worth of blocks, not accounting for forks, diff --git a/polkadot/node/overseer/src/tests.rs b/polkadot/node/overseer/src/tests.rs index 298783f41805..c17613fb7ea5 100644 --- a/polkadot/node/overseer/src/tests.rs +++ b/polkadot/node/overseer/src/tests.rs @@ -1074,6 +1074,7 @@ fn overseer_all_subsystems_receive_signals_and_messages() { #[test] fn context_holds_onto_message_until_enough_signals_received() { + const CHANNEL_CAPACITY: usize = 64; let (candidate_validation_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); let (candidate_backing_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); let (statement_distribution_bounded_tx, _) = metered::channel(CHANNEL_CAPACITY); diff --git a/polkadot/node/service/Cargo.toml b/polkadot/node/service/Cargo.toml index 03814af19cd2..667f9b86d3e4 100644 --- a/polkadot/node/service/Cargo.toml +++ b/polkadot/node/service/Cargo.toml @@ -78,7 +78,7 @@ futures = "0.3.21" hex-literal = "0.4.1" gum = { package = "tracing-gum", path = "../gum" } serde = { version = "1.0.188", features = ["derive"] } -serde_json = "1.0.96" +serde_json = "1.0.107" thiserror = "1.0.48" kvdb = "0.13.0" kvdb-rocksdb = { version = "0.19.0", optional = true } diff --git a/polkadot/node/service/src/lib.rs b/polkadot/node/service/src/lib.rs index 7f4eadaba7f8..75e853d0ef85 100644 --- a/polkadot/node/service/src/lib.rs +++ b/polkadot/node/service/src/lib.rs @@ -894,6 +894,7 @@ pub fn new_full( import_queue, block_announce_validator_builder: None, warp_sync_params: Some(WarpSyncParams::WithProvider(warp_sync)), + block_relay: None, })?; if config.offchain_worker.enabled { diff --git a/polkadot/node/subsystem-test-helpers/src/lib.rs b/polkadot/node/subsystem-test-helpers/src/lib.rs index fe6b106bf46e..3f92513498c4 100644 --- a/polkadot/node/subsystem-test-helpers/src/lib.rs +++ b/polkadot/node/subsystem-test-helpers/src/lib.rs @@ -20,7 +20,7 @@ use polkadot_node_subsystem::{ messages::AllMessages, overseer, FromOrchestra, OverseerSignal, SpawnGlue, SpawnedSubsystem, - SubsystemError, SubsystemResult, + SubsystemError, SubsystemResult, TrySendError, }; use polkadot_node_subsystem_util::TimeoutExt; @@ -160,6 +160,14 @@ where self.tx.send(msg.into()).await.expect("test overseer no longer live"); } + fn try_send_message( + &mut self, + msg: OutgoingMessage, + ) -> Result<(), TrySendError> { + self.tx.unbounded_send(msg.into()).expect("test overseer no longer live"); + Ok(()) + } + async fn send_messages(&mut self, msgs: I) where I: IntoIterator + Send, diff --git a/polkadot/node/subsystem-types/Cargo.toml b/polkadot/node/subsystem-types/Cargo.toml index f6965cf647c3..a1c00cb0652e 100644 --- a/polkadot/node/subsystem-types/Cargo.toml +++ b/polkadot/node/subsystem-types/Cargo.toml @@ -14,7 +14,7 @@ polkadot-node-primitives = { path = "../primitives" } polkadot-node-network-protocol = { path = "../network/protocol" } polkadot-statement-table = { path = "../../statement-table" } polkadot-node-jaeger = { path = "../jaeger" } -orchestra = "0.0.5" +orchestra = { version = "0.3.3", default-features = false, features=["futures_channel"] } sc-network = { path = "../../../substrate/client/network" } sp-api = { path = "../../../substrate/primitives/api" } sp-consensus-babe = { path = "../../../substrate/primitives/consensus/babe" } diff --git a/polkadot/node/subsystem-util/Cargo.toml b/polkadot/node/subsystem-util/Cargo.toml index 0d5ae7a0e8e6..d9364e2c2c0f 100644 --- a/polkadot/node/subsystem-util/Cargo.toml +++ b/polkadot/node/subsystem-util/Cargo.toml @@ -29,7 +29,7 @@ polkadot-node-network-protocol = { path = "../network/protocol" } polkadot-primitives = { path = "../../primitives" } polkadot-node-primitives = { path = "../primitives" } polkadot-overseer = { path = "../overseer" } -metered = { package = "prioritized-metered-channel", version = "0.2.0" } +metered = { package = "prioritized-metered-channel", version = "0.5.1", default-features = false, features=["futures_channel"] } sp-core = { path = "../../../substrate/primitives/core" } sp-application-crypto = { path = "../../../substrate/primitives/application-crypto" } diff --git a/polkadot/node/test/service/Cargo.toml b/polkadot/node/test/service/Cargo.toml index d730a601d5a7..ce4148b459ee 100644 --- a/polkadot/node/test/service/Cargo.toml +++ b/polkadot/node/test/service/Cargo.toml @@ -58,7 +58,7 @@ substrate-test-client = { path = "../../../../substrate/test-utils/client" } [dev-dependencies] pallet-balances = { path = "../../../../substrate/frame/balances", default-features = false } -serde_json = "1.0.96" +serde_json = "1.0.107" substrate-test-utils = { path = "../../../../substrate/test-utils" } tokio = { version = "1.24.2", features = ["macros"] } diff --git a/polkadot/parachain/test-parachains/adder/collator/Cargo.toml b/polkadot/parachain/test-parachains/adder/collator/Cargo.toml index 7079ab732704..a2ce15c21db2 100644 --- a/polkadot/parachain/test-parachains/adder/collator/Cargo.toml +++ b/polkadot/parachain/test-parachains/adder/collator/Cargo.toml @@ -11,14 +11,9 @@ license.workspace = true name = "adder-collator" path = "src/main.rs" -[[bin]] -name = "adder_collator_puppet_worker" -path = "bin/puppet_worker.rs" -required-features = ["test-utils"] - [dependencies] parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive"] } -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } futures = "0.3.21" futures-timer = "3.0.2" log = "0.4.17" @@ -33,25 +28,17 @@ polkadot-node-subsystem = { path = "../../../../node/subsystem" } sc-cli = { path = "../../../../../substrate/client/cli" } sp-core = { path = "../../../../../substrate/primitives/core" } sc-service = { path = "../../../../../substrate/client/service" } -# This one is tricky. Even though it is not used directly by the collator, we still need it for the -# `puppet_worker` binary, which is required for the integration test. However, this shouldn't be -# a big problem since it is used transitively anyway. -polkadot-node-core-pvf = { path = "../../../../node/core/pvf", features = ["test-utils"], optional = true } [dev-dependencies] polkadot-parachain-primitives = { path = "../../.." } polkadot-test-service = { path = "../../../../node/test/service" } +polkadot-node-core-pvf = { path = "../../../../node/core/pvf", features = ["test-utils"] } substrate-test-utils = { path = "../../../../../substrate/test-utils" } sc-service = { path = "../../../../../substrate/client/service" } sp-keyring = { path = "../../../../../substrate/primitives/keyring" } -# For the puppet worker, depend on ourselves with the test-utils feature. -test-parachain-adder-collator = { path = "", features = ["test-utils"] } tokio = { version = "1.24.2", features = ["macros"] } [features] network-protocol-staging = [ "polkadot-cli/network-protocol-staging" ] -# This feature is used to export test code to other crates without putting it in the production build. -# This is also used by the `puppet_worker` binary. -test-utils = [ "polkadot-node-core-pvf/test-utils" ] diff --git a/polkadot/parachain/test-parachains/adder/collator/bin/puppet_worker.rs b/polkadot/parachain/test-parachains/adder/collator/bin/puppet_worker.rs deleted file mode 100644 index 7f93519d8454..000000000000 --- a/polkadot/parachain/test-parachains/adder/collator/bin/puppet_worker.rs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -polkadot_node_core_pvf::decl_puppet_worker_main!(); diff --git a/polkadot/parachain/test-parachains/adder/collator/tests/integration.rs b/polkadot/parachain/test-parachains/adder/collator/tests/integration.rs index 6b481f961a42..85abf8bf36b9 100644 --- a/polkadot/parachain/test-parachains/adder/collator/tests/integration.rs +++ b/polkadot/parachain/test-parachains/adder/collator/tests/integration.rs @@ -17,8 +17,6 @@ //! Integration test that ensures that we can build and include parachain //! blocks of the adder parachain. -const PUPPET_EXE: &str = env!("CARGO_BIN_EXE_adder_collator_puppet_worker"); - // If this test is failing, make sure to run all tests with the `real-overseer` feature being // enabled. @@ -41,8 +39,12 @@ async fn collating_using_adder_collator() { true, ); + let mut workers_path = std::env::current_exe().unwrap(); + workers_path.pop(); + workers_path.pop(); + // start alice - let alice = polkadot_test_service::run_validator_node(alice_config, Some(PUPPET_EXE.into())); + let alice = polkadot_test_service::run_validator_node(alice_config, Some(workers_path.clone())); let bob_config = polkadot_test_service::node_config( || {}, @@ -53,7 +55,7 @@ async fn collating_using_adder_collator() { ); // start bob - let bob = polkadot_test_service::run_validator_node(bob_config, Some(PUPPET_EXE.into())); + let bob = polkadot_test_service::run_validator_node(bob_config, Some(workers_path)); let collator = test_parachain_adder_collator::Collator::new(); diff --git a/polkadot/parachain/test-parachains/undying/collator/Cargo.toml b/polkadot/parachain/test-parachains/undying/collator/Cargo.toml index 0f1fd60a9000..9a5e580af1b9 100644 --- a/polkadot/parachain/test-parachains/undying/collator/Cargo.toml +++ b/polkadot/parachain/test-parachains/undying/collator/Cargo.toml @@ -11,15 +11,10 @@ publish = false name = "undying-collator" path = "src/main.rs" -[[bin]] -name = "undying_collator_puppet_worker" -path = "bin/puppet_worker.rs" -required-features = ["test-utils"] - [dependencies] parity-scale-codec = { version = "3.6.1", default-features = false, features = ["derive"] } -clap = { version = "4.4.2", features = ["derive"] } -futures = "0.3.19" +clap = { version = "4.4.3", features = ["derive"] } +futures = "0.3.21" futures-timer = "3.0.2" log = "0.4.17" @@ -33,24 +28,14 @@ polkadot-node-subsystem = { path = "../../../../node/subsystem" } sc-cli = { path = "../../../../../substrate/client/cli" } sp-core = { path = "../../../../../substrate/primitives/core" } sc-service = { path = "../../../../../substrate/client/service" } -# This one is tricky. Even though it is not used directly by the collator, we still need it for the -# `puppet_worker` binary, which is required for the integration test. However, this shouldn't be -# a big problem since it is used transitively anyway. -polkadot-node-core-pvf = { path = "../../../../node/core/pvf", features = ["test-utils"], optional = true } [dev-dependencies] polkadot-parachain-primitives = { path = "../../.." } polkadot-test-service = { path = "../../../../node/test/service" } -# For the puppet worker, depend on ourselves with the test-utils feature. -test-parachain-undying-collator = { path = "", features = ["test-utils"] } +polkadot-node-core-pvf = { path = "../../../../node/core/pvf", features = ["test-utils"] } substrate-test-utils = { path = "../../../../../substrate/test-utils" } sc-service = { path = "../../../../../substrate/client/service" } sp-keyring = { path = "../../../../../substrate/primitives/keyring" } tokio = { version = "1.24.2", features = ["macros"] } - -[features] -# This feature is used to export test code to other crates without putting it in the production build. -# This is also used by the `puppet_worker` binary. -test-utils = [ "polkadot-node-core-pvf/test-utils" ] diff --git a/polkadot/parachain/test-parachains/undying/collator/bin/puppet_worker.rs b/polkadot/parachain/test-parachains/undying/collator/bin/puppet_worker.rs deleted file mode 100644 index 7f93519d8454..000000000000 --- a/polkadot/parachain/test-parachains/undying/collator/bin/puppet_worker.rs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -polkadot_node_core_pvf::decl_puppet_worker_main!(); diff --git a/polkadot/parachain/test-parachains/undying/collator/tests/integration.rs b/polkadot/parachain/test-parachains/undying/collator/tests/integration.rs index a98a7ff6eefc..8be535b9bb4c 100644 --- a/polkadot/parachain/test-parachains/undying/collator/tests/integration.rs +++ b/polkadot/parachain/test-parachains/undying/collator/tests/integration.rs @@ -17,8 +17,6 @@ //! Integration test that ensures that we can build and include parachain //! blocks of the `Undying` parachain. -const PUPPET_EXE: &str = env!("CARGO_BIN_EXE_undying_collator_puppet_worker"); - // If this test is failing, make sure to run all tests with the `real-overseer` feature being // enabled. #[tokio::test(flavor = "multi_thread")] @@ -40,8 +38,12 @@ async fn collating_using_undying_collator() { true, ); + let mut workers_path = std::env::current_exe().unwrap(); + workers_path.pop(); + workers_path.pop(); + // start alice - let alice = polkadot_test_service::run_validator_node(alice_config, Some(PUPPET_EXE.into())); + let alice = polkadot_test_service::run_validator_node(alice_config, Some(workers_path.clone())); let bob_config = polkadot_test_service::node_config( || {}, @@ -52,7 +54,7 @@ async fn collating_using_undying_collator() { ); // start bob - let bob = polkadot_test_service::run_validator_node(bob_config, Some(PUPPET_EXE.into())); + let bob = polkadot_test_service::run_validator_node(bob_config, Some(workers_path)); let collator = test_parachain_undying_collator::Collator::new(1_000, 1); diff --git a/polkadot/primitives/test-helpers/src/lib.rs b/polkadot/primitives/test-helpers/src/lib.rs index 8ee5f180d342..d532d6ff57f4 100644 --- a/polkadot/primitives/test-helpers/src/lib.rs +++ b/polkadot/primitives/test-helpers/src/lib.rs @@ -159,6 +159,11 @@ pub fn dummy_pvd(parent_head: HeadData, relay_parent_number: u32) -> PersistedVa } } +/// Creates a meaningless signature +pub fn dummy_signature() -> polkadot_primitives::ValidatorSignature { + sp_core::crypto::UncheckedFrom::unchecked_from([1u8; 64]) +} + /// Create a meaningless candidate, returning its receipt and PVD. pub fn make_candidate( relay_parent_hash: Hash, @@ -244,6 +249,11 @@ pub fn resign_candidate_descriptor_with_collator>( descriptor.signature = signature; } +/// Extracts validators's public keus (`ValidatorId`) from `Sr25519Keyring` +pub fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec { + val_ids.iter().map(|v| v.public().into()).collect() +} + /// Builder for `CandidateReceipt`. pub struct TestCandidateBuilder { pub para_id: ParaId, @@ -298,7 +308,3 @@ impl rand::RngCore for AlwaysZeroRng { Ok(()) } } - -pub fn dummy_signature() -> polkadot_primitives::ValidatorSignature { - sp_core::crypto::UncheckedFrom::unchecked_from([1u8; 64]) -} diff --git a/polkadot/runtime/common/Cargo.toml b/polkadot/runtime/common/Cargo.toml index 83aa70cd308a..17617bf4ada3 100644 --- a/polkadot/runtime/common/Cargo.toml +++ b/polkadot/runtime/common/Cargo.toml @@ -58,13 +58,12 @@ pallet-babe = { path = "../../../substrate/frame/babe" } pallet-treasury = { path = "../../../substrate/frame/treasury" } sp-keystore = { path = "../../../substrate/primitives/keystore" } sp-keyring = { path = "../../../substrate/primitives/keyring" } -serde_json = "1.0.96" +serde_json = "1.0.107" libsecp256k1 = "0.7.0" test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../primitives/test-helpers" } [features] default = [ "std" ] -experimental = [ "frame-support/experimental" ] no_std = [] std = [ "bitvec/std", diff --git a/polkadot/runtime/common/src/assigned_slots/migration.rs b/polkadot/runtime/common/src/assigned_slots/migration.rs index 656e0836bba3..0e88b27a1ff8 100644 --- a/polkadot/runtime/common/src/assigned_slots/migration.rs +++ b/polkadot/runtime/common/src/assigned_slots/migration.rs @@ -63,7 +63,6 @@ pub mod v1 { /// [`MigrateToV1`] wrapped in a /// [`VersionedMigration`](frame_support::migrations::VersionedMigration), ensuring the /// migration is only performed when on-chain version is 0. - #[cfg(feature = "experimental")] pub type VersionCheckedMigrateToV1 = frame_support::migrations::VersionedMigration< 0, 1, diff --git a/polkadot/runtime/common/src/paras_registrar/migration.rs b/polkadot/runtime/common/src/paras_registrar/migration.rs index 7d3dda54e0ed..b767985489d3 100644 --- a/polkadot/runtime/common/src/paras_registrar/migration.rs +++ b/polkadot/runtime/common/src/paras_registrar/migration.rs @@ -60,7 +60,6 @@ impl> OnRuntimeUpgrade } } -#[cfg(feature = "experimental")] pub type VersionCheckedMigrateToV1 = frame_support::migrations::VersionedMigration< 0, diff --git a/polkadot/runtime/kusama/Cargo.toml b/polkadot/runtime/kusama/Cargo.toml index 6b0370633f0b..565b34637e29 100644 --- a/polkadot/runtime/kusama/Cargo.toml +++ b/polkadot/runtime/kusama/Cargo.toml @@ -77,7 +77,7 @@ pallet-recovery = { path = "../../../substrate/frame/recovery", default-features pallet-referenda = { path = "../../../substrate/frame/referenda", default-features = false } pallet-scheduler = { path = "../../../substrate/frame/scheduler", default-features = false } pallet-session = { path = "../../../substrate/frame/session", default-features = false } -pallet-society = { path = "../../../substrate/frame/society", default-features = false, features = ["experimental"] } +pallet-society = { path = "../../../substrate/frame/society", default-features = false } frame-support = { path = "../../../substrate/frame/support", default-features = false } pallet-staking = { path = "../../../substrate/frame/staking", default-features = false } pallet-state-trie-migration = { path = "../../../substrate/frame/state-trie-migration", default-features = false } @@ -103,7 +103,7 @@ frame-system-benchmarking = { path = "../../../substrate/frame/system/benchmarki pallet-election-provider-support-benchmarking = { path = "../../../substrate/frame/election-provider-support/benchmarking", default-features = false, optional = true } hex-literal = "0.4.1" -runtime-common = { package = "polkadot-runtime-common", path = "../common", default-features = false, features = ["experimental"] } +runtime-common = { package = "polkadot-runtime-common", path = "../common", default-features = false } runtime-parachains = { package = "polkadot-runtime-parachains", path = "../parachains", default-features = false } primitives = { package = "polkadot-primitives", path = "../../primitives", default-features = false } @@ -116,7 +116,7 @@ tiny-keccak = { version = "2.0.2", features = ["keccak"] } keyring = { package = "sp-keyring", path = "../../../substrate/primitives/keyring" } sp-trie = { path = "../../../substrate/primitives/trie" } separator = "0.4.1" -serde_json = "1.0.96" +serde_json = "1.0.107" remote-externalities = { package = "frame-remote-externalities" , path = "../../../substrate/utils/frame/remote-externalities" } tokio = { version = "1.24.2", features = ["macros"] } sp-tracing = { path = "../../../substrate/primitives/tracing", default-features = false } diff --git a/polkadot/runtime/parachains/Cargo.toml b/polkadot/runtime/parachains/Cargo.toml index 550c0ef75346..9f1ec57257f8 100644 --- a/polkadot/runtime/parachains/Cargo.toml +++ b/polkadot/runtime/parachains/Cargo.toml @@ -62,7 +62,7 @@ test-helpers = { package = "polkadot-primitives-test-helpers", path = "../../pri sp-tracing = { path = "../../../substrate/primitives/tracing" } thousands = "0.2.0" assert_matches = "1" -serde_json = "1.0.96" +serde_json = "1.0.107" [features] default = [ "std" ] diff --git a/polkadot/runtime/parachains/src/paras/tests.rs b/polkadot/runtime/parachains/src/paras/tests.rs index a024525c817a..c511c65d4733 100644 --- a/polkadot/runtime/parachains/src/paras/tests.rs +++ b/polkadot/runtime/parachains/src/paras/tests.rs @@ -17,11 +17,11 @@ use super::*; use frame_support::{assert_err, assert_ok, assert_storage_noop}; use keyring::Sr25519Keyring; -use primitives::{BlockNumber, ValidatorId, PARACHAIN_KEY_TYPE_ID}; +use primitives::{BlockNumber, PARACHAIN_KEY_TYPE_ID}; use sc_keystore::LocalKeystore; use sp_keystore::{Keystore, KeystorePtr}; use std::sync::Arc; -use test_helpers::{dummy_head_data, dummy_validation_code}; +use test_helpers::{dummy_head_data, dummy_validation_code, validator_pubkeys}; use crate::{ configuration::HostConfiguration, @@ -39,10 +39,6 @@ static VALIDATORS: &[Sr25519Keyring] = &[ Sr25519Keyring::Ferdie, ]; -fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec { - val_ids.iter().map(|v| v.public().into()).collect() -} - fn sign_and_include_pvf_check_statement(stmt: PvfCheckStatement) { let validators = &[ Sr25519Keyring::Alice, diff --git a/polkadot/runtime/parachains/src/shared/tests.rs b/polkadot/runtime/parachains/src/shared/tests.rs index 91891ba8d75b..ae12b4b3fc16 100644 --- a/polkadot/runtime/parachains/src/shared/tests.rs +++ b/polkadot/runtime/parachains/src/shared/tests.rs @@ -22,10 +22,7 @@ use crate::{ use assert_matches::assert_matches; use keyring::Sr25519Keyring; use primitives::Hash; - -fn validator_pubkeys(val_ids: &[Sr25519Keyring]) -> Vec { - val_ids.iter().map(|v| v.public().into()).collect() -} +use test_helpers::validator_pubkeys; #[test] fn tracker_earliest_block_number() { diff --git a/polkadot/runtime/polkadot/Cargo.toml b/polkadot/runtime/polkadot/Cargo.toml index 0b9498347ca4..a659a4553220 100644 --- a/polkadot/runtime/polkadot/Cargo.toml +++ b/polkadot/runtime/polkadot/Cargo.toml @@ -82,7 +82,7 @@ pallet-whitelist = { path = "../../../substrate/frame/whitelist", default-featur pallet-vesting = { path = "../../../substrate/frame/vesting", default-features = false } pallet-utility = { path = "../../../substrate/frame/utility", default-features = false } frame-election-provider-support = { path = "../../../substrate/frame/election-provider-support", default-features = false } -pallet-xcm = { path = "../../xcm/pallet-xcm", default-features = false, features=["experimental"] } +pallet-xcm = { path = "../../xcm/pallet-xcm", default-features = false } pallet-xcm-benchmarks = { path = "../../xcm/pallet-xcm-benchmarks", default-features = false, optional = true } frame-benchmarking = { path = "../../../substrate/frame/benchmarking", default-features = false, optional = true } @@ -94,7 +94,7 @@ pallet-session-benchmarking = { path = "../../../substrate/frame/session/benchma pallet-nomination-pools-benchmarking = { path = "../../../substrate/frame/nomination-pools/benchmarking", default-features = false, optional = true } hex-literal = { version = "0.4.1", optional = true } -runtime-common = { package = "polkadot-runtime-common", path = "../common", default-features = false, features = ["experimental"] } +runtime-common = { package = "polkadot-runtime-common", path = "../common", default-features = false } runtime-parachains = { package = "polkadot-runtime-parachains", path = "../parachains", default-features = false } primitives = { package = "polkadot-primitives", path = "../../primitives", default-features = false } @@ -107,7 +107,7 @@ hex-literal = "0.4.1" tiny-keccak = { version = "2.0.2", features = ["keccak"] } keyring = { package = "sp-keyring", path = "../../../substrate/primitives/keyring" } sp-trie = { path = "../../../substrate/primitives/trie" } -serde_json = "1.0.96" +serde_json = "1.0.107" separator = "0.4.1" remote-externalities = { package = "frame-remote-externalities" , path = "../../../substrate/utils/frame/remote-externalities" } tokio = { version = "1.24.2", features = ["macros"] } diff --git a/polkadot/runtime/rococo/Cargo.toml b/polkadot/runtime/rococo/Cargo.toml index a181250cfa37..8f38659b84f5 100644 --- a/polkadot/runtime/rococo/Cargo.toml +++ b/polkadot/runtime/rococo/Cargo.toml @@ -65,7 +65,7 @@ pallet-proxy = { path = "../../../substrate/frame/proxy", default-features = fal pallet-recovery = { path = "../../../substrate/frame/recovery", default-features = false } pallet-scheduler = { path = "../../../substrate/frame/scheduler", default-features = false } pallet-session = { path = "../../../substrate/frame/session", default-features = false } -pallet-society = { path = "../../../substrate/frame/society", default-features = false, features = ["experimental"] } +pallet-society = { path = "../../../substrate/frame/society", default-features = false } pallet-sudo = { path = "../../../substrate/frame/sudo", default-features = false } frame-support = { path = "../../../substrate/frame/support", default-features = false } pallet-staking = { path = "../../../substrate/frame/staking", default-features = false } @@ -76,7 +76,7 @@ pallet-tips = { path = "../../../substrate/frame/tips", default-features = false pallet-treasury = { path = "../../../substrate/frame/treasury", default-features = false } pallet-utility = { path = "../../../substrate/frame/utility", default-features = false } pallet-vesting = { path = "../../../substrate/frame/vesting", default-features = false } -pallet-xcm = { path = "../../xcm/pallet-xcm", default-features = false, features=["experimental"] } +pallet-xcm = { path = "../../xcm/pallet-xcm", default-features = false } pallet-xcm-benchmarks = { path = "../../xcm/pallet-xcm-benchmarks", default-features = false, optional = true } frame-benchmarking = { path = "../../../substrate/frame/benchmarking", default-features = false, optional = true } @@ -84,7 +84,7 @@ frame-try-runtime = { path = "../../../substrate/frame/try-runtime", default-fea frame-system-benchmarking = { path = "../../../substrate/frame/system/benchmarking", default-features = false, optional = true } hex-literal = { version = "0.4.1" } -runtime-common = { package = "polkadot-runtime-common", path = "../common", default-features = false, features = ["experimental"] } +runtime-common = { package = "polkadot-runtime-common", path = "../common", default-features = false } runtime-parachains = { package = "polkadot-runtime-parachains", path = "../parachains", default-features = false } primitives = { package = "polkadot-primitives", path = "../../primitives", default-features = false } polkadot-parachain-primitives = { path = "../../parachain", default-features = false } @@ -99,7 +99,7 @@ keyring = { package = "sp-keyring", path = "../../../substrate/primitives/keyrin remote-externalities = { package = "frame-remote-externalities" , path = "../../../substrate/utils/frame/remote-externalities" } sp-trie = { path = "../../../substrate/primitives/trie" } separator = "0.4.1" -serde_json = "1.0.96" +serde_json = "1.0.107" sp-tracing = { path = "../../../substrate/primitives/tracing", default-features = false } tokio = { version = "1.24.2", features = ["macros"] } diff --git a/polkadot/runtime/test-runtime/Cargo.toml b/polkadot/runtime/test-runtime/Cargo.toml index 1e411bc901ce..a8f26eb5b3d9 100644 --- a/polkadot/runtime/test-runtime/Cargo.toml +++ b/polkadot/runtime/test-runtime/Cargo.toml @@ -70,7 +70,7 @@ hex-literal = "0.4.1" tiny-keccak = { version = "2.0.2", features = ["keccak"] } keyring = { package = "sp-keyring", path = "../../../substrate/primitives/keyring" } sp-trie = { path = "../../../substrate/primitives/trie" } -serde_json = "1.0.96" +serde_json = "1.0.107" [build-dependencies] substrate-wasm-builder = { path = "../../../substrate/utils/wasm-builder" } diff --git a/polkadot/runtime/westend/Cargo.toml b/polkadot/runtime/westend/Cargo.toml index de561c8ac68b..f4bb66f68cdf 100644 --- a/polkadot/runtime/westend/Cargo.toml +++ b/polkadot/runtime/westend/Cargo.toml @@ -83,7 +83,7 @@ pallet-nomination-pools-runtime-api = { path = "../../../substrate/frame/nominat pallet-treasury = { path = "../../../substrate/frame/treasury", default-features = false } pallet-utility = { path = "../../../substrate/frame/utility", default-features = false } pallet-vesting = { path = "../../../substrate/frame/vesting", default-features = false } -pallet-xcm = { path = "../../xcm/pallet-xcm", default-features = false, features=["experimental"] } +pallet-xcm = { path = "../../xcm/pallet-xcm", default-features = false } pallet-xcm-benchmarks = { path = "../../xcm/pallet-xcm-benchmarks", default-features = false, optional = true } frame-benchmarking = { path = "../../../substrate/frame/benchmarking", default-features = false, optional = true } @@ -95,7 +95,7 @@ pallet-offences-benchmarking = { path = "../../../substrate/frame/offences/bench pallet-session-benchmarking = { path = "../../../substrate/frame/session/benchmarking", default-features = false, optional = true } hex-literal = { version = "0.4.1", optional = true } -runtime-common = { package = "polkadot-runtime-common", path = "../common", default-features = false, features = ["experimental"] } +runtime-common = { package = "polkadot-runtime-common", path = "../common", default-features = false } primitives = { package = "polkadot-primitives", path = "../../primitives", default-features = false } polkadot-parachain-primitives = { path = "../../parachain", default-features = false } runtime-parachains = { package = "polkadot-runtime-parachains", path = "../parachains", default-features = false } @@ -108,7 +108,7 @@ xcm-builder = { package = "staging-xcm-builder", path = "../../xcm/xcm-builder", hex-literal = "0.4.1" tiny-keccak = { version = "2.0.2", features = ["keccak"] } keyring = { package = "sp-keyring", path = "../../../substrate/primitives/keyring" } -serde_json = "1.0.96" +serde_json = "1.0.107" remote-externalities = { package = "frame-remote-externalities" , path = "../../../substrate/utils/frame/remote-externalities" } tokio = { version = "1.24.2", features = ["macros"] } sp-tracing = { path = "../../../substrate/primitives/tracing", default-features = false } diff --git a/polkadot/src/bin/execute-worker.rs b/polkadot/src/bin/execute-worker.rs index 72cab799d753..1deb36580986 100644 --- a/polkadot/src/bin/execute-worker.rs +++ b/polkadot/src/bin/execute-worker.rs @@ -19,5 +19,5 @@ polkadot_node_core_pvf_common::decl_worker_main!( "execute-worker", polkadot_node_core_pvf_execute_worker::worker_entrypoint, - env!("SUBSTRATE_CLI_IMPL_VERSION") + polkadot_cli::NODE_VERSION, ); diff --git a/polkadot/src/bin/prepare-worker.rs b/polkadot/src/bin/prepare-worker.rs index 695f66cc7b7d..d731f8a30d06 100644 --- a/polkadot/src/bin/prepare-worker.rs +++ b/polkadot/src/bin/prepare-worker.rs @@ -19,5 +19,5 @@ polkadot_node_core_pvf_common::decl_worker_main!( "prepare-worker", polkadot_node_core_pvf_prepare_worker::worker_entrypoint, - env!("SUBSTRATE_CLI_IMPL_VERSION") + polkadot_cli::NODE_VERSION, ); diff --git a/polkadot/utils/generate-bags/Cargo.toml b/polkadot/utils/generate-bags/Cargo.toml index 98a0a9b6a885..99948cb68b18 100644 --- a/polkadot/utils/generate-bags/Cargo.toml +++ b/polkadot/utils/generate-bags/Cargo.toml @@ -6,7 +6,7 @@ edition.workspace = true license.workspace = true [dependencies] -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } generate-bags = { path = "../../../substrate/utils/frame/generate-bags" } sp-io = { path = "../../../substrate/primitives/io" } diff --git a/polkadot/utils/remote-ext-tests/bags-list/Cargo.toml b/polkadot/utils/remote-ext-tests/bags-list/Cargo.toml index bbdddf750644..ed49f77bb81a 100644 --- a/polkadot/utils/remote-ext-tests/bags-list/Cargo.toml +++ b/polkadot/utils/remote-ext-tests/bags-list/Cargo.toml @@ -19,6 +19,6 @@ sp-tracing = { path = "../../../../substrate/primitives/tracing" } frame-system = { path = "../../../../substrate/frame/system" } sp-core = { path = "../../../../substrate/primitives/core" } -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } log = "0.4.17" tokio = { version = "1.24.2", features = ["macros"] } diff --git a/polkadot/utils/staking-miner/.gitignore b/polkadot/utils/staking-miner/.gitignore deleted file mode 100644 index db7cff848330..000000000000 --- a/polkadot/utils/staking-miner/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.key -*.bin diff --git a/polkadot/utils/staking-miner/Cargo.toml b/polkadot/utils/staking-miner/Cargo.toml deleted file mode 100644 index 4b012e3ac73f..000000000000 --- a/polkadot/utils/staking-miner/Cargo.toml +++ /dev/null @@ -1,54 +0,0 @@ -[[bin]] -name = "staging-staking-miner" -path = "src/main.rs" - -[package] -name = "staging-staking-miner" -version = "1.0.0" -authors.workspace = true -edition.workspace = true -license.workspace = true -publish = false - -[dependencies] -codec = { package = "parity-scale-codec", version = "3.6.1" } -clap = { version = "4.4.2", features = ["derive", "env"] } -tracing-subscriber = { version = "0.3.11", features = ["env-filter"] } -jsonrpsee = { version = "0.16.2", features = ["ws-client", "macros"] } -log = "0.4.17" -paste = "1.0.7" -serde = "1.0.188" -serde_json = "1.0" -thiserror = "1.0.48" -tokio = { version = "1.24.2", features = ["macros", "rt-multi-thread", "sync"] } -remote-externalities = { package = "frame-remote-externalities" , path = "../../../substrate/utils/frame/remote-externalities" } -signal-hook-tokio = { version = "0.3", features = ["futures-v0_3"] } -sp-core = { path = "../../../substrate/primitives/core" } -sp-version = { path = "../../../substrate/primitives/version" } -sp-state-machine = { path = "../../../substrate/primitives/state-machine" } -sp-runtime = { path = "../../../substrate/primitives/runtime" } -sp-npos-elections = { path = "../../../substrate/primitives/npos-elections" } -sc-transaction-pool-api = { path = "../../../substrate/client/transaction-pool/api" } - -frame-system = { path = "../../../substrate/frame/system" } -frame-support = { path = "../../../substrate/frame/support" } -frame-election-provider-support = { path = "../../../substrate/frame/election-provider-support" } -pallet-election-provider-multi-phase = { path = "../../../substrate/frame/election-provider-multi-phase" } -pallet-staking = { path = "../../../substrate/frame/staking" } -pallet-balances = { path = "../../../substrate/frame/balances" } -pallet-transaction-payment = { path = "../../../substrate/frame/transaction-payment" } - -core-primitives = { package = "polkadot-core-primitives", path = "../../core-primitives" } - -runtime-common = { package = "polkadot-runtime-common", path = "../../runtime/common" } -polkadot-runtime = { path = "../../runtime/polkadot" } -kusama-runtime = { package = "staging-kusama-runtime", path = "../../runtime/kusama" } -westend-runtime = { path = "../../runtime/westend" } -exitcode = "1.1" - -sub-tokens = { git = "https://github.com/paritytech/substrate-debug-kit", branch = "master" } -signal-hook = "0.3" -futures-util = "0.3" - -[dev-dependencies] -assert_cmd = "2.0.4" diff --git a/polkadot/utils/staking-miner/README.md b/polkadot/utils/staking-miner/README.md deleted file mode 100644 index 90a00eeac089..000000000000 --- a/polkadot/utils/staking-miner/README.md +++ /dev/null @@ -1,81 +0,0 @@ -# Staking Miner - -Substrate chains validators compute a basic solution for the NPoS election. The optimization of the solution is -computing-intensive and can be delegated to the `staking-miner`. The `staking-miner` does not act as validator and -focuses solely on the optimization of the solution. - -The staking miner connects to a specified chain and keeps listening to new Signed phase of the -[pallet-election-provider-multi-phase](https://crates.parity.io/pallet_election_provider_multi_phase/index.html) in -order to submit solutions to the NPoS election. When the correct time comes, it computes its solution and submit it to -the chain. The default miner algorithm is -[sequential-phragmen](https://crates.parity.io/sp_npos_elections/phragmen/fn.seq_phragmen_core.html)] with a -configurable number of balancing iterations that improve the score. - -Running the staking-miner requires passing the seed of a funded account in order to pay the fees for the transactions -that will be sent. The same account's balance is used to reserve deposits as well. The best solution in each round is -rewarded. All correct solutions will get their bond back. Any invalid solution will lose their bond. - -You can check the help with: -``` -staking-miner --help -``` - -## Building - -You can build from the root of the Polkadot repository using: -``` -cargo build --profile production --locked --package staking-miner --bin staking-miner -``` - -## Docker - -There are 2 options to build a staking-miner Docker image: -- injected binary: the binary is first built on a Linux host and then injected into a Docker base image. This method - only works if you have a Linux host or access to a pre-built binary from a Linux host. -- multi-stage: the binary is entirely built within the multi-stage Docker image. There is no requirement on the host in - terms of OS and the host does not even need to have any Rust toolchain installed. - -### Building the injected image - -First build the binary as documented [above](#building). You may then inject the binary into a Docker base image: -`parity/base-bin` (running the command from the root of the Polkadot repository): -``` -TODO: UPDATE THAT -docker build -t staking-miner -f scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile target/release -``` - -### Building the multi-stage image - -Unlike the injected image that requires a Linux pre-built binary, this option does not requires a Linux host, nor Rust -to be installed. The trade-off however is that it takes a little longer to build and this option is less ideal for CI -tasks. You may build the multi-stage image the root of the Polkadot repository with: -``` -TODO: UPDATE THAT -docker build -t staking-miner -f docker/dockerfiles/staking-miner/staking-miner_builder.Dockerfile . -``` - -### Running - -A Docker container, especially one holding one of your `SEED` should be kept as secure as possible. While it won't -prevent a malicious actor to read your `SEED` if they gain access to your container, it is nonetheless recommended -running this container in `read-only` mode: - -``` -# The following line starts with an extra space on purpose: - SEED=0x1234... - -docker run --rm -i \ - --name staking-miner \ - --read-only \ - -e RUST_LOG=info \ - -e SEED=$SEED \ - -e URI=wss://your-node:9944 \ - staking-miner dry-run -``` - -### Test locally - -Make sure you've built Polkadot, then: - -1. `cargo run -p polkadot --features fast-runtime -- --chain polkadot-dev --tmp --alice -lruntime=debug` -2. `cargo run -p staking-miner -- --uri ws://localhost:9944 monitor --seed-or-path //Alice phrag-mms` diff --git a/polkadot/utils/staking-miner/src/dry_run.rs b/polkadot/utils/staking-miner/src/dry_run.rs deleted file mode 100644 index 7e46f630a1f5..000000000000 --- a/polkadot/utils/staking-miner/src/dry_run.rs +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! The dry-run command. - -use crate::{opts::DryRunConfig, prelude::*, rpc::*, signer::Signer, Error, SharedRpcClient}; -use codec::Encode; -use frame_support::traits::Currency; -use sp_core::Bytes; -use sp_npos_elections::ElectionScore; - -/// Forcefully create the snapshot. This can be used to compute the election at anytime. -fn force_create_snapshot(ext: &mut Ext) -> Result<(), Error> { - ext.execute_with(|| { - if >::exists() { - log::info!(target: LOG_TARGET, "snapshot already exists."); - Ok(()) - } else { - log::info!(target: LOG_TARGET, "creating a fake snapshot now."); - >::create_snapshot().map(|_| ()).map_err(Into::into) - } - }) -} - -/// Helper method to print the encoded size of the snapshot. -async fn print_info( - rpc: &SharedRpcClient, - ext: &mut Ext, - raw_solution: &EPM::RawSolution>, - extrinsic: &Bytes, -) where - ::Currency: Currency, -{ - ext.execute_with(|| { - log::info!( - target: LOG_TARGET, - "Snapshot Metadata: {:?}", - >::snapshot_metadata() - ); - log::info!( - target: LOG_TARGET, - "Snapshot Encoded Length: {:?}", - >::snapshot() - .expect("snapshot must exist before calling `measure_snapshot_size`") - .encode() - .len() - ); - - let snapshot_size = - >::snapshot_metadata().expect("snapshot must exist by now; qed."); - let deposit = EPM::Pallet::::deposit_for(raw_solution, snapshot_size); - - let score = { - let ElectionScore { minimal_stake, sum_stake, sum_stake_squared } = raw_solution.score; - [Token::from(minimal_stake), Token::from(sum_stake), Token::from(sum_stake_squared)] - }; - - log::info!( - target: LOG_TARGET, - "solution score {:?} / deposit {:?} / length {:?}", - score, - Token::from(deposit), - raw_solution.encode().len(), - ); - }); - - let info = rpc.payment_query_info(&extrinsic, None).await; - - log::info!( - target: LOG_TARGET, - "payment_queryInfo: (fee = {}) {:?}", - info.as_ref() - .map(|d| Token::from(d.partial_fee)) - .unwrap_or_else(|_| Token::from(0)), - info, - ); -} - -/// Find the stake threshold in order to have at most `count` voters. -#[allow(unused)] -fn find_threshold(ext: &mut Ext, count: usize) { - ext.execute_with(|| { - let mut voters = >::snapshot() - .expect("snapshot must exist before calling `measure_snapshot_size`") - .voters; - voters.sort_by_key(|(_voter, weight, _targets)| std::cmp::Reverse(*weight)); - match voters.get(count) { - Some(threshold_voter) => println!("smallest allowed voter is {:?}", threshold_voter), - None => { - println!("requested truncation to {} voters but had only {}", count, voters.len()); - println!("smallest current voter: {:?}", voters.last()); - }, - } - }) -} - -macro_rules! dry_run_cmd_for { ($runtime:ident) => { paste::paste! { - /// Execute the dry-run command. - pub(crate) async fn []( - rpc: SharedRpcClient, - config: DryRunConfig, - signer: Signer, - ) -> Result<(), Error<$crate::[<$runtime _runtime_exports>]::Runtime>> { - use $crate::[<$runtime _runtime_exports>]::*; - let pallets = if config.force_snapshot { - vec!["Staking".to_string(), "BagsList".to_string()] - } else { - Default::default() - }; - let mut ext = crate::create_election_ext::(rpc.clone(), config.at, pallets).await?; - if config.force_snapshot { - force_create_snapshot::(&mut ext)?; - }; - - log::debug!(target: LOG_TARGET, "solving with {:?}", config.solver); - let raw_solution = crate::mine_with::(&config.solver, &mut ext, false)?; - - let nonce = crate::get_account_info::(&rpc, &signer.account, config.at) - .await? - .map(|i| i.nonce) - .expect("signer account is checked to exist upon startup; it can only die if it \ - transfers funds out of it, or get slashed. If it does not exist at this point, \ - it is likely due to a bug, or the signer got slashed. Terminating." - ); - let tip = 0 as Balance; - let era = sp_runtime::generic::Era::Immortal; - let extrinsic = ext.execute_with(|| create_uxt(raw_solution.clone(), signer.clone(), nonce, tip, era)); - - let bytes = sp_core::Bytes(extrinsic.encode().to_vec()); - print_info::(&rpc, &mut ext, &raw_solution, &bytes).await; - - let feasibility_result = ext.execute_with(|| { - EPM::Pallet::::feasibility_check(raw_solution.clone(), EPM::ElectionCompute::Signed) - }); - log::info!(target: LOG_TARGET, "feasibility result is {:?}", feasibility_result.map(|_| ())); - - let dispatch_result = ext.execute_with(|| { - // manually tweak the phase. - EPM::CurrentPhase::::put(EPM::Phase::Signed); - EPM::Pallet::::submit(frame_system::RawOrigin::Signed(signer.account).into(), Box::new(raw_solution)) - }); - log::info!(target: LOG_TARGET, "dispatch result is {:?}", dispatch_result); - - let dry_run_fut = rpc.dry_run(&bytes, None); - let outcome: sp_runtime::ApplyExtrinsicResult = await_request_and_decode(dry_run_fut).await.map_err::, _>(Into::into)?; - log::info!(target: LOG_TARGET, "dry-run outcome is {:?}", outcome); - Ok(()) - } -}}} - -dry_run_cmd_for!(polkadot); -dry_run_cmd_for!(kusama); -dry_run_cmd_for!(westend); diff --git a/polkadot/utils/staking-miner/src/emergency_solution.rs b/polkadot/utils/staking-miner/src/emergency_solution.rs deleted file mode 100644 index 9ea9f90756e2..000000000000 --- a/polkadot/utils/staking-miner/src/emergency_solution.rs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! The emergency-solution command. - -use crate::{prelude::*, EmergencySolutionConfig, Error, SharedRpcClient}; -use codec::Encode; -use std::io::Write; - -macro_rules! emergency_solution_cmd_for { ($runtime:ident) => { paste::paste! { - /// Execute the emergency-solution command. - pub(crate) async fn []( - client: SharedRpcClient, - config: EmergencySolutionConfig, - ) -> Result<(), Error<$crate::[<$runtime _runtime_exports>]::Runtime>> { - use $crate::[<$runtime _runtime_exports>]::*; - - let mut ext = crate::create_election_ext::(client, config.at, vec![]).await?; - let raw_solution = crate::mine_with::(&config.solver, &mut ext, false)?; - - ext.execute_with(|| { - assert!(EPM::Pallet::::current_phase().is_emergency()); - - log::info!(target: LOG_TARGET, "mined solution with {:?}", &raw_solution.score); - - let ready_solution = EPM::Pallet::::feasibility_check(raw_solution, EPM::ElectionCompute::Signed)?; - let encoded_size = ready_solution.encoded_size(); - let score = ready_solution.score; - let mut supports = ready_solution.supports.into_inner(); - // maybe truncate. - if let Some(take) = config.take { - log::info!(target: LOG_TARGET, "truncating {} winners to {}", supports.len(), take); - supports.sort_unstable_by_key(|(_, s)| s.total); - supports.truncate(take); - } - - // write to file and stdout. - let encoded_support = supports.encode(); - let mut supports_file = std::fs::File::create("solution.supports.bin")?; - supports_file.write_all(&encoded_support)?; - - log::info!(target: LOG_TARGET, "ReadySolution: size {:?} / score = {:?}", encoded_size, score); - log::trace!(target: LOG_TARGET, "Supports: {}", sp_core::hexdisplay::HexDisplay::from(&encoded_support)); - - Ok(()) - }) - } -}}} - -emergency_solution_cmd_for!(polkadot); -emergency_solution_cmd_for!(kusama); -emergency_solution_cmd_for!(westend); diff --git a/polkadot/utils/staking-miner/src/main.rs b/polkadot/utils/staking-miner/src/main.rs deleted file mode 100644 index 90b2c7366a1b..000000000000 --- a/polkadot/utils/staking-miner/src/main.rs +++ /dev/null @@ -1,665 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! # Polkadot Staking Miner. -//! -//! Simple bot capable of monitoring a polkadot (and cousins) chain and submitting solutions to the -//! `pallet-election-provider-multi-phase`. See `--help` for more details. -//! -//! # Implementation Notes: -//! -//! - First draft: Be aware that this is the first draft and there might be bugs, or undefined -//! behaviors. Don't attach this bot to an account with lots of funds. -//! - Quick to crash: The bot is written so that it only continues to work if everything goes well. -//! In case of any failure (RPC, logic, IO), it will crash. This was a decision to simplify the -//! development. It is intended to run this bot with a `restart = true` way, so that it reports it -//! crash, but resumes work thereafter. - -// Silence erroneous warning about unsafe not being required whereas it is -// see https://github.com/rust-lang/rust/issues/49112 -#![allow(unused_unsafe)] - -mod dry_run; -mod emergency_solution; -mod monitor; -mod opts; -mod prelude; -mod rpc; -mod runtime_versions; -mod signer; - -pub(crate) use prelude::*; -pub(crate) use signer::get_account_info; - -use crate::opts::*; -use clap::Parser; -use frame_election_provider_support::NposSolver; -use frame_support::traits::Get; -use futures_util::StreamExt; -use jsonrpsee::ws_client::{WsClient, WsClientBuilder}; -use remote_externalities::{Builder, Mode, OnlineConfig, Transport}; -use rpc::{RpcApiClient, SharedRpcClient}; -use runtime_versions::RuntimeVersions; -use signal_hook::consts::signal::*; -use signal_hook_tokio::Signals; -use sp_npos_elections::BalancingConfig; -use std::{ops::Deref, sync::Arc, time::Duration}; -use tracing_subscriber::{fmt, EnvFilter}; - -pub(crate) enum AnyRuntime { - Polkadot, - Kusama, - Westend, -} - -pub(crate) static mut RUNTIME: AnyRuntime = AnyRuntime::Polkadot; - -macro_rules! construct_runtime_prelude { - ($runtime:ident) => { paste::paste! { - pub(crate) mod [<$runtime _runtime_exports>] { - pub(crate) use crate::prelude::EPM; - pub(crate) use [<$runtime _runtime>]::*; - pub(crate) use crate::monitor::[] as monitor_cmd; - pub(crate) use crate::dry_run::[] as dry_run_cmd; - pub(crate) use crate::emergency_solution::[] as emergency_solution_cmd; - pub(crate) use private::{[] as create_uxt}; - - mod private { - use super::*; - pub(crate) fn []( - raw_solution: EPM::RawSolution>, - signer: crate::signer::Signer, - nonce: crate::prelude::Nonce, - tip: crate::prelude::Balance, - era: sp_runtime::generic::Era, - ) -> UncheckedExtrinsic { - use codec::Encode as _; - use sp_core::Pair as _; - use sp_runtime::traits::StaticLookup as _; - - let crate::signer::Signer { account, pair, .. } = signer; - - let local_call = EPMCall::::submit { raw_solution: Box::new(raw_solution) }; - let call: RuntimeCall = as std::convert::TryInto>::try_into(local_call) - .expect("election provider pallet must exist in the runtime, thus \ - inner call can be converted, qed." - ); - - let extra: SignedExtra = crate::[](nonce, tip, era); - let raw_payload = SignedPayload::new(call, extra).expect("creating signed payload infallible; qed."); - let signature = raw_payload.using_encoded(|payload| { - pair.sign(payload) - }); - let (call, extra, _) = raw_payload.deconstruct(); - let address = ::Lookup::unlookup(account); - let extrinsic = UncheckedExtrinsic::new_signed(call, address, signature.into(), extra); - log::debug!( - target: crate::LOG_TARGET, "constructed extrinsic {} with length {}", - sp_core::hexdisplay::HexDisplay::from(&extrinsic.encode()), - extrinsic.encode().len(), - ); - extrinsic - } - } - }} - }; -} - -// NOTE: we might be able to use some code from the bridges repo here. -fn signed_ext_builder_polkadot( - nonce: Nonce, - tip: Balance, - era: sp_runtime::generic::Era, -) -> polkadot_runtime_exports::SignedExtra { - use polkadot_runtime_exports::Runtime; - ( - frame_system::CheckNonZeroSender::::new(), - frame_system::CheckSpecVersion::::new(), - frame_system::CheckTxVersion::::new(), - frame_system::CheckGenesis::::new(), - frame_system::CheckMortality::::from(era), - frame_system::CheckNonce::::from(nonce), - frame_system::CheckWeight::::new(), - pallet_transaction_payment::ChargeTransactionPayment::::from(tip), - runtime_common::claims::PrevalidateAttests::::new(), - ) -} - -fn signed_ext_builder_kusama( - nonce: Nonce, - tip: Balance, - era: sp_runtime::generic::Era, -) -> kusama_runtime_exports::SignedExtra { - use kusama_runtime_exports::Runtime; - ( - frame_system::CheckNonZeroSender::::new(), - frame_system::CheckSpecVersion::::new(), - frame_system::CheckTxVersion::::new(), - frame_system::CheckGenesis::::new(), - frame_system::CheckMortality::::from(era), - frame_system::CheckNonce::::from(nonce), - frame_system::CheckWeight::::new(), - pallet_transaction_payment::ChargeTransactionPayment::::from(tip), - ) -} - -fn signed_ext_builder_westend( - nonce: Nonce, - tip: Balance, - era: sp_runtime::generic::Era, -) -> westend_runtime_exports::SignedExtra { - use westend_runtime_exports::Runtime; - ( - frame_system::CheckNonZeroSender::::new(), - frame_system::CheckSpecVersion::::new(), - frame_system::CheckTxVersion::::new(), - frame_system::CheckGenesis::::new(), - frame_system::CheckMortality::::from(era), - frame_system::CheckNonce::::from(nonce), - frame_system::CheckWeight::::new(), - pallet_transaction_payment::ChargeTransactionPayment::::from(tip), - ) -} - -construct_runtime_prelude!(polkadot); -construct_runtime_prelude!(kusama); -construct_runtime_prelude!(westend); - -// NOTE: this is no longer used extensively, most of the per-runtime stuff us delegated to -// `construct_runtime_prelude` and macro's the import directly from it. A part of the code is also -// still generic over `T`. My hope is to still make everything generic over a `Runtime`, but sadly -// that is not currently possible as each runtime has its unique `Call`, and all Calls are not -// sharing any generic trait. In other words, to create the `UncheckedExtrinsic` of each chain, you -// need the concrete `Call` of that chain as well. -#[macro_export] -macro_rules! any_runtime { - ($($code:tt)*) => { - unsafe { - match $crate::RUNTIME { - $crate::AnyRuntime::Polkadot => { - #[allow(unused)] - use $crate::polkadot_runtime_exports::*; - $($code)* - }, - $crate::AnyRuntime::Kusama => { - #[allow(unused)] - use $crate::kusama_runtime_exports::*; - $($code)* - }, - $crate::AnyRuntime::Westend => { - #[allow(unused)] - use $crate::westend_runtime_exports::*; - $($code)* - } - } - } - } -} - -/// Same as [`any_runtime`], but instead of returning a `Result`, this simply returns `()`. Useful -/// for situations where the result is not useful and un-ergonomic to handle. -#[macro_export] -macro_rules! any_runtime_unit { - ($($code:tt)*) => { - unsafe { - match $crate::RUNTIME { - $crate::AnyRuntime::Polkadot => { - #[allow(unused)] - use $crate::polkadot_runtime_exports::*; - let _ = $($code)*; - }, - $crate::AnyRuntime::Kusama => { - #[allow(unused)] - use $crate::kusama_runtime_exports::*; - let _ = $($code)*; - }, - $crate::AnyRuntime::Westend => { - #[allow(unused)] - use $crate::westend_runtime_exports::*; - let _ = $($code)*; - } - } - } - } -} - -#[derive(frame_support::DebugNoBound, thiserror::Error)] -enum Error { - Io(#[from] std::io::Error), - JsonRpsee(#[from] jsonrpsee::core::Error), - RpcHelperError(#[from] rpc::RpcHelperError), - Codec(#[from] codec::Error), - Crypto(sp_core::crypto::SecretStringError), - RemoteExternalities(&'static str), - PalletMiner(EPM::unsigned::MinerError), - PalletElection(EPM::ElectionError), - PalletFeasibility(EPM::FeasibilityError), - AccountDoesNotExists, - IncorrectPhase, - AlreadySubmitted, - VersionMismatch, - StrategyNotSatisfied, - Other(String), -} - -impl From for Error { - fn from(e: sp_core::crypto::SecretStringError) -> Error { - Error::Crypto(e) - } -} - -impl From for Error { - fn from(e: EPM::unsigned::MinerError) -> Error { - Error::PalletMiner(e) - } -} - -impl From> for Error { - fn from(e: EPM::ElectionError) -> Error { - Error::PalletElection(e) - } -} - -impl From for Error { - fn from(e: EPM::FeasibilityError) -> Error { - Error::PalletFeasibility(e) - } -} - -impl std::fmt::Display for Error { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - as std::fmt::Debug>::fmt(self, f) - } -} - -frame_support::parameter_types! { - /// Number of balancing iterations for a solution algorithm. Set based on the [`Solvers`] CLI - /// config. - pub static BalanceIterations: usize = 10; - pub static Balancing: Option = Some( BalancingConfig { iterations: BalanceIterations::get(), tolerance: 0 } ); -} - -/// Build the Ext at hash with all the data of `ElectionProviderMultiPhase` and any additional -/// pallets. -async fn create_election_ext( - client: SharedRpcClient, - at: Option, - additional: Vec, -) -> Result> -where - T: EPM::Config, -{ - use frame_support::{storage::generator::StorageMap, traits::PalletInfo}; - use sp_core::hashing::twox_128; - - let mut pallets = vec![::PalletInfo::name::>() - .expect("Pallet always has name; qed.") - .to_string()]; - pallets.extend(additional); - Builder::::new() - .mode(Mode::Online(OnlineConfig { - transport: Transport::Uri(client.uri().to_owned()), - at, - pallets, - hashed_prefixes: vec![>::prefix_hash()], - hashed_keys: vec![[twox_128(b"System"), twox_128(b"Number")].concat()], - ..Default::default() - })) - .build() - .await - .map_err(|why| Error::::RemoteExternalities(why)) - .map(|rx| rx.inner_ext) -} - -/// Compute the election. It expects to NOT be `Phase::Off`. In other words, the snapshot must -/// exists on the given externalities. -fn mine_solution( - ext: &mut Ext, - do_feasibility: bool, -) -> Result>, Error> -where - T: EPM::Config, - S: NposSolver< - Error = <::Solver as NposSolver>::Error, - AccountId = <::Solver as NposSolver>::AccountId, - >, -{ - ext.execute_with(|| { - let (solution, _) = >::mine_solution().map_err::, _>(Into::into)?; - if do_feasibility { - let _ = >::feasibility_check( - solution.clone(), - EPM::ElectionCompute::Signed, - )?; - } - Ok(solution) - }) -} - -/// Mine a solution with the given `solver`. -fn mine_with( - solver: &Solver, - ext: &mut Ext, - do_feasibility: bool, -) -> Result>, Error> -where - T: EPM::Config, - T::Solver: NposSolver, -{ - use frame_election_provider_support::{PhragMMS, SequentialPhragmen}; - - match solver { - Solver::SeqPhragmen { iterations } => { - BalanceIterations::set(*iterations); - mine_solution::< - T, - SequentialPhragmen< - ::AccountId, - sp_runtime::Perbill, - Balancing, - >, - >(ext, do_feasibility) - }, - Solver::PhragMMS { iterations } => { - BalanceIterations::set(*iterations); - mine_solution::< - T, - PhragMMS<::AccountId, sp_runtime::Perbill, Balancing>, - >(ext, do_feasibility) - }, - } -} - -#[allow(unused)] -fn mine_dpos(ext: &mut Ext) -> Result<(), Error> { - ext.execute_with(|| { - use std::collections::BTreeMap; - use EPM::RoundSnapshot; - let RoundSnapshot { voters, .. } = EPM::Snapshot::::get().unwrap(); - let desired_targets = EPM::DesiredTargets::::get().unwrap(); - let mut candidates_and_backing = BTreeMap::::new(); - voters.into_iter().for_each(|(who, stake, targets)| { - if targets.is_empty() { - println!("target = {:?}", (who, stake, targets)); - return - } - let share: u128 = (stake as u128) / (targets.len() as u128); - for target in targets { - *candidates_and_backing.entry(target.clone()).or_default() += share - } - }); - - let mut candidates_and_backing = - candidates_and_backing.into_iter().collect::>(); - candidates_and_backing.sort_by_key(|(_, total_stake)| *total_stake); - let winners = candidates_and_backing - .into_iter() - .rev() - .take(desired_targets as usize) - .collect::>(); - let score = { - let min_staker = *winners.last().map(|(_, stake)| stake).unwrap(); - let sum_stake = winners.iter().fold(0u128, |acc, (_, stake)| acc + stake); - let sum_squared = winners.iter().fold(0u128, |acc, (_, stake)| acc + stake); - [min_staker, sum_stake, sum_squared] - }; - println!("mined a dpos-like solution with score = {:?}", score); - Ok(()) - }) -} - -pub(crate) async fn check_versions( - rpc: &SharedRpcClient, - print: bool, -) -> Result<(), Error> { - let linked_version = T::Version::get(); - let on_chain_version = rpc - .runtime_version(None) - .await - .expect("runtime version RPC should always work; qed"); - - let do_print = || { - log::info!( - target: LOG_TARGET, - "linked version {:?}", - (&linked_version.spec_name, &linked_version.spec_version) - ); - log::info!( - target: LOG_TARGET, - "on-chain version {:?}", - (&on_chain_version.spec_name, &on_chain_version.spec_version) - ); - }; - - if print { - do_print(); - } - - // we relax the checking here a bit, which should not cause any issues in production (a chain - // that messes up its spec name is highly unlikely), but it allows us to do easier testing. - if linked_version.spec_name != on_chain_version.spec_name || - linked_version.spec_version != on_chain_version.spec_version - { - if !print { - do_print(); - } - log::error!( - target: LOG_TARGET, - "VERSION MISMATCH: any transaction will fail with bad-proof" - ); - Err(Error::VersionMismatch) - } else { - Ok(()) - } -} - -/// Control how we exit the application -fn controlled_exit(code: i32) { - log::info!(target: LOG_TARGET, "Exiting application"); - std::process::exit(code); -} - -/// Handles the various signal and exit the application -/// when appropriate. -async fn handle_signals(mut signals: Signals) { - let mut keyboard_sig_count: u8 = 0; - while let Some(signal) = signals.next().await { - match signal { - // Interrupts come from the keyboard - SIGQUIT | SIGINT => { - if keyboard_sig_count >= 1 { - log::info!( - target: LOG_TARGET, - "Received keyboard termination signal #{}/{}, quitting...", - keyboard_sig_count + 1, - 2 - ); - controlled_exit(exitcode::OK); - } - keyboard_sig_count += 1; - log::warn!( - target: LOG_TARGET, - "Received keyboard termination signal #{}, if you keep doing that I will really quit", - keyboard_sig_count - ); - }, - - SIGKILL | SIGTERM => { - log::info!(target: LOG_TARGET, "Received SIGKILL | SIGTERM, quitting..."); - controlled_exit(exitcode::OK); - }, - _ => unreachable!(), - } - } -} - -#[tokio::main] -async fn main() { - fmt().with_env_filter(EnvFilter::from_default_env()).init(); - - let Opt { uri, command, connection_timeout, request_timeout } = Opt::parse(); - log::debug!(target: LOG_TARGET, "attempting to connect to {:?}", uri); - - let signals = Signals::new(&[SIGTERM, SIGINT, SIGQUIT]).expect("Failed initializing Signals"); - let handle = signals.handle(); - let signals_task = tokio::spawn(handle_signals(signals)); - - let rpc = loop { - match SharedRpcClient::new( - &uri, - Duration::from_secs(connection_timeout as u64), - Duration::from_secs(request_timeout as u64), - ) - .await - { - Ok(client) => break client, - Err(why) => { - log::warn!( - target: LOG_TARGET, - "failed to connect to client due to {:?}, retrying soon..", - why - ); - tokio::time::sleep(std::time::Duration::from_millis(2500)).await; - }, - } - }; - - let chain: String = rpc.system_chain().await.expect("system_chain infallible; qed."); - match chain.to_lowercase().as_str() { - "polkadot" | "development" => { - sp_core::crypto::set_default_ss58_version( - sp_core::crypto::Ss58AddressFormatRegistry::PolkadotAccount.into(), - ); - sub_tokens::dynamic::set_name("DOT"); - sub_tokens::dynamic::set_decimal_points(10_000_000_000); - // safety: this program will always be single threaded, thus accessing global static is - // safe. - unsafe { - RUNTIME = AnyRuntime::Polkadot; - } - }, - "kusama" | "kusama-dev" => { - sp_core::crypto::set_default_ss58_version( - sp_core::crypto::Ss58AddressFormatRegistry::KusamaAccount.into(), - ); - sub_tokens::dynamic::set_name("KSM"); - sub_tokens::dynamic::set_decimal_points(1_000_000_000_000); - // safety: this program will always be single threaded, thus accessing global static is - // safe. - unsafe { - RUNTIME = AnyRuntime::Kusama; - } - }, - "westend" => { - sp_core::crypto::set_default_ss58_version( - sp_core::crypto::Ss58AddressFormatRegistry::PolkadotAccount.into(), - ); - sub_tokens::dynamic::set_name("WND"); - sub_tokens::dynamic::set_decimal_points(1_000_000_000_000); - // safety: this program will always be single threaded, thus accessing global static is - // safe. - unsafe { - RUNTIME = AnyRuntime::Westend; - } - }, - _ => { - eprintln!("unexpected chain: {:?}", chain); - return - }, - } - log::info!(target: LOG_TARGET, "connected to chain {:?}", chain); - - any_runtime_unit! { - check_versions::(&rpc, true).await - }; - - let outcome = any_runtime! { - match command { - Command::Monitor(monitor_config) => - { - let signer_account = any_runtime! { - signer::signer_uri_from_string::(&monitor_config.seed_or_path , &rpc) - .await - .expect("Provided account is invalid, terminating.") - }; - monitor_cmd(rpc, monitor_config, signer_account).await - .map_err(|e| { - log::error!(target: LOG_TARGET, "Monitor error: {:?}", e); - })}, - Command::DryRun(dryrun_config) => { - let signer_account = any_runtime! { - signer::signer_uri_from_string::(&dryrun_config.seed_or_path , &rpc) - .await - .expect("Provided account is invalid, terminating.") - }; - dry_run_cmd(rpc, dryrun_config, signer_account).await - .map_err(|e| { - log::error!(target: LOG_TARGET, "DryRun error: {:?}", e); - })}, - Command::EmergencySolution(emergency_solution_config) => - emergency_solution_cmd(rpc, emergency_solution_config).await - .map_err(|e| { - log::error!(target: LOG_TARGET, "EmergencySolution error: {:?}", e); - }), - Command::Info(info_opts) => { - let remote_runtime_version = rpc.runtime_version(None).await.expect("runtime_version infallible; qed."); - - let builtin_version = any_runtime! { - Version::get() - }; - - let versions = RuntimeVersions::new(&remote_runtime_version, &builtin_version); - - if !info_opts.json { - println!("{}", versions); - } else { - let versions = serde_json::to_string_pretty(&versions).expect("Failed serializing version info"); - println!("{}", versions); - } - Ok(()) - } - } - }; - log::info!(target: LOG_TARGET, "round of execution finished. outcome = {:?}", outcome); - - handle.close(); - let _ = signals_task.await; -} - -#[cfg(test)] -mod tests { - use super::*; - - fn get_version() -> sp_version::RuntimeVersion { - T::Version::get() - } - - #[test] - fn any_runtime_works() { - unsafe { - RUNTIME = AnyRuntime::Polkadot; - } - let polkadot_version = any_runtime! { get_version::() }; - - unsafe { - RUNTIME = AnyRuntime::Kusama; - } - let kusama_version = any_runtime! { get_version::() }; - - assert_eq!(polkadot_version.spec_name, "polkadot".into()); - assert_eq!(kusama_version.spec_name, "kusama".into()); - } -} diff --git a/polkadot/utils/staking-miner/src/monitor.rs b/polkadot/utils/staking-miner/src/monitor.rs deleted file mode 100644 index 607ecb6baa42..000000000000 --- a/polkadot/utils/staking-miner/src/monitor.rs +++ /dev/null @@ -1,478 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! The monitor command. - -use crate::{ - prelude::*, rpc::*, signer::Signer, Error, MonitorConfig, SharedRpcClient, SubmissionStrategy, -}; -use codec::Encode; -use jsonrpsee::core::Error as RpcError; -use sc_transaction_pool_api::TransactionStatus; -use sp_core::storage::StorageKey; -use sp_runtime::Perbill; -use std::sync::Arc; -use tokio::sync::{mpsc, Mutex}; -use EPM::{signed::SubmissionIndicesOf, SignedSubmissionOf}; - -/// Ensure that now is the signed phase. -async fn ensure_signed_phase>( - rpc: &SharedRpcClient, - at: B::Hash, -) -> Result<(), Error> { - let key = StorageKey(EPM::CurrentPhase::::hashed_key().to_vec()); - let phase = rpc - .get_storage_and_decode::>(&key, Some(at)) - .await - .map_err::, _>(Into::into)? - .unwrap_or_default(); - - if phase.is_signed() { - Ok(()) - } else { - Err(Error::IncorrectPhase) - } -} - -/// Ensure that our current `us` have not submitted anything previously. -async fn ensure_no_previous_solution( - rpc: &SharedRpcClient, - at: Hash, - us: &AccountId, -) -> Result<(), Error> -where - T: EPM::Config + frame_system::Config, - B: BlockT, -{ - let indices_key = StorageKey(EPM::SignedSubmissionIndices::::hashed_key().to_vec()); - - let indices: SubmissionIndicesOf = rpc - .get_storage_and_decode(&indices_key, Some(at)) - .await - .map_err::, _>(Into::into)? - .unwrap_or_default(); - - for (_score, _bn, idx) in indices { - let key = StorageKey(EPM::SignedSubmissionsMap::::hashed_key_for(idx)); - - if let Some(submission) = rpc - .get_storage_and_decode::>(&key, Some(at)) - .await - .map_err::, _>(Into::into)? - { - if &submission.who == us { - return Err(Error::AlreadySubmitted) - } - } - } - - Ok(()) -} - -/// `true` if `our_score` should pass the onchain `best_score` with the given strategy. -pub(crate) fn score_passes_strategy( - our_score: sp_npos_elections::ElectionScore, - best_score: sp_npos_elections::ElectionScore, - strategy: SubmissionStrategy, -) -> bool { - match strategy { - SubmissionStrategy::Always => true, - SubmissionStrategy::IfLeading => - our_score == best_score || - our_score.strict_threshold_better(best_score, Perbill::zero()), - SubmissionStrategy::ClaimBetterThan(epsilon) => - our_score.strict_threshold_better(best_score, epsilon), - SubmissionStrategy::ClaimNoWorseThan(epsilon) => - !best_score.strict_threshold_better(our_score, epsilon), - } -} - -/// Reads all current solutions and checks the scores according to the `SubmissionStrategy`. -async fn ensure_strategy_met( - rpc: &SharedRpcClient, - at: Hash, - score: sp_npos_elections::ElectionScore, - strategy: SubmissionStrategy, - max_submissions: u32, -) -> Result<(), Error> { - // don't care about current scores. - if matches!(strategy, SubmissionStrategy::Always) { - return Ok(()) - } - - let indices_key = StorageKey(EPM::SignedSubmissionIndices::::hashed_key().to_vec()); - - let indices: SubmissionIndicesOf = rpc - .get_storage_and_decode(&indices_key, Some(at)) - .await - .map_err::, _>(Into::into)? - .unwrap_or_default(); - - if indices.len() >= max_submissions as usize { - log::debug!(target: LOG_TARGET, "The submissions queue is full"); - } - - // default score is all zeros, any score is better than it. - let best_score = indices.last().map(|(score, _, _)| *score).unwrap_or_default(); - log::debug!(target: LOG_TARGET, "best onchain score is {:?}", best_score); - - if score_passes_strategy(score, best_score, strategy) { - Ok(()) - } else { - Err(Error::StrategyNotSatisfied) - } -} - -async fn get_latest_head( - rpc: &SharedRpcClient, - mode: &str, -) -> Result> { - if mode == "head" { - match rpc.block_hash(None).await { - Ok(Some(hash)) => Ok(hash), - Ok(None) => Err(Error::Other("Best head not found".into())), - Err(e) => Err(e.into()), - } - } else { - rpc.finalized_head().await.map_err(Into::into) - } -} - -macro_rules! monitor_cmd_for { ($runtime:tt) => { paste::paste! { - - /// The monitor command. - pub(crate) async fn []( - rpc: SharedRpcClient, - config: MonitorConfig, - signer: Signer, - ) -> Result<(), Error<$crate::[<$runtime _runtime_exports>]::Runtime>> { - use $crate::[<$runtime _runtime_exports>]::*; - type StakingMinerError = Error<$crate::[<$runtime _runtime_exports>]::Runtime>; - - let heads_subscription = || - if config.listen == "head" { - rpc.subscribe_new_heads() - } else { - rpc.subscribe_finalized_heads() - }; - - let mut subscription = heads_subscription().await?; - let (tx, mut rx) = mpsc::unbounded_channel::(); - let submit_lock = Arc::new(Mutex::new(())); - - loop { - let at = tokio::select! { - maybe_rp = subscription.next() => { - match maybe_rp { - Some(Ok(r)) => r, - Some(Err(e)) => { - log::error!(target: LOG_TARGET, "subscription failed to decode Header {:?}, this is bug please file an issue", e); - return Err(e.into()); - } - // The subscription was dropped, should only happen if: - // - the connection was closed. - // - the subscription could not keep up with the server. - None => { - log::warn!(target: LOG_TARGET, "subscription to `subscribeNewHeads/subscribeFinalizedHeads` terminated. Retrying.."); - subscription = heads_subscription().await?; - continue - } - } - }, - maybe_err = rx.recv() => { - match maybe_err { - Some(err) => return Err(err), - None => unreachable!("at least one sender kept in the main loop should always return Some; qed"), - } - } - }; - - // Spawn task and non-recoverable errors are sent back to the main task - // such as if the connection has been closed. - tokio::spawn( - send_and_watch_extrinsic(rpc.clone(), tx.clone(), at, signer.clone(), config.clone(), submit_lock.clone()) - ); - } - - /// Construct extrinsic at given block and watch it. - async fn send_and_watch_extrinsic( - rpc: SharedRpcClient, - tx: mpsc::UnboundedSender, - at: Header, - signer: Signer, - config: MonitorConfig, - submit_lock: Arc>, - ) { - - async fn flatten( - handle: tokio::task::JoinHandle> - ) -> Result { - match handle.await { - Ok(Ok(result)) => Ok(result), - Ok(Err(err)) => Err(err), - Err(err) => panic!("tokio spawn task failed; kill task: {:?}", err), - } - } - - let hash = at.hash(); - log::trace!(target: LOG_TARGET, "new event at #{:?} ({:?})", at.number, hash); - - // block on this because if this fails there is no way to recover from - // that error i.e, upgrade/downgrade required. - if let Err(err) = crate::check_versions::(&rpc, false).await { - let _ = tx.send(err.into()); - return; - } - - let rpc1 = rpc.clone(); - let rpc2 = rpc.clone(); - let account = signer.account.clone(); - - let signed_phase_fut = tokio::spawn(async move { - ensure_signed_phase::(&rpc1, hash).await - }); - - tokio::time::sleep(std::time::Duration::from_secs(config.delay as u64)).await; - - let no_prev_sol_fut = tokio::spawn(async move { - ensure_no_previous_solution::(&rpc2, hash, &account).await - }); - - // Run the calls in parallel and return once all has completed or any failed. - if let Err(err) = tokio::try_join!(flatten(signed_phase_fut), flatten(no_prev_sol_fut)) { - log::debug!(target: LOG_TARGET, "Skipping block {}; {}", at.number, err); - return; - } - - let _lock = submit_lock.lock().await; - - let mut ext = match crate::create_election_ext::(rpc.clone(), Some(hash), vec![]).await { - Ok(ext) => ext, - Err(err) => { - log::debug!(target: LOG_TARGET, "Skipping block {}; {}", at.number, err); - return; - } - }; - - // mine a solution, and run feasibility check on it as well. - let raw_solution = match crate::mine_with::(&config.solver, &mut ext, true) { - Ok(r) => r, - Err(err) => { - let _ = tx.send(err.into()); - return; - } - }; - - let score = raw_solution.score; - log::info!(target: LOG_TARGET, "mined solution with {:?}", score); - - let nonce = match crate::get_account_info::(&rpc, &signer.account, Some(hash)).await { - Ok(maybe_account) => { - let acc = maybe_account.expect(crate::signer::SIGNER_ACCOUNT_WILL_EXIST); - acc.nonce - } - Err(err) => { - let _ = tx.send(err); - return; - } - }; - - let tip = 0 as Balance; - let period = ::BlockHashCount::get() / 2; - let current_block = at.number.saturating_sub(1); - let era = sp_runtime::generic::Era::mortal(period.into(), current_block.into()); - - log::trace!( - target: LOG_TARGET, "transaction mortality: {:?} -> {:?}", - era.birth(current_block.into()), - era.death(current_block.into()), - ); - - let extrinsic = ext.execute_with(|| create_uxt(raw_solution, signer.clone(), nonce, tip, era)); - let bytes = sp_core::Bytes(extrinsic.encode()); - - let rpc1 = rpc.clone(); - let rpc2 = rpc.clone(); - let rpc3 = rpc.clone(); - - let latest_head = match get_latest_head::(&rpc, &config.listen).await { - Ok(hash) => hash, - Err(e) => { - log::debug!(target: LOG_TARGET, "Skipping to submit at block {}; {}", at.number, e); - return; - } - }; - - let ensure_strategy_met_fut = tokio::spawn(async move { - ensure_strategy_met::( - &rpc1, - latest_head, - score, - config.submission_strategy, - SignedMaxSubmissions::get() - ).await - }); - - let ensure_signed_phase_fut = tokio::spawn(async move { - ensure_signed_phase::(&rpc2, latest_head).await - }); - - let account = signer.account.clone(); - let no_prev_sol_fut = tokio::spawn(async move { - ensure_no_previous_solution::(&rpc3, latest_head, &account).await - }); - - // Run the calls in parallel and return once all has completed or any failed. - if let Err(err) = tokio::try_join!( - flatten(ensure_strategy_met_fut), - flatten(ensure_signed_phase_fut), - flatten(no_prev_sol_fut), - ) { - log::debug!(target: LOG_TARGET, "Skipping to submit at block {}; {}", at.number, err); - return; - } - - let mut tx_subscription = match rpc.watch_extrinsic(&bytes).await { - Ok(sub) => sub, - Err(RpcError::RestartNeeded(e)) => { - let _ = tx.send(RpcError::RestartNeeded(e).into()); - return - }, - Err(why) => { - // This usually happens when we've been busy with mining for a few blocks, and - // now we're receiving the subscriptions of blocks in which we were busy. In - // these blocks, we still don't have a solution, so we re-compute a new solution - // and submit it with an outdated `Nonce`, which yields most often `Stale` - // error. NOTE: to improve this overall, and to be able to introduce an array of - // other fancy features, we should make this multi-threaded and do the - // computation outside of this callback. - log::warn!( - target: LOG_TARGET, - "failing to submit a transaction {:?}. ignore block: {}", - why, at.number - ); - return; - }, - }; - - while let Some(rp) = tx_subscription.next().await { - let status_update = match rp { - Ok(r) => r, - Err(e) => { - log::error!(target: LOG_TARGET, "subscription failed to decode TransactionStatus {:?}, this is a bug please file an issue", e); - let _ = tx.send(e.into()); - return; - }, - }; - - log::trace!(target: LOG_TARGET, "status update {:?}", status_update); - match status_update { - TransactionStatus::Ready | - TransactionStatus::Broadcast(_) | - TransactionStatus::Future => continue, - TransactionStatus::InBlock((hash, _)) => { - log::info!(target: LOG_TARGET, "included at {:?}", hash); - let key = StorageKey( - frame_support::storage::storage_prefix(b"System", b"Events").to_vec(), - ); - - let events = match rpc.get_storage_and_decode::< - Vec::Hash>>, - >(&key, Some(hash)) - .await { - Ok(rp) => rp.unwrap_or_default(), - Err(RpcHelperError::JsonRpsee(RpcError::RestartNeeded(e))) => { - let _ = tx.send(RpcError::RestartNeeded(e).into()); - return; - } - // Decoding or other RPC error => just terminate the task. - Err(e) => { - log::warn!(target: LOG_TARGET, "get_storage [key: {:?}, hash: {:?}] failed: {:?}; skip block: {}", - key, hash, e, at.number - ); - return; - } - }; - - log::info!(target: LOG_TARGET, "events at inclusion {:?}", events); - }, - TransactionStatus::Retracted(hash) => { - log::info!(target: LOG_TARGET, "Retracted at {:?}", hash); - }, - TransactionStatus::Finalized((hash, _)) => { - log::info!(target: LOG_TARGET, "Finalized at {:?}", hash); - break - }, - _ => { - log::warn!( - target: LOG_TARGET, - "Stopping listen due to other status {:?}", - status_update - ); - break - }, - }; - } - } - } -}}} - -monitor_cmd_for!(polkadot); -monitor_cmd_for!(kusama); -monitor_cmd_for!(westend); - -#[cfg(test)] -pub mod tests { - use super::*; - - #[test] - fn score_passes_strategy_works() { - let s = |x| sp_npos_elections::ElectionScore { minimal_stake: x, ..Default::default() }; - let two = Perbill::from_percent(2); - - // anything passes Always - assert!(score_passes_strategy(s(0), s(0), SubmissionStrategy::Always)); - assert!(score_passes_strategy(s(5), s(0), SubmissionStrategy::Always)); - assert!(score_passes_strategy(s(5), s(10), SubmissionStrategy::Always)); - - // if leading - assert!(score_passes_strategy(s(0), s(0), SubmissionStrategy::IfLeading)); - assert!(score_passes_strategy(s(1), s(0), SubmissionStrategy::IfLeading)); - assert!(score_passes_strategy(s(2), s(0), SubmissionStrategy::IfLeading)); - assert!(!score_passes_strategy(s(5), s(10), SubmissionStrategy::IfLeading)); - assert!(!score_passes_strategy(s(9), s(10), SubmissionStrategy::IfLeading)); - assert!(score_passes_strategy(s(10), s(10), SubmissionStrategy::IfLeading)); - - // if better by 2% - assert!(!score_passes_strategy(s(50), s(100), SubmissionStrategy::ClaimBetterThan(two))); - assert!(!score_passes_strategy(s(100), s(100), SubmissionStrategy::ClaimBetterThan(two))); - assert!(!score_passes_strategy(s(101), s(100), SubmissionStrategy::ClaimBetterThan(two))); - assert!(!score_passes_strategy(s(102), s(100), SubmissionStrategy::ClaimBetterThan(two))); - assert!(score_passes_strategy(s(103), s(100), SubmissionStrategy::ClaimBetterThan(two))); - assert!(score_passes_strategy(s(150), s(100), SubmissionStrategy::ClaimBetterThan(two))); - - // if no less than 2% worse - assert!(!score_passes_strategy(s(50), s(100), SubmissionStrategy::ClaimNoWorseThan(two))); - assert!(!score_passes_strategy(s(97), s(100), SubmissionStrategy::ClaimNoWorseThan(two))); - assert!(score_passes_strategy(s(98), s(100), SubmissionStrategy::ClaimNoWorseThan(two))); - assert!(score_passes_strategy(s(99), s(100), SubmissionStrategy::ClaimNoWorseThan(two))); - assert!(score_passes_strategy(s(100), s(100), SubmissionStrategy::ClaimNoWorseThan(two))); - assert!(score_passes_strategy(s(101), s(100), SubmissionStrategy::ClaimNoWorseThan(two))); - assert!(score_passes_strategy(s(102), s(100), SubmissionStrategy::ClaimNoWorseThan(two))); - assert!(score_passes_strategy(s(103), s(100), SubmissionStrategy::ClaimNoWorseThan(two))); - assert!(score_passes_strategy(s(150), s(100), SubmissionStrategy::ClaimNoWorseThan(two))); - } -} diff --git a/polkadot/utils/staking-miner/src/opts.rs b/polkadot/utils/staking-miner/src/opts.rs deleted file mode 100644 index 4cf4d0a76519..000000000000 --- a/polkadot/utils/staking-miner/src/opts.rs +++ /dev/null @@ -1,366 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -use crate::prelude::*; -use clap::Parser; -use sp_runtime::Perbill; -use std::str::FromStr; - -#[derive(Debug, Clone, Parser)] -#[cfg_attr(test, derive(PartialEq))] -#[command(author, version, about)] -pub(crate) struct Opt { - /// The `ws` node to connect to. - #[arg(long, short, default_value = DEFAULT_URI, env = "URI", global = true)] - pub uri: String, - - /// WS connection timeout in number of seconds. - #[arg(long, default_value_t = 60)] - pub connection_timeout: usize, - - /// WS request timeout in number of seconds. - #[arg(long, default_value_t = 60 * 10)] - pub request_timeout: usize, - - #[command(subcommand)] - pub command: Command, -} - -#[derive(Debug, Clone, Parser)] -#[cfg_attr(test, derive(PartialEq))] -pub(crate) enum Command { - /// Monitor for the phase being signed, then compute. - Monitor(MonitorConfig), - - /// Just compute a solution now, and don't submit it. - DryRun(DryRunConfig), - - /// Provide a solution that can be submitted to the chain as an emergency response. - EmergencySolution(EmergencySolutionConfig), - - /// Return information about the current version - Info(InfoOpts), -} - -#[derive(Debug, Clone, Parser)] -#[cfg_attr(test, derive(PartialEq))] -pub(crate) struct MonitorConfig { - /// The path to a file containing the seed of the account. If the file is not found, the seed - /// is used as-is. - /// - /// Can also be provided via the `SEED` environment variable. - /// - /// WARNING: Don't use an account with a large stash for this. Based on how the bot is - /// configured, it might re-try and lose funds through transaction fees/deposits. - #[arg(long, short, env = "SEED")] - pub seed_or_path: String, - - /// They type of event to listen to. - /// - /// Typically, finalized is safer and there is no chance of anything going wrong, but it can be - /// slower. It is recommended to use finalized, if the duration of the signed phase is longer - /// than the the finality delay. - #[arg(long, default_value = "head", value_parser = ["head", "finalized"])] - pub listen: String, - - /// The solver algorithm to use. - #[command(subcommand)] - pub solver: Solver, - - /// Submission strategy to use. - /// - /// Possible options: - /// - /// `--submission-strategy if-leading`: only submit if leading. - /// - /// `--submission-strategy always`: always submit. - /// - /// `--submission-strategy "percent-better percent"`: submit if the submission is `n` percent - /// better. - /// - /// `--submission-strategy "no-worse-than percent"`: submit if submission is no more than - /// `n` percent worse. - #[clap(long, default_value = "if-leading")] - pub submission_strategy: SubmissionStrategy, - - /// Delay in number seconds to wait until starting mining a solution. - /// - /// At every block when a solution is attempted - /// a delay can be enforced to avoid submitting at - /// "same time" and risk potential races with other miners. - /// - /// When this is enabled and there are competing solutions, your solution might not be - /// submitted if the scores are equal. - #[arg(long, default_value_t = 0)] - pub delay: usize, -} - -#[derive(Debug, Clone, Parser)] -#[cfg_attr(test, derive(PartialEq))] -pub(crate) struct DryRunConfig { - /// The path to a file containing the seed of the account. If the file is not found, the seed - /// is used as-is. - /// - /// Can also be provided via the `SEED` environment variable. - /// - /// WARNING: Don't use an account with a large stash for this. Based on how the bot is - /// configured, it might re-try and lose funds through transaction fees/deposits. - #[arg(long, short, env = "SEED")] - pub seed_or_path: String, - - /// The block hash at which scraping happens. If none is provided, the latest head is used. - #[arg(long)] - pub at: Option, - - /// The solver algorithm to use. - #[command(subcommand)] - pub solver: Solver, - - /// Force create a new snapshot, else expect one to exist onchain. - #[arg(long)] - pub force_snapshot: bool, -} - -#[derive(Debug, Clone, Parser)] -#[cfg_attr(test, derive(PartialEq))] -pub(crate) struct EmergencySolutionConfig { - /// The block hash at which scraping happens. If none is provided, the latest head is used. - #[arg(long)] - pub at: Option, - - /// The solver algorithm to use. - #[command(subcommand)] - pub solver: Solver, - - /// The number of top backed winners to take. All are taken, if not provided. - pub take: Option, -} - -#[derive(Debug, Clone, Parser)] -#[cfg_attr(test, derive(PartialEq))] -pub(crate) struct InfoOpts { - /// Serialize the output as json - #[arg(long, short)] - pub json: bool, -} - -/// Submission strategy to use. -#[derive(Debug, Copy, Clone)] -#[cfg_attr(test, derive(PartialEq))] -pub enum SubmissionStrategy { - /// Always submit. - Always, - /// Only submit if at the time, we are the best (or equal to it). - IfLeading, - /// Submit if we are no worse than `Perbill` worse than the best. - ClaimNoWorseThan(Perbill), - /// Submit if we are leading, or if the solution that's leading is more that the given - /// `Perbill` better than us. This helps detect obviously fake solutions and still combat them. - ClaimBetterThan(Perbill), -} - -#[derive(Debug, Clone, Parser)] -#[cfg_attr(test, derive(PartialEq))] -pub(crate) enum Solver { - SeqPhragmen { - #[arg(long, default_value_t = 10)] - iterations: usize, - }, - PhragMMS { - #[arg(long, default_value_t = 10)] - iterations: usize, - }, -} - -/// Custom `impl` to parse `SubmissionStrategy` from CLI. -/// -/// Possible options: -/// * --submission-strategy if-leading: only submit if leading -/// * --submission-strategy always: always submit -/// * --submission-strategy "percent-better percent": submit if submission is `n` percent better. -/// * --submission-strategy "no-worse-than percent": submit if submission is no more than `n` -/// percent worse. -impl FromStr for SubmissionStrategy { - type Err = String; - - fn from_str(s: &str) -> Result { - let s = s.trim(); - - let res = if s == "if-leading" { - Self::IfLeading - } else if s == "always" { - Self::Always - } else if let Some(percent) = s.strip_prefix("no-worse-than ") { - let percent: u32 = percent.parse().map_err(|e| format!("{:?}", e))?; - Self::ClaimNoWorseThan(Perbill::from_percent(percent)) - } else if let Some(percent) = s.strip_prefix("percent-better ") { - let percent: u32 = percent.parse().map_err(|e| format!("{:?}", e))?; - Self::ClaimBetterThan(Perbill::from_percent(percent)) - } else { - return Err(s.into()) - }; - Ok(res) - } -} - -#[cfg(test)] -mod test_super { - use super::*; - - #[test] - fn cli_monitor_works() { - let opt = Opt::try_parse_from([ - env!("CARGO_PKG_NAME"), - "--uri", - "hi", - "monitor", - "--seed-or-path", - "//Alice", - "--listen", - "head", - "--delay", - "12", - "seq-phragmen", - ]) - .unwrap(); - - assert_eq!( - opt, - Opt { - uri: "hi".to_string(), - connection_timeout: 60, - request_timeout: 10 * 60, - command: Command::Monitor(MonitorConfig { - seed_or_path: "//Alice".to_string(), - listen: "head".to_string(), - solver: Solver::SeqPhragmen { iterations: 10 }, - submission_strategy: SubmissionStrategy::IfLeading, - delay: 12, - }), - } - ); - } - - #[test] - fn cli_dry_run_works() { - let opt = Opt::try_parse_from([ - env!("CARGO_PKG_NAME"), - "--uri", - "hi", - "dry-run", - "--seed-or-path", - "//Alice", - "phrag-mms", - ]) - .unwrap(); - - assert_eq!( - opt, - Opt { - uri: "hi".to_string(), - connection_timeout: 60, - request_timeout: 10 * 60, - command: Command::DryRun(DryRunConfig { - seed_or_path: "//Alice".to_string(), - at: None, - solver: Solver::PhragMMS { iterations: 10 }, - force_snapshot: false, - }), - } - ); - } - - #[test] - fn cli_emergency_works() { - let opt = Opt::try_parse_from([ - env!("CARGO_PKG_NAME"), - "--uri", - "hi", - "emergency-solution", - "99", - "phrag-mms", - "--iterations", - "1337", - ]) - .unwrap(); - - assert_eq!( - opt, - Opt { - uri: "hi".to_string(), - connection_timeout: 60, - request_timeout: 10 * 60, - command: Command::EmergencySolution(EmergencySolutionConfig { - take: Some(99), - at: None, - solver: Solver::PhragMMS { iterations: 1337 } - }), - } - ); - } - - #[test] - fn cli_info_works() { - let opt = Opt::try_parse_from([env!("CARGO_PKG_NAME"), "--uri", "hi", "info"]).unwrap(); - - assert_eq!( - opt, - Opt { - uri: "hi".to_string(), - connection_timeout: 60, - request_timeout: 10 * 60, - command: Command::Info(InfoOpts { json: false }) - } - ); - } - - #[test] - fn cli_request_conn_timeout_works() { - let opt = Opt::try_parse_from([ - env!("CARGO_PKG_NAME"), - "--uri", - "hi", - "--request-timeout", - "10", - "--connection-timeout", - "9", - "info", - ]) - .unwrap(); - - assert_eq!( - opt, - Opt { - uri: "hi".to_string(), - connection_timeout: 9, - request_timeout: 10, - command: Command::Info(InfoOpts { json: false }) - } - ); - } - - #[test] - fn submission_strategy_from_str_works() { - use std::str::FromStr; - - assert_eq!(SubmissionStrategy::from_str("if-leading"), Ok(SubmissionStrategy::IfLeading)); - assert_eq!(SubmissionStrategy::from_str("always"), Ok(SubmissionStrategy::Always)); - assert_eq!( - SubmissionStrategy::from_str(" percent-better 99 "), - Ok(SubmissionStrategy::ClaimBetterThan(Perbill::from_percent(99))) - ); - } -} diff --git a/polkadot/utils/staking-miner/src/prelude.rs b/polkadot/utils/staking-miner/src/prelude.rs deleted file mode 100644 index fb701ece2384..000000000000 --- a/polkadot/utils/staking-miner/src/prelude.rs +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! Types that we don't fetch from a particular runtime and just assume that they are constant all -//! of the place. -//! -//! It is actually easy to convert the rest as well, but it'll be a lot of noise in our codebase, -//! needing to sprinkle `any_runtime` in a few extra places. - -/// The account id type. -pub type AccountId = core_primitives::AccountId; -/// The block number type. -pub type BlockNumber = core_primitives::BlockNumber; -/// The balance type. -pub type Balance = core_primitives::Balance; -/// Index of a transaction in the chain. -pub type Nonce = core_primitives::Nonce; -/// The hash type. We re-export it here, but we can easily get it from block as well. -pub type Hash = core_primitives::Hash; -/// The header type. We re-export it here, but we can easily get it from block as well. -pub type Header = core_primitives::Header; -/// The block type. -pub type Block = core_primitives::Block; - -pub use sp_runtime::traits::{Block as BlockT, Header as HeaderT}; - -/// Default URI to connect to. -pub const DEFAULT_URI: &str = "wss://rpc.polkadot.io:443"; -/// The logging target. -pub const LOG_TARGET: &str = "staking-miner"; - -/// The election provider pallet. -pub use pallet_election_provider_multi_phase as EPM; - -/// The externalities type. -pub type Ext = sp_state_machine::TestExternalities>; - -/// The key pair type being used. We "strongly" assume sr25519 for simplicity. -pub type Pair = sp_core::sr25519::Pair; - -/// A dynamic token type used to represent account balances. -pub type Token = sub_tokens::dynamic::DynamicToken; diff --git a/polkadot/utils/staking-miner/src/rpc.rs b/polkadot/utils/staking-miner/src/rpc.rs deleted file mode 100644 index 2d25616e2a17..000000000000 --- a/polkadot/utils/staking-miner/src/rpc.rs +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! JSON-RPC related types and helpers. - -use super::*; -use jsonrpsee::{ - core::{Error as RpcError, RpcResult}, - proc_macros::rpc, -}; -use pallet_transaction_payment::RuntimeDispatchInfo; -use sc_transaction_pool_api::TransactionStatus; -use sp_core::{storage::StorageKey, Bytes}; -use sp_version::RuntimeVersion; -use std::{future::Future, time::Duration}; - -#[derive(frame_support::DebugNoBound, thiserror::Error)] -pub(crate) enum RpcHelperError { - JsonRpsee(#[from] jsonrpsee::core::Error), - Codec(#[from] codec::Error), -} - -impl std::fmt::Display for RpcHelperError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - ::fmt(self, f) - } -} - -#[rpc(client)] -pub trait RpcApi { - /// Fetch system name. - #[method(name = "system_chain")] - async fn system_chain(&self) -> RpcResult; - - /// Fetch a storage key. - #[method(name = "state_getStorage")] - async fn storage(&self, key: &StorageKey, hash: Option) -> RpcResult>; - - /// Fetch the runtime version. - #[method(name = "state_getRuntimeVersion")] - async fn runtime_version(&self, at: Option) -> RpcResult; - - /// Fetch the payment query info. - #[method(name = "payment_queryInfo")] - async fn payment_query_info( - &self, - encoded_xt: &Bytes, - at: Option<&Hash>, - ) -> RpcResult>; - - /// Dry run an extrinsic at a given block. Return SCALE encoded - /// [`sp_runtime::ApplyExtrinsicResult`]. - #[method(name = "system_dryRun")] - async fn dry_run(&self, extrinsic: &Bytes, at: Option) -> RpcResult; - - /// Get hash of the n-th block in the canon chain. - /// - /// By default returns latest block hash. - #[method(name = "chain_getBlockHash", aliases = ["chain_getHead"], blocking)] - fn block_hash(&self, hash: Option) -> RpcResult>; - - /// Get hash of the last finalized block in the canon chain. - #[method(name = "chain_getFinalizedHead", aliases = ["chain_getFinalisedHead"], blocking)] - fn finalized_head(&self) -> RpcResult; - - /// Submit an extrinsic to watch. - /// - /// See [`TransactionStatus`](sc_transaction_pool_api::TransactionStatus) for details on - /// transaction life cycle. - #[subscription( - name = "author_submitAndWatchExtrinsic" => "author_extrinsicUpdate", - unsubscribe = "author_unwatchExtrinsic", - item = TransactionStatus - )] - fn watch_extrinsic(&self, bytes: &Bytes); - - /// New head subscription. - #[subscription( - name = "chain_subscribeNewHeads" => "newHead", - unsubscribe = "chain_unsubscribeNewHeads", - item = Header - )] - fn subscribe_new_heads(&self); - - /// Finalized head subscription. - #[subscription( - name = "chain_subscribeFinalizedHeads" => "chain_finalizedHead", - unsubscribe = "chain_unsubscribeFinalizedHeads", - item = Header - )] - fn subscribe_finalized_heads(&self); -} - -type Uri = String; - -/// Wraps a shared web-socket JSON-RPC client that can be cloned. -#[derive(Clone, Debug)] -pub(crate) struct SharedRpcClient(Arc, Uri); - -impl Deref for SharedRpcClient { - type Target = WsClient; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl SharedRpcClient { - /// Get the URI of the client. - pub fn uri(&self) -> &str { - &self.1 - } - - /// Create a new shared JSON-RPC web-socket client. - pub(crate) async fn new( - uri: &str, - connection_timeout: Duration, - request_timeout: Duration, - ) -> Result { - let client = WsClientBuilder::default() - .connection_timeout(connection_timeout) - .max_request_body_size(u32::MAX) - .request_timeout(request_timeout) - .max_concurrent_requests(u32::MAX as usize) - .build(uri) - .await?; - Ok(Self(Arc::new(client), uri.to_owned())) - } - - /// Get a storage item and decode it as `T`. - /// - /// # Return value: - /// - /// The function returns: - /// - /// * `Ok(Some(val))` if successful. - /// * `Ok(None)` if the storage item was not found. - /// * `Err(e)` if the JSON-RPC call failed. - pub(crate) async fn get_storage_and_decode<'a, T: codec::Decode>( - &self, - key: &StorageKey, - hash: Option, - ) -> Result, RpcHelperError> { - if let Some(bytes) = self.storage(key, hash).await? { - let decoded = ::decode(&mut &*bytes.0) - .map_err::(Into::into)?; - Ok(Some(decoded)) - } else { - Ok(None) - } - } -} - -/// Takes a future that returns `Bytes` and tries to decode those bytes into the type `Dec`. -/// Warning: don't use for storage, it will fail for non-existent storage items. -/// -/// # Return value: -/// -/// The function returns: -/// -/// * `Ok(val)` if successful. -/// * `Err(RpcHelperError::JsonRpsee)` if the JSON-RPC call failed. -/// * `Err(RpcHelperError::Codec)` if `Bytes` could not be decoded. -pub(crate) async fn await_request_and_decode<'a, Dec: codec::Decode>( - req: impl Future>, -) -> Result { - let bytes = req.await?; - Dec::decode(&mut &*bytes.0).map_err::(Into::into) -} diff --git a/polkadot/utils/staking-miner/src/runtime_versions.rs b/polkadot/utils/staking-miner/src/runtime_versions.rs deleted file mode 100644 index 38af05ead241..000000000000 --- a/polkadot/utils/staking-miner/src/runtime_versions.rs +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -use sp_version::RuntimeVersion; -use std::fmt; - -#[derive(Debug, serde::Serialize)] -pub(crate) struct RuntimeWrapper<'a>(pub &'a RuntimeVersion); - -impl<'a> fmt::Display for RuntimeWrapper<'a> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let width = 16; - - writeln!( - f, - r#" impl_name : {impl_name:>width$} - spec_name : {spec_name:>width$} - spec_version : {spec_version:>width$} - transaction_version : {transaction_version:>width$} - impl_version : {impl_version:>width$} - authoringVersion : {authoring_version:>width$} - state_version : {state_version:>width$}"#, - spec_name = self.0.spec_name.to_string(), - impl_name = self.0.impl_name.to_string(), - spec_version = self.0.spec_version, - impl_version = self.0.impl_version, - authoring_version = self.0.authoring_version, - transaction_version = self.0.transaction_version, - state_version = self.0.state_version, - ) - } -} - -impl<'a> From<&'a RuntimeVersion> for RuntimeWrapper<'a> { - fn from(r: &'a RuntimeVersion) -> Self { - RuntimeWrapper(r) - } -} - -#[derive(Debug, serde::Serialize)] -pub(crate) struct RuntimeVersions<'a> { - /// The `RuntimeVersion` linked in the staking-miner - pub linked: RuntimeWrapper<'a>, - - /// The `RuntimeVersion` reported by the node we connect to via RPC - pub remote: RuntimeWrapper<'a>, - - /// This `bool` reports whether both remote and linked `RuntimeVersion` are compatible - /// and if the staking-miner is expected to work properly against the remote runtime - compatible: bool, -} - -impl<'a> RuntimeVersions<'a> { - pub fn new( - remote_runtime_version: &'a RuntimeVersion, - linked_runtime_version: &'a RuntimeVersion, - ) -> Self { - Self { - remote: remote_runtime_version.into(), - linked: linked_runtime_version.into(), - compatible: are_runtimes_compatible(remote_runtime_version, linked_runtime_version), - } - } -} - -/// Check whether runtimes are compatible. Currently we only support equality. -fn are_runtimes_compatible(r1: &RuntimeVersion, r2: &RuntimeVersion) -> bool { - r1 == r2 -} - -impl<'a> fmt::Display for RuntimeVersions<'a> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let _ = write!(f, "- linked:\n{}", self.linked); - let _ = write!(f, "- remote :\n{}", self.remote); - write!(f, "Compatible: {}", if self.compatible { "YES" } else { "NO" }) - } -} diff --git a/polkadot/utils/staking-miner/src/signer.rs b/polkadot/utils/staking-miner/src/signer.rs deleted file mode 100644 index e6677ccd3a66..000000000000 --- a/polkadot/utils/staking-miner/src/signer.rs +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! Wrappers around creating a signer account. - -use crate::{prelude::*, rpc::SharedRpcClient, AccountId, Error, Nonce, Pair, LOG_TARGET}; -use frame_system::AccountInfo; -use sp_core::{crypto::Pair as _, storage::StorageKey}; - -pub(crate) const SIGNER_ACCOUNT_WILL_EXIST: &str = - "signer account is checked to exist upon startup; it can only die if it transfers funds out \ - of it, or get slashed. If it does not exist at this point, it is likely due to a bug, or the \ - signer got slashed. Terminating."; - -/// Some information about the signer. Redundant at this point, but makes life easier. -#[derive(Clone)] -pub(crate) struct Signer { - /// The account id. - pub(crate) account: AccountId, - - /// The full crypto key-pair. - pub(crate) pair: Pair, -} - -pub(crate) async fn get_account_info + EPM::Config>( - rpc: &SharedRpcClient, - who: &T::AccountId, - maybe_at: Option, -) -> Result>, Error> { - rpc.get_storage_and_decode::>( - &StorageKey(>::hashed_key_for(&who)), - maybe_at, - ) - .await - .map_err(Into::into) -} - -/// Read the signer account's URI -pub(crate) async fn signer_uri_from_string< - T: frame_system::Config< - AccountId = AccountId, - Nonce = Nonce, - AccountData = pallet_balances::AccountData, - Hash = Hash, - > + EPM::Config, ->( - mut seed_or_path: &str, - client: &SharedRpcClient, -) -> Result> { - seed_or_path = seed_or_path.trim(); - - let seed = match std::fs::read(seed_or_path) { - Ok(s) => String::from_utf8(s).map_err(|_| Error::::AccountDoesNotExists)?, - Err(_) => seed_or_path.to_string(), - }; - let seed = seed.trim(); - - let pair = Pair::from_string(seed, None)?; - let account = T::AccountId::from(pair.public()); - let _info = get_account_info::(client, &account, None) - .await? - .ok_or(Error::::AccountDoesNotExists)?; - log::info!( - target: LOG_TARGET, - "loaded account {:?}, free: {:?}, info: {:?}", - &account, - Token::from(_info.data.free), - _info - ); - Ok(Signer { account, pair }) -} diff --git a/polkadot/utils/staking-miner/tests/cli.rs b/polkadot/utils/staking-miner/tests/cli.rs deleted file mode 100644 index 1ced1239e553..000000000000 --- a/polkadot/utils/staking-miner/tests/cli.rs +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -use assert_cmd::{cargo::cargo_bin, Command}; -use serde_json::{Result, Value}; - -#[test] -fn cli_version_works() { - let crate_name = env!("CARGO_PKG_NAME"); - let output = Command::new(cargo_bin(crate_name)).arg("--version").output().unwrap(); - - assert!(output.status.success(), "command returned with non-success exit code"); - let version = String::from_utf8_lossy(&output.stdout).trim().to_owned(); - - assert_eq!(version, format!("{} {}", crate_name, env!("CARGO_PKG_VERSION"))); -} - -#[test] -fn cli_info_works() { - let crate_name = env!("CARGO_PKG_NAME"); - let output = Command::new(cargo_bin(crate_name)) - .arg("info") - .arg("--json") - .env("RUST_LOG", "none") - .output() - .unwrap(); - - assert!(output.status.success(), "command returned with non-success exit code"); - let info = String::from_utf8_lossy(&output.stdout).trim().to_owned(); - let v: Result = serde_json::from_str(&info); - let v = v.unwrap(); - assert!(!v["builtin"].to_string().is_empty()); - assert!(!v["builtin"]["spec_name"].to_string().is_empty()); - assert!(!v["builtin"]["spec_version"].to_string().is_empty()); - assert!(!v["remote"].to_string().is_empty()); -} diff --git a/polkadot/xcm/pallet-xcm/Cargo.toml b/polkadot/xcm/pallet-xcm/Cargo.toml index 17ee9840e8f4..4946a8d11ce9 100644 --- a/polkadot/xcm/pallet-xcm/Cargo.toml +++ b/polkadot/xcm/pallet-xcm/Cargo.toml @@ -32,8 +32,6 @@ xcm-builder = { package = "staging-xcm-builder", path = "../xcm-builder" } [features] default = [ "std" ] -# Enable `VersionedMigration` for the migrations using the experimental feature. -experimental = [ "frame-support/experimental" ] std = [ "bounded-collections/std", "codec/std", diff --git a/polkadot/xcm/pallet-xcm/src/migration.rs b/polkadot/xcm/pallet-xcm/src/migration.rs index e28ca56563c6..ba3cdb5c51ed 100644 --- a/polkadot/xcm/pallet-xcm/src/migration.rs +++ b/polkadot/xcm/pallet-xcm/src/migration.rs @@ -65,7 +65,6 @@ pub mod v1 { /// /// Wrapped in [`frame_support::migrations::VersionedMigration`] so the pre/post checks don't /// begin failing after the upgrade is enacted on-chain. - #[cfg(feature = "experimental")] pub type VersionCheckedMigrateToV1 = frame_support::migrations::VersionedMigration< 0, 1, diff --git a/polkadot/xcm/procedural/Cargo.toml b/polkadot/xcm/procedural/Cargo.toml index 6545fb68764d..3e137c42843b 100644 --- a/polkadot/xcm/procedural/Cargo.toml +++ b/polkadot/xcm/procedural/Cargo.toml @@ -11,5 +11,5 @@ proc-macro = true [dependencies] proc-macro2 = "1.0.56" quote = "1.0.28" -syn = "2.0.31" +syn = "2.0.33" Inflector = "0.11.4" diff --git a/polkadot/xcm/xcm-builder/src/universal_exports.rs b/polkadot/xcm/xcm-builder/src/universal_exports.rs index 0ee627e0ee90..d1b8c8c721fe 100644 --- a/polkadot/xcm/xcm-builder/src/universal_exports.rs +++ b/polkadot/xcm/xcm-builder/src/universal_exports.rs @@ -163,15 +163,19 @@ impl, - xcm: &mut Option>, + msg: &mut Option>, ) -> SendResult { let d = dest.ok_or(MissingArgument)?; let devolved = ensure_is_remote(UniversalLocation::get(), d).map_err(|_| NotApplicable)?; let (remote_network, remote_location) = devolved; - let xcm = xcm.take().ok_or(MissingArgument)?; + let xcm = msg.take().ok_or(MissingArgument)?; - let (bridge, maybe_payment) = - Bridges::exporter_for(&remote_network, &remote_location, &xcm).ok_or(NotApplicable)?; + // find exporter + let Some((bridge, maybe_payment)) = Bridges::exporter_for(&remote_network, &remote_location, &xcm) else { + // We need to make sure that msg is not consumed in case of `NotApplicable`. + *msg = Some(xcm); + return Err(SendError::NotApplicable) + }; ensure!(maybe_payment.is_none(), Unroutable); // `xcm` should already end with `SetTopic` - if it does, then extract and derive into @@ -224,13 +228,19 @@ impl, - xcm: &mut Option>, + msg: &mut Option>, ) -> SendResult { let d = *dest.as_ref().ok_or(MissingArgument)?; let devolved = ensure_is_remote(UniversalLocation::get(), d).map_err(|_| NotApplicable)?; let (remote_network, remote_location) = devolved; + let xcm = msg.take().ok_or(MissingArgument)?; - let xcm = xcm.take().ok_or(MissingArgument)?; + // find exporter + let Some((bridge, maybe_payment)) = Bridges::exporter_for(&remote_network, &remote_location, &xcm) else { + // We need to make sure that msg is not consumed in case of `NotApplicable`. + *msg = Some(xcm); + return Err(SendError::NotApplicable) + }; // `xcm` should already end with `SetTopic` - if it does, then extract and derive into // an onward topic ID. @@ -239,9 +249,6 @@ impl None, }; - let (bridge, maybe_payment) = - Bridges::exporter_for(&remote_network, &remote_location, &xcm).ok_or(NotApplicable)?; - let local_from_bridge = UniversalLocation::get().invert_target(&bridge).map_err(|_| Unroutable)?; let export_instruction = @@ -452,4 +459,80 @@ mod tests { let x = ensure_is_remote((), (Parent, Polkadot, Parachain(1000))); assert_eq!(x, Err((Parent, Polkadot, Parachain(1000)).into())); } + + pub struct OkSender; + impl SendXcm for OkSender { + type Ticket = (); + + fn validate( + _destination: &mut Option, + _message: &mut Option>, + ) -> SendResult { + Ok(((), MultiAssets::new())) + } + + fn deliver(_ticket: Self::Ticket) -> Result { + Ok([0; 32]) + } + } + + /// Generic test case asserting that dest and msg is not consumed by `validate` implementation + /// of `SendXcm` in case of expected result. + fn ensure_validate_does_not_consume_dest_or_msg( + dest: MultiLocation, + assert_result: impl Fn(SendResult), + ) { + let mut dest_wrapper = Some(dest); + let msg = Xcm::<()>::new(); + let mut msg_wrapper = Some(msg.clone()); + + assert_result(S::validate(&mut dest_wrapper, &mut msg_wrapper)); + + // ensure dest and msg are untouched + assert_eq!(Some(dest), dest_wrapper); + assert_eq!(Some(msg), msg_wrapper); + } + + #[test] + fn remote_exporters_does_not_consume_dest_or_msg_on_not_applicable() { + frame_support::parameter_types! { + pub Local: NetworkId = ByGenesis([0; 32]); + pub UniversalLocation: InteriorMultiLocation = X2(GlobalConsensus(Local::get()), Parachain(1234)); + pub DifferentRemote: NetworkId = ByGenesis([22; 32]); + // no routers + pub BridgeTable: Vec<(NetworkId, MultiLocation, Option)> = vec![]; + } + + // check with local destination (should be remote) + let local_dest = (Parent, Parachain(5678)).into(); + assert!(ensure_is_remote(UniversalLocation::get(), local_dest).is_err()); + + ensure_validate_does_not_consume_dest_or_msg::< + UnpaidRemoteExporter, OkSender, UniversalLocation>, + >(local_dest, |result| assert_eq!(Err(NotApplicable), result)); + + ensure_validate_does_not_consume_dest_or_msg::< + SovereignPaidRemoteExporter< + NetworkExportTable, + OkSender, + UniversalLocation, + >, + >(local_dest, |result| assert_eq!(Err(NotApplicable), result)); + + // check with not applicable destination + let remote_dest = (Parent, Parent, DifferentRemote::get()).into(); + assert!(ensure_is_remote(UniversalLocation::get(), remote_dest).is_ok()); + + ensure_validate_does_not_consume_dest_or_msg::< + UnpaidRemoteExporter, OkSender, UniversalLocation>, + >(remote_dest, |result| assert_eq!(Err(NotApplicable), result)); + + ensure_validate_does_not_consume_dest_or_msg::< + SovereignPaidRemoteExporter< + NetworkExportTable, + OkSender, + UniversalLocation, + >, + >(remote_dest, |result| assert_eq!(Err(NotApplicable), result)); + } } diff --git a/cumulus/scripts/bridges_update_subtree.sh b/scripts/bridges_update_subtree.sh similarity index 100% rename from cumulus/scripts/bridges_update_subtree.sh rename to scripts/bridges_update_subtree.sh diff --git a/substrate/bin/node-template/node/Cargo.toml b/substrate/bin/node-template/node/Cargo.toml index 38eba5ee0e1f..b510ed34c23a 100644 --- a/substrate/bin/node-template/node/Cargo.toml +++ b/substrate/bin/node-template/node/Cargo.toml @@ -17,7 +17,7 @@ targets = ["x86_64-unknown-linux-gnu"] name = "node-template" [dependencies] -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } futures = { version = "0.3.21", features = ["thread-pool"]} sc-cli = { path = "../../../client/cli" } diff --git a/substrate/bin/node-template/node/src/service.rs b/substrate/bin/node-template/node/src/service.rs index 7303f5cd6dd6..403202829241 100644 --- a/substrate/bin/node-template/node/src/service.rs +++ b/substrate/bin/node-template/node/src/service.rs @@ -183,6 +183,7 @@ pub fn new_full(config: Configuration) -> Result { import_queue, block_announce_validator_builder: None, warp_sync_params: Some(WarpSyncParams::WithProvider(warp_sync)), + block_relay: None, })?; if config.offchain_worker.enabled { diff --git a/substrate/bin/node/bench/Cargo.toml b/substrate/bin/node/bench/Cargo.toml index a02d7393740a..05c13e312b21 100644 --- a/substrate/bin/node/bench/Cargo.toml +++ b/substrate/bin/node/bench/Cargo.toml @@ -13,7 +13,7 @@ publish = false [dependencies] array-bytes = "6.1" -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } log = "0.4.17" node-primitives = { path = "../primitives" } node-testing = { path = "../testing" } @@ -22,7 +22,7 @@ sc-client-api = { path = "../../../client/api" } sp-runtime = { path = "../../../primitives/runtime" } sp-state-machine = { path = "../../../primitives/state-machine" } serde = "1.0.188" -serde_json = "1.0.85" +serde_json = "1.0.107" derive_more = { version = "0.99.17", default-features = false, features = ["display"] } kvdb = "0.13.0" kvdb-rocksdb = "0.19.0" diff --git a/substrate/bin/node/cli/Cargo.toml b/substrate/bin/node/cli/Cargo.toml index 28701db482d2..84d8de11a35f 100644 --- a/substrate/bin/node/cli/Cargo.toml +++ b/substrate/bin/node/cli/Cargo.toml @@ -38,7 +38,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] # third-party dependencies array-bytes = "6.1" -clap = { version = "4.4.2", features = ["derive"], optional = true } +clap = { version = "4.4.3", features = ["derive"], optional = true } codec = { package = "parity-scale-codec", version = "3.6.1" } serde = { version = "1.0.188", features = ["derive"] } jsonrpsee = { version = "0.16.2", features = ["server"] } @@ -106,7 +106,7 @@ sc-cli = { path = "../../../client/cli", optional = true} frame-benchmarking-cli = { path = "../../../utils/frame/benchmarking-cli", optional = true} node-inspect = { path = "../inspect", optional = true} try-runtime-cli = { path = "../../../utils/frame/try-runtime/cli", optional = true} -serde_json = "1.0.85" +serde_json = "1.0.107" [dev-dependencies] sc-keystore = { path = "../../../client/keystore" } @@ -135,7 +135,7 @@ pallet-timestamp = { path = "../../../frame/timestamp" } substrate-cli-test-utils = { path = "../../../test-utils/cli" } [build-dependencies] -clap = { version = "4.4.2", optional = true } +clap = { version = "4.4.3", optional = true } clap_complete = { version = "4.0.2", optional = true } node-inspect = { path = "../inspect", optional = true} frame-benchmarking-cli = { path = "../../../utils/frame/benchmarking-cli", optional = true} diff --git a/substrate/bin/node/cli/src/service.rs b/substrate/bin/node/cli/src/service.rs index e49c60fe2fb7..977c90e73e9f 100644 --- a/substrate/bin/node/cli/src/service.rs +++ b/substrate/bin/node/cli/src/service.rs @@ -388,6 +388,7 @@ pub fn new_full_base( import_queue, block_announce_validator_builder: None, warp_sync_params: Some(WarpSyncParams::WithProvider(warp_sync)), + block_relay: None, })?; let role = config.role.clone(); diff --git a/substrate/bin/node/inspect/Cargo.toml b/substrate/bin/node/inspect/Cargo.toml index 2b6a5e1aea18..f28ceb459571 100644 --- a/substrate/bin/node/inspect/Cargo.toml +++ b/substrate/bin/node/inspect/Cargo.toml @@ -13,7 +13,7 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } codec = { package = "parity-scale-codec", version = "3.6.1" } thiserror = "1.0" sc-cli = { path = "../../../client/cli" } diff --git a/substrate/bin/utils/chain-spec-builder/Cargo.toml b/substrate/bin/utils/chain-spec-builder/Cargo.toml index fcc97e7809bb..c626a1612eb1 100644 --- a/substrate/bin/utils/chain-spec-builder/Cargo.toml +++ b/substrate/bin/utils/chain-spec-builder/Cargo.toml @@ -22,7 +22,7 @@ crate-type = ["rlib"] [dependencies] ansi_term = "0.12.1" -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } rand = "0.8" node-cli = { path = "../../node/cli" } sc-chain-spec = { path = "../../../client/chain-spec" } diff --git a/substrate/bin/utils/subkey/Cargo.toml b/substrate/bin/utils/subkey/Cargo.toml index 98276863f8de..8a5f4ec49c54 100644 --- a/substrate/bin/utils/subkey/Cargo.toml +++ b/substrate/bin/utils/subkey/Cargo.toml @@ -17,5 +17,5 @@ path = "src/main.rs" name = "subkey" [dependencies] -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } sc-cli = { path = "../../../client/cli" } diff --git a/substrate/client/chain-spec/Cargo.toml b/substrate/client/chain-spec/Cargo.toml index 6acd57ef5168..e326d84ae50c 100644 --- a/substrate/client/chain-spec/Cargo.toml +++ b/substrate/client/chain-spec/Cargo.toml @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] memmap2 = "0.5.0" serde = { version = "1.0.188", features = ["derive"] } -serde_json = "1.0.85" +serde_json = "1.0.107" sc-client-api = { path = "../api" } sc-chain-spec-derive = { path = "derive" } sc-executor = { path = "../executor" } diff --git a/substrate/client/chain-spec/derive/Cargo.toml b/substrate/client/chain-spec/derive/Cargo.toml index b5ecc6107b94..ff1ce5141e69 100644 --- a/substrate/client/chain-spec/derive/Cargo.toml +++ b/substrate/client/chain-spec/derive/Cargo.toml @@ -18,4 +18,4 @@ proc-macro = true proc-macro-crate = "1.1.3" proc-macro2 = "1.0.56" quote = "1.0.28" -syn = "2.0.31" +syn = "2.0.33" diff --git a/substrate/client/cli/Cargo.toml b/substrate/client/cli/Cargo.toml index 06967a890931..b781e8782ec7 100644 --- a/substrate/client/cli/Cargo.toml +++ b/substrate/client/cli/Cargo.toml @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] array-bytes = "6.1" chrono = "0.4.27" -clap = { version = "4.4.2", features = ["derive", "string"] } +clap = { version = "4.4.3", features = ["derive", "string"] } fdlimit = "0.2.1" futures = "0.3.21" libp2p-identity = { version = "0.1.3", features = ["peerid", "ed25519"]} @@ -26,7 +26,7 @@ rand = "0.8.5" regex = "1.6.0" rpassword = "7.0.0" serde = "1.0.188" -serde_json = "1.0.85" +serde_json = "1.0.107" thiserror = "1.0.48" tiny-bip39 = "1.0.0" tokio = { version = "1.22.0", features = ["signal", "rt-multi-thread", "parking_lot"] } diff --git a/substrate/client/consensus/babe/rpc/Cargo.toml b/substrate/client/consensus/babe/rpc/Cargo.toml index bfae5ad3fd0c..f54edb296842 100644 --- a/substrate/client/consensus/babe/rpc/Cargo.toml +++ b/substrate/client/consensus/babe/rpc/Cargo.toml @@ -30,7 +30,7 @@ sp-keystore = { path = "../../../../primitives/keystore" } sp-runtime = { path = "../../../../primitives/runtime" } [dev-dependencies] -serde_json = "1.0.85" +serde_json = "1.0.107" tokio = "1.22.0" sc-consensus = { path = "../../common" } sc-keystore = { path = "../../../keystore" } diff --git a/substrate/client/consensus/beefy/rpc/Cargo.toml b/substrate/client/consensus/beefy/rpc/Cargo.toml index f9c3b4a99dfe..74733ea9edd9 100644 --- a/substrate/client/consensus/beefy/rpc/Cargo.toml +++ b/substrate/client/consensus/beefy/rpc/Cargo.toml @@ -23,7 +23,7 @@ sp-core = { path = "../../../../primitives/core" } sp-runtime = { path = "../../../../primitives/runtime" } [dev-dependencies] -serde_json = "1.0.85" +serde_json = "1.0.107" sc-rpc = { path = "../../../rpc", features = ["test-helpers"]} substrate-test-runtime-client = { path = "../../../../test-utils/runtime/client" } tokio = { version = "1.22.0", features = ["macros"] } diff --git a/substrate/client/consensus/grandpa/Cargo.toml b/substrate/client/consensus/grandpa/Cargo.toml index dfd9916850e5..472bdd1c5b82 100644 --- a/substrate/client/consensus/grandpa/Cargo.toml +++ b/substrate/client/consensus/grandpa/Cargo.toml @@ -25,7 +25,7 @@ log = "0.4.17" parity-scale-codec = { version = "3.6.1", features = ["derive"] } parking_lot = "0.12.1" rand = "0.8.5" -serde_json = "1.0.85" +serde_json = "1.0.107" thiserror = "1.0" fork-tree = { path = "../../../utils/fork-tree" } prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../../utils/prometheus" } diff --git a/substrate/client/consensus/grandpa/src/import.rs b/substrate/client/consensus/grandpa/src/import.rs index 8481b3958472..ca5b7c400bfb 100644 --- a/substrate/client/consensus/grandpa/src/import.rs +++ b/substrate/client/consensus/grandpa/src/import.rs @@ -552,7 +552,6 @@ where .into(), )) } - assert!(block.justifications.is_some()); let mut authority_set = self.authority_set.inner_locked(); authority_set.authority_set_changes.insert(number); crate::aux_schema::update_authority_set::( diff --git a/substrate/client/keystore/Cargo.toml b/substrate/client/keystore/Cargo.toml index 9ead4265a84a..6303ff5c27e1 100644 --- a/substrate/client/keystore/Cargo.toml +++ b/substrate/client/keystore/Cargo.toml @@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] array-bytes = "6.1" parking_lot = "0.12.1" -serde_json = "1.0.85" +serde_json = "1.0.107" thiserror = "1.0" sp-application-crypto = { path = "../../primitives/application-crypto" } sp-core = { path = "../../primitives/core" } diff --git a/substrate/client/merkle-mountain-range/rpc/Cargo.toml b/substrate/client/merkle-mountain-range/rpc/Cargo.toml index 6aa32333af97..05d8547d243f 100644 --- a/substrate/client/merkle-mountain-range/rpc/Cargo.toml +++ b/substrate/client/merkle-mountain-range/rpc/Cargo.toml @@ -23,4 +23,4 @@ sp-runtime = { path = "../../../primitives/runtime" } anyhow = "1" [dev-dependencies] -serde_json = "1.0.85" +serde_json = "1.0.107" diff --git a/substrate/client/network/Cargo.toml b/substrate/client/network/Cargo.toml index 72e38ce498d3..8b188176fc5e 100644 --- a/substrate/client/network/Cargo.toml +++ b/substrate/client/network/Cargo.toml @@ -34,7 +34,7 @@ partial_sort = "0.2.0" pin-project = "1.0.12" rand = "0.8.5" serde = { version = "1.0.188", features = ["derive"] } -serde_json = "1.0.85" +serde_json = "1.0.107" smallvec = "1.11.0" thiserror = "1.0" unsigned-varint = { version = "0.7.1", features = ["futures", "asynchronous_codec"] } diff --git a/substrate/client/network/common/src/sync.rs b/substrate/client/network/common/src/sync.rs index 461c4ae411d6..5a6f90b290d2 100644 --- a/substrate/client/network/common/src/sync.rs +++ b/substrate/client/network/common/src/sync.rs @@ -22,12 +22,12 @@ pub mod message; pub mod metrics; pub mod warp; -use crate::{role::Roles, sync::message::BlockAnnounce, types::ReputationChange}; +use crate::{role::Roles, types::ReputationChange}; use futures::Stream; use libp2p_identity::PeerId; -use message::{BlockData, BlockRequest, BlockResponse}; +use message::{BlockAnnounce, BlockRequest, BlockResponse}; use sc_consensus::{import_queue::RuntimeOrigin, IncomingBlock}; use sp_consensus::BlockOrigin; use sp_runtime::{ @@ -226,28 +226,6 @@ impl fmt::Debug for OpaqueStateResponse { } } -/// Wrapper for implementation-specific block request. -/// -/// NOTE: Implementation must be able to encode and decode it for network purposes. -pub struct OpaqueBlockRequest(pub Box); - -impl fmt::Debug for OpaqueBlockRequest { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("OpaqueBlockRequest").finish() - } -} - -/// Wrapper for implementation-specific block response. -/// -/// NOTE: Implementation must be able to encode and decode it for network purposes. -pub struct OpaqueBlockResponse(pub Box); - -impl fmt::Debug for OpaqueBlockResponse { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("OpaqueBlockResponse").finish() - } -} - /// Provides high-level status of syncing. #[async_trait::async_trait] pub trait SyncStatusProvider: Send + Sync { @@ -392,13 +370,6 @@ pub trait ChainSync: Send { /// Return some key metrics. fn metrics(&self) -> Metrics; - /// Access blocks from implementation-specific block response. - fn block_response_into_blocks( - &self, - request: &BlockRequest, - response: OpaqueBlockResponse, - ) -> Result>, String>; - /// Advance the state of `ChainSync` fn poll(&mut self, cx: &mut std::task::Context) -> Poll<()>; diff --git a/substrate/client/network/sync/src/block_relay_protocol.rs b/substrate/client/network/sync/src/block_relay_protocol.rs new file mode 100644 index 000000000000..7a313458bf03 --- /dev/null +++ b/substrate/client/network/sync/src/block_relay_protocol.rs @@ -0,0 +1,72 @@ +// Copyright Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Block relay protocol related definitions. + +use futures::channel::oneshot; +use libp2p::PeerId; +use sc_network::request_responses::{ProtocolConfig, RequestFailure}; +use sc_network_common::sync::message::{BlockData, BlockRequest}; +use sp_runtime::traits::Block as BlockT; +use std::sync::Arc; + +/// The serving side of the block relay protocol. It runs a single instance +/// of the server task that processes the incoming protocol messages. +#[async_trait::async_trait] +pub trait BlockServer: Send { + /// Starts the protocol processing. + async fn run(&mut self); +} + +/// The client side stub to download blocks from peers. This is a handle +/// that can be used to initiate concurrent downloads. +#[async_trait::async_trait] +pub trait BlockDownloader: Send + Sync { + /// Performs the protocol specific sequence to fetch the blocks from the peer. + /// Output: if the download succeeds, the response is a `Vec` which is + /// in a format specific to the protocol implementation. The block data + /// can be extracted from this response using [`BlockDownloader::block_response_into_blocks`]. + async fn download_blocks( + &self, + who: PeerId, + request: BlockRequest, + ) -> Result, RequestFailure>, oneshot::Canceled>; + + /// Parses the protocol specific response to retrieve the block data. + fn block_response_into_blocks( + &self, + request: &BlockRequest, + response: Vec, + ) -> Result>, BlockResponseError>; +} + +/// Errors returned by [`BlockDownloader::block_response_into_blocks`]. +#[derive(Debug)] +pub enum BlockResponseError { + /// Failed to decode the response bytes. + DecodeFailed(String), + + /// Failed to extract the blocks from the decoded bytes. + ExtractionFailed(String), +} + +/// Block relay specific params for network creation, specified in +/// ['sc_service::BuildNetworkParams']. +pub struct BlockRelayParams { + pub server: Box>, + pub downloader: Arc>, + pub request_response_config: ProtocolConfig, +} diff --git a/substrate/client/network/sync/src/block_request_handler.rs b/substrate/client/network/sync/src/block_request_handler.rs index d90a00b37673..c24083f63287 100644 --- a/substrate/client/network/sync/src/block_request_handler.rs +++ b/substrate/client/network/sync/src/block_request_handler.rs @@ -18,29 +18,35 @@ //! `crate::request_responses::RequestResponsesBehaviour`. use crate::{ - schema::v1::{block_request::FromBlock, BlockResponse, Direction}, + block_relay_protocol::{BlockDownloader, BlockRelayParams, BlockResponseError, BlockServer}, + schema::v1::{ + block_request::FromBlock as FromBlockSchema, BlockRequest as BlockRequestSchema, + BlockResponse as BlockResponseSchema, BlockResponse, Direction, + }, + service::network::NetworkServiceHandle, MAX_BLOCKS_IN_RESPONSE, }; -use codec::{Decode, Encode}; +use codec::{Decode, DecodeAll, Encode}; use futures::{channel::oneshot, stream::StreamExt}; use libp2p::PeerId; use log::debug; use prost::Message; -use schnellru::{ByLength, LruMap}; - use sc_client_api::BlockBackend; use sc_network::{ config::ProtocolId, - request_responses::{IncomingRequest, OutgoingResponse, ProtocolConfig}, + request_responses::{ + IfDisconnected, IncomingRequest, OutgoingResponse, ProtocolConfig, RequestFailure, + }, + types::ProtocolName, }; -use sc_network_common::sync::message::BlockAttributes; +use sc_network_common::sync::message::{BlockAttributes, BlockData, BlockRequest, FromBlock}; +use schnellru::{ByLength, LruMap}; use sp_blockchain::HeaderBackend; use sp_runtime::{ generic::BlockId, traits::{Block as BlockT, Header, One, Zero}, }; - use std::{ cmp::min, hash::{Hash, Hasher}, @@ -129,7 +135,8 @@ enum SeenRequestsValue { Fulfilled(usize), } -/// Handler for incoming block requests from a remote peer. +/// The full block server implementation of [`BlockServer`]. It handles +/// the incoming block requests from a remote peer. pub struct BlockRequestHandler { client: Arc, request_receiver: async_channel::Receiver, @@ -146,11 +153,12 @@ where { /// Create a new [`BlockRequestHandler`]. pub fn new( + network: NetworkServiceHandle, protocol_id: &ProtocolId, fork_id: Option<&str>, client: Arc, num_peer_hint: usize, - ) -> (Self, ProtocolConfig) { + ) -> BlockRelayParams { // Reserve enough request slots for one request per peer when we are at the maximum // number of peers. let capacity = std::cmp::max(num_peer_hint, 1); @@ -170,11 +178,15 @@ where let capacity = ByLength::new(num_peer_hint.max(1) as u32 * 2); let seen_requests = LruMap::new(capacity); - (Self { client, request_receiver, seen_requests }, protocol_config) + BlockRelayParams { + server: Box::new(Self { client, request_receiver, seen_requests }), + downloader: Arc::new(FullBlockDownloader::new(protocol_config.name.clone(), network)), + request_response_config: protocol_config, + } } /// Run [`BlockRequestHandler`]. - pub async fn run(mut self) { + async fn process_requests(&mut self) { while let Some(request) = self.request_receiver.next().await { let IncomingRequest { peer, payload, pending_response } = request; @@ -197,11 +209,11 @@ where let request = crate::schema::v1::BlockRequest::decode(&payload[..])?; let from_block_id = match request.from_block.ok_or(HandleRequestError::MissingFromField)? { - FromBlock::Hash(ref h) => { + FromBlockSchema::Hash(ref h) => { let h = Decode::decode(&mut h.as_ref())?; BlockId::::Hash(h) }, - FromBlock::Number(ref n) => { + FromBlockSchema::Number(ref n) => { let n = Decode::decode(&mut n.as_ref())?; BlockId::::Number(n) }, @@ -448,6 +460,17 @@ where } } +#[async_trait::async_trait] +impl BlockServer for BlockRequestHandler +where + B: BlockT, + Client: HeaderBackend + BlockBackend + Send + Sync + 'static, +{ + async fn run(&mut self) { + self.process_requests().await; + } +} + #[derive(Debug, thiserror::Error)] enum HandleRequestError { #[error("Failed to decode request: {0}.")] @@ -465,3 +488,122 @@ enum HandleRequestError { #[error("Failed to send response.")] SendResponse, } + +/// The full block downloader implementation of [`BlockDownloader]. +pub struct FullBlockDownloader { + protocol_name: ProtocolName, + network: NetworkServiceHandle, +} + +impl FullBlockDownloader { + fn new(protocol_name: ProtocolName, network: NetworkServiceHandle) -> Self { + Self { protocol_name, network } + } + + /// Extracts the blocks from the response schema. + fn blocks_from_schema( + &self, + request: &BlockRequest, + response: BlockResponseSchema, + ) -> Result>, String> { + response + .blocks + .into_iter() + .map(|block_data| { + Ok(BlockData:: { + hash: Decode::decode(&mut block_data.hash.as_ref())?, + header: if !block_data.header.is_empty() { + Some(Decode::decode(&mut block_data.header.as_ref())?) + } else { + None + }, + body: if request.fields.contains(BlockAttributes::BODY) { + Some( + block_data + .body + .iter() + .map(|body| Decode::decode(&mut body.as_ref())) + .collect::, _>>()?, + ) + } else { + None + }, + indexed_body: if request.fields.contains(BlockAttributes::INDEXED_BODY) { + Some(block_data.indexed_body) + } else { + None + }, + receipt: if !block_data.receipt.is_empty() { + Some(block_data.receipt) + } else { + None + }, + message_queue: if !block_data.message_queue.is_empty() { + Some(block_data.message_queue) + } else { + None + }, + justification: if !block_data.justification.is_empty() { + Some(block_data.justification) + } else if block_data.is_empty_justification { + Some(Vec::new()) + } else { + None + }, + justifications: if !block_data.justifications.is_empty() { + Some(DecodeAll::decode_all(&mut block_data.justifications.as_ref())?) + } else { + None + }, + }) + }) + .collect::>() + .map_err(|error: codec::Error| error.to_string()) + } +} + +#[async_trait::async_trait] +impl BlockDownloader for FullBlockDownloader { + async fn download_blocks( + &self, + who: PeerId, + request: BlockRequest, + ) -> Result, RequestFailure>, oneshot::Canceled> { + // Build the request protobuf. + let bytes = BlockRequestSchema { + fields: request.fields.to_be_u32(), + from_block: match request.from { + FromBlock::Hash(h) => Some(FromBlockSchema::Hash(h.encode())), + FromBlock::Number(n) => Some(FromBlockSchema::Number(n.encode())), + }, + direction: request.direction as i32, + max_blocks: request.max.unwrap_or(0), + support_multiple_justifications: true, + } + .encode_to_vec(); + + let (tx, rx) = oneshot::channel(); + self.network.start_request( + who, + self.protocol_name.clone(), + bytes, + tx, + IfDisconnected::ImmediateError, + ); + rx.await + } + + fn block_response_into_blocks( + &self, + request: &BlockRequest, + response: Vec, + ) -> Result>, BlockResponseError> { + // Decode the response protobuf + let response_schema = BlockResponseSchema::decode(response.as_slice()) + .map_err(|error| BlockResponseError::DecodeFailed(error.to_string()))?; + + // Extract the block data from the protobuf + self.blocks_from_schema::(request, response_schema) + .map_err(|error| BlockResponseError::ExtractionFailed(error.to_string())) + } +} diff --git a/substrate/client/network/sync/src/engine.rs b/substrate/client/network/sync/src/engine.rs index 23847d16c972..c93ba89c6220 100644 --- a/substrate/client/network/sync/src/engine.rs +++ b/substrate/client/network/sync/src/engine.rs @@ -23,6 +23,7 @@ use crate::{ block_announce_validator::{ BlockAnnounceValidationResult, BlockAnnounceValidator as BlockAnnounceValidatorStream, }, + block_relay_protocol::BlockDownloader, service::{self, chain_sync::ToServiceCommand}, warp::WarpSyncParams, ChainSync, ClientError, SyncingService, @@ -300,7 +301,7 @@ where warp_sync_params: Option>, network_service: service::network::NetworkServiceHandle, import_queue: Box>, - block_request_protocol_name: ProtocolName, + block_downloader: Arc>, state_request_protocol_name: ProtocolName, warp_sync_protocol_name: Option, rx: sc_utils::mpsc::TracingUnboundedReceiver>, @@ -392,7 +393,7 @@ where metrics_registry, network_service.clone(), import_queue, - block_request_protocol_name, + block_downloader, state_request_protocol_name, warp_sync_protocol_name, )?; diff --git a/substrate/client/network/sync/src/lib.rs b/substrate/client/network/sync/src/lib.rs index df0ed2c45410..20c0034966d5 100644 --- a/substrate/client/network/sync/src/lib.rs +++ b/substrate/client/network/sync/src/lib.rs @@ -29,13 +29,14 @@ //! order to update it. use crate::{ + block_relay_protocol::{BlockDownloader, BlockResponseError}, blocks::BlockCollection, schema::v1::{StateRequest, StateResponse}, state::StateSync, warp::{WarpProofImportResult, WarpSync, WarpSyncConfig}, }; -use codec::{Decode, DecodeAll, Encode}; +use codec::Encode; use extra_requests::ExtraRequests; use futures::{channel::oneshot, task::Poll, Future, FutureExt}; use libp2p::{request_response::OutboundFailure, PeerId}; @@ -63,8 +64,8 @@ use sc_network_common::{ }, warp::{EncodedProof, WarpProofRequest, WarpSyncPhase, WarpSyncProgress}, BadPeer, ChainSync as ChainSyncT, ImportResult, Metrics, OnBlockData, OnBlockJustification, - OnStateData, OpaqueBlockRequest, OpaqueBlockResponse, OpaqueStateRequest, - OpaqueStateResponse, PeerInfo, PeerRequest, SyncMode, SyncState, SyncStatus, + OnStateData, OpaqueStateRequest, OpaqueStateResponse, PeerInfo, PeerRequest, SyncMode, + SyncState, SyncStatus, }, }; use sp_arithmetic::traits::Saturating; @@ -93,6 +94,7 @@ mod extra_requests; mod futures_stream; mod schema; +pub mod block_relay_protocol; pub mod block_request_handler; pub mod blocks; pub mod engine; @@ -320,8 +322,8 @@ pub struct ChainSync { network_service: service::network::NetworkServiceHandle, /// Protocol name used for block announcements block_announce_protocol_name: ProtocolName, - /// Protocol name used to send out block requests - block_request_protocol_name: ProtocolName, + /// Block downloader stub + block_downloader: Arc>, /// Protocol name used to send out state requests state_request_protocol_name: ProtocolName, /// Protocol name used to send out warp sync requests @@ -1167,72 +1169,6 @@ where } } - fn block_response_into_blocks( - &self, - request: &BlockRequest, - response: OpaqueBlockResponse, - ) -> Result>, String> { - let response: Box = response.0.downcast().map_err(|_error| { - "Failed to downcast opaque block response during encoding, this is an \ - implementation bug." - .to_string() - })?; - - response - .blocks - .into_iter() - .map(|block_data| { - Ok(BlockData:: { - hash: Decode::decode(&mut block_data.hash.as_ref())?, - header: if !block_data.header.is_empty() { - Some(Decode::decode(&mut block_data.header.as_ref())?) - } else { - None - }, - body: if request.fields.contains(BlockAttributes::BODY) { - Some( - block_data - .body - .iter() - .map(|body| Decode::decode(&mut body.as_ref())) - .collect::, _>>()?, - ) - } else { - None - }, - indexed_body: if request.fields.contains(BlockAttributes::INDEXED_BODY) { - Some(block_data.indexed_body) - } else { - None - }, - receipt: if !block_data.receipt.is_empty() { - Some(block_data.receipt) - } else { - None - }, - message_queue: if !block_data.message_queue.is_empty() { - Some(block_data.message_queue) - } else { - None - }, - justification: if !block_data.justification.is_empty() { - Some(block_data.justification) - } else if block_data.is_empty_justification { - Some(Vec::new()) - } else { - None - }, - justifications: if !block_data.justifications.is_empty() { - Some(DecodeAll::decode_all(&mut block_data.justifications.as_ref())?) - } else { - None - }, - }) - }) - .collect::>() - .map_err(|error: codec::Error| error.to_string()) - } - fn poll(&mut self, cx: &mut std::task::Context) -> Poll<()> { self.process_outbound_requests(); @@ -1248,30 +1184,18 @@ where } fn send_block_request(&mut self, who: PeerId, request: BlockRequest) { - let (tx, rx) = oneshot::channel(); - let opaque_req = self.create_opaque_block_request(&request); - if self.peers.contains_key(&who) { - self.pending_responses - .insert(who, Box::pin(async move { (who, PeerRequest::Block(request), rx.await) })); - } - - match self.encode_block_request(&opaque_req) { - Ok(data) => { - self.network_service.start_request( - who, - self.block_request_protocol_name.clone(), - data, - tx, - IfDisconnected::ImmediateError, - ); - }, - Err(err) => { - log::warn!( - target: LOG_TARGET, - "Failed to encode block request {opaque_req:?}: {err:?}", - ); - }, + let downloader = self.block_downloader.clone(); + self.pending_responses.insert( + who, + Box::pin(async move { + ( + who, + PeerRequest::Block(request.clone()), + downloader.download_blocks(who, request).await, + ) + }), + ); } } } @@ -1301,7 +1225,7 @@ where metrics_registry: Option<&Registry>, network_service: service::network::NetworkServiceHandle, import_queue: Box>, - block_request_protocol_name: ProtocolName, + block_downloader: Arc>, state_request_protocol_name: ProtocolName, warp_sync_protocol_name: Option, ) -> Result<(Self, NonDefaultSetConfig), ClientError> { @@ -1337,7 +1261,7 @@ where import_existing: false, gap_sync: None, network_service, - block_request_protocol_name, + block_downloader, state_request_protocol_name, warp_sync_config, warp_sync_target_block_header: None, @@ -1710,13 +1634,6 @@ where } } - fn decode_block_response(response: &[u8]) -> Result { - let response = schema::v1::BlockResponse::decode(response) - .map_err(|error| format!("Failed to decode block response: {error}"))?; - - Ok(OpaqueBlockResponse(Box::new(response))) - } - fn decode_state_response(response: &[u8]) -> Result { let response = StateResponse::decode(response) .map_err(|error| format!("Failed to decode state response: {error}"))?; @@ -1780,22 +1697,8 @@ where &mut self, peer_id: PeerId, request: BlockRequest, - response: OpaqueBlockResponse, + blocks: Vec>, ) -> Option> { - let blocks = match self.block_response_into_blocks(&request, response) { - Ok(blocks) => blocks, - Err(err) => { - debug!( - target: LOG_TARGET, - "Failed to decode block response from {}: {}", - peer_id, - err, - ); - self.network_service.report_peer(peer_id, rep::BAD_MESSAGE); - return None - }, - }; - let block_response = BlockResponse:: { id: request.id, blocks }; let blocks_range = || match ( @@ -1915,22 +1818,34 @@ where match response { Ok(Ok(resp)) => match request { PeerRequest::Block(req) => { - let response = match Self::decode_block_response(&resp[..]) { - Ok(proto) => proto, - Err(e) => { + match self.block_downloader.block_response_into_blocks(&req, resp) { + Ok(blocks) => { + if let Some(import) = self.on_block_response(id, req, blocks) { + return Poll::Ready(import) + } + }, + Err(BlockResponseError::DecodeFailed(e)) => { debug!( target: LOG_TARGET, - "Failed to decode block response from peer {id:?}: {e:?}.", + "Failed to decode block response from peer {:?}: {:?}.", + id, + e ); self.network_service.report_peer(id, rep::BAD_MESSAGE); self.network_service .disconnect_peer(id, self.block_announce_protocol_name.clone()); continue }, - }; - - if let Some(import) = self.on_block_response(id, req, response) { - return Poll::Ready(import) + Err(BlockResponseError::ExtractionFailed(e)) => { + debug!( + target: LOG_TARGET, + "Failed to extract blocks from peer response {:?}: {:?}.", + id, + e + ); + self.network_service.report_peer(id, rep::BAD_MESSAGE); + continue + }, } }, PeerRequest::State => { @@ -2010,31 +1925,6 @@ where Poll::Pending } - /// Create implementation-specific block request. - fn create_opaque_block_request(&self, request: &BlockRequest) -> OpaqueBlockRequest { - OpaqueBlockRequest(Box::new(schema::v1::BlockRequest { - fields: request.fields.to_be_u32(), - from_block: match request.from { - FromBlock::Hash(h) => Some(schema::v1::block_request::FromBlock::Hash(h.encode())), - FromBlock::Number(n) => - Some(schema::v1::block_request::FromBlock::Number(n.encode())), - }, - direction: request.direction as i32, - max_blocks: request.max.unwrap_or(0), - support_multiple_justifications: true, - })) - } - - fn encode_block_request(&self, request: &OpaqueBlockRequest) -> Result, String> { - let request: &schema::v1::BlockRequest = request.0.downcast_ref().ok_or_else(|| { - "Failed to downcast opaque block response during encoding, this is an \ - implementation bug." - .to_string() - })?; - - Ok(request.encode_to_vec()) - } - fn encode_state_request(&self, request: &OpaqueStateRequest) -> Result, String> { let request: &StateRequest = request.0.downcast_ref().ok_or_else(|| { "Failed to downcast opaque state response during encoding, this is an \ @@ -2909,7 +2799,7 @@ fn validate_blocks( #[cfg(test)] mod test { use super::*; - use crate::service::network::NetworkServiceProvider; + use crate::{mock::MockBlockDownloader, service::network::NetworkServiceProvider}; use futures::executor::block_on; use sc_block_builder::BlockBuilderProvider; use sc_network_common::{ @@ -2947,7 +2837,7 @@ mod test { None, chain_sync_network_handle, import_queue, - ProtocolName::from("block-request"), + Arc::new(MockBlockDownloader::new()), ProtocolName::from("state-request"), None, ) @@ -3013,7 +2903,7 @@ mod test { None, chain_sync_network_handle, import_queue, - ProtocolName::from("block-request"), + Arc::new(MockBlockDownloader::new()), ProtocolName::from("state-request"), None, ) @@ -3187,7 +3077,7 @@ mod test { None, chain_sync_network_handle, import_queue, - ProtocolName::from("block-request"), + Arc::new(MockBlockDownloader::new()), ProtocolName::from("state-request"), None, ) @@ -3313,7 +3203,7 @@ mod test { None, chain_sync_network_handle, import_queue, - ProtocolName::from("block-request"), + Arc::new(MockBlockDownloader::new()), ProtocolName::from("state-request"), None, ) @@ -3470,7 +3360,7 @@ mod test { None, chain_sync_network_handle, import_queue, - ProtocolName::from("block-request"), + Arc::new(MockBlockDownloader::new()), ProtocolName::from("state-request"), None, ) @@ -3612,7 +3502,7 @@ mod test { None, chain_sync_network_handle, import_queue, - ProtocolName::from("block-request"), + Arc::new(MockBlockDownloader::new()), ProtocolName::from("state-request"), None, ) @@ -3756,7 +3646,7 @@ mod test { None, chain_sync_network_handle, import_queue, - ProtocolName::from("block-request"), + Arc::new(MockBlockDownloader::new()), ProtocolName::from("state-request"), None, ) @@ -3801,7 +3691,7 @@ mod test { None, chain_sync_network_handle, import_queue, - ProtocolName::from("block-request"), + Arc::new(MockBlockDownloader::new()), ProtocolName::from("state-request"), None, ) @@ -3853,7 +3743,7 @@ mod test { None, chain_sync_network_handle, import_queue, - ProtocolName::from("block-request"), + Arc::new(MockBlockDownloader::new()), ProtocolName::from("state-request"), None, ) diff --git a/substrate/client/network/sync/src/mock.rs b/substrate/client/network/sync/src/mock.rs index d37095c17d2c..859f9fb9c54b 100644 --- a/substrate/client/network/sync/src/mock.rs +++ b/substrate/client/network/sync/src/mock.rs @@ -16,15 +16,17 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -//! Contains a mock implementation of `ChainSync` that can be used -//! for testing calls made to `ChainSync`. +//! Contains mock implementations of `ChainSync` and 'BlockDownloader'. -use futures::task::Poll; +use crate::block_relay_protocol::{BlockDownloader as BlockDownloaderT, BlockResponseError}; + +use futures::{channel::oneshot, task::Poll}; use libp2p::PeerId; +use sc_network::RequestFailure; use sc_network_common::sync::{ message::{BlockAnnounce, BlockData, BlockRequest, BlockResponse}, - BadPeer, ChainSync as ChainSyncT, Metrics, OnBlockData, OnBlockJustification, - OpaqueBlockResponse, PeerInfo, SyncStatus, + BadPeer, ChainSync as ChainSyncT, Metrics, OnBlockData, OnBlockJustification, PeerInfo, + SyncStatus, }; use sp_runtime::traits::{Block as BlockT, NumberFor}; @@ -79,11 +81,6 @@ mockall::mock! { ); fn peer_disconnected(&mut self, who: &PeerId); fn metrics(&self) -> Metrics; - fn block_response_into_blocks( - &self, - request: &BlockRequest, - response: OpaqueBlockResponse, - ) -> Result>, String>; fn poll<'a>( &mut self, cx: &mut std::task::Context<'a>, @@ -95,3 +92,21 @@ mockall::mock! { ); } } + +mockall::mock! { + pub BlockDownloader {} + + #[async_trait::async_trait] + impl BlockDownloaderT for BlockDownloader { + async fn download_blocks( + &self, + who: PeerId, + request: BlockRequest, + ) -> Result, RequestFailure>, oneshot::Canceled>; + fn block_response_into_blocks( + &self, + request: &BlockRequest, + response: Vec, + ) -> Result>, BlockResponseError>; + } +} diff --git a/substrate/client/network/test/src/lib.rs b/substrate/client/network/test/src/lib.rs index d350b0e54ae1..11505903e35d 100644 --- a/substrate/client/network/test/src/lib.rs +++ b/substrate/client/network/test/src/lib.rs @@ -798,12 +798,18 @@ pub trait TestNetFactory: Default + Sized + Send { let fork_id = Some(String::from("test-fork-id")); - let block_request_protocol_config = { - let (handler, protocol_config) = - BlockRequestHandler::new(&protocol_id, None, client.clone(), 50); - self.spawn_task(handler.run().boxed()); - protocol_config - }; + let (chain_sync_network_provider, chain_sync_network_handle) = + NetworkServiceProvider::new(); + let mut block_relay_params = BlockRequestHandler::new( + chain_sync_network_handle.clone(), + &protocol_id, + None, + client.clone(), + 50, + ); + self.spawn_task(Box::pin(async move { + block_relay_params.server.run().await; + })); let state_request_protocol_config = { let (handler, protocol_config) = @@ -848,8 +854,6 @@ pub trait TestNetFactory: Default + Sized + Send { let block_announce_validator = config .block_announce_validator .unwrap_or_else(|| Box::new(DefaultBlockAnnounceValidator)); - let (chain_sync_network_provider, chain_sync_network_handle) = - NetworkServiceProvider::new(); let (tx, rx) = sc_utils::mpsc::tracing_unbounded("mpsc_syncing_engine_protocol", 100_000); let (engine, sync_service, block_announce_config) = @@ -864,7 +868,7 @@ pub trait TestNetFactory: Default + Sized + Send { Some(warp_sync_params), chain_sync_network_handle, import_queue.service(), - block_request_protocol_config.name.clone(), + block_relay_params.downloader, state_request_protocol_config.name.clone(), Some(warp_protocol_config.name.clone()), rx, @@ -877,7 +881,7 @@ pub trait TestNetFactory: Default + Sized + Send { full_net_config.add_request_response_protocol(config); } for config in [ - block_request_protocol_config, + block_relay_params.request_response_config, state_request_protocol_config, light_client_request_protocol_config, warp_protocol_config, diff --git a/substrate/client/network/test/src/service.rs b/substrate/client/network/test/src/service.rs index 68e780545bb1..62d7f9f9d1bb 100644 --- a/substrate/client/network/test/src/service.rs +++ b/substrate/client/network/test/src/service.rs @@ -156,12 +156,18 @@ impl TestNetworkBuilder { let fork_id = Some(String::from("test-fork-id")); let mut full_net_config = FullNetworkConfiguration::new(&network_config); - let block_request_protocol_config = { - let (handler, protocol_config) = - BlockRequestHandler::new(&protocol_id, None, client.clone(), 50); - tokio::spawn(handler.run().boxed()); - protocol_config - }; + let (chain_sync_network_provider, chain_sync_network_handle) = + self.chain_sync_network.unwrap_or(NetworkServiceProvider::new()); + let mut block_relay_params = BlockRequestHandler::new( + chain_sync_network_handle.clone(), + &protocol_id, + None, + client.clone(), + 50, + ); + tokio::spawn(Box::pin(async move { + block_relay_params.server.run().await; + })); let state_request_protocol_config = { let (handler, protocol_config) = @@ -177,8 +183,6 @@ impl TestNetworkBuilder { protocol_config }; - let (chain_sync_network_provider, chain_sync_network_handle) = - self.chain_sync_network.unwrap_or(NetworkServiceProvider::new()); let (tx, rx) = sc_utils::mpsc::tracing_unbounded("mpsc_syncing_engine_protocol", 100_000); let (engine, chain_sync_service, block_announce_config) = SyncingEngine::new( Roles::from(&config::Role::Full), @@ -191,7 +195,7 @@ impl TestNetworkBuilder { None, chain_sync_network_handle, import_queue.service(), - block_request_protocol_config.name.clone(), + block_relay_params.downloader, state_request_protocol_config.name.clone(), None, rx, @@ -214,7 +218,7 @@ impl TestNetworkBuilder { } for config in [ - block_request_protocol_config, + block_relay_params.request_response_config, state_request_protocol_config, light_client_request_protocol_config, ] { diff --git a/substrate/client/rpc-api/Cargo.toml b/substrate/client/rpc-api/Cargo.toml index fb0d0a8a1f8a..a2ee090b1c23 100644 --- a/substrate/client/rpc-api/Cargo.toml +++ b/substrate/client/rpc-api/Cargo.toml @@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] codec = { package = "parity-scale-codec", version = "3.6.1" } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } serde = { version = "1.0.188", features = ["derive"] } -serde_json = "1.0.85" +serde_json = "1.0.107" thiserror = "1.0" sc-chain-spec = { path = "../chain-spec" } sc-transaction-pool-api = { path = "../transaction-pool/api" } diff --git a/substrate/client/rpc-servers/Cargo.toml b/substrate/client/rpc-servers/Cargo.toml index 94a0d815f2e6..674a8db39cb5 100644 --- a/substrate/client/rpc-servers/Cargo.toml +++ b/substrate/client/rpc-servers/Cargo.toml @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] jsonrpsee = { version = "0.16.2", features = ["server"] } log = "0.4.17" -serde_json = "1.0.85" +serde_json = "1.0.107" tokio = { version = "1.22.0", features = ["parking_lot"] } prometheus-endpoint = { package = "substrate-prometheus-endpoint", path = "../../utils/prometheus" } tower-http = { version = "0.4.0", features = ["cors"] } diff --git a/substrate/client/rpc-spec-v2/src/chain_head/api.rs b/substrate/client/rpc-spec-v2/src/chain_head/api.rs index 682cd690dd10..d93c4018b60f 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/api.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/api.rs @@ -30,7 +30,7 @@ pub trait ChainHeadApi { /// /// This method is unstable and subject to change in the future. #[subscription( - name = "chainHead_unstable_follow", + name = "chainHead_unstable_follow" => "chainHead_unstable_followEvent", unsubscribe = "chainHead_unstable_unfollow", item = FollowEvent, )] diff --git a/substrate/client/rpc-spec-v2/src/transaction/api.rs b/substrate/client/rpc-spec-v2/src/transaction/api.rs index c226ab86787e..3eb6cb204f24 100644 --- a/substrate/client/rpc-spec-v2/src/transaction/api.rs +++ b/substrate/client/rpc-spec-v2/src/transaction/api.rs @@ -29,7 +29,7 @@ pub trait TransactionApi { /// See [`TransactionEvent`](crate::transaction::event::TransactionEvent) for details on /// transaction life cycle. #[subscription( - name = "transaction_unstable_submitAndWatch" => "transaction_unstable_submitExtrinsic", + name = "transaction_unstable_submitAndWatch" => "transaction_unstable_watchEvent", unsubscribe = "transaction_unstable_unwatch", item = TransactionEvent, )] diff --git a/substrate/client/rpc/Cargo.toml b/substrate/client/rpc/Cargo.toml index c7e9977cb252..64aaa7c94aac 100644 --- a/substrate/client/rpc/Cargo.toml +++ b/substrate/client/rpc/Cargo.toml @@ -18,7 +18,7 @@ futures = "0.3.21" jsonrpsee = { version = "0.16.2", features = ["server"] } log = "0.4.17" parking_lot = "0.12.1" -serde_json = "1.0.85" +serde_json = "1.0.107" sc-block-builder = { path = "../block-builder" } sc-chain-spec = { path = "../chain-spec" } sc-client-api = { path = "../api" } diff --git a/substrate/client/service/Cargo.toml b/substrate/client/service/Cargo.toml index 87b341fe3123..ccf23bc8994b 100644 --- a/substrate/client/service/Cargo.toml +++ b/substrate/client/service/Cargo.toml @@ -35,7 +35,7 @@ futures-timer = "3.0.1" exit-future = "0.2.0" pin-project = "1.0.12" serde = "1.0.188" -serde_json = "1.0.85" +serde_json = "1.0.107" sc-keystore = { path = "../keystore" } sp-runtime = { path = "../../primitives/runtime" } sp-trie = { path = "../../primitives/trie" } diff --git a/substrate/client/service/src/builder.rs b/substrate/client/service/src/builder.rs index 917b3be8dc7c..3838accde023 100644 --- a/substrate/client/service/src/builder.rs +++ b/substrate/client/service/src/builder.rs @@ -50,10 +50,10 @@ use sc_network_bitswap::BitswapRequestHandler; use sc_network_common::role::Roles; use sc_network_light::light_client_requests::handler::LightClientRequestHandler; use sc_network_sync::{ - block_request_handler::BlockRequestHandler, engine::SyncingEngine, - service::network::NetworkServiceProvider, state_request_handler::StateRequestHandler, - warp::WarpSyncParams, warp_request_handler::RequestHandler as WarpSyncRequestHandler, - SyncingService, + block_relay_protocol::BlockRelayParams, block_request_handler::BlockRequestHandler, + engine::SyncingEngine, service::network::NetworkServiceProvider, + state_request_handler::StateRequestHandler, warp::WarpSyncParams, + warp_request_handler::RequestHandler as WarpSyncRequestHandler, SyncingService, }; use sc_rpc::{ author::AuthorApiServer, @@ -696,6 +696,9 @@ pub struct BuildNetworkParams<'a, TBl: BlockT, TExPool, TImpQu, TCl> { Option) -> Box + Send> + Send>>, /// Optional warp sync params. pub warp_sync_params: Option>, + /// User specified block relay params. If not specified, the default + /// block request handler will be used. + pub block_relay: Option>, } /// Build the network service, the network status sinks and an RPC sender. @@ -734,6 +737,7 @@ where import_queue, block_announce_validator_builder, warp_sync_params, + block_relay, } = params; if warp_sync_params.is_none() && config.network.sync_mode.is_warp() { @@ -757,19 +761,26 @@ where Box::new(DefaultBlockAnnounceValidator) }; - let (block_request_protocol_config, block_request_protocol_name) = { - // Allow both outgoing and incoming requests. - let (handler, protocol_config) = BlockRequestHandler::new( - &protocol_id, - config.chain_spec.fork_id(), - client.clone(), - net_config.network_config.default_peers_set.in_peers as usize + - net_config.network_config.default_peers_set.out_peers as usize, - ); - let config_name = protocol_config.name.clone(); - spawn_handle.spawn("block-request-handler", Some("networking"), handler.run()); - (protocol_config, config_name) + let (chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); + let (mut block_server, block_downloader, block_request_protocol_config) = match block_relay { + Some(params) => (params.server, params.downloader, params.request_response_config), + None => { + // Custom protocol was not specified, use the default block handler. + // Allow both outgoing and incoming requests. + let params = BlockRequestHandler::new( + chain_sync_network_handle.clone(), + &protocol_id, + config.chain_spec.fork_id(), + client.clone(), + config.network.default_peers_set.in_peers as usize + + config.network.default_peers_set.out_peers as usize, + ); + (params.server, params.downloader, params.request_response_config) + }, }; + spawn_handle.spawn("block-request-handler", Some("networking"), async move { + block_server.run().await; + }); let (state_request_protocol_config, state_request_protocol_name) = { let num_peer_hint = net_config.network_config.default_peers_set_num_full as usize + @@ -860,7 +871,6 @@ where spawn_handle.spawn("peer-store", Some("networking"), peer_store.run()); let (tx, rx) = sc_utils::mpsc::tracing_unbounded("mpsc_syncing_engine_protocol", 100_000); - let (chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); let (engine, sync_service, block_announce_config) = SyncingEngine::new( Roles::from(&config.role), client.clone(), @@ -872,7 +882,7 @@ where warp_sync_params, chain_sync_network_handle, import_queue.service(), - block_request_protocol_name, + block_downloader, state_request_protocol_name, warp_request_protocol_name, rx, diff --git a/substrate/client/service/src/client/client.rs b/substrate/client/service/src/client/client.rs index a0983d823e5b..09c1673884aa 100644 --- a/substrate/client/service/src/client/client.rs +++ b/substrate/client/service/src/client/client.rs @@ -603,8 +603,6 @@ where .block_gap .map_or(false, |(start, _)| *import_headers.post().number() == start); - assert!(justifications.is_some() && finalized || justifications.is_none() || gap_block); - // the block is lower than our last finalized block so it must revert // finality, refusing import. if status == blockchain::BlockStatus::Unknown && diff --git a/substrate/client/storage-monitor/Cargo.toml b/substrate/client/storage-monitor/Cargo.toml index 5354819e3a38..6d425749a91b 100644 --- a/substrate/client/storage-monitor/Cargo.toml +++ b/substrate/client/storage-monitor/Cargo.toml @@ -9,7 +9,7 @@ description = "Storage monitor service for substrate" homepage = "https://substrate.io" [dependencies] -clap = { version = "4.4.2", features = ["derive", "string"] } +clap = { version = "4.4.3", features = ["derive", "string"] } log = "0.4.17" fs4 = "0.6.3" sc-client-db = { path = "../db", default-features = false} diff --git a/substrate/client/sync-state-rpc/Cargo.toml b/substrate/client/sync-state-rpc/Cargo.toml index 88d268cc93d9..e582d0b7e4ff 100644 --- a/substrate/client/sync-state-rpc/Cargo.toml +++ b/substrate/client/sync-state-rpc/Cargo.toml @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] codec = { package = "parity-scale-codec", version = "3.6.1" } jsonrpsee = { version = "0.16.2", features = ["client-core", "server", "macros"] } serde = { version = "1.0.188", features = ["derive"] } -serde_json = "1.0.85" +serde_json = "1.0.107" thiserror = "1.0.48" sc-chain-spec = { path = "../chain-spec" } sc-client-api = { path = "../api" } diff --git a/substrate/client/sysinfo/Cargo.toml b/substrate/client/sysinfo/Cargo.toml index 5834d8dec077..fdf987ed45f2 100644 --- a/substrate/client/sysinfo/Cargo.toml +++ b/substrate/client/sysinfo/Cargo.toml @@ -21,7 +21,7 @@ rand = "0.8.5" rand_pcg = "0.3.1" regex = "1" serde = { version = "1.0.188", features = ["derive"] } -serde_json = "1.0.85" +serde_json = "1.0.107" sc-telemetry = { path = "../telemetry" } sp-core = { path = "../../primitives/core" } sp-io = { path = "../../primitives/io" } diff --git a/substrate/client/telemetry/Cargo.toml b/substrate/client/telemetry/Cargo.toml index 2be5ad5c1435..6d59b9dfc1a6 100644 --- a/substrate/client/telemetry/Cargo.toml +++ b/substrate/client/telemetry/Cargo.toml @@ -23,6 +23,6 @@ pin-project = "1.0.12" sc-utils = { path = "../utils" } rand = "0.8.5" serde = { version = "1.0.188", features = ["derive"] } -serde_json = "1.0.85" +serde_json = "1.0.107" thiserror = "1.0.48" wasm-timer = "0.2.5" diff --git a/substrate/client/tracing/proc-macro/Cargo.toml b/substrate/client/tracing/proc-macro/Cargo.toml index 8c444ffb606b..bf45111de88c 100644 --- a/substrate/client/tracing/proc-macro/Cargo.toml +++ b/substrate/client/tracing/proc-macro/Cargo.toml @@ -18,4 +18,4 @@ proc-macro = true proc-macro-crate = "1.1.3" proc-macro2 = "1.0.56" quote = { version = "1.0.28", features = ["proc-macro"] } -syn = { version = "2.0.31", features = ["proc-macro", "full", "extra-traits", "parsing"] } +syn = { version = "2.0.33", features = ["proc-macro", "full", "extra-traits", "parsing"] } diff --git a/substrate/client/transaction-pool/api/src/lib.rs b/substrate/client/transaction-pool/api/src/lib.rs index a132cbc46e9b..73cc513708d2 100644 --- a/substrate/client/transaction-pool/api/src/lib.rs +++ b/substrate/client/transaction-pool/api/src/lib.rs @@ -22,6 +22,7 @@ pub mod error; use async_trait::async_trait; +use codec::Codec; use futures::{Future, Stream}; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use sp_core::offchain::TransactionPoolExt; @@ -187,7 +188,7 @@ pub trait TransactionPool: Send + Sync { /// Block type. type Block: BlockT; /// Transaction hash type. - type Hash: Hash + Eq + Member + Serialize + DeserializeOwned; + type Hash: Hash + Eq + Member + Serialize + DeserializeOwned + Codec; /// In-pool transaction type. type InPoolTransaction: InPoolTransaction< Transaction = TransactionFor, diff --git a/substrate/client/utils/src/metrics.rs b/substrate/client/utils/src/metrics.rs index 6bbdbe2e2e59..308e90cb2537 100644 --- a/substrate/client/utils/src/metrics.rs +++ b/substrate/client/utils/src/metrics.rs @@ -24,7 +24,10 @@ use prometheus::{ Error as PrometheusError, Registry, }; -use prometheus::{core::GenericCounterVec, Opts}; +use prometheus::{ + core::{GenericCounterVec, GenericGaugeVec}, + Opts, +}; lazy_static! { pub static ref TOKIO_THREADS_TOTAL: GenericCounter = @@ -36,18 +39,32 @@ lazy_static! { } lazy_static! { - pub static ref UNBOUNDED_CHANNELS_COUNTER : GenericCounterVec = GenericCounterVec::new( - Opts::new("substrate_unbounded_channel_len", "Items in each mpsc::unbounded instance"), - &["entity", "action"] // 'name of channel, send|received|dropped + pub static ref UNBOUNDED_CHANNELS_COUNTER: GenericCounterVec = GenericCounterVec::new( + Opts::new( + "substrate_unbounded_channel_len", + "Items sent/received/dropped on each mpsc::unbounded instance" + ), + &["entity", "action"], // name of channel, send|received|dropped + ).expect("Creating of statics doesn't fail. qed"); + pub static ref UNBOUNDED_CHANNELS_SIZE: GenericGaugeVec = GenericGaugeVec::new( + Opts::new( + "substrate_unbounded_channel_size", + "Size (number of messages to be processed) of each mpsc::unbounded instance", + ), + &["entity"], // name of channel ).expect("Creating of statics doesn't fail. qed"); - } +pub static SENT_LABEL: &'static str = "send"; +pub static RECEIVED_LABEL: &'static str = "received"; +pub static DROPPED_LABEL: &'static str = "dropped"; + /// Register the statics to report to registry pub fn register_globals(registry: &Registry) -> Result<(), PrometheusError> { registry.register(Box::new(TOKIO_THREADS_ALIVE.clone()))?; registry.register(Box::new(TOKIO_THREADS_TOTAL.clone()))?; registry.register(Box::new(UNBOUNDED_CHANNELS_COUNTER.clone()))?; + registry.register(Box::new(UNBOUNDED_CHANNELS_SIZE.clone()))?; Ok(()) } diff --git a/substrate/client/utils/src/mpsc.rs b/substrate/client/utils/src/mpsc.rs index 039e03f9e618..c24a5bd8904a 100644 --- a/substrate/client/utils/src/mpsc.rs +++ b/substrate/client/utils/src/mpsc.rs @@ -20,7 +20,9 @@ pub use async_channel::{TryRecvError, TrySendError}; -use crate::metrics::UNBOUNDED_CHANNELS_COUNTER; +use crate::metrics::{ + DROPPED_LABEL, RECEIVED_LABEL, SENT_LABEL, UNBOUNDED_CHANNELS_COUNTER, UNBOUNDED_CHANNELS_SIZE, +}; use async_channel::{Receiver, Sender}; use futures::{ stream::{FusedStream, Stream}, @@ -102,7 +104,10 @@ impl TracingUnboundedSender { /// Proxy function to `async_channel::Sender::try_send`. pub fn unbounded_send(&self, msg: T) -> Result<(), TrySendError> { self.inner.try_send(msg).map(|s| { - UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[self.name, "send"]).inc(); + UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[self.name, SENT_LABEL]).inc(); + UNBOUNDED_CHANNELS_SIZE + .with_label_values(&[self.name]) + .set(self.inner.len().saturated_into()); if self.inner.len() >= self.queue_size_warning && self.warning_fired @@ -140,7 +145,10 @@ impl TracingUnboundedReceiver { /// that discounts the messages taken out. pub fn try_recv(&mut self) -> Result { self.inner.try_recv().map(|s| { - UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[self.name, "received"]).inc(); + UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[self.name, RECEIVED_LABEL]).inc(); + UNBOUNDED_CHANNELS_SIZE + .with_label_values(&[self.name]) + .set(self.inner.len().saturated_into()); s }) } @@ -155,14 +163,16 @@ impl Drop for TracingUnboundedReceiver { fn drop(&mut self) { // Close the channel to prevent any further messages to be sent into the channel self.close(); - // the number of messages about to be dropped + // The number of messages about to be dropped let count = self.inner.len(); - // discount the messages + // Discount the messages if count > 0 { UNBOUNDED_CHANNELS_COUNTER - .with_label_values(&[self.name, "dropped"]) + .with_label_values(&[self.name, DROPPED_LABEL]) .inc_by(count.saturated_into()); } + // Reset the size metric to 0 + UNBOUNDED_CHANNELS_SIZE.with_label_values(&[self.name]).set(0); // Drain all the pending messages in the channel since they can never be accessed, // this can be removed once https://github.com/smol-rs/async-channel/issues/23 is // resolved @@ -180,7 +190,10 @@ impl Stream for TracingUnboundedReceiver { match Pin::new(&mut s.inner).poll_next(cx) { Poll::Ready(msg) => { if msg.is_some() { - UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[s.name, "received"]).inc(); + UNBOUNDED_CHANNELS_COUNTER.with_label_values(&[s.name, RECEIVED_LABEL]).inc(); + UNBOUNDED_CHANNELS_SIZE + .with_label_values(&[s.name]) + .set(s.inner.len().saturated_into()); } Poll::Ready(msg) }, diff --git a/substrate/docs/CODEOWNERS b/substrate/docs/CODEOWNERS deleted file mode 100644 index 63294d90e9d0..000000000000 --- a/substrate/docs/CODEOWNERS +++ /dev/null @@ -1,74 +0,0 @@ -# Lists some code owners. -# -# A codeowner just oversees some part of the codebase. If an owned file is changed then the -# corresponding codeowner receives a review request. An approval of the codeowner is -# not required for merging a PR though. -# -# **This is pretty much an experiment at the moment**. Feel free to remove yourself at any time if -# you do not want to receive review requests any longer. -# -# For details about syntax, see: -# https://help.github.com/en/articles/about-code-owners -# But here are some important notes: -# -# - Glob syntax is git-like, e.g. `/core` means the core directory in the root, unlike `core` which -# can be everywhere. -# - Multiple owners are supported. -# - Either handle (e.g, @pepyakin) or email can be used. Keep in mind, that handles might work better because they -# are more recognizable on GitHub, you can use them for mentioning unlike an email. -# - The latest matching rule, if multiple, takes precedence. - -# CI -/.github/ @paritytech/ci -/.gitlab-ci.yml @paritytech/ci -/scripts/ci/ @paritytech/ci - -# WASM executor, low-level client <-> WASM interface and other WASM-related code -/client/allocator/ @koute -/client/executor/ @koute -/primitives/panic-handler/ @koute -/primitives/runtime-interface/ @koute -/primitives/wasm-interface/ @koute -/utils/wasm-builder/ @koute - -# Systems-related bits and bobs on the client side -/client/sysinfo/ @koute -/client/tracing/ @koute - -# Documentation audit -/primitives/runtime @paritytech/docs-audit -/primitives/arithmetic @paritytech/docs-audit -# /primitives/core (to be added later) -# /primitives/io (to be added later) - -# FRAME -/frame/ @paritytech/frame-coders @paritytech/docs-audit -/frame/nfts/ @jsidorenko @paritytech/docs-audit -/frame/state-trie-migration/ @paritytech/frame-coders @cheme -/frame/uniques/ @jsidorenko @paritytech/docs-audit - -# GRANDPA, BABE, consensus stuff -/client/consensus/babe/ @andresilva -/client/consensus/grandpa/ @andresilva -/client/consensus/pow/ @sorpaas -/client/consensus/slots/ @andresilva -/frame/babe/ @andresilva -/frame/grandpa/ @andresilva -/primitives/consensus/pow/ @sorpaas - -# BEEFY, MMR -/frame/beefy/ @acatangiu -/frame/beefy-mmr/ @acatangiu -/frame/merkle-mountain-range/ @acatangiu -/primitives/merkle-mountain-range/ @acatangiu - -# Contracts -/frame/contracts/ @athei @paritytech/docs-audit - -# NPoS and election -/frame/election-provider-multi-phase/ @paritytech/staking-core @paritytech/docs-audit -/frame/election-provider-support/ @paritytech/staking-core @paritytech/docs-audit -/frame/elections-phragmen/ @paritytech/staking-core @paritytech/docs-audit -/frame/nomination-pools/ @paritytech/staking-core @paritytech/docs-audit -/frame/staking/ @paritytech/staking-core @paritytech/docs-audit -/primitives/npos-elections/ @paritytech/staking-core @paritytech/docs-audit diff --git a/substrate/frame/asset-rate/src/benchmarking.rs b/substrate/frame/asset-rate/src/benchmarking.rs index 12edcf5aee02..21d53a89e397 100644 --- a/substrate/frame/asset-rate/src/benchmarking.rs +++ b/substrate/frame/asset-rate/src/benchmarking.rs @@ -54,7 +54,7 @@ mod benchmarks { fn create() -> Result<(), BenchmarkError> { let asset_kind: T::AssetKind = T::BenchmarkHelper::create_asset_kind(SEED); #[extrinsic_call] - _(RawOrigin::Root, asset_kind.clone(), default_conversion_rate()); + _(RawOrigin::Root, Box::new(asset_kind.clone()), default_conversion_rate()); assert_eq!( pallet_asset_rate::ConversionRateToNative::::get(asset_kind), @@ -68,12 +68,12 @@ mod benchmarks { let asset_kind: T::AssetKind = T::BenchmarkHelper::create_asset_kind(SEED); assert_ok!(AssetRate::::create( RawOrigin::Root.into(), - asset_kind.clone(), + Box::new(asset_kind.clone()), default_conversion_rate() )); #[extrinsic_call] - _(RawOrigin::Root, asset_kind.clone(), FixedU128::from_u32(2)); + _(RawOrigin::Root, Box::new(asset_kind.clone()), FixedU128::from_u32(2)); assert_eq!( pallet_asset_rate::ConversionRateToNative::::get(asset_kind), @@ -87,12 +87,12 @@ mod benchmarks { let asset_kind: T::AssetKind = T::BenchmarkHelper::create_asset_kind(SEED); assert_ok!(AssetRate::::create( RawOrigin::Root.into(), - asset_kind.clone(), + Box::new(asset_kind.clone()), default_conversion_rate() )); #[extrinsic_call] - _(RawOrigin::Root, asset_kind.clone()); + _(RawOrigin::Root, Box::new(asset_kind.clone())); assert!(pallet_asset_rate::ConversionRateToNative::::get(asset_kind).is_none()); Ok(()) diff --git a/substrate/frame/asset-rate/src/lib.rs b/substrate/frame/asset-rate/src/lib.rs index 8b55f3d1d402..c3dc551f876d 100644 --- a/substrate/frame/asset-rate/src/lib.rs +++ b/substrate/frame/asset-rate/src/lib.rs @@ -61,6 +61,7 @@ use frame_support::traits::{fungible::Inspect, tokens::ConversionFromAssetBalance}; use sp_runtime::{traits::Zero, FixedPointNumber, FixedU128}; +use sp_std::boxed::Box; pub use pallet::*; pub use weights::WeightInfo; @@ -155,18 +156,18 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::create())] pub fn create( origin: OriginFor, - asset_kind: T::AssetKind, + asset_kind: Box, rate: FixedU128, ) -> DispatchResult { T::CreateOrigin::ensure_origin(origin)?; ensure!( - !ConversionRateToNative::::contains_key(asset_kind.clone()), + !ConversionRateToNative::::contains_key(asset_kind.as_ref()), Error::::AlreadyExists ); - ConversionRateToNative::::set(asset_kind.clone(), Some(rate)); + ConversionRateToNative::::set(asset_kind.as_ref(), Some(rate)); - Self::deposit_event(Event::AssetRateCreated { asset_kind, rate }); + Self::deposit_event(Event::AssetRateCreated { asset_kind: *asset_kind, rate }); Ok(()) } @@ -178,13 +179,13 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::update())] pub fn update( origin: OriginFor, - asset_kind: T::AssetKind, + asset_kind: Box, rate: FixedU128, ) -> DispatchResult { T::UpdateOrigin::ensure_origin(origin)?; let mut old = FixedU128::zero(); - ConversionRateToNative::::mutate(asset_kind.clone(), |maybe_rate| { + ConversionRateToNative::::mutate(asset_kind.as_ref(), |maybe_rate| { if let Some(r) = maybe_rate { old = *r; *r = rate; @@ -195,7 +196,11 @@ pub mod pallet { } })?; - Self::deposit_event(Event::AssetRateUpdated { asset_kind, old, new: rate }); + Self::deposit_event(Event::AssetRateUpdated { + asset_kind: *asset_kind, + old, + new: rate, + }); Ok(()) } @@ -205,16 +210,16 @@ pub mod pallet { /// - O(1) #[pallet::call_index(2)] #[pallet::weight(T::WeightInfo::remove())] - pub fn remove(origin: OriginFor, asset_kind: T::AssetKind) -> DispatchResult { + pub fn remove(origin: OriginFor, asset_kind: Box) -> DispatchResult { T::RemoveOrigin::ensure_origin(origin)?; ensure!( - ConversionRateToNative::::contains_key(asset_kind.clone()), + ConversionRateToNative::::contains_key(asset_kind.as_ref()), Error::::UnknownAssetKind ); - ConversionRateToNative::::remove(asset_kind.clone()); + ConversionRateToNative::::remove(asset_kind.as_ref()); - Self::deposit_event(Event::AssetRateRemoved { asset_kind }); + Self::deposit_event(Event::AssetRateRemoved { asset_kind: *asset_kind }); Ok(()) } } diff --git a/substrate/frame/asset-rate/src/tests.rs b/substrate/frame/asset-rate/src/tests.rs index 8990ba9fc28d..452265476093 100644 --- a/substrate/frame/asset-rate/src/tests.rs +++ b/substrate/frame/asset-rate/src/tests.rs @@ -29,7 +29,11 @@ const ASSET_ID: u32 = 42; fn create_works() { new_test_ext().execute_with(|| { assert!(pallet_asset_rate::ConversionRateToNative::::get(ASSET_ID).is_none()); - assert_ok!(AssetRate::create(RuntimeOrigin::root(), ASSET_ID, FixedU128::from_float(0.1))); + assert_ok!(AssetRate::create( + RuntimeOrigin::root(), + Box::new(ASSET_ID), + FixedU128::from_float(0.1) + )); assert_eq!( pallet_asset_rate::ConversionRateToNative::::get(ASSET_ID), @@ -42,10 +46,18 @@ fn create_works() { fn create_existing_throws() { new_test_ext().execute_with(|| { assert!(pallet_asset_rate::ConversionRateToNative::::get(ASSET_ID).is_none()); - assert_ok!(AssetRate::create(RuntimeOrigin::root(), ASSET_ID, FixedU128::from_float(0.1))); + assert_ok!(AssetRate::create( + RuntimeOrigin::root(), + Box::new(ASSET_ID), + FixedU128::from_float(0.1) + )); assert_noop!( - AssetRate::create(RuntimeOrigin::root(), ASSET_ID, FixedU128::from_float(0.1)), + AssetRate::create( + RuntimeOrigin::root(), + Box::new(ASSET_ID), + FixedU128::from_float(0.1) + ), Error::::AlreadyExists ); }); @@ -54,9 +66,13 @@ fn create_existing_throws() { #[test] fn remove_works() { new_test_ext().execute_with(|| { - assert_ok!(AssetRate::create(RuntimeOrigin::root(), ASSET_ID, FixedU128::from_float(0.1))); + assert_ok!(AssetRate::create( + RuntimeOrigin::root(), + Box::new(ASSET_ID), + FixedU128::from_float(0.1) + )); - assert_ok!(AssetRate::remove(RuntimeOrigin::root(), ASSET_ID,)); + assert_ok!(AssetRate::remove(RuntimeOrigin::root(), Box::new(ASSET_ID),)); assert!(pallet_asset_rate::ConversionRateToNative::::get(ASSET_ID).is_none()); }); } @@ -65,7 +81,7 @@ fn remove_works() { fn remove_unknown_throws() { new_test_ext().execute_with(|| { assert_noop!( - AssetRate::remove(RuntimeOrigin::root(), ASSET_ID,), + AssetRate::remove(RuntimeOrigin::root(), Box::new(ASSET_ID),), Error::::UnknownAssetKind ); }); @@ -74,8 +90,16 @@ fn remove_unknown_throws() { #[test] fn update_works() { new_test_ext().execute_with(|| { - assert_ok!(AssetRate::create(RuntimeOrigin::root(), ASSET_ID, FixedU128::from_float(0.1))); - assert_ok!(AssetRate::update(RuntimeOrigin::root(), ASSET_ID, FixedU128::from_float(0.5))); + assert_ok!(AssetRate::create( + RuntimeOrigin::root(), + Box::new(ASSET_ID), + FixedU128::from_float(0.1) + )); + assert_ok!(AssetRate::update( + RuntimeOrigin::root(), + Box::new(ASSET_ID), + FixedU128::from_float(0.5) + )); assert_eq!( pallet_asset_rate::ConversionRateToNative::::get(ASSET_ID), @@ -88,7 +112,11 @@ fn update_works() { fn update_unknown_throws() { new_test_ext().execute_with(|| { assert_noop!( - AssetRate::update(RuntimeOrigin::root(), ASSET_ID, FixedU128::from_float(0.5)), + AssetRate::update( + RuntimeOrigin::root(), + Box::new(ASSET_ID), + FixedU128::from_float(0.5) + ), Error::::UnknownAssetKind ); }); @@ -97,7 +125,11 @@ fn update_unknown_throws() { #[test] fn convert_works() { new_test_ext().execute_with(|| { - assert_ok!(AssetRate::create(RuntimeOrigin::root(), ASSET_ID, FixedU128::from_float(2.51))); + assert_ok!(AssetRate::create( + RuntimeOrigin::root(), + Box::new(ASSET_ID), + FixedU128::from_float(2.51) + )); let conversion = , diff --git a/substrate/frame/bags-list/Cargo.toml b/substrate/frame/bags-list/Cargo.toml index 4d6f3d768aad..21cdb4f89830 100644 --- a/substrate/frame/bags-list/Cargo.toml +++ b/substrate/frame/bags-list/Cargo.toml @@ -27,7 +27,7 @@ frame-election-provider-support = { path = "../election-provider-support", defau # third party log = { version = "0.4.17", default-features = false } -docify = "0.2.1" +docify = "0.2.3" aquamarine = { version = "0.3.2" } # Optional imports for benchmarking diff --git a/substrate/frame/beefy-mmr/src/lib.rs b/substrate/frame/beefy-mmr/src/lib.rs index b12eb95f650f..a0bf7cdcf86a 100644 --- a/substrate/frame/beefy-mmr/src/lib.rs +++ b/substrate/frame/beefy-mmr/src/lib.rs @@ -79,7 +79,7 @@ impl Convert> for BeefyEc .to_eth_address() .map(|v| v.to_vec()) .map_err(|_| { - log::error!(target: "runtime::beefy", "Failed to convert BEEFY PublicKey to ETH address!"); + log::debug!(target: "runtime::beefy", "Failed to convert BEEFY PublicKey to ETH address!"); }) .unwrap_or_default() } @@ -199,7 +199,20 @@ impl Pallet { .cloned() .map(T::BeefyAuthorityToMerkleLeaf::convert) .collect::>(); + let default_eth_addr = [0u8; 20]; let len = beefy_addresses.len() as u32; + let uninitialized_addresses = beefy_addresses + .iter() + .filter(|&addr| addr.as_slice().eq(&default_eth_addr)) + .count(); + if uninitialized_addresses > 0 { + log::error!( + target: "runtime::beefy", + "Failed to convert {} out of {} BEEFY PublicKeys to ETH addresses!", + uninitialized_addresses, + len, + ); + } let keyset_commitment = binary_merkle_tree::merkle_root::< ::Hashing, _, diff --git a/substrate/frame/beefy/src/default_weights.rs b/substrate/frame/beefy/src/default_weights.rs index 091d58f47f97..8042f0c932eb 100644 --- a/substrate/frame/beefy/src/default_weights.rs +++ b/substrate/frame/beefy/src/default_weights.rs @@ -49,4 +49,8 @@ impl crate::WeightInfo for () { // fetching set id -> session index mappings .saturating_add(DbWeight::get().reads(2)) } + + fn set_new_genesis() -> Weight { + DbWeight::get().writes(1) + } } diff --git a/substrate/frame/beefy/src/lib.rs b/substrate/frame/beefy/src/lib.rs index 77e74436dd67..0760446753a6 100644 --- a/substrate/frame/beefy/src/lib.rs +++ b/substrate/frame/beefy/src/lib.rs @@ -33,7 +33,7 @@ use frame_system::{ use log; use sp_runtime::{ generic::DigestItem, - traits::{IsMember, Member}, + traits::{IsMember, Member, One}, RuntimeAppPublic, }; use sp_session::{GetSessionNumber, GetValidatorCount}; @@ -62,7 +62,7 @@ const LOG_TARGET: &str = "runtime::beefy"; #[frame_support::pallet] pub mod pallet { use super::*; - use frame_system::pallet_prelude::BlockNumberFor; + use frame_system::{ensure_root, pallet_prelude::BlockNumberFor}; #[pallet::config] pub trait Config: frame_system::Config { @@ -152,8 +152,8 @@ pub mod pallet { StorageMap<_, Twox64Concat, sp_consensus_beefy::ValidatorSetId, SessionIndex>; /// Block number where BEEFY consensus is enabled/started. - /// By changing this (through governance or sudo), BEEFY consensus is effectively - /// restarted from the new block number. + /// By changing this (through privileged `set_new_genesis()`), BEEFY consensus is effectively + /// restarted from the newly set block number. #[pallet::storage] #[pallet::getter(fn genesis_block)] pub(super) type GenesisBlock = @@ -174,7 +174,7 @@ pub mod pallet { fn default() -> Self { // BEEFY genesis will be first BEEFY-MANDATORY block, // use block number one instead of chain-genesis. - let genesis_block = Some(sp_runtime::traits::One::one()); + let genesis_block = Some(One::one()); Self { authorities: Vec::new(), genesis_block } } } @@ -198,6 +198,8 @@ pub mod pallet { InvalidEquivocationProof, /// A given equivocation report is valid but already previously reported. DuplicateOffenceReport, + /// Submitted configuration is invalid. + InvalidConfiguration, } #[pallet::call] @@ -265,6 +267,23 @@ pub mod pallet { )?; Ok(Pays::No.into()) } + + /// Reset BEEFY consensus by setting a new BEEFY genesis at `delay_in_blocks` blocks in the + /// future. + /// + /// Note: `delay_in_blocks` has to be at least 1. + #[pallet::call_index(2)] + #[pallet::weight(::WeightInfo::set_new_genesis())] + pub fn set_new_genesis( + origin: OriginFor, + delay_in_blocks: BlockNumberFor, + ) -> DispatchResult { + ensure_root(origin)?; + ensure!(delay_in_blocks >= One::one(), Error::::InvalidConfiguration); + let genesis_block = frame_system::Pallet::::block_number() + delay_in_blocks; + GenesisBlock::::put(Some(genesis_block)); + Ok(()) + } } #[pallet::validate_unsigned] @@ -452,4 +471,5 @@ impl IsMember for Pallet { pub trait WeightInfo { fn report_equivocation(validator_count: u32, max_nominators_per_validator: u32) -> Weight; + fn set_new_genesis() -> Weight; } diff --git a/substrate/frame/beefy/src/tests.rs b/substrate/frame/beefy/src/tests.rs index e04dc330d0c0..bf1b204e0260 100644 --- a/substrate/frame/beefy/src/tests.rs +++ b/substrate/frame/beefy/src/tests.rs @@ -791,3 +791,25 @@ fn valid_equivocation_reports_dont_pay_fees() { assert_eq!(post_info.pays_fee, Pays::Yes); }) } + +#[test] +fn set_new_genesis_works() { + let authorities = test_authorities(); + + new_test_ext_raw_authorities(authorities).execute_with(|| { + start_era(1); + + let new_genesis_delay = 10u64; + // the call for setting new genesis should work + assert_ok!(Beefy::set_new_genesis(RuntimeOrigin::root(), new_genesis_delay,)); + let expected = System::block_number() + new_genesis_delay; + // verify new genesis was set + assert_eq!(Beefy::genesis_block(), Some(expected)); + + // setting delay < 1 should fail + assert_err!( + Beefy::set_new_genesis(RuntimeOrigin::root(), 0u64,), + Error::::InvalidConfiguration, + ); + }); +} diff --git a/substrate/frame/contracts/Cargo.toml b/substrate/frame/contracts/Cargo.toml index 237ab9303e25..d5c809e1bf7c 100644 --- a/substrate/frame/contracts/Cargo.toml +++ b/substrate/frame/contracts/Cargo.toml @@ -9,7 +9,7 @@ homepage = "https://substrate.io" repository.workspace = true description = "FRAME pallet for WASM contracts" readme = "README.md" -include = ["src/**/*", "README.md", "CHANGELOG.md"] +include = ["src/**/*", "build.rs", "README.md", "CHANGELOG.md"] [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/substrate/frame/contracts/fixtures/run_out_of_gas_start_fn.wat b/substrate/frame/contracts/fixtures/run_out_of_gas_start_fn.wat new file mode 100644 index 000000000000..6591d7ede78c --- /dev/null +++ b/substrate/frame/contracts/fixtures/run_out_of_gas_start_fn.wat @@ -0,0 +1,10 @@ +(module + (import "env" "memory" (memory 1 1)) + (start $start) + (func $start + (loop $inf (br $inf)) ;; just run out of gas + (unreachable) + ) + (func (export "call")) + (func (export "deploy")) +) diff --git a/substrate/frame/contracts/proc-macro/Cargo.toml b/substrate/frame/contracts/proc-macro/Cargo.toml index ecc5bc627b70..8779f97b9cde 100644 --- a/substrate/frame/contracts/proc-macro/Cargo.toml +++ b/substrate/frame/contracts/proc-macro/Cargo.toml @@ -17,7 +17,7 @@ proc-macro = true [dependencies] proc-macro2 = "1.0.56" quote = "1.0.28" -syn = { version = "2.0.31", features = ["full"] } +syn = { version = "2.0.33", features = ["full"] } [dev-dependencies] diff --git a/substrate/frame/contracts/src/benchmarking/sandbox.rs b/substrate/frame/contracts/src/benchmarking/sandbox.rs index 34974b02ea0c..c3abbcad5f2b 100644 --- a/substrate/frame/contracts/src/benchmarking/sandbox.rs +++ b/substrate/frame/contracts/src/benchmarking/sandbox.rs @@ -58,7 +58,13 @@ impl From<&WasmModule> for Sandbox { .add_fuel(u64::MAX) .expect("We've set up engine to fuel consuming mode; qed"); - let entry_point = instance.get_export(&store, "call").unwrap().into_func().unwrap(); + let entry_point = instance + .start(&mut store) + .unwrap() + .get_export(&store, "call") + .unwrap() + .into_func() + .unwrap(); Self { entry_point, store } } } diff --git a/substrate/frame/contracts/src/exec.rs b/substrate/frame/contracts/src/exec.rs index fdb30310ef70..f93e7a2b21a5 100644 --- a/substrate/frame/contracts/src/exec.rs +++ b/substrate/frame/contracts/src/exec.rs @@ -21,7 +21,7 @@ use crate::{ storage::{self, meter::Diff, WriteOutcome}, BalanceOf, CodeHash, CodeInfo, CodeInfoOf, Config, ContractInfo, ContractInfoOf, DebugBufferVec, Determinism, Error, Event, Nonce, Origin, Pallet as Contracts, Schedule, - WasmBlob, LOG_TARGET, + LOG_TARGET, }; use frame_support::{ crypto::ecdsa::ECDSAExt, @@ -318,6 +318,22 @@ pub trait Ext: sealing::Sealed { /// Returns a nonce that is incremented for every instantiated contract. fn nonce(&mut self) -> u64; + /// Increment the reference count of a of a stored code by one. + /// + /// # Errors + /// + /// [`Error::CodeNotFound`] is returned if no stored code found having the specified + /// `code_hash`. + fn increment_refcount(code_hash: CodeHash) -> Result<(), DispatchError>; + + /// Decrement the reference count of a stored code by one. + /// + /// # Note + /// + /// A contract whose reference count dropped to zero isn't automatically removed. A + /// `remove_code` transaction must be submitted by the original uploader to do so. + fn decrement_refcount(code_hash: CodeHash); + /// Adds a delegate dependency to [`ContractInfo`]'s `delegate_dependencies` field. /// /// This ensures that the delegated contract is not removed while it is still in use. It @@ -381,22 +397,6 @@ pub trait Executable: Sized { gas_meter: &mut GasMeter, ) -> Result; - /// Increment the reference count of a of a stored code by one. - /// - /// # Errors - /// - /// [`Error::CodeNotFound`] is returned if no stored code found having the specified - /// `code_hash`. - fn increment_refcount(code_hash: CodeHash) -> Result<(), DispatchError>; - - /// Decrement the reference count of a stored code by one. - /// - /// # Note - /// - /// A contract whose reference count dropped to zero isn't automatically removed. A - /// `remove_code` transaction must be submitted by the original uploader to do so. - fn decrement_refcount(code_hash: CodeHash); - /// Execute the specified exported function and return the result. /// /// When the specified function is `Constructor` the executable is stored and its @@ -1285,10 +1285,10 @@ where info.queue_trie_for_deletion(); ContractInfoOf::::remove(&frame.account_id); - E::decrement_refcount(info.code_hash); + Self::decrement_refcount(info.code_hash); for (code_hash, deposit) in info.delegate_dependencies() { - E::decrement_refcount(*code_hash); + Self::decrement_refcount(*code_hash); frame .nested_storage .charge_deposit(frame.account_id.clone(), StorageDeposit::Refund(*deposit)); @@ -1491,8 +1491,8 @@ where frame.nested_storage.charge_deposit(frame.account_id.clone(), deposit); - E::increment_refcount(hash)?; - E::decrement_refcount(prev_hash); + Self::increment_refcount(hash)?; + Self::decrement_refcount(prev_hash); Contracts::::deposit_event( vec![T::Hashing::hash_of(&frame.account_id), hash, prev_hash], Event::ContractCodeUpdated { @@ -1525,6 +1525,25 @@ where } } + fn increment_refcount(code_hash: CodeHash) -> Result<(), DispatchError> { + >::mutate(code_hash, |existing| -> Result<(), DispatchError> { + if let Some(info) = existing { + *info.refcount_mut() = info.refcount().saturating_add(1); + Ok(()) + } else { + Err(Error::::CodeNotFound.into()) + } + }) + } + + fn decrement_refcount(code_hash: CodeHash) { + >::mutate(code_hash, |existing| { + if let Some(info) = existing { + *info.refcount_mut() = info.refcount().saturating_sub(1); + } + }); + } + fn add_delegate_dependency( &mut self, code_hash: CodeHash, @@ -1537,7 +1556,7 @@ where let deposit = T::CodeHashLockupDepositPercent::get().mul_ceil(code_info.deposit()); info.add_delegate_dependency(code_hash, deposit)?; - >::increment_refcount(code_hash)?; + Self::increment_refcount(code_hash)?; frame .nested_storage .charge_deposit(frame.account_id.clone(), StorageDeposit::Charge(deposit)); @@ -1552,8 +1571,7 @@ where let info = frame.contract_info.get(&frame.account_id); let deposit = info.remove_delegate_dependency(code_hash)?; - >::decrement_refcount(*code_hash); - + Self::decrement_refcount(*code_hash); frame .nested_storage .charge_deposit(frame.account_id.clone(), StorageDeposit::Refund(deposit)); @@ -1600,11 +1618,7 @@ mod tests { use pallet_contracts_primitives::ReturnFlags; use pretty_assertions::assert_eq; use sp_runtime::{traits::Hash, DispatchError}; - use std::{ - cell::RefCell, - collections::hash_map::{Entry, HashMap}, - rc::Rc, - }; + use std::{cell::RefCell, collections::hash_map::HashMap, rc::Rc}; type System = frame_system::Pallet; @@ -1635,7 +1649,6 @@ mod tests { func_type: ExportedFunction, code_hash: CodeHash, code_info: CodeInfo, - refcount: u64, } #[derive(Default, Clone)] @@ -1664,37 +1677,11 @@ mod tests { func_type, code_hash: hash, code_info: CodeInfo::::new(ALICE), - refcount: 1, }, ); hash }) } - - fn increment_refcount(code_hash: CodeHash) -> Result<(), DispatchError> { - Loader::mutate(|loader| { - match loader.map.entry(code_hash) { - Entry::Vacant(_) => Err(>::CodeNotFound)?, - Entry::Occupied(mut entry) => entry.get_mut().refcount += 1, - } - Ok(()) - }) - } - - fn decrement_refcount(code_hash: CodeHash) { - use std::collections::hash_map::Entry::Occupied; - Loader::mutate(|loader| { - let mut entry = match loader.map.entry(code_hash) { - Occupied(e) => e, - _ => panic!("code_hash does not exist"), - }; - let refcount = &mut entry.get_mut().refcount; - *refcount -= 1; - if *refcount == 0 { - entry.remove(); - } - }); - } } impl Executable for MockExecutable { @@ -1707,14 +1694,6 @@ mod tests { }) } - fn increment_refcount(code_hash: CodeHash) -> Result<(), DispatchError> { - MockLoader::increment_refcount(code_hash) - } - - fn decrement_refcount(code_hash: CodeHash) { - MockLoader::decrement_refcount(code_hash); - } - fn execute>( self, ext: &mut E, @@ -1722,7 +1701,7 @@ mod tests { input_data: Vec, ) -> ExecResult { if let &Constructor = function { - Self::increment_refcount(self.code_hash).unwrap(); + E::increment_refcount(self.code_hash).unwrap(); } // # Safety // @@ -1733,7 +1712,7 @@ mod tests { // The transmute is necessary because `execute` has to be generic over all // `E: Ext`. However, `MockExecutable` can't be generic over `E` as it would // constitute a cycle. - let ext = unsafe { std::mem::transmute(ext) }; + let ext = unsafe { mem::transmute(ext) }; if function == &self.func_type { (self.func)(MockCtx { ext, input_data }, &self) } else { diff --git a/substrate/frame/contracts/src/lib.rs b/substrate/frame/contracts/src/lib.rs index e22e4a3f9ff8..7d516fbe2496 100644 --- a/substrate/frame/contracts/src/lib.rs +++ b/substrate/frame/contracts/src/lib.rs @@ -103,7 +103,9 @@ pub mod weights; #[cfg(test)] mod tests; use crate::{ - exec::{AccountIdOf, ErrorOrigin, ExecError, Executable, Key, MomentOf, Stack as ExecStack}, + exec::{ + AccountIdOf, ErrorOrigin, ExecError, Executable, Ext, Key, MomentOf, Stack as ExecStack, + }, gas::GasMeter, storage::{meter::Meter as StorageMeter, ContractInfo, DeletionQueueManager}, wasm::{CodeInfo, WasmBlob}, @@ -658,8 +660,8 @@ pub mod pallet { } else { return Err(>::ContractNotFound.into()) }; - >::increment_refcount(code_hash)?; - >::decrement_refcount(contract.code_hash); + >>::increment_refcount(code_hash)?; + >>::decrement_refcount(contract.code_hash); Self::deposit_event( vec![T::Hashing::hash_of(&dest), code_hash, contract.code_hash], Event::ContractCodeUpdated { diff --git a/substrate/frame/contracts/src/tests.rs b/substrate/frame/contracts/src/tests.rs index 8d6c5c5ac728..0fea2b155950 100644 --- a/substrate/frame/contracts/src/tests.rs +++ b/substrate/frame/contracts/src/tests.rs @@ -862,6 +862,27 @@ fn deposit_event_max_value_limit() { }); } +// Fail out of fuel (ref_time weight) inside the start function. +#[test] +fn run_out_of_fuel_start_fun() { + let (wasm, _code_hash) = compile_module::("run_out_of_gas_start_fn").unwrap(); + ExtBuilder::default().existential_deposit(50).build().execute_with(|| { + let _ = ::Currency::set_balance(&ALICE, 1_000_000); + assert_err_ignore_postinfo!( + Contracts::instantiate_with_code( + RuntimeOrigin::signed(ALICE), + 0, + Weight::from_parts(1_000_000_000_000, u64::MAX), + None, + wasm, + vec![], + vec![], + ), + Error::::OutOfGas, + ); + }); +} + // Fail out of fuel (ref_time weight) in the engine. #[test] fn run_out_of_fuel_engine() { diff --git a/substrate/frame/contracts/src/wasm/mod.rs b/substrate/frame/contracts/src/wasm/mod.rs index 5fc65e314ad9..77e94b16777b 100644 --- a/substrate/frame/contracts/src/wasm/mod.rs +++ b/substrate/frame/contracts/src/wasm/mod.rs @@ -49,7 +49,7 @@ use frame_support::{ use sp_core::Get; use sp_runtime::{DispatchError, RuntimeDebug}; use sp_std::prelude::*; -use wasmi::{Instance, Linker, Memory, MemoryType, StackLimits, Store}; +use wasmi::{InstancePre, Linker, Memory, MemoryType, StackLimits, Store}; const BYTES_PER_PAGE: usize = 64 * 1024; @@ -164,7 +164,30 @@ impl WasmBlob { /// /// Applies all necessary checks before removing the code. pub fn remove(origin: &T::AccountId, code_hash: CodeHash) -> DispatchResult { - Self::try_remove_code(origin, code_hash) + >::try_mutate_exists(&code_hash, |existing| { + if let Some(code_info) = existing { + ensure!(code_info.refcount == 0, >::CodeInUse); + ensure!(&code_info.owner == origin, BadOrigin); + let _ = T::Currency::release( + &HoldReason::CodeUploadDepositReserve.into(), + &code_info.owner, + code_info.deposit, + BestEffort, + ); + let deposit_released = code_info.deposit; + let remover = code_info.owner.clone(); + + *existing = None; + >::remove(&code_hash); + >::deposit_event( + vec![code_hash], + Event::CodeRemoved { code_hash, deposit_released, remover }, + ); + Ok(()) + } else { + Err(>::CodeNotFound.into()) + } + }) } /// Creates and returns an instance of the supplied code. @@ -179,7 +202,7 @@ impl WasmBlob { determinism: Determinism, stack_limits: StackLimits, allow_deprecated: AllowDeprecatedInterface, - ) -> Result<(Store, Memory, Instance), &'static str> + ) -> Result<(Store, Memory, InstancePre), &'static str> where E: Environment, { @@ -217,9 +240,7 @@ impl WasmBlob { let instance = linker .instantiate(&mut store, &contract.module) - .map_err(|_| "can't instantiate module with provided definitions")? - .ensure_no_start(&mut store) - .map_err(|_| "start function is forbidden but found in the module")?; + .map_err(|_| "can't instantiate module with provided definitions")?; Ok((store, memory, instance)) } @@ -261,45 +282,6 @@ impl WasmBlob { }) } - /// Try to remove code together with all associated information. - fn try_remove_code(origin: &T::AccountId, code_hash: CodeHash) -> DispatchResult { - >::try_mutate_exists(&code_hash, |existing| { - if let Some(code_info) = existing { - ensure!(code_info.refcount == 0, >::CodeInUse); - ensure!(&code_info.owner == origin, BadOrigin); - let _ = T::Currency::release( - &HoldReason::CodeUploadDepositReserve.into(), - &code_info.owner, - code_info.deposit, - BestEffort, - ); - let deposit_released = code_info.deposit; - let remover = code_info.owner.clone(); - - *existing = None; - >::remove(&code_hash); - >::deposit_event( - vec![code_hash], - Event::CodeRemoved { code_hash, deposit_released, remover }, - ); - Ok(()) - } else { - Err(>::CodeNotFound.into()) - } - }) - } - - /// Load code with the given code hash. - fn load_code( - code_hash: CodeHash, - gas_meter: &mut GasMeter, - ) -> Result<(CodeVec, CodeInfo), DispatchError> { - let code_info = >::get(code_hash).ok_or(Error::::CodeNotFound)?; - gas_meter.charge(CodeLoadToken(code_info.code_len))?; - let code = >::get(code_hash).ok_or(Error::::CodeNotFound)?; - Ok((code, code_info)) - } - /// Create the module without checking the passed code. /// /// # Note @@ -318,12 +300,6 @@ impl WasmBlob { } impl CodeInfo { - /// Return the refcount of the module. - #[cfg(test)] - pub fn refcount(&self) -> u64 { - self.refcount - } - #[cfg(test)] pub fn new(owner: T::AccountId) -> Self { CodeInfo { @@ -335,6 +311,16 @@ impl CodeInfo { } } + /// Returns reference count of the module. + pub fn refcount(&self) -> u64 { + self.refcount + } + + /// Return mutable reference to the refcount of the module. + pub fn refcount_mut(&mut self) -> &mut u64 { + &mut self.refcount + } + /// Returns the deposit of the module. pub fn deposit(&self) -> BalanceOf { self.deposit @@ -346,29 +332,12 @@ impl Executable for WasmBlob { code_hash: CodeHash, gas_meter: &mut GasMeter, ) -> Result { - let (code, code_info) = Self::load_code(code_hash, gas_meter)?; + let code_info = >::get(code_hash).ok_or(Error::::CodeNotFound)?; + gas_meter.charge(CodeLoadToken(code_info.code_len))?; + let code = >::get(code_hash).ok_or(Error::::CodeNotFound)?; Ok(Self { code, code_info, code_hash }) } - fn increment_refcount(code_hash: CodeHash) -> Result<(), DispatchError> { - >::mutate(code_hash, |existing| -> Result<(), DispatchError> { - if let Some(info) = existing { - info.refcount = info.refcount.saturating_add(1); - Ok(()) - } else { - Err(Error::::CodeNotFound.into()) - } - }) - } - - fn decrement_refcount(code_hash: CodeHash) { - >::mutate(code_hash, |existing| { - if let Some(info) = existing { - info.refcount = info.refcount.saturating_sub(1); - } - }); - } - fn execute>( self, ext: &mut E, @@ -410,25 +379,38 @@ impl Executable for WasmBlob { .add_fuel(fuel_limit) .expect("We've set up engine to fuel consuming mode; qed"); - let exported_func = instance - .get_export(&store, function.identifier()) - .and_then(|export| export.into_func()) - .ok_or_else(|| { - log::error!(target: LOG_TARGET, "failed to find entry point"); - Error::::CodeRejected - })?; + // Sync this frame's gas meter with the engine's one. + let process_result = |mut store: Store>, result| { + let engine_consumed_total = + store.fuel_consumed().expect("Fuel metering is enabled; qed"); + let gas_meter = store.data_mut().ext().gas_meter_mut(); + gas_meter.charge_fuel(engine_consumed_total)?; + store.into_data().to_execution_result(result) + }; + // Start function should already see the correct refcount in case it will be ever inspected. if let &ExportedFunction::Constructor = function { - WasmBlob::::increment_refcount(self.code_hash)?; + E::increment_refcount(self.code_hash)?; } - let result = exported_func.call(&mut store, &[], &mut []); - let engine_consumed_total = store.fuel_consumed().expect("Fuel metering is enabled; qed"); - // Sync this frame's gas meter with the engine's one. - let gas_meter = store.data_mut().ext().gas_meter_mut(); - gas_meter.charge_fuel(engine_consumed_total)?; - - store.into_data().to_execution_result(result) + // Any abort in start function (includes `return` + `terminate`) will make us skip the + // call into the subsequent exported function. This means that calling `return` returns data + // from the whole contract execution. + match instance.start(&mut store) { + Ok(instance) => { + let exported_func = instance + .get_export(&store, function.identifier()) + .and_then(|export| export.into_func()) + .ok_or_else(|| { + log::error!(target: LOG_TARGET, "failed to find entry point"); + Error::::CodeRejected + })?; + + let result = exported_func.call(&mut store, &[], &mut []); + process_result(store, result) + }, + Err(err) => process_result(store, Err(err)), + } } fn code_hash(&self) -> &CodeHash { @@ -740,7 +722,10 @@ mod tests { fn nonce(&mut self) -> u64 { 995 } - + fn increment_refcount(_code_hash: CodeHash) -> Result<(), DispatchError> { + Ok(()) + } + fn decrement_refcount(_code_hash: CodeHash) {} fn add_delegate_dependency( &mut self, code: CodeHash, @@ -748,7 +733,6 @@ mod tests { self.delegate_dependencies.borrow_mut().insert(code); Ok(()) } - fn remove_delegate_dependency( &mut self, code: &CodeHash, @@ -790,11 +774,20 @@ mod tests { executable.execute(ext.borrow_mut(), entry_point, input_data) } - /// Execute the supplied code. + /// Execute the `call` function within the supplied code. fn execute>(wat: &str, input_data: Vec, ext: E) -> ExecResult { execute_internal(wat, input_data, ext, &ExportedFunction::Call, true, false) } + /// Execute the `deploy` function within the supplied code. + fn execute_instantiate>( + wat: &str, + input_data: Vec, + ext: E, + ) -> ExecResult { + execute_internal(wat, input_data, ext, &ExportedFunction::Constructor, true, false) + } + /// Execute the supplied code with disabled unstable functions. /// /// In our test config unstable functions are disabled so that we can test them. @@ -1878,32 +1871,47 @@ mod tests { assert_ok!(execute(CODE_VALUE_TRANSFERRED, vec![], MockExt::default())); } - const START_FN_ILLEGAL: &str = r#" + const START_FN_DOES_RUN: &str = r#" (module - (import "seal0" "seal_return" (func $seal_return (param i32 i32 i32))) + (import "seal0" "seal_deposit_event" (func $seal_deposit_event (param i32 i32 i32 i32))) (import "env" "memory" (memory 1 1)) (start $start) (func $start - (unreachable) + (call $seal_deposit_event + (i32.const 0) ;; Pointer to the start of topics buffer + (i32.const 0) ;; The length of the topics buffer. + (i32.const 0) ;; Pointer to the start of the data buffer + (i32.const 13) ;; Length of the buffer + ) ) - (func (export "call") - (unreachable) - ) + (func (export "call")) - (func (export "deploy") - (unreachable) - ) + (func (export "deploy")) - (data (i32.const 8) "\01\02\03\04") + (data (i32.const 0) "\00\01\2A\00\00\00\00\00\00\00\E5\14\00") ) "#; #[test] - fn start_fn_illegal() { - let output = execute(START_FN_ILLEGAL, vec![], MockExt::default()); - assert_err!(output, >::CodeRejected,); + fn start_fn_does_run_on_call() { + let mut ext = MockExt::default(); + execute(START_FN_DOES_RUN, vec![], &mut ext).unwrap(); + assert_eq!( + ext.events[0].1, + [0x00_u8, 0x01, 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe5, 0x14, 0x00] + ); + } + + #[test] + fn start_fn_does_run_on_deploy() { + let mut ext = MockExt::default(); + execute_instantiate(START_FN_DOES_RUN, vec![], &mut ext).unwrap(); + assert_eq!( + ext.events[0].1, + [0x00_u8, 0x01, 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe5, 0x14, 0x00] + ); } const CODE_TIMESTAMP_NOW: &str = r#" diff --git a/substrate/frame/election-provider-multi-phase/src/lib.rs b/substrate/frame/election-provider-multi-phase/src/lib.rs index f26a6f40d426..0d751e3f9cb0 100644 --- a/substrate/frame/election-provider-multi-phase/src/lib.rs +++ b/substrate/frame/election-provider-multi-phase/src/lib.rs @@ -149,7 +149,8 @@ //! while this binary lives in the Polkadot repository, this particular subcommand of it can work //! against any substrate-based chain. //! -//! See the `staking-miner` documentation in the Polkadot repository for more information. +//! See the [`staking-miner`](https://github.com/paritytech/staking-miner-v2) docs for more +//! information. //! //! ## Feasible Solution (correct solution) //! diff --git a/substrate/frame/election-provider-support/solution-type/Cargo.toml b/substrate/frame/election-provider-support/solution-type/Cargo.toml index 7693da0372d9..50381d838696 100644 --- a/substrate/frame/election-provider-support/solution-type/Cargo.toml +++ b/substrate/frame/election-provider-support/solution-type/Cargo.toml @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] proc-macro = true [dependencies] -syn = { version = "2.0.31", features = ["full", "visit"] } +syn = { version = "2.0.33", features = ["full", "visit"] } quote = "1.0.28" proc-macro2 = "1.0.56" proc-macro-crate = "1.1.3" diff --git a/substrate/frame/election-provider-support/solution-type/fuzzer/Cargo.toml b/substrate/frame/election-provider-support/solution-type/fuzzer/Cargo.toml index 69ce42559a61..fe84c4c0ef94 100644 --- a/substrate/frame/election-provider-support/solution-type/fuzzer/Cargo.toml +++ b/substrate/frame/election-provider-support/solution-type/fuzzer/Cargo.toml @@ -13,7 +13,7 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } honggfuzz = "0.5" rand = { version = "0.8", features = ["std", "small_rng"] } diff --git a/substrate/frame/fast-unstake/Cargo.toml b/substrate/frame/fast-unstake/Cargo.toml index a2aa769620cb..224067d4fa70 100644 --- a/substrate/frame/fast-unstake/Cargo.toml +++ b/substrate/frame/fast-unstake/Cargo.toml @@ -27,7 +27,7 @@ frame-election-provider-support = { path = "../election-provider-support", defau frame-benchmarking = { path = "../benchmarking", default-features = false, optional = true} -docify = "0.2.1" +docify = "0.2.3" [dev-dependencies] pallet-staking-reward-curve = { path = "../staking/reward-curve" } diff --git a/substrate/frame/nomination-pools/src/lib.rs b/substrate/frame/nomination-pools/src/lib.rs index c4bebc5a1d03..485cdada7173 100644 --- a/substrate/frame/nomination-pools/src/lib.rs +++ b/substrate/frame/nomination-pools/src/lib.rs @@ -3134,6 +3134,10 @@ impl Pallet { // reward math rounds down, we might accumulate some dust here. let pending_rewards_lt_leftover_bal = RewardPool::::current_balance(id) >= pools_members_pending_rewards.get(&id).copied().unwrap_or_default(); + + // this is currently broken in Kusama, a fix is being worked on in + // . until it is fixed, log a + // warning instead of panicing with an `ensure` statement. if !pending_rewards_lt_leftover_bal { log::warn!( "pool {:?}, sum pending rewards = {:?}, remaining balance = {:?}", @@ -3142,10 +3146,6 @@ impl Pallet { RewardPool::::current_balance(id) ); } - ensure!( - pending_rewards_lt_leftover_bal, - "The sum of the pending rewards must be less than the leftover balance." - ); Ok(()) })?; diff --git a/substrate/frame/paged-list/Cargo.toml b/substrate/frame/paged-list/Cargo.toml index f3165a2f5f92..9fbc3fdb2c62 100644 --- a/substrate/frame/paged-list/Cargo.toml +++ b/substrate/frame/paged-list/Cargo.toml @@ -13,7 +13,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive"] } -docify = "0.2.1" +docify = "0.2.3" scale-info = { version = "2.0.0", default-features = false, features = ["derive"] } frame-benchmarking = { path = "../benchmarking", default-features = false, optional = true} diff --git a/substrate/frame/scheduler/Cargo.toml b/substrate/frame/scheduler/Cargo.toml index 3ad49ef5e662..7fa14e861e8c 100644 --- a/substrate/frame/scheduler/Cargo.toml +++ b/substrate/frame/scheduler/Cargo.toml @@ -20,7 +20,7 @@ sp-io = { path = "../../primitives/io", default-features = false} sp-runtime = { path = "../../primitives/runtime", default-features = false} sp-std = { path = "../../primitives/std", default-features = false} sp-weights = { path = "../../primitives/weights", default-features = false} -docify = "0.2.1" +docify = "0.2.3" [dev-dependencies] pallet-preimage = { path = "../preimage" } diff --git a/substrate/frame/society/Cargo.toml b/substrate/frame/society/Cargo.toml index 1510e17b8b57..d38875836aa2 100644 --- a/substrate/frame/society/Cargo.toml +++ b/substrate/frame/society/Cargo.toml @@ -34,8 +34,6 @@ sp-io = { path = "../../primitives/io" } [features] default = [ "std" ] -# Enable `VersionedMigration` for migrations using this feature. -experimental = [ "frame-support/experimental" ] std = [ "codec/std", "frame-benchmarking?/std", diff --git a/substrate/frame/society/src/migrations.rs b/substrate/frame/society/src/migrations.rs index b50b0e088a6e..553eea1a7952 100644 --- a/substrate/frame/society/src/migrations.rs +++ b/substrate/frame/society/src/migrations.rs @@ -95,7 +95,6 @@ impl< /// [`VersionUncheckedMigrateToV2`] wrapped in a [`frame_support::migrations::VersionedMigration`], /// ensuring the migration is only performed when on-chain version is 0. -#[cfg(feature = "experimental")] pub type VersionCheckedMigrateToV2 = frame_support::migrations::VersionedMigration< 0, diff --git a/substrate/frame/staking/reward-curve/Cargo.toml b/substrate/frame/staking/reward-curve/Cargo.toml index e1b97d278da1..fc1f1b4b3ee8 100644 --- a/substrate/frame/staking/reward-curve/Cargo.toml +++ b/substrate/frame/staking/reward-curve/Cargo.toml @@ -18,7 +18,7 @@ proc-macro = true proc-macro-crate = "1.1.3" proc-macro2 = "1.0.56" quote = "1.0.28" -syn = { version = "2.0.31", features = ["full", "visit"] } +syn = { version = "2.0.33", features = ["full", "visit"] } [dev-dependencies] sp-runtime = { path = "../../../primitives/runtime" } diff --git a/substrate/frame/sudo/Cargo.toml b/substrate/frame/sudo/Cargo.toml index a75a0c504f98..bdd4252414a6 100644 --- a/substrate/frame/sudo/Cargo.toml +++ b/substrate/frame/sudo/Cargo.toml @@ -22,6 +22,8 @@ sp-io = { path = "../../primitives/io", default-features = false} sp-runtime = { path = "../../primitives/runtime", default-features = false} sp-std = { path = "../../primitives/std", default-features = false} +docify = "0.2.3" + [dev-dependencies] sp-core = { path = "../../primitives/core" } diff --git a/substrate/frame/sudo/src/lib.rs b/substrate/frame/sudo/src/lib.rs index f735469558c7..0c869bec7f07 100644 --- a/substrate/frame/sudo/src/lib.rs +++ b/substrate/frame/sudo/src/lib.rs @@ -7,7 +7,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -15,41 +15,32 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! # Sudo Pallet -//! -//! - [`Config`] -//! - [`Call`] -//! -//! ## Overview -//! -//! The Sudo pallet allows for a single account (called the "sudo key") -//! to execute dispatchable functions that require a `Root` call -//! or designate a new account to replace them as the sudo key. -//! Only one account can be the sudo key at a time. -//! -//! ## Interface +//! > Made with *Substrate*, for *Polkadot*. //! -//! ### Dispatchable Functions +//! [![github]](https://github.com/paritytech/polkadot-sdk/tree/master/substrate/frame/sudo) +//! [![polkadot]](https://polkadot.network) //! -//! Only the sudo key can call the dispatchable functions from the Sudo pallet. +//! [github]: https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github +//! [polkadot]: https://img.shields.io/badge/polkadot-E6007A?style=for-the-badge&logo=polkadot&logoColor=white //! -//! * `sudo` - Make a `Root` call to a dispatchable function. -//! * `set_key` - Assign a new account to be the sudo key. +//! # Sudo Pallet //! -//! ## Usage +//! A pallet to provide a way to execute privileged runtime calls using a specified sudo ("superuser +//! do") account. //! -//! ### Executing Privileged Functions +//! ## Pallet API //! -//! The Sudo pallet itself is not intended to be used within other pallets. -//! Instead, you can build "privileged functions" (i.e. functions that require `Root` origin) in -//! other pallets. You can execute these privileged functions by calling `sudo` with the sudo key -//! account. Privileged functions cannot be directly executed via an extrinsic. +//! See the [`pallet`] module for more information about the interfaces this pallet exposes, +//! including its configuration trait, dispatchables, storage items, events and errors. //! -//! Learn more about privileged functions and `Root` origin in the [`Origin`] type documentation. +//! ## Overview //! -//! ### Simple Code Snippet +//! In Substrate blockchains, pallets may contain dispatchable calls that can only be called at +//! the system level of the chain (i.e. dispatchables that require a `Root` origin). +//! Setting a privileged account, called the _sudo key_, allows you to make such calls as an +//! extrinisic. //! -//! This is an example of a pallet that exposes a privileged function: +//! Here's an example of a privileged function in another pallet: //! //! ``` //! #[frame_support::pallet] @@ -76,27 +67,58 @@ //! } //! } //! } -//! # fn main() {} //! ``` //! -//! ### Signed Extension +//! With the Sudo pallet configured in your chain's runtime you can execute this privileged +//! function by constructing a call using the [`sudo`](Pallet::sudo) dispatchable. +//! +//! To use this pallet in your runtime, a sudo key must be specified in the [`GenesisConfig`] of +//! the pallet. You can change this key at anytime once your chain is live using the +//! [`set_key`](Pallet::set_key) dispatchable, however only one sudo key can be set at a +//! time. The pallet also allows you to make a call using +//! [`sudo_unchecked_weight`](Pallet::sudo_unchecked_weight), which allows the sudo account to +//! execute a call with a custom weight. +//! +//! 
+//! Note: this pallet is not meant to be used inside other pallets. It is only
+//! meant to be used by constructing runtime calls from outside the runtime.
+//!
+//! +//! This pallet also defines a [`SignedExtension`](sp_runtime::traits::SignedExtension) called +//! [`CheckOnlySudoAccount`] to ensure that only signed transactions by the sudo account are +//! accepted by the transaction pool. The intended use of this signed extension is to prevent other +//! accounts from spamming the transaction pool for the initial phase of a chain, during which +//! developers may only want a sudo account to be able to make transactions. +//! +//! Learn more about the `Root` origin in the [`RawOrigin`](frame_system::RawOrigin) type +//! documentation. //! -//! The Sudo pallet defines the following extension: +//! ### Examples //! -//! - [`CheckOnlySudoAccount`]: Ensures that the signed transactions are only valid if they are -//! signed by sudo account. +//! 1. You can make a privileged runtime call using `sudo` with an account that matches the sudo +//! key. +#![doc = docify::embed!("src/tests.rs", sudo_basics)] //! -//! ## Genesis Config +//! 2. Only an existing sudo key can set a new one. +#![doc = docify::embed!("src/tests.rs", set_key_basics)] //! -//! The Sudo pallet depends on the [`GenesisConfig`]. -//! You need to set an initial superuser account as the sudo `key`. +//! 3. You can also make non-privileged calls using `sudo_as`. +#![doc = docify::embed!("src/tests.rs", sudo_as_emits_events_correctly)] //! -//! ## Related Pallets +//! ## Low Level / Implementation Details //! -//! * [Democracy](../pallet_democracy/index.html) +//! This pallet checks that the caller of its dispatchables is a signed account and ensures that the +//! caller matches the sudo key in storage. +//! A caller of this pallet's dispatchables does not pay any fees to dispatch a call. If the account +//! making one of these calls is not the sudo key, the pallet returns a [`Error::RequireSudo`] +//! error. //! -//! [`Origin`]: https://docs.substrate.io/main-docs/build/origins/ +//! Once an origin is verified, sudo calls use `dispatch_bypass_filter` from the +//! [`UnfilteredDispatchable`](frame_support::traits::UnfilteredDispatchable) trait to allow call +//! execution without enforcing any further origin checks. +#![deny(missing_docs)] #![cfg_attr(not(feature = "std"), no_std)] use sp_runtime::{traits::StaticLookup, DispatchResult}; @@ -261,12 +283,21 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { - /// A sudo just took place. \[result\] - Sudid { sudo_result: DispatchResult }, - /// The \[sudoer\] just switched identity; the old key is supplied if one existed. - KeyChanged { old_sudoer: Option }, - /// A sudo just took place. \[result\] - SudoAsDone { sudo_result: DispatchResult }, + /// A sudo call just took place. + Sudid { + /// The result of the call made by the sudo user. + sudo_result: DispatchResult, + }, + /// The sudo key has been updated. + KeyChanged { + /// The old sudo key if one was previously set. + old_sudoer: Option, + }, + /// A [sudo_as](Pallet::sudo_as) call just took place. + SudoAsDone { + /// The result of the call made by the sudo user. + sudo_result: DispatchResult, + }, } #[pallet::error] diff --git a/substrate/frame/sudo/src/tests.rs b/substrate/frame/sudo/src/tests.rs index c854fed8f073..6963ba2e6a05 100644 --- a/substrate/frame/sudo/src/tests.rs +++ b/substrate/frame/sudo/src/tests.rs @@ -34,6 +34,7 @@ fn test_setup_works() { }); } +#[docify::export] #[test] fn sudo_basics() { // Configure a default test environment and set the root `key` to 1. @@ -134,6 +135,7 @@ fn sudo_unchecked_weight_emits_events_correctly() { }) } +#[docify::export] #[test] fn set_key_basics() { new_test_ext(1).execute_with(|| { @@ -195,6 +197,7 @@ fn sudo_as_basics() { }); } +#[docify::export] #[test] fn sudo_as_emits_events_correctly() { new_test_ext(1).execute_with(|| { diff --git a/substrate/frame/sudo/src/weights.rs b/substrate/frame/sudo/src/weights.rs index 6a0197d1469b..0cdd0c8a81f2 100644 --- a/substrate/frame/sudo/src/weights.rs +++ b/substrate/frame/sudo/src/weights.rs @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Autogenerated weights for pallet_sudo +//! Autogenerated weights for pallet_sudo. //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev //! DATE: 2023-06-16, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` diff --git a/substrate/frame/support/Cargo.toml b/substrate/frame/support/Cargo.toml index 864e3bd9ad30..62a7b5e41fe7 100644 --- a/substrate/frame/support/Cargo.toml +++ b/substrate/frame/support/Cargo.toml @@ -42,8 +42,8 @@ sp-core-hashing-proc-macro = { path = "../../primitives/core/hashing/proc-macro" k256 = { version = "0.13.1", default-features = false, features = ["ecdsa"] } environmental = { version = "1.1.4", default-features = false } sp-genesis-builder = { path = "../../primitives/genesis-builder", default-features=false} -serde_json = { version = "1.0.85", default-features = false, features = ["alloc"] } -docify = "0.2.1" +serde_json = { version = "1.0.107", default-features = false, features = ["alloc"] } +docify = "0.2.3" static_assertions = "1.1.0" aquamarine = { version = "0.3.2" } diff --git a/substrate/frame/support/procedural/Cargo.toml b/substrate/frame/support/procedural/Cargo.toml index 9ca86b9fccb1..b582457e4b8d 100644 --- a/substrate/frame/support/procedural/Cargo.toml +++ b/substrate/frame/support/procedural/Cargo.toml @@ -17,11 +17,11 @@ proc-macro = true [dependencies] derive-syn-parse = "0.1.5" Inflector = "0.11.4" -cfg-expr = "0.15.4" +cfg-expr = "0.15.5" itertools = "0.10.3" proc-macro2 = "1.0.56" quote = "1.0.28" -syn = { version = "2.0.31", features = ["full"] } +syn = { version = "2.0.33", features = ["full"] } frame-support-procedural-tools = { path = "tools" } proc-macro-warning = { version = "0.4.2", default-features = false } macro_magic = { version = "0.4.2", features = ["proc_support"] } diff --git a/substrate/frame/support/procedural/tools/Cargo.toml b/substrate/frame/support/procedural/tools/Cargo.toml index 46c264256513..211dc3bd66a5 100644 --- a/substrate/frame/support/procedural/tools/Cargo.toml +++ b/substrate/frame/support/procedural/tools/Cargo.toml @@ -15,5 +15,5 @@ targets = ["x86_64-unknown-linux-gnu"] proc-macro-crate = "1.1.3" proc-macro2 = "1.0.56" quote = "1.0.28" -syn = { version = "2.0.31", features = ["full", "visit", "extra-traits"] } +syn = { version = "2.0.33", features = ["full", "visit", "extra-traits"] } frame-support-procedural-tools-derive = { path = "derive" } diff --git a/substrate/frame/support/procedural/tools/derive/Cargo.toml b/substrate/frame/support/procedural/tools/derive/Cargo.toml index da01e6f4f6af..472e288c3df8 100644 --- a/substrate/frame/support/procedural/tools/derive/Cargo.toml +++ b/substrate/frame/support/procedural/tools/derive/Cargo.toml @@ -17,4 +17,4 @@ proc-macro = true [dependencies] proc-macro2 = "1.0.56" quote = { version = "1.0.28", features = ["proc-macro"] } -syn = { version = "2.0.31", features = ["proc-macro", "full", "extra-traits", "parsing"] } +syn = { version = "2.0.33", features = ["proc-macro", "full", "extra-traits", "parsing"] } diff --git a/substrate/frame/support/src/migrations.rs b/substrate/frame/support/src/migrations.rs index 6fc1834f01ad..8f490030c25b 100644 --- a/substrate/frame/support/src/migrations.rs +++ b/substrate/frame/support/src/migrations.rs @@ -24,9 +24,7 @@ use sp_core::Get; use sp_io::{hashing::twox_128, storage::clear_prefix, KillStorageResult}; use sp_std::marker::PhantomData; -/// EXPERIMENTAL: The API of this feature may change. -/// -/// Make it easier to write versioned runtime upgrades. +/// Handles storage migration pallet versioning. /// /// [`VersionedMigration`] allows developers to write migrations without worrying about checking and /// setting storage versions. Instead, the developer wraps their migration in this struct which @@ -69,14 +67,12 @@ use sp_std::marker::PhantomData; /// // other migrations... /// ); /// ``` -#[cfg(feature = "experimental")] pub struct VersionedMigration { _marker: PhantomData<(Inner, Pallet, Weight)>, } /// A helper enum to wrap the pre_upgrade bytes like an Option before passing them to post_upgrade. /// This enum is used rather than an Option to make the API clearer to the developer. -#[cfg(feature = "experimental")] #[derive(codec::Encode, codec::Decode)] pub enum VersionedPostUpgradeData { /// The migration ran, inner vec contains pre_upgrade data. @@ -91,7 +87,6 @@ pub enum VersionedPostUpgradeData { /// version of the pallets storage matches `From`, and after the upgrade set the on-chain storage to /// `To`. If the versions do not match, it writes a log notifying the developer that the migration /// is a noop. -#[cfg(feature = "experimental")] impl< const FROM: u16, const TO: u16, diff --git a/substrate/frame/support/test/tests/versioned_migration.rs b/substrate/frame/support/test/tests/versioned_migration.rs index a0766f6bec25..03dbd948df92 100644 --- a/substrate/frame/support/test/tests/versioned_migration.rs +++ b/substrate/frame/support/test/tests/versioned_migration.rs @@ -17,7 +17,7 @@ //! Tests for [`VersionedMigration`] -#![cfg(all(feature = "experimental", feature = "try-runtime"))] +#![cfg(feature = "try-runtime")] use frame_support::{ construct_runtime, derive_impl, diff --git a/substrate/frame/transaction-payment/Cargo.toml b/substrate/frame/transaction-payment/Cargo.toml index 1db63bd134ad..5c65ebd4c730 100644 --- a/substrate/frame/transaction-payment/Cargo.toml +++ b/substrate/frame/transaction-payment/Cargo.toml @@ -26,7 +26,7 @@ sp-runtime = { path = "../../primitives/runtime", default-features = false} sp-std = { path = "../../primitives/std", default-features = false} [dev-dependencies] -serde_json = "1.0.85" +serde_json = "1.0.107" pallet-balances = { path = "../balances" } [features] diff --git a/substrate/frame/transaction-payment/asset-tx-payment/Cargo.toml b/substrate/frame/transaction-payment/asset-tx-payment/Cargo.toml index edf05a053317..d3f040e9893f 100644 --- a/substrate/frame/transaction-payment/asset-tx-payment/Cargo.toml +++ b/substrate/frame/transaction-payment/asset-tx-payment/Cargo.toml @@ -30,7 +30,7 @@ scale-info = { version = "2.5.0", default-features = false, features = ["derive" serde = { version = "1.0.188", optional = true } [dev-dependencies] -serde_json = "1.0.85" +serde_json = "1.0.107" sp-storage = { path = "../../../primitives/storage", default-features = false} diff --git a/substrate/frame/tx-pause/Cargo.toml b/substrate/frame/tx-pause/Cargo.toml index 356693d90f04..6d96cb8abe79 100644 --- a/substrate/frame/tx-pause/Cargo.toml +++ b/substrate/frame/tx-pause/Cargo.toml @@ -7,7 +7,6 @@ license = "Apache-2.0" homepage = "https://substrate.io" repository.workspace = true description = "FRAME transaction pause pallet" -readme = "README.md" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] diff --git a/substrate/primitives/api/proc-macro/Cargo.toml b/substrate/primitives/api/proc-macro/Cargo.toml index 6819d139f450..d53b9b702e7b 100644 --- a/substrate/primitives/api/proc-macro/Cargo.toml +++ b/substrate/primitives/api/proc-macro/Cargo.toml @@ -17,7 +17,7 @@ proc-macro = true [dependencies] quote = "1.0.28" -syn = { version = "2.0.31", features = ["full", "fold", "extra-traits", "visit"] } +syn = { version = "2.0.33", features = ["full", "fold", "extra-traits", "visit"] } proc-macro2 = "1.0.56" blake2 = { version = "0.10.4", default-features = false } proc-macro-crate = "1.1.3" diff --git a/substrate/primitives/core/Cargo.toml b/substrate/primitives/core/Cargo.toml index 4e5186f16878..9f1f0127d7cd 100644 --- a/substrate/primitives/core/Cargo.toml +++ b/substrate/primitives/core/Cargo.toml @@ -13,7 +13,6 @@ documentation = "https://docs.rs/sp-core" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -arrayvec = { version = "0.7.2", default-features = false } codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive","max-encoded-len"] } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } log = { version = "0.4.17", default-features = false } @@ -58,7 +57,7 @@ sp-runtime-interface = { path = "../runtime-interface", default-features = false # bls crypto w3f-bls = { version = "0.1.3", default-features = false, optional = true} # bandersnatch crypto -bandersnatch_vrfs = { git = "https://github.com/w3f/ring-vrf", rev = "3119f51", default-features = false, optional = true } +bandersnatch_vrfs = { git = "https://github.com/w3f/ring-vrf", rev = "f4fe253", default-features = false, optional = true } [dev-dependencies] criterion = "0.4.0" @@ -76,7 +75,6 @@ bench = false default = [ "std" ] std = [ "array-bytes", - "arrayvec/std", "bandersnatch_vrfs/getrandom", "blake2/std", "bounded-collections/std", diff --git a/substrate/primitives/core/hashing/proc-macro/Cargo.toml b/substrate/primitives/core/hashing/proc-macro/Cargo.toml index ef17473a6f39..5e7cd9e3a7a4 100644 --- a/substrate/primitives/core/hashing/proc-macro/Cargo.toml +++ b/substrate/primitives/core/hashing/proc-macro/Cargo.toml @@ -17,5 +17,5 @@ proc-macro = true [dependencies] quote = "1.0.28" -syn = { version = "2.0.31", features = ["full", "parsing"] } +syn = { version = "2.0.33", features = ["full", "parsing"] } sp-core-hashing = { path = "..", default-features = false} diff --git a/substrate/primitives/core/src/bandersnatch.rs b/substrate/primitives/core/src/bandersnatch.rs index 01f3538188a4..832ef6c77bb3 100644 --- a/substrate/primitives/core/src/bandersnatch.rs +++ b/substrate/primitives/core/src/bandersnatch.rs @@ -53,9 +53,8 @@ const SEED_SERIALIZED_LEN: usize = 32; // Short-Weierstrass form serialized sizes. const PUBLIC_SERIALIZED_LEN: usize = 33; const SIGNATURE_SERIALIZED_LEN: usize = 65; +const RING_SIGNATURE_SERIALIZED_LEN: usize = 755; const PREOUT_SERIALIZED_LEN: usize = 33; -const PEDERSEN_SIGNATURE_SERIALIZED_LEN: usize = 163; -const RING_PROOF_SERIALIZED_LEN: usize = 592; // Max size of serialized ring-vrf context params. // @@ -69,7 +68,7 @@ const RING_PROOF_SERIALIZED_LEN: usize = 592; // 2048 → 295 KB // NOTE: This is quite big but looks like there is an upcoming fix // in the backend. -const RING_CONTEXT_SERIALIZED_LEN: usize = 147752; +const RING_CONTEXT_SERIALIZED_LEN: usize = 147748; /// Bandersnatch public key. #[cfg_attr(feature = "full_crypto", derive(Hash))] @@ -278,7 +277,7 @@ impl TraitPair for Pair { let mut raw = [0; PUBLIC_SERIALIZED_LEN]; public .serialize_compressed(raw.as_mut_slice()) - .expect("key buffer length is good; qed"); + .expect("serialization length is constant and checked by test; qed"); Public::unchecked_from(raw) } @@ -356,7 +355,7 @@ pub mod vrf { let mut bytes = [0; PREOUT_SERIALIZED_LEN]; self.0 .serialize_compressed(bytes.as_mut_slice()) - .expect("preout serialization can't fail"); + .expect("serialization length is constant and checked by test; qed"); bytes.encode() } } @@ -506,7 +505,7 @@ pub mod vrf { } fn vrf_output(&self, input: &Self::VrfInput) -> Self::VrfOutput { - let output = self.secret.0.vrf_preout(&input.0); + let output = self.secret.vrf_preout(&input.0); VrfOutput(output) } } @@ -539,24 +538,26 @@ pub mod vrf { #[cfg(feature = "full_crypto")] impl Pair { fn vrf_sign_gen(&self, data: &VrfSignData) -> VrfSignature { - let ios: Vec<_> = data - .inputs - .iter() - .map(|i| self.secret.clone().0.vrf_inout(i.0.clone())) - .collect(); + let ios = core::array::from_fn(|i| { + let input = data.inputs[i].0.clone(); + self.secret.vrf_inout(input) + }); - let signature: ThinVrfSignature = - self.secret.sign_thin_vrf(data.transcript.clone(), ios.as_slice()); + let thin_signature: ThinVrfSignature = + self.secret.sign_thin_vrf(data.transcript.clone(), &ios); - let mut sign_bytes = [0; SIGNATURE_SERIALIZED_LEN]; - signature - .signature - .serialize_compressed(sign_bytes.as_mut_slice()) - .expect("serialization can't fail"); - - let outputs: Vec<_> = signature.preoutputs.into_iter().map(VrfOutput).collect(); + let outputs: Vec<_> = thin_signature.preouts.into_iter().map(VrfOutput).collect(); let outputs = VrfIosVec::truncate_from(outputs); - VrfSignature { signature: Signature(sign_bytes), outputs } + + let mut signature = + VrfSignature { signature: Signature([0; SIGNATURE_SERIALIZED_LEN]), outputs }; + + thin_signature + .proof + .serialize_compressed(signature.signature.0.as_mut_slice()) + .expect("serialization length is constant and checked by test; qed"); + + signature } /// Generate an arbitrary number of bytes from the given `context` and VRF `input`. @@ -566,7 +567,7 @@ pub mod vrf { input: &VrfInput, ) -> [u8; N] { let transcript = Transcript::new_labeled(context); - let inout = self.secret.clone().0.vrf_inout(input.0.clone()); + let inout = self.secret.vrf_inout(input.0.clone()); inout.vrf_output_bytes(transcript) } } @@ -581,30 +582,23 @@ pub mod vrf { return false }; - let Ok(preouts) = signature - .outputs - .iter() - .map(|o| o.0.clone()) - .collect::>() - .into_inner() - else { - return false - }; + let preouts: [bandersnatch_vrfs::VrfPreOut; N] = + core::array::from_fn(|i| signature.outputs[i].0.clone()); // Deserialize only the proof, the rest has already been deserialized // This is another hack used because backend signature type is generic over // the number of ios. - let Ok(signature) = + let Ok(proof) = ThinVrfSignature::<0>::deserialize_compressed(signature.signature.as_ref()) - .map(|s| s.signature) + .map(|s| s.proof) else { return false }; - let signature = ThinVrfSignature { signature, preoutputs: preouts }; + let signature = ThinVrfSignature { proof, preouts }; let inputs = data.inputs.iter().map(|i| i.0.clone()); - signature.verify_thin_vrf(data.transcript.clone(), inputs, &public).is_ok() + public.verify_thin_vrf(data.transcript.clone(), inputs, &signature).is_ok() } } @@ -627,7 +621,7 @@ pub mod vrf { pub mod ring_vrf { use super::{vrf::*, *}; pub use bandersnatch_vrfs::ring::{RingProof, RingProver, RingVerifier, KZG}; - use bandersnatch_vrfs::{CanonicalDeserialize, PedersenVrfSignature, PublicKey}; + use bandersnatch_vrfs::{CanonicalDeserialize, PublicKey}; /// Context used to produce ring signatures. #[derive(Clone)] @@ -649,7 +643,7 @@ pub mod ring_vrf { let mut pks = Vec::with_capacity(public_keys.len()); for public_key in public_keys { let pk = PublicKey::deserialize_compressed(public_key.as_slice()).ok()?; - pks.push(pk.0 .0.into()); + pks.push(pk.0.into()); } let prover_key = self.0.prover_key(pks); @@ -662,7 +656,7 @@ pub mod ring_vrf { let mut pks = Vec::with_capacity(public_keys.len()); for public_key in public_keys { let pk = PublicKey::deserialize_compressed(public_key.as_slice()).ok()?; - pks.push(pk.0 .0.into()); + pks.push(pk.0.into()); } let verifier_key = self.0.verifier_key(pks); @@ -676,7 +670,7 @@ pub mod ring_vrf { let mut buf = Box::new([0; RING_CONTEXT_SERIALIZED_LEN]); self.0 .serialize_compressed(buf.as_mut_slice()) - .expect("preout serialization can't fail"); + .expect("serialization length is constant and checked by test; qed"); buf.encode() } } @@ -711,10 +705,8 @@ pub mod ring_vrf { pub struct RingVrfSignature { /// VRF (pre)outputs. pub outputs: VrfIosVec, - /// Pedersen VRF signature. - pub signature: [u8; PEDERSEN_SIGNATURE_SERIALIZED_LEN], - /// Ring proof. - pub ring_proof: [u8; RING_PROOF_SERIALIZED_LEN], + /// Ring signature. + pub signature: [u8; RING_SIGNATURE_SERIALIZED_LEN], } #[cfg(feature = "full_crypto")] @@ -741,31 +733,27 @@ pub mod ring_vrf { data: &VrfSignData, prover: &RingProver, ) -> RingVrfSignature { - let ios: Vec<_> = data - .inputs - .iter() - .map(|i| self.secret.clone().0.vrf_inout(i.0.clone())) - .collect(); + let ios = core::array::from_fn(|i| { + let input = data.inputs[i].0.clone(); + self.secret.vrf_inout(input) + }); let ring_signature: bandersnatch_vrfs::RingVrfSignature = - self.secret.sign_ring_vrf(data.transcript.clone(), ios.as_slice(), prover); + bandersnatch_vrfs::RingProver { ring_prover: prover, secret: &self.secret } + .sign_ring_vrf(data.transcript.clone(), &ios); - let outputs: Vec<_> = ring_signature.preoutputs.into_iter().map(VrfOutput).collect(); + let outputs: Vec<_> = ring_signature.preouts.into_iter().map(VrfOutput).collect(); let outputs = VrfIosVec::truncate_from(outputs); - let mut signature = [0; PEDERSEN_SIGNATURE_SERIALIZED_LEN]; - ring_signature - .signature - .serialize_compressed(signature.as_mut_slice()) - .expect("ped-signature serialization can't fail"); + let mut signature = + RingVrfSignature { outputs, signature: [0; RING_SIGNATURE_SERIALIZED_LEN] }; - let mut ring_proof = [0; RING_PROOF_SERIALIZED_LEN]; ring_signature - .ring_proof - .serialize_compressed(ring_proof.as_mut_slice()) - .expect("ring-proof serialization can't fail"); + .proof + .serialize_compressed(signature.signature.as_mut_slice()) + .expect("serialization length is constant and checked by test; qed"); - RingVrfSignature { outputs, signature, ring_proof } + signature } } @@ -774,7 +762,7 @@ pub mod ring_vrf { /// /// The signature is verifiable if it has been produced by a member of the ring /// from which the [`RingVerifier`] has been constructed. - pub fn verify(&self, data: &VrfSignData, verifier: &RingVerifier) -> bool { + pub fn ring_vrf_verify(&self, data: &VrfSignData, verifier: &RingVerifier) -> bool { const _: () = assert!(MAX_VRF_IOS == 3, "`MAX_VRF_IOS` expected to be 3"); let preouts_len = self.outputs.len(); if preouts_len != data.inputs.len() { @@ -782,43 +770,37 @@ pub mod ring_vrf { } // Workaround to overcome backend signature generic over the number of IOs. match preouts_len { - 0 => self.verify_gen::<0>(data, verifier), - 1 => self.verify_gen::<1>(data, verifier), - 2 => self.verify_gen::<2>(data, verifier), - 3 => self.verify_gen::<3>(data, verifier), + 0 => self.ring_vrf_verify_gen::<0>(data, verifier), + 1 => self.ring_vrf_verify_gen::<1>(data, verifier), + 2 => self.ring_vrf_verify_gen::<2>(data, verifier), + 3 => self.ring_vrf_verify_gen::<3>(data, verifier), _ => unreachable!(), } } - fn verify_gen(&self, data: &VrfSignData, verifier: &RingVerifier) -> bool { - let Ok(preoutputs) = self - .outputs - .iter() - .map(|o| o.0.clone()) - .collect::>() - .into_inner() - else { - return false - }; - - let Ok(signature) = - PedersenVrfSignature::deserialize_compressed(self.signature.as_slice()) + fn ring_vrf_verify_gen( + &self, + data: &VrfSignData, + verifier: &RingVerifier, + ) -> bool { + let Ok(vrf_signature) = + bandersnatch_vrfs::RingVrfSignature::<0>::deserialize_compressed( + self.signature.as_slice(), + ) else { return false }; - let Ok(ring_proof) = RingProof::deserialize_compressed(self.ring_proof.as_slice()) - else { - return false - }; + let preouts: [bandersnatch_vrfs::VrfPreOut; N] = + core::array::from_fn(|i| self.outputs[i].0.clone()); - let ring_signature = - bandersnatch_vrfs::RingVrfSignature { signature, preoutputs, ring_proof }; + let signature = + bandersnatch_vrfs::RingVrfSignature { proof: vrf_signature.proof, preouts }; let inputs = data.inputs.iter().map(|i| i.0.clone()); - ring_signature - .verify_ring_vrf(data.transcript.clone(), inputs, verifier) + bandersnatch_vrfs::RingVerifier(verifier) + .verify_ring_vrf(data.transcript.clone(), inputs, &signature) .is_ok() } } @@ -840,16 +822,41 @@ mod tests { } #[test] - fn assumptions_sanity_check() { - // Backend - let ring_ctx = RingContext::new_testing(); - let pair = SecretKey::from_seed(DEV_SEED); - let public = pair.to_public(); + fn backend_assumptions_sanity_check() { + let kzg = KZG::testing_kzg_setup([0; 32], RING_DOMAIN_SIZE as u32); + assert_eq!(kzg.max_keyset_size(), RING_DOMAIN_SIZE - 257); + assert_eq!(kzg.compressed_size(), RING_CONTEXT_SERIALIZED_LEN); + + let pks: Vec<_> = (0..16) + .map(|i| SecretKey::from_seed(&[i as u8; 32]).to_public().0.into()) + .collect(); - assert_eq!(public.0.size_of_serialized(), PUBLIC_SERIALIZED_LEN); - assert_eq!(ring_ctx.max_keyset_size(), RING_DOMAIN_SIZE - 257); + let secret = SecretKey::from_seed(&[0u8; 32]); - // Wrapper + let public = secret.to_public(); + assert_eq!(public.compressed_size(), PUBLIC_SERIALIZED_LEN); + + let input = VrfInput::new(b"foo", &[]); + let preout = secret.vrf_preout(&input.0); + assert_eq!(preout.compressed_size(), PREOUT_SERIALIZED_LEN); + + let prover_key = kzg.prover_key(pks); + let ring_prover = kzg.init_ring_prover(prover_key, 0); + + let data = VrfSignData::new_unchecked(b"mydata", &[b"tdata"], None); + + let thin_signature: bandersnatch_vrfs::ThinVrfSignature<0> = + secret.sign_thin_vrf(data.transcript.clone(), &[]); + assert_eq!(thin_signature.compressed_size(), SIGNATURE_SERIALIZED_LEN); + + let ring_signature: bandersnatch_vrfs::RingVrfSignature<0> = + bandersnatch_vrfs::RingProver { ring_prover: &ring_prover, secret: &secret } + .sign_ring_vrf(data.transcript.clone(), &[]); + assert_eq!(ring_signature.compressed_size(), RING_SIGNATURE_SERIALIZED_LEN); + } + + #[test] + fn max_vrf_ios_bound_respected() { let inputs: Vec<_> = (0..MAX_VRF_IOS - 1).map(|_| VrfInput::new(b"", &[])).collect(); let mut sign_data = VrfSignData::new(b"", &[b""], inputs).unwrap(); let res = sign_data.push_vrf_input(VrfInput::new(b"", b"")); @@ -987,7 +994,7 @@ mod tests { let signature = pair.ring_vrf_sign(&data, &prover); let verifier = ring_ctx.verifier(&pks).unwrap(); - assert!(signature.verify(&data, &verifier)); + assert!(signature.ring_vrf_verify(&data, &verifier)); } #[test] @@ -1006,7 +1013,7 @@ mod tests { let signature = pair.ring_vrf_sign(&data, &prover); let verifier = ring_ctx.verifier(&pks).unwrap(); - assert!(!signature.verify(&data, &verifier)); + assert!(!signature.ring_vrf_verify(&data, &verifier)); } #[test] @@ -1062,10 +1069,8 @@ mod tests { let bytes = expected.encode(); - let expected_len = data.inputs.len() * PREOUT_SERIALIZED_LEN + - PEDERSEN_SIGNATURE_SERIALIZED_LEN + - RING_PROOF_SERIALIZED_LEN + - 1; + let expected_len = + data.inputs.len() * PREOUT_SERIALIZED_LEN + RING_SIGNATURE_SERIALIZED_LEN + 1; assert_eq!(bytes.len(), expected_len); let decoded = RingVrfSignature::decode(&mut bytes.as_slice()).unwrap(); diff --git a/substrate/primitives/debug-derive/Cargo.toml b/substrate/primitives/debug-derive/Cargo.toml index 292c07e092f9..82f4e8cf6048 100644 --- a/substrate/primitives/debug-derive/Cargo.toml +++ b/substrate/primitives/debug-derive/Cargo.toml @@ -18,7 +18,7 @@ proc-macro = true [dependencies] quote = "1.0.28" -syn = "2.0.31" +syn = "2.0.33" proc-macro2 = "1.0.56" [features] diff --git a/substrate/primitives/genesis-builder/Cargo.toml b/substrate/primitives/genesis-builder/Cargo.toml index fb92f0223d06..8f083e2d7d54 100644 --- a/substrate/primitives/genesis-builder/Cargo.toml +++ b/substrate/primitives/genesis-builder/Cargo.toml @@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] sp-api = { path = "../api", default-features = false} sp-runtime = { path = "../runtime", default-features = false} sp-std = { path = "../std", default-features = false} -serde_json = { version = "1.0.85", default-features = false, features = ["alloc"] } +serde_json = { version = "1.0.107", default-features = false, features = ["alloc"] } [features] default = [ "std" ] diff --git a/substrate/primitives/npos-elections/fuzzer/Cargo.toml b/substrate/primitives/npos-elections/fuzzer/Cargo.toml index 86927786c58f..1e9f0517df12 100644 --- a/substrate/primitives/npos-elections/fuzzer/Cargo.toml +++ b/substrate/primitives/npos-elections/fuzzer/Cargo.toml @@ -14,7 +14,7 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } honggfuzz = "0.5" rand = { version = "0.8", features = ["std", "small_rng"] } sp-npos-elections = { path = ".." } diff --git a/substrate/primitives/rpc/Cargo.toml b/substrate/primitives/rpc/Cargo.toml index 21447dafb050..4739c96740e9 100644 --- a/substrate/primitives/rpc/Cargo.toml +++ b/substrate/primitives/rpc/Cargo.toml @@ -18,4 +18,4 @@ serde = { version = "1.0.188", features = ["derive"] } sp-core = { path = "../core" } [dev-dependencies] -serde_json = "1.0.85" +serde_json = "1.0.107" diff --git a/substrate/primitives/runtime-interface/proc-macro/Cargo.toml b/substrate/primitives/runtime-interface/proc-macro/Cargo.toml index e01343d6dc0a..0884efc56d4e 100644 --- a/substrate/primitives/runtime-interface/proc-macro/Cargo.toml +++ b/substrate/primitives/runtime-interface/proc-macro/Cargo.toml @@ -20,4 +20,4 @@ Inflector = "0.11.4" proc-macro-crate = "1.1.3" proc-macro2 = "1.0.56" quote = "1.0.28" -syn = { version = "2.0.31", features = ["full", "visit", "fold", "extra-traits"] } +syn = { version = "2.0.33", features = ["full", "visit", "fold", "extra-traits"] } diff --git a/substrate/primitives/runtime/Cargo.toml b/substrate/primitives/runtime/Cargo.toml index 7050c27bc849..fa3c3ae2e6ea 100644 --- a/substrate/primitives/runtime/Cargo.toml +++ b/substrate/primitives/runtime/Cargo.toml @@ -32,7 +32,7 @@ sp-weights = { path = "../weights", default-features = false} [dev-dependencies] rand = "0.8.5" -serde_json = "1.0.85" +serde_json = "1.0.107" zstd = { version = "0.12.4", default-features = false } sp-api = { path = "../api" } sp-state-machine = { path = "../state-machine" } diff --git a/substrate/primitives/state-machine/Cargo.toml b/substrate/primitives/state-machine/Cargo.toml index 8546345e5cae..ec5d9b5ea14e 100644 --- a/substrate/primitives/state-machine/Cargo.toml +++ b/substrate/primitives/state-machine/Cargo.toml @@ -27,7 +27,7 @@ sp-externalities = { path = "../externalities", default-features = false} sp-panic-handler = { path = "../panic-handler", optional = true} sp-std = { path = "../std", default-features = false} sp-trie = { path = "../trie", default-features = false} -trie-db = { version = "0.27.1", default-features = false } +trie-db = { version = "0.28.0", default-features = false } [dev-dependencies] array-bytes = "6.1" diff --git a/substrate/primitives/trie/Cargo.toml b/substrate/primitives/trie/Cargo.toml index 0bce597414b1..0b54514f6003 100644 --- a/substrate/primitives/trie/Cargo.toml +++ b/substrate/primitives/trie/Cargo.toml @@ -29,7 +29,7 @@ parking_lot = { version = "0.12.1", optional = true } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } thiserror = { version = "1.0.48", optional = true } tracing = { version = "0.1.29", optional = true } -trie-db = { version = "0.27.0", default-features = false } +trie-db = { version = "0.28.0", default-features = false } trie-root = { version = "0.18.0", default-features = false } sp-core = { path = "../core", default-features = false} sp-std = { path = "../std", default-features = false} @@ -38,7 +38,7 @@ schnellru = { version = "0.2.1", optional = true } [dev-dependencies] array-bytes = "6.1" criterion = "0.4.0" -trie-bench = "0.37.0" +trie-bench = "0.38.0" trie-standardmap = "0.16.0" sp-runtime = { path = "../runtime" } diff --git a/substrate/primitives/trie/src/recorder.rs b/substrate/primitives/trie/src/recorder.rs index 728dc836205b..154cee3f37dc 100644 --- a/substrate/primitives/trie/src/recorder.rs +++ b/substrate/primitives/trie/src/recorder.rs @@ -379,6 +379,17 @@ impl>> trie_db::TrieRecord // that the value doesn't exist in the trie. self.update_recorded_keys(full_key, RecordedForKey::Value); }, + TrieAccess::InlineValue { full_key } => { + tracing::trace!( + target: LOG_TARGET, + key = ?sp_core::hexdisplay::HexDisplay::from(&full_key), + "Recorded inline value access for key", + ); + + // A value was accessed that is stored inline a node and we recorded all trie nodes + // to access this value. + self.update_recorded_keys(full_key, RecordedForKey::Value); + }, }; self.encoded_size_estimation.fetch_add(encoded_size_update, Ordering::Relaxed); diff --git a/substrate/primitives/version/proc-macro/Cargo.toml b/substrate/primitives/version/proc-macro/Cargo.toml index 25b5e7906f62..a3478e3e5ca9 100644 --- a/substrate/primitives/version/proc-macro/Cargo.toml +++ b/substrate/primitives/version/proc-macro/Cargo.toml @@ -19,7 +19,7 @@ proc-macro = true codec = { package = "parity-scale-codec", version = "3.6.1", features = [ "derive" ] } proc-macro2 = "1.0.56" quote = "1.0.28" -syn = { version = "2.0.31", features = ["full", "fold", "extra-traits", "visit"] } +syn = { version = "2.0.33", features = ["full", "fold", "extra-traits", "visit"] } [dev-dependencies] sp-version = { path = ".." } diff --git a/substrate/scripts/ci/node-template-release/Cargo.toml b/substrate/scripts/ci/node-template-release/Cargo.toml index 6c4cee8ece33..63611ae8da6b 100644 --- a/substrate/scripts/ci/node-template-release/Cargo.toml +++ b/substrate/scripts/ci/node-template-release/Cargo.toml @@ -11,7 +11,7 @@ publish = false targets = ["x86_64-unknown-linux-gnu"] [dependencies] -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } flate2 = "1.0" fs_extra = "1.3" glob = "0.3" diff --git a/substrate/test-utils/client/Cargo.toml b/substrate/test-utils/client/Cargo.toml index 91c30f5a8ece..12863ac184c1 100644 --- a/substrate/test-utils/client/Cargo.toml +++ b/substrate/test-utils/client/Cargo.toml @@ -18,7 +18,7 @@ async-trait = "0.1.57" codec = { package = "parity-scale-codec", version = "3.6.1" } futures = "0.3.21" serde = "1.0.188" -serde_json = "1.0.85" +serde_json = "1.0.107" sc-client-api = { path = "../../client/api" } sc-client-db = { path = "../../client/db", default-features = false, features = [ "test-helpers", diff --git a/substrate/test-utils/runtime/Cargo.toml b/substrate/test-utils/runtime/Cargo.toml index 4f587d55e79d..727d46cb8f53 100644 --- a/substrate/test-utils/runtime/Cargo.toml +++ b/substrate/test-utils/runtime/Cargo.toml @@ -40,7 +40,7 @@ pallet-timestamp = { path = "../../frame/timestamp", default-features = false} sp-consensus-grandpa = { path = "../../primitives/consensus/grandpa", default-features = false, features = ["serde"] } sp-trie = { path = "../../primitives/trie", default-features = false} sp-transaction-pool = { path = "../../primitives/transaction-pool", default-features = false} -trie-db = { version = "0.27.0", default-features = false } +trie-db = { version = "0.28.0", default-features = false } sc-service = { path = "../../client/service", default-features = false, features = ["test-helpers"], optional = true} sp-state-machine = { path = "../../primitives/state-machine", default-features = false} sp-externalities = { path = "../../primitives/externalities", default-features = false} @@ -49,7 +49,7 @@ sp-externalities = { path = "../../primitives/externalities", default-features = array-bytes = { version = "6.1", optional = true } log = { version = "0.4.17", default-features = false } serde = { version = "1.0.188", features = ["alloc", "derive"], default-features = false } -serde_json = { version = "1.0.85", default-features = false, features = ["alloc"] } +serde_json = { version = "1.0.107", default-features = false, features = ["alloc"] } [dev-dependencies] futures = "0.3.21" diff --git a/substrate/utils/build-script-utils/src/version.rs b/substrate/utils/build-script-utils/src/version.rs index 4ee5376ed322..309c6d71d77b 100644 --- a/substrate/utils/build-script-utils/src/version.rs +++ b/substrate/utils/build-script-utils/src/version.rs @@ -41,6 +41,7 @@ pub fn generate_cargo_keys() { } }; + println!("cargo:rustc-env=SUBSTRATE_CLI_COMMIT_HASH={commit}"); println!("cargo:rustc-env=SUBSTRATE_CLI_IMPL_VERSION={}", get_version(&commit)) } diff --git a/substrate/utils/frame/benchmarking-cli/Cargo.toml b/substrate/utils/frame/benchmarking-cli/Cargo.toml index 85be451fa60c..017e4b4d5034 100644 --- a/substrate/utils/frame/benchmarking-cli/Cargo.toml +++ b/substrate/utils/frame/benchmarking-cli/Cargo.toml @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] array-bytes = "6.1" chrono = "0.4" -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } codec = { package = "parity-scale-codec", version = "3.6.1" } comfy-table = { version = "7.0.1", default-features = false } handlebars = "4.2.2" @@ -27,7 +27,7 @@ log = "0.4.17" rand = { version = "0.8.4", features = ["small_rng"] } rand_pcg = "0.3.1" serde = "1.0.188" -serde_json = "1.0.85" +serde_json = "1.0.107" thiserror = "1.0.48" thousands = "0.2.0" frame-benchmarking = { path = "../../../frame/benchmarking" } diff --git a/substrate/utils/frame/frame-utilities-cli/Cargo.toml b/substrate/utils/frame/frame-utilities-cli/Cargo.toml index d191506a2acb..89aef5338650 100644 --- a/substrate/utils/frame/frame-utilities-cli/Cargo.toml +++ b/substrate/utils/frame/frame-utilities-cli/Cargo.toml @@ -11,7 +11,7 @@ documentation = "https://docs.rs/substrate-frame-cli" readme = "README.md" [dependencies] -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } frame-support = { path = "../../../frame/support" } frame-system = { path = "../../../frame/system" } sc-cli = { path = "../../../client/cli" } diff --git a/substrate/utils/frame/generate-bags/node-runtime/Cargo.toml b/substrate/utils/frame/generate-bags/node-runtime/Cargo.toml index 43005ca5c4a3..b06674a62cb3 100644 --- a/substrate/utils/frame/generate-bags/node-runtime/Cargo.toml +++ b/substrate/utils/frame/generate-bags/node-runtime/Cargo.toml @@ -14,4 +14,4 @@ kitchensink-runtime = { path = "../../../../bin/node/runtime" } generate-bags = { path = ".." } # third-party -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } diff --git a/substrate/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml b/substrate/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml index d82377593c71..8ce9d06c2d11 100644 --- a/substrate/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml +++ b/substrate/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml @@ -19,7 +19,7 @@ serde = { version = "1", features = ["derive"] } sp-core = { path = "../../../../primitives/core" } sp-state-machine = { path = "../../../../primitives/state-machine" } sp-trie = { path = "../../../../primitives/trie" } -trie-db = "0.27.0" +trie-db = "0.28.0" jsonrpsee = { version = "0.16.2", features = ["client-core", "server", "macros"] } diff --git a/substrate/utils/frame/try-runtime/cli/Cargo.toml b/substrate/utils/frame/try-runtime/cli/Cargo.toml index 7d1c204422ae..c736c3f6cc56 100644 --- a/substrate/utils/frame/try-runtime/cli/Cargo.toml +++ b/substrate/utils/frame/try-runtime/cli/Cargo.toml @@ -35,12 +35,12 @@ frame-try-runtime = { path = "../../../../frame/try-runtime", optional = true} substrate-rpc-client = { path = "../../rpc/client" } async-trait = "0.1.57" -clap = { version = "4.4.2", features = ["derive"] } +clap = { version = "4.4.3", features = ["derive"] } hex = { version = "0.4.3", default-features = false } log = "0.4.17" parity-scale-codec = "3.6.1" serde = "1.0.188" -serde_json = "1.0.85" +serde_json = "1.0.107" zstd = { version = "0.12.4", default-features = false } [dev-dependencies] diff --git a/substrate/utils/wasm-builder/src/builder.rs b/substrate/utils/wasm-builder/src/builder.rs index 208b56077669..e0a30644b231 100644 --- a/substrate/utils/wasm-builder/src/builder.rs +++ b/substrate/utils/wasm-builder/src/builder.rs @@ -203,7 +203,10 @@ fn generate_crate_skip_build_env_name() -> String { /// Checks if the build of the WASM binary should be skipped. fn check_skip_build() -> bool { env::var(crate::SKIP_BUILD_ENV).is_ok() || - env::var(generate_crate_skip_build_env_name()).is_ok() + env::var(generate_crate_skip_build_env_name()).is_ok() || + // If we are running in docs.rs, let's skip building. + // https://docs.rs/about/builds#detecting-docsrs + env::var("DOCS_RS").is_ok() } /// Provide a dummy WASM binary if there doesn't exist one.