Merge branch 'master' into tsv-disabling-node-side

paritytech · Sep 16, 2023 · 5e7655f · 5e7655f
2 parents 0d0129c + 76c68ee
commit 5e7655f
Show file tree

Hide file tree

Showing 377 changed files with 3,708 additions and 5,086 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -0,0 +1,75 @@
+# Lists some code owners.
+#
+# A codeowner just oversees some part of the codebase. If an owned file is changed then the
+# corresponding codeowner receives a review request. An approval of the codeowner might be
+# required for merging a PR (depends on repository settings).
+#
+# For details about syntax, see:
+# https://help.github.com/en/articles/about-code-owners
+# But here are some important notes:
+#
+# - Glob syntax is git-like, e.g. `/core` means the core directory in the root, unlike `core`
+#   which can be everywhere.
+# - Multiple owners are supported.
+# - Either handle (e.g, @github_user or @github/team) or email can be used. Keep in mind,
+#   that handles might work better because they are more recognizable on GitHub,
+#   eyou can use them for mentioning unlike an email.
+# - The latest matching rule, if multiple, takes precedence.
+
+# CI
+/.github/ @paritytech/ci @paritytech/release-engineering
+/.gitlab-ci.yml @paritytech/ci
+/.gitlab/ @paritytech/ci
+
+# XCM
+/polkadot/xcm/ @paritytech/xcm
+
+# WASM executor, low-level client <-> WASM interface and other WASM-related code
+/substrate/client/allocator/ @koute
+/substrate/client/executor/ @koute
+/substrate/primitives/panic-handler/ @koute
+/substrate/primitives/runtime-interface/ @koute
+/substrate/primitives/wasm-interface/ @koute
+/substrate/utils/wasm-builder/ @koute
+
+# Systems-related bits and bobs on the client side
+/substrate/client/sysinfo/ @koute
+/substrate/client/tracing/ @koute
+
+# Documentation audit
+/substrate/primitives/runtime @paritytech/docs-audit
+/substrate/primitives/arithmetic @paritytech/docs-audit
+# /primitives/core (to be added later)
+# /primitives/io (to be added later)
+
+# FRAME
+/substrate/frame/ @paritytech/frame-coders @paritytech/docs-audit
+/substrate/frame/nfts/ @jsidorenko @paritytech/docs-audit
+/substrate/frame/state-trie-migration/ @paritytech/frame-coders @cheme
+/substrate/frame/uniques/ @jsidorenko @paritytech/docs-audit
+
+# GRANDPA, BABE, consensus stuff
+/substrate/client/consensus/babe/ @andresilva
+/substrate/client/consensus/grandpa/ @andresilva
+/substrate/client/consensus/pow/ @sorpaas
+/substrate/client/consensus/slots/ @andresilva
+/substrate/frame/babe/ @andresilva
+/substrate/frame/grandpa/ @andresilva
+/substrate/primitives/consensus/pow/ @sorpaas
+
+# BEEFY, MMR
+/substrate/frame/beefy/ @acatangiu
+/substrate/frame/beefy-mmr/ @acatangiu
+/substrate/frame/merkle-mountain-range/ @acatangiu
+/substrate/primitives/merkle-mountain-range/ @acatangiu
+
+# Contracts
+/substrate/frame/contracts/ @athei @paritytech/docs-audit
+
+# NPoS and election
+/substrate/frame/election-provider-multi-phase/ @paritytech/staking-core @paritytech/docs-audit
+/substrate/frame/election-provider-support/ @paritytech/staking-core @paritytech/docs-audit
+/substrate/frame/elections-phragmen/ @paritytech/staking-core @paritytech/docs-audit
+/substrate/frame/nomination-pools/ @paritytech/staking-core @paritytech/docs-audit
+/substrate/frame/staking/ @paritytech/staking-core @paritytech/docs-audit
+/substrate/primitives/npos-elections/ @paritytech/staking-core @paritytech/docs-audit
diff --git a/.github/pr-custom-review.yml b/.github/pr-custom-review.yml
@@ -8,7 +8,7 @@ rules:
     check_type: changed_files
     condition:
       include: ^\.gitlab-ci\.yml|^docker/.*|^\.github/.*|^\.gitlab/.*|^\.config/nextest.toml|^\.cargo/.*
-      exclude: ^./gitlab/pipeline/zombienet.yml$
+      exclude: ^\.gitlab/pipeline/zombienet.*
     min_approvals: 2
     teams:
       - ci
@@ -39,7 +39,7 @@ rules:
   # if there are any changes in the bridges subtree (in case of backport changes back to bridges repo)
   - name: Bridges subtree files
     check_type: changed_files
-    condition: ^cumulus/bridges/.*
+    condition: ^bridges/.*
     min_approvals: 1
     teams:
       - bridges-core

diff --git a/.github/workflows/release-50_publish-docker.yml b/.github/workflows/release-50_publish-docker.yml
@@ -1,10 +1,7 @@
 name: Release - Publish Docker Image
 
-# This workflow listens to pubished releases or can be triggered manually.
-# It includes releases and rc candidates.
-# It fetches the binaries, checks sha256 and GPG
-# signatures, then builds an injected docker
-# image and publishes it.
+# This workflow listens to published releases or can be triggered manually.
+# It builds and published releases and rc candidates.
 
 on:
   #TODO: activate automated run later
@@ -13,6 +10,24 @@ on:
   #    - published
   workflow_dispatch:
     inputs:
+      image_type:
+        description: Type of the image to be published
+        required: true
+        default: rc
+        type: choice
+        options:
+          - rc
+          - release
+
+      binary:
+        description: Binary to be published
+        required: true
+        default: polkadot
+        type: choice
+        options:
+          - polkadot
+          - polkadot-parachain
+
       release_id:
         description: |
           Release ID.
@@ -22,33 +37,25 @@ on:
             jq '.[] | { name: .name, id: .id }'
         required: true
         type: string
-      image_type:
-        description: Type of the image to be published
-        required: true
-        default: rc
-        type: choice
-        options:
-          - rc
-          - release
+
       registry:
         description: Container registry
         required: true
         type: string
         default: docker.io
+
+      # The owner is often the same than the Docker Hub username but does ont have to be.
+      # In our case, it is not.
       owner:
         description: Owner of the container image repo
         required: true
         type: string
         default: parity
-      binary:
-        description: Binary to be published
+
+      version:
+        description: version to build/release
+        default: v0.9.18
         required: true
-        default: polkadot
-        type: choice
-        options:
-          - polkadot
-          - staking-miner
-          - polkadot-parachain
 
 permissions:
   contents: write
@@ -66,7 +73,8 @@ env:
   IMAGE_TYPE: ${{ inputs.image_type }}
 
 jobs:
-  fetch-artifacts:
+  fetch-artifacts: # this job will be triggered for the polkadot-parachain rc and release or polkadot rc image build
+    if: ${{ inputs.binary == 'polkadot-parachain' || inputs.image_type == 'rc' }}
     runs-on: ubuntu-latest
 
     steps:
@@ -102,7 +110,8 @@ jobs:
           path: |
             ./release-artifacts/${{ env.BINARY }}/**/*
 
-  build-container:
+  build-container: # this job will be triggered for the polkadot-parachain rc and release or polkadot rc image build
+    if: ${{ inputs.binary == 'polkadot-parachain' || inputs.image_type == 'rc' }}
     runs-on: ubuntu-latest
     needs: fetch-artifacts
 
@@ -158,8 +167,8 @@ jobs:
           echo "tag=latest" >> $GITHUB_OUTPUT
           echo "release=${release}" >> $GITHUB_OUTPUT
 
-      - name: Build Injected Container image for polkadot/staking-miner
-        if: ${{ env.BINARY == 'polkadot' || env.BINARY == 'staking-miner' }}
+      - name: Build Injected Container image for polkadot rc
+        if: ${{ env.BINARY == 'polkadot' }}
         env:
           ARTIFACTS_FOLDER: ./release-artifacts
           IMAGE_NAME: ${{ env.BINARY }}
@@ -204,3 +213,73 @@ jobs:
         run: |
           echo "Checking tag ${RELEASE_TAG} for image ${REGISTRY}/${DOCKER_OWNER}/${BINARY}"
           $ENGINE run -i ${REGISTRY}/${DOCKER_OWNER}/${BINARY}:${RELEASE_TAG} --version
+
+  fetch-latest-debian-package-version: # this job will be triggered for polkadot release build
+    if: ${{ inputs.binary == 'polkadot' && inputs.image_type == 'release' }}
+    runs-on: ubuntu-latest
+    outputs:
+      polkadot_apt_version: ${{ steps.fetch-latest-apt.outputs.polkadot_apt_version }}
+    container:
+      image: paritytech/parity-keyring
+      options: --user root
+    steps:
+      - name: Get version
+        id: fetch-latest-apt
+        run: |
+          apt update
+          apt show polkadot
+          version=$(apt show polkadot 2>/dev/null | grep "Version:" | awk '{print $2}')
+          echo "polkadot_apt_version=v$version" >> $GITHUB_OUTPUT
+          echo "You passed ${{ inputs.version }} but this is ignored"
+          echo "We use the version from the Debian Package: $version"
+
+  build-polkadot-release-container: # this job will be triggered for polkadot release build
+    if: ${{ inputs.binary == 'polkadot' && inputs.image_type == 'release' }}
+    runs-on: ubuntu-latest
+    needs: fetch-latest-debian-package-version
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@95cb08cb2672c73d4ffd2f422e6d11953d2a9c70 # v2.1.0
+
+      - name: Cache Docker layers
+        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2
+        with:
+         path: /tmp/.buildx-cache
+         key: ${{ runner.os }}-buildx-${{ github.sha }}
+         restore-keys: |
+           ${{ runner.os }}-buildx-
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Fetch values
+        id: fetch-data
+        run: |
+          date=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
+          echo "date=$date" >> $GITHUB_OUTPUT
+
+      - name: Build and push
+        id:   docker_build
+        uses: docker/build-push-action@v4
+        with:
+          push: true
+          file: docker/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile
+          # TODO: The owner should be used below but buildx does not resolve the VARs
+          # TODO: It would be good to get rid of this GHA that we don't really need.
+          tags: |
+            parity/polkadot:latest
+            parity/polkadot:${{ needs.fetch-latest-debian-package-version.outputs.polkadot_apt_version }}
+          build-args: |
+            VCS_REF=${{ github.ref }}
+            POLKADOT_VERSION=${{ needs.fetch-latest-debian-package-version.outputs.polkadot_apt_version }}
+            BUILD_DATE=${{ steps.fetch-data.outputs.date }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+      - name: Image digest
+        run: echo ${{ steps.docker_build.outputs.digest }}
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -21,7 +21,7 @@ workflow:
     - if: $CI_COMMIT_BRANCH
 
 variables:
-  CI_IMAGE: "paritytech/ci-unified:bullseye-1.70.0-2023-05-23-v20230706"
+  CI_IMAGE: !reference [.ci-unified, variables, CI_IMAGE]
   # BUILDAH_IMAGE is defined in group variables
   BUILDAH_COMMAND: "buildah --storage-driver overlay2"
   RELENG_SCRIPTS_BRANCH: "master"
@@ -114,7 +114,7 @@ default:
     - !reference [.rust-info-script, script]
     - !reference [.rusty-cachier, before_script]
   tags:
-    - linux-docker-vm-c2
+    - linux-docker
 
 # rusty-cachier's hidden job. Parts of this job are used to instrument the pipeline's other real jobs with rusty-cachier
 # rusty-cachier's commands are described here: https://gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client#description
@@ -213,6 +213,10 @@ include:
   - project: parity/infrastructure/ci_cd/shared
     ref: v0.2
     file: /common/timestamp.yml
+  # ci image
+  - project: parity/infrastructure/ci_cd/shared
+    ref: main
+    file: /common/ci-unified.yml
 # This job cancels the whole pipeline if any of provided jobs fail.
 # In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests
 # to fail the pipeline as soon as possible to shorten the feedback loop.

diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml
@@ -77,26 +77,6 @@ build-malus:
     - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
     - cp -r ./docker/* ./artifacts
 
-build-staking-miner:
-  stage: build
-  extends:
-    - .docker-env
-    - .common-refs
-    # - .collect-artifacts
-  # DAG
-  needs:
-    - job: build-malus
-      artifacts: false
-  script:
-    - time cargo build -q --locked --release --package staging-staking-miner
-    # # pack artifacts
-    # - mkdir -p ./artifacts
-    # - mv ./target/release/staking-miner ./artifacts/.
-    # - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION
-    # - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG
-    # - echo "staking-miner = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
-    # - cp -r ./scripts/* ./artifacts
-
 build-rustdoc:
   stage: build
   extends:
@@ -131,6 +111,9 @@ build-rustdoc:
             sed -i "s|</head>|$script_content</head>|" "$file"
         }
         export -f process_file
+        # xargs runs process_file in seperate shells without access to outer variables.
+        # to make script_content available inside process_file, export it as an env var here.
+        export script_content
 
         # Modify .html files in parallel using xargs, otherwise it can take a long time.
         find "$path" -name '*.html' | xargs -I {} -P "$(nproc)" bash -c 'process_file "$@"' _ {}
@@ -163,7 +146,7 @@ build-short-benchmark:
     - .run-immediately
     - .collect-artifacts
   script:
-    - cargo build --profile release --locked --features=runtime-benchmarks --bin polkadot --workspace
+    - cargo build --profile release --locked --features=runtime-benchmarks,on-chain-release-build --bin polkadot --workspace
     - mkdir -p artifacts
     - target/release/polkadot --version
     - cp ./target/release/polkadot ./artifacts/
@@ -290,7 +273,7 @@ build-short-benchmark-cumulus:
     - .run-immediately
     - .collect-artifacts
   script:
-    - cargo build --profile release --locked --features=runtime-benchmarks -p polkadot-parachain-bin --bin polkadot-parachain
+    - cargo build --profile release --locked --features=runtime-benchmarks,on-chain-release-build -p polkadot-parachain-bin --bin polkadot-parachain --workspace
     - mkdir -p artifacts
     - target/release/polkadot-parachain --version
     - cp ./target/release/polkadot-parachain ./artifacts/
@@ -358,7 +341,7 @@ build-subkey-linux:
   extends: .build-subkey
   # DAG
   needs:
-    - job: build-staking-miner
+    - job: build-malus
       artifacts: false
 # tbd
 # build-subkey-macos:

diff --git a/.gitlab/pipeline/publish.yml b/.gitlab/pipeline/publish.yml
@@ -328,24 +328,6 @@ build-push-image-substrate-pr:
 #       # this artifact is used in zombienet-tests job
 #       dotenv: ./artifacts/malus.env
 
-# publish-staking-miner-image:
-#   stage: publish
-#   extends:
-#     - .kubernetes-env
-#     - .build-push-image
-#     - .publish-refs
-#   variables:
-#     CI_IMAGE: ${BUILDAH_IMAGE}
-#     # scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile
-#     DOCKERFILE: ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile
-#     IMAGE_NAME: docker.io/paritytech/staking-miner
-#     GIT_STRATEGY: none
-#     DOCKER_USER: ${Docker_Hub_User_Parity}
-#     DOCKER_PASS: ${Docker_Hub_Pass_Parity}
-#   needs:
-#     - job: build-staking-miner
-#       artifacts: true
-
 # substrate
 
 # publish-substrate-image-pr: