From 709b1033f0873b629f314cf4dfd86eae1cd06920 Mon Sep 17 00:00:00 2001
From: alvicsam <alvicsam@gmail.com>
Date: Wed, 3 Jan 2024 13:10:28 +0100
Subject: [PATCH 01/15] [DNM] test codecov

---
 .gitlab-ci.yml | 389 ++++++++++++++++++++++++++-----------------------
 1 file changed, 208 insertions(+), 181 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index dc4b3cf162e1..d3da06580bfd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -208,18 +208,18 @@ default:
     - if: $CI_COMMIT_REF_NAME =~ /^gh-readonly-queue.*$/ # merge queues
 
 include:
-  # check jobs
-  - .gitlab/pipeline/check.yml
-  # test jobs
-  - .gitlab/pipeline/test.yml
-  # build jobs
-  - .gitlab/pipeline/build.yml
-  # short-benchmarks jobs
-  - .gitlab/pipeline/short-benchmarks.yml
-  # publish jobs
-  - .gitlab/pipeline/publish.yml
-  # zombienet jobs
-  - .gitlab/pipeline/zombienet.yml
+  # # check jobs
+  # - .gitlab/pipeline/check.yml
+  # # test jobs
+  # - .gitlab/pipeline/test.yml
+  # # build jobs
+  # - .gitlab/pipeline/build.yml
+  # # short-benchmarks jobs
+  # - .gitlab/pipeline/short-benchmarks.yml
+  # # publish jobs
+  # - .gitlab/pipeline/publish.yml
+  # # zombienet jobs
+  # - .gitlab/pipeline/zombienet.yml
   # ci image
   - project: parity/infrastructure/ci_cd/shared
     ref: main
@@ -227,175 +227,202 @@ include:
   - project: parity/infrastructure/ci_cd/shared
     ref: main
     file: /common/forklift.yml
+
+codecov:
+  stage: test
+  timeout: 2h
+  extends:
+    - .docker-env
+    - .common-refs
+  variables:
+    # For codecov it's sufficient to run the fuzz tests only once.
+    QUICKCHECK_TESTS: 1
+    RUSTDOCFLAGS: "-Cpanic=abort"
+  script:
+    - rustup component add llvm-tools-preview
+    - date
+    - CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='cargo-test-%p-%m.profraw' time cargo nextest run
+      --workspace
+      --locked
+      --release
+      --no-fail-fast
+      --partition count:1/5
+      --features try-runtime,experimental > /dev/null || echo "some errors"
+    - mkdir -p target/coverage
+    - date
+    - grcov . --binary-path ./target/release/deps/ -s . -t lcov --branch --ignore-not-existing --ignore '../*' --ignore "/*" -o target/coverage/tests.lcov
+    # - codecov --token "${CODECOV_TOKEN}" --file target/coverage/tests.lcov
+  after_script:
+    - date
 # This job cancels the whole pipeline if any of provided jobs fail.
 # In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests
 # to fail the pipeline as soon as possible to shorten the feedback loop.
-.cancel-pipeline-template:
-  stage: .post
-  rules:
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
-      when: on_failure
-  variables:
-    PROJECT_ID: "${CI_PROJECT_ID}"
-    PROJECT_NAME: "${CI_PROJECT_NAME}"
-    PIPELINE_ID: "${CI_PIPELINE_ID}"
-    FAILED_JOB_URL: "${FAILED_JOB_URL}"
-    FAILED_JOB_NAME: "${FAILED_JOB_NAME}"
-    PR_NUM: "${PR_NUM}"
-  trigger:
-    project: "parity/infrastructure/ci_cd/pipeline-stopper"
-
-remove-cancel-pipeline-message:
-  stage: .post
-  rules:
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
-  variables:
-    PROJECT_ID: "${CI_PROJECT_ID}"
-    PROJECT_NAME: "${CI_PROJECT_NAME}"
-    PIPELINE_ID: "${CI_PIPELINE_ID}"
-    FAILED_JOB_URL: "https://gitlab.com"
-    FAILED_JOB_NAME: "nope"
-    PR_NUM: "${CI_COMMIT_REF_NAME}"
-  trigger:
-    project: "parity/infrastructure/ci_cd/pipeline-stopper"
-# need to copy jobs this way because otherwise gitlab will wait
-# for all 3 jobs to finish instead of cancelling if one fails
-cancel-pipeline-test-linux-stable1:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "test-linux-stable 1/3"
-
-cancel-pipeline-test-linux-stable2:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "test-linux-stable 2/3"
-
-cancel-pipeline-test-linux-stable3:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "test-linux-stable 3/3"
-
-cancel-pipeline-test-linux-stable-additional-tests:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "test-linux-stable-additional-tests"
-
-cancel-pipeline-test-linux-stable-slow:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "test-linux-stable-slow"
-
-cancel-pipeline-cargo-check-benches1:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "cargo-check-benches 1/2"
-
-cancel-pipeline-cargo-check-benches2:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "cargo-check-benches 2/2"
-
-cancel-pipeline-test-linux-stable-int:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: test-linux-stable-int
-
-cancel-pipeline-cargo-check-each-crate-1:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "cargo-check-each-crate 1/6"
-
-cancel-pipeline-cargo-check-each-crate-2:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "cargo-check-each-crate 2/6"
-
-cancel-pipeline-cargo-check-each-crate-3:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "cargo-check-each-crate 3/6"
-
-cancel-pipeline-cargo-check-each-crate-4:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "cargo-check-each-crate 4/6"
-
-cancel-pipeline-cargo-check-each-crate-5:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "cargo-check-each-crate 5/6"
-
-cancel-pipeline-cargo-check-each-crate-6:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: "cargo-check-each-crate 6/6"
-
-cancel-pipeline-cargo-check-each-crate-macos:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: cargo-check-each-crate-macos
-
-cancel-pipeline-check-tracing:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: check-tracing
-
-cancel-pipeline-cargo-clippy:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: cargo-clippy
-
-cancel-pipeline-build-linux-stable:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: build-linux-stable
-
-cancel-pipeline-build-linux-stable-cumulus:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: build-linux-stable-cumulus
-
-cancel-pipeline-build-linux-substrate:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: build-linux-substrate
-
-cancel-pipeline-test-node-metrics:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: test-node-metrics
-
-cancel-pipeline-test-frame-ui:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: test-frame-ui
-
-cancel-pipeline-quick-benchmarks:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: quick-benchmarks
-
-cancel-pipeline-check-try-runtime:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: check-try-runtime
-
-cancel-pipeline-test-frame-examples-compile-to-wasm:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: test-frame-examples-compile-to-wasm
-
-cancel-pipeline-build-short-benchmark:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: build-short-benchmark
-
-cancel-pipeline-check-runtime-migration-rococo:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: check-runtime-migration-rococo
-
-cancel-pipeline-check-runtime-migration-westend:
-  extends: .cancel-pipeline-template
-  needs:
-    - job: check-runtime-migration-westend
+# .cancel-pipeline-template:
+#   stage: .post
+#   rules:
+#     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
+#       when: on_failure
+#   variables:
+#     PROJECT_ID: "${CI_PROJECT_ID}"
+#     PROJECT_NAME: "${CI_PROJECT_NAME}"
+#     PIPELINE_ID: "${CI_PIPELINE_ID}"
+#     FAILED_JOB_URL: "${FAILED_JOB_URL}"
+#     FAILED_JOB_NAME: "${FAILED_JOB_NAME}"
+#     PR_NUM: "${PR_NUM}"
+#   trigger:
+#     project: "parity/infrastructure/ci_cd/pipeline-stopper"
+
+# remove-cancel-pipeline-message:
+#   stage: .post
+#   rules:
+#     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
+#   variables:
+#     PROJECT_ID: "${CI_PROJECT_ID}"
+#     PROJECT_NAME: "${CI_PROJECT_NAME}"
+#     PIPELINE_ID: "${CI_PIPELINE_ID}"
+#     FAILED_JOB_URL: "https://gitlab.com"
+#     FAILED_JOB_NAME: "nope"
+#     PR_NUM: "${CI_COMMIT_REF_NAME}"
+#   trigger:
+#     project: "parity/infrastructure/ci_cd/pipeline-stopper"
+# # need to copy jobs this way because otherwise gitlab will wait
+# # for all 3 jobs to finish instead of cancelling if one fails
+# cancel-pipeline-test-linux-stable1:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: "test-linux-stable 1/3"
+
+# cancel-pipeline-test-linux-stable2:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: "test-linux-stable 2/3"
+
+# cancel-pipeline-test-linux-stable3:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: "test-linux-stable 3/3"
+
+# cancel-pipeline-test-linux-stable-additional-tests:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: "test-linux-stable-additional-tests"
+
+# cancel-pipeline-test-linux-stable-slow:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: "test-linux-stable-slow"
+
+# cancel-pipeline-cargo-check-benches1:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: "cargo-check-benches 1/2"
+
+# cancel-pipeline-cargo-check-benches2:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: "cargo-check-benches 2/2"
+
+# cancel-pipeline-test-linux-stable-int:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: test-linux-stable-int
+
+# cancel-pipeline-cargo-check-each-crate-1:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: "cargo-check-each-crate 1/6"
+
+# cancel-pipeline-cargo-check-each-crate-2:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: "cargo-check-each-crate 2/6"
+
+# cancel-pipeline-cargo-check-each-crate-3:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: "cargo-check-each-crate 3/6"
+
+# cancel-pipeline-cargo-check-each-crate-4:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: "cargo-check-each-crate 4/6"
+
+# cancel-pipeline-cargo-check-each-crate-5:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: "cargo-check-each-crate 5/6"
+
+# cancel-pipeline-cargo-check-each-crate-6:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: "cargo-check-each-crate 6/6"
+
+# cancel-pipeline-cargo-check-each-crate-macos:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: cargo-check-each-crate-macos
+
+# cancel-pipeline-check-tracing:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: check-tracing
+
+# cancel-pipeline-cargo-clippy:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: cargo-clippy
+
+# cancel-pipeline-build-linux-stable:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: build-linux-stable
+
+# cancel-pipeline-build-linux-stable-cumulus:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: build-linux-stable-cumulus
+
+# cancel-pipeline-build-linux-substrate:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: build-linux-substrate
+
+# cancel-pipeline-test-node-metrics:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: test-node-metrics
+
+# cancel-pipeline-test-frame-ui:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: test-frame-ui
+
+# cancel-pipeline-quick-benchmarks:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: quick-benchmarks
+
+# cancel-pipeline-check-try-runtime:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: check-try-runtime
+
+# cancel-pipeline-test-frame-examples-compile-to-wasm:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: test-frame-examples-compile-to-wasm
+
+# cancel-pipeline-build-short-benchmark:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: build-short-benchmark
+
+# cancel-pipeline-check-runtime-migration-rococo:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: check-runtime-migration-rococo
+
+# cancel-pipeline-check-runtime-migration-westend:
+#   extends: .cancel-pipeline-template
+#   needs:
+#     - job: check-runtime-migration-westend

From f345493b4d3e76a17d421691f68425e21bd944b7 Mon Sep 17 00:00:00 2001
From: alvicsam <alvicsam@gmail.com>
Date: Wed, 3 Jan 2024 16:45:41 +0100
Subject: [PATCH 02/15] add termination to slow tests

---
 .config/nextest.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.config/nextest.toml b/.config/nextest.toml
index 1e18f8b5589c..ecfe20f6ebf6 100644
--- a/.config/nextest.toml
+++ b/.config/nextest.toml
@@ -76,7 +76,7 @@ fail-fast = true
 # which will cause slow tests to be terminated after the specified number of
 # periods have passed.
 # Example: slow-timeout = { period = "60s", terminate-after = 2 }
-slow-timeout = { period = "60s" }
+slow-timeout = { period = "60s", terminate-after = 3 }
 
 # Treat a test as leaky if after the process is shut down, standard output and standard error
 # aren't closed within this duration.

From 06ca995504eca08a673be519b425b8d8ce4b1fdd Mon Sep 17 00:00:00 2001
From: alvicsam <alvicsam@gmail.com>
Date: Thu, 4 Jan 2024 15:07:16 +0100
Subject: [PATCH 03/15] increase timeout

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d3da06580bfd..3e301ac9bcf5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -230,7 +230,7 @@ include:
 
 codecov:
   stage: test
-  timeout: 2h
+  timeout: 3h
   extends:
     - .docker-env
     - .common-refs

From 1a2799636cacc2f664a1812a43a5692f4ab97b7e Mon Sep 17 00:00:00 2001
From: alvicsam <alvicsam@gmail.com>
Date: Thu, 4 Jan 2024 16:39:02 +0100
Subject: [PATCH 04/15] try codecov with 3 nextest jobs

---
 .config/nextest.toml |  2 +-
 .gitlab-ci.yml       | 54 +++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/.config/nextest.toml b/.config/nextest.toml
index ecfe20f6ebf6..1e18f8b5589c 100644
--- a/.config/nextest.toml
+++ b/.config/nextest.toml
@@ -76,7 +76,7 @@ fail-fast = true
 # which will cause slow tests to be terminated after the specified number of
 # periods have passed.
 # Example: slow-timeout = { period = "60s", terminate-after = 2 }
-slow-timeout = { period = "60s", terminate-after = 3 }
+slow-timeout = { period = "60s" }
 
 # Treat a test as leaky if after the process is shut down, standard output and standard error
 # aren't closed within this duration.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3e301ac9bcf5..528c9395baa8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -228,7 +228,7 @@ include:
     ref: main
     file: /common/forklift.yml
 
-codecov:
+.codecov:
   stage: test
   timeout: 3h
   extends:
@@ -254,6 +254,58 @@ codecov:
     # - codecov --token "${CODECOV_TOKEN}" --file target/coverage/tests.lcov
   after_script:
     - date
+
+test-linux-stable:
+  stage: test
+  extends:
+    - .docker-env
+    - .common-refs
+    - .run-immediately
+    - .pipeline-stopper-artifacts
+  variables:
+    RUST_TOOLCHAIN: stable
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings -Cinstrument-coverage"
+    CARGO_INCREMENTAL: 0
+    LLVM_PROFILE_FILE: "cargo-test-%p-%m.profraw"
+  parallel: 3
+  script:
+    - rustup component add llvm-tools-preview
+    # Build all but only execute 'runtime' tests.
+    - echo "Node index - ${CI_NODE_INDEX}. Total amount - ${CI_NODE_TOTAL}"
+    # add experimental to features after https://github.com/paritytech/substrate/pull/14502 is merged
+    # "upgrade_version_checks_should_work" is currently failing
+    - |
+      time cargo nextest run \
+        --workspace \
+        --locked \
+        --release \
+        --no-fail-fast \
+        --features try-runtime,experimental,ci-only-tests \
+        --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
+    # codecov
+    - mkdir -p target/coverage
+    - date
+    - grcov . --binary-path ./target/release/deps/ -s . -t lcov --branch --ignore-not-existing --ignore '../*' --ignore "/*" -o target/coverage/tests.lcov
+    # - codecov --token "${CODECOV_TOKEN}" --file target/coverage/tests.lcov
+    # Upload tests results to Elasticsearch
+    - echo "Upload test results to Elasticsearch"
+    - cat target/nextest/default/junit.xml | xq . > target/nextest/default/junit.json
+    - |
+      curl -v -XPOST --http1.1 \
+      -u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD} \
+      https://elasticsearch.parity-build.parity.io/unit-tests/_doc/${CI_JOB_ID} \
+      -H 'Content-Type: application/json' \
+      -d @target/nextest/default/junit.json || echo "failed to upload junit report"
+    # run runtime-api tests with `enable-staging-api` feature on the 1st node
+    - if [ ${CI_NODE_INDEX} == 1 ]; then time cargo nextest run -p sp-api-test --features enable-staging-api; fi
+  artifacts:
+    when: always
+    paths:
+      - target/nextest/default/junit.xml
+    reports:
+      junit: target/nextest/default/junit.xml
 # This job cancels the whole pipeline if any of provided jobs fail.
 # In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests
 # to fail the pipeline as soon as possible to shorten the feedback loop.

From 3ca2e5200b6c53e0ba394fcf5b170ab03bc5d6cc Mon Sep 17 00:00:00 2001
From: alvicsam <alvicsam@gmail.com>
Date: Thu, 4 Jan 2024 16:46:32 +0100
Subject: [PATCH 05/15] rm dep

---
 .gitlab-ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 528c9395baa8..8e8cd5f0c3c8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -260,7 +260,6 @@ test-linux-stable:
   extends:
     - .docker-env
     - .common-refs
-    - .run-immediately
     - .pipeline-stopper-artifacts
   variables:
     RUST_TOOLCHAIN: stable

From 3d9a2b0bd8edea2095e343f76dc158f658fbedf7 Mon Sep 17 00:00:00 2001
From: alvicsam <alvicsam@gmail.com>
Date: Mon, 8 Jan 2024 10:24:59 +0100
Subject: [PATCH 06/15] df -h

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8e8cd5f0c3c8..96168d3f745e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -282,7 +282,7 @@ test-linux-stable:
         --release \
         --no-fail-fast \
         --features try-runtime,experimental,ci-only-tests \
-        --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
+        --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL} || df -h
     # codecov
     - mkdir -p target/coverage
     - date

From 725995cbf0374494668bd278a520c5e5966342c6 Mon Sep 17 00:00:00 2001
From: alvicsam <alvicsam@gmail.com>
Date: Mon, 8 Jan 2024 13:05:42 +0100
Subject: [PATCH 07/15] mv codecov

---
 .gitlab-ci.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 96168d3f745e..2cffdeabb994 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -282,12 +282,7 @@ test-linux-stable:
         --release \
         --no-fail-fast \
         --features try-runtime,experimental,ci-only-tests \
-        --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL} || df -h
-    # codecov
-    - mkdir -p target/coverage
-    - date
-    - grcov . --binary-path ./target/release/deps/ -s . -t lcov --branch --ignore-not-existing --ignore '../*' --ignore "/*" -o target/coverage/tests.lcov
-    # - codecov --token "${CODECOV_TOKEN}" --file target/coverage/tests.lcov
+        --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
     # Upload tests results to Elasticsearch
     - echo "Upload test results to Elasticsearch"
     - cat target/nextest/default/junit.xml | xq . > target/nextest/default/junit.json
@@ -299,6 +294,11 @@ test-linux-stable:
       -d @target/nextest/default/junit.json || echo "failed to upload junit report"
     # run runtime-api tests with `enable-staging-api` feature on the 1st node
     - if [ ${CI_NODE_INDEX} == 1 ]; then time cargo nextest run -p sp-api-test --features enable-staging-api; fi
+    # codecov
+    - mkdir -p target/coverage
+    - date
+    - grcov . --binary-path ./target/release/deps/ -s . -t lcov --branch --ignore-not-existing --ignore '../*' --ignore "/*" -o target/coverage/tests.lcov
+    # - codecov --token "${CODECOV_TOKEN}" --file target/coverage/tests.lcov
   artifacts:
     when: always
     paths:

From b9c7be0f794f5e9a9558741214ce55003194d2b8 Mon Sep 17 00:00:00 2001
From: alvicsam <alvicsam@gmail.com>
Date: Mon, 8 Jan 2024 15:42:09 +0100
Subject: [PATCH 08/15] compare with coverage and without

---
 .gitlab-ci.yml | 46 +++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 45 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2cffdeabb994..bd2cf56f183a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -255,7 +255,7 @@ include:
   after_script:
     - date
 
-test-linux-stable:
+test-linux-stable-codecov:
   stage: test
   extends:
     - .docker-env
@@ -305,6 +305,50 @@ test-linux-stable:
       - target/nextest/default/junit.xml
     reports:
       junit: target/nextest/default/junit.xml
+
+test-linux-stable:
+  stage: test
+  extends:
+    - .docker-env
+    - .common-refs
+    - .run-immediately
+    - .pipeline-stopper-artifacts
+  variables:
+    RUST_TOOLCHAIN: stable
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
+  parallel: 3
+  script:
+    # Build all but only execute 'runtime' tests.
+    - echo "Node index - ${CI_NODE_INDEX}. Total amount - ${CI_NODE_TOTAL}"
+    # add experimental to features after https://github.com/paritytech/substrate/pull/14502 is merged
+    # "upgrade_version_checks_should_work" is currently failing
+    - |
+      time cargo nextest run \
+        --workspace \
+        --locked \
+        --release \
+        --no-fail-fast \
+        --features try-runtime,experimental,ci-only-tests \
+        --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
+    # Upload tests results to Elasticsearch
+    - echo "Upload test results to Elasticsearch"
+    - cat target/nextest/default/junit.xml | xq . > target/nextest/default/junit.json
+    - |
+      curl -v -XPOST --http1.1 \
+      -u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD} \
+      https://elasticsearch.parity-build.parity.io/unit-tests/_doc/${CI_JOB_ID} \
+      -H 'Content-Type: application/json' \
+      -d @target/nextest/default/junit.json || echo "failed to upload junit report"
+    # run runtime-api tests with `enable-staging-api` feature on the 1st node
+    - if [ ${CI_NODE_INDEX} == 1 ]; then time cargo nextest run -p sp-api-test --features enable-staging-api; fi
+  artifacts:
+    when: always
+    paths:
+      - target/nextest/default/junit.xml
+    reports:
+      junit: target/nextest/default/junit.xml
 # This job cancels the whole pipeline if any of provided jobs fail.
 # In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests
 # to fail the pipeline as soon as possible to shorten the feedback loop.

From 960975ddb164e11fcc8506816d410d324e5b790d Mon Sep 17 00:00:00 2001
From: alvicsam <alvicsam@gmail.com>
Date: Mon, 8 Jan 2024 15:42:30 +0100
Subject: [PATCH 09/15] rm dep

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bd2cf56f183a..5dbb6c6c9a85 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -311,7 +311,7 @@ test-linux-stable:
   extends:
     - .docker-env
     - .common-refs
-    - .run-immediately
+    # - .run-immediately
     - .pipeline-stopper-artifacts
   variables:
     RUST_TOOLCHAIN: stable

From 75a88ede298e5c2ab98177bd5035851a6605d4aa Mon Sep 17 00:00:00 2001
From: alvicsam <alvicsam@gmail.com>
Date: Tue, 9 Jan 2024 10:09:27 +0100
Subject: [PATCH 10/15] restart pipeline


From 251145a402d13ee74f64751d99d60c76b7d0808a Mon Sep 17 00:00:00 2001
From: alvicsam <alvicsam@gmail.com>
Date: Wed, 13 Mar 2024 16:44:08 +0100
Subject: [PATCH 11/15] fix forklift, add more options

---
 .gitlab-ci.yml | 36 +++++++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 7 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5dbb6c6c9a85..2465ea1223a2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -121,9 +121,8 @@ default:
   before_script:
     - 'curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift  -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION}/forklift_${FL_FORKLIFT_VERSION}_linux_amd64"'
     - chmod +x forklift
-    - mkdir .forklift
-    - cp $FL_FORKLIFT_CONFIG .forklift/config.toml
-    - export FORKLIFT_PACKAGE_SUFFIX=${CI_JOB_NAME/ [0-9 \/]*}
+    - mkdir ~/.forklift
+    - cp $FL_FORKLIFT_CONFIG ~/.forklift/config.toml
     - shopt -s expand_aliases
     - export PATH=$PATH:$(pwd)
     - |
@@ -131,12 +130,8 @@ default:
       echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'"
       alias cargo="forklift cargo"
       fi
-    - ls -al
-    - rm -f forklift.sock
-    - forklift clean
     #
     - echo "FL_FORKLIFT_VERSION ${FL_FORKLIFT_VERSION}"
-    - echo "FORKLIFT_PACKAGE_SUFFIX $FORKLIFT_PACKAGE_SUFFIX"
 
 .common-refs:
   rules:
@@ -306,6 +301,33 @@ test-linux-stable-codecov:
     reports:
       junit: target/nextest/default/junit.xml
 
+# requires seccomp adjustments
+# https://github.com/xd009642/tarpaulin?tab=readme-ov-file#docker
+test-linux-stable-tarpaulin:
+  stage: test
+  extends:
+    - .docker-env
+    - .common-refs
+    - .pipeline-stopper-artifacts
+  script:
+    - cargo install cargo-tarpaulin
+    - cargo tarpaulin
+
+test-linux-stable-llvm-cov:
+  stage: test
+  extends:
+    - .docker-env
+    - .common-refs
+    - .pipeline-stopper-artifacts
+  script:
+    - rustup component add llvm-tools-preview
+    - cargo install cargo-llvm-cov
+    - cargo llvm-cov nextest --workspace \
+      --locked \
+      --release \
+      --no-fail-fast \
+      --features try-runtime,experimental,ci-only-tests
+
 test-linux-stable:
   stage: test
   extends:

From 315dd71044241644b719d17ffbfffd00b613fabd Mon Sep 17 00:00:00 2001
From: alvicsam <alvicsam@gmail.com>
Date: Mon, 18 Mar 2024 11:38:45 +0100
Subject: [PATCH 12/15] disable forklift

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2465ea1223a2..cb6d4f65a942 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -112,7 +112,7 @@ default:
     - !reference [.common-before-script, before_script]
     - !reference [.prepare-env, before_script]
     - !reference [.rust-info-script, script]
-    - !reference [.forklift-cache, before_script]
+    # - !reference [.forklift-cache, before_script]
   tags:
     - linux-docker
 

From 01c7d48fd681d338c2fbde68f69e0605aa5f8b95 Mon Sep 17 00:00:00 2001
From: alvicsam <alvicsam@gmail.com>
Date: Mon, 18 Mar 2024 11:39:40 +0100
Subject: [PATCH 13/15] disable taurpaulin

---
 .gitlab-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cb6d4f65a942..4bd17c70015d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -303,7 +303,7 @@ test-linux-stable-codecov:
 
 # requires seccomp adjustments
 # https://github.com/xd009642/tarpaulin?tab=readme-ov-file#docker
-test-linux-stable-tarpaulin:
+.test-linux-stable-tarpaulin:
   stage: test
   extends:
     - .docker-env

From f738bb4f4bb9faf01c93d2c80f4b3e22e0e1a3fe Mon Sep 17 00:00:00 2001
From: alvicsam <alvicsam@gmail.com>
Date: Mon, 18 Mar 2024 12:33:20 +0100
Subject: [PATCH 14/15] update test commands

---
 .gitlab-ci.yml | 41 ++++++++---------------------------------
 1 file changed, 8 insertions(+), 33 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4bd17c70015d..cec0d250879b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -272,21 +272,13 @@ test-linux-stable-codecov:
     # "upgrade_version_checks_should_work" is currently failing
     - |
       time cargo nextest run \
+        --filter-expr 'not deps(/polkadot-subsystem-bench/)' \
         --workspace \
         --locked \
         --release \
         --no-fail-fast \
-        --features try-runtime,experimental,ci-only-tests \
+        --features try-runtime,experimental,riscv,ci-only-tests \
         --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
-    # Upload tests results to Elasticsearch
-    - echo "Upload test results to Elasticsearch"
-    - cat target/nextest/default/junit.xml | xq . > target/nextest/default/junit.json
-    - |
-      curl -v -XPOST --http1.1 \
-      -u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD} \
-      https://elasticsearch.parity-build.parity.io/unit-tests/_doc/${CI_JOB_ID} \
-      -H 'Content-Type: application/json' \
-      -d @target/nextest/default/junit.json || echo "failed to upload junit report"
     # run runtime-api tests with `enable-staging-api` feature on the 1st node
     - if [ ${CI_NODE_INDEX} == 1 ]; then time cargo nextest run -p sp-api-test --features enable-staging-api; fi
     # codecov
@@ -294,12 +286,6 @@ test-linux-stable-codecov:
     - date
     - grcov . --binary-path ./target/release/deps/ -s . -t lcov --branch --ignore-not-existing --ignore '../*' --ignore "/*" -o target/coverage/tests.lcov
     # - codecov --token "${CODECOV_TOKEN}" --file target/coverage/tests.lcov
-  artifacts:
-    when: always
-    paths:
-      - target/nextest/default/junit.xml
-    reports:
-      junit: target/nextest/default/junit.xml
 
 # requires seccomp adjustments
 # https://github.com/xd009642/tarpaulin?tab=readme-ov-file#docker
@@ -323,10 +309,11 @@ test-linux-stable-llvm-cov:
     - rustup component add llvm-tools-preview
     - cargo install cargo-llvm-cov
     - cargo llvm-cov nextest --workspace \
+      --filter-expr 'not deps(/polkadot-subsystem-bench/)' \
       --locked \
       --release \
       --no-fail-fast \
-      --features try-runtime,experimental,ci-only-tests
+      --features try-runtime,experimental,riscv,ci-only-tests
 
 test-linux-stable:
   stage: test
@@ -348,29 +335,17 @@ test-linux-stable:
     # "upgrade_version_checks_should_work" is currently failing
     - |
       time cargo nextest run \
+        --filter-expr 'not deps(/polkadot-subsystem-bench/)' \
         --workspace \
         --locked \
         --release \
         --no-fail-fast \
-        --features try-runtime,experimental,ci-only-tests \
+        --features try-runtime,experimental,riscv,ci-only-tests \
         --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
-    # Upload tests results to Elasticsearch
-    - echo "Upload test results to Elasticsearch"
-    - cat target/nextest/default/junit.xml | xq . > target/nextest/default/junit.json
-    - |
-      curl -v -XPOST --http1.1 \
-      -u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD} \
-      https://elasticsearch.parity-build.parity.io/unit-tests/_doc/${CI_JOB_ID} \
-      -H 'Content-Type: application/json' \
-      -d @target/nextest/default/junit.json || echo "failed to upload junit report"
     # run runtime-api tests with `enable-staging-api` feature on the 1st node
     - if [ ${CI_NODE_INDEX} == 1 ]; then time cargo nextest run -p sp-api-test --features enable-staging-api; fi
-  artifacts:
-    when: always
-    paths:
-      - target/nextest/default/junit.xml
-    reports:
-      junit: target/nextest/default/junit.xml
+  timeout: 90m
+#
 # This job cancels the whole pipeline if any of provided jobs fail.
 # In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests
 # to fail the pipeline as soon as possible to shorten the feedback loop.

From ac6c7e78ba6b6b07b52752e260baaba4242cd1da Mon Sep 17 00:00:00 2001
From: alvicsam <alvicsam@gmail.com>
Date: Wed, 10 Apr 2024 12:22:49 +0200
Subject: [PATCH 15/15] rm filter expr

---
 .gitlab-ci.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a3a239d29ef2..62a4d0c944ca 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -286,7 +286,6 @@ test-linux-stable-codecov:
     # "upgrade_version_checks_should_work" is currently failing
     - |
       time cargo nextest run \
-        --filter-expr 'not deps(/polkadot-subsystem-bench/)' \
         --workspace \
         --locked \
         --release \
@@ -323,7 +322,6 @@ test-linux-stable-llvm-cov:
     - rustup component add llvm-tools-preview
     - cargo install cargo-llvm-cov
     - cargo llvm-cov nextest --workspace \
-      --filter-expr 'not deps(/polkadot-subsystem-bench/)' \
       --locked \
       --release \
       --no-fail-fast \
@@ -349,7 +347,6 @@ test-linux-stable:
     # "upgrade_version_checks_should_work" is currently failing
     - |
       time cargo nextest run \
-        --filter-expr 'not deps(/polkadot-subsystem-bench/)' \
         --workspace \
         --locked \
         --release \