[ci] Add short benchmarks to the pipeline (#5188)

* [ci] Add short benchmarks to the pipeline

* fix bench

* divide build and bench

* debug

* debug

* remove build from bench script

* run benches on ci runners

* return full pipeline

* fix pipeline

* Update scripts/run_short_benches_for_runtime.sh

Co-authored-by: Mara Robin B. <mara@broda.me>

* change short benchmark script

* allow short bench fail

Co-authored-by: Mara Robin B. <mara@broda.me>
Co-authored-by: parity-processbot <>

polkadot/.gitlab-ci.yml

@@ -577,20 +577,6 @@ generate-impl-guide:
   script:
     - mdbook build roadmap/implementers-guide
 
-check-runtime-benchmarks:
-  stage:                           stage3
-  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
-  needs:
-    - job:                         test-node-metrics
-      artifacts:                   false
-  <<:                              *test-refs
-  <<:                              *docker-env
-  <<:                              *compiler-info
-  script:
-    # Check that everything compiles with `runtime-benchmarks` feature flag.
-    - cargo check --features runtime-benchmarks --all
-    - sccache -s
-
 check-try-runtime:
   stage:                           stage3
   # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
@@ -620,6 +606,19 @@ check-no-default-features:
     - pushd cli && cargo check --no-default-features --features "service" && popd
     - sccache -s
 
+build-short-benchmark:
+  stage:                           stage3
+  <<:                              *test-refs
+  <<:                              *docker-env
+  <<:                              *collect-artifacts
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  needs:
+    - job:                         test-node-metrics
+      artifacts:                   false
+  script:
+    - cargo +nightly build --profile release --locked --features=runtime-benchmarks
+    - mkdir artifacts
+    - cp ./target/release/polkadot ./artifacts/
 
 deploy-parity-testnet:
   stage:                           stage3
@@ -804,6 +803,32 @@ publish-rustdoc:
   after_script:
     - rm -rf .git/ ./*
 
+# Run all pallet benchmarks only once to check if there are any errors
+short-benchmark-polkadot:          &short-bench
+  stage:                           stage4
+  <<:                              *test-refs
+  <<:                              *docker-env
+  # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
+  needs:
+    - job:                         build-short-benchmark
+      artifacts:                   true
+  variables:
+    RUNTIME:                       polkadot
+  # FIXME: https://github.com/paritytech/substrate/pull/11109
+  allow_failure:                   true
+  script:
+    - ./artifacts/polkadot benchmark --execution wasm --wasm-execution compiled --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 1 --repeat 1
+
+short-benchmark-kusama:
+  <<:                              *short-bench
+  variables:
+    RUNTIME:                       kusama
+
+short-benchmark-westend:
+  <<:                              *short-bench
+  variables:
+    RUNTIME:                       westend
+
 #### stage:                        .post
 
 # This job cancels the whole pipeline if any of provided jobs fail.
@@ -814,8 +839,6 @@ cancel-pipeline:
   needs:
     - job:                         test-linux-stable
       artifacts:                   false
-    - job:                         check-runtime-benchmarks
-      artifacts:                   false
     - job:                         check-try-runtime
       artifacts:                   false
   rules:
@@ -825,3 +848,5 @@ cancel-pipeline:
     PROJECT_ID:                    "${CI_PROJECT_ID}"
     PIPELINE_ID:                   "${CI_PIPELINE_ID}"
   trigger:                         "parity/infrastructure/ci_cd/pipeline-stopper"
+
+