[CI] Add manual Gitlab job for running all benchmarks (#3515)

* initial weights job

* add artifact

* revertme: changes to test branch

* add benchmarking instructions

* Revert "revertme: changes to test branch"

This reverts commit 2eab22037223967e66a70a16fda14f58e41c6ced.

* add kusama + westend weights jobs

* fix chevdor comments

* add temporary changes for testing again

* fix

* fix

* test sccache fix

* Revert "add temporary changes for testing again"

This reverts commit bb5a7660151f404994c85abfff31502aac89c1d1.

* whitespace

polkadot/.github/ISSUE_TEMPLATE/release.md

@@ -134,8 +134,22 @@ date to include them.
 
 ### Benchmarks
 
-Run the benchmarking suite with the new runtime and update any function weights
-if necessary.
+There are three benchmarking machines reserved for updating the weights at
+release-time. To initialise a benchmark run for each production runtime
+(westend, kusama, polkadot):
+* Go to https://gitlab.parity.io/parity/polkadot/-/pipelines?page=1&scope=branches&ref=master
+* Click the link to the last pipeline run for master
+* Start each of the manual jobs:
+  * 'update_westend_weights'
+  * 'update_polkadot_weights'
+  * 'update_kusama_weights'
+* When these jobs have completed (it takes a few hours), a git PATCH file will
+    be available to download as an artifact. 
+* On your local machine, branch off master
+* Download the patch file and apply it to your branch with `git patch patchfile.patch`
+* Commit the changes to your branch and submit a PR against master
+* The weights should be (Currently manually) checked to make sure there are no
+    big outliers (i.e., twice or half the weight).
 
 ### Polkadot JS
 

polkadot/.gitlab-ci.yml

@@ -326,6 +326,33 @@ publish-adder-collator-image:
       # this artifact is used in trigger-simnet job
       dotenv: ./artifacts/collator.env
 
+.update_weights:                   &update-weights
+  stage:                           build
+  when:                            manual
+  tags:
+    - weights
+  variables:
+    RUNTIME:                       polkadot
+  artifacts:
+    paths:
+      - ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
+  script: |
+    ./scripts/run_benches_for_runtime.sh $RUNTIME
+    git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
+
+update_polkadot_weights:
+  <<:                              *update-weights
+
+update_kusama_weights:
+  <<:                              *update-weights
+  variables:
+    RUNTIME:                       kusama
+
+update_westend_weights:
+  <<:                              *update-weights
+  variables:
+    RUNTIME:                       westend
+
 #### stage:                        publish
 
 publish-s3-release:                &publish-s3

polkadot/scripts/run_all_benches.sh

@@ -10,13 +10,6 @@ runtimes=(
   westend
 )
 
-# cargo build --locked --release
 for runtime in "${runtimes[@]}"; do
-  cargo +nightly run --release --features=runtime-benchmarks --locked benchmark --chain "${runtime}-dev" --execution=wasm --wasm-execution=compiled --pallet "*" --extrinsic "*" --repeat 0 | sed -r -e 's/Pallet: "([a-z_:]+)".*/\1/' | uniq | grep -v frame_system > "${runtime}_pallets"
-  while read -r line; do
-    pallet="$(echo "$line" | cut -d' ' -f1)";
-    echo "Runtime: $runtime. Pallet: $pallet";
-    cargo +nightly run --release --features=runtime-benchmarks -- benchmark --chain="${runtime}-dev" --steps=50 --repeat=20 --pallet="$pallet" --extrinsic="*" --execution=wasm --wasm-execution=compiled --heap-pages=4096 --header=./file_header.txt --output="./runtime/${runtime}/src/weights/${pallet/::/_}.rs"
-  done < "${runtime}_pallets"
-  rm "${runtime}_pallets"
+  "$(dirname "$0")/run_benches_for_runtime.sh" "$runtime"
 done

polkadot/scripts/run_benches_for_runtime.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Runs all benchmarks for all pallets, for a given runtime, provided by $1
+# Should be run on a reference machine to gain accurate benchmarks
+# current reference machine: https://github.com/paritytech/substrate/pull/5848
+
+runtime="$1"
+standard_args="--release --locked --features=runtime-benchmarks"
+
+echo "[+] Running all benchmarks for $runtime"
+
+# shellcheck disable=SC2086
+cargo +nightly run $standard_args benchmark \
+    --chain "${runtime}-dev" \
+    --execution=wasm \
+    --wasm-execution=compiled \
+    --pallet "*" \
+    --extrinsic "*" \
+    --repeat 0 | \
+  sed -r -e 's/Pallet: "([a-z_:]+)".*/\1/' | \
+  uniq | \
+  grep -v frame_system > "${runtime}_pallets"
+
+# For each pallet found in the previous command, run benches on each function
+while read -r line; do
+  pallet="$(echo "$line" | cut -d' ' -f1)";
+  echo "Runtime: $runtime. Pallet: $pallet";
+# shellcheck disable=SC2086
+cargo +nightly run $standard_args -- benchmark \
+  --chain="${runtime}-dev" \
+  --steps=50 \
+  --repeat=20 \
+  --pallet="$pallet" \
+  --extrinsic="*" \
+  --execution=wasm \
+  --wasm-execution=compiled \
+  --heap-pages=4096 \
+  --header=./file_header.txt \
+  --output="./runtime/${runtime}/src/weights/${pallet/::/_}.rs"
+done < "${runtime}_pallets"
+rm "${runtime}_pallets"