Move scripts used in CI to the new location (#5198)

* Move CI scripts and update references

* Update paths in .gitlab-ci.yml

* Removed outdated entries from CODEOWNERS

polkadot/scripts/ci/github/check_labels.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+#shellcheck source=../common/lib.sh
+source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh"
+
+repo="$GITHUB_REPOSITORY"
+pr="$GITHUB_PR"
+
+ensure_labels() {
+  for label in "$@"; do
+    if has_label "$repo" "$pr" "$label"; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+# Must have one of the following labels
+releasenotes_labels=(
+  'B0-silent'
+  'B1-releasenotes'
+  'B7-runtimenoteworthy'
+)
+
+# Must be an ordered list of priorities, lowest first
+priority_labels=(
+  'C1-low 📌'
+  'C3-medium 📣'
+  'C7-high ❗️'
+  'C9-critical ‼️'
+)
+
+audit_labels=(
+  'D1-audited 👍'
+  'D2-notlive 💤'
+  'D3-trivial 🧸'
+  'D5-nicetohaveaudit ⚠️'
+  'D9-needsaudit 👮'
+)
+
+echo "[+] Checking release notes (B) labels for $CI_COMMIT_BRANCH"
+if ensure_labels "${releasenotes_labels[@]}";  then
+  echo "[+] Release notes label detected. All is well."
+else
+  echo "[!] Release notes label not detected. Please add one of: ${releasenotes_labels[*]}"
+  exit 1
+fi
+
+echo "[+] Checking release priority (C) labels for $CI_COMMIT_BRANCH"
+if ensure_labels "${priority_labels[@]}";  then
+  echo "[+] Release priority label detected. All is well."
+else
+  echo "[!] Release priority label not detected. Please add one of: ${priority_labels[*]}"
+  exit 1
+fi
+
+if has_runtime_changes "${BASE_SHA}" "${HEAD_SHA}"; then
+  echo "[+] Runtime changes detected. Checking audit (D) labels"
+  if ensure_labels "${audit_labels[@]}";  then
+    echo "[+] Release audit label detected. All is well."
+  else
+    echo "[!] Release audit label not detected. Please add one of: ${audit_labels[*]}"
+    exit 1
+  fi
+fi
+
+# If the priority is anything other than the lowest, we *must not* have a B0-silent
+# label
+if has_label "$repo" "$GITHUB_PR" 'B0-silent' &&
+  ! has_label "$repo" "$GITHUB_PR" "${priority_labels[0]}"; then
+  echo "[!] Changes with a priority higher than C1-low *MUST* have a B- label that is not B0-Silent"
+  exit 1
+fi
+
+exit 0

polkadot/scripts/ci/github/extrinsic-ordering-filter.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# This script is used in a Github Workflow. It helps filtering out what is interesting
+# when comparing metadata and spot what would require a tx version bump.
+
+# shellcheck disable=SC2002,SC2086
+
+FILE=$1
+
+# Higlight indexes that were deleted
+function find_deletions() {
+    echo "\n## Deletions\n"
+    RES=$(cat "$FILE" | grep -n '\[\-\]' | tr -s " ")
+    if [ "$RES" ]; then
+        echo "$RES" | awk '{ printf "%s\\n", $0 }'
+    else
+        echo "n/a"
+    fi
+}
+
+# Highlight indexes that have been deleted
+function find_index_changes() {
+    echo "\n## Index changes\n"
+    RES=$(cat "$FILE" | grep -E -n -i 'idx:\s*([0-9]+)\s*(->)\s*([0-9]+)' | tr -s " ")
+    if [ "$RES" ]; then
+        echo "$RES" | awk '{ printf "%s\\n", $0 }'
+    else
+        echo "n/a"
+    fi
+}
+
+# Highlight values that decreased
+function find_decreases() {
+    echo "\n## Decreases\n"
+    OUT=$(cat "$FILE" | grep -E -i -o '([0-9]+)\s*(->)\s*([0-9]+)' | awk '$1 > $3 { printf "%s;", $0 }')
+    IFS=$';' LIST=("$OUT")
+    unset RES
+    for line in "${LIST[@]}"; do
+        RES="$RES\n$(cat "$FILE" | grep -E -i -n \"$line\" | tr -s " ")"
+    done
+
+    if [ "$RES" ]; then
+        echo "$RES" | awk '{ printf "%s\\n", $0 }' | sort -u -g | uniq
+    else
+        echo "n/a"
+    fi
+}
+
+echo "\n------------------------------ SUMMARY -------------------------------"
+echo "\n⚠️ This filter is here to help spotting changes that should be reviewed carefully."
+echo "\n⚠️ It catches only index changes, deletions and value decreases".
+
+find_deletions "$FILE"
+find_index_changes "$FILE"
+find_decreases "$FILE"
+echo "\n----------------------------------------------------------------------\n"

polkadot/scripts/ci/github/generate_release_text.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require 'base64'
+require 'changelogerator'
+require 'erb'
+require 'git'
+require 'json'
+require 'octokit'
+require 'toml'
+require_relative './lib.rb'
+
+# A logger only active when NOT running in CI
+def logger(s)
+  puts "▶ DEBUG: %s" % [s] if ENV['CI'] != 'true'
+end
+
+# Check if all the required ENV are set
+# This is especially convenient when testing locally
+def check_env()
+  if ENV['CI'] != 'true' then
+    logger("Running locally")
+    vars = ['GITHUB_REF', 'GITHUB_TOKEN', 'GITHUB_WORKSPACE', 'GITHUB_REPOSITORY', 'RUSTC_STABLE', 'RUSTC_NIGHTLY']
+    vars.each { |x|
+      env = (ENV[x] || "")
+      if env.length > 0 then
+        logger("- %s:\tset: %s, len: %d" % [x, env.length > 0 || false, env.length])
+      else
+        logger("- %s:\tset: %s, len: %d" % [x, env.length > 0 || false, env.length])
+      end
+    }
+  end
+end
+
+check_env()
+
+current_ref = ENV['GITHUB_REF']
+token = ENV['GITHUB_TOKEN']
+
+logger("Connecting to Github")
+github_client = Octokit::Client.new(
+  access_token: token
+)
+
+polkadot_path = ENV['GITHUB_WORKSPACE'] + '/polkadot/'
+
+# Generate an ERB renderer based on the template .erb file
+renderer = ERB.new(
+  File.read(File.join(polkadot_path, 'scripts/ci/github/polkadot_release.erb')),
+  trim_mode: '<>'
+)
+
+# get ref of last polkadot release
+last_ref = 'refs/tags/' + github_client.latest_release(ENV['GITHUB_REPOSITORY']).tag_name
+logger("Last ref: " + last_ref)
+
+logger("Generate changelog for Polkadot")
+polkadot_cl = Changelog.new(
+  'paritytech/polkadot', last_ref, current_ref, token: token
+)
+
+# Gets the substrate commit hash used for a given polkadot ref
+def get_substrate_commit(client, ref)
+  cargo = TOML::Parser.new(
+    Base64.decode64(
+      client.contents(
+        ENV['GITHUB_REPOSITORY'],
+        path: 'Cargo.lock',
+        query: { ref: ref.to_s }
+      ).content
+    )
+  ).parsed
+  cargo['package'].find { |p| p['name'] == 'sc-cli' }['source'].split('#').last
+end
+
+substrate_prev_sha = get_substrate_commit(github_client, last_ref)
+substrate_cur_sha = get_substrate_commit(github_client, current_ref)
+
+logger("Generate changelog for Substrate")
+substrate_cl = Changelog.new(
+  'paritytech/substrate', substrate_prev_sha, substrate_cur_sha,
+  token: token,
+  prefix: true
+)
+
+# Combine all changes into a single array and filter out companions
+all_changes = (polkadot_cl.changes + substrate_cl.changes).reject do |c|
+  c[:title] =~ /[Cc]ompanion/
+end
+
+# Set all the variables needed for a release
+
+misc_changes = Changelog.changes_with_label(all_changes, 'B1-releasenotes')
+client_changes = Changelog.changes_with_label(all_changes, 'B5-clientnoteworthy')
+runtime_changes = Changelog.changes_with_label(all_changes, 'B7-runtimenoteworthy')
+
+# Add the audit status for runtime changes
+runtime_changes.each do |c|
+  if c[:labels].any? { |l| l[:name] == 'D1-audited 👍' }
+    c[:pretty_title] = "✅ `audited` #{c[:pretty_title]}"
+    next
+  end
+  if c[:labels].any? { |l| l[:name] == 'D2-notlive 💤' }
+    c[:pretty_title] = "✅ `not live` #{c[:pretty_title]}"
+    next
+  end
+  if c[:labels].any? { |l| l[:name] == 'D3-trivial 🧸' }
+    c[:pretty_title] = "✅ `trivial` #{c[:pretty_title]}"
+    next
+  end
+  if c[:labels].any? { |l| l[:name] == 'D5-nicetohaveaudit ⚠️' }
+    c[:pretty_title] = "⏳ `pending non-critical audit` #{c[:pretty_title]}"
+    next
+  end
+  if c[:labels].any? { |l| l[:name] == 'D9-needsaudit 👮' }
+    c[:pretty_title] = "❌ `AWAITING AUDIT` #{c[:pretty_title]}"
+    next
+  end
+  c[:pretty_title] = "⭕️ `unknown audit requirements` #{c[:pretty_title]}"
+end
+
+# The priority of users upgraded is determined by the highest-priority
+# *Client* change
+release_priority = Changelog.highest_priority_for_changes(client_changes)
+
+# Pulled from the previous Github step
+rustc_stable = ENV['RUSTC_STABLE']
+rustc_nightly = ENV['RUSTC_NIGHTLY']
+polkadot_runtime = get_runtime('polkadot', polkadot_path)
+kusama_runtime = get_runtime('kusama', polkadot_path)
+westend_runtime = get_runtime('westend', polkadot_path)
+rococo_runtime = get_runtime('rococo', polkadot_path)
+
+# These json files should have been downloaded as part of the build-runtimes
+# github action
+
+polkadot_json = JSON.parse(
+  File.read(
+    "#{ENV['GITHUB_WORKSPACE']}/polkadot-srtool-json/polkadot_srtool_output.json"
+  )
+)
+
+kusama_json = JSON.parse(
+  File.read(
+    "#{ENV['GITHUB_WORKSPACE']}/kusama-srtool-json/kusama_srtool_output.json"
+  )
+)
+
+puts renderer.result

polkadot/scripts/ci/github/lib.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# Gets the runtime version for a given runtime from the filesystem.
+# Optionally accepts a path that is the root of the project which defaults to
+# the current working directory
+def get_runtime(runtime: nil, path: '.', runtime_dir: 'runtime')
+  File.open(path + "/#{runtime_dir}/#{runtime}/src/lib.rs") do |f|
+    f.find { |l| l =~ /spec_version/ }.match(/[0-9]+/)[0]
+  end
+end

polkadot/scripts/ci/github/polkadot_release.erb

@@ -0,0 +1,42 @@
+<%= print release_priority[:text] %> <%= puts " due to changes: *#{Changelog.changes_with_label(all_changes, release_priority[:label]).map(&:pretty_title).join(", ")}*" if release_priority[:priority] > 1 %>
+
+Native runtimes:
+
+- Polkadot: **<%= polkadot_runtime %>**
+- Kusama: **<%= kusama_runtime %>**
+- Westend: **<%= westend_runtime %>**
+
+This release was tested against the following versions of `rustc`. Other versions may work.
+
+- <%= rustc_stable %>
+- <%= rustc_nightly %>
+
+WASM runtimes built with [<%= polkadot_json['info']['generator']['name'] %> v<%= polkadot_json['info']['generator']['version'] %>](https://github.com/paritytech/srtool) using `<%= polkadot_json['rustc'] %>`.
+
+Proposal hashes:
+* `polkadot_runtime-v<%= polkadot_runtime %>.compact.compressed.wasm`: `<%= polkadot_json['runtimes']['compressed']['prop'] %>`
+* `kusama_runtime-v<%= kusama_runtime %>.compact.compressed.wasm`: `<%= kusama_json['runtimes']['compressed']['prop'] %>`
+
+<% unless misc_changes.empty? %>
+## Changes
+
+<% misc_changes.each do |c| %>
+* <%= c[:pretty_title] %>
+<% end %>
+<% end %>
+
+<% unless client_changes.empty? %>
+## Client
+
+<% client_changes.each do |c| %>
+* <%= c[:pretty_title] %>
+<% end %>
+<% end %>
+
+<% unless runtime_changes.empty? %>
+## Runtime
+
+<% runtime_changes.each do |c| %>
+* <%= c[:pretty_title] %>
+<% end %>
+<% end %>

polkadot/scripts/ci/github/run_fuzzer.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+timeout --signal INT 5h cargo hfuzz run $1
+status=$?
+
+if [ $status -ne 124 ]; then
+  echo "Found a panic!"
+  # TODO: provide Minimal Reproducible Input
+  # TODO: message on Matrix
+  exit 1
+else
+  echo "Didn't find any problem in 5 hours of fuzzing"
+fi