CI: Make alert_pending_release.sh filter substrate changes based on github labels... (#850)

* CI: Split common functions into lib.sh and make...

... alert_pending_release.sh check the substrate changes for various
labels. If labelled, will be included in the changelog, otherwise will
silently be left out.

* replace non-standard /bin/bash with /usr/bin/env bash

* CI: make lib.sh /bin/sh-compatible

polkadot/scripts/gitlab/alert_pending_release.sh

@@ -1,19 +1,7 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
-# structure_message $content $formatted_content (optional)
+#shellcheck source=lib.sh
-structure_message() {
+source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh"
-  if [ -z "$2" ]; then
-    body=$(jq -Rs --arg body "$1" '{"msgtype": "m.text", $body}' < /dev/null)
-  else
-    body=$(jq -Rs --arg body "$1" --arg formatted_body "$2" '{"msgtype": "m.text", $body, "format": "org.matrix.custom.html", $formatted_body}' < /dev/null)
-  fi
-  echo "$body"
-}
-
-# send_message $body (json formatted) $room_id $access_token
-send_message() {
-curl -XPOST -d "$1" "https://matrix.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3"
-}
 
 # Receive keys
 # trusted_keys=(

polkadot/scripts/gitlab/lib.sh

@@ -0,0 +1,73 @@
+#!/bin/sh
+
+api_base="https://api.github.com/repos"
+
+# Function to take 2 git tags/commits and get any lines from commit messages
+# that contain something that looks like a PR reference: e.g., (#1234)
+sanitised_git_logs(){
+  git --no-pager log --pretty=format:"%s" "$1..$2" |
+  # Only find messages referencing a PR
+  grep -E '\(#[0-9]+\)' |
+  # Strip any asterisks
+  sed 's/^* //g' |
+  # And add them all back
+  sed 's/^/* /g'
+}
+
+# Checks whether a tag on github has been verified
+# repo: 'organization/repo'
+# tagver: 'v1.2.3'
+# Usage: check_tag $repo $tagver
+check_tag () {
+  repo=$1
+  tagver=$2
+  tag_out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$repo/git/refs/tags/$tagver")
+  tag_sha=$(echo "$tag_out" | jq -r .object.sha)
+  object_url=$(echo "$tag_out" | jq -r .object.url)
+  if [ "$tag_sha" = "null" ]; then
+    return 2
+  fi
+  verified_str=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$object_url" | jq -r .verification.verified)
+  if [ "$verified_str" = "true" ]; then
+    # Verified, everything is good
+    return 0
+  else
+    # Not verified. Bad juju.
+    return 1
+  fi
+}
+
+# Checks whether a given PR has a given label.
+# repo: 'organization/repo'
+# pr_id: 12345
+# label: B1-silent
+# Usage: has_label $repo $pr_id $label
+has_label(){
+  repo="$1"
+  pr_id="$2"
+  label="$3"
+  out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$repo/pulls/$pr_id")
+  [ -n "$(echo "$out" | jq ".labels | .[] | select(.name==\"$label\")")" ]
+}
+
+# Formats a message into a JSON string for posting to Matrix
+# message: 'any plaintext message'
+# formatted_message: '<strong>optional message formatted in <em>html</em></strong>' 
+# Usage: structure_message $content $formatted_content (optional)
+structure_message() {
+  if [ -z "$2" ]; then
+    body=$(jq -Rs --arg body "$1" '{"msgtype": "m.text", $body}' < /dev/null)
+  else
+    body=$(jq -Rs --arg body "$1" --arg formatted_body "$2" '{"msgtype": "m.text", $body, "format": "org.matrix.custom.html", $formatted_body}' < /dev/null)
+  fi
+  echo "$body"
+}
+
+# Post a message to a matrix room
+# body: '{body: "JSON string produced by structure_message"}'
+# room_id: !fsfSRjgjBWEWffws:matrix.parity.io
+# access_token: see https://matrix.org/docs/guides/client-server-api/
+# Usage: send_message $body (json formatted) $room_id $access_token
+send_message() {
+curl -XPOST -d "$1" "https://matrix.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3"
+}

polkadot/scripts/gitlab/publish_draft_release.sh

@@ -1,55 +1,19 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
-# Function to take 2 git tags/commits and get any lines from commit messages
+# shellcheck source=lib.sh
-# that contain something that looks like a PR reference: e.g., (#1234)
+source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh"
-sanitised_git_logs(){
-  git --no-pager log --pretty=format:"%s" "$1..$2" |
-  # Only find messages referencing a PR
-  grep -E '\(#[0-9]+\)' |
-  # Strip any asterisks
-  sed 's/^* //g' |
-  # And add them all back
-  sed 's/^/* /g'
-}
-
-check_tag () {
-  tagver=$1
-  tag_out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/git/refs/tags/$tagver")
-  tag_sha=$(echo "$tag_out" | jq -r .object.sha)
-  object_url=$(echo "$tag_out" | jq -r .object.url)
-  if [ "$tag_sha" == "null" ]; then
-    return 2
-  fi
-  verified_str=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$object_url" | jq -r .verification.verified)
-  if [ "$verified_str" == "true" ]; then
-    # Verified, everything is good
-    return 0
-  else
-    # Not verified. Bad juju.
-    return 1
-  fi
-}
-
-# structure_message $content $formatted_content (optional)
-structure_message() {
-  if [ -z "$2" ]; then
-    body=$(jq -Rs --arg body "$1" '{"msgtype": "m.text", $body}' < /dev/null)
-  else
-    body=$(jq -Rs --arg body "$1" --arg formatted_body "$2" '{"msgtype": "m.text", $body, "format": "org.matrix.custom.html", $formatted_body}' < /dev/null)
-  fi
-  echo "$body"
-}
-
-# send_message $body (json formatted) $room_id $access_token
-send_message() {
-curl -XPOST -d "$1" "https://matrix.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3"
-}
 
 # Set initial variables
-api_base="https://api.github.com/repos/paritytech/polkadot"
 substrate_repo="https://github.com/paritytech/substrate"
 substrate_dir='./substrate'
 
+# Substrate labels for PRs we want to include in the release notes
+labels=(
+  'B1-runtimenoteworthy'
+  'B1-clientnoteworthy'
+  'B1-apinoteworthy'
+)
+
 # Cloning repos to ensure freshness
 echo "[+] Cloning substrate to generate list of changes"
 git clone $substrate_repo $substrate_dir
@@ -61,22 +25,26 @@ echo "[+] Version: $version; Previous version: $last_version"
 
 # Check that a signed tag exists on github for this version
 echo '[+] Checking tag has been signed'
-check_tag "$version"
+check_tag "paritytech/polkadot" "$version"
 case $? in
   0) echo '[+] Tag found and has been signed'
     ;;
   1) echo '[!] Tag found but has not been signed. Aborting release.'; exit 1
     ;;
-  2) echo '[!] Tag not found. Aborting release.'; exit 
+  2) echo '[!] Tag not found. Aborting release.'; exit
 esac
 
 # Start with referencing current native runtime
 # and find any referenced PRs since last release
+# Note: Drop any changes that begin with '[contracts]' or 'contracts:'
 spec=$(grep spec_version runtime/kusama/src/lib.rs | tail -n 1 | grep -Eo '[0-9]{4}')
 echo "[+] Spec version: $spec"
 release_text="Native for runtime $spec.
 
-$(sanitised_git_logs "$last_version" "$version")"
+$(sanitised_git_logs "$last_version" "$version" | \
+  sed '/^\[contracts\].*/d' | \
+  sed '/^contracts:.*/d' \
+)"
 
 # Get substrate changes between last polkadot version and current
 cur_substrate_commit=$(grep -A 2 'name = "sc-cli"' Cargo.lock | grep -E -o '[a-f0-9]{40}')
@@ -86,7 +54,23 @@ old_substrate_commit=$(grep -A 2 'name = "sc-cli"' Cargo.lock | grep -E -o '[a-f
 pushd $substrate_dir || exit
   git checkout polkadot-master > /dev/null
   git pull > /dev/null
-  substrate_changes="$(sanitised_git_logs "$old_substrate_commit" "$cur_substrate_commit" | sed 's/(#/(paritytech\/substrate#/')"
+  all_substrate_changes="$(sanitised_git_logs "$old_substrate_commit" "$cur_substrate_commit" | sed 's/(#/(paritytech\/substrate#/')"
+  substrate_changes=""
+  echo "[+] Iterating through substrate changes to find labelled PRs"
+  while IFS= read -r line; do
+    pr_id=$(echo "$line" | sed -E 's/.*#([0-9]+)\)$/\1/')
+
+    # Skip if the PR has the silent label - this allows us to skip a few requests
+    if has_label 'paritytech/substrate' "$pr_id" 'B0-silent'; then
+      continue
+    fi
+    for label in "${labels[@]}"; do
+      if has_label 'paritytech/substrate' "$pr_id" "$label"; then
+        substrate_changes="$substrate_changes
+$line"
+      fi
+    done
+  done <<< "$all_substrate_changes"
 popd || exit
 
 echo "[+] Changes generated. Removing temporary repos"
@@ -143,6 +127,6 @@ formatted_msg_body=$(cat <<EOF
 Draft release created: $html_url
 EOF
 )
-send_message "$(structure_message "$msg_body" "$formatted_msg_body")" "$MATRIX_ROCCESS_TOKEN"
+send_message "$(structure_message "$msg_body" "$formatted_msg_body")" "$MATRIX_ACCESS_TOKEN"
 
 echo "[+] Done! Maybe the release worked..."