From 25e2448597269a6270bcbb5dc5937f578ec2fd8b Mon Sep 17 00:00:00 2001 From: s3krit Date: Tue, 18 Feb 2020 15:59:22 +0100 Subject: [PATCH] CI: Make alert_pending_release.sh filter substrate changes based on github labels... (#850) * CI: Split common functions into lib.sh and make... ... alert_pending_release.sh check the substrate changes for various labels. If labelled, will be included in the changelog, otherwise will silently be left out. * replace non-standard /bin/bash with /usr/bin/env bash * CI: make lib.sh /bin/sh-compatible --- .../scripts/gitlab/alert_pending_release.sh | 18 +--- polkadot/scripts/gitlab/lib.sh | 73 ++++++++++++++++ .../scripts/gitlab/publish_draft_release.sh | 86 ++++++++----------- 3 files changed, 111 insertions(+), 66 deletions(-) create mode 100755 polkadot/scripts/gitlab/lib.sh diff --git a/polkadot/scripts/gitlab/alert_pending_release.sh b/polkadot/scripts/gitlab/alert_pending_release.sh index a88677fc5f..0adf4ae98c 100755 --- a/polkadot/scripts/gitlab/alert_pending_release.sh +++ b/polkadot/scripts/gitlab/alert_pending_release.sh @@ -1,19 +1,7 @@ -#!/bin/bash +#!/usr/bin/env bash -# structure_message $content $formatted_content (optional) -structure_message() { - if [ -z "$2" ]; then - body=$(jq -Rs --arg body "$1" '{"msgtype": "m.text", $body}' < /dev/null) - else - body=$(jq -Rs --arg body "$1" --arg formatted_body "$2" '{"msgtype": "m.text", $body, "format": "org.matrix.custom.html", $formatted_body}' < /dev/null) - fi - echo "$body" -} - -# send_message $body (json formatted) $room_id $access_token -send_message() { -curl -XPOST -d "$1" "https://matrix.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3" -} +#shellcheck source=lib.sh +source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh" # Receive keys # trusted_keys=( diff --git a/polkadot/scripts/gitlab/lib.sh b/polkadot/scripts/gitlab/lib.sh new file mode 100755 index 0000000000..bc0e06a6d4 --- /dev/null +++ b/polkadot/scripts/gitlab/lib.sh @@ -0,0 +1,73 @@ +#!/bin/sh + +api_base="https://api.github.com/repos" + +# Function to take 2 git tags/commits and get any lines from commit messages +# that contain something that looks like a PR reference: e.g., (#1234) +sanitised_git_logs(){ + git --no-pager log --pretty=format:"%s" "$1..$2" | + # Only find messages referencing a PR + grep -E '\(#[0-9]+\)' | + # Strip any asterisks + sed 's/^* //g' | + # And add them all back + sed 's/^/* /g' +} + +# Checks whether a tag on github has been verified +# repo: 'organization/repo' +# tagver: 'v1.2.3' +# Usage: check_tag $repo $tagver +check_tag () { + repo=$1 + tagver=$2 + tag_out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$repo/git/refs/tags/$tagver") + tag_sha=$(echo "$tag_out" | jq -r .object.sha) + object_url=$(echo "$tag_out" | jq -r .object.url) + if [ "$tag_sha" = "null" ]; then + return 2 + fi + verified_str=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$object_url" | jq -r .verification.verified) + if [ "$verified_str" = "true" ]; then + # Verified, everything is good + return 0 + else + # Not verified. Bad juju. + return 1 + fi +} + +# Checks whether a given PR has a given label. +# repo: 'organization/repo' +# pr_id: 12345 +# label: B1-silent +# Usage: has_label $repo $pr_id $label +has_label(){ + repo="$1" + pr_id="$2" + label="$3" + out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/$repo/pulls/$pr_id") + [ -n "$(echo "$out" | jq ".labels | .[] | select(.name==\"$label\")")" ] +} + +# Formats a message into a JSON string for posting to Matrix +# message: 'any plaintext message' +# formatted_message: 'optional message formatted in html' +# Usage: structure_message $content $formatted_content (optional) +structure_message() { + if [ -z "$2" ]; then + body=$(jq -Rs --arg body "$1" '{"msgtype": "m.text", $body}' < /dev/null) + else + body=$(jq -Rs --arg body "$1" --arg formatted_body "$2" '{"msgtype": "m.text", $body, "format": "org.matrix.custom.html", $formatted_body}' < /dev/null) + fi + echo "$body" +} + +# Post a message to a matrix room +# body: '{body: "JSON string produced by structure_message"}' +# room_id: !fsfSRjgjBWEWffws:matrix.parity.io +# access_token: see https://matrix.org/docs/guides/client-server-api/ +# Usage: send_message $body (json formatted) $room_id $access_token +send_message() { +curl -XPOST -d "$1" "https://matrix.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3" +} diff --git a/polkadot/scripts/gitlab/publish_draft_release.sh b/polkadot/scripts/gitlab/publish_draft_release.sh index d4615b3485..0e9b004529 100755 --- a/polkadot/scripts/gitlab/publish_draft_release.sh +++ b/polkadot/scripts/gitlab/publish_draft_release.sh @@ -1,55 +1,19 @@ -#!/bin/bash +#!/usr/bin/env bash -# Function to take 2 git tags/commits and get any lines from commit messages -# that contain something that looks like a PR reference: e.g., (#1234) -sanitised_git_logs(){ - git --no-pager log --pretty=format:"%s" "$1..$2" | - # Only find messages referencing a PR - grep -E '\(#[0-9]+\)' | - # Strip any asterisks - sed 's/^* //g' | - # And add them all back - sed 's/^/* /g' -} - -check_tag () { - tagver=$1 - tag_out=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$api_base/git/refs/tags/$tagver") - tag_sha=$(echo "$tag_out" | jq -r .object.sha) - object_url=$(echo "$tag_out" | jq -r .object.url) - if [ "$tag_sha" == "null" ]; then - return 2 - fi - verified_str=$(curl -H "Authorization: token $GITHUB_RELEASE_TOKEN" -s "$object_url" | jq -r .verification.verified) - if [ "$verified_str" == "true" ]; then - # Verified, everything is good - return 0 - else - # Not verified. Bad juju. - return 1 - fi -} - -# structure_message $content $formatted_content (optional) -structure_message() { - if [ -z "$2" ]; then - body=$(jq -Rs --arg body "$1" '{"msgtype": "m.text", $body}' < /dev/null) - else - body=$(jq -Rs --arg body "$1" --arg formatted_body "$2" '{"msgtype": "m.text", $body, "format": "org.matrix.custom.html", $formatted_body}' < /dev/null) - fi - echo "$body" -} - -# send_message $body (json formatted) $room_id $access_token -send_message() { -curl -XPOST -d "$1" "https://matrix.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3" -} +# shellcheck source=lib.sh +source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh" # Set initial variables -api_base="https://api.github.com/repos/paritytech/polkadot" substrate_repo="https://github.com/paritytech/substrate" substrate_dir='./substrate' +# Substrate labels for PRs we want to include in the release notes +labels=( + 'B1-runtimenoteworthy' + 'B1-clientnoteworthy' + 'B1-apinoteworthy' +) + # Cloning repos to ensure freshness echo "[+] Cloning substrate to generate list of changes" git clone $substrate_repo $substrate_dir @@ -61,22 +25,26 @@ echo "[+] Version: $version; Previous version: $last_version" # Check that a signed tag exists on github for this version echo '[+] Checking tag has been signed' -check_tag "$version" +check_tag "paritytech/polkadot" "$version" case $? in 0) echo '[+] Tag found and has been signed' ;; 1) echo '[!] Tag found but has not been signed. Aborting release.'; exit 1 ;; - 2) echo '[!] Tag not found. Aborting release.'; exit + 2) echo '[!] Tag not found. Aborting release.'; exit esac # Start with referencing current native runtime # and find any referenced PRs since last release +# Note: Drop any changes that begin with '[contracts]' or 'contracts:' spec=$(grep spec_version runtime/kusama/src/lib.rs | tail -n 1 | grep -Eo '[0-9]{4}') echo "[+] Spec version: $spec" release_text="Native for runtime $spec. -$(sanitised_git_logs "$last_version" "$version")" +$(sanitised_git_logs "$last_version" "$version" | \ + sed '/^\[contracts\].*/d' | \ + sed '/^contracts:.*/d' \ +)" # Get substrate changes between last polkadot version and current cur_substrate_commit=$(grep -A 2 'name = "sc-cli"' Cargo.lock | grep -E -o '[a-f0-9]{40}') @@ -86,7 +54,23 @@ old_substrate_commit=$(grep -A 2 'name = "sc-cli"' Cargo.lock | grep -E -o '[a-f pushd $substrate_dir || exit git checkout polkadot-master > /dev/null git pull > /dev/null - substrate_changes="$(sanitised_git_logs "$old_substrate_commit" "$cur_substrate_commit" | sed 's/(#/(paritytech\/substrate#/')" + all_substrate_changes="$(sanitised_git_logs "$old_substrate_commit" "$cur_substrate_commit" | sed 's/(#/(paritytech\/substrate#/')" + substrate_changes="" + echo "[+] Iterating through substrate changes to find labelled PRs" + while IFS= read -r line; do + pr_id=$(echo "$line" | sed -E 's/.*#([0-9]+)\)$/\1/') + + # Skip if the PR has the silent label - this allows us to skip a few requests + if has_label 'paritytech/substrate' "$pr_id" 'B0-silent'; then + continue + fi + for label in "${labels[@]}"; do + if has_label 'paritytech/substrate' "$pr_id" "$label"; then + substrate_changes="$substrate_changes +$line" + fi + done + done <<< "$all_substrate_changes" popd || exit echo "[+] Changes generated. Removing temporary repos" @@ -143,6 +127,6 @@ formatted_msg_body=$(cat <