From 497772da184f47f973c36a6be203fd6a698c9189 Mon Sep 17 00:00:00 2001 From: Cecile Tonglet Date: Thu, 6 Aug 2020 06:52:03 +0200 Subject: [PATCH] Ignore checks for companion PRs (#1455) --- polkadot/.editorconfig | 6 + polkadot/.gitlab-ci.yml | 25 +- polkadot/scripts/gitlab/check_runtime.sh | 272 +++++++++--------- .../gitlab/check_runtime_benchmarks.sh | 8 + polkadot/scripts/gitlab/check_web_wasm.sh | 15 + polkadot/scripts/gitlab/lib.sh | 15 + .../scripts/gitlab/test_deterministic_wasm.sh | 17 ++ polkadot/scripts/gitlab/test_linux_stable.sh | 8 + 8 files changed, 210 insertions(+), 156 deletions(-) create mode 100755 polkadot/scripts/gitlab/check_runtime_benchmarks.sh create mode 100755 polkadot/scripts/gitlab/check_web_wasm.sh create mode 100755 polkadot/scripts/gitlab/test_deterministic_wasm.sh create mode 100755 polkadot/scripts/gitlab/test_linux_stable.sh diff --git a/polkadot/.editorconfig b/polkadot/.editorconfig index f511aad460..dda39da09e 100644 --- a/polkadot/.editorconfig +++ b/polkadot/.editorconfig @@ -14,3 +14,9 @@ indent_style=space indent_size=2 tab_width=8 end_of_line=lf + +[*.sh] +indent_style=space +indent_size=2 +tab_width=8 +end_of_line=lf diff --git a/polkadot/.gitlab-ci.yml b/polkadot/.gitlab-ci.yml index 11312ad649..1f2ef6ded2 100644 --- a/polkadot/.gitlab-ci.yml +++ b/polkadot/.gitlab-ci.yml @@ -99,17 +99,7 @@ test-deterministic-wasm: <<: *docker-env except: script: - # build runtime - - WASM_BUILD_NO_COLOR=1 cargo build --verbose --release -p kusama-runtime -p polkadot-runtime -p westend-runtime - # make checksum - - sha256sum target/release/wbuild/target/wasm32-unknown-unknown/release/*.wasm > checksum.sha256 - # clean up – FIXME: can we reuse some of the artifacts? - - cargo clean - # build again - - WASM_BUILD_NO_COLOR=1 cargo build --verbose --release -p kusama-runtime -p polkadot-runtime -p westend-runtime - # confirm checksum - - sha256sum -c checksum.sha256 - + - ./scripts/gitlab/test_deterministic_wasm.sh test-linux-stable: &test stage: test @@ -123,7 +113,7 @@ test-linux-stable: &test RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" TARGET: native script: - - time cargo test --all --release --verbose --locked --features runtime-benchmarks + - ./scripts/gitlab/test_linux_stable.sh - sccache -s check-web-wasm: &test @@ -134,14 +124,7 @@ check-web-wasm: &test script: # WASM support is in progress. As more and more crates support WASM, we # should add entries here. See https://github.com/paritytech/polkadot/issues/625 - - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path runtime/polkadot/Cargo.toml - - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path runtime/kusama/Cargo.toml - - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path erasure-coding/Cargo.toml - - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path parachain/Cargo.toml - - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path primitives/Cargo.toml - - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path rpc/Cargo.toml - - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path statement-table/Cargo.toml - - time cargo build --locked --target=wasm32-unknown-unknown --manifest-path cli/Cargo.toml --no-default-features --features browser + - ./scripts/gitlab/check_web_wasm.sh - sccache -s check-runtime-benchmarks: &test @@ -151,7 +134,7 @@ check-runtime-benchmarks: &test <<: *compiler_info script: # Check that the node will compile with `runtime-benchmarks` feature flag. - - time cargo check --features runtime-benchmarks + - ./scripts/gitlab/check_runtime_benchmarks.sh - sccache -s build-wasm-release: diff --git a/polkadot/scripts/gitlab/check_runtime.sh b/polkadot/scripts/gitlab/check_runtime.sh index 30fdcd455c..0c635c8824 100755 --- a/polkadot/scripts/gitlab/check_runtime.sh +++ b/polkadot/scripts/gitlab/check_runtime.sh @@ -34,13 +34,13 @@ boldprint "make sure the master branch is available in shallow clones" git fetch --depth="${GIT_DEPTH:-100}" origin master runtimes=( - "kusama" - "polkadot" - "westend" + "kusama" + "polkadot" + "westend" ) common_dirs=( - "common" + "common" ) # Helper function to join elements in an array with a multi-char delimiter @@ -52,176 +52,178 @@ runtime_regex="^runtime/$(join_by '|^runtime/' "${runtimes[@]}" "${common_dirs[@ boldprint "check if the wasm sources changed since ${LATEST_TAG}" if ! git diff --name-only "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" \ - | grep -E -q -e "$runtime_regex" + | grep -E -q -e "$runtime_regex" then - boldprint "no changes to any runtime source code detected" - # continue checking if Cargo.lock was updated with a new substrate reference - # and if that change includes a {spec|impl}_version update. + boldprint "no changes to any runtime source code detected" + # continue checking if Cargo.lock was updated with a new substrate reference + # and if that change includes a {spec|impl}_version update. - SUBSTRATE_REFS_CHANGED="$( - git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \ - | sed -n -r "s~^[\+\-]source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | wc -l - )" + SUBSTRATE_REFS_CHANGED="$( + git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \ + | sed -n -r "s~^[\+\-]source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | wc -l + )" - # check Cargo.lock for substrate ref change - case "${SUBSTRATE_REFS_CHANGED}" in - (0) - boldprint "substrate refs not changed in Cargo.lock" - exit 0 - ;; - (2) - boldprint "substrate refs updated since ${LATEST_TAG}" - ;; - (*) - boldprint "check unsupported: more than one commit targeted in repo ${SUBSTRATE_REPO_CARGO}" - exit 1 - esac + # check Cargo.lock for substrate ref change + case "${SUBSTRATE_REFS_CHANGED}" in + (0) + boldprint "substrate refs not changed in Cargo.lock" + exit 0 + ;; + (2) + boldprint "substrate refs updated since ${LATEST_TAG}" + ;; + (*) + boldprint "check unsupported: more than one commit targeted in repo ${SUBSTRATE_REPO_CARGO}" + exit 1 + esac - SUBSTRATE_PREV_REF="$( - git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \ - | sed -n -r "s~^\-source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | head -n 1 - )" + SUBSTRATE_PREV_REF="$( + git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \ + | sed -n -r "s~^\-source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | head -n 1 + )" - SUBSTRATE_NEW_REF="$( - git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \ - | sed -n -r "s~^\+source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | head -n 1 - )" + SUBSTRATE_NEW_REF="$( + git diff "refs/tags/${LATEST_TAG}...${CI_COMMIT_SHA}" Cargo.lock \ + | sed -n -r "s~^\+source = \"${SUBSTRATE_REPO_CARGO}#([a-f0-9]+)\".*$~\1~p" | sort -u | head -n 1 + )" - boldcat <<-EOT - previous substrate commit id ${SUBSTRATE_PREV_REF} - new substrate commit id ${SUBSTRATE_NEW_REF} - EOT + boldcat < ${add_spec_version} +spec_version: ${sub_spec_version} -> ${add_spec_version} - EOT - continue +EOT + continue - else - # check for impl_version updates: if only the impl versions changed, we assume - # there is no consensus-critical logic that has changed. + else + # check for impl_version updates: if only the impl versions changed, we assume + # there is no consensus-critical logic that has changed. - add_impl_version="$( - git diff refs/tags/"${LATEST_TAG}...${CI_COMMIT_SHA}" "runtime/${RUNTIME}/src/lib.rs" \ - | sed -n -r 's/^\+[[:space:]]+impl_version: +([0-9]+),$/\1/p' - )" - sub_impl_version="$( - git diff refs/tags/"${LATEST_TAG}...${CI_COMMIT_SHA}" "runtime/${RUNTIME}/src/lib.rs" \ - | sed -n -r 's/^\-[[:space:]]+impl_version: +([0-9]+),$/\1/p' - )" + add_impl_version="$( + git diff refs/tags/"${LATEST_TAG}...${CI_COMMIT_SHA}" "runtime/${RUNTIME}/src/lib.rs" \ + | sed -n -r 's/^\+[[:space:]]+impl_version: +([0-9]+),$/\1/p' + )" + sub_impl_version="$( + git diff refs/tags/"${LATEST_TAG}...${CI_COMMIT_SHA}" "runtime/${RUNTIME}/src/lib.rs" \ + | sed -n -r 's/^\-[[:space:]]+impl_version: +([0-9]+),$/\1/p' + )" - # see if the impl version changed - if [ "${add_impl_version}" != "${sub_impl_version}" ] - then - boldcat <<-EOT + # see if the impl version changed + if [ "${add_impl_version}" != "${sub_impl_version}" ] + then + boldcat < ${add_impl_version} +impl_version: ${sub_impl_version} -> ${add_impl_version} - EOT - continue - fi +EOT + continue + fi - - boldcat <<-EOT - wasm source files changed or the spec version in the substrate reference in - the Cargo.lock but not the spec/impl version. If changes made do not alter - logic, just bump 'impl_version'. If they do change logic, bump - 'spec_version'. - - source file directories: - - runtime - - versions file: ${RUNTIME} - - EOT - - exit 1 - fi + failed_runtime_checks+=("$RUNTIME") + fi done -exit 0 +if [ ${#failed_runtime_checks} -gt 0 ]; then + boldcat </dev/null 2>&1 && pwd )/lib.sh" + +skip_if_companion_pr + +time cargo check --features runtime-benchmarks diff --git a/polkadot/scripts/gitlab/check_web_wasm.sh b/polkadot/scripts/gitlab/check_web_wasm.sh new file mode 100755 index 0000000000..056e90d1e3 --- /dev/null +++ b/polkadot/scripts/gitlab/check_web_wasm.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +#shellcheck source=lib.sh +source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh" + +skip_if_companion_pr + +time cargo build --locked --target=wasm32-unknown-unknown --manifest-path runtime/polkadot/Cargo.toml +time cargo build --locked --target=wasm32-unknown-unknown --manifest-path runtime/kusama/Cargo.toml +time cargo build --locked --target=wasm32-unknown-unknown --manifest-path erasure-coding/Cargo.toml +time cargo build --locked --target=wasm32-unknown-unknown --manifest-path parachain/Cargo.toml +time cargo build --locked --target=wasm32-unknown-unknown --manifest-path primitives/Cargo.toml +time cargo build --locked --target=wasm32-unknown-unknown --manifest-path rpc/Cargo.toml +time cargo build --locked --target=wasm32-unknown-unknown --manifest-path statement-table/Cargo.toml +time cargo build --locked --target=wasm32-unknown-unknown --manifest-path cli/Cargo.toml --no-default-features --features browser diff --git a/polkadot/scripts/gitlab/lib.sh b/polkadot/scripts/gitlab/lib.sh index ab2e7c86f6..993cab35e3 100755 --- a/polkadot/scripts/gitlab/lib.sh +++ b/polkadot/scripts/gitlab/lib.sh @@ -96,3 +96,18 @@ curl -XPOST -d "$1" "https://matrix.parity.io/_matrix/client/r0/rooms/$2/send/m. # Pretty-printing functions boldprint () { printf "|\n| \033[1m%s\033[0m\n|\n" "${@}"; } boldcat () { printf "|\n"; while read -r l; do printf "| \033[1m%s\033[0m\n" "${l}"; done; printf "|\n" ; } + +skip_if_companion_pr() { + url="https://api.github.com/repos/paritytech/polkadot/pulls/${CI_COMMIT_REF_NAME}" + echo "[+] API URL: $url" + + pr_title=$(curl -sSL -H "Authorization: token ${GITHUB_PR_TOKEN}" "$url" | jq -r .title) + echo "[+] PR title: $pr_title" + + if echo "$pr_title" | grep -qi '^companion'; then + echo "[!] PR is a companion PR. Build is already done in substrate" + exit 0 + else + echo "[+] PR is not a companion PR. Proceeding test" + fi +} diff --git a/polkadot/scripts/gitlab/test_deterministic_wasm.sh b/polkadot/scripts/gitlab/test_deterministic_wasm.sh new file mode 100755 index 0000000000..f2bf2a4f05 --- /dev/null +++ b/polkadot/scripts/gitlab/test_deterministic_wasm.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +#shellcheck source=lib.sh +source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh" + +skip_if_companion_pr + +# build runtime +WASM_BUILD_NO_COLOR=1 cargo build --verbose --release -p kusama-runtime -p polkadot-runtime -p westend-runtime +# make checksum +sha256sum target/release/wbuild/target/wasm32-unknown-unknown/release/*.wasm > checksum.sha256 +# clean up - FIXME: can we reuse some of the artifacts? +cargo clean +# build again +WASM_BUILD_NO_COLOR=1 cargo build --verbose --release -p kusama-runtime -p polkadot-runtime -p westend-runtime +# confirm checksum +sha256sum -c checksum.sha256 diff --git a/polkadot/scripts/gitlab/test_linux_stable.sh b/polkadot/scripts/gitlab/test_linux_stable.sh new file mode 100755 index 0000000000..f523e24434 --- /dev/null +++ b/polkadot/scripts/gitlab/test_linux_stable.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +#shellcheck source=lib.sh +source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/lib.sh" + +skip_if_companion_pr + +time cargo test --all --release --verbose --locked --features runtime-benchmarks