CI: Rewrite check-each-crate in python (#13238)

* CI: Rewrite `check-each-crate` in python

This is a much better readable version of the script and it should also work on Macos and not
siltently fail ;)

* Fix dumb bugs and print everything to stderr

* Don't buffer Python output

* 🤦

* 🤦 🤦

* Use check all until we have more macos runners

* Update scripts/ci/gitlab/check-each-crate.py

Co-authored-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

* Update scripts/ci/gitlab/pipeline/test.yml

Co-authored-by: Vladimir Istyufeev <vladimir@parity.io>
Co-authored-by: Oliver Tale-Yazdi <oliver.tale-yazdi@parity.io>

substrate/scripts/ci/gitlab/check-each-crate.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+# A script that checks each workspace crate individually.
+# It's relevant to check workspace crates individually because otherwise their compilation problems
+# due to feature misconfigurations won't be caught, as exemplified by
+# https://github.com/paritytech/substrate/issues/12705
+#
+# `check-each-crate.py target_group groups_total`
+#
+# - `target_group`: Integer starting from 1, the group this script should execute.
+# - `groups_total`: Integer starting from 1, total number of groups.
+
+import subprocess, sys
+
+# Get all crates
+output = subprocess.check_output(["cargo", "tree", "--locked", "--workspace", "--depth", "0", "--prefix", "none"])
+
+# Convert the output into a proper list
+crates = []
+for line in output.splitlines():
+	if line != b"":
+		crates.append(line.decode('utf8').split(" ")[0])
+
+# Make the list unique and sorted
+crates = list(set(crates))
+crates.sort()
+
+target_group = int(sys.argv[1]) - 1
+groups_total = int(sys.argv[2])
+
+if len(crates) == 0:
+	print("No crates detected!", file=sys.stderr)
+	sys.exit(1)
+
+print(f"Total crates: {len(crates)}", file=sys.stderr)
+
+crates_per_group = len(crates) // groups_total
+
+# If this is the last runner, we need to take care of crates
+# after the group that we lost because of the integer division.
+if target_group + 1 == groups_total:
+	overflow_crates = len(crates) % groups_total
+else:
+	overflow_crates = 0
+
+print(f"Crates per group: {crates_per_group}", file=sys.stderr)
+
+# Check each crate
+for i in range(0, crates_per_group + overflow_crates):
+	crate = crates_per_group * target_group + i
+
+	print(f"Checking {crates[crate]}", file=sys.stderr)
+
+	res = subprocess.run(["cargo", "check", "--locked", "-p", crates[crate]])
+
+	if res.returncode != 0:
+		sys.exit(1)

substrate/scripts/ci/gitlab/check-each-crate.sh

@@ -1,54 +0,0 @@
-#!/usr/bin/env bash
-
-## A script that checks each workspace crate individually.
-## It's relevant to check workspace crates individually because otherwise their compilation problems
-## due to feature misconfigurations won't be caught, as exemplified by
-## https://github.com/paritytech/substrate/issues/12705
-
-set -Eeu -o pipefail
-shopt -s inherit_errexit
-
-set -vx
-
-target_group="$1"
-groups_total="$2"
-
-readarray -t workspace_crates < <(\
-  cargo tree --workspace --depth 0 --prefix none |
-  awk '{ if (length($1) == 0 || substr($1, 1, 1) == "[") { skip } else { print $1 } }' |
-  sort |
-  uniq
-)
-
-crates_total=${#workspace_crates[*]}
-if [ "$crates_total" -lt 1 ]; then
-  >&2 echo "No crates detected for $PWD"
-  exit 1
-fi
-
-if [ "$crates_total" -lt "$groups_total" ]; then
-  # `crates_total / groups_total` would result in 0, so round it up to 1
-  crates_per_group=1
-else
-  # We add `crates_total % groups_total > 0` (which evaluates to 1 in case there's a remainder for
-  # `crates_total % groups_total`) to round UP `crates_total / groups_total` 's
-  # potentially-fractional result to the nearest integer. Doing that ensures that we'll not miss any
-  # crate in case `crates_total / groups_total` would normally result in a fractional number, since
-  # in those cases Bash would round DOWN the result to the nearest integer. For example, if
-  # `crates_total = 5` and `groups_total = 2`, then `crates_total / groups_total` would round down
-  # to 2; since the loop below would then step by 2, we'd miss the 5th crate.
-  crates_per_group=$(( (crates_total / groups_total) + (crates_total % groups_total > 0) ))
-fi
-
-group=1
-for ((i=0; i < crates_total; i += crates_per_group)); do
-  if [ $group -eq "$target_group" ]; then
-    crates_in_group=("${workspace_crates[@]:$i:$crates_per_group}")
-    echo "crates in the group: ${crates_in_group[*]}" >/dev/null # >/dev/null due to "set -x"
-    for crate in "${crates_in_group[@]}"; do
-      cargo check --locked --release -p "$crate"
-    done
-    break
-  fi
-  group=$(( group + 1 ))
-done

substrate/scripts/ci/gitlab/pipeline/test.yml

@@ -411,7 +411,7 @@ cargo-check-each-crate:
     CI_JOB_NAME:                   cargo-check-each-crate
   script:
     - rusty-cachier snapshot create
-    - time ./scripts/ci/gitlab/check-each-crate.sh "$CI_NODE_INDEX" "$CI_NODE_TOTAL"
+    - PYTHONUNBUFFERED=x time ./scripts/ci/gitlab/check-each-crate.py "$CI_NODE_INDEX" "$CI_NODE_TOTAL"
     # need to update cache only from one job
     - if [ "$CI_NODE_INDEX" == 1 ]; then rusty-cachier cache upload; fi
   parallel: 2
@@ -449,6 +449,7 @@ cargo-check-each-crate-macos:
   script:
     # TODO: enable rusty-cachier once it supports Mac
     # TODO: use parallel jobs, as per cargo-check-each-crate, once more Mac runners are available
-    - time ./scripts/ci/gitlab/check-each-crate.sh 1 1
+    # - time ./scripts/ci/gitlab/check-each-crate.py 1 1
+    - time cargo check --workspace --locked
   tags:
     - osx