More improvements for the crate publishing pipeline (#13153)

* more improvements for the crate publishing pipeline

* move default definitions to the publish-crates script

* add script to check the crate publishing pipeline at the start

* fix yaml references

* move more variables to .crates-publishing-pipeline

* separate .crates-publishing-pipeline from .crates-publishing-variables

* clean up redundant and unused code

substrate/.gitlab-ci.yml

@@ -229,19 +229,21 @@ variables:
     # this job runs only on nightly pipeline with the mentioned variable, against `master` branch
     - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly"
 
-.crate-publishing-pipeline:
-  rules:
-    - if: $CI_COMMIT_REF_NAME != "master"
-      when: never
+.crates-publishing-variables:
+  variables:
+    CRATESIO_CRATES_OWNER:         parity-crate-owner
+    REPO:                          substrate
+    REPO_OWNER:                    paritytech
 
-.scheduled-crate-publishing-pipeline:
+.crates-publishing-pipeline:
+  extends:                         .crates-publishing-variables
   rules:
-    - !reference [.crate-publishing-pipeline, rules]
-    - if: $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "automatic-crate-publishing"
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $CI_COMMIT_REF_NAME == "master" && $PIPELINE == "automatic-crate-publishing"
 
 .crates-publishing-template:
-  stage:                           test
-  extends:                         .docker-env
+  extends:
+    - .docker-env
+    - .crates-publishing-variables
   # collect artifacts even on failure so that we know how the crates were generated (they'll be
   # generated to the artifacts folder according to SPUB_TMP further down)
   artifacts:
@@ -251,12 +253,7 @@ variables:
     paths:
       - artifacts/
   variables:
-    CRATESIO_API: https://crates.io/api
-    CRATESIO_CRATES_OWNER: parity-crate-owner
-    GH_API: https://api.github.com
-    REPO: substrate
-    REPO_OWNER: paritytech
-    SPUB_TMP: artifacts
+    SPUB_TMP:                      artifacts
 
 #### stage:                       .pre
 
@@ -274,6 +271,18 @@ skip-if-draft:
      - ./scripts/ci/gitlab/skip_if_draft.sh
   allow_failure:                   true
 
+check-crates-publishing-pipeline:
+  stage:                           .pre
+  extends:
+    - .kubernetes-env
+    - .crates-publishing-pipeline
+  script:
+    - git clone
+        --depth 1
+        --branch "$RELENG_SCRIPTS_BRANCH"
+        https://github.com/paritytech/releng-scripts.git
+    - ONLY_CHECK_PIPELINE=true ./releng-scripts/publish-crates
+
 include:
   # check jobs
   - scripts/ci/gitlab/pipeline/check.yml

substrate/scripts/ci/gitlab/pipeline/publish.yml

@@ -184,19 +184,19 @@ publish-draft-release:
 
 .publish-crates-template:
   stage:                           publish
-  extends:                         .crates-publishing-template
+  extends:
+    - .crates-publishing-template
+    - .crates-publishing-pipeline
   # We don't want multiple jobs racing to publish crates as it's redundant and they might overwrite
   # the releases of one another. Use resource_group to ensure that at most one instance of this job
   # is running at any given time.
   resource_group:                  crates-publishing
-  variables:
-    # crates.io rate limits crates publishing by 1 per minute, so a delay needs to be inserted
-    # slightly higher than that after publishing each crate. The value is specified in seconds.
-    SPUB_AFTER_PUBLISH_DELAY: 64
-  # We might have to publish lots of crates at a time. Given the 1 minute delay introduced above and
-  # taking into account the 202 (as of Dec 07, 2022) publishable Substrate crates, that would equate
-  # to roughly 202 minutes of delay, or 3h and 22 minutes. As such, the job needs to have a much
-  # higher timeout than average.
+  # crates.io currently rate limits crate publishing at 1 per minute:
+  # https://github.com/paritytech/release-engineering/issues/123#issuecomment-1335509748
+  # Taking into account the 202 (as of Dec 07, 2022) publishable Substrate crates, in the worst
+  # case, due to the rate limits alone, we'd have to wait through at least 202 minutes of delay.
+  # Taking into account also the verification steps and extra synchronization delays after
+  # publishing the crate, the job needs to have a much higher timeout than average.
   timeout:                         9h
   # A custom publishing environment is used for us to be able to set up protected secrets
   # specifically for it
@@ -211,15 +211,9 @@ publish-draft-release:
     - rusty-cachier cache upload
 
 publish-crates:
-  extends:
-    - .publish-crates-template
-    - .scheduled-crate-publishing-pipeline
-  needs:
-    - job:                         publish-crates-locally
-      artifacts:                   false
+  extends:                         .publish-crates-template
 
 publish-crates-manual:
   extends:                         .publish-crates-template
-  rules:                           !reference [.crate-publishing-pipeline, rules]
   when:                            manual
-  allow_failure:                   true
+  interruptible:                   false

substrate/scripts/ci/gitlab/pipeline/test.yml

@@ -14,7 +14,7 @@ find-fail-ci-phrase:
   script:
     - set +e
     - rg --line-number --hidden --type rust --glob '!{.git,target}' "$ASSERT_REGEX" .; exit_status=$?
-    - if [ $exit_status -eq 0 ]; then 
+    - if [ $exit_status -eq 0 ]; then
         echo "$ASSERT_REGEX was found, exiting with 1";
         exit 1;
       else
@@ -417,9 +417,14 @@ cargo-check-each-crate:
   parallel: 2
 
 publish-crates-locally:
+  stage:                           test
   extends:
     - .test-refs
     - .crates-publishing-template
+  # When lots of crates are taken into account (for example on master where all crates are tested)
+  # the job might take a long time, as evidenced by:
+  # https://gitlab.parity.io/parity/mirrors/substrate/-/jobs/2269364
+  timeout:                         4h
   script:
     - rusty-cachier snapshot create
     - git clone