WIP: fix Simnet trigger (#8493)

* CI: trigger guard [skip ci]

* CI: variables and dotenv [skip ci]

* CI: correct variable for simnet trigger

* CI: correct rules for simnet trigger

* fix linting issue :)

* CI: trigger simnet via API

Co-authored-by: radupopa2010 <radupopa2010@yahoo.com>

substrate/.gitlab-ci.yml

@@ -26,7 +26,6 @@ stages:
   - build
   - publish
   - deploy
-  - flaming-fir
 
 workflow:
   rules:
@@ -84,7 +83,6 @@ default:
     - if: $CI_PIPELINE_SOURCE == "web"
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "tags"
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
 
@@ -95,7 +93,6 @@ default:
     - if: $CI_PIPELINE_SOURCE == "web"
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "tags"
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
 
@@ -114,7 +111,6 @@ default:
       when: never
     - if: $CI_PIPELINE_SOURCE == "web"
     - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "tags"
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
       when: manual
@@ -127,7 +123,6 @@ default:
     - if: $CI_PIPELINE_SOURCE == "web"
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "tags"
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
 
 .nightly-pipeline:                 &nightly-pipeline
@@ -337,7 +332,7 @@ test-linux-stable:                 &test-linux
   script:
     # this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests
     - time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path bin/node/cli/Cargo.toml
-    - time cargo test -p frame-support-test --features=conditional-storage --manifest-path frame/support/test/Cargo.toml
+    - time cargo test -p frame-support-test --features=conditional-storage --manifest-path frame/support/test/Cargo.toml # does not reuse cache 1 min 44 sec
     - SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout
     - sccache -s
 
@@ -348,7 +343,6 @@ unleash-check:
     - if: $CI_PIPELINE_SOURCE == "pipeline"
       when: never
     - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "tags"
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
   script:
     - cargo install cargo-unleash ${CARGO_UNLEASH_INSTALL_PARAMS}
@@ -584,15 +578,16 @@ build-rust-doc:
     - buildah info
     - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
     - buildah push --format=v2s2 "$IMAGE_NAME:latest"
-    # pass artifacts to the trigget-simnet job
-    - echo "VERSION=${VERSION}" > build.env
-    - echo "TRIGGERER=${CI_PROJECT_NAME}" >> build.env
   after_script:
     - buildah logout "$IMAGE_NAME"
+    # pass artifacts to the trigger-simnet job
+    - echo "IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/$PRODUCT/build.env
+    - echo "IMAGE_TAG=${VERSION}" >> ./artifacts/$PRODUCT/build.env
 
 publish-docker-substrate:
   stage:                           publish
   <<:                              *build-push-docker-image
+  <<:                              *build-refs
   needs:
     - job:                         build-linux-substrate
       artifacts:                   true
@@ -603,7 +598,7 @@ publish-docker-substrate:
     reports:
       # this artifact is used in trigger-simnet job
       # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
-      dotenv: artifacts/substrate/build.env
+      dotenv: ./artifacts/substrate/build.env
 
 publish-docker-subkey:
   stage:                           publish
@@ -716,10 +711,19 @@ deploy-prometheus-alerting-rules:
 
 trigger-simnet:
   stage:                           deploy
-  <<:                              *nightly-pipeline
+  image:                           paritytech/tools:latest
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME == "master"
   needs:
     - job:                         publish-docker-substrate
-  trigger:
-    project:                       parity/simnet
-    branch:                        master
-    strategy:                      depend
+  # `build.env` brings here `$IMAGE_NAME` and `$IMAGE_TAG` (`$VERSION` here, 
+  # i.e. `2643-0.8.29-5f689e0a-6b24dc54`).
+  variables:
+    TRGR_PROJECT:                  ${CI_PROJECT_NAME}
+    TRGR_REF:                      ${CI_COMMIT_REF_NAME}
+    # simnet project ID
+    DWNSTRM_ID:                    332
+  script:
+    # API trigger for a simnet job
+    - ./scripts/gitlab/trigger_pipeline.sh

substrate/.maintain/gitlab/trigger_pipeline.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -eu
+# API trigger another project's pipeline
+curl --silent \
+    -X POST \
+    -F "token=${CI_JOB_TOKEN}" \
+    -F "ref=master" \
+    -F "variables[TRGR_PROJECT]=${TRGR_PROJECT}" \
+    -F "variables[TRGR_REF]=${TRGR_REF}" \
+    -F "variables[IMAGE_NAME]=${IMAGE_NAME}" \
+    -F "variables[IMAGE_TAG]=${IMAGE_TAG}" \
+    "https://${CI_SERVER_HOST}/api/v4/projects/${DWNSTRM_ID}/trigger/pipeline" | \
+        tee pipeline
+
+PIPELINE_ID=$(cat pipeline | jq ".id")
+echo "\nWaiting on ${PIPELINE_ID} status..."
+
+# This part polls for the triggered pipeline status, the native
+# `trigger` job does not return this status via API.
+# This is a workaround for a Gitlab bug, waits here until
+# https://gitlab.com/gitlab-org/gitlab/-/issues/326137 gets fixed.
+# The timeout is 360 curls with 8 sec interval, roughly an hour.
+
+function get_status() {
+    curl --silent \
+        --header "PRIVATE-TOKEN: ${PIPELINE_TOKEN}" \
+        "https://${CI_SERVER_HOST}/api/v4/projects/${DWNSTRM_ID}/pipelines/${PIPELINE_ID}" | \
+            jq --raw-output ".status";
+}
+
+for i in $(seq 1 360); do
+    STATUS=$(get_status);
+    echo "Triggered pipeline status is ${STATUS}";
+    if [[ ${STATUS} =~ ^(pending|running|created)$ ]]; then
+        echo "Busy...";
+    elif [[ ${STATUS} =~ ^(failed|canceled|skipped|manual)$ ]]; then
+        exit 1;
+    elif [[ ${STATUS} =~ ^(success)$ ]]; then
+        exit 0;
+    else
+        exit 1;
+    fi
+sleep 8;
+done