Simnet v8 move tests (#3647)

* Update to simnet v8

* rm duplicate

* add tags also

* revertme: add simnet image version for testing CI

* add info regarding the image names and values

* fix lint error

* one more try to fix the lint

* hardcode simnet version and revertme

* hardcode simnet

* more hardcodding

* I wonder how may silly mistakes I can make

* simnet image name needs to be hardcoded

* Update .gitlab-ci.yml

Co-authored-by: Denis Pisarev <denis.pisarev@parity.io>

* Update .gitlab-ci.yml

Co-authored-by: Denis Pisarev <denis.pisarev@parity.io>

* remove revertme changes

* rm unused script

Co-authored-by: Denis Pisarev <denis.pisarev@parity.io>

polkadot/.gitlab-ci.yml

@@ -293,14 +293,6 @@ build-rustdoc:
     - buildah push --format=v2s2 "$IMAGE_NAME:$EXTRATAG"
   after_script:
     - buildah logout "$IMAGE_NAME"
-    # pass artifacts to the trigger-simnet job
-    - echo "IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/build.env
-    - echo "IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/build.env
-  artifacts:
-    reports:
-      # this artifact is used in trigger-simnet job
-      # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
-      dotenv: ./artifacts/build.env
 
 publish-polkadot-image:
   stage:                           build
@@ -327,6 +319,15 @@ publish-polkadot-image:
   needs:
     - job:                         test-build-linux-stable
       artifacts:                   true
+  after_script:
+    # pass artifacts to the simnet-tests job
+    # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
+    - echo "PARACHAINS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/parachains.env
+    - echo "PARACHAINS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/parachains.env
+  artifacts:
+    reports:
+      # this artifact is used in simnet-tests job
+      dotenv: ./artifacts/parachains.env
 
 publish-adder-collator-image:
   # service image for Simnet
@@ -347,11 +348,12 @@ publish-adder-collator-image:
       artifacts:                   true
   after_script:
     - buildah logout "$IMAGE_NAME"
-    # pass artifacts to the trigger-simnet job
+    # pass artifacts to the simnet-tests job
-    - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" > ./artifacts/collator.env
+    - echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env
+    - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env
   artifacts:
     reports:
-      # this artifact is used in trigger-simnet job
+      # this artifact is used in simnet-tests job
       dotenv: ./artifacts/collator.env
 
 .update_weights:                   &update-weights
@@ -479,9 +481,9 @@ deploy-polkasync-kusama:
   allow_failure:                   true
   trigger:                         "parity/infrastructure/parity-testnet"
 
-trigger-simnet:
+simnet-tests:
   stage:                           deploy
-  image:                           paritytech/tools:latest
+  image:                           "docker.io/paritytech/simnet:${SIMNET_REF}"
   <<:                              *kubernetes-env
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
@@ -491,18 +493,26 @@ trigger-simnet:
   needs:
     - job:                         publish-polkadot-image
     - job:                         publish-adder-collator-image
-  # `build.env` brings here `$IMAGE_NAME` and `$IMAGE_TAG` (`$EXTRATAG` here,
+  # `parachains.env` brings here `$PARACHAINS_IMAGE_NAME` and `$PARACHAINS_IMAGE_TAG` 
-  # i.e. `2643-0.8.29-5f689e0a-6b24dc54`).
+  # (`$EXTRATAG` here, # i.e. `2643-0.8.29-5f689e0a-6b24dc54`).
-  # `collator.env` bears adder-collator unique build tag. In non-triggered builds it
+  # `collator.env` brings here `$COLLATOR_IMAGE_NAME` and  `$COLLATOR_IMAGE_TAG` 
-  # can be called by `master` tag.
+  # For local tests with docker `$PARACHAINS_IMAGE_TAG` and `$COLLATOR_IMAGE_TAG` 
-  # Simnet uses an image published on PRs with this exact version for triggered runs
+  # can be replaced with `master` tag.
-  # on commits. And parity/rococo:rococo-v1 for the runs not launched by this job.
+
-  variables:
+  # SIMNET_REF is a gitlab variable
-    TRGR_PROJECT:                  ${CI_PROJECT_NAME}
+  before_script:
-    TRGR_REF:                      ${CI_COMMIT_REF_NAME}
+    - echo "Simnet Tests Config
-    # Simnet project ID
+        ${SIMNET_IMAGE_NAME} ${SIMNET_REF}
-    DWNSTRM_ID:                    332
+        ${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}
+        ${COLLATOR_IMAGE_NAME} ${COLLATOR_IMAGE_TAG}"
   script:
-    # API trigger for a Simnet job, argument value is set in the project variables
+    - /home/nonroot/simnet/gurke/scripts/run-test-environment-manager.sh
-    - ./scripts/gitlab/trigger_pipeline.sh --simnet-version=${SIMNET_REF}
+        --github-remote-dir="https://github.com/paritytech/polkadot/tree/master/simnet_tests"
+        --config="simnet_tests/configs/simple_rococo_testnet.toml"
+        --image="${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}"
+        --image-2="${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}"
+        --image-simnet="docker.io/paritytech/simnet:${SIMNET_REF}"
   allow_failure:                   true
+  retry: 2
+  tags:
+    - parity-simnet

polkadot/scripts/gitlab/trigger_pipeline.sh

@@ -1,202 +0,0 @@
-#!/bin/bash
-
-set -eou pipefail
-
-# This script is to trigger Simnet pipeline.
-# See help article for more details.
-
-SCRIPT_NAME="$0"
-SCRIPT_PATH=$(dirname "$0")               # relative
-SCRIPT_PATH=$(cd "${SCRIPT_PATH}" && pwd) # absolutized and normalized
-SIMNET_VERSION=""
-
-function usage {
-  cat << EOF
-This script is to trigger Simnet pipeline.
-It's designed to be launched locally and from CI.
-The required argumants for both cases are listed below.
-
-Usage: ${SCRIPT_NAME} OPTION
-
-OPTIONS
-
-    -h, --help                  Print this help message.
-
-  Mandatory in both cases:
-
-    -s, --simnet-version        Simnet version to trigger.
-                                E.g.: v4
-
-    -u, --upstream-project      Triggering project.
-                                E.g.: polkadot
-
-    -r, --upstream-ref          The branch or tag name for which project is built.
-                                E.g.: master
-
-    -d, --downstream-id         Downstream project's ID to trigger.
-                                E.g.: 332 (simnet project id)
-
-    -n, --image-name            Name of image to test.
-                                E.g.: docker.io/paritypr/synth-wave
-
-    -i, --image-tag             Tag of the image to test.
-                                E.g.: master
-
-    -c, --collator-image-tag    Tag of collator image. Image name is hardcoded.
-                                E.g.: master
-
-  Required for local launch:
-
-    -g, --ci-server-fqdn        FQDN of your gitlab server.
-                                E.g.: gitlab.parity.io
-
-    -t, --trigger-token         Gitlab trigger token. This must be defined in
-                                project -> settings -> CI/CD -> Pipeline triggers
-                                Defaults to CI_JOB_TOKEN
-                                https://stackoverflow.com/questions/42746634/gitlab-trigger-api-returns-404
-
-    -a, --access-token          Gitlab  peronal access token or it defaults to
-                                PIPELINE_TOKEN (gitlab variable)
-                                https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html
-
-EXAMPLES
-  ${SCRIPT_NAME} -s v4
-  ${SCRIPT_NAME} --simnet-version=v4
-
-  Local test example. You need to set the 2 vars before running: TR_TOKEN and PERS_TOKEN
-  ${SCRIPT_NAME} --simnet-version=v4 \\
-                 --upstream-project=polkadot \\
-                 --upstream-ref=master \\
-                 --image-name=docker.io/paritypr/synth-wave \\
-                 --image-tag=master \\
-                 --collator-image-tag=master \\
-                 --ci-server-fqdn=gitlab.parity.io \\
-                 --downstream-id=332 \\
-                 --trigger-token="\${TR_TOKEN}" \\
-                 --access-token="\${PERS_TOKEN}"
-EOF
-}
-
-function main {
-  # Main entry point for the script.
-  parse_args "$@"
-  check_args
-  trigger_pipeline
-  check_pipeline
-  poll_pipeline
-}
-
-function parse_args {
-  # shellcheck disable=SC2214
-  while getopts c:u:r:i:n:g:t:r:a:s:h-: OPT; do
-    # support long options: https://stackoverflow.com/a/28466267/519360
-    if [ "${OPT}" = "-" ]; then   # long option: reformulate OPT and OPTARG
-      OPT="${OPTARG%%=*}"         # extract long option name
-      OPTARG="${OPTARG#$OPT}"     # extract long option argument (may be empty)
-      OPTARG="${OPTARG#=}"        # if long option argument, remove assigning `=`
-    fi
-    case "${OPT}" in
-      h | help )                usage     ; exit 0 ;;
-      s | simnet-version )      needs_arg ; SIMNET_VERSION="${OPTARG}" ;;
-      u | upstream-project )    needs_arg ; TRGR_PROJECT="${OPTARG}" ;;
-      r | upstream-ref )        needs_arg ; TRGR_REF="${OPTARG}" ;;
-      n | image-name )          needs_arg ; IMAGE_NAME="${OPTARG}" ;;
-      i | image-tag )           needs_arg ; IMAGE_TAG="${OPTARG}" ;;
-      c | collator-image-tag )  needs_arg ; COLLATOR_IMAGE_TAG="${OPTARG}" ;;
-      g | ci-server-fqdn )      needs_arg ; CI_SERVER_HOST="${OPTARG}" ;;
-      d | downstream-id )       needs_arg ; DWNSTRM_ID="${OPTARG}" ;;
-      t | trigger-token )       needs_arg ; CI_JOB_TOKEN="${OPTARG}" ;;
-      a | access-token )        needs_arg ; PIPELINE_TOKEN="${OPTARG}" ;;
-      ??* )                     log DIE "Illegal option --${OPT}" ;;  # bad long option
-      ? )                       exit 2 ;;  # bad short option (error reported via getopts)
-    esac
-  done
-  shift $((OPTIND-1)) # remove parsed options and args from $@ list
-
-}
-
-function check_args {
-  if [[ -z "${SIMNET_VERSION}" ]] ; then
-    log DIE "Must specify value for mandatory argument -s,--simnet-version
-
-$(usage)"
-  fi
-}
-
-function needs_arg {
-  if [ -z "${OPTARG}" ]; then
-    log DIE "No arg for --${OPT} option"
-  fi
-}
-
-function trigger_pipeline {
-  # API trigger another project's pipeline.
-  log INFO "Triggering Simnet pipeline."
-
-  curl --silent \
-      -X POST \
-      -F "token=${CI_JOB_TOKEN}" \
-      -F "ref=${SIMNET_VERSION}" \
-      -F "variables[TRGR_PROJECT]=${TRGR_PROJECT}" \
-      -F "variables[TRGR_REF]=${TRGR_REF}" \
-      -F "variables[IMAGE_NAME]=${IMAGE_NAME}" \
-      -F "variables[IMAGE_TAG]=${IMAGE_TAG}" \
-      -F "variables[COLLATOR_IMAGE_TAG]=${COLLATOR_IMAGE_TAG}" \
-      "https://${CI_SERVER_HOST}/api/v4/projects/${DWNSTRM_ID}/trigger/pipeline" | \
-          tee pipeline;
-}
-
-function check_pipeline {
-  PIPELINE_ID=$(jq ".id" pipeline)
-  PIPELINE_URL=$(jq ".web_url" pipeline)
-  echo
-  log INFO "Simnet pipeline ${PIPELINE_URL} was successfully triggered."
-  log INFO "Now we're polling it to obtain the distinguished status."
-}
-
-function poll_pipeline {
-  # This is a workaround for a Gitlab bug, waits here until
-  # https://gitlab.com/gitlab-org/gitlab/-/issues/326137 gets fixed.
-  # The timeout is 360 curls with 8 sec interval, roughly an hour.
-  log INFO "Waiting on ${PIPELINE_ID} status..."
-
-# shellcheck disable=SC2034
-  for i in {1..360}; do
-      STATUS=$(get_status);
-      log INFO "Triggered pipeline status is ${STATUS}";
-      if [[ ${STATUS} =~ ^(pending|running|created)$ ]]; then
-          echo;
-      elif [[ ${STATUS} =~ ^(failed|canceled|skipped|manual)$ ]]; then
-          log DIE "Something's broken in: ${PIPELINE_URL}";
-      elif [[ ${STATUS} =~ ^(success)$ ]]; then
-          log INFO "Look how green it is: ${PIPELINE_URL}"
-          exit 0
-      else
-          log DIE "Something else has happened in ${PIPELINE_URL}"
-      fi
-  sleep 8;
-  done
-}
-
-function get_status() {
-    curl --silent \
-        --header "PRIVATE-TOKEN: ${PIPELINE_TOKEN}" \
-        "https://${CI_SERVER_HOST}/api/v4/projects/${DWNSTRM_ID}/pipelines/${PIPELINE_ID}" | \
-            jq --raw-output ".status";
-}
-
-function log {
-  local lvl msg fmt
-  lvl=$1 msg=$2
-  fmt='+%Y-%m-%d %H:%M:%S'
-  lg_date=$(date "${fmt}")
-  if [[ "${lvl}" = "DIE" ]] ; then
-    lvl="ERROR"
-    echo  "${lg_date} - ${lvl} - ${msg}"
-    exit 1
-  else
-    echo "${lg_date} - ${lvl} - ${msg}"
-  fi
-}
-
-main "$@"