zombienet: warp-sync integration test added (#12675)

* zombienet: warp-sync integration test added

* spelling

* Readme corrected

* dir name updated

* Check second phase of warp sync

* zombienet pipeline enable + naive test network

* zombienet stage added

* paritypr/substrate-debug image added for zombienet testing

* debugs added

* debugs added

* buildah problem fixed

* rollback

* runner tag

* test name corrected

* dir renamed (regex problem)

* common code clean up

* common code clean up

* fix

* warp sync test improvements

* full sha used

as short is too short (https://gitlab.parity.io/parity/mirrors/substrate/-/jobs/2051228#L38)

* disable tracing for nodes

* COMMON_USER -> DOCKERIO_USER

* refs reworked

* paritypr/substrate image used

* DOCKERIO -> DOCKER

* generate-ws-db toml cleanup

* improvements

* fix

* raw chain spec used

* zombienet v1.3.18 used

* zombienet: warp sync test enabled

* chain-spec path corrected

* log parsing improved

Checking if log does not container error or verification failed messages

* warp sync test: removed validators

* fix

* review remarks applied

* dir test name changed: 0000_block_building -> 0000-block-building

* transaction finalized test added

* transaction finalized test: error handling improved

* trigger CI job

* trigger CI job

* trigger CI job

* trigger CI job

* Explicitly touch `version.rs` to invalidate the related cache

* zombienet add logs as artifacts

* Revert "Explicitly touch `version.rs` to invalidate the related cache"

This reverts commit 9d00ccfe897a280581156c281961a32665dba6d5.

* file naming changed

Co-authored-by: parity-processbot <>
Co-authored-by: Vladimir Istyufeev <vladimir@parity.io>
Co-authored-by: Javier Viola <javier@parity.io>

substrate/scripts/ci/gitlab/pipeline/build.yml

@@ -86,7 +86,7 @@ build-linux-substrate:
   extends:
     - .collect-artifacts
     - .docker-env
-    - .build-refs
+    - .publish-refs
   variables:
     # this variable gets overriden by "rusty-cachier environment inject", use the value as default
     CARGO_TARGET_DIR:              "$CI_PROJECT_DIR/target"

substrate/scripts/ci/gitlab/pipeline/publish.yml

@@ -2,32 +2,33 @@
 # This file is part of .gitlab-ci.yml
 # Here are all jobs that are executed during "publish" stage
 
-.build-push-docker-image:
+.build-push-docker-image-common:
   extends:
-    - .build-refs
     - .kubernetes-env
+  stage:                           publish
   variables:
     CI_IMAGE:                      $BUILDAH_IMAGE
     GIT_STRATEGY:                  none
     DOCKERFILE:                    $PRODUCT.Dockerfile
-    IMAGE_NAME:                    docker.io/parity/$PRODUCT
+    IMAGE_NAME:                    docker.io/$IMAGE_PATH
   before_script:
     - cd ./artifacts/$PRODUCT/
     - VERSION="$(cat ./VERSION)"
     - echo "${PRODUCT} version = ${VERSION}"
     - test -z "${VERSION}" && exit 1
   script:
-    - test "$Docker_Hub_User_Parity" -a "$Docker_Hub_Pass_Parity" ||
+    - test "$DOCKER_USER" -a "$DOCKER_PASS" ||
         ( echo "no docker credentials provided"; exit 1 )
     - buildah bud
         --format=docker
         --build-arg VCS_REF="${CI_COMMIT_SHA}"
         --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+        --build-arg IMAGE_NAME="${IMAGE_PATH}"
         --tag "$IMAGE_NAME:$VERSION"
         --tag "$IMAGE_NAME:latest"
         --file "$DOCKERFILE" .
-    - echo "$Docker_Hub_Pass_Parity" |
-        buildah login --username "$Docker_Hub_User_Parity" --password-stdin docker.io
+    - echo "$DOCKER_PASS" |
+        buildah login --username "$DOCKER_USER" --password-stdin docker.io
     - buildah info
     - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
     - buildah push --format=v2s2 "$IMAGE_NAME:latest"
@@ -38,8 +39,27 @@
     - echo "SUBSTRATE_IMAGE_TAG=${IMAGE_TAG}"   | tee -a ./artifacts/$PRODUCT/build.env
     - cat ./artifacts/$PRODUCT/build.env
 
+.build-push-docker-image:
+  extends:
+    - .publish-refs
+    - .build-push-docker-image-common
+  variables:
+    IMAGE_PATH:                    parity/$PRODUCT
+    DOCKER_USER:                   $Docker_Hub_User_Parity
+    DOCKER_PASS:                   $Docker_Hub_Pass_Parity
+
+
+# publish image to docker.io/paritypr, (e.g. for later use in zombienet testing)
+.build-push-image-temporary:
+  extends:
+    - .build-refs
+    - .build-push-docker-image-common
+  variables:
+    IMAGE_PATH:                    paritypr/$PRODUCT
+    DOCKER_USER:                   $PARITYPR_USER
+    DOCKER_PASS:                   $PARITYPR_PASS
+
 publish-docker-substrate:
-  stage:                           publish
   extends:                         .build-push-docker-image
   needs:
     - job:                         build-linux-substrate
@@ -47,8 +67,21 @@ publish-docker-substrate:
   variables:
     PRODUCT:                       substrate
 
+publish-docker-substrate-temporary:
+  extends:                         .build-push-image-temporary
+  needs:
+    - job:                         build-linux-substrate
+      artifacts:                   true
+  variables:
+    PRODUCT:                       substrate
+  artifacts:
+    reports:
+      # this artifact is used in zombienet-tests job
+      # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
+      dotenv: ./artifacts/$PRODUCT/build.env
+    expire_in:                     24h
+
 publish-docker-subkey:
-  stage:                           publish
   extends:                         .build-push-docker-image
   needs:
     - job:                         build-subkey-linux
@@ -59,7 +92,7 @@ publish-docker-subkey:
 publish-s3-release:
   stage:                           publish
   extends:
-    - .build-refs
+    - .publish-refs
     - .kubernetes-env
   needs:
     - job:                         build-linux-substrate

substrate/scripts/ci/gitlab/pipeline/zombienet.yml

@@ -0,0 +1,53 @@
+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "zombienet" stage
+
+# common settings for all zombienet jobs
+.zombienet-common:
+  before_script:
+    - echo "Zombie-net Tests Config"
+    - echo "${ZOMBIENET_IMAGE}"
+    - echo "${SUBSTRATE_IMAGE_NAME} ${SUBSTRATE_IMAGE_TAG}"
+    - echo "${GH_DIR}"
+    - export DEBUG=zombie,zombie::network-node
+    - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${SUBSTRATE_IMAGE_NAME}:${SUBSTRATE_IMAGE_TAG}
+    - echo "${ZOMBIENET_INTEGRATION_TEST_IMAGE}"
+  stage:                           zombienet
+  image:                           "${ZOMBIENET_IMAGE}"
+  needs:
+    - job:                         publish-docker-substrate-temporary
+  extends:
+    - .kubernetes-env
+    - .zombienet-refs
+  variables:
+    GH_DIR:                        "https://github.com/paritytech/substrate/tree/${CI_COMMIT_SHA}/zombienet"
+    FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: 1
+  artifacts:
+    name:                          "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
+    when:                          always
+    expire_in:                     2 days
+    paths:
+      - ./zombienet-logs
+  after_script:
+    - mkdir -p ./zombienet-logs
+    - cp /tmp/zombie*/logs/* ./zombienet-logs/
+  allow_failure:                   true
+  retry: 2
+  tags:
+    - zombienet-polkadot-integration-test
+
+zombienet-0000-block-building:
+  extends:
+    - .zombienet-common
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}/0000-block-building"
+        --test="block-building.zndsl"
+
+
+zombienet-0001-basic-warp-sync:
+  extends:
+    - .zombienet-common
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}/0001-basic-warp-sync"
+        --test="test-warp-sync.zndsl"