zombienet: warp-sync integration test added (#12675)

* zombienet: warp-sync integration test added

* spelling

* Readme corrected

* dir name updated

* Check second phase of warp sync

* zombienet pipeline enable + naive test network

* zombienet stage added

* paritypr/substrate-debug image added for zombienet testing

* debugs added

* debugs added

* buildah problem fixed

* rollback

* runner tag

* test name corrected

* dir renamed (regex problem)

* common code clean up

* common code clean up

* fix

* warp sync test improvements

* full sha used

as short is too short (https://gitlab.parity.io/parity/mirrors/substrate/-/jobs/2051228#L38)

* disable tracing for nodes

* COMMON_USER -> DOCKERIO_USER

* refs reworked

* paritypr/substrate image used

* DOCKERIO -> DOCKER

* generate-ws-db toml cleanup

* improvements

* fix

* raw chain spec used

* zombienet v1.3.18 used

* zombienet: warp sync test enabled

* chain-spec path corrected

* log parsing improved

Checking if log does not container error or verification failed messages

* warp sync test: removed validators

* fix

* review remarks applied

* dir test name changed: 0000_block_building -> 0000-block-building

* transaction finalized test added

* transaction finalized test: error handling improved

* trigger CI job

* trigger CI job

* trigger CI job

* trigger CI job

* Explicitly touch `version.rs` to invalidate the related cache

* zombienet add logs as artifacts

* Revert "Explicitly touch `version.rs` to invalidate the related cache"

This reverts commit 9d00ccfe897a280581156c281961a32665dba6d5.

* file naming changed

Co-authored-by: parity-processbot <>
Co-authored-by: Vladimir Istyufeev <vladimir@parity.io>
Co-authored-by: Javier Viola <javier@parity.io>

substrate/.gitlab-ci.yml

@@ -33,6 +33,7 @@ stages:
   - test
   - build
   - publish
+  - zombienet
   - deploy
   - notify
 
@@ -52,6 +53,7 @@ variables:
   BUILDAH_IMAGE:                   "quay.io/buildah/stable:v1.27"
   RUSTY_CACHIER_SINGLE_BRANCH:     master
   RUSTY_CACHIER_DONT_OPERATE_ON_MAIN_BRANCH: "true"
+  ZOMBIENET_IMAGE:                 "docker.io/paritytech/zombienet:v1.3.22"
 
 default:
   retry:
@@ -166,7 +168,7 @@ default:
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
 
-.build-refs:
+.publish-refs:
   rules:
     - if: $CI_PIPELINE_SOURCE == "pipeline"
       when: never
@@ -175,6 +177,20 @@ default:
     - if: $CI_COMMIT_REF_NAME == "master"
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
 
+.build-refs:
+  # publish-refs + PRs
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "web"
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
+
+.zombienet-refs:
+  extends: .build-refs
+
 .nightly-pipeline:
   rules:
     # this job runs only on nightly pipeline with the mentioned variable, against `master` branch
@@ -224,6 +240,8 @@ include:
   - scripts/ci/gitlab/pipeline/build.yml
   # publish jobs
   - scripts/ci/gitlab/pipeline/publish.yml
+  # zombienet jobs
+  - scripts/ci/gitlab/pipeline/zombienet.yml
 
 #### stage:                        deploy
 

substrate/scripts/ci/docker/subkey.Dockerfile

@@ -3,10 +3,11 @@ FROM docker.io/library/ubuntu:20.04
 # metadata
 ARG VCS_REF
 ARG BUILD_DATE
+ARG IMAGE_NAME
 
 LABEL io.parity.image.authors="devops-team@parity.io" \
 	io.parity.image.vendor="Parity Technologies" \
-	io.parity.image.title="parity/subkey" \
+	io.parity.image.title="${IMAGE_NAME}" \
 	io.parity.image.description="Subkey: key generating utility for Substrate." \
 	io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/scripts/ci/docker/subkey.Dockerfile" \
 	io.parity.image.revision="${VCS_REF}" \

substrate/scripts/ci/docker/substrate.Dockerfile

@@ -3,10 +3,11 @@ FROM docker.io/library/ubuntu:20.04
 # metadata
 ARG VCS_REF
 ARG BUILD_DATE
+ARG IMAGE_NAME
 
 LABEL io.parity.image.authors="devops-team@parity.io" \
 	io.parity.image.vendor="Parity Technologies" \
-	io.parity.image.title="parity/substrate" \
+	io.parity.image.title="${IMAGE_NAME}" \
 	io.parity.image.description="Substrate: The platform for blockchain innovators." \
 	io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/scripts/ci/docker/Dockerfile" \
 	io.parity.image.revision="${VCS_REF}" \

substrate/scripts/ci/gitlab/pipeline/build.yml

@@ -86,7 +86,7 @@ build-linux-substrate:
   extends:
     - .collect-artifacts
     - .docker-env
-    - .build-refs
+    - .publish-refs
   variables:
     # this variable gets overriden by "rusty-cachier environment inject", use the value as default
     CARGO_TARGET_DIR:              "$CI_PROJECT_DIR/target"

substrate/scripts/ci/gitlab/pipeline/publish.yml

@@ -2,32 +2,33 @@
 # This file is part of .gitlab-ci.yml
 # Here are all jobs that are executed during "publish" stage
 
-.build-push-docker-image:
+.build-push-docker-image-common:
   extends:
-    - .build-refs
     - .kubernetes-env
+  stage:                           publish
   variables:
     CI_IMAGE:                      $BUILDAH_IMAGE
     GIT_STRATEGY:                  none
     DOCKERFILE:                    $PRODUCT.Dockerfile
-    IMAGE_NAME:                    docker.io/parity/$PRODUCT
+    IMAGE_NAME:                    docker.io/$IMAGE_PATH
   before_script:
     - cd ./artifacts/$PRODUCT/
     - VERSION="$(cat ./VERSION)"
     - echo "${PRODUCT} version = ${VERSION}"
     - test -z "${VERSION}" && exit 1
   script:
-    - test "$Docker_Hub_User_Parity" -a "$Docker_Hub_Pass_Parity" ||
+    - test "$DOCKER_USER" -a "$DOCKER_PASS" ||
         ( echo "no docker credentials provided"; exit 1 )
     - buildah bud
         --format=docker
         --build-arg VCS_REF="${CI_COMMIT_SHA}"
         --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+        --build-arg IMAGE_NAME="${IMAGE_PATH}"
         --tag "$IMAGE_NAME:$VERSION"
         --tag "$IMAGE_NAME:latest"
         --file "$DOCKERFILE" .
-    - echo "$Docker_Hub_Pass_Parity" |
+    - echo "$DOCKER_PASS" |
-        buildah login --username "$Docker_Hub_User_Parity" --password-stdin docker.io
+        buildah login --username "$DOCKER_USER" --password-stdin docker.io
     - buildah info
     - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
     - buildah push --format=v2s2 "$IMAGE_NAME:latest"
@@ -38,8 +39,27 @@
     - echo "SUBSTRATE_IMAGE_TAG=${IMAGE_TAG}"   | tee -a ./artifacts/$PRODUCT/build.env
     - cat ./artifacts/$PRODUCT/build.env
 
+.build-push-docker-image:
+  extends:
+    - .publish-refs
+    - .build-push-docker-image-common
+  variables:
+    IMAGE_PATH:                    parity/$PRODUCT
+    DOCKER_USER:                   $Docker_Hub_User_Parity
+    DOCKER_PASS:                   $Docker_Hub_Pass_Parity
+
+
+# publish image to docker.io/paritypr, (e.g. for later use in zombienet testing)
+.build-push-image-temporary:
+  extends:
+    - .build-refs
+    - .build-push-docker-image-common
+  variables:
+    IMAGE_PATH:                    paritypr/$PRODUCT
+    DOCKER_USER:                   $PARITYPR_USER
+    DOCKER_PASS:                   $PARITYPR_PASS
+
 publish-docker-substrate:
-  stage:                           publish
   extends:                         .build-push-docker-image
   needs:
     - job:                         build-linux-substrate
@@ -47,8 +67,21 @@ publish-docker-substrate:
   variables:
     PRODUCT:                       substrate
 
+publish-docker-substrate-temporary:
+  extends:                         .build-push-image-temporary
+  needs:
+    - job:                         build-linux-substrate
+      artifacts:                   true
+  variables:
+    PRODUCT:                       substrate
+  artifacts:
+    reports:
+      # this artifact is used in zombienet-tests job
+      # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
+      dotenv: ./artifacts/$PRODUCT/build.env
+    expire_in:                     24h
+
 publish-docker-subkey:
-  stage:                           publish
   extends:                         .build-push-docker-image
   needs:
     - job:                         build-subkey-linux
@@ -59,7 +92,7 @@ publish-docker-subkey:
 publish-s3-release:
   stage:                           publish
   extends:
-    - .build-refs
+    - .publish-refs
     - .kubernetes-env
   needs:
     - job:                         build-linux-substrate

substrate/scripts/ci/gitlab/pipeline/zombienet.yml

@@ -0,0 +1,53 @@
+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "zombienet" stage
+
+# common settings for all zombienet jobs
+.zombienet-common:
+  before_script:
+    - echo "Zombie-net Tests Config"
+    - echo "${ZOMBIENET_IMAGE}"
+    - echo "${SUBSTRATE_IMAGE_NAME} ${SUBSTRATE_IMAGE_TAG}"
+    - echo "${GH_DIR}"
+    - export DEBUG=zombie,zombie::network-node
+    - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${SUBSTRATE_IMAGE_NAME}:${SUBSTRATE_IMAGE_TAG}
+    - echo "${ZOMBIENET_INTEGRATION_TEST_IMAGE}"
+  stage:                           zombienet
+  image:                           "${ZOMBIENET_IMAGE}"
+  needs:
+    - job:                         publish-docker-substrate-temporary
+  extends:
+    - .kubernetes-env
+    - .zombienet-refs
+  variables:
+    GH_DIR:                        "https://github.com/paritytech/substrate/tree/${CI_COMMIT_SHA}/zombienet"
+    FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: 1
+  artifacts:
+    name:                          "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
+    when:                          always
+    expire_in:                     2 days
+    paths:
+      - ./zombienet-logs
+  after_script:
+    - mkdir -p ./zombienet-logs
+    - cp /tmp/zombie*/logs/* ./zombienet-logs/
+  allow_failure:                   true
+  retry: 2
+  tags:
+    - zombienet-polkadot-integration-test
+
+zombienet-0000-block-building:
+  extends:
+    - .zombienet-common
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}/0000-block-building"
+        --test="block-building.zndsl"
+
+
+zombienet-0001-basic-warp-sync:
+  extends:
+    - .zombienet-common
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}/0001-basic-warp-sync"
+        --test="test-warp-sync.zndsl"

substrate/zombienet/0000-block-building/block-building.toml

@@ -0,0 +1,15 @@
+[settings]
+enable_tracing = false
+
+[relaychain]
+default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}"
+default_command = "substrate"
+chain = "local"
+
+  [[relaychain.nodes]]
+  name = "alice"
+  validator = true
+
+  [[relaychain.nodes]]
+  name = "bob"
+  validator = true

substrate/zombienet/0000-block-building/block-building.zndsl

@@ -0,0 +1,20 @@
+Description: Block building
+Network: ./block-building.toml
+Creds: config
+
+alice: is up
+bob: is up
+
+alice: reports node_roles is 4
+bob: reports node_roles is 4
+
+alice: reports peers count is at least 1
+bob: reports peers count is at least 1
+
+alice: reports block height is at least 5 within 20 seconds
+bob: reports block height is at least 5 within 20 seconds
+
+alice: count of log lines containing "error" is 0 within 2 seconds
+bob: count of log lines containing "error" is 0 within 2 seconds
+
+alice: js-script ./transaction-gets-finalized.js within 30 seconds

substrate/zombienet/0000-block-building/transaction-gets-finalized.js

@@ -0,0 +1,59 @@
+//based on: https://polkadot.js.org/docs/api/examples/promise/transfer-events
+
+const assert = require("assert");
+
+async function run(nodeName, networkInfo, args) {
+  const {wsUri, userDefinedTypes} = networkInfo.nodesByName[nodeName];
+  const api = await zombie.connect(wsUri, userDefinedTypes);
+
+  // Construct the keyring after the API (crypto has an async init)
+  const keyring = new zombie.Keyring({ type: "sr25519" });
+
+  // Add Alice to our keyring with a hard-derivation path (empty phrase, so uses dev)
+  const alice = keyring.addFromUri('//Alice');
+  const bob = '5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty';
+
+  // Create a extrinsic, transferring 10^20 units to Bob
+  const transfer = api.tx.balances.transfer(bob, 10n**20n);
+
+  let transaction_success_event = false;
+  try {
+    await new Promise( async (resolve, reject) => {
+      const unsubscribe = await transfer
+        .signAndSend(alice, { nonce: -1 }, ({ events = [], status }) => {
+          console.log('Transaction status:', status.type);
+
+          if (status.isInBlock) {
+            console.log('Included at block hash', status.asInBlock.toHex());
+            console.log('Events:');
+
+            events.forEach(({ event: { data, method, section }, phase }) => {
+              console.log('\t', phase.toString(), `: ${section}.${method}`, data.toString());
+
+              if (section=="system" && method =="ExtrinsicSuccess") {
+                transaction_success_event = true;
+              }
+            });
+          } else if (status.isFinalized) {
+            console.log('Finalized block hash', status.asFinalized.toHex());
+            unsubscribe();
+            if (transaction_success_event) {
+              resolve();
+            } else {
+              reject("ExtrinsicSuccess has not been seen");
+            }
+          } else if (status.isError) {
+            unsubscribe();
+            reject("Transaction status.isError");
+          }
+
+        });
+    });
+  } catch (error) {
+    assert.fail("Transfer promise failed, error: " + error);
+  }
+
+  assert.ok("test passed");
+}
+
+module.exports = { run }

substrate/zombienet/0001-basic-warp-sync/README.md

@@ -0,0 +1,101 @@
+# Test design
+The `warp-sync` test works on predefined database which is stored in the cloud and
+fetched by the test. `alice` and `bob` nodes are spun up using this database snapshot in full node mode.
+
+As `warp-sync` requires at least 3 peers, the test spawns the `charlie` full node which uses the same database snapshot.
+
+The `dave` node executed with `--sync warp` syncs database with the rest of the network.
+
+# How to prepare database
+Database was prepared using the following zombienet file (`generate-warp-sync-database.toml`):
+```
+[relaychain]
+default_image = "docker.io/parity/substrate:master"
+default_command = "substrate"
+
+chain = "gen-db"
+
+chain_spec_path = "chain-spec.json"
+
+  [[relaychain.nodes]]
+  name = "alice"
+  validator = true
+
+  [[relaychain.nodes]]
+  name = "bob"
+  validator = true
+```
+
+The zombienet shall be executed with the following command, and run for some period of time to allow for few grandpa eras.
+```
+./zombienet-linux spawn --dir ./db-test-gen --provider native generate-warp-sync-database.toml
+```
+
+Once the zombienet is stopped, the database snapshot
+(`{alice,bob}/data/chains/local_testnet/db/` dirs) was created using the following
+commands:
+```bash
+mkdir -p db-snapshot/{alice,bob}/data/chains/local_testnet/db/  
+cp -r db-test-gen/alice/data/chains/local_testnet/db/full db-snapshot/alice/data/chains/local_testnet/db/  
+cp -r db-test-gen/bob/data/chains/local_testnet/db/full   db-snapshot/bob/data/chains/local_testnet/db/
+```
+
+The file format should be `tar.gz`. File shall contain `local_testnet` folder and its subfolders, e.g.:
+```
+$ tar tzf chains.tgz | head
+local_testnet/
+local_testnet/db/
+local_testnet/db/full/
+...
+local_testnet/db/full/000469.log
+```
+
+Sample command to prepare archive:
+```
+tar -C db-snapshot/alice/data/chains/ -czf chains.tgz local_testnet
+```
+
+Also refer to: [zombienet#578](https://github.com/paritytech/zombienet/issues/578)
+
+The `raw` chain-spec shall also be saved: `db-test-gen/gen-db-raw.json`.
+
+# Where to upload database
+The access to this [bucket](https://console.cloud.google.com/storage/browser/zombienet-db-snaps/) is required.
+
+Sample public path is: `https://storage.googleapis.com/zombienet-db-snaps/substrate/0001-basic-warp-sync/chains-0bb3f0be2ce41b5615b224215bcc8363aa0416a6.tgz`.
+
+The database file path should be `substrate/XXXX-test-name/file-SHA1SUM.tgz`, where `SHA1SUM` is a `sha1sum` of the file.
+
+# Chain spec
+Chain spec was simply built with:
+```
+substrate build-spec --chain=local > chain-spec.json
+```
+
+Please note that `chain-spec.json` committed into repository is `raw` version produced by `zombienet` during database snapshot generation. Zombienet applies some modifications to plain versions of chain-spec.
+
+# Run the test
+Test can be run with the following command:
+```
+zombienet-linux test --dir db-snapshot --provider native test-warp-sync.zndsl
+```
+
+*NOTE*: currently blocked by: [zombienet#578](https://github.com/paritytech/zombienet/issues/578)
+
+
+# Save some time hack
+Substrate can be patched to reduce the grandpa session period.
+```
+diff --git a/bin/node/runtime/src/constants.rs b/bin/node/runtime/src/constants.rs
+index 23fb13cfb0..89f8646291 100644
+--- a/bin/node/runtime/src/constants.rs
++++ b/bin/node/runtime/src/constants.rs
+@@ -63,7 +63,7 @@ pub mod time {
+ 
+    // NOTE: Currently it is not possible to change the epoch duration after the chain has started.
+    //       Attempting to do so will brick block production.
+-   pub const EPOCH_DURATION_IN_BLOCKS: BlockNumber = 10 * MINUTES;
++   pub const EPOCH_DURATION_IN_BLOCKS: BlockNumber = 1 * MINUTES / 2;
+    pub const EPOCH_DURATION_IN_SLOTS: u64 = {
+        const SLOT_FILL_RATE: f64 = MILLISECS_PER_BLOCK as f64 / SLOT_DURATION as f64
+```

substrate/zombienet/0001-basic-warp-sync/generate-warp-sync-database.toml

@@ -0,0 +1,17 @@
+# this file is not intended to be executed in CI stage
+[relaychain]
+default_image = "docker.io/parity/substrate:latest"
+default_command = "substrate"
+
+# refer to ./README.md for more details on how to create snapshot and spec
+chain = "gen-db"
+chain_spec_path = "chain-spec.json"
+
+
+  [[relaychain.nodes]]
+  name = "alice"
+  validator = true
+
+  [[relaychain.nodes]]
+  name = "bob"
+  validator = true

substrate/zombienet/0001-basic-warp-sync/test-warp-sync.toml

@@ -0,0 +1,30 @@
+[settings]
+enable_tracing = false
+
+[relaychain]
+default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}"
+default_command = "substrate"
+
+chain = "gen-db"
+chain_spec_path = "zombienet/0001-basic-warp-sync/chain-spec.json"
+
+  [[relaychain.nodes]]
+  name = "alice"
+  validator = false
+  db_snapshot="https://storage.googleapis.com/zombienet-db-snaps/substrate/0001-basic-warp-sync/chains-0bb3f0be2ce41b5615b224215bcc8363aa0416a6.tgz"
+
+  [[relaychain.nodes]]
+  name = "bob"
+  validator = false
+  db_snapshot="https://storage.googleapis.com/zombienet-db-snaps/substrate/0001-basic-warp-sync/chains-0bb3f0be2ce41b5615b224215bcc8363aa0416a6.tgz"
+
+  #we need at least 3 nodes for warp sync
+  [[relaychain.nodes]]
+  name = "charlie"
+  validator = false
+  db_snapshot="https://storage.googleapis.com/zombienet-db-snaps/substrate/0001-basic-warp-sync/chains-0bb3f0be2ce41b5615b224215bcc8363aa0416a6.tgz"
+
+  [[relaychain.nodes]]
+  name = "dave"
+  validator = false
+  args = ["--sync warp"]

substrate/zombienet/0001-basic-warp-sync/test-warp-sync.zndsl

@@ -0,0 +1,35 @@
+Description: Warp sync
+Network: ./test-warp-sync.toml
+Creds: config
+
+alice: is up
+bob: is up
+charlie: is up
+dave: is up
+
+alice: reports node_roles is 1
+bob: reports node_roles is 1
+charlie: reports node_roles is 1
+dave: reports node_roles is 1
+
+alice: reports peers count is at least 3 within 60 seconds
+bob: reports peers count is at least 3 within 60 seconds
+charlie: reports peers count is at least 3 within 60 seconds
+dave: reports peers count is at least 3 within 60 seconds
+
+
+# db snapshot has 12133 blocks
+dave: reports block height is at least 1 within 60 seconds
+dave: reports block height is at least 12132 within 60 seconds
+dave: reports block height is at least 12133 within 60 seconds
+
+alice: reports block height is at least 12133 within 60 seconds
+bob: reports block height is at least 12133 within 60 seconds
+charlie: reports block height is at least 12133 within 60 seconds
+
+dave: log line matches "Warp sync is complete" within 60 seconds
+# workaround for: https://github.com/paritytech/zombienet/issues/580
+dave: count of log lines containing  "Block history download is complete" is 1 within 10 seconds
+
+dave: count of log lines containing "error" is 0 within 10 seconds
+dave: count of log lines containing "verification failed" is 0 within 10 seconds