zombienet: warp-sync integration test added (#12675)

* zombienet: warp-sync integration test added * spelling * Readme corrected * dir name updated * Check second phase of warp sync * zombienet pipeline enable + naive test network * zombienet stage added * paritypr/substrate-debug image added for zombienet testing * debugs added * debugs added * buildah problem fixed * rollback * runner tag * test name corrected * dir renamed (regex problem) * common code clean up * common code clean up * fix * warp sync test improvements * full sha used as short is too short (https://gitlab.parity.io/parity/mirrors/substrate/-/jobs/2051228#L38) * disable tracing for nodes * COMMON_USER -> DOCKERIO_USER * refs reworked * paritypr/substrate image used * DOCKERIO -> DOCKER * generate-ws-db toml cleanup * improvements * fix * raw chain spec used * zombienet v1.3.18 used * zombienet: warp sync test enabled * chain-spec path corrected * log parsing improved Checking if log does not container error or verification failed messages * warp sync test: removed validators * fix * review remarks applied * dir test name changed: 0000_block_building -> 0000-block-building * transaction finalized test added * transaction finalized test: error handling improved * trigger CI job * trigger CI job * trigger CI job * trigger CI job * Explicitly touch `version.rs` to invalidate the related cache * zombienet add logs as artifacts * Revert "Explicitly touch `version.rs` to invalidate the related cache" This reverts commit 9d00ccfe897a280581156c281961a32665dba6d5. * file naming changed Co-authored-by: parity-processbot <> Co-authored-by: Vladimir Istyufeev <vladimir@parity.io> Co-authored-by: Javier Viola <javier@parity.io>
2026-05-30 21:01:02 +00:00 · 2022-12-07 23:44:40 +01:00
parent 8751f88fc7
commit 39cb3b06cd
14 changed files with 588 additions and 13 deletions
@@ -33,6 +33,7 @@ stages:
  - test
  - build
  - publish
+  - zombienet
  - deploy
  - notify

@@ -52,6 +53,7 @@ variables:
  BUILDAH_IMAGE:                   "quay.io/buildah/stable:v1.27"
  RUSTY_CACHIER_SINGLE_BRANCH:     master
  RUSTY_CACHIER_DONT_OPERATE_ON_MAIN_BRANCH: "true"
+  ZOMBIENET_IMAGE:                 "docker.io/paritytech/zombienet:v1.3.22"

 default:
  retry:
@@ -166,7 +168,7 @@ default:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs

-.build-refs:
+.publish-refs:
  rules:
    - if: $CI_PIPELINE_SOURCE == "pipeline"
      when: never
@@ -175,6 +177,20 @@ default:
    - if: $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1

+.build-refs:
+  # publish-refs + PRs
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "web"
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_REF_NAME == "master"
+    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
+
+.zombienet-refs:
+  extends: .build-refs
+
 .nightly-pipeline:
  rules:
    # this job runs only on nightly pipeline with the mentioned variable, against `master` branch
@@ -224,6 +240,8 @@ include:
  - scripts/ci/gitlab/pipeline/build.yml
  # publish jobs
  - scripts/ci/gitlab/pipeline/publish.yml
+  # zombienet jobs
+  - scripts/ci/gitlab/pipeline/zombienet.yml

 #### stage:                        deploy

@@ -3,10 +3,11 @@ FROM docker.io/library/ubuntu:20.04
 # metadata
 ARG VCS_REF
 ARG BUILD_DATE
+ARG IMAGE_NAME

 LABEL io.parity.image.authors="devops-team@parity.io" \
 	io.parity.image.vendor="Parity Technologies" \
-	io.parity.image.title="parity/subkey" \
+	io.parity.image.title="${IMAGE_NAME}" \
 	io.parity.image.description="Subkey: key generating utility for Substrate." \
 	io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/scripts/ci/docker/subkey.Dockerfile" \
 	io.parity.image.revision="${VCS_REF}" \
@@ -3,10 +3,11 @@ FROM docker.io/library/ubuntu:20.04
 # metadata
 ARG VCS_REF
 ARG BUILD_DATE
+ARG IMAGE_NAME

 LABEL io.parity.image.authors="devops-team@parity.io" \
 	io.parity.image.vendor="Parity Technologies" \
-	io.parity.image.title="parity/substrate" \
+	io.parity.image.title="${IMAGE_NAME}" \
 	io.parity.image.description="Substrate: The platform for blockchain innovators." \
 	io.parity.image.source="https://github.com/paritytech/substrate/blob/${VCS_REF}/scripts/ci/docker/Dockerfile" \
 	io.parity.image.revision="${VCS_REF}" \
@@ -86,7 +86,7 @@ build-linux-substrate:
  extends:
    - .collect-artifacts
    - .docker-env
-    - .build-refs
+    - .publish-refs
  variables:
    # this variable gets overriden by "rusty-cachier environment inject", use the value as default
    CARGO_TARGET_DIR:              "$CI_PROJECT_DIR/target"
@@ -2,32 +2,33 @@
 # This file is part of .gitlab-ci.yml
 # Here are all jobs that are executed during "publish" stage

-.build-push-docker-image:
+.build-push-docker-image-common:
  extends:
-    - .build-refs
    - .kubernetes-env
+  stage:                           publish
  variables:
    CI_IMAGE:                      $BUILDAH_IMAGE
    GIT_STRATEGY:                  none
    DOCKERFILE:                    $PRODUCT.Dockerfile
-    IMAGE_NAME:                    docker.io/parity/$PRODUCT
+    IMAGE_NAME:                    docker.io/$IMAGE_PATH
  before_script:
    - cd ./artifacts/$PRODUCT/
    - VERSION="$(cat ./VERSION)"
    - echo "${PRODUCT} version = ${VERSION}"
    - test -z "${VERSION}" && exit 1
  script:
-    - test "$Docker_Hub_User_Parity" -a "$Docker_Hub_Pass_Parity" ||
+    - test "$DOCKER_USER" -a "$DOCKER_PASS" ||
        ( echo "no docker credentials provided"; exit 1 )
    - buildah bud
        --format=docker
        --build-arg VCS_REF="${CI_COMMIT_SHA}"
        --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
+        --build-arg IMAGE_NAME="${IMAGE_PATH}"
        --tag "$IMAGE_NAME:$VERSION"
        --tag "$IMAGE_NAME:latest"
        --file "$DOCKERFILE" .
-    - echo "$Docker_Hub_Pass_Parity" |
-        buildah login --username "$Docker_Hub_User_Parity" --password-stdin docker.io
+    - echo "$DOCKER_PASS" |
+        buildah login --username "$DOCKER_USER" --password-stdin docker.io
    - buildah info
    - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
    - buildah push --format=v2s2 "$IMAGE_NAME:latest"
@@ -38,8 +39,27 @@
    - echo "SUBSTRATE_IMAGE_TAG=${IMAGE_TAG}"   | tee -a ./artifacts/$PRODUCT/build.env
    - cat ./artifacts/$PRODUCT/build.env

+.build-push-docker-image:
+  extends:
+    - .publish-refs
+    - .build-push-docker-image-common
+  variables:
+    IMAGE_PATH:                    parity/$PRODUCT
+    DOCKER_USER:                   $Docker_Hub_User_Parity
+    DOCKER_PASS:                   $Docker_Hub_Pass_Parity
+
+
+# publish image to docker.io/paritypr, (e.g. for later use in zombienet testing)
+.build-push-image-temporary:
+  extends:
+    - .build-refs
+    - .build-push-docker-image-common
+  variables:
+    IMAGE_PATH:                    paritypr/$PRODUCT
+    DOCKER_USER:                   $PARITYPR_USER
+    DOCKER_PASS:                   $PARITYPR_PASS
+
 publish-docker-substrate:
-  stage:                           publish
  extends:                         .build-push-docker-image
  needs:
    - job:                         build-linux-substrate
@@ -47,8 +67,21 @@ publish-docker-substrate:
  variables:
    PRODUCT:                       substrate

+publish-docker-substrate-temporary:
+  extends:                         .build-push-image-temporary
+  needs:
+    - job:                         build-linux-substrate
+      artifacts:                   true
+  variables:
+    PRODUCT:                       substrate
+  artifacts:
+    reports:
+      # this artifact is used in zombienet-tests job
+      # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
+      dotenv: ./artifacts/$PRODUCT/build.env
+    expire_in:                     24h
+
 publish-docker-subkey:
-  stage:                           publish
  extends:                         .build-push-docker-image
  needs:
    - job:                         build-subkey-linux
@@ -59,7 +92,7 @@ publish-docker-subkey:
 publish-s3-release:
  stage:                           publish
  extends:
-    - .build-refs
+    - .publish-refs
    - .kubernetes-env
  needs:
    - job:                         build-linux-substrate
@@ -0,0 +1,53 @@
+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "zombienet" stage
+
+# common settings for all zombienet jobs
+.zombienet-common:
+  before_script:
+    - echo "Zombie-net Tests Config"
+    - echo "${ZOMBIENET_IMAGE}"
+    - echo "${SUBSTRATE_IMAGE_NAME} ${SUBSTRATE_IMAGE_TAG}"
+    - echo "${GH_DIR}"
+    - export DEBUG=zombie,zombie::network-node
+    - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${SUBSTRATE_IMAGE_NAME}:${SUBSTRATE_IMAGE_TAG}
+    - echo "${ZOMBIENET_INTEGRATION_TEST_IMAGE}"
+  stage:                           zombienet
+  image:                           "${ZOMBIENET_IMAGE}"
+  needs:
+    - job:                         publish-docker-substrate-temporary
+  extends:
+    - .kubernetes-env
+    - .zombienet-refs
+  variables:
+    GH_DIR:                        "https://github.com/paritytech/substrate/tree/${CI_COMMIT_SHA}/zombienet"
+    FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: 1
+  artifacts:
+    name:                          "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
+    when:                          always
+    expire_in:                     2 days
+    paths:
+      - ./zombienet-logs
+  after_script:
+    - mkdir -p ./zombienet-logs
+    - cp /tmp/zombie*/logs/* ./zombienet-logs/
+  allow_failure:                   true
+  retry: 2
+  tags:
+    - zombienet-polkadot-integration-test
+
+zombienet-0000-block-building:
+  extends:
+    - .zombienet-common
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}/0000-block-building"
+        --test="block-building.zndsl"
+
+
+zombienet-0001-basic-warp-sync:
+  extends:
+    - .zombienet-common
+  script:
+    - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh
+        --github-remote-dir="${GH_DIR}/0001-basic-warp-sync"
+        --test="test-warp-sync.zndsl"
@@ -0,0 +1,15 @@
+[settings]
+enable_tracing = false
+
+[relaychain]
+default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}"
+default_command = "substrate"
+chain = "local"
+
+  [[relaychain.nodes]]
+  name = "alice"
+  validator = true
+
+  [[relaychain.nodes]]
+  name = "bob"
+  validator = true
@@ -0,0 +1,20 @@
+Description: Block building
+Network: ./block-building.toml
+Creds: config
+
+alice: is up
+bob: is up
+
+alice: reports node_roles is 4
+bob: reports node_roles is 4
+
+alice: reports peers count is at least 1
+bob: reports peers count is at least 1
+
+alice: reports block height is at least 5 within 20 seconds
+bob: reports block height is at least 5 within 20 seconds
+
+alice: count of log lines containing "error" is 0 within 2 seconds
+bob: count of log lines containing "error" is 0 within 2 seconds
+
+alice: js-script ./transaction-gets-finalized.js within 30 seconds
@@ -0,0 +1,59 @@
+//based on: https://polkadot.js.org/docs/api/examples/promise/transfer-events
+
+const assert = require("assert");
+
+async function run(nodeName, networkInfo, args) {
+  const {wsUri, userDefinedTypes} = networkInfo.nodesByName[nodeName];
+  const api = await zombie.connect(wsUri, userDefinedTypes);
+
+  // Construct the keyring after the API (crypto has an async init)
+  const keyring = new zombie.Keyring({ type: "sr25519" });
+
+  // Add Alice to our keyring with a hard-derivation path (empty phrase, so uses dev)
+  const alice = keyring.addFromUri('//Alice');
+  const bob = '5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty';
+
+  // Create a extrinsic, transferring 10^20 units to Bob
+  const transfer = api.tx.balances.transfer(bob, 10n**20n);
+
+  let transaction_success_event = false;
+  try {
+    await new Promise( async (resolve, reject) => {
+      const unsubscribe = await transfer
+        .signAndSend(alice, { nonce: -1 }, ({ events = [], status }) => {
+          console.log('Transaction status:', status.type);
+
+          if (status.isInBlock) {
+            console.log('Included at block hash', status.asInBlock.toHex());
+            console.log('Events:');
+
+            events.forEach(({ event: { data, method, section }, phase }) => {
+              console.log('\t', phase.toString(), `: ${section}.${method}`, data.toString());
+
+              if (section=="system" && method =="ExtrinsicSuccess") {
+                transaction_success_event = true;
+              }
+            });
+          } else if (status.isFinalized) {
+            console.log('Finalized block hash', status.asFinalized.toHex());
+            unsubscribe();
+            if (transaction_success_event) {
+              resolve();
+            } else {
+              reject("ExtrinsicSuccess has not been seen");
+            }
+          } else if (status.isError) {
+            unsubscribe();
+            reject("Transaction status.isError");
+          }
+
+        });
+    });
+  } catch (error) {
+    assert.fail("Transfer promise failed, error: " + error);
+  }
+
+  assert.ok("test passed");
+}
+
+module.exports = { run }
@@ -0,0 +1,101 @@
+# Test design
+The `warp-sync` test works on predefined database which is stored in the cloud and
+fetched by the test. `alice` and `bob` nodes are spun up using this database snapshot in full node mode.
+
+As `warp-sync` requires at least 3 peers, the test spawns the `charlie` full node which uses the same database snapshot.
+
+The `dave` node executed with `--sync warp` syncs database with the rest of the network.
+
+# How to prepare database
+Database was prepared using the following zombienet file (`generate-warp-sync-database.toml`):
+```
+[relaychain]
+default_image = "docker.io/parity/substrate:master"
+default_command = "substrate"
+
+chain = "gen-db"
+
+chain_spec_path = "chain-spec.json"
+
+  [[relaychain.nodes]]
+  name = "alice"
+  validator = true
+
+  [[relaychain.nodes]]
+  name = "bob"
+  validator = true
+```
+
+The zombienet shall be executed with the following command, and run for some period of time to allow for few grandpa eras.
+```
+./zombienet-linux spawn --dir ./db-test-gen --provider native generate-warp-sync-database.toml
+```
+
+Once the zombienet is stopped, the database snapshot
+(`{alice,bob}/data/chains/local_testnet/db/` dirs) was created using the following
+commands:
+```bash
+mkdir -p db-snapshot/{alice,bob}/data/chains/local_testnet/db/  
+cp -r db-test-gen/alice/data/chains/local_testnet/db/full db-snapshot/alice/data/chains/local_testnet/db/  
+cp -r db-test-gen/bob/data/chains/local_testnet/db/full   db-snapshot/bob/data/chains/local_testnet/db/
+```
+
+The file format should be `tar.gz`. File shall contain `local_testnet` folder and its subfolders, e.g.:
+```
+$ tar tzf chains.tgz | head
+local_testnet/
+local_testnet/db/
+local_testnet/db/full/
+...
+local_testnet/db/full/000469.log
+```
+
+Sample command to prepare archive:
+```
+tar -C db-snapshot/alice/data/chains/ -czf chains.tgz local_testnet
+```
+
+Also refer to: [zombienet#578](https://github.com/paritytech/zombienet/issues/578)
+
+The `raw` chain-spec shall also be saved: `db-test-gen/gen-db-raw.json`.
+
+# Where to upload database
+The access to this [bucket](https://console.cloud.google.com/storage/browser/zombienet-db-snaps/) is required.
+
+Sample public path is: `https://storage.googleapis.com/zombienet-db-snaps/substrate/0001-basic-warp-sync/chains-0bb3f0be2ce41b5615b224215bcc8363aa0416a6.tgz`.
+
+The database file path should be `substrate/XXXX-test-name/file-SHA1SUM.tgz`, where `SHA1SUM` is a `sha1sum` of the file.
+
+# Chain spec
+Chain spec was simply built with:
+```
+substrate build-spec --chain=local > chain-spec.json
+```
+
+Please note that `chain-spec.json` committed into repository is `raw` version produced by `zombienet` during database snapshot generation. Zombienet applies some modifications to plain versions of chain-spec.
+
+# Run the test
+Test can be run with the following command:
+```
+zombienet-linux test --dir db-snapshot --provider native test-warp-sync.zndsl
+```
+
+*NOTE*: currently blocked by: [zombienet#578](https://github.com/paritytech/zombienet/issues/578)
+
+
+# Save some time hack
+Substrate can be patched to reduce the grandpa session period.
+```
+diff --git a/bin/node/runtime/src/constants.rs b/bin/node/runtime/src/constants.rs
+index 23fb13cfb0..89f8646291 100644
+--- a/bin/node/runtime/src/constants.rs
+++ b/bin/node/runtime/src/constants.rs
+@@ -63,7 +63,7 @@ pub mod time {
+ 
+    // NOTE: Currently it is not possible to change the epoch duration after the chain has started.
+    //       Attempting to do so will brick block production.
+-   pub const EPOCH_DURATION_IN_BLOCKS: BlockNumber = 10 * MINUTES;
+   pub const EPOCH_DURATION_IN_BLOCKS: BlockNumber = 1 * MINUTES / 2;
+    pub const EPOCH_DURATION_IN_SLOTS: u64 = {
+        const SLOT_FILL_RATE: f64 = MILLISECS_PER_BLOCK as f64 / SLOT_DURATION as f64
+```
@@ -0,0 +1,17 @@
+# this file is not intended to be executed in CI stage
+[relaychain]
+default_image = "docker.io/parity/substrate:latest"
+default_command = "substrate"
+
+# refer to ./README.md for more details on how to create snapshot and spec
+chain = "gen-db"
+chain_spec_path = "chain-spec.json"
+
+
+  [[relaychain.nodes]]
+  name = "alice"
+  validator = true
+
+  [[relaychain.nodes]]
+  name = "bob"
+  validator = true
@@ -0,0 +1,30 @@
+[settings]
+enable_tracing = false
+
+[relaychain]
+default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}"
+default_command = "substrate"
+
+chain = "gen-db"
+chain_spec_path = "zombienet/0001-basic-warp-sync/chain-spec.json"
+
+  [[relaychain.nodes]]
+  name = "alice"
+  validator = false
+  db_snapshot="https://storage.googleapis.com/zombienet-db-snaps/substrate/0001-basic-warp-sync/chains-0bb3f0be2ce41b5615b224215bcc8363aa0416a6.tgz"
+
+  [[relaychain.nodes]]
+  name = "bob"
+  validator = false
+  db_snapshot="https://storage.googleapis.com/zombienet-db-snaps/substrate/0001-basic-warp-sync/chains-0bb3f0be2ce41b5615b224215bcc8363aa0416a6.tgz"
+
+  #we need at least 3 nodes for warp sync
+  [[relaychain.nodes]]
+  name = "charlie"
+  validator = false
+  db_snapshot="https://storage.googleapis.com/zombienet-db-snaps/substrate/0001-basic-warp-sync/chains-0bb3f0be2ce41b5615b224215bcc8363aa0416a6.tgz"
+
+  [[relaychain.nodes]]
+  name = "dave"
+  validator = false
+  args = ["--sync warp"]
@@ -0,0 +1,35 @@
+Description: Warp sync
+Network: ./test-warp-sync.toml
+Creds: config
+
+alice: is up
+bob: is up
+charlie: is up
+dave: is up
+
+alice: reports node_roles is 1
+bob: reports node_roles is 1
+charlie: reports node_roles is 1
+dave: reports node_roles is 1
+
+alice: reports peers count is at least 3 within 60 seconds
+bob: reports peers count is at least 3 within 60 seconds
+charlie: reports peers count is at least 3 within 60 seconds
+dave: reports peers count is at least 3 within 60 seconds
+
+
+# db snapshot has 12133 blocks
+dave: reports block height is at least 1 within 60 seconds
+dave: reports block height is at least 12132 within 60 seconds
+dave: reports block height is at least 12133 within 60 seconds
+
+alice: reports block height is at least 12133 within 60 seconds
+bob: reports block height is at least 12133 within 60 seconds
+charlie: reports block height is at least 12133 within 60 seconds
+
+dave: log line matches "Warp sync is complete" within 60 seconds
+# workaround for: https://github.com/paritytech/zombienet/issues/580
+dave: count of log lines containing  "Block history download is complete" is 1 within 10 seconds
+
+dave: count of log lines containing "error" is 0 within 10 seconds
+dave: count of log lines containing "verification failed" is 0 within 10 seconds