Zombienet add tests (#1321)

* changes to read json spec in test binary

* add zombienet tests

* fmt

* use {{COL_IMAGE}} and clean config

* add comment and use relay image from env

* use test-parachain image from pr

* fix warns

* fix warns

* fmt

* typo

* fix ci to use zombienet image

* fix spawn nodes for test

* reorg test

* add within to test

* remove check for full node collators is up

* add tests for pov, mirate solo to para, sync blocks

* bump zombienet image

* add job dep with artifacts

* add sleep for test

* fix after merge

* fmt

* bump zombienet version

* changes from clap

* use base/shared params

* fmt

* debug ci

* add upgrade test

* update js test for debug

* less debug in test

* print assertion

* fix upgrade test

* Collator key only needed if we run as collator

* [Fix] Benchmark build artifact folder creation (#1518)

* Trivial networking changes for Substrate PR #11940 (#1486)

* Trivial networking changes for Substrate PR https://github.com/paritytech/substrate/pull/11940

* Apply formatting rules

* update lockfile for {"polkadot", "substrate"}

Co-authored-by: parity-processbot <>

* bump zombienet version

* update network def for test

* typo

Co-authored-by: Sebastian Kunert <skunert49@gmail.com>
Co-authored-by: Roman Useinov <roman.useinov@gmail.com>
Co-authored-by: Nazar Mokrynskyi <nazar@mokrynskyi.com>

cumulus/zombienet_tests/0001-sync_blocks_from_tip_without_connected_collator.feature

@@ -1,4 +1,4 @@
-Description: Small Network test
+Description: Sync blocks from tip without connected collator test
 Network: ./0001-sync_blocks_from_tip_without_connected_collator.toml
 Creds: config
 
@@ -7,11 +7,9 @@ alice: is up
 bob: is up
 charlie: is up
 dave: is up
-ferdie: is up
-eve: is up
 
 alice: parachain 2000 is registered within 225 seconds
 alice: parachain 2000 block height is at least 10 within 250 seconds
 
 ferdie: reports block height is at least 12 within 250 seconds
-eve: reports block height is at least 12 within 250 seconds
+eve: reports block height is at least 12 within 250 seconds

cumulus/zombienet_tests/0001-sync_blocks_from_tip_without_connected_collator.toml

@@ -1,5 +1,5 @@
 [relaychain]
-default_image = "docker.io/paritypr/polkadot-debug:master"
+default_image = "{{RELAY_IMAGE}}"
 default_command = "polkadot"
 default_args = [ "-lparachain=debug" ]
 
@@ -11,9 +11,7 @@ chain = "rococo-local"
 
   [[relaychain.nodes]]
   name = "bob"
-  image = "docker.io/paritypr/polkadot-debug:5236-0.9.18-c55660e9-be16bd72"
   validator = true
-  args = ["--database=paritydb-experimental"]
 
 [[parachains]]
 id = 2000
@@ -23,30 +21,30 @@ cumulus_based = true
   [[parachains.collators]]
   name = "charlie"
   validator = true
-  image = "docker.io/parity/polkadot-collator:latest"
-  command = "test-collator"
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"
   args = ["-lparachain=debug"]
 
   # run dave as parachain full node
   [[parachains.collators]]
   name = "dave"
   validator = false
-  image = "docker.io/parity/polkadot-collator:latest"
-  command = "test-collator"
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"
   args = ["-lparachain=debug"]
 
   # run eve as parachain full node that is only connected to dave
   [[parachains.collators]]
   name = "eve"
   validator = false
-  image = "docker.io/parity/polkadot-collator:latest"
-  command = "test-collator"
-  args = ["--reserved-only", "--reserved-nodes {{'dave'|zombie}}"]
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"
+  args = ["--reserved-only", "--reserved-nodes {{'dave'|zombie('multiAddress')}}"]
 
   # run ferdie as parachain full node that is only connected to dave
   [[parachains.collators]]
   name = "ferdie"
   validator = false
-  image = "docker.io/parity/polkadot-collator:latest"
-  command = "test-collator"
-  args = ["--reserved-only", "--reserved-nodes {{'dave'|zombie}}"]
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"
+  args = ["--reserved-only", "--reserved-nodes {{'dave'|zombie('multiAddress')}}", "--relay-chain-rpc-url {{'alice'|zombie('wsUri')}}"]

cumulus/zombienet_tests/0002-pov_recovery.feature

@@ -1,4 +1,4 @@
-Description: Small Network test
+Description: PoV recovery test
 Network: ./0002-pov_recovery.toml
 Creds: config
 

cumulus/zombienet_tests/0002-pov_recovery.toml

@@ -1,5 +1,5 @@
 [relaychain]
-default_image = "docker.io/paritypr/polkadot-debug:master"
+default_image = "{{RELAY_IMAGE}}"
 default_command = "polkadot"
 default_args = [ "-lparachain=debug" ]
 
@@ -11,9 +11,7 @@ chain = "rococo-local"
 
   [[relaychain.nodes]]
   name = "bob"
-  image = "docker.io/paritypr/polkadot-debug:5236-0.9.18-c55660e9-be16bd72"
   validator = true
-  args = ["--database=paritydb-experimental"]
 
 [[parachains]]
 id = 2000
@@ -23,8 +21,8 @@ cumulus_based = true
   [[parachains.collators]]
   name = "charlie"
   validator = true
-  image = "docker.io/parity/polkadot-collator:latest"
-  command = "test-collator"
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"
   args = ["-lparachain=debug"]
 
   # Run dave as parachain collator and eve as parachain full node
@@ -33,14 +31,14 @@ cumulus_based = true
   [[parachains.collators]]
   name = "dave"
   validator = true
-  image = "docker.io/parity/polkadot-collator:latest"
-  command = "test-collator"
-  args = ["--reserved-only", "--reserved-nodes {{'charlie'|zombie}}", "--use-null-consensus", "--disable-block-announcements"]
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"
+  args = ["--reserved-only", "--reserved-nodes {{'charlie'|zombie('multiAddress')}}", "--use-null-consensus", "--disable-block-announcements"]
 
   # run eve as parachain full node that is only connected to dave
   [[parachains.collators]]
   name = "eve"
   validator = false
-  image = "docker.io/parity/polkadot-collator:latest"
-  command = "test-collator"
-  args = ["--reserved-only", "--reserved-nodes {{'charlie'|zombie}}", "--use-null-consensus", "--disable-block-announcements"]
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"
+  args = ["--reserved-only", "--reserved-nodes {{'charlie'|zombie('multiAddress')}}", "--use-null-consensus", "--disable-block-announcements"]

cumulus/zombienet_tests/0003-full_node_catching_up.feature

@@ -1,8 +1,7 @@
-Description: Small Network test
-Network: ./0003-test_full_node_catching_up.toml
+Description: Full node catching up test
+Network: ./0003-full_node_catching_up.toml
 Creds: config
 
-
 alice: is up
 bob: is up
 charlie: is up

cumulus/zombienet_tests/0003-full_node_catching_up.toml

@@ -1,5 +1,5 @@
 [relaychain]
-default_image = "docker.io/paritypr/polkadot-debug:master"
+default_image = "{{RELAY_IMAGE}}"
 default_command = "polkadot"
 default_args = [ "-lparachain=debug" ]
 
@@ -11,9 +11,7 @@ chain = "rococo-local"
 
   [[relaychain.nodes]]
   name = "bob"
-  image = "docker.io/paritypr/polkadot-debug:5236-0.9.18-c55660e9-be16bd72"
   validator = true
-  args = ["--database=paritydb-experimental"]
 
 [[parachains]]
 id = 2000
@@ -23,22 +21,22 @@ cumulus_based = true
   [[parachains.collators]]
   name = "charlie"
   validator = true
-  image = "docker.io/parity/polkadot-collator:latest"
-  command = "test-collator"
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"
   args = ["-lparachain=debug"]
 
   # run cumulus dave (a parachain full node) and wait for it to sync some blocks
   [[parachains.collators]]
   name = "dave"
   validator = false
-  image = "docker.io/parity/polkadot-collator:latest"
-  command = "test-collator"
-  args = ["--reserved-only", "--reserved-nodes {{'charlie'|zombie}}"]
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"
+  args = ["--reserved-only", "--reserved-nodes {{'charlie'|zombie('multiAddress')}}"]
 
   # run cumulus eve (a parachain full node) and wait for it to sync some blocks
   [[parachains.collators]]
   name = "eve"
   validator = false
-  image = "docker.io/parity/polkadot-collator:latest"
-  command = "test-collator"
-  args = ["--reserved-only", "--reserved-nodes {{'charlie'|zombie}}"]
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"
+  args = ["--reserved-only", "--reserved-nodes {{'charlie'|zombie('multiAddress')}}", "--relay-chain-rpc-url {{'alice'|zombie('wsUri')}}"]

cumulus/zombienet_tests/0004-runtime_upgrade.feature

@@ -0,0 +1,13 @@
+Description: Runtime Upgrade test
+Network: ./0004-runtime_upgrade.toml
+Creds: config
+
+alice: is up
+bob: is up
+charlie: is up
+dave: is up
+alice: parachain 2000 is registered within 225 seconds
+charlie: reports block height is at least 5 within 250 seconds
+charlie: parachain 2000 perform upgrade with /tmp/wasm_binary_spec_version_incremented.rs.compact.compressed.wasm within 200 seconds
+dave: reports block height is at least 20 within 250 seconds
+dave: js-script ./runtime_upgrade.js within 200 seconds

cumulus/zombienet_tests/0004-runtime_upgrade.toml

@@ -0,0 +1,33 @@
+[relaychain]
+default_image = "{{RELAY_IMAGE}}"
+default_command = "polkadot"
+default_args = [ "-lparachain=debug" ]
+
+chain = "rococo-local"
+
+  [[relaychain.nodes]]
+  name = "alice"
+  validator = true
+
+  [[relaychain.nodes]]
+  name = "bob"
+  validator = true
+
+[[parachains]]
+id = 2000
+cumulus_based = true
+
+  # run charlie as parachain collator
+  [[parachains.collators]]
+  name = "charlie"
+  validator = true
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"
+  args = ["-lparachain=debug"]
+
+  # Run dave as parachain full node
+  [[parachains.collators]]
+  name = "dave"
+  validator = false
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"

cumulus/zombienet_tests/0005-migrate_solo_to_para.feature

@@ -0,0 +1,13 @@
+Description: Migrate solo to para
+Network: ./0005-migrate_solo_to_para.toml
+Creds: config
+
+alice: is up
+bob: is up
+dave: is up
+eve: is up
+alice: parachain 2000 is registered within 225 seconds
+alice: reports block height is at least 10 within 250 seconds
+alice: parachain 2000 block height is at least 10 within 250 seconds
+eve: reports block height is 0 within 20 seconds
+dave: js-script ./migrate_solo_to_para.js with "dave,2000-1,eve" within 200 seconds

cumulus/zombienet_tests/0005-migrate_solo_to_para.toml

@@ -0,0 +1,45 @@
+[relaychain]
+default_image = "{{RELAY_IMAGE}}"
+default_command = "polkadot"
+default_args = [ "-lparachain=debug" ]
+
+chain = "rococo-local"
+
+  [[relaychain.nodes]]
+  name = "alice"
+  validator = true
+
+  [[relaychain.nodes]]
+  name = "bob"
+  validator = true
+
+
+[[parachains]]
+id = 2000
+cumulus_based = true
+
+  # run the solo chain (in our case this is also already a parachain, but as it has a different genesis it will not produce any blocks.)
+  [[parachains.collators]]
+  name = "dave"
+  validator = true
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"
+  args = ["-lparachain=debug"]
+
+[[parachains]]
+id = 2000
+cumulus_based = true
+add_to_genesis = false
+register_para = false
+# Set some random value in the genesis state to create a different genesis hash.
+[parachains.genesis.runtime.runtime_genesis_config.sudo]
+key = "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty"
+
+  # run the parachain that will be used to return the header of the solo chain.
+  [[parachains.collators]]
+  name = "eve"
+  validator = true
+  add_to_bootnodes = true
+  image = "{{COL_IMAGE}}"
+  command = "test-parachain"
+  args = ["-lparachain=debug"]

cumulus/zombienet_tests/migrate_solo_to_para.js

@@ -0,0 +1,57 @@
+const assert = require("assert");
+const polkadotApi = require("@polkadot/api");
+const utilCrypto = require("@polkadot/util-crypto");
+const fs = require("fs").promises;
+
+async function connect(apiUrl, types) {
+    const provider = new polkadotApi.WsProvider(apiUrl);
+    const api = new polkadotApi.ApiPromise({ provider, types });
+    await api.isReady;
+    return api;
+}
+
+async function run(nodeName, networkInfo, args) {
+    const [paraNode, partialPath, soloNode ] = args;
+    const {wsUri, userDefinedTypes} = networkInfo.nodesByName[paraNode];
+    const {wsUri: wsUri_solo, userDefinedTypes: userDefinedTypes_solo } = networkInfo.nodesByName[soloNode];
+    const para = await connect(wsUri, userDefinedTypes);
+    const solo = await connect(wsUri_solo, userDefinedTypes_solo);
+
+    await utilCrypto.cryptoWaitReady();
+
+    // account to submit tx
+    const keyring = new polkadotApi.Keyring({ type: "sr25519" });
+    const alice = keyring.addFromUri("//Alice");
+
+    // get genesis to update
+    const filePath = `${networkInfo.tmpDir}/${partialPath}/genesis-state`;
+    const customHeader = await fs.readFile(filePath);
+
+    // get current header
+    await para.tx.testPallet.setCustomValidationHeadData(customHeader.toString()).signAndSend(alice);
+
+    let parachain_best;
+    let count = 0;
+
+    assertParachainBest = async (parachain_best) => {
+        const current = await para.rpc.chain.getHeader();
+        assert.equal(parachain_best.toHuman().number, current.toHuman().number, "parachain should not produce more blocks");
+    }
+
+
+    await new Promise( async (resolve, reject) => {
+        const unsubscribe = await solo.rpc.chain.subscribeNewHeads(async (header) => {
+            console.log(`Solo chain is at block: #${header.number}`);
+            count++;
+            if(count === 2) parachain_best = await para.rpc.chain.getHeader();
+
+            if(count > 4) {
+                unsubscribe();
+                await assertParachainBest(parachain_best);
+                resolve();
+            }
+        });
+    });
+}
+
+module.exports = { run }

cumulus/zombienet_tests/runtime_upgrade.js

@@ -0,0 +1,24 @@
+const assert = require("assert");
+
+async function run(nodeName, networkInfo, args) {
+    const {wsUri, userDefinedTypes} = networkInfo.nodesByName[nodeName];
+    const api = await zombie.connect(wsUri, userDefinedTypes);
+
+    // get blockhash/runtimeVersion at block 1
+    const hashAtBlock1 = await api.rpc.chain.getBlockHash(1);
+    const versionAtBlock1 = await api.rpc.state.getRuntimeVersion(hashAtBlock1.toHuman());
+
+    // get blockhash/runtimeVersion at current head
+    const currentHeader = await api.rpc.chain.getHeader();
+    const hashAtCurrent = await api.rpc.chain.getBlockHash(currentHeader.number.toHuman());
+    const versionAtCurrent = await api.rpc.state.getRuntimeVersion(hashAtCurrent.toHuman());
+
+    const oldVersionIncremented = parseInt(versionAtBlock1.specVersion.toHuman(),10) + 1;
+    console.log("current", versionAtCurrent.specVersion.toHuman());
+    console.log("oldVersionIncremented", oldVersionIncremented);
+
+    // 2 == 2
+    assert.equal( oldVersionIncremented, versionAtCurrent.specVersion.toHuman(), "Running version should be the incremented version");
+}
+
+module.exports = { run }