feat: Rebrand Polkadot/Substrate references to PezkuwiChain

This commit systematically rebrands various references from Parity Technologies'
Polkadot/Substrate ecosystem to PezkuwiChain within the kurdistan-sdk.

Key changes include:
- Updated external repository URLs (zombienet-sdk, parity-db, parity-scale-codec, wasm-instrument) to point to pezkuwichain forks.
- Modified internal documentation and code comments to reflect PezkuwiChain naming and structure.
- Replaced direct references to  with  or specific paths within the  for XCM, Pezkuwi, and other modules.
- Cleaned up deprecated  issue and PR references in various  and  files, particularly in  and  modules.
- Adjusted image and logo URLs in documentation to point to PezkuwiChain assets.
- Removed or rephrased comments related to external Polkadot/Substrate PRs and issues.

This is a significant step towards fully customizing the SDK for the PezkuwiChain ecosystem.

pezcumulus/zombienet/zombienet-sdk/Cargo.toml

@@ -0,0 +1,38 @@
+[package]
+name = "pezcumulus-zombienet-sdk-tests"
+version = "0.1.0"
+description = "Zomebienet-sdk tests for pezcumulus."
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+publish = false
+
+[dependencies]
+anyhow = { workspace = true }
+env_logger = { workspace = true }
+log = { workspace = true }
+pezkuwi-primitives = { workspace = true, default-features = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+tokio = { workspace = true, features = ["rt-multi-thread"] }
+futures = { workspace = true }
+zombienet-sdk = { workspace = true }
+zombienet-orchestrator = { workspace = true }
+zombienet-configuration = { workspace = true }
+pezcumulus-zombienet-sdk-helpers = { workspace = true }
+pezsp-statement-store = { workspace = true, default-features = true }
+pezsc-statement-store = { workspace = true, default-features = true }
+pezsp-keyring = { workspace = true, default-features = true }
+pezsp-core = { workspace = true, default-features = true }
+codec = { workspace = true }
+rstest = { workspace = true }
+
+[features]
+zombie-ci = []
+runtime-benchmarks = [
+  "pezcumulus-zombienet-sdk-helpers/runtime-benchmarks",
+  "pezkuwi-primitives/runtime-benchmarks",
+  "pezsc-statement-store/runtime-benchmarks",
+  "pezsp-keyring/runtime-benchmarks",
+  "pezsp-statement-store/runtime-benchmarks",
+]

pezcumulus/zombienet/zombienet-sdk/README.md

@@ -0,0 +1,22 @@
+# How to run locally
+
+As a prerequisite, the `test-teyrchain` and `pezkuwi` binaries need to be installed or available under `$PATH`.
+
+The following commands need to be run from the repository root:
+```
+# install test-teyrchain
+cargo install --path ./pezcumulus/test/service --locked
+# install pezkuwi
+cargo install --path ./pezkuwi --locked
+```
+
+The following command launches the tests:
+
+```
+ZOMBIE_PROVIDER=native cargo test --release -p pezcumulus-zombienet-sdk-tests --features zombie-ci
+```
+
+You can also just use `run.sh` that setups everything for you and runs the tests.
+
+In addition, you can specify a base directory with `ZOMBIENET_SDK_BASE_DIR=/my/dir/of/choice`. All chain files and logs
+will be placed in that directory.

pezcumulus/zombienet/zombienet-sdk/run.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -e
+
+cargo build --release -p pezcumulus-test-service --bin test-teyrchain -p pezkuwi --bin pezkuwi-prepare-worker --bin pezkuwi-execute-worker --bin pezkuwi -p pezkuwi-teyrchain-bin --bin pezkuwi-teyrchain
+
+RELEASE_DIR=$(dirname "$(cargo locate-project --workspace --message-format plain)")/target/release
+
+export PATH=$RELEASE_DIR:$PATH
+ZOMBIE_PROVIDER=native cargo test --release -p pezcumulus-zombienet-sdk-tests --features zombie-ci "$@"

pezcumulus/zombienet/zombienet-sdk/src/lib.rs

@@ -0,0 +1,2 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0

pezcumulus/zombienet/zombienet-sdk/tests/tests.rs

@@ -0,0 +1,8 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+#[cfg(feature = "zombie-ci")]
+mod zombie_ci;
+
+#[cfg(feature = "zombie-ci")]
+mod utils;

pezcumulus/zombienet/zombienet-sdk/tests/utils.rs

@@ -0,0 +1,22 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+use zombienet_sdk::{LocalFileSystem, Network, NetworkConfig};
+
+pub const BEST_BLOCK_METRIC: &str = "block_height{status=\"best\"}";
+
+pub async fn initialize_network(
+	config: NetworkConfig,
+) -> Result<Network<LocalFileSystem>, anyhow::Error> {
+	// Spawn network
+	let spawn_fn = zombienet_sdk::environment::get_spawn_fn();
+	let network = spawn_fn(config).await?;
+
+	// Do not terminate network after the test is finished.
+	// This is needed for CI to get logs from k8s.
+	// Network shall be terminated from CI after logs are downloaded.
+	// NOTE! For local execution (native provider) below call has no effect.
+	network.detach().await;
+
+	Ok(network)
+}

pezcumulus/zombienet/zombienet-sdk/tests/zombie_ci/bootnodes.rs

@@ -0,0 +1,132 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+use anyhow::anyhow;
+use tokio::time::Duration;
+
+use crate::utils::initialize_network;
+
+use cumulus_zombienet_sdk_helpers::wait_for_nth_session_change;
+use zombienet_orchestrator::network::node::LogLineCountOptions;
+use zombienet_sdk::{
+	subxt::{OnlineClient, PolkadotConfig},
+	NetworkConfig, NetworkConfigBuilder,
+};
+
+async fn build_network_config() -> Result<NetworkConfig, anyhow::Error> {
+	// images are not relevant for `native`, but we leave it here in case we use `k8s` some day
+	let images = zombienet_sdk::environment::get_images_from_env();
+	log::info!("Using images: {images:?}");
+
+	NetworkConfigBuilder::new()
+		.with_relaychain(|r| {
+			let r = r
+				.with_chain("pezkuwichain-local")
+				.with_default_command("pezkuwi")
+				.with_default_image(images.polkadot.as_str())
+				// Not strictly necessary for the test, but to keep it consistent
+				// with the teyrchain part we also pass `--no-mdns` to the relaychain.
+				.with_default_args(vec!["-lteyrchain=debug".into(), "--no-mdns".into()])
+				// Have to set a `with_node` outside of the loop below, so that `r` has the right
+				// type.
+				.with_node(|node| node.with_name("validator-0"));
+			(1..3).fold(r, |acc, i| acc.with_node(|node| node.with_name(&format!("validator-{i}"))))
+		})
+		.with_teyrchain(|p| {
+			p.with_id(1000)
+				.with_default_command("pezkuwi-teyrchain")
+				.with_default_image(images.pezcumulus.as_str())
+				.with_chain("asset-hub-pezkuwichain-local")
+				// Do not put bootnodes into the chain-spec nor command line arguments.
+				.without_default_bootnodes()
+				// Disable mdns to rely only on DHT bootnode discovery mechanism.
+				.with_default_args(vec![
+					"-lbootnodes=trace".into(),
+					"--no-mdns".into(),
+					"--discover-local".into(),
+					"--".into(),
+					"--no-mdns".into(),
+				])
+				.with_collator(|n| n.with_name("alpha"))
+				.with_collator(|n| n.with_name("beta"))
+		})
+		.with_global_settings(|global_settings| match std::env::var("ZOMBIENET_SDK_BASE_DIR") {
+			Ok(val) => global_settings.with_base_dir(val),
+			_ => global_settings,
+		})
+		.build()
+		.map_err(|e| {
+			let errs = e.into_iter().map(|e| e.to_string()).collect::<Vec<_>>().join(" ");
+			anyhow!("config errs: {errs}")
+		})
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn dht_bootnodes_test() -> Result<(), anyhow::Error> {
+	let _ = env_logger::try_init_from_env(
+		env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
+	);
+
+	log::info!("Spawning network");
+	let config = build_network_config().await?;
+	let mut network = initialize_network(config).await?;
+
+	let relay_node = network.get_node("validator-0")?;
+	let relay_client: OnlineClient<PolkadotConfig> = relay_node.wait_client().await?;
+
+	let alpha = network.get_node("alpha")?;
+
+	log::info!("Make sure the collators connect to each other");
+	alpha
+		.wait_metric_with_timeout("bizinikiwi_sync_peers", |count| count == 1.0, 300u64)
+		.await?;
+
+	// In case of initial failure (alpha was first to start) the discovery is retried in 30 seconds,
+	// so timeout in double that time.
+	let log_line_options = LogLineCountOptions::new(|n| n == 1, Duration::from_secs(60), false);
+
+	log::info!("Make sure the DHT bootnode discovery was successful");
+	let result = alpha
+		.wait_log_line_count_with_timeout(
+			".* Teyrchain bootnode discovery on the relay chain DHT succeeded",
+			false,
+			log_line_options.clone(),
+		)
+		.await?;
+	assert!(result.success());
+
+	log::info!(
+		"First two collators successfully connected via DHT bootnodes. \
+		 Waiting for two full sessions (~3min) before spawning a third collator."
+	);
+
+	// Wait for two full sessions (three session changes) and spawn a new collator to check the
+	// bootnode is also advertised with the new epoch key (republishing works).
+	let mut blocks_sub = relay_client.blocks().subscribe_all().await?;
+	wait_for_nth_session_change(&mut blocks_sub, 3).await?;
+	drop(blocks_sub);
+
+	log::info!("Spawning the third collator.");
+	network.add_collator("gamma", Default::default(), 1000).await?;
+
+	let gamma = network.get_node("gamma")?;
+
+	log::info!("Make sure the new collator has connected to the existing collators");
+	gamma
+		.wait_metric_with_timeout("bizinikiwi_sync_peers", |count| count == 2.0, 300u64)
+		.await?;
+
+	log::info!("Make sure the DHT bootnode discovery was successful");
+	let result = gamma
+		.wait_log_line_count_with_timeout(
+			".* Teyrchain bootnode discovery on the relay chain DHT succeeded",
+			false,
+			log_line_options,
+		)
+		.await?;
+	assert!(result.success());
+
+	log::info!("Test finished successfully.");
+
+	Ok(())
+}

pezcumulus/zombienet/zombienet-sdk/tests/zombie_ci/elastic_scaling/mod.rs

@@ -0,0 +1,8 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+mod multiple_blocks_per_slot;
+mod pov_recovery;
+mod slot_based_authoring;
+mod slot_based_rp_offset;
+mod upgrade_to_3_cores;

pezcumulus/zombienet/zombienet-sdk/tests/zombie_ci/elastic_scaling/multiple_blocks_per_slot.rs

@@ -0,0 +1,123 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+use anyhow::anyhow;
+
+use crate::utils::initialize_network;
+
+use cumulus_zombienet_sdk_helpers::{assert_finality_lag, assert_para_throughput, assign_cores};
+use pezkuwi_primitives::Id as ParaId;
+use serde_json::json;
+use zombienet_sdk::{
+	subxt::{OnlineClient, PolkadotConfig},
+	NetworkConfig, NetworkConfigBuilder,
+};
+
+const PARA_ID: u32 = 2400;
+
+/// This test spawns a teyrchain network.
+/// Initially, one core is assigned. We expect the teyrchain to produce 1 block per relay.
+/// As we increase the number of cores via `assign_core`, we expect the block pace to increase too.
+/// **Note:** The runtime in use here has 6s slot duration, so multiple blocks will be produced per
+/// slot.
+#[tokio::test(flavor = "multi_thread")]
+async fn elastic_scaling_multiple_blocks_per_slot() -> Result<(), anyhow::Error> {
+	let _ = env_logger::try_init_from_env(
+		env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
+	);
+
+	log::info!("Spawning network");
+	let config = build_network_config().await?;
+	let network = initialize_network(config).await?;
+
+	let relay_node = network.get_node("validator-0")?;
+	let para_node_elastic = network.get_node("collator-1")?;
+
+	let relay_client: OnlineClient<PolkadotConfig> = relay_node.wait_client().await?;
+	assert_para_throughput(
+		&relay_client,
+		10,
+		[(ParaId::from(PARA_ID), 3..18)].into_iter().collect(),
+	)
+	.await?;
+	assert_finality_lag(&para_node_elastic.wait_client().await?, 5).await?;
+
+	assign_cores(relay_node, PARA_ID, vec![2, 3]).await?;
+
+	assert_para_throughput(
+		&relay_client,
+		15,
+		[(ParaId::from(PARA_ID), 39..46)].into_iter().collect(),
+	)
+	.await?;
+	assert_finality_lag(&para_node_elastic.wait_client().await?, 20).await?;
+
+	assign_cores(relay_node, PARA_ID, vec![4, 5, 6]).await?;
+
+	assert_para_throughput(
+		&relay_client,
+		10,
+		[(ParaId::from(PARA_ID), 52..61)].into_iter().collect(),
+	)
+	.await?;
+	assert_finality_lag(&para_node_elastic.wait_client().await?, 30).await?;
+	log::info!("Test finished successfully");
+	Ok(())
+}
+
+async fn build_network_config() -> Result<NetworkConfig, anyhow::Error> {
+	// images are not relevant for `native`, but we leave it here in case we use `k8s` some day
+	let images = zombienet_sdk::environment::get_images_from_env();
+	log::info!("Using images: {images:?}");
+
+	NetworkConfigBuilder::new()
+		.with_relaychain(|r| {
+			let r = r
+				.with_chain("pezkuwichain-local")
+				.with_default_command("pezkuwi")
+				.with_default_image(images.polkadot.as_str())
+				.with_default_args(vec![("-lteyrchain=trace").into()])
+				.with_default_resources(|resources| {
+					// These settings are applicable only for `k8s` provider.
+					// Leaving them in case we switch to `k8s` some day.
+					resources.with_request_cpu(4).with_request_memory("4G")
+				})
+				.with_genesis_overrides(json!({
+					"configuration": {
+						"config": {
+							"scheduler_params": {
+								"num_cores": 7,
+								"max_validators_per_core": 1
+							}
+						}
+					}
+				}))
+				// Have to set a `with_node` outside of the loop below, so that `r` has the right
+				// type.
+				.with_node(|node| node.with_name("validator-0"));
+			(1..9).fold(r, |acc, i| acc.with_node(|node| node.with_name(&format!("validator-{i}"))))
+		})
+		.with_teyrchain(|p| {
+			p.with_id(PARA_ID)
+				.with_default_command("test-teyrchain")
+				.with_default_image(images.pezcumulus.as_str())
+				.with_chain("elastic-scaling-multi-block-slot")
+				.with_default_args(vec![
+					("--authoring").into(),
+					("slot-based").into(),
+					("-lteyrchain=trace,aura=debug").into(),
+				])
+				.with_collator(|n| n.with_name("collator-0"))
+				.with_collator(|n| n.with_name("collator-1"))
+				.with_collator(|n| n.with_name("collator-2"))
+		})
+		.with_global_settings(|global_settings| match std::env::var("ZOMBIENET_SDK_BASE_DIR") {
+			Ok(val) => global_settings.with_base_dir(val),
+			_ => global_settings,
+		})
+		.build()
+		.map_err(|e| {
+			let errs = e.into_iter().map(|e| e.to_string()).collect::<Vec<_>>().join(" ");
+			anyhow!("config errs: {errs}")
+		})
+}

pezcumulus/zombienet/zombienet-sdk/tests/zombie_ci/elastic_scaling/pov_recovery.rs

@@ -0,0 +1,218 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+use anyhow::anyhow;
+use std::{sync::Arc, time::Duration};
+
+use crate::utils::{initialize_network, BEST_BLOCK_METRIC};
+
+use cumulus_zombienet_sdk_helpers::{
+	assert_para_is_registered, assert_para_throughput, assign_cores,
+};
+use pezkuwi_primitives::Id as ParaId;
+use serde_json::json;
+use zombienet_orchestrator::network::node::LogLineCountOptions;
+use zombienet_sdk::{
+	subxt::{OnlineClient, PolkadotConfig},
+	NetworkConfig, NetworkConfigBuilder, RegistrationStrategy,
+};
+
+const PARA_ID: u32 = 2100;
+
+/// This test checks if teyrchain node is importing blocks using PoV recovery even
+/// after more cores have been assigned for the teyrchain.
+#[tokio::test(flavor = "multi_thread")]
+async fn elastic_scaling_pov_recovery() -> Result<(), anyhow::Error> {
+	let _ = env_logger::try_init_from_env(
+		env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
+	);
+
+	log::info!("Spawning network with relay chain only");
+	let config = build_network_config().await?;
+	let mut network = initialize_network(config).await?;
+
+	let alice = network.get_node("alice")?;
+	let collator_elastic = network.get_node("collator-elastic")?;
+
+	log::info!("Checking if alice is up");
+	assert!(alice.wait_until_is_up(60u64).await.is_ok());
+
+	log::info!("Checking if collator-elastic is up");
+	assert!(collator_elastic.wait_until_is_up(60u64).await.is_ok());
+
+	assign_cores(alice, PARA_ID, vec![0, 1]).await?;
+
+	log::info!("Waiting 20 blocks to register teyrchain");
+	// Wait 20 blocks and register teyrchain. This part is important for pov-recovery.
+	// We need to make sure that the recovering node is able to see all relay-chain
+	// notifications containing the candidates to recover.
+	assert!(alice
+		.wait_metric_with_timeout(BEST_BLOCK_METRIC, |b| b >= 20.0, 250u64)
+		.await
+		.is_ok());
+
+	log::info!("Registering teyrchain para_id = {PARA_ID}");
+	let relay_client: OnlineClient<PolkadotConfig> = alice.wait_client().await?;
+	network.register_parachain(PARA_ID).await?;
+
+	log::info!("Ensuring teyrchain is registered within 30 blocks");
+	assert_para_is_registered(&relay_client, ParaId::from(PARA_ID), 30).await?;
+
+	log::info!("Ensuring teyrchain making progress");
+	assert_para_throughput(
+		&relay_client,
+		20,
+		[(ParaId::from(PARA_ID), 40..65)].into_iter().collect(),
+	)
+	.await?;
+
+	let collator_elastic = network.get_node("collator-elastic")?;
+
+	log::info!("Checking block production");
+	assert!(collator_elastic
+		.wait_metric_with_timeout(BEST_BLOCK_METRIC, |b| b >= 40.0, 225u64)
+		.await
+		.is_ok());
+
+	// We want to make sure that none of the consensus hook checks fail, even if the chain makes
+	// progress. If below log line occurred 1 or more times then test failed.
+	log::info!("Ensuring none of the consensus hook checks fail at {}", collator_elastic.name());
+	let result = collator_elastic
+		.wait_log_line_count_with_timeout(
+			"set_validation_data inherent needs to be present in every block",
+			false,
+			LogLineCountOptions::no_occurences_within_timeout(Duration::from_secs(10)),
+		)
+		.await?;
+
+	assert!(result.success(), "Consensus hook failed at {}: {:?}", collator_elastic.name(), result);
+
+	// Wait (up to 10 seconds) until pattern occurs more than 35 times
+	let options = LogLineCountOptions {
+		predicate: Arc::new(|n| n > 35),
+		timeout: Duration::from_secs(10),
+		wait_until_timeout_elapses: false,
+	};
+
+	let name = "recovery-target";
+	log::info!("Ensuring blocks are imported using PoV recovery by {name}");
+	let result = network
+		.get_node(name)?
+		.wait_log_line_count_with_timeout(
+			"Importing blocks retrieved using pov_recovery",
+			false,
+			options,
+		)
+		.await?;
+
+	assert!(result.success(), "Failed importing blocks using PoV recovery by {name}: {result:?}");
+
+	log::info!("Test finished successfully");
+	Ok(())
+}
+
+async fn build_network_config() -> Result<NetworkConfig, anyhow::Error> {
+	// images are not relevant for `native`, but we leave it here in case we use `k8s` some day
+	let images = zombienet_sdk::environment::get_images_from_env();
+	log::info!("Using images: {images:?}");
+
+	// Network setup:
+	// - relaychain nodes:
+	// 	 - alice
+	// 	   - validator
+	// 	 - validator[0-3]
+	// 	   - validator
+	// 	   - synchronize only with alice
+	// - teyrchain nodes
+	//   - recovery-target
+	//     - full node
+	//   - collator-elastic
+	//     - collator which is the only one producing blocks
+	NetworkConfigBuilder::new()
+		.with_relaychain(|r| {
+			let r = r
+				.with_chain("pezkuwichain-local")
+				.with_default_command("pezkuwi")
+				.with_default_image(images.polkadot.as_str())
+				.with_default_resources(|resources| {
+					// These settings are applicable only for `k8s` provider.
+					// Leaving them in case we switch to `k8s` some day.
+					resources
+						.with_request_cpu(1)
+						.with_request_memory("2G")
+						.with_limit_cpu(2)
+						.with_limit_memory("4G")
+				})
+				.with_genesis_overrides(json!({
+					"configuration": {
+						"config": {
+							"scheduler_params": {
+								"num_cores": 2,
+								"max_validators_per_core": 1
+							},
+							"approval_voting_params": {
+								"max_approval_coalesce_count": 5
+							}
+						}
+					}
+				}))
+				// Have to set a `with_node` outside of the loop below, so that `r` has the right
+				// type.
+				.with_node(|node| node.with_name("alice").with_args(vec![]));
+
+			(0..4).fold(r, |acc, i| {
+				acc.with_node(|node| {
+					node.with_name(&format!("validator-{i}")).with_args(vec![
+						("-lruntime=debug,teyrchain=trace").into(),
+						("--reserved-only").into(),
+						("--reserved-nodes", "{{ZOMBIE:alice:multiaddr}}").into(),
+					])
+				})
+			})
+		})
+		.with_teyrchain(|p| {
+			p.with_id(PARA_ID)
+				.with_chain("elastic-scaling")
+				.with_registration_strategy(RegistrationStrategy::Manual)
+				.with_default_command("test-teyrchain")
+				.with_default_image(images.pezcumulus.as_str())
+				.with_default_resources(|resources| {
+					// These settings are applicable only for `k8s` provider.
+					// Leaving them in case we switch to `k8s` some day.
+					resources
+						.with_request_cpu(1)
+						.with_request_memory("2G")
+						.with_limit_cpu(2)
+						.with_limit_memory("4G")
+				})
+				.with_collator(|n|
+					n.with_name("recovery-target")
+						.validator(false)
+						.with_args(vec![
+						("-lteyrchain::availability=trace,sync=debug,teyrchain=debug,pezcumulus-pov-recovery=debug,pezcumulus-consensus=debug").into(),
+						("--disable-block-announcements").into(),
+						("--in-peers", "0").into(),
+						("--out-peers", "0").into(),
+						("--").into(),
+						("--reserved-only").into(),
+						("--reserved-nodes", "{{ZOMBIE:alice:multiaddr}}").into()
+					]))
+				.with_collator(|n| n.with_name("collator-elastic")
+					.with_args(vec![
+						("-laura=trace,runtime=info,pezcumulus-consensus=trace,consensus::common=trace,teyrchain::collation-generation=trace,teyrchain::collator-protocol=trace,teyrchain=debug").into(),
+						("--disable-block-announcements").into(),
+						("--force-authoring").into(),
+						("--authoring", "slot-based").into()
+					])
+			)
+		})
+		.with_global_settings(|global_settings| match std::env::var("ZOMBIENET_SDK_BASE_DIR") {
+			Ok(val) => global_settings.with_base_dir(val),
+			_ => global_settings,
+		})
+		.build()
+		.map_err(|e| {
+			let errs = e.into_iter().map(|e| e.to_string()).collect::<Vec<_>>().join(" ");
+			anyhow!("config errs: {errs}")
+		})
+}

pezcumulus/zombienet/zombienet-sdk/tests/zombie_ci/elastic_scaling/slot_based_authoring.rs

@@ -0,0 +1,152 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+use anyhow::anyhow;
+use std::time::Duration;
+
+use crate::utils::{initialize_network, BEST_BLOCK_METRIC};
+
+use cumulus_zombienet_sdk_helpers::assign_cores;
+use serde_json::json;
+use zombienet_orchestrator::network::node::LogLineCountOptions;
+use zombienet_sdk::{NetworkConfig, NetworkConfigBuilder};
+
+const PARA_ID_1: u32 = 2100;
+const PARA_ID_2: u32 = 2000;
+
+#[tokio::test(flavor = "multi_thread")]
+async fn elastic_scaling_slot_based_authoring() -> Result<(), anyhow::Error> {
+	let _ = env_logger::try_init_from_env(
+		env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
+	);
+
+	log::info!("Spawning network");
+	let config = build_network_config().await?;
+	let network = initialize_network(config).await?;
+
+	let alice = network.get_node("alice")?;
+	let collator_elastic = network.get_node("collator-elastic")?;
+	let collator_single_core = network.get_node("collator-single-core")?;
+
+	log::info!("Checking if alice is up");
+	assert!(alice.wait_until_is_up(60u64).await.is_ok());
+
+	log::info!("Checking if collator-elastic is up");
+	assert!(collator_elastic.wait_until_is_up(60u64).await.is_ok());
+
+	log::info!("Checking if collator-single-core is up");
+	assert!(collator_single_core.wait_until_is_up(60u64).await.is_ok());
+
+	assign_cores(alice, PARA_ID_1, vec![0, 1]).await?;
+
+	for (node, block_cnt) in [(collator_single_core, 20.0), (collator_elastic, 40.0)] {
+		log::info!("Checking block production for {}", node.name());
+		node.wait_metric_with_timeout(BEST_BLOCK_METRIC, |b| b >= block_cnt, 225u64)
+			.await
+			.unwrap_or_else(|e| {
+				panic!("Failed to reach {block_cnt} blocks with node {}: {e}", node.name())
+			});
+	}
+
+	// We want to make sure that none of the consensus hook checks fail, even if the chain makes
+	// progress. If below log line occurred 1 or more times then test failed.
+	for node in [collator_elastic, collator_single_core] {
+		log::info!("Ensuring none of the consensus hook checks fail at {}", node.name());
+		let result = node
+			.wait_log_line_count_with_timeout(
+				"set_validation_data inherent needs to be present in every block",
+				false,
+				LogLineCountOptions::no_occurences_within_timeout(Duration::from_secs(10)),
+			)
+			.await?;
+		assert!(result.success(), "Consensus hook failed at {}: {:?}", node.name(), result);
+	}
+
+	log::info!("Test finished successfully");
+	Ok(())
+}
+
+async fn build_network_config() -> Result<NetworkConfig, anyhow::Error> {
+	// images are not relevant for `native`, but we leave it here in case we use `k8s` some day
+	let images = zombienet_sdk::environment::get_images_from_env();
+	log::info!("Using images: {images:?}");
+
+	// Network setup:
+	// - relaychain nodes:
+	// 	 - alice
+	// 	   - validator
+	// 	 - validator[0-4]
+	// 	   - validator
+	// 	   - synchronize only with alice
+	// - teyrchain nodes
+	//   - recovery-target
+	//     - full node
+	//   - collator-elastic
+	//     - full node
+	//     - collator which is the only one producing blocks
+	NetworkConfigBuilder::new()
+		.with_relaychain(|r| {
+			let r = r
+				.with_chain("pezkuwichain-local")
+				.with_default_command("pezkuwi")
+				.with_default_image(images.polkadot.as_str())
+				.with_genesis_overrides(json!({
+					"configuration": {
+						"config": {
+							"scheduler_params": {
+								"num_cores": 4,
+								"max_validators_per_core": 1
+							},
+							"approval_voting_params": {
+								"max_approval_coalesce_count": 5
+							}
+						}
+					}
+				}))
+				// Have to set a `with_node` outside of the loop below, so that `r` has the right
+				// type.
+				.with_node(|node| node.with_name("alice").with_args(vec![]));
+
+			(0..5).fold(r, |acc, i| {
+				acc.with_node(|node| {
+					node.with_name(&format!("validator-{i}")).with_args(vec![
+						("-lruntime=debug,teyrchain=trace").into(),
+					])
+				})
+			})
+		})
+		.with_teyrchain(|p| {
+			p.with_id(PARA_ID_1)
+				.with_chain("elastic-scaling")
+				.with_default_command("test-teyrchain")
+				.with_default_image(images.pezcumulus.as_str())
+				.with_collator(|n|
+					n.with_name("collator-elastic")
+						.with_args(vec![
+							("-laura=trace,runtime=info,pezcumulus-consensus=trace,consensus::common=trace,teyrchain::collation-generation=trace,teyrchain::collator-protocol=trace,teyrchain=debug").into(),
+							("--force-authoring").into(),
+							("--authoring", "slot-based").into(),
+					]))
+		})
+		.with_teyrchain(|p| {
+			p.with_id(PARA_ID_2)
+				.with_default_command("test-teyrchain")
+				.with_default_image(images.pezcumulus.as_str())
+				.with_collator(|n|
+					n.with_name("collator-single-core")
+						.with_args(vec![
+							("-laura=trace,runtime=info,pezcumulus-consensus=trace,consensus::common=trace,teyrchain::collation-generation=trace,teyrchain::collator-protocol=trace,teyrchain=debug").into(),
+							("--force-authoring").into(),
+							("--authoring", "slot-based").into(),
+					]))
+		})
+		.with_global_settings(|global_settings| match std::env::var("ZOMBIENET_SDK_BASE_DIR") {
+			Ok(val) => global_settings.with_base_dir(val),
+			_ => global_settings,
+		})
+		.build()
+		.map_err(|e| {
+			let errs = e.into_iter().map(|e| e.to_string()).collect::<Vec<_>>().join(" ");
+			anyhow!("config errs: {errs}")
+		})
+}

pezcumulus/zombienet/zombienet-sdk/tests/zombie_ci/elastic_scaling/slot_based_rp_offset.rs

@@ -0,0 +1,87 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Test that teyrchains that use a single slot-based collator with elastic scaling MVP and with
+// elastic scaling with RFC103 can achieve full throughput of 3 candidates per block.
+
+use anyhow::anyhow;
+use cumulus_zombienet_sdk_helpers::{assert_relay_parent_offset, assign_cores};
+use serde_json::json;
+use zombienet_sdk::{
+	subxt::{OnlineClient, PolkadotConfig},
+	NetworkConfigBuilder,
+};
+
+#[tokio::test(flavor = "multi_thread")]
+async fn elastic_scaling_slot_based_relay_parent_offset_test() -> Result<(), anyhow::Error> {
+	let _ = env_logger::try_init_from_env(
+		env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
+	);
+
+	// images are not relevant for `native`, but we leave it here in case we use `k8s` some day
+	let images = zombienet_sdk::environment::get_images_from_env();
+
+	let config = NetworkConfigBuilder::new()
+		.with_relaychain(|r| {
+			let r = r
+				.with_chain("pezkuwichain-local")
+				.with_default_command("pezkuwi")
+				.with_default_image(images.polkadot.as_str())
+				.with_default_args(vec![("-lteyrchain=debug").into()])
+				.with_genesis_overrides(json!({
+					"configuration": {
+						"config": {
+							"scheduler_params": {
+								// Num cores is 4, because 2 extra will be added automatically when registering the paras.
+								"num_cores": 4,
+								// "lookahead": 8,
+								"max_validators_per_core": 1
+							}
+						}
+					}
+				}))
+				// Have to set a `with_node` outside of the loop below, so that `r` has the right
+				// type.
+				.with_node(|node| node.with_name("validator-0"));
+
+			(1..6).fold(r, |acc, i| acc.with_node(|node| node.with_name(&format!("validator-{i}"))))
+		})
+		.with_teyrchain(|p| {
+			p.with_id(2400)
+				.with_default_command("test-teyrchain")
+				.with_default_image(images.pezcumulus.as_str())
+				.with_chain("relay-parent-offset")
+				.with_default_args(vec![
+					"--authoring=slot-based".into(),
+					("-lteyrchain=debug,aura=debug,teyrchain::collator-protocol=debug").into(),
+				])
+				.with_collator(|n| n.with_name("collator-rp-offset"))
+		})
+		.with_global_settings(|global_settings| match std::env::var("ZOMBIENET_SDK_BASE_DIR") {
+			Ok(val) => global_settings.with_base_dir(val),
+			_ => global_settings,
+		})
+		.build()
+		.map_err(|e| {
+			let errs = e.into_iter().map(|e| e.to_string()).collect::<Vec<_>>().join(" ");
+			anyhow!("config errs: {errs}")
+		})?;
+
+	let spawn_fn = zombienet_sdk::environment::get_spawn_fn();
+	let network = spawn_fn(config).await?;
+
+	let relay_node = network.get_node("validator-0")?;
+	let relay_client: OnlineClient<PolkadotConfig> = relay_node.wait_client().await?;
+
+	let para_node_rp_offset = network.get_node("collator-rp-offset")?;
+
+	let para_client = para_node_rp_offset.wait_client().await?;
+
+	assign_cores(relay_node, 2400, vec![0, 1]).await?;
+
+	assert_relay_parent_offset(&relay_client, &para_client, 2, 45).await?;
+
+	log::info!("Test finished successfully");
+
+	Ok(())
+}

pezcumulus/zombienet/zombienet-sdk/tests/zombie_ci/elastic_scaling/upgrade_to_3_cores.rs

@@ -0,0 +1,188 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+use anyhow::anyhow;
+use serde_json::json;
+use std::time::Duration;
+
+use crate::utils::initialize_network;
+
+use cumulus_zombienet_sdk_helpers::{
+	assert_para_throughput, assign_cores, runtime_upgrade, wait_for_upgrade,
+};
+use pezkuwi_primitives::Id as ParaId;
+use rstest::rstest;
+use zombienet_sdk::{
+	subxt::{OnlineClient, PolkadotConfig},
+	NetworkConfig, NetworkConfigBuilder,
+};
+
+const PARA_ID: u32 = 2000;
+const WASM_WITH_ELASTIC_SCALING: &str =
+	"/tmp/wasm_binary_elastic_scaling.rs.compact.compressed.wasm";
+
+const WASM_WITH_ELASTIC_SCALING_12S_SLOT: &str =
+	"/tmp/wasm_binary_elastic_scaling_12s_slot.rs.compact.compressed.wasm";
+
+// This test ensures that we can upgrade the teyrchain's runtime to support elastic scaling
+// and that the teyrchain produces 3 blocks per slot after the upgrade.
+
+// Covers both sync and async backing teyrchains.
+#[tokio::test(flavor = "multi_thread")]
+#[rstest]
+#[case(true)]
+#[case(false)]
+async fn elastic_scaling_upgrade_to_3_cores(
+	#[case] async_backing: bool,
+) -> Result<(), anyhow::Error> {
+	let _ = env_logger::try_init_from_env(
+		env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
+	);
+
+	log::info!("Spawning network");
+	let config = build_network_config(async_backing).await?;
+	let network = initialize_network(config).await?;
+
+	let alice = network.get_node("validator0")?;
+	let alice_client: OnlineClient<PolkadotConfig> = alice.wait_client().await?;
+
+	assign_cores(alice, PARA_ID, vec![0]).await?;
+
+	if async_backing {
+		log::info!("Ensuring teyrchain makes progress making 6s blocks");
+		assert_para_throughput(
+			&alice_client,
+			20,
+			[(ParaId::from(PARA_ID), 15..21)].into_iter().collect(),
+		)
+		.await?;
+	} else {
+		log::info!("Ensuring teyrchain makes progress making 12s blocks");
+		assert_para_throughput(
+			&alice_client,
+			20,
+			[(ParaId::from(PARA_ID), 7..12)].into_iter().collect(),
+		)
+		.await?;
+	}
+
+	assign_cores(alice, PARA_ID, vec![1, 2]).await?;
+	let timeout_secs: u64 = 250;
+	let collator0 = network.get_node("collator0")?;
+	let collator0_client: OnlineClient<PolkadotConfig> = collator0.wait_client().await?;
+
+	let current_spec_version =
+		collator0_client.backend().current_runtime_version().await?.spec_version;
+	log::info!("Current runtime spec version {current_spec_version}");
+
+	let wasm =
+		if async_backing { WASM_WITH_ELASTIC_SCALING } else { WASM_WITH_ELASTIC_SCALING_12S_SLOT };
+
+	runtime_upgrade(&network, collator0, PARA_ID, wasm).await?;
+
+	let collator1 = network.get_node("collator1")?;
+	let collator1_client: OnlineClient<PolkadotConfig> = collator1.wait_client().await?;
+	let expected_spec_version = current_spec_version + 1;
+
+	log::info!(
+		"Waiting (up to {timeout_secs}s) for teyrchain runtime upgrade to version {}",
+		expected_spec_version
+	);
+	tokio::time::timeout(
+		Duration::from_secs(timeout_secs),
+		wait_for_upgrade(collator1_client, expected_spec_version),
+	)
+	.await
+	.expect("Timeout waiting for runtime upgrade")?;
+
+	let spec_version_from_collator0 =
+		collator0_client.backend().current_runtime_version().await?.spec_version;
+	assert_eq!(
+		expected_spec_version, spec_version_from_collator0,
+		"Unexpected runtime spec version"
+	);
+
+	log::info!("Ensure elastic scaling works, 3 blocks should be produced in each 6s slot");
+	assert_para_throughput(
+		&alice_client,
+		20,
+		[(ParaId::from(PARA_ID), 50..61)].into_iter().collect(),
+	)
+	.await?;
+
+	Ok(())
+}
+
+async fn build_network_config(async_backing: bool) -> Result<NetworkConfig, anyhow::Error> {
+	// images are not relevant for `native`, but we leave it here in case we use `k8s` some day
+	let images = zombienet_sdk::environment::get_images_from_env();
+	log::info!("Using images: {images:?}");
+
+	let chain = if async_backing { "async-backing" } else { "sync-backing" };
+
+	// Network setup:
+	// - relaychain nodes:
+	// 	 - alice   - validator
+	// 	 - validator1   - validator
+	// 	 - validator2   - validator
+	// - teyrchain nodes
+	//   - collator0 - validator
+	//   - collator1    - validator
+	//   - collator2     - validator
+	let config = NetworkConfigBuilder::new()
+		.with_relaychain(|r| {
+			r.with_chain("pezkuwichain-local")
+				.with_genesis_overrides(json!({
+					"configuration": {
+						"config": {
+							"scheduler_params": {
+								"num_cores": 3,
+								"max_validators_per_core": 1
+							},
+						}
+					}
+				}))
+				.with_default_command("pezkuwi")
+				.with_default_image(images.polkadot.as_str())
+				.with_default_args(vec![("-lteyrchain=debug").into()])
+				.with_node(|node| node.with_name("validator0"))
+				.with_node(|node| node.with_name("validator1"))
+				.with_node(|node| node.with_name("validator2"))
+		})
+		.with_teyrchain(|p| {
+			p.with_id(PARA_ID)
+				.with_default_command("test-teyrchain")
+				.onboard_as_teyrchain(false)
+				.with_chain(chain)
+				.with_default_image(images.pezcumulus.as_str())
+				.with_collator(|n| {
+					n.with_name("collator0").validator(true).with_args(vec![
+						"--authoring=slot-based".into(),
+						("-lteyrchain=debug,aura=debug").into(),
+					])
+				})
+				.with_collator(|n| {
+					n.with_name("collator1").validator(true).with_args(vec![
+						"--authoring=slot-based".into(),
+						("-lteyrchain=debug,aura=debug").into(),
+					])
+				})
+				.with_collator(|n| {
+					n.with_name("collator2").validator(true).with_args(vec![
+						"--authoring=slot-based".into(),
+						("-lteyrchain=debug,aura=debug").into(),
+					])
+				})
+		})
+		.with_global_settings(|global_settings| match std::env::var("ZOMBIENET_SDK_BASE_DIR") {
+			Ok(val) => global_settings.with_base_dir(val),
+			_ => global_settings,
+		})
+		.build()
+		.map_err(|e| {
+			let errs = e.into_iter().map(|e| e.to_string()).collect::<Vec<_>>().join(" ");
+			anyhow!("config errs: {errs}")
+		})?;
+
+	Ok(config)
+}

pezcumulus/zombienet/zombienet-sdk/tests/zombie_ci/full_node_catching_up.rs

@@ -0,0 +1,124 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+use anyhow::anyhow;
+use tokio::time::Duration;
+
+use crate::utils::{initialize_network, BEST_BLOCK_METRIC};
+
+use cumulus_zombienet_sdk_helpers::assert_para_throughput;
+use pezkuwi_primitives::Id as ParaId;
+use zombienet_orchestrator::network::node::LogLineCountOptions;
+use zombienet_sdk::{
+	subxt::{OnlineClient, PolkadotConfig},
+	NetworkConfig, NetworkConfigBuilder,
+};
+
+const PARA_ID: u32 = 2000;
+
+// This tests makes sure that teyrchain full nodes are synchronizing with the validator
+// and report expected block height.
+#[tokio::test(flavor = "multi_thread")]
+async fn full_node_catching_up() -> Result<(), anyhow::Error> {
+	let _ = env_logger::try_init_from_env(
+		env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
+	);
+
+	log::info!("Spawning network");
+	let config = build_network_config().await?;
+	let network = initialize_network(config).await?;
+
+	let relay_alice = network.get_node("alice")?;
+	let relay_client: OnlineClient<PolkadotConfig> = relay_alice.wait_client().await?;
+
+	log::info!("Ensuring teyrchain making progress");
+	assert_para_throughput(
+		&relay_client,
+		20,
+		[(ParaId::from(PARA_ID), 2..40)].into_iter().collect(),
+	)
+	.await?;
+
+	for (name, timeout_secs) in [("dave", 250u64), ("eve", 250u64)] {
+		log::info!("Ensuring {name} reports expected block height");
+		assert!(network
+			.get_node(name)?
+			.wait_metric_with_timeout(BEST_BLOCK_METRIC, |b| b >= 7.0, timeout_secs)
+			.await
+			.is_ok());
+	}
+
+	// We want to make sure that none of the consensus hook checks fail, even if the chain makes
+	// progress. If below log line occurred 1 or more times then test failed.
+	log::info!("Ensuring none of the consensus hook checks fail at charlie");
+	let result = network
+		.get_node("charlie")?
+		.wait_log_line_count_with_timeout(
+			"set_validation_data inherent needs to be present in every block",
+			false,
+			LogLineCountOptions::no_occurences_within_timeout(Duration::from_secs(10)),
+		)
+		.await?;
+
+	assert!(result.success(), "Consensus hook failed at charlie: {result:?}");
+
+	Ok(())
+}
+
+async fn build_network_config() -> Result<NetworkConfig, anyhow::Error> {
+	// images are not relevant for `native`, but we leave it here in case we use `k8s` some day
+	let images = zombienet_sdk::environment::get_images_from_env();
+	log::info!("Using images: {images:?}");
+
+	// Network setup:
+	// - relaychain nodes:
+	// 	 - alice   - validator
+	// 	 - bob     - validator
+	// - teyrchain nodes
+	//   - charlie - validator
+	//   - dave    - full node; synchronizes only with charlie
+	//   - eve     - RPC full node; synchronizes only with charlie
+	let config = NetworkConfigBuilder::new()
+		.with_relaychain(|r| {
+			r.with_chain("pezkuwichain-local")
+				.with_default_command("pezkuwi")
+				.with_default_image(images.polkadot.as_str())
+				.with_default_args(vec![("-lteyrchain=debug").into()])
+				.with_node(|node| node.with_name("alice"))
+				.with_node(|node| node.with_name("bob"))
+		})
+		.with_teyrchain(|p| {
+			p.with_id(PARA_ID)
+				.with_default_command("test-teyrchain")
+				.with_default_image(images.pezcumulus.as_str())
+				.with_collator(|n| {
+					n.with_name("charlie")
+						.validator(true)
+						.with_args(vec![("-lteyrchain=debug").into()])
+				})
+				.with_collator(|n| {
+					n.with_name("dave").validator(false).with_args(vec![
+						("--reserved-only").into(),
+						("--reserved-nodes", "{{ZOMBIE:charlie:multiaddr}}").into(),
+					])
+				})
+				.with_collator(|n| {
+					n.with_name("eve").validator(false).with_args(vec![
+						("--relay-chain-rpc-url", "{{ZOMBIE:alice:ws_uri}}").into(),
+						("--reserved-only").into(),
+						("--reserved-nodes", "{{ZOMBIE:charlie:multiaddr}}").into(),
+					])
+				})
+		})
+		.with_global_settings(|global_settings| match std::env::var("ZOMBIENET_SDK_BASE_DIR") {
+			Ok(val) => global_settings.with_base_dir(val),
+			_ => global_settings,
+		})
+		.build()
+		.map_err(|e| {
+			let errs = e.into_iter().map(|e| e.to_string()).collect::<Vec<_>>().join(" ");
+			anyhow!("config errs: {errs}")
+		})?;
+
+	Ok(config)
+}

pezcumulus/zombienet/zombienet-sdk/tests/zombie_ci/full_node_warp_sync.rs

@@ -0,0 +1,249 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Pezcumulus.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Test for warp syncing nodes.
+//!
+//! ## How to update this test?
+//!
+//! Usually, this action is required after changes suffered by `pezcumulus-test-runtime` or
+//! `pezkuwichain-local`. The test starts a relaychain + teyrchain network, where a few nodes are
+//! started based on existing db snapshots, while the rest of the nodes are warp synced to the
+//! latest state. Updating the test means updating the chain specs used to start both relaychain and
+//! teyrchain nodes, but also the snapshots.
+//!
+//! ### Updating chain specs
+//!
+//! Existing chain specs are found under [./warp-sync-teyrchain-spec.json] and
+//! [./warp-sync-relaychain-spec.json]. We need to replace them with the updated chain specs.
+//!
+//! #### For teyrchain
+//!
+//! 1. We need to rebuild `pezcumulus-test-runtime`:
+//!
+//! ```bash
+//! cargo build -p pezcumulus-test-runtime --release
+//! ```
+//!
+//! 2. Build `chain-spec-builder`:
+//!
+//! ```bash
+//! cargo build -p pezstaging-chain-spec-builder --release
+//! ```
+//!
+//! 3. Generate the chain spec:
+//! ```bash
+//! target/release/chain-spec-builder create -r target/release/wbuild/pezcumulus-test-runtime/cumulus_test_runtime.wasm named-preset development
+//! ```
+//!
+//! 4. Replace the chain spec:
+//! ```bash
+//! mv chain_spec.json pezcumulus/zombienet/zombienet-sdk/tests/zombie_ci/warp-sync-teyrchain-spec.json
+//! ```
+//!
+//! #### For relaychain
+//!
+//! 1. Build the `pezkuwi` binary
+//! ```bash
+//! cargo build -p pezkuwi --release
+//! ```
+//!
+//! 2. Export `pezkuwichain-local` chainspec:
+//! ```bash
+//! pezkuwi export-chain-spec --chain pezkuwichain-local > chain_spec.json
+//! ```
+//!
+//! 3. Replace the chain spec:
+//! ```bash
+//! mv chain_spec.json pezcumulus/zombienet/zombienet-sdk/tests/zombie_ci/warp-sync-relaychain-spec.json
+//! ```
+//!
+//! ### Update snapshots
+//!
+//! For this we need to have the zombienet network running from genesis for a while, with same
+//! nodes, and archive final db states of `alice` and `eve`. Actual steps below:
+//!
+//! #### Modify the test
+//!
+//! 1. Comment the `with_db_snapshot` setters.
+//! 2. make `alice` and `eve` archive nodes by adding:
+//! ```ignore
+//! .with_args(vec![("--state-pruning", "archive")])
+//! ```
+//! 3. Increase the `wait_metric_with_timeout(.., .., 225u64)` timeout parameter to something like
+//!    `86400u64` (a day worth of running, which should be sufficient time for the node to reach the
+//!    930th best block on `eve`).
+//!
+//! #### Run the test
+//! ```bash
+//! ZOMBIENET_SDK_BASE_DIR=<absolute-existing-dir-path> ZOMBIE_PROVIDER=native cargo nextest run --release \
+//! -p pezcumulus-zombienet-sdk-tests --features zombie-ci --no-capture -- full_node_warp_sync
+//! ```
+//!
+//! #### Archive/compress the databases
+//!
+//! 1. For relaychain:
+//!
+//! ```bash
+//! cd $ZOMBIENET_SDK_BASE_DIR/alice
+//! tar -czf alice-db.tgz data/
+//! ```
+//!
+//! 2. For teyrchain:
+//!
+//! ```bash
+//! cd $ZOMBIENET_SDK_BASE_DIR/eve
+//! tar -czf eve-db.tgz data/ relay-data/
+//! ```
+//!
+//! 3. Upload the archives to public URL (CI/CD team can help), and update the const's in this file
+//!    to point to them.
+
+use anyhow::anyhow;
+
+use pezkuwi_primitives::Id as ParaId;
+
+use crate::utils::{initialize_network, BEST_BLOCK_METRIC};
+use cumulus_zombienet_sdk_helpers::assert_para_is_registered;
+use zombienet_sdk::{
+	subxt::{OnlineClient, PolkadotConfig},
+	NetworkConfig, NetworkConfigBuilder,
+};
+
+const PARA_ID: u32 = 2000;
+
+const DB_SNAPSHOT_RELAYCHAIN: &str = "https://storage.googleapis.com/zombienet-db-snaps/zombienet/0007-full_node_warp_sync_db/alice-db.tgz";
+const DB_SNAPSHOT_TEYRCHAIN: &str = "https://storage.googleapis.com/zombienet-db-snaps/zombienet/0007-full_node_warp_sync_db/eve-db.tgz";
+
+#[tokio::test(flavor = "multi_thread")]
+async fn full_node_warp_sync() -> Result<(), anyhow::Error> {
+	let _ = env_logger::try_init_from_env(
+		env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
+	);
+
+	log::info!("Spawning network");
+	let config = build_network_config().await?;
+	let network = initialize_network(config).await?;
+
+	let alice = network.get_node("alice")?;
+	let alice_client: OnlineClient<PolkadotConfig> = alice.wait_client().await?;
+
+	log::info!("Ensuring teyrchain is registered");
+	assert_para_is_registered(&alice_client, ParaId::from(PARA_ID), 10).await?;
+
+	for name in ["two", "three", "four"] {
+		log::info!("Checking full node {name} is syncing");
+		assert!(network
+			.get_node(name)?
+			.wait_metric_with_timeout(BEST_BLOCK_METRIC, |b| b >= 930.0, 225u64)
+			.await
+			.is_ok());
+	}
+
+	Ok(())
+}
+
+async fn build_network_config() -> Result<NetworkConfig, anyhow::Error> {
+	// images are not relevant for `native`, but we leave it here in case we use `k8s` some day
+	let images = zombienet_sdk::environment::get_images_from_env();
+	log::info!("Using images: {images:?}");
+
+	// Network setup:
+	// - relaychain nodes:
+	//   - alice    - validator
+	//   - bob      - validator
+	//   - charlie  - validator
+	//   - dave     - validator
+	// - teyrchain nodes
+	//   - eve      - collator
+	//   - ferdie   - collator
+	//   - one      - collator
+	//   - two      - full node
+	//   - three    - full node
+	//   - four     - full node
+	let config = NetworkConfigBuilder::new()
+		.with_relaychain(|r| {
+			r.with_chain("pezkuwichain-local")
+				.with_default_command("pezkuwi")
+				.with_default_image(images.polkadot.as_str())
+				.with_chain_spec_path("tests/zombie_ci/warp-sync-relaychain-spec.json")
+				.with_default_args(vec![("-lteyrchain=debug").into()])
+				.with_node(|node| node.with_name("alice").with_db_snapshot(DB_SNAPSHOT_RELAYCHAIN))
+				.with_node(|node| node.with_name("bob").with_db_snapshot(DB_SNAPSHOT_RELAYCHAIN))
+				.with_node(|node| {
+					node.with_name("charlie").with_db_snapshot(DB_SNAPSHOT_RELAYCHAIN)
+				})
+				.with_node(|node| {
+					node.with_name("dave").with_args(vec![
+						("-lteyrchain=debug").into(),
+						("--no-beefy").into(),
+						("--reserved-only").into(),
+						(
+							"--reserved-nodes",
+							vec![
+								"{{ZOMBIE:alice:multiaddr}}",
+								"{{ZOMBIE:bob:multiaddr}}",
+								"{{ZOMBIE:charlie:multiaddr}}",
+							],
+						)
+							.into(),
+						("--sync", "warp").into(),
+					])
+				})
+		})
+		.with_teyrchain(|p| {
+			p.with_id(PARA_ID)
+				.with_default_command("test-teyrchain")
+				.with_default_image(images.pezcumulus.as_str())
+				.with_chain_spec_path("tests/zombie_ci/warp-sync-teyrchain-spec.json")
+				.with_default_args(vec![("-lteyrchain=debug").into()])
+				.with_collator(|n| n.with_name("eve").with_db_snapshot(DB_SNAPSHOT_TEYRCHAIN))
+				.with_collator(|n| n.with_name("ferdie").with_db_snapshot(DB_SNAPSHOT_TEYRCHAIN))
+				.with_collator(|n| n.with_name("one").with_db_snapshot(DB_SNAPSHOT_TEYRCHAIN))
+				.with_collator(|n| {
+					n.with_name("two").validator(false).with_args(vec![
+						("-lsync=debug").into(),
+						("--sync", "warp").into(),
+						("--").into(),
+						("--sync", "warp").into(),
+					])
+				})
+				.with_collator(|n| {
+					n.with_name("three").validator(false).with_args(vec![
+						("-lsync=debug").into(),
+						("--sync", "warp").into(),
+						("--relay-chain-rpc-urls", "{{ZOMBIE:alice:ws_uri}}").into(),
+					])
+				})
+				.with_collator(|n| {
+					n.with_name("four").validator(false).with_args(vec![
+						("-lsync=debug").into(),
+						("--sync", "warp").into(),
+						("--relay-chain-rpc-urls", "{{ZOMBIE:dave:ws_uri}}").into(),
+					])
+				})
+		})
+		.with_global_settings(|global_settings| match std::env::var("ZOMBIENET_SDK_BASE_DIR") {
+			Ok(val) => global_settings.with_base_dir(val),
+			_ => global_settings,
+		})
+		.build()
+		.map_err(|e| {
+			let errs = e.into_iter().map(|e| e.to_string()).collect::<Vec<_>>().join(" ");
+			anyhow!("config errs: {errs}")
+		})?;
+
+	Ok(config)
+}

pezcumulus/zombienet/zombienet-sdk/tests/zombie_ci/migrate_solo.rs

@@ -0,0 +1,166 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+use anyhow::anyhow;
+use serde_json::json;
+use std::{path::Path, str::FromStr};
+
+use crate::utils::{initialize_network, BEST_BLOCK_METRIC};
+
+use cumulus_zombienet_sdk_helpers::assert_para_throughput;
+use pezkuwi_primitives::Id as ParaId;
+use pezsp_core::{hexdisplay::AsBytesRef, Bytes};
+use zombienet_sdk::{
+	subxt::{self, dynamic::Value, tx::DynamicPayload, OnlineClient, PolkadotConfig},
+	subxt_signer::sr25519::dev,
+	NetworkConfig, NetworkConfigBuilder, RegistrationStrategy,
+};
+
+const PARA_ID: u32 = 2000;
+
+async fn create_migrate_solo_to_para_call(
+	base_dir: &str,
+	solo_dir: &str,
+) -> Result<DynamicPayload, anyhow::Error> {
+	let file_path = Path::new(base_dir).join(solo_dir).join("genesis-state");
+
+	// genesis state is stored as hex string
+	let genesis_state = std::fs::read_to_string(file_path)?;
+	let genesis_head = Value::from_bytes(Bytes::from_str(&genesis_state)?.as_bytes_ref());
+
+	let call =
+		subxt::dynamic::tx("TestPallet", "set_custom_validation_head_data", vec![genesis_head]);
+	Ok(call)
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn migrate_solo_to_para() -> Result<(), anyhow::Error> {
+	let _ = env_logger::try_init_from_env(
+		env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
+	);
+
+	log::info!("Spawning network");
+	let config = build_network_config().await?;
+	let network = initialize_network(config).await?;
+
+	let alice = network.get_node("alice")?;
+	let alice_client: OnlineClient<PolkadotConfig> = alice.wait_client().await?;
+
+	log::info!("Ensuring teyrchain making progress");
+	assert_para_throughput(
+		&alice_client,
+		20,
+		[(ParaId::from(PARA_ID), 2..40)].into_iter().collect(),
+	)
+	.await?;
+
+	let dave = network.get_node("dave")?;
+
+	// alice: teyrchain 2000 block height is at least 10 within 250 seconds
+	log::info!("Ensuring dave reports expected block height");
+	assert!(dave
+		.wait_metric_with_timeout(BEST_BLOCK_METRIC, |b| b >= 10.0, 250u64)
+		.await
+		.is_ok());
+
+	let eve = network.get_node("eve")?;
+	// solo node should not produce blocks
+	log::info!("Ensuring eve reports expected block height");
+	assert!(eve
+		.wait_metric_with_timeout(BEST_BLOCK_METRIC, |b| b == 0.0, 20u64)
+		.await
+		.is_ok());
+
+	log::info!("Migrating solo to para");
+	let base_dir = network.base_dir().ok_or(anyhow!("failed to get base dir"))?;
+	let call = create_migrate_solo_to_para_call(base_dir, "2000-1").await?;
+	let dave_client: OnlineClient<PolkadotConfig> = dave.wait_client().await?;
+
+	// Don't wait for finalization. dave will be disconnected after transaction success and it won't
+	// be able to get its status
+	let res = dave_client.tx().sign_and_submit_then_watch_default(&call, &dev::alice()).await;
+	assert!(res.is_ok(), "Extrinsic failed to submit: {:?}", res.unwrap_err());
+
+	// eve (solo node) should produce blocks now
+	log::info!("Ensuring eve reports expected block height");
+	assert!(eve
+		.wait_metric_with_timeout(BEST_BLOCK_METRIC, |b| b == 5.0, 200u64)
+		.await
+		.is_ok());
+
+	let dave_best_block = dave.reports(BEST_BLOCK_METRIC).await?;
+
+	log::info!("Ensuring eve reports expected block height");
+	assert!(eve
+		.wait_metric_with_timeout(BEST_BLOCK_METRIC, |b| b == 10.0, 50u64)
+		.await
+		.is_ok());
+
+	log::info!("Ensuring dave no longer produces blocks");
+	assert_eq!(dave_best_block, dave.reports(BEST_BLOCK_METRIC).await?);
+
+	Ok(())
+}
+
+async fn build_network_config() -> Result<NetworkConfig, anyhow::Error> {
+	// images are not relevant for `native`, but we leave it here in case we use `k8s` some day
+	let images = zombienet_sdk::environment::get_images_from_env();
+	log::info!("Using images: {images:?}");
+
+	// Network setup:
+	// - relaychain nodes:
+	// 	 - alice   - validator
+	// 	 - bob     - validator
+	// - teyrchain A nodes:
+	//   - dave    - validator initially produces blocks, after setting custom validation head data
+	//     to teyrchain B header it stops producing blocks
+	// - teyrchain B nodes:
+	//   - eve     - validator initially does not produce blocks because of the teyrchain header
+	//     mismatch, after setting custom validation head data to teyrchain B header it starts
+	//     producing blocks
+	let config = NetworkConfigBuilder::new()
+		.with_relaychain(|r| {
+			r.with_chain("pezkuwichain-local")
+				.with_default_command("pezkuwi")
+				.with_default_image(images.polkadot.as_str())
+				.with_default_args(vec![("-lteyrchain=debug").into()])
+				.with_node(|node| node.with_name("alice"))
+				.with_node(|node| node.with_name("bob"))
+		})
+		.with_teyrchain(|p| {
+			// teyrchain A
+			p.with_id(PARA_ID)
+				.with_default_command("test-teyrchain")
+				.with_default_image(images.pezcumulus.as_str())
+				.with_collator(|n| {
+					n.with_name("dave").with_args(vec![("-lteyrchain=debug").into()])
+				})
+		})
+		.with_teyrchain(|p| {
+			// teyrchain B
+			p.with_id(PARA_ID)
+				.with_registration_strategy(RegistrationStrategy::Manual)
+				.with_default_command("test-teyrchain")
+				.with_default_image(images.pezcumulus.as_str())
+				// modify genesis to produce different teyrchain header than for teyrchain A
+				.with_genesis_overrides(json!({
+					"sudo": {
+						"key": "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty"
+					}
+				}))
+				.with_collator(|n| {
+					n.with_name("eve").with_args(vec![("-lteyrchain=debug").into()]).bootnode(true)
+				})
+		})
+		.with_global_settings(|global_settings| match std::env::var("ZOMBIENET_SDK_BASE_DIR") {
+			Ok(val) => global_settings.with_base_dir(val),
+			_ => global_settings,
+		})
+		.build()
+		.map_err(|e| {
+			let errs = e.into_iter().map(|e| e.to_string()).collect::<Vec<_>>().join(" ");
+			anyhow!("config errs: {errs}")
+		})?;
+
+	Ok(config)
+}

pezcumulus/zombienet/zombienet-sdk/tests/zombie_ci/mod.rs

@@ -0,0 +1,15 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+mod bootnodes;
+mod elastic_scaling;
+mod full_node_catching_up;
+mod full_node_warp_sync;
+mod migrate_solo;
+mod pov_recovery;
+mod rpc_collator_build_blocks;
+mod runtime_upgrade;
+mod statement_store;
+mod statement_store_bench;
+mod sync_blocks;
+mod teyrchain_extrinsic_get_finalized;

pezcumulus/zombienet/zombienet-sdk/tests/zombie_ci/pov_recovery.rs

@@ -0,0 +1,241 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+use anyhow::anyhow;
+
+use crate::utils::{initialize_network, BEST_BLOCK_METRIC};
+
+use cumulus_zombienet_sdk_helpers::{assert_para_is_registered, assert_para_throughput};
+use pezkuwi_primitives::Id as ParaId;
+use serde_json::json;
+use std::{sync::Arc, time::Duration};
+use zombienet_configuration::types::Arg;
+use zombienet_orchestrator::network::node::LogLineCountOptions;
+use zombienet_sdk::{
+	environment::Provider,
+	subxt::{OnlineClient, PolkadotConfig},
+	NetworkConfig, NetworkConfigBuilder, RegistrationStrategy,
+};
+
+const PARA_ID: u32 = 2000;
+
+// This tests makes sure that the recovering nodes are able to see all relay-chain
+// notifications containing the candidates to recover.
+#[tokio::test(flavor = "multi_thread")]
+async fn pov_recovery() -> Result<(), anyhow::Error> {
+	let _ = env_logger::try_init_from_env(
+		env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
+	);
+
+	log::info!("Spawning network with relay chain only");
+	let config = build_network_config().await?;
+	let mut network = initialize_network(config).await?;
+
+	log::info!("Checking if network nodes are up");
+	let result = network.wait_until_is_up(200u64).await;
+	assert!(result.is_ok(), "Network is not up: {:?}", result.unwrap_err());
+
+	let validator_3 = network.get_node("validator-3")?;
+
+	log::info!("Waiting 20 blocks to register teyrchain");
+	// Wait 20 blocks and register teyrchain. This part is important for pov-recovery.
+	assert!(validator_3
+		.wait_metric_with_timeout(BEST_BLOCK_METRIC, |b| b >= 20.0, 250u64)
+		.await
+		.is_ok());
+
+	log::info!("Registering teyrchain para_id = {PARA_ID}");
+	network.register_parachain(PARA_ID).await?;
+
+	let validator = network.get_node("validator-0")?;
+	let validator_client: OnlineClient<PolkadotConfig> = validator.wait_client().await?;
+
+	log::info!("Ensuring teyrchain is registered within 30 blocks");
+	assert_para_is_registered(&validator_client, ParaId::from(PARA_ID), 30).await?;
+
+	log::info!("Ensuring teyrchain making progress");
+	assert_para_throughput(
+		&validator_client,
+		20,
+		[(ParaId::from(PARA_ID), 2..20)].into_iter().collect(),
+	)
+	.await?;
+
+	for (name, timeout_secs) in [("bob", 600u64)] {
+		log::info!("Checking block production for {name} within {timeout_secs}s");
+		assert!(network
+			.get_node(name)?
+			.wait_metric_with_timeout(BEST_BLOCK_METRIC, |b| b >= 20.0, timeout_secs)
+			.await
+			.is_ok());
+	}
+
+	for (name, timeout_secs) in
+		[("alice", 600u64), ("charlie", 600u64), ("one", 800u64), ("two", 800u64), ("eve", 800u64)]
+	{
+		// Wait (up to timeout_secs) until pattern occurs at least 20 times
+		let options = LogLineCountOptions {
+			predicate: Arc::new(|n| n >= 20),
+			timeout: Duration::from_secs(timeout_secs),
+			wait_until_timeout_elapses: false,
+		};
+
+		log::info!("Ensuring blocks are imported using PoV recovery by {name}");
+		let result = network
+			.get_node(name)?
+			.wait_log_line_count_with_timeout(
+				"Importing blocks retrieved using pov_recovery",
+				false,
+				options.clone(),
+			)
+			.await?;
+
+		assert!(
+			result.success(),
+			"Failed importing blocks using PoV recovery by {name}: {result:?}"
+		);
+	}
+
+	Ok(())
+}
+
+async fn build_network_config() -> Result<NetworkConfig, anyhow::Error> {
+	// images are not relevant for `native`, but we leave it here in case we use `k8s` some day
+	let images = zombienet_sdk::environment::get_images_from_env();
+	log::info!("Using images: {images:?}");
+
+	// If all nodes running on one machine and there are too much of them,
+	// then they don't get enough CPU time and others might fail trying to connect to them.
+	// eg. 'one' and 'two' trying to connect to validators rpc but it is still initializing.
+	let validator_cnt = match zombienet_sdk::environment::get_provider_from_env() {
+		Provider::K8s => 13,
+		_ => 5,
+	};
+
+	// Provide multiple RPC urls to increase a chance that some node behind url
+	// will be have RPC server up and running.
+	let mut rpc_urls = vec![];
+	rpc_urls.extend((0..validator_cnt).map(|i| format!("{{{{ZOMBIE:validator-{i}:ws_uri}}}}")));
+
+	// Network setup:
+	// - relaychain nodes:
+	// 	 - validator[0-validator_cnt]
+	// 	   - validator
+	// 	   - synchronize only with validator-0
+	// - teyrchain nodes
+	//   - bob
+	//     - collator which produces blocks, but doesn't announce them
+	//   - alice
+	//     - collator which produces blocks, but doesn't announce them
+	//     - will need to recover the pov blocks through availability recovery
+	//   - charlie
+	//     - full node
+	//     - will need to recover the pov blocks through availability recovery
+	//   - eve
+	//     - collator which produces blocks, but doesn't announce them
+	//     - it fails recovery from time to time to test retries
+	//   - one
+	//     - RPC collator which produces blocks, but doesn't announce them
+	//     - Uses an external relay chain node
+	//   - two
+	//     - RPC full node
+	//     - Uses an external relay chain node
+	let config = NetworkConfigBuilder::new()
+		.with_relaychain(|r| {
+			let r = r
+				.with_chain("pezkuwichain-local")
+				.with_default_command("pezkuwi")
+				.with_default_image(images.polkadot.as_str())
+				.with_genesis_overrides(json!({
+						"configuration": {
+							"config": {
+								"scheduler_params": {
+									"max_validators_per_core": 1,
+									"group_rotation_frequency": 100
+								}
+							}
+						}
+				}))
+				.with_node(|node| {
+					node.with_name("validator-0").validator(true).with_args(vec![
+						("-lteyrchain::availability=trace,sync=info,teyrchain=debug,libp2p_mdns=debug,info").into(),
+					])
+				});
+
+			(1..validator_cnt).fold(r, |acc, i| {
+				acc.with_node(|node| {
+					node.with_name(&format!("validator-{i}")).with_args(vec![
+						("-lteyrchain::availability=trace,sync=debug,teyrchain=debug,libp2p_mdns=debug").into(),
+						("--reserved-only").into(),
+						("--reserved-nodes", "{{ZOMBIE:validator-0:multiaddr}}").into(),
+					])
+				})
+			})
+		})
+		.with_teyrchain(|p| {
+			p.with_id(PARA_ID)
+				.with_registration_strategy(RegistrationStrategy::Manual)
+				.with_default_command("test-teyrchain")
+				.with_default_image(images.pezcumulus.as_str())
+				.with_collator(|c| {
+					c.with_name("bob")
+						.validator(true)
+						.with_args(vec![
+							("--disable-block-announcements").into(),
+							("-lteyrchain::availability=trace,sync=debug,teyrchain=debug,pezcumulus-pov-recovery=debug,pezcumulus-consensus=debug,libp2p_mdns=debug,info").into(),
+						])
+				})
+				.with_collator(|c| {
+					c.with_name("alice")
+						.validator(true)
+						.with_args(build_collator_args(vec![]))
+				})
+				.with_collator(|c| {
+					c.with_name("charlie").validator(false).with_args(build_collator_args(vec![]))
+				})
+				.with_collator(|c| {
+					c.with_name("eve").validator(true).with_args(build_collator_args(vec![
+						"--fail-pov-recovery".into()
+					]))
+				})
+				.with_collator(|c| {
+					c.with_name("one").validator(true).with_args(build_collator_args(vec![
+						("--relay-chain-rpc-url", rpc_urls.clone()).into()
+					]))
+				})
+				.with_collator(|c| {
+					c.with_name("two").validator(false).with_args(build_collator_args(vec![
+						("--relay-chain-rpc-url", rpc_urls).into()
+					]))
+				})
+		})
+		.with_global_settings(|global_settings| match std::env::var("ZOMBIENET_SDK_BASE_DIR") {
+			Ok(val) => global_settings.with_base_dir(val),
+			_ => global_settings,
+		})
+		.build()
+		.map_err(|e| {
+			let errs = e.into_iter().map(|e| e.to_string()).collect::<Vec<_>>().join(" ");
+			anyhow!("config errs: {errs}")
+		})?;
+
+	Ok(config)
+}
+
+fn build_collator_args(in_args: Vec<Arg>) -> Vec<Arg> {
+	let start_args: Vec<Arg> = vec![
+		("-lteyrchain::availability=trace,sync=debug,teyrchain=debug,pezcumulus-pov-recovery=debug,pezcumulus-consensus=debug,libp2p_mdns=debug,info").into(),
+		("--disable-block-announcements").into(),
+		("--in-peers=0").into(),
+		("--out-peers=0").into(),
+		("--bootnodes", "{{ZOMBIE:bob:multiaddr}}").into(),
+	];
+
+	let remaining_args: Vec<Arg> = vec![
+		("--").into(),
+		("--reserved-only").into(),
+		("--reserved-nodes", "{{ZOMBIE:validator-0:multiaddr}}").into(),
+	];
+
+	[start_args, in_args, remaining_args].concat()
+}

pezcumulus/zombienet/zombienet-sdk/tests/zombie_ci/rpc_collator_build_blocks.rs

@@ -0,0 +1,162 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+use anyhow::anyhow;
+use std::time::Duration;
+
+use crate::utils::{initialize_network, BEST_BLOCK_METRIC};
+
+use cumulus_zombienet_sdk_helpers::assert_para_throughput;
+use pezkuwi_primitives::Id as ParaId;
+use zombienet_orchestrator::network::node::LogLineCountOptions;
+use zombienet_sdk::{
+	subxt::{OnlineClient, PolkadotConfig},
+	NetworkConfig, NetworkConfigBuilder,
+};
+
+const PARA_ID: u32 = 2000;
+
+// This tests makes sure that RPC collator is able to build blocks
+#[tokio::test(flavor = "multi_thread")]
+async fn rpc_collator_builds_blocks() -> Result<(), anyhow::Error> {
+	let _ = env_logger::try_init_from_env(
+		env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
+	);
+
+	log::info!("Spawning network");
+	let config = build_network_config().await?;
+	let network = initialize_network(config).await?;
+
+	let alice = network.get_node("alice")?;
+	let alice_client: OnlineClient<PolkadotConfig> = alice.wait_client().await?;
+
+	log::info!("Ensuring teyrchain making progress");
+	assert_para_throughput(
+		&alice_client,
+		20,
+		[(ParaId::from(PARA_ID), 2..40)].into_iter().collect(),
+	)
+	.await?;
+
+	let dave = network.get_node("dave")?;
+	let eve = network.get_node("eve")?;
+	for (node, timeout_secs) in [(eve, 250u64), (dave, 250u64)] {
+		log::info!("Ensuring {} reports expected block height", node.name());
+		assert!(node
+			.wait_metric_with_timeout(BEST_BLOCK_METRIC, |b| b >= 12.0, timeout_secs)
+			.await
+			.is_ok());
+	}
+
+	log::info!("Restaring 'one' after 1 second");
+	network.get_node("one")?.restart(Some(Duration::from_secs(1))).await?;
+
+	log::info!("Ensuring dave reports expected block height");
+	let dave = network.get_node("dave")?;
+	assert!(dave
+		.wait_metric_with_timeout(BEST_BLOCK_METRIC, |b| b >= 20.0, 200u64)
+		.await
+		.is_ok());
+
+	log::info!("Restaring 'two' after 2 seconds");
+	network.get_node("two")?.restart(Some(Duration::from_secs(1))).await?;
+
+	log::info!("Restaring 'three' after 20 seconds");
+	network.get_node("three")?.restart(Some(Duration::from_secs(20))).await?;
+
+	log::info!("Checking if dave is up");
+	assert!(dave.wait_until_is_up(10u64).await.is_ok());
+
+	log::info!("Ensuring dave reports expected block height");
+	assert!(dave
+		.wait_metric_with_timeout(BEST_BLOCK_METRIC, |b| b >= 30.0, 200u64)
+		.await
+		.is_ok());
+
+	// We want to make sure that none of the consensus hook checks fail, even if the chain makes
+	// progress. If below log line occurred 1 or more times then test failed.
+	for node in [eve, dave] {
+		log::info!("Ensuring none of the consensus hook checks fail at {}", node.name());
+		let result = node
+			.wait_log_line_count_with_timeout(
+				"set_validation_data inherent needs to be present in every block",
+				false,
+				LogLineCountOptions::no_occurences_within_timeout(Duration::from_secs(10)),
+			)
+			.await?;
+
+		assert!(result.success(), "Consensus hook failed at {}: {:?}", node.name(), result);
+	}
+
+	Ok(())
+}
+
+async fn build_network_config() -> Result<NetworkConfig, anyhow::Error> {
+	// images are not relevant for `native`, but we leave it here in case we use `k8s` some day
+	let images = zombienet_sdk::environment::get_images_from_env();
+	log::info!("Using images: {images:?}");
+
+	// Network setup:
+	// - relaychain nodes:
+	// 	 - alice   - validator
+	// 	 - bob     - validator
+	// 	 - charlie - validator
+	// 	 - one     - full node
+	// 	 - two     - full node
+	// 	 - three   - full node
+	// - teyrchain nodes
+	//   - dave    - validator; gets relay chain data only from full nodes (which are bootnodes too)
+	//   - eve     - validator; gets relay chain data only from full nodes (which are bootnodes too)
+	let config = NetworkConfigBuilder::new()
+		.with_relaychain(|r| {
+			r.with_chain("pezkuwichain-local")
+				.with_default_command("pezkuwi")
+				.with_default_image(images.polkadot.as_str())
+				.with_default_args(vec![("-lteyrchain=debug").into()])
+				.with_node(|node| node.with_name("alice"))
+				.with_node(|node| node.with_name("bob"))
+				.with_node(|node| node.with_name("charlie"))
+				.with_node(|node| node.with_name("one").validator(false))
+				.with_node(|node| node.with_name("two").validator(false))
+				.with_node(|node| node.with_name("three").validator(false))
+		})
+		.with_teyrchain(|p| {
+			p.with_id(PARA_ID)
+				.with_default_command("test-teyrchain")
+				.with_default_image(images.pezcumulus.as_str())
+				.with_default_args(vec![
+					("-lteyrchain=trace,blockchain-rpc-client=debug").into(),
+					(
+						"--bootnodes",
+						vec![
+							"{{ZOMBIE:one:multiaddr}}",
+							"{{ZOMBIE:two:multiaddr}}",
+							"{{ZOMBIE:three:multiaddr}}",
+						],
+					)
+						.into(),
+					(
+						"--relay-chain-rpc-url",
+						vec![
+							"{{ZOMBIE:one:ws_uri}}",
+							"{{ZOMBIE:two:ws_uri}}",
+							"{{ZOMBIE:three:ws_uri}}",
+						],
+					)
+						.into(),
+				])
+				.with_collator(|n| n.with_name("dave"))
+				.with_collator(|n| n.with_name("eve"))
+		})
+		.with_global_settings(|global_settings| match std::env::var("ZOMBIENET_SDK_BASE_DIR") {
+			Ok(val) => global_settings.with_base_dir(val),
+			_ => global_settings,
+		})
+		.build()
+		.map_err(|e| {
+			let errs = e.into_iter().map(|e| e.to_string()).collect::<Vec<_>>().join(" ");
+			anyhow!("config errs: {errs}")
+		})?;
+
+	Ok(config)
+}

pezcumulus/zombienet/zombienet-sdk/tests/zombie_ci/runtime_upgrade.rs

@@ -0,0 +1,128 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+use anyhow::anyhow;
+use std::time::Duration;
+
+use crate::utils::initialize_network;
+
+use cumulus_zombienet_sdk_helpers::{assert_para_throughput, wait_for_upgrade};
+use pezkuwi_primitives::Id as ParaId;
+use zombienet_configuration::types::AssetLocation;
+use zombienet_sdk::{
+	subxt::{OnlineClient, PolkadotConfig},
+	tx_helper::{ChainUpgrade, RuntimeUpgradeOptions},
+	NetworkConfig, NetworkConfigBuilder,
+};
+
+const PARA_ID: u32 = 2000;
+const WASM_WITH_SPEC_VERSION_INCREMENTED: &str =
+	"/tmp/wasm_binary_spec_version_incremented.rs.compact.compressed.wasm";
+
+// This tests makes sure that it is possible to upgrade teyrchain's runtime
+// and teyrchain produces blocks after such upgrade.
+#[tokio::test(flavor = "multi_thread")]
+async fn runtime_upgrade() -> Result<(), anyhow::Error> {
+	let _ = env_logger::try_init_from_env(
+		env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
+	);
+
+	log::info!("Spawning network");
+	let config = build_network_config().await?;
+	let network = initialize_network(config).await?;
+
+	let alice = network.get_node("alice")?;
+	let alice_client: OnlineClient<PolkadotConfig> = alice.wait_client().await?;
+
+	log::info!("Ensuring teyrchain making progress");
+	assert_para_throughput(
+		&alice_client,
+		20,
+		[(ParaId::from(PARA_ID), 2..40)].into_iter().collect(),
+	)
+	.await?;
+
+	let timeout_secs: u64 = 250;
+	let charlie = network.get_node("charlie")?;
+	let charlie_client: OnlineClient<PolkadotConfig> = charlie.wait_client().await?;
+
+	let current_spec_version =
+		charlie_client.backend().current_runtime_version().await?.spec_version;
+	log::info!("Current runtime spec version {current_spec_version}");
+
+	log::info!("Performing runtime upgrade");
+	network
+		.parachain(PARA_ID)
+		.unwrap()
+		.perform_runtime_upgrade(
+			charlie,
+			RuntimeUpgradeOptions::new(AssetLocation::from(WASM_WITH_SPEC_VERSION_INCREMENTED)),
+		)
+		.await?;
+
+	let dave = network.get_node("dave")?;
+	let dave_client: OnlineClient<PolkadotConfig> = dave.wait_client().await?;
+	let expected_spec_version = current_spec_version + 1;
+
+	log::info!(
+		"Waiting (up to {timeout_secs}s) for teyrchain runtime upgrade to version {}",
+		expected_spec_version
+	);
+	tokio::time::timeout(
+		Duration::from_secs(timeout_secs),
+		wait_for_upgrade(dave_client, expected_spec_version),
+	)
+	.await
+	.expect("Timeout waiting for runtime upgrade")?;
+
+	let spec_version_from_charlie =
+		charlie_client.backend().current_runtime_version().await?.spec_version;
+	assert_eq!(expected_spec_version, spec_version_from_charlie, "Unexpected runtime spec version");
+
+	Ok(())
+}
+
+async fn build_network_config() -> Result<NetworkConfig, anyhow::Error> {
+	// images are not relevant for `native`, but we leave it here in case we use `k8s` some day
+	let images = zombienet_sdk::environment::get_images_from_env();
+	log::info!("Using images: {images:?}");
+
+	// Network setup:
+	// - relaychain nodes:
+	// 	 - alice   - validator
+	// 	 - bob     - validator
+	// - teyrchain nodes
+	//   - charlie - validator
+	//   - dave    - full node
+	let config = NetworkConfigBuilder::new()
+		.with_relaychain(|r| {
+			r.with_chain("pezkuwichain-local")
+				.with_default_command("pezkuwi")
+				.with_default_image(images.polkadot.as_str())
+				.with_default_args(vec![("-lteyrchain=debug").into()])
+				.with_node(|node| node.with_name("alice"))
+				.with_node(|node| node.with_name("bob"))
+		})
+		.with_teyrchain(|p| {
+			p.with_id(PARA_ID)
+				.with_default_command("test-teyrchain")
+				.with_default_image(images.pezcumulus.as_str())
+				.with_collator(|n| {
+					n.with_name("charlie")
+						.validator(true)
+						.with_args(vec![("-lteyrchain=debug").into()])
+				})
+				.with_collator(|n| n.with_name("dave").validator(false))
+		})
+		.with_global_settings(|global_settings| match std::env::var("ZOMBIENET_SDK_BASE_DIR") {
+			Ok(val) => global_settings.with_base_dir(val),
+			_ => global_settings,
+		})
+		.build()
+		.map_err(|e| {
+			let errs = e.into_iter().map(|e| e.to_string()).collect::<Vec<_>>().join(" ");
+			anyhow!("config errs: {errs}")
+		})?;
+
+	Ok(config)
+}

pezcumulus/zombienet/zombienet-sdk/tests/zombie_ci/statement_store.rs

@@ -0,0 +1,103 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Test that people-zagros enables the statement store in the node and that statements are
+// propagated to peers.
+
+use anyhow::anyhow;
+use pezsp_core::{Bytes, Encode};
+use zombienet_sdk::{subxt::ext::subxt_rpcs::rpc_params, NetworkConfigBuilder};
+
+#[tokio::test(flavor = "multi_thread")]
+async fn statement_store() -> Result<(), anyhow::Error> {
+	let _ = env_logger::try_init_from_env(
+		env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
+	);
+	// images are not relevant for `native`, but we leave it here in case we use `k8s` some day
+	let images = zombienet_sdk::environment::get_images_from_env();
+
+	let config = NetworkConfigBuilder::new()
+		.with_relaychain(|r| {
+			let r = r
+				.with_chain("zagros-local")
+				.with_default_command("pezkuwi")
+				.with_default_image(images.polkadot.as_str())
+				.with_default_args(vec!["-lteyrchain=debug".into()])
+				// Have to set a `with_node` outside of the loop below, so that `r` has the right
+				// type.
+				.with_node(|node| node.with_name("validator-0"));
+
+			(1..6).fold(r, |acc, i| acc.with_node(|node| node.with_name(&format!("validator-{i}"))))
+		})
+		.with_teyrchain(|p| {
+			p.with_id(2400)
+				.with_default_command("pezkuwi-teyrchain")
+				.with_default_image(images.pezcumulus.as_str())
+				.with_chain("people-zagros-local")
+				.with_default_args(vec![
+					"--force-authoring".into(),
+					"-lteyrchain=debug".into(),
+					"--enable-statement-store".into(),
+				])
+				.with_collator(|n| n.with_name("charlie"))
+				.with_collator(|n| n.with_name("dave"))
+		})
+		.with_global_settings(|global_settings| match std::env::var("ZOMBIENET_SDK_BASE_DIR") {
+			Ok(val) => global_settings.with_base_dir(val),
+			_ => global_settings,
+		})
+		.build()
+		.map_err(|e| {
+			let errs = e.into_iter().map(|e| e.to_string()).collect::<Vec<_>>().join(" ");
+			anyhow!("config errs: {errs}")
+		})?;
+
+	let spawn_fn = zombienet_sdk::environment::get_spawn_fn();
+	let network = spawn_fn(config).await?;
+	assert!(network.wait_until_is_up(60).await.is_ok());
+
+	let charlie = network.get_node("charlie")?;
+	let dave = network.get_node("dave")?;
+
+	let charlie_rpc = charlie.rpc().await?;
+	let dave_rpc = dave.rpc().await?;
+
+	// Create the statement "1,2,3" signed by dave.
+	let mut statement = pezsp_statement_store::Statement::new();
+	statement.set_plain_data(vec![1, 2, 3]);
+	let dave = pezsp_keyring::Sr25519Keyring::Dave;
+	statement.sign_sr25519_private(&dave.pair());
+	let statement: Bytes = statement.encode().into();
+
+	// Submit the statement to charlie.
+	let _: () = charlie_rpc.request("statement_submit", rpc_params![statement.clone()]).await?;
+
+	// Ensure that charlie stored the statement.
+	let charlie_dump: Vec<Bytes> = charlie_rpc.request("statement_dump", rpc_params![]).await?;
+	if charlie_dump != vec![statement.clone()] {
+		return Err(anyhow!("Charlie did not store the statement"));
+	}
+
+	// Query dave until it receives the statement, stop if 20 seconds passed.
+	let query_start_time = std::time::SystemTime::now();
+	let stop_after_secs = 20;
+	loop {
+		let dave_dump: Vec<Bytes> = dave_rpc.request("statement_dump", rpc_params![]).await?;
+		if !dave_dump.is_empty() {
+			if dave_dump != vec![statement.clone()] {
+				return Err(anyhow!("Dave statement store is not the expected one"));
+			}
+			break;
+		}
+
+		let elapsed =
+			query_start_time.elapsed().map_err(|_| anyhow!("Failed to get elapsed time"))?;
+		if elapsed.as_secs() > stop_after_secs {
+			return Err(anyhow!("Dave did not receive the statement in time"));
+		}
+
+		tokio::time::sleep(core::time::Duration::from_secs(1)).await;
+	}
+
+	Ok(())
+}

pezcumulus/zombienet/zombienet-sdk/tests/zombie_ci/statement_store_bench.rs

@@ -0,0 +1,764 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Benchmarking statement store performance
+
+use anyhow::anyhow;
+use codec::{Decode, Encode};
+use log::{debug, info, trace};
+use pezsc_statement_store::{DEFAULT_MAX_TOTAL_SIZE, DEFAULT_MAX_TOTAL_STATEMENTS};
+use pezsp_core::{blake2_256, sr25519, Bytes, Pair};
+use pezsp_statement_store::{Channel, Statement, Topic};
+use std::{cell::Cell, collections::HashMap, time::Duration};
+use tokio::time::timeout;
+use zombienet_sdk::{
+	subxt::{backend::rpc::RpcClient, ext::subxt_rpcs::rpc_params},
+	LocalFileSystem, Network, NetworkConfigBuilder,
+};
+
+const GROUP_SIZE: u32 = 6;
+const PARTICIPANT_SIZE: u32 = GROUP_SIZE * 8333; // Target ~50,000 total
+const MESSAGE_SIZE: usize = 512;
+const MESSAGE_COUNT: usize = 1;
+const MAX_RETRIES: u32 = 100;
+const RETRY_DELAY_MS: u64 = 500;
+const PROPAGATION_DELAY_MS: u64 = 2000;
+const TIMEOUT_MS: u64 = 3000;
+
+/// Single-node benchmark.
+///
+/// Tests statement-store performance of a single node without relying on statement gossip between
+/// nodes. Measures the maximum speed of statement exchange. Network spawned with only 2 collator
+/// nodes. All participants connect to one collator node, allowing them to fetch new statements
+/// immediately without waiting for propagation.
+///
+/// # Output
+/// Logs aggregated statistics across all participants:
+/// - Average messages sent/received per participant
+/// - Execution time (min/avg/max) across all participants
+/// - Retry attempts (min/avg/max) needed for statement propagation
+#[tokio::test(flavor = "multi_thread")]
+async fn statement_store_one_node_bench() -> Result<(), anyhow::Error> {
+	let _ = env_logger::try_init_from_env(
+		env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
+	);
+
+	let collator_names = ["alice", "bob"];
+	let network = spawn_network(&collator_names).await?;
+
+	info!("Starting statement store benchmark with {} participants", PARTICIPANT_SIZE);
+
+	let target_node = collator_names[0];
+	let node = network.get_node(target_node)?;
+	let rpc_client = node.rpc().await?;
+	info!("Created single RPC client for target node: {}", target_node);
+
+	let mut participants = Vec::with_capacity(PARTICIPANT_SIZE as usize);
+	for i in 0..(PARTICIPANT_SIZE) as usize {
+		participants.push(Participant::new(i as u32, rpc_client.clone()));
+	}
+
+	let handles: Vec<_> = participants
+		.into_iter()
+		.map(|mut p| tokio::spawn(async move { p.run().await }))
+		.collect();
+
+	let mut all_stats = Vec::new();
+	for handle in handles {
+		let stats = handle.await??;
+		all_stats.push(stats);
+	}
+
+	let aggregated_stats = ParticipantStats::aggregate(all_stats);
+	aggregated_stats.log_summary();
+
+	Ok(())
+}
+
+/// Multi-node benchmark.
+///
+/// Tests statement store performance with participants distributed across multiple collator nodes.
+/// No two participants in a group are connected to the same node, so statements must be propagated
+/// to other nodes before being received. Network spawned with 6 collator nodes. Each participant in
+/// a group connects to a different collator node
+///
+/// # Output
+/// Logs aggregated statistics across all participants:
+/// - Average messages sent/received per participant
+/// - Execution time (min/avg/max) across all participants
+/// - Retry attempts (min/avg/max) needed for statement propagation
+#[tokio::test(flavor = "multi_thread")]
+async fn statement_store_many_nodes_bench() -> Result<(), anyhow::Error> {
+	let _ = env_logger::try_init_from_env(
+		env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
+	);
+
+	let collator_names = ["alice", "bob", "charlie", "dave", "eve", "ferdie"];
+	let network = spawn_network(&collator_names).await?;
+
+	info!("Starting statement store benchmark with {} participants", PARTICIPANT_SIZE);
+
+	let mut rpc_clients = Vec::new();
+	for &name in &collator_names {
+		let node = network.get_node(name)?;
+		let rpc_client = node.rpc().await?;
+		rpc_clients.push(rpc_client);
+	}
+	info!("Created RPC clients for {} collator nodes", rpc_clients.len());
+
+	let mut participants = Vec::with_capacity(PARTICIPANT_SIZE as usize);
+	for i in 0..(PARTICIPANT_SIZE) as usize {
+		let client_idx = i % collator_names.len();
+		participants.push(Participant::new(i as u32, rpc_clients[client_idx].clone()));
+	}
+	info!(
+		"{} participants were distributed across {} nodes: {} participants per node",
+		PARTICIPANT_SIZE,
+		collator_names.len(),
+		PARTICIPANT_SIZE as usize / collator_names.len()
+	);
+
+	let handles: Vec<_> = participants
+		.into_iter()
+		.map(|mut participant| tokio::spawn(async move { participant.run().await }))
+		.collect();
+
+	let mut all_stats = Vec::new();
+	for handle in handles {
+		let stats = handle.await??;
+		all_stats.push(stats);
+	}
+
+	let aggregated_stats = ParticipantStats::aggregate(all_stats);
+	aggregated_stats.log_summary();
+
+	Ok(())
+}
+
+/// Memory stress benchmark.
+///
+/// Tests statement store memory usage under extreme load. Network spawned with 6 collator nodes.
+/// Concurrent tasks send statements to a single target node until the store is full. The test ends
+/// when all statements are propagated.
+///
+/// # Output
+/// Logs real-time metrics every 5 seconds with the following data per node:
+/// - Submitted statements: total count, percentage of capacity, submission rate
+/// - Propagated statements: total count, percentage of propagation capacity, propagation rate
+/// - Elapsed time since test start
+/// - Final completion status when submit capacity is reached across all nodes
+#[tokio::test(flavor = "multi_thread")]
+async fn statement_store_memory_stress_bench() -> Result<(), anyhow::Error> {
+	let _ = env_logger::try_init_from_env(
+		env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
+	);
+
+	let collator_names = ["alice", "bob", "charlie", "dave", "eve", "ferdie"];
+	let network = spawn_network(&collator_names).await?;
+
+	let target_node = collator_names[0];
+	let node = network.get_node(target_node)?;
+	let rpc_client = node.rpc().await?;
+	info!("Created single RPC client for target node: {}", target_node);
+
+	let total_tasks = 64 * 1024;
+	let payload_size = 1024;
+	let submit_capacity =
+		DEFAULT_MAX_TOTAL_STATEMENTS.min(DEFAULT_MAX_TOTAL_SIZE / payload_size) as u64;
+	let statements_per_task = submit_capacity / total_tasks as u64;
+	let num_collators = collator_names.len() as u64;
+	let propogation_capacity = submit_capacity * (num_collators - 1); // 5x per node
+	let start_time = std::time::Instant::now();
+
+	info!("Starting memory stress benchmark with {} tasks, each submitting {} statements of {}B payload, total submit capacity per node: {}, total propagation capacity: {}",
+		total_tasks, statements_per_task, payload_size, submit_capacity, propogation_capacity);
+
+	for _ in 0..total_tasks {
+		let rpc_client = rpc_client.clone();
+		tokio::spawn(async move {
+			let (keyring, _) = sr25519::Pair::generate();
+			let public = keyring.public().0;
+
+			for statement_count in 0..statements_per_task {
+				let mut statement = Statement::new();
+				let topic =
+					|idx: usize| blake2_256(format!("{idx}{statement_count}{public:?}").as_bytes());
+				statement.set_topic(0, topic(0));
+				statement.set_topic(1, topic(1));
+				statement.set_topic(2, topic(2));
+				statement.set_topic(3, topic(3));
+				statement.set_plain_data(vec![0u8; payload_size]);
+				statement.sign_sr25519_private(&keyring);
+
+				loop {
+					let statement_bytes: Bytes = statement.encode().into();
+					let Err(err) = rpc_client
+						.request::<()>("statement_submit", rpc_params![statement_bytes])
+						.await
+					else {
+						break; // Successfully submitted
+					};
+
+					if err.to_string().contains("Statement store error: Store is full") {
+						info!("Statement store is full, {}/{} statements submitted, `statements_per_task` overestimated", statement_count, statements_per_task);
+						break;
+					}
+
+					info!(
+						"Failed to submit statement, retrying in {}ms: {:?}",
+						RETRY_DELAY_MS, err
+					);
+					tokio::time::sleep(Duration::from_millis(RETRY_DELAY_MS)).await;
+				}
+			}
+		});
+	}
+
+	info!("All {} tasks spawned in {:.2}s", total_tasks, start_time.elapsed().as_secs_f64());
+
+	let mut prev_submitted: HashMap<&str, u64> = HashMap::new();
+	let mut prev_propagated: HashMap<&str, u64> = HashMap::new();
+	for &name in &collator_names {
+		prev_submitted.insert(name, 0);
+		prev_propagated.insert(name, 0);
+	}
+
+	loop {
+		let interval = 5;
+		tokio::time::sleep(Duration::from_secs(interval)).await;
+		let elapsed = start_time.elapsed().as_secs();
+
+		// Collect submitted metrics
+		let mut submitted_metrics = Vec::new();
+		for &name in &collator_names {
+			let node = network.get_node(name)?;
+			let prev_count = prev_submitted.get(name).copied().unwrap_or(0);
+
+			let current_count = Cell::new(0.0f64);
+			node.wait_metric_with_timeout(
+				"bizinikiwi_sub_statement_store_submitted_statements",
+				|count| {
+					current_count.set(count);
+					true
+				},
+				30u64,
+			)
+			.await?;
+
+			let count = current_count.get() as u64;
+			let delta = count - prev_count;
+			let rate = delta / interval;
+			submitted_metrics.push((name, count, rate));
+			prev_submitted.insert(name, count);
+		}
+
+		// Collect propagated metrics
+		let mut propagated_metrics = Vec::new();
+		for &name in &collator_names {
+			let node = network.get_node(name)?;
+			let prev_count = prev_propagated.get(name).copied().unwrap_or(0);
+
+			let current_count = Cell::new(0.0f64);
+			node.wait_metric_with_timeout(
+				"bizinikiwi_sync_propagated_statements",
+				|count| {
+					current_count.set(count);
+					true
+				},
+				30u64,
+			)
+			.await?;
+
+			let count = current_count.get() as u64;
+			let delta = count - prev_count;
+			let rate = delta / interval;
+			propagated_metrics.push((name, count, rate));
+			prev_propagated.insert(name, count);
+		}
+
+		info!("[{:>3}s]  Statements  submitted                 propagated", elapsed);
+		for i in 0..collator_names.len() {
+			let (sub_name, sub_count, sub_rate) = submitted_metrics[i];
+			let (prop_name, prop_count, prop_rate) = propagated_metrics[i];
+			assert_eq!(sub_name, prop_name);
+
+			let sub_percentage = sub_count * 100 / submit_capacity;
+			let prop_percentage = prop_count * 100 / propogation_capacity;
+
+			info!(
+				"         {:<8}  {:>8} {:>3}% {:>8}/s   {:>8} {:>3}% {:>8}/s",
+				sub_name,
+				sub_count,
+				sub_percentage,
+				sub_rate,
+				prop_count,
+				prop_percentage,
+				prop_rate
+			);
+		}
+
+		let total_submitted: u64 = submitted_metrics.iter().map(|(_, count, _)| *count).sum();
+		if total_submitted == submit_capacity * num_collators {
+			info!("Reached total submit capacity of {} statements per node in {}s, benchmark completed successfully", submit_capacity, elapsed);
+			break;
+		}
+	}
+
+	Ok(())
+}
+
+/// Spawns a network using a custom chain spec (people-pezkuwichain-spec.json) which validates any
+/// signed statement in the statement-store without additional verification.
+async fn spawn_network(collators: &[&str]) -> Result<Network<LocalFileSystem>, anyhow::Error> {
+	assert!(collators.len() >= 2);
+	let images = zombienet_sdk::environment::get_images_from_env();
+	let config = NetworkConfigBuilder::new()
+		.with_relaychain(|r| {
+			r.with_chain("pezkuwichain-local")
+				.with_default_command("pezkuwi")
+				.with_default_image(images.polkadot.as_str())
+				.with_default_args(vec!["-lteyrchain=debug".into()])
+				.with_node(|node| node.with_name("validator-0"))
+				.with_node(|node| node.with_name("validator-1"))
+		})
+		.with_teyrchain(|p| {
+			let p = p
+				.with_id(2400)
+				.with_default_command("pezkuwi-teyrchain")
+				.with_default_image(images.pezcumulus.as_str())
+				.with_chain_spec_path("tests/zombie_ci/people-pezkuwichain-spec.json")
+				.with_default_args(vec![
+					"--force-authoring".into(),
+					"-lstatement-store=info,statement-gossip=info,error".into(),
+					"--enable-statement-store".into(),
+					"--rpc-max-connections=50000".into(),
+				])
+				// Have to set outside of the loop below, so that `p` has the right type.
+				.with_collator(|n| n.with_name(collators[0]));
+
+			collators[1..]
+				.iter()
+				.fold(p, |acc, &name| acc.with_collator(|n| n.with_name(name)))
+		})
+		.with_global_settings(|global_settings| match std::env::var("ZOMBIENET_SDK_BASE_DIR") {
+			Ok(val) => global_settings.with_base_dir(val),
+			_ => global_settings,
+		})
+		.build()
+		.map_err(|e| {
+			let errs = e.into_iter().map(|e| e.to_string()).collect::<Vec<_>>().join(" ");
+			anyhow!("config errs: {errs}")
+		})?;
+
+	let spawn_fn = zombienet_sdk::environment::get_spawn_fn();
+	let network = spawn_fn(config).await?;
+	assert!(network.wait_until_is_up(60).await.is_ok());
+
+	Ok(network)
+}
+
+#[derive(Encode, Decode, Debug, Clone)]
+struct StatementMessage {
+	message_id: u32,
+	data: Vec<u8>,
+}
+
+#[derive(Debug, Clone)]
+struct ParticipantStats {
+	total_time: Duration,
+	sent_count: u32,
+	received_count: u32,
+	retry_count: u32,
+}
+
+#[derive(Debug)]
+struct AggregatedStats {
+	participants: u32,
+	sent: u32,
+	received: u32,
+	min_time: Duration,
+	max_time: Duration,
+	avg_time: Duration,
+	min_retries: u32,
+	max_retries: u32,
+	avg_retries: u32,
+}
+
+impl ParticipantStats {
+	fn aggregate(all_stats: Vec<ParticipantStats>) -> AggregatedStats {
+		let participants = all_stats.len() as u32;
+		let sent = all_stats.iter().map(|s| s.sent_count).sum::<u32>() / participants;
+		let received = all_stats.iter().map(|s| s.received_count).sum::<u32>() / participants;
+
+		let min_time = all_stats.iter().map(|s| s.total_time).min().unwrap_or(Duration::ZERO);
+		let max_time = all_stats.iter().map(|s| s.total_time).max().unwrap_or(Duration::ZERO);
+		let avg_time = Duration::from_secs_f64(
+			all_stats.iter().map(|s| s.total_time.as_secs_f64()).sum::<f64>() / participants as f64,
+		);
+
+		let min_retries = all_stats.iter().map(|s| s.retry_count).min().unwrap_or(0);
+		let max_retries = all_stats.iter().map(|s| s.retry_count).max().unwrap_or(0);
+		let avg_retries = all_stats.iter().map(|s| s.retry_count).sum::<u32>() / participants;
+
+		AggregatedStats {
+			participants,
+			sent,
+			received,
+			min_time,
+			max_time,
+			avg_time,
+			min_retries,
+			max_retries,
+			avg_retries,
+		}
+	}
+}
+
+impl AggregatedStats {
+	fn log_summary(&self) {
+		info!("Statement store benchmark completed with {} participants", self.participants);
+		info!(
+			"Participants: {}, each sent: {}, received: {}",
+			self.participants, self.sent, self.received
+		);
+		info!("Summary        min       avg       max");
+		info!(
+			" {:<8} {:>8}  {:>8}  {:>8}",
+			"time, s",
+			self.min_time.as_secs(),
+			self.avg_time.as_secs(),
+			self.max_time.as_secs(),
+		);
+		info!(
+			" {:<8} {:>8}  {:>8}  {:>8}",
+			"retries", self.min_retries, self.avg_retries, self.max_retries
+		);
+	}
+}
+
+struct Participant {
+	idx: u32,
+	keyring: sr25519::Pair,
+	session_key: sr25519::Pair,
+	group_members: Vec<u32>,
+	session_keys: HashMap<u32, sr25519::Public>,
+	sent_messages: HashMap<(u32, u32), bool>,
+	received_messages: HashMap<(u32, u32), bool>,
+	sent_count: u32,
+	received_count: u32,
+	pending_messages: HashMap<u32, Option<u32>>,
+	retry_count: u32,
+	rpc_client: RpcClient,
+}
+
+impl Participant {
+	fn log_target(&self) -> String {
+		format!("participant_{}", self.idx)
+	}
+
+	fn new(idx: u32, rpc_client: RpcClient) -> Self {
+		debug!(target: &format!("participant_{idx}"), "Initializing participant {}", idx);
+		let (keyring, _) = sr25519::Pair::generate();
+		let (session_key, _) = sr25519::Pair::generate();
+
+		let group_start = (idx / GROUP_SIZE) * GROUP_SIZE;
+		let group_end = group_start + GROUP_SIZE;
+		let group_members: Vec<u32> = (group_start..group_end).filter(|&i| i != idx).collect();
+
+		Self {
+			keyring,
+			session_key,
+			idx,
+			group_members,
+			session_keys: HashMap::new(),
+			sent_messages: HashMap::new(),
+			received_messages: HashMap::new(),
+			pending_messages: HashMap::new(),
+			sent_count: 0,
+			received_count: 0,
+			retry_count: 0,
+			rpc_client,
+		}
+	}
+
+	async fn wait_for_retry(&mut self) -> Result<(), anyhow::Error> {
+		if self.retry_count >= MAX_RETRIES {
+			return Err(anyhow!("No more retry attempts for participant {}", self.idx));
+		}
+
+		self.retry_count += 1;
+		if self.retry_count % 10 == 0 {
+			debug!(target: &self.log_target(), "Retry attempt {}", self.retry_count);
+		}
+		tokio::time::sleep(tokio::time::Duration::from_millis(RETRY_DELAY_MS)).await;
+
+		Ok(())
+	}
+
+	async fn wait_for_propagation(&mut self) {
+		trace!(target: &self.log_target(), "Waiting {}ms for propagation", PROPAGATION_DELAY_MS);
+		tokio::time::sleep(tokio::time::Duration::from_millis(PROPAGATION_DELAY_MS)).await;
+	}
+
+	async fn statement_submit(&mut self, statement: Statement) -> Result<(), anyhow::Error> {
+		let statement_bytes: Bytes = statement.encode().into();
+		let _: () = self
+			.rpc_client
+			.request("statement_submit", rpc_params![statement_bytes])
+			.await?;
+
+		self.sent_count += 1;
+		trace!(target: &self.log_target(), "Submitted statement (counter: {})", self.sent_count);
+
+		Ok(())
+	}
+
+	async fn statement_broadcasts_statement(
+		&mut self,
+		topics: Vec<Topic>,
+	) -> Result<Vec<Statement>, anyhow::Error> {
+		let statements: Vec<Bytes> = self
+			.rpc_client
+			.request("statement_broadcastsStatement", rpc_params![topics])
+			.await?;
+
+		let mut decoded_statements = Vec::new();
+		for statement_bytes in &statements {
+			let statement = Statement::decode(&mut &statement_bytes[..])?;
+			decoded_statements.push(statement);
+		}
+
+		self.received_count += decoded_statements.len() as u32;
+		trace!(target: &self.log_target(), "Received {} statements (counter: {})", decoded_statements.len(), self.received_count);
+
+		Ok(decoded_statements)
+	}
+
+	fn create_session_key_statement(&self) -> Statement {
+		let mut statement = Statement::new();
+		statement.set_channel(channel_public_key());
+		statement.set_priority(self.sent_count);
+		statement.set_topic(0, topic_public_key());
+		statement.set_topic(1, topic_idx(self.idx));
+		statement.set_plain_data(self.session_key.public().to_vec());
+		statement.sign_sr25519_private(&self.keyring);
+
+		statement
+	}
+
+	fn create_message_statement(&mut self, receiver_idx: u32) -> Option<(Statement, u32)> {
+		let receiver_session_key = self.session_keys.get(&receiver_idx)?;
+
+		let message_id = self.sent_count;
+		let mut data = vec![0u8; MESSAGE_SIZE];
+		for (i, byte) in data.iter_mut().enumerate() {
+			*byte = (i % 256) as u8; // Simple pattern for testing
+		}
+
+		let request = StatementMessage { message_id, data };
+		let request_data = request.encode();
+		let mut statement = Statement::new();
+
+		let topic0 = topic_message();
+		let topic1 = topic_pair(&self.session_key.public(), receiver_session_key);
+		let channel = channel_message(&self.session_key.public(), receiver_session_key);
+
+		statement.set_topic(0, topic0);
+		statement.set_topic(1, topic1);
+		statement.set_channel(channel);
+		statement.set_priority(self.sent_count);
+		statement.set_plain_data(request_data);
+		statement.sign_sr25519_private(&self.keyring);
+
+		Some((statement, message_id))
+	}
+
+	async fn send_session_key(&mut self) -> Result<(), anyhow::Error> {
+		let statement = self.create_session_key_statement();
+		self.statement_submit(statement).await
+	}
+
+	async fn receive_session_keys(&mut self) -> Result<(), anyhow::Error> {
+		let mut pending = self.group_members.clone();
+
+		trace!(target: &self.log_target(), "Pending session keys to receive: {:?}", pending.len());
+		loop {
+			let mut completed_this_round = Vec::new();
+			for &idx in &pending {
+				match timeout(
+					Duration::from_millis(TIMEOUT_MS),
+					self.statement_broadcasts_statement(vec![topic_public_key(), topic_idx(idx)]),
+				)
+				.await
+				{
+					Ok(Ok(statements)) if !statements.is_empty() => {
+						if let Some(statement) = statements.first() {
+							let data = statement.data().expect("Must contain session_key");
+							let session_key = sr25519::Public::from_raw(data[..].try_into()?);
+							self.session_keys.insert(idx, session_key);
+							completed_this_round.push(idx);
+						}
+					},
+					res => {
+						debug!(target: &self.log_target(), "No statements received for idx {:?}: {:?}", idx, res);
+					},
+				}
+			}
+
+			pending.retain(|x| !completed_this_round.contains(x));
+			if pending.is_empty() {
+				break;
+			}
+			trace!(target: &self.log_target(), "Session keys left to receive: {:?}, waiting {}ms for retry", pending.len(), RETRY_DELAY_MS);
+			self.wait_for_retry().await?;
+		}
+
+		assert_eq!(
+			self.session_keys.len(),
+			self.group_members.len(),
+			"Not every session key received"
+		);
+
+		Ok(())
+	}
+
+	async fn send_messages(&mut self, round: usize) -> Result<(), anyhow::Error> {
+		let group_members = self.group_members.clone();
+		for receiver_idx in group_members {
+			let (statement, request_id) =
+				self.create_message_statement(receiver_idx).expect("Receiver must present");
+			self.statement_submit(statement).await?;
+			self.sent_messages.insert((receiver_idx, request_id), false);
+		}
+
+		assert_eq!(
+			self.sent_messages.len(),
+			self.group_members.len() * (round + 1),
+			"Not every request sent"
+		);
+
+		Ok(())
+	}
+
+	async fn receive_messages(&mut self, round: usize) -> Result<(), anyhow::Error> {
+		let mut pending: Vec<(u32, sr25519::Public)> =
+			self.session_keys.iter().map(|(&idx, &key)| (idx, key)).collect();
+		let own_session_key = self.session_key.public();
+
+		trace!(target: &self.log_target(), "Pending messages to receive: {:?}", pending.len());
+		loop {
+			let mut completed_this_round = Vec::new();
+			for &(sender_idx, sender_session_key) in &pending {
+				match timeout(
+					Duration::from_millis(TIMEOUT_MS),
+					self.statement_broadcasts_statement(vec![
+						topic_message(),
+						topic_pair(&sender_session_key, &own_session_key),
+					]),
+				)
+				.await
+				{
+					Ok(Ok(statements)) if !statements.is_empty() => {
+						if let Some(statement) = statements.first() {
+							let data = statement.data().expect("Must contain request");
+							let req = StatementMessage::decode(&mut &data[..])?;
+
+							if let std::collections::hash_map::Entry::Vacant(e) =
+								self.received_messages.entry((sender_idx, req.message_id))
+							{
+								e.insert(false);
+								self.pending_messages.insert(sender_idx, Some(req.message_id));
+								completed_this_round.push((sender_idx, sender_session_key));
+							}
+						}
+					},
+					res => {
+						debug!(target: &self.log_target(), "No statements received for sender {:?}: {:?}", sender_idx, res);
+					},
+				}
+			}
+
+			pending.retain(|x| !completed_this_round.contains(x));
+			if pending.is_empty() {
+				break;
+			}
+			trace!(target: &self.log_target(), "Messages left to receive: {:?}, waiting {}ms for retry", pending.len(), RETRY_DELAY_MS);
+			self.wait_for_retry().await?;
+		}
+
+		assert_eq!(
+			self.received_messages.len(),
+			self.group_members.len() * (round + 1),
+			"Not every request received"
+		);
+		assert_eq!(
+			self.pending_messages.values().filter(|i| i.is_some()).count(),
+			self.group_members.len(),
+			"Not every request received"
+		);
+
+		Ok(())
+	}
+
+	async fn run(&mut self) -> Result<ParticipantStats, anyhow::Error> {
+		let start_time = std::time::Instant::now();
+
+		debug!(target: &self.log_target(), "Session keys exchange");
+		self.send_session_key().await?;
+		trace!(target: &self.log_target(), "Session keys sent");
+		self.wait_for_propagation().await;
+		trace!(target: &self.log_target(), "Session keys requests started");
+		self.receive_session_keys().await?;
+		trace!(target: &self.log_target(), "Session keys received");
+
+		for round in 0..MESSAGE_COUNT {
+			debug!(target: &self.log_target(), "Messages exchange, round {}", round + 1);
+			self.send_messages(round).await?;
+			trace!(target: &self.log_target(), "Messages sent");
+			self.wait_for_propagation().await;
+			trace!(target: &self.log_target(), "Messages requests started");
+			self.receive_messages(round).await?;
+			trace!(target: &self.log_target(), "Messages received");
+		}
+
+		let elapsed = start_time.elapsed();
+
+		Ok(ParticipantStats {
+			total_time: elapsed,
+			sent_count: self.sent_count,
+			received_count: self.received_count,
+			retry_count: self.retry_count,
+		})
+	}
+}
+
+fn topic_public_key() -> Topic {
+	blake2_256(b"public key")
+}
+
+fn topic_idx(idx: u32) -> Topic {
+	blake2_256(&idx.to_le_bytes())
+}
+
+fn topic_pair(sender: &sr25519::Public, receiver: &sr25519::Public) -> Topic {
+	let mut data = Vec::new();
+	data.extend_from_slice(sender.as_ref());
+	data.extend_from_slice(receiver.as_ref());
+	blake2_256(&data)
+}
+
+fn topic_message() -> Topic {
+	blake2_256(b"message")
+}
+
+fn channel_public_key() -> Channel {
+	[0u8; 32]
+}
+
+fn channel_message(sender: &sr25519::Public, receiver: &sr25519::Public) -> Channel {
+	let mut data = Vec::new();
+	data.extend_from_slice(b"message");
+	data.extend_from_slice(sender.as_ref());
+	data.extend_from_slice(receiver.as_ref());
+	blake2_256(&data)
+}

pezcumulus/zombienet/zombienet-sdk/tests/zombie_ci/sync_blocks.rs

@@ -0,0 +1,121 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+use anyhow::anyhow;
+
+use crate::utils::{initialize_network, BEST_BLOCK_METRIC};
+
+use cumulus_zombienet_sdk_helpers::assert_para_throughput;
+use pezkuwi_primitives::Id as ParaId;
+use zombienet_sdk::{
+	subxt::{OnlineClient, PolkadotConfig},
+	NetworkConfig, NetworkConfigBuilder,
+};
+
+const PARA_ID: u32 = 2000;
+
+#[tokio::test(flavor = "multi_thread")]
+async fn sync_blocks_from_tip_without_connected_collator() -> Result<(), anyhow::Error> {
+	let _ = env_logger::try_init_from_env(
+		env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
+	);
+	log::info!("Spawning network");
+	let config = build_network_config().await?;
+	let network = initialize_network(config).await?;
+
+	let relay_alice = network.get_node("alice")?;
+
+	let relay_client: OnlineClient<PolkadotConfig> = relay_alice.wait_client().await?;
+
+	log::info!("Ensuring teyrchain making progress");
+	assert_para_throughput(
+		&relay_client,
+		10,
+		[(ParaId::from(PARA_ID), 5..11)].into_iter().collect(),
+	)
+	.await?;
+
+	let para_ferdie = network.get_node("ferdie")?;
+	let para_eve = network.get_node("eve")?;
+
+	log::info!("Ensuring eve is connected to 1 peer only");
+	assert!(para_eve.assert("sub_libp2p_peers_count", 1).await?);
+
+	log::info!("Ensuring ferdie is connected to 1 peer only");
+	assert!(para_ferdie.assert("sub_libp2p_peers_count", 1).await?);
+
+	log::info!("Ensuring ferdie reports expected block height");
+	assert!(para_ferdie
+		.wait_metric_with_timeout(BEST_BLOCK_METRIC, |b| b > 12.0, 250u64)
+		.await
+		.is_ok());
+	log::info!("Ensuring eve reports expected block height");
+	assert!(para_eve
+		.wait_metric_with_timeout(BEST_BLOCK_METRIC, |b| b > 12.0, 250u64)
+		.await
+		.is_ok());
+
+	Ok(())
+}
+
+async fn build_network_config() -> Result<NetworkConfig, anyhow::Error> {
+	// images are not relevant for `native`, but we leave it here in case we use `k8s` some day
+	let images = zombienet_sdk::environment::get_images_from_env();
+	log::info!("Using images: {images:?}");
+
+	// Network setup:
+	// - relaychain Nodes:
+	// 	 - alice
+	// 	 - bob
+	// - teyrchain Nodes:
+	//   - charlie - collator
+	//   - dave    - full node
+	//   - eve     - full node; connected only to dave,
+	//   - ferdie  - full node; connected only to dave; gets relay chain data only from alice
+	let config = NetworkConfigBuilder::new()
+		.with_relaychain(|r| {
+			r.with_chain("pezkuwichain-local")
+				.with_default_command("pezkuwi")
+				.with_default_image(images.polkadot.as_str())
+				.with_default_args(vec![("-lteyrchain=debug").into()])
+				.with_default_resources(|resources| {
+					// These settings are applicable only for `k8s` provider.
+					// Leaving them in case we switch to `k8s` some day.
+					resources.with_request_cpu(2).with_request_memory("2G")
+				})
+				.with_node(|node| node.with_name("alice"))
+				.with_node(|node| node.with_name("bob"))
+		})
+		.with_teyrchain(|p| {
+			p.with_id(PARA_ID)
+				.with_default_command("test-teyrchain")
+				.with_default_image(images.pezcumulus.as_str())
+				.with_default_args(vec![("-lteyrchain=debug").into()])
+				.with_collator(|n| n.with_name("dave").validator(false))
+				.with_collator(|n| n.with_name("charlie").validator(true))
+				.with_collator(|n| {
+					n.with_name("eve").validator(false).with_args(vec![
+						"--reserved-only".into(),
+						("--reserved-nodes", "{{ZOMBIE:dave:multiAddress}}").into(),
+					])
+				})
+				.with_collator(|n| {
+					n.with_name("ferdie").validator(false).with_args(vec![
+						"--reserved-only".into(),
+						("--reserved-nodes", "{{ZOMBIE:dave:multiaddr}}").into(),
+						("--relay-chain-rpc-url", "{{ZOMBIE:alice:ws_uri}}").into(),
+					])
+				})
+		})
+		.with_global_settings(|global_settings| match std::env::var("ZOMBIENET_SDK_BASE_DIR") {
+			Ok(val) => global_settings.with_base_dir(val),
+			_ => global_settings,
+		})
+		.build()
+		.map_err(|e| {
+			let errs = e.into_iter().map(|e| e.to_string()).collect::<Vec<_>>().join(" ");
+			anyhow!("config errs: {errs}")
+		})?;
+
+	Ok(config)
+}

pezcumulus/zombienet/zombienet-sdk/tests/zombie_ci/teyrchain_extrinsic_get_finalized.rs

@@ -0,0 +1,170 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+use anyhow::anyhow;
+use tokio::time::Duration;
+
+use crate::utils::{initialize_network, BEST_BLOCK_METRIC};
+use cumulus_zombienet_sdk_helpers::submit_extrinsic_and_wait_for_finalization_success_with_timeout;
+use zombienet_orchestrator::network::node::{LogLineCount, LogLineCountOptions};
+use zombienet_sdk::{
+	subxt::{self, dynamic::Value, OnlineClient, PolkadotConfig},
+	subxt_signer::sr25519::dev,
+	NetworkConfig, NetworkConfigBuilder,
+};
+
+const PARA_ID: u32 = 2000;
+
+#[tokio::test(flavor = "multi_thread")]
+async fn teyrchain_extrinsic_gets_finalized() -> Result<(), anyhow::Error> {
+	log::info!("Spawning network");
+	let config = build_network_config().await?;
+	let network = initialize_network(config).await?;
+
+	let alice = network.get_node("alice")?;
+	let bob = network.get_node("bob")?;
+	let charlie = network.get_node("charlie")?;
+
+	for node in [alice, bob, charlie] {
+		log::info!("Ensuring {} reports 4 node roles", node.name());
+		assert!(node.wait_metric_with_timeout("node_roles", |p| p == 4.0, 60u64).await.is_ok());
+	}
+
+	for node in [alice, bob] {
+		log::info!("Ensuring {} is connected to at least 1 peer", node.name());
+		assert!(node
+			.wait_metric_with_timeout("sub_libp2p_peers_count", |p| p >= 1.0, 60u64)
+			.await
+			.is_ok());
+	}
+
+	for (node, block_height, timeout) in
+		[(alice, 5.0, 60u64), (bob, 5.0, 60u64), (charlie, 2.0, 120u64)]
+	{
+		log::info!("Ensuring all nodes report expected block height");
+		assert!(node
+			.wait_metric_with_timeout(BEST_BLOCK_METRIC, |b| b > block_height, timeout)
+			.await
+			.is_ok());
+	}
+
+	for node in [alice, bob, charlie] {
+		log::info!("Ensuring {} does not report any error", node.name());
+
+		let result = node
+			.wait_log_line_count_with_timeout(
+				".*kernel security feature.*not available.*error.",
+				false,
+				// Do not wait for more logs. The lines we are looking for appear at the node
+				// startup, which we must have clearly reached if we are here.
+				LogLineCountOptions::new(|n| n == 0, Duration::from_secs(0), false),
+			)
+			.await?;
+
+		// If above line appeared then increase expected error count
+		let mut error_cnt_expected = if result.success() { 0 } else { 1 };
+
+		let result = node
+			.wait_log_line_count_with_timeout(
+				r"error=Pool\(InvalidTransaction\(InvalidTransaction::MandatoryValidation\)\)",
+				false,
+				LogLineCountOptions::new(move |n| n > 0, Duration::from_secs(2), true),
+			)
+			.await?;
+
+		// We want to search for lines containing 'error' but we need to ignore below line (which
+		// occasionally appears):
+		//   error=Pool(InvalidTransaction(InvalidTransaction::MandatoryValidation))
+		// In a perfect world, we could use a regex with look-around, e.g.
+		//  (?!.*error=Pool(InvalidTransaction(InvalidTransaction::MandatoryValidation))).*error.*
+		// but unfortunately, `regex` crate used by zombienet-sdk does not support look-arounds.
+		//
+		// Therefore, we will do as follows:
+		// - First, search for lines containing:
+		//   error=Pool(InvalidTransaction(InvalidTransaction::MandatoryValidation))
+		// - Add the number of such occurrences to the expected number of 'error' lines
+		// - Then, search for lines containing: error
+		if let LogLineCount::TargetReached(cnt) = result {
+			error_cnt_expected += cnt;
+		}
+
+		let result = node
+			.wait_log_line_count_with_timeout(
+				"error",
+				false,
+				// Do not wait here, we already waited in previous step
+				LogLineCountOptions::new(
+					move |n| n == error_cnt_expected,
+					Duration::from_secs(0),
+					false,
+				),
+			)
+			.await?;
+		assert!(result.success(), "node {} reported error: {:?}", node.name(), result);
+	}
+
+	log::info!("Ensuring teyrchain extrinsic gets finalized");
+	let call = subxt::dynamic::tx("System", "remark", vec![Value::from_bytes("xxx".as_bytes())]);
+	let charlie_client: OnlineClient<PolkadotConfig> = charlie.wait_client().await?;
+
+	let res = submit_extrinsic_and_wait_for_finalization_success_with_timeout(
+		&charlie_client,
+		&call,
+		&dev::alice(),
+		600u64,
+	)
+	.await;
+	assert!(res.is_ok(), "Extrinsic failed to finalize: {:?}", res.unwrap_err());
+
+	Ok(())
+}
+
+async fn build_network_config() -> Result<NetworkConfig, anyhow::Error> {
+	let _ = env_logger::try_init_from_env(
+		env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
+	);
+	log::info!("Spawning network");
+
+	// images are not relevant for `native`, but we leave it here in case we use `k8s` some day
+	let images = zombienet_sdk::environment::get_images_from_env();
+	log::info!("Using images: {images:?}");
+
+	// Network setup:
+	// - relaychain Nodes:
+	// 	 - alice
+	// 	 - bob
+	// - teyrchain Nodes:
+	//   - charlie - collator
+	let config = NetworkConfigBuilder::new()
+		.with_relaychain(|r| {
+			r.with_chain("pezkuwichain-local")
+				.with_default_command("pezkuwi")
+				.with_default_image(images.polkadot.as_str())
+				.with_node(|node| node.with_name("alice"))
+				.with_node(|node| node.with_name("bob"))
+		})
+		.with_teyrchain(|p| {
+			p.with_id(PARA_ID)
+				.with_chain("asset-hub-pezkuwichain-local")
+				.with_default_command("pezkuwi-teyrchain")
+				.with_default_image(images.pezcumulus.as_str())
+				.with_collator(|n| {
+					n.with_name("charlie").validator(true).with_args(vec![
+						("--force-authoring").into(),
+						("-ltxpool=trace").into(),
+						("--pool-type=fork-aware").into(),
+					])
+				})
+		})
+		.with_global_settings(|global_settings| match std::env::var("ZOMBIENET_SDK_BASE_DIR") {
+			Ok(val) => global_settings.with_base_dir(val),
+			_ => global_settings,
+		})
+		.build()
+		.map_err(|e| {
+			let errs = e.into_iter().map(|e| e.to_string()).collect::<Vec<_>>().join(" ");
+			anyhow!("config errs: {errs}")
+		})?;
+
+	Ok(config)
+}