// Copyright (C) Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 use anyhow::anyhow; use crate::utils::{initialize_network, BEST_BLOCK_METRIC}; use cumulus_zombienet_sdk_helpers::{assert_para_is_registered, assert_para_throughput}; use pezkuwi_primitives::Id as ParaId; use serde_json::json; use std::{sync::Arc, time::Duration}; use zombienet_configuration::types::Arg; use zombienet_orchestrator::network::node::LogLineCountOptions; use zombienet_sdk::{ environment::Provider, subxt::{OnlineClient, PolkadotConfig}, NetworkConfig, NetworkConfigBuilder, RegistrationStrategy, }; const PARA_ID: u32 = 2000; // This tests makes sure that the recovering nodes are able to see all relay-chain // notifications containing the candidates to recover. #[tokio::test(flavor = "multi_thread")] async fn pov_recovery() -> Result<(), anyhow::Error> { let _ = env_logger::try_init_from_env( env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"), ); log::info!("Spawning network with relay chain only"); let config = build_network_config().await?; let mut network = initialize_network(config).await?; log::info!("Checking if network nodes are up"); let result = network.wait_until_is_up(200u64).await; assert!(result.is_ok(), "Network is not up: {:?}", result.unwrap_err()); let validator_3 = network.get_node("validator-3")?; log::info!("Waiting 20 blocks to register teyrchain"); // Wait 20 blocks and register teyrchain. This part is important for pov-recovery. assert!(validator_3 .wait_metric_with_timeout(BEST_BLOCK_METRIC, |b| b >= 20.0, 250u64) .await .is_ok()); log::info!("Registering teyrchain para_id = {PARA_ID}"); network.register_parachain(PARA_ID).await?; let validator = network.get_node("validator-0")?; let validator_client: OnlineClient = validator.wait_client().await?; log::info!("Ensuring teyrchain is registered within 30 blocks"); assert_para_is_registered(&validator_client, ParaId::from(PARA_ID), 30).await?; log::info!("Ensuring teyrchain making progress"); assert_para_throughput( &validator_client, 20, [(ParaId::from(PARA_ID), 2..20)].into_iter().collect(), ) .await?; for (name, timeout_secs) in [("bob", 600u64)] { log::info!("Checking block production for {name} within {timeout_secs}s"); assert!(network .get_node(name)? .wait_metric_with_timeout(BEST_BLOCK_METRIC, |b| b >= 20.0, timeout_secs) .await .is_ok()); } for (name, timeout_secs) in [("alice", 600u64), ("charlie", 600u64), ("one", 800u64), ("two", 800u64), ("eve", 800u64)] { // Wait (up to timeout_secs) until pattern occurs at least 20 times let options = LogLineCountOptions { predicate: Arc::new(|n| n >= 20), timeout: Duration::from_secs(timeout_secs), wait_until_timeout_elapses: false, }; log::info!("Ensuring blocks are imported using PoV recovery by {name}"); let result = network .get_node(name)? .wait_log_line_count_with_timeout( "Importing blocks retrieved using pov_recovery", false, options.clone(), ) .await?; assert!( result.success(), "Failed importing blocks using PoV recovery by {name}: {result:?}" ); } Ok(()) } async fn build_network_config() -> Result { // images are not relevant for `native`, but we leave it here in case we use `k8s` some day let images = zombienet_sdk::environment::get_images_from_env(); log::info!("Using images: {images:?}"); // If all nodes running on one machine and there are too much of them, // then they don't get enough CPU time and others might fail trying to connect to them. // eg. 'one' and 'two' trying to connect to validators rpc but it is still initializing. let validator_cnt = match zombienet_sdk::environment::get_provider_from_env() { Provider::K8s => 13, _ => 5, }; // Provide multiple RPC urls to increase a chance that some node behind url // will be have RPC server up and running. let mut rpc_urls = vec![]; rpc_urls.extend((0..validator_cnt).map(|i| format!("{{{{ZOMBIE:validator-{i}:ws_uri}}}}"))); // Network setup: // - relaychain nodes: // - validator[0-validator_cnt] // - validator // - synchronize only with validator-0 // - teyrchain nodes // - bob // - collator which produces blocks, but doesn't announce them // - alice // - collator which produces blocks, but doesn't announce them // - will need to recover the pov blocks through availability recovery // - charlie // - full node // - will need to recover the pov blocks through availability recovery // - eve // - collator which produces blocks, but doesn't announce them // - it fails recovery from time to time to test retries // - one // - RPC collator which produces blocks, but doesn't announce them // - Uses an external relay chain node // - two // - RPC full node // - Uses an external relay chain node let config = NetworkConfigBuilder::new() .with_relaychain(|r| { let r = r .with_chain("pezkuwichain-local") .with_default_command("pezkuwi") .with_default_image(images.polkadot.as_str()) .with_genesis_overrides(json!({ "configuration": { "config": { "scheduler_params": { "max_validators_per_core": 1, "group_rotation_frequency": 100 } } } })) .with_node(|node| { node.with_name("validator-0").validator(true).with_args(vec![ ("-lteyrchain::availability=trace,sync=info,teyrchain=debug,libp2p_mdns=debug,info").into(), ]) }); (1..validator_cnt).fold(r, |acc, i| { acc.with_node(|node| { node.with_name(&format!("validator-{i}")).with_args(vec![ ("-lteyrchain::availability=trace,sync=debug,teyrchain=debug,libp2p_mdns=debug").into(), ("--reserved-only").into(), ("--reserved-nodes", "{{ZOMBIE:validator-0:multiaddr}}").into(), ]) }) }) }) .with_teyrchain(|p| { p.with_id(PARA_ID) .with_registration_strategy(RegistrationStrategy::Manual) .with_default_command("test-teyrchain") .with_default_image(images.pezcumulus.as_str()) .with_collator(|c| { c.with_name("bob") .validator(true) .with_args(vec![ ("--disable-block-announcements").into(), ("-lteyrchain::availability=trace,sync=debug,teyrchain=debug,pezcumulus-pov-recovery=debug,pezcumulus-consensus=debug,libp2p_mdns=debug,info").into(), ]) }) .with_collator(|c| { c.with_name("alice") .validator(true) .with_args(build_collator_args(vec![])) }) .with_collator(|c| { c.with_name("charlie").validator(false).with_args(build_collator_args(vec![])) }) .with_collator(|c| { c.with_name("eve").validator(true).with_args(build_collator_args(vec![ "--fail-pov-recovery".into() ])) }) .with_collator(|c| { c.with_name("one").validator(true).with_args(build_collator_args(vec![ ("--relay-chain-rpc-url", rpc_urls.clone()).into() ])) }) .with_collator(|c| { c.with_name("two").validator(false).with_args(build_collator_args(vec![ ("--relay-chain-rpc-url", rpc_urls).into() ])) }) }) .with_global_settings(|global_settings| match std::env::var("ZOMBIENET_SDK_BASE_DIR") { Ok(val) => global_settings.with_base_dir(val), _ => global_settings, }) .build() .map_err(|e| { let errs = e.into_iter().map(|e| e.to_string()).collect::>().join(" "); anyhow!("config errs: {errs}") })?; Ok(config) } fn build_collator_args(in_args: Vec) -> Vec { let start_args: Vec = vec![ ("-lteyrchain::availability=trace,sync=debug,teyrchain=debug,pezcumulus-pov-recovery=debug,pezcumulus-consensus=debug,libp2p_mdns=debug,info").into(), ("--disable-block-announcements").into(), ("--in-peers=0").into(), ("--out-peers=0").into(), ("--bootnodes", "{{ZOMBIE:bob:multiaddr}}").into(), ]; let remaining_args: Vec = vec![ ("--").into(), ("--reserved-only").into(), ("--reserved-nodes", "{{ZOMBIE:validator-0:multiaddr}}").into(), ]; [start_args, in_args, remaining_args].concat() }