feat: Vendor pezkuwi-subxt and pezkuwi-zombienet-sdk into monorepo

- Add pezkuwi-subxt crates to vendor/pezkuwi-subxt
- Add pezkuwi-zombienet-sdk crates to vendor/pezkuwi-zombienet-sdk
- Convert git dependencies to path dependencies
- Add vendor crates to workspace members
- Remove test/example crates from vendor (not needed for SDK)
- Fix feature propagation issues detected by zepter
- Fix workspace inheritance for internal dependencies
- All 606 crates now in workspace
- All 6919 internal dependency links verified correct
- No git dependencies remaining
This commit is contained in:
2025-12-22 23:31:24 +03:00
parent 4c8f281051
commit 70ddb6516f
386 changed files with 76759 additions and 36 deletions
@@ -0,0 +1,2 @@
/target
/Cargo.lock
@@ -0,0 +1,29 @@
[package]
name = "zombienet-configuration"
version.workspace = true
authors.workspace = true
edition.workspace = true
rust-version.workspace = true
publish = true
license.workspace = true
repository.workspace = true
description = "Zombienet sdk config builder, allow to build a network configuration"
keywords = ["zombienet", "configuration", "sdk"]
[dependencies]
regex = { workspace = true }
lazy_static = { workspace = true }
multiaddr = { workspace = true }
url = { workspace = true, features = ["serde"] }
thiserror = { workspace = true }
anyhow = { workspace = true }
serde = { workspace = true, features = ["derive"] }
toml = { workspace = true }
serde_json = { workspace = true }
reqwest = { workspace = true }
tokio = { workspace = true, features = ["fs"] }
tracing = { workspace = true }
# zombienet deps
support = { workspace = true }
@@ -0,0 +1,383 @@
use std::{
error::Error,
fmt::Display,
net::IpAddr,
path::{Path, PathBuf},
str::FromStr,
};
use multiaddr::Multiaddr;
use serde::{Deserialize, Serialize};
use crate::{
shared::{
errors::{ConfigError, FieldError},
helpers::{merge_errors, merge_errors_vecs},
types::Duration,
},
utils::{default_as_true, default_node_spawn_timeout, default_timeout},
};
/// Global settings applied to an entire network.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct GlobalSettings {
/// Global bootnodes to use (we will then add more)
#[serde(skip_serializing_if = "std::vec::Vec::is_empty", default)]
bootnodes_addresses: Vec<Multiaddr>,
// TODO: parse both case in zombienet node version to avoid renamed ?
/// Global spawn timeout
#[serde(rename = "timeout", default = "default_timeout")]
network_spawn_timeout: Duration,
// TODO: not used yet
/// Node spawn timeout
#[serde(default = "default_node_spawn_timeout")]
node_spawn_timeout: Duration,
// TODO: not used yet
/// Local ip to use for construct the direct links
local_ip: Option<IpAddr>,
/// Directory to use as base dir
/// Used to reuse the same files (database) from a previous run,
/// also note that we will override the content of some of those files.
base_dir: Option<PathBuf>,
/// Number of concurrent spawning process to launch, None means try to spawn all at the same time.
spawn_concurrency: Option<usize>,
/// If enabled, will launch a task to monitor nodes' liveness and tear down the network if there are any.
#[serde(default = "default_as_true")]
tear_down_on_failure: bool,
}
impl GlobalSettings {
/// External bootnode address.
pub fn bootnodes_addresses(&self) -> Vec<&Multiaddr> {
self.bootnodes_addresses.iter().collect()
}
/// Global spawn timeout in seconds.
pub fn network_spawn_timeout(&self) -> Duration {
self.network_spawn_timeout
}
/// Individual node spawn timeout in seconds.
pub fn node_spawn_timeout(&self) -> Duration {
self.node_spawn_timeout
}
/// Local IP used to expose local services (including RPC, metrics and monitoring).
pub fn local_ip(&self) -> Option<&IpAddr> {
self.local_ip.as_ref()
}
/// Base directory to use (instead a random tmp one)
/// All the artifacts will be created in this directory.
pub fn base_dir(&self) -> Option<&Path> {
self.base_dir.as_deref()
}
/// Number of concurrent spawning process to launch
pub fn spawn_concurrency(&self) -> Option<usize> {
self.spawn_concurrency
}
/// A flag to tear down the network if there are any unresponsive nodes detected.
pub fn tear_down_on_failure(&self) -> bool {
self.tear_down_on_failure
}
}
impl Default for GlobalSettings {
fn default() -> Self {
Self {
bootnodes_addresses: Default::default(),
network_spawn_timeout: default_timeout(),
node_spawn_timeout: default_node_spawn_timeout(),
local_ip: Default::default(),
base_dir: Default::default(),
spawn_concurrency: Default::default(),
tear_down_on_failure: true,
}
}
}
/// A global settings builder, used to build [`GlobalSettings`] declaratively with fields validation.
#[derive(Default)]
pub struct GlobalSettingsBuilder {
config: GlobalSettings,
errors: Vec<anyhow::Error>,
}
impl GlobalSettingsBuilder {
pub fn new() -> Self {
Self::default()
}
// Transition to the next state of the builder.
fn transition(config: GlobalSettings, errors: Vec<anyhow::Error>) -> Self {
Self { config, errors }
}
/// Set the external bootnode address.
///
/// Note: Bootnode address replacements are NOT supported here.
/// Only arguments (`args`) support dynamic replacements. Bootnode addresses must be a valid address.
pub fn with_raw_bootnodes_addresses<T>(self, bootnodes_addresses: Vec<T>) -> Self
where
T: TryInto<Multiaddr> + Display + Copy,
T::Error: Error + Send + Sync + 'static,
{
let mut addrs = vec![];
let mut errors = vec![];
for (index, addr) in bootnodes_addresses.into_iter().enumerate() {
match addr.try_into() {
Ok(addr) => addrs.push(addr),
Err(error) => errors.push(
FieldError::BootnodesAddress(index, addr.to_string(), error.into()).into(),
),
}
}
Self::transition(
GlobalSettings {
bootnodes_addresses: addrs,
..self.config
},
merge_errors_vecs(self.errors, errors),
)
}
/// Set global spawn timeout in seconds.
pub fn with_network_spawn_timeout(self, timeout: Duration) -> Self {
Self::transition(
GlobalSettings {
network_spawn_timeout: timeout,
..self.config
},
self.errors,
)
}
/// Set individual node spawn timeout in seconds.
pub fn with_node_spawn_timeout(self, timeout: Duration) -> Self {
Self::transition(
GlobalSettings {
node_spawn_timeout: timeout,
..self.config
},
self.errors,
)
}
/// Set local IP used to expose local services (including RPC, metrics and monitoring).
pub fn with_local_ip(self, local_ip: &str) -> Self {
match IpAddr::from_str(local_ip) {
Ok(local_ip) => Self::transition(
GlobalSettings {
local_ip: Some(local_ip),
..self.config
},
self.errors,
),
Err(error) => Self::transition(
self.config,
merge_errors(self.errors, FieldError::LocalIp(error.into()).into()),
),
}
}
/// Set the directory to use as base (instead of a random tmp one).
pub fn with_base_dir(self, base_dir: impl Into<PathBuf>) -> Self {
Self::transition(
GlobalSettings {
base_dir: Some(base_dir.into()),
..self.config
},
self.errors,
)
}
/// Set the spawn concurrency
pub fn with_spawn_concurrency(self, spawn_concurrency: usize) -> Self {
Self::transition(
GlobalSettings {
spawn_concurrency: Some(spawn_concurrency),
..self.config
},
self.errors,
)
}
/// Set the `tear_down_on_failure` flag
pub fn with_tear_down_on_failure(self, tear_down_on_failure: bool) -> Self {
Self::transition(
GlobalSettings {
tear_down_on_failure,
..self.config
},
self.errors,
)
}
/// Seals the builder and returns a [`GlobalSettings`] if there are no validation errors, else returns errors.
pub fn build(self) -> Result<GlobalSettings, Vec<anyhow::Error>> {
if !self.errors.is_empty() {
return Err(self
.errors
.into_iter()
.map(|error| ConfigError::GlobalSettings(error).into())
.collect::<Vec<_>>());
}
Ok(self.config)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn global_settings_config_builder_should_succeeds_and_returns_a_global_settings_config() {
let global_settings_config = GlobalSettingsBuilder::new()
.with_raw_bootnodes_addresses(vec![
"/ip4/10.41.122.55/tcp/45421",
"/ip4/51.144.222.10/tcp/2333",
])
.with_network_spawn_timeout(600)
.with_node_spawn_timeout(120)
.with_local_ip("10.0.0.1")
.with_base_dir("/home/nonroot/mynetwork")
.with_spawn_concurrency(5)
.with_tear_down_on_failure(true)
.build()
.unwrap();
let bootnodes_addresses: Vec<Multiaddr> = vec![
"/ip4/10.41.122.55/tcp/45421".try_into().unwrap(),
"/ip4/51.144.222.10/tcp/2333".try_into().unwrap(),
];
assert_eq!(
global_settings_config.bootnodes_addresses(),
bootnodes_addresses.iter().collect::<Vec<_>>()
);
assert_eq!(global_settings_config.network_spawn_timeout(), 600);
assert_eq!(global_settings_config.node_spawn_timeout(), 120);
assert_eq!(
global_settings_config
.local_ip()
.unwrap()
.to_string()
.as_str(),
"10.0.0.1"
);
assert_eq!(
global_settings_config.base_dir().unwrap(),
Path::new("/home/nonroot/mynetwork")
);
assert_eq!(global_settings_config.spawn_concurrency().unwrap(), 5);
assert!(global_settings_config.tear_down_on_failure());
}
#[test]
fn global_settings_config_builder_should_succeeds_when_node_spawn_timeout_is_missing() {
let global_settings_config = GlobalSettingsBuilder::new()
.with_raw_bootnodes_addresses(vec![
"/ip4/10.41.122.55/tcp/45421",
"/ip4/51.144.222.10/tcp/2333",
])
.with_network_spawn_timeout(600)
.with_local_ip("10.0.0.1")
.build()
.unwrap();
let bootnodes_addresses: Vec<Multiaddr> = vec![
"/ip4/10.41.122.55/tcp/45421".try_into().unwrap(),
"/ip4/51.144.222.10/tcp/2333".try_into().unwrap(),
];
assert_eq!(
global_settings_config.bootnodes_addresses(),
bootnodes_addresses.iter().collect::<Vec<_>>()
);
assert_eq!(global_settings_config.network_spawn_timeout(), 600);
assert_eq!(global_settings_config.node_spawn_timeout(), 600);
assert_eq!(
global_settings_config
.local_ip()
.unwrap()
.to_string()
.as_str(),
"10.0.0.1"
);
}
#[test]
fn global_settings_builder_should_fails_and_returns_an_error_if_one_bootnode_address_is_invalid(
) {
let errors = GlobalSettingsBuilder::new()
.with_raw_bootnodes_addresses(vec!["/ip4//tcp/45421"])
.build()
.unwrap_err();
assert_eq!(errors.len(), 1);
assert_eq!(
errors.first().unwrap().to_string(),
"global_settings.bootnodes_addresses[0]: '/ip4//tcp/45421' failed to parse: invalid IPv4 address syntax"
);
}
#[test]
fn global_settings_builder_should_fails_and_returns_multiple_errors_if_multiple_bootnodes_addresses_are_invalid(
) {
let errors = GlobalSettingsBuilder::new()
.with_raw_bootnodes_addresses(vec!["/ip4//tcp/45421", "//10.42.153.10/tcp/43111"])
.build()
.unwrap_err();
assert_eq!(errors.len(), 2);
assert_eq!(
errors.first().unwrap().to_string(),
"global_settings.bootnodes_addresses[0]: '/ip4//tcp/45421' failed to parse: invalid IPv4 address syntax"
);
assert_eq!(
errors.get(1).unwrap().to_string(),
"global_settings.bootnodes_addresses[1]: '//10.42.153.10/tcp/43111' unknown protocol string: "
);
}
#[test]
fn global_settings_builder_should_fails_and_returns_an_error_if_local_ip_is_invalid() {
let errors = GlobalSettingsBuilder::new()
.with_local_ip("invalid")
.build()
.unwrap_err();
assert_eq!(errors.len(), 1);
assert_eq!(
errors.first().unwrap().to_string(),
"global_settings.local_ip: invalid IP address syntax"
);
}
#[test]
fn global_settings_builder_should_fails_and_returns_multiple_errors_if_multiple_fields_are_invalid(
) {
let errors = GlobalSettingsBuilder::new()
.with_raw_bootnodes_addresses(vec!["/ip4//tcp/45421", "//10.42.153.10/tcp/43111"])
.with_local_ip("invalid")
.build()
.unwrap_err();
assert_eq!(errors.len(), 3);
assert_eq!(
errors.first().unwrap().to_string(),
"global_settings.bootnodes_addresses[0]: '/ip4//tcp/45421' failed to parse: invalid IPv4 address syntax"
);
assert_eq!(
errors.get(1).unwrap().to_string(),
"global_settings.bootnodes_addresses[1]: '//10.42.153.10/tcp/43111' unknown protocol string: "
);
assert_eq!(
errors.get(2).unwrap().to_string(),
"global_settings.local_ip: invalid IP address syntax"
);
}
}
@@ -0,0 +1,137 @@
use std::marker::PhantomData;
use serde::{Deserialize, Serialize};
use crate::shared::{macros::states, types::ParaId};
/// HRMP channel configuration, with fine-grained configuration options.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct HrmpChannelConfig {
sender: ParaId,
recipient: ParaId,
max_capacity: u32,
max_message_size: u32,
}
impl HrmpChannelConfig {
/// The sending parachain ID.
pub fn sender(&self) -> ParaId {
self.sender
}
/// The receiving parachain ID.
pub fn recipient(&self) -> ParaId {
self.recipient
}
/// The maximum capacity of messages in the channel.
pub fn max_capacity(&self) -> u32 {
self.max_capacity
}
/// The maximum size of a message in the channel.
pub fn max_message_size(&self) -> u32 {
self.max_message_size
}
}
states! {
Initial,
WithSender,
WithRecipient
}
/// HRMP channel configuration builder, used to build an [`HrmpChannelConfig`] declaratively with fields validation.
pub struct HrmpChannelConfigBuilder<State> {
config: HrmpChannelConfig,
_state: PhantomData<State>,
}
impl Default for HrmpChannelConfigBuilder<Initial> {
fn default() -> Self {
Self {
config: HrmpChannelConfig {
sender: 0,
recipient: 0,
max_capacity: 8,
max_message_size: 512,
},
_state: PhantomData,
}
}
}
impl<A> HrmpChannelConfigBuilder<A> {
fn transition<B>(&self, config: HrmpChannelConfig) -> HrmpChannelConfigBuilder<B> {
HrmpChannelConfigBuilder {
config,
_state: PhantomData,
}
}
}
impl HrmpChannelConfigBuilder<Initial> {
pub fn new() -> Self {
Self::default()
}
/// Set the sending parachain ID.
pub fn with_sender(self, sender: ParaId) -> HrmpChannelConfigBuilder<WithSender> {
self.transition(HrmpChannelConfig {
sender,
..self.config
})
}
}
impl HrmpChannelConfigBuilder<WithSender> {
/// Set the receiving parachain ID.
pub fn with_recipient(self, recipient: ParaId) -> HrmpChannelConfigBuilder<WithRecipient> {
self.transition(HrmpChannelConfig {
recipient,
..self.config
})
}
}
impl HrmpChannelConfigBuilder<WithRecipient> {
/// Set the max capacity of messages in the channel.
pub fn with_max_capacity(self, max_capacity: u32) -> Self {
self.transition(HrmpChannelConfig {
max_capacity,
..self.config
})
}
/// Set the maximum size of a message in the channel.
pub fn with_max_message_size(self, max_message_size: u32) -> Self {
self.transition(HrmpChannelConfig {
max_message_size,
..self.config
})
}
pub fn build(self) -> HrmpChannelConfig {
self.config
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn hrmp_channel_config_builder_should_build_a_new_hrmp_channel_config_correctly() {
let hrmp_channel_config = HrmpChannelConfigBuilder::new()
.with_sender(1000)
.with_recipient(2000)
.with_max_capacity(50)
.with_max_message_size(100)
.build();
assert_eq!(hrmp_channel_config.sender(), 1000);
assert_eq!(hrmp_channel_config.recipient(), 2000);
assert_eq!(hrmp_channel_config.max_capacity(), 50);
assert_eq!(hrmp_channel_config.max_message_size(), 100);
}
}
@@ -0,0 +1,102 @@
//! This crate is used to create type safe configuration for Zombienet SDK using nested builders.
//!
//!
//! The main entry point of this crate is the [`NetworkConfigBuilder`] which is used to build a full network configuration
//! but all inner builders are also exposed to allow more granular control over the configuration.
//!
//! **Note**: Not all options can be checked at compile time and some will be checked at runtime when spawning a
//! network (e.g.: supported args for a specific node version).
//!
//! # Example
//! ```
//! use zombienet_configuration::NetworkConfigBuilder;
//!
//! let simple_configuration = NetworkConfigBuilder::new()
//! .with_relaychain(|relaychain| {
//! relaychain
//! .with_chain("polkadot")
//! .with_random_nominators_count(10)
//! .with_default_resources(|resources| {
//! resources
//! .with_limit_cpu("1000m")
//! .with_request_memory("1Gi")
//! .with_request_cpu(100_000)
//! })
//! .with_node(|node| {
//! node.with_name("node")
//! .with_command("command")
//! .validator(true)
//! })
//! })
//! .with_parachain(|parachain| {
//! parachain
//! .with_id(1000)
//! .with_chain("myparachain1")
//! .with_initial_balance(100_000)
//! .with_default_image("myimage:version")
//! .with_collator(|collator| {
//! collator
//! .with_name("collator1")
//! .with_command("command1")
//! .validator(true)
//! })
//! })
//! .with_parachain(|parachain| {
//! parachain
//! .with_id(2000)
//! .with_chain("myparachain2")
//! .with_initial_balance(50_0000)
//! .with_collator(|collator| {
//! collator
//! .with_name("collator2")
//! .with_command("command2")
//! .validator(true)
//! })
//! })
//! .with_hrmp_channel(|hrmp_channel1| {
//! hrmp_channel1
//! .with_sender(1)
//! .with_recipient(2)
//! .with_max_capacity(200)
//! .with_max_message_size(500)
//! })
//! .with_hrmp_channel(|hrmp_channel2| {
//! hrmp_channel2
//! .with_sender(2)
//! .with_recipient(1)
//! .with_max_capacity(100)
//! .with_max_message_size(250)
//! })
//! .with_global_settings(|global_settings| {
//! global_settings
//! .with_network_spawn_timeout(1200)
//! .with_node_spawn_timeout(240)
//! })
//! .build();
//!
//! assert!(simple_configuration.is_ok())
//! ```
#![allow(clippy::expect_fun_call)]
mod global_settings;
mod hrmp_channel;
mod network;
mod relaychain;
pub mod shared;
mod teyrchain;
mod utils;
pub use global_settings::{GlobalSettings, GlobalSettingsBuilder};
pub use hrmp_channel::{HrmpChannelConfig, HrmpChannelConfigBuilder};
pub use network::{NetworkConfig, NetworkConfigBuilder, WithRelaychain};
pub use relaychain::{RelaychainConfig, RelaychainConfigBuilder};
// re-export shared
pub use shared::{node::NodeConfig, types};
pub use teyrchain::{
states as para_states, RegistrationStrategy, TeyrchainConfig, TeyrchainConfigBuilder,
};
// Backward compatibility aliases for external crates that use Polkadot SDK terminology
// These allow zombienet-orchestrator and other external crates to work with our renamed types
pub type ParachainConfig = TeyrchainConfig;
pub type ParachainConfigBuilder<S, C> = TeyrchainConfigBuilder<S, C>;
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,6 @@
pub mod errors;
pub mod helpers;
pub mod macros;
pub mod node;
pub mod resources;
pub mod types;
@@ -0,0 +1,116 @@
use super::types::{ParaId, Port};
/// An error at the configuration level.
#[derive(thiserror::Error, Debug)]
pub enum ConfigError {
#[error("relaychain.{0}")]
Relaychain(anyhow::Error),
#[error("teyrchain[{0}].{1}")]
Teyrchain(ParaId, anyhow::Error),
#[error("global_settings.{0}")]
GlobalSettings(anyhow::Error),
#[error("nodes['{0}'].{1}")]
Node(String, anyhow::Error),
#[error("collators['{0}'].{1}")]
Collator(String, anyhow::Error),
}
/// An error at the field level.
#[derive(thiserror::Error, Debug)]
pub enum FieldError {
#[error("name: {0}")]
Name(anyhow::Error),
#[error("chain: {0}")]
Chain(anyhow::Error),
#[error("image: {0}")]
Image(anyhow::Error),
#[error("default_image: {0}")]
DefaultImage(anyhow::Error),
#[error("command: {0}")]
Command(anyhow::Error),
#[error("default_command: {0}")]
DefaultCommand(anyhow::Error),
#[error("bootnodes_addresses[{0}]: '{1}' {2}")]
BootnodesAddress(usize, String, anyhow::Error),
#[error("genesis_wasm_generator: {0}")]
GenesisWasmGenerator(anyhow::Error),
#[error("genesis_state_generator: {0}")]
GenesisStateGenerator(anyhow::Error),
#[error("local_ip: {0}")]
LocalIp(anyhow::Error),
#[error("default_resources.{0}")]
DefaultResources(anyhow::Error),
#[error("resources.{0}")]
Resources(anyhow::Error),
#[error("request_memory: {0}")]
RequestMemory(anyhow::Error),
#[error("request_cpu: {0}")]
RequestCpu(anyhow::Error),
#[error("limit_memory: {0}")]
LimitMemory(anyhow::Error),
#[error("limit_cpu: {0}")]
LimitCpu(anyhow::Error),
#[error("ws_port: {0}")]
WsPort(anyhow::Error),
#[error("rpc_port: {0}")]
RpcPort(anyhow::Error),
#[error("prometheus_port: {0}")]
PrometheusPort(anyhow::Error),
#[error("p2p_port: {0}")]
P2pPort(anyhow::Error),
#[error("session_key: {0}")]
SessionKey(anyhow::Error),
#[error("registration_strategy: {0}")]
RegistrationStrategy(anyhow::Error),
}
/// A conversion error for shared types across fields.
#[derive(thiserror::Error, Debug, Clone)]
pub enum ConversionError {
#[error("'{0}' shouldn't contains whitespace")]
ContainsWhitespaces(String),
#[error("'{}' doesn't match regex '{}'", .value, .regex)]
DoesntMatchRegex { value: String, regex: String },
#[error("can't be empty")]
CantBeEmpty,
#[error("deserialize error")]
DeserializeError(String),
}
/// A validation error for shared types across fields.
#[derive(thiserror::Error, Debug, Clone)]
pub enum ValidationError {
#[error("'{0}' is already used across config")]
PortAlreadyUsed(Port),
#[error("can't be empty")]
CantBeEmpty(),
}
@@ -0,0 +1,118 @@
use std::{cell::RefCell, collections::HashSet, rc::Rc};
use support::constants::{BORROWABLE, THIS_IS_A_BUG};
use tracing::warn;
use super::{
errors::ValidationError,
types::{ParaId, Port, ValidationContext},
};
pub fn merge_errors(errors: Vec<anyhow::Error>, new_error: anyhow::Error) -> Vec<anyhow::Error> {
let mut errors = errors;
errors.push(new_error);
errors
}
pub fn merge_errors_vecs(
errors: Vec<anyhow::Error>,
new_errors: Vec<anyhow::Error>,
) -> Vec<anyhow::Error> {
let mut errors = errors;
for new_error in new_errors.into_iter() {
errors.push(new_error);
}
errors
}
/// Generates a unique name from a base name and the names already present in a
/// [`ValidationContext`].
///
/// Uses [`generate_unique_node_name_from_names()`] internally to ensure uniqueness.
/// Logs a warning if the generated name differs from the original due to duplicates.
pub fn generate_unique_node_name(
node_name: impl Into<String>,
validation_context: Rc<RefCell<ValidationContext>>,
) -> String {
let mut context = validation_context
.try_borrow_mut()
.expect(&format!("{BORROWABLE}, {THIS_IS_A_BUG}"));
generate_unique_node_name_from_names(node_name, &mut context.used_nodes_names)
}
/// Returns `node_name` if it is not already in `names`.
///
/// Otherwise, appends an incrementing `-{counter}` suffix until a unique name is found,
/// then returns it. Logs a warning when a duplicate is detected.
pub fn generate_unique_node_name_from_names(
node_name: impl Into<String>,
names: &mut HashSet<String>,
) -> String {
let node_name = node_name.into();
if names.insert(node_name.clone()) {
return node_name;
}
let mut counter = 1;
let mut candidate = node_name.clone();
while names.contains(&candidate) {
candidate = format!("{node_name}-{counter}");
counter += 1;
}
warn!(
original = %node_name,
adjusted = %candidate,
"Duplicate node name detected."
);
names.insert(candidate.clone());
candidate
}
pub fn ensure_value_is_not_empty(value: &str) -> Result<(), anyhow::Error> {
if value.is_empty() {
Err(ValidationError::CantBeEmpty().into())
} else {
Ok(())
}
}
pub fn ensure_port_unique(
port: Port,
validation_context: Rc<RefCell<ValidationContext>>,
) -> Result<(), anyhow::Error> {
let mut context = validation_context
.try_borrow_mut()
.expect(&format!("{BORROWABLE}, {THIS_IS_A_BUG}"));
if !context.used_ports.contains(&port) {
context.used_ports.push(port);
return Ok(());
}
Err(ValidationError::PortAlreadyUsed(port).into())
}
pub fn generate_unique_para_id(
para_id: ParaId,
validation_context: Rc<RefCell<ValidationContext>>,
) -> String {
let mut context = validation_context
.try_borrow_mut()
.expect(&format!("{BORROWABLE}, {THIS_IS_A_BUG}"));
if let Some(suffix) = context.used_para_ids.get_mut(&para_id) {
*suffix += 1;
format!("{para_id}-{suffix}")
} else {
// insert 0, since will be used next time.
context.used_para_ids.insert(para_id, 0);
para_id.to_string()
}
}
@@ -0,0 +1,11 @@
// Helper to define states of a type.
// We use an enum with no variants because it can't be constructed by definition.
macro_rules! states {
($($ident:ident),*) => {
$(
pub enum $ident {}
)*
};
}
pub(crate) use states;
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,489 @@
use std::error::Error;
use lazy_static::lazy_static;
use regex::Regex;
use serde::{
de::{self},
ser::SerializeStruct,
Deserialize, Serialize,
};
use support::constants::{SHOULD_COMPILE, THIS_IS_A_BUG};
use super::{
errors::{ConversionError, FieldError},
helpers::merge_errors,
};
/// A resource quantity used to define limits (k8s/podman only).
/// It can be constructed from a `&str` or u64, if it fails, it returns a [`ConversionError`].
/// Possible optional prefixes are: m, K, M, G, T, P, E, Ki, Mi, Gi, Ti, Pi, Ei
///
/// # Examples
///
/// ```
/// use zombienet_configuration::shared::resources::ResourceQuantity;
///
/// let quantity1: ResourceQuantity = "100000".try_into().unwrap();
/// let quantity2: ResourceQuantity = "1000m".try_into().unwrap();
/// let quantity3: ResourceQuantity = "1Gi".try_into().unwrap();
/// let quantity4: ResourceQuantity = 10_000.into();
///
/// assert_eq!(quantity1.as_str(), "100000");
/// assert_eq!(quantity2.as_str(), "1000m");
/// assert_eq!(quantity3.as_str(), "1Gi");
/// assert_eq!(quantity4.as_str(), "10000");
/// ```
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ResourceQuantity(String);
impl ResourceQuantity {
pub fn as_str(&self) -> &str {
&self.0
}
}
impl TryFrom<&str> for ResourceQuantity {
type Error = ConversionError;
fn try_from(value: &str) -> Result<Self, Self::Error> {
lazy_static! {
static ref RE: Regex = Regex::new(r"^\d+(.\d+)?(m|K|M|G|T|P|E|Ki|Mi|Gi|Ti|Pi|Ei)?$")
.expect(&format!("{SHOULD_COMPILE}, {THIS_IS_A_BUG}"));
}
if !RE.is_match(value) {
return Err(ConversionError::DoesntMatchRegex {
value: value.to_string(),
regex: r"^\d+(.\d+)?(m|K|M|G|T|P|E|Ki|Mi|Gi|Ti|Pi|Ei)?$".to_string(),
});
}
Ok(Self(value.to_string()))
}
}
impl From<u64> for ResourceQuantity {
fn from(value: u64) -> Self {
Self(value.to_string())
}
}
/// Resources limits used in the context of podman/k8s.
#[derive(Debug, Default, Clone, PartialEq)]
pub struct Resources {
request_memory: Option<ResourceQuantity>,
request_cpu: Option<ResourceQuantity>,
limit_memory: Option<ResourceQuantity>,
limit_cpu: Option<ResourceQuantity>,
}
#[derive(Serialize, Deserialize)]
struct ResourcesField {
memory: Option<ResourceQuantity>,
cpu: Option<ResourceQuantity>,
}
impl Serialize for Resources {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let mut state = serializer.serialize_struct("Resources", 2)?;
if self.request_memory.is_some() || self.request_memory.is_some() {
state.serialize_field(
"requests",
&ResourcesField {
memory: self.request_memory.clone(),
cpu: self.request_cpu.clone(),
},
)?;
} else {
state.skip_field("requests")?;
}
if self.limit_memory.is_some() || self.limit_memory.is_some() {
state.serialize_field(
"limits",
&ResourcesField {
memory: self.limit_memory.clone(),
cpu: self.limit_cpu.clone(),
},
)?;
} else {
state.skip_field("limits")?;
}
state.end()
}
}
struct ResourcesVisitor;
impl<'de> de::Visitor<'de> for ResourcesVisitor {
type Value = Resources;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
formatter.write_str("a resources object")
}
fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
where
A: de::MapAccess<'de>,
{
let mut resources: Resources = Resources::default();
while let Some((key, value)) = map.next_entry::<String, ResourcesField>()? {
match key.as_str() {
"requests" => {
resources.request_memory = value.memory;
resources.request_cpu = value.cpu;
},
"limits" => {
resources.limit_memory = value.memory;
resources.limit_cpu = value.cpu;
},
_ => {
return Err(de::Error::unknown_field(
&key,
&["requests", "limits", "cpu", "memory"],
))
},
}
}
Ok(resources)
}
}
impl<'de> Deserialize<'de> for Resources {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
deserializer.deserialize_any(ResourcesVisitor)
}
}
impl Resources {
/// Memory limit applied to requests.
pub fn request_memory(&self) -> Option<&ResourceQuantity> {
self.request_memory.as_ref()
}
/// CPU limit applied to requests.
pub fn request_cpu(&self) -> Option<&ResourceQuantity> {
self.request_cpu.as_ref()
}
/// Overall memory limit applied.
pub fn limit_memory(&self) -> Option<&ResourceQuantity> {
self.limit_memory.as_ref()
}
/// Overall CPU limit applied.
pub fn limit_cpu(&self) -> Option<&ResourceQuantity> {
self.limit_cpu.as_ref()
}
}
/// A resources builder, used to build a [`Resources`] declaratively with fields validation.
#[derive(Debug, Default)]
pub struct ResourcesBuilder {
config: Resources,
errors: Vec<anyhow::Error>,
}
impl ResourcesBuilder {
pub fn new() -> ResourcesBuilder {
Self::default()
}
fn transition(config: Resources, errors: Vec<anyhow::Error>) -> Self {
Self { config, errors }
}
/// Set the requested memory for a pod. This is the minimum memory allocated for a pod.
pub fn with_request_memory<T>(self, quantity: T) -> Self
where
T: TryInto<ResourceQuantity>,
T::Error: Error + Send + Sync + 'static,
{
match quantity.try_into() {
Ok(quantity) => Self::transition(
Resources {
request_memory: Some(quantity),
..self.config
},
self.errors,
),
Err(error) => Self::transition(
self.config,
merge_errors(self.errors, FieldError::RequestMemory(error.into()).into()),
),
}
}
/// Set the requested CPU limit for a pod. This is the minimum CPU allocated for a pod.
pub fn with_request_cpu<T>(self, quantity: T) -> Self
where
T: TryInto<ResourceQuantity>,
T::Error: Error + Send + Sync + 'static,
{
match quantity.try_into() {
Ok(quantity) => Self::transition(
Resources {
request_cpu: Some(quantity),
..self.config
},
self.errors,
),
Err(error) => Self::transition(
self.config,
merge_errors(self.errors, FieldError::RequestCpu(error.into()).into()),
),
}
}
/// Set the overall memory limit for a pod. This is the maximum memory threshold for a pod.
pub fn with_limit_memory<T>(self, quantity: T) -> Self
where
T: TryInto<ResourceQuantity>,
T::Error: Error + Send + Sync + 'static,
{
match quantity.try_into() {
Ok(quantity) => Self::transition(
Resources {
limit_memory: Some(quantity),
..self.config
},
self.errors,
),
Err(error) => Self::transition(
self.config,
merge_errors(self.errors, FieldError::LimitMemory(error.into()).into()),
),
}
}
/// Set the overall CPU limit for a pod. This is the maximum CPU threshold for a pod.
pub fn with_limit_cpu<T>(self, quantity: T) -> Self
where
T: TryInto<ResourceQuantity>,
T::Error: Error + Send + Sync + 'static,
{
match quantity.try_into() {
Ok(quantity) => Self::transition(
Resources {
limit_cpu: Some(quantity),
..self.config
},
self.errors,
),
Err(error) => Self::transition(
self.config,
merge_errors(self.errors, FieldError::LimitCpu(error.into()).into()),
),
}
}
/// Seals the builder and returns a [`Resources`] if there are no validation errors, else returns errors.
pub fn build(self) -> Result<Resources, Vec<anyhow::Error>> {
if !self.errors.is_empty() {
return Err(self.errors);
}
Ok(self.config)
}
}
#[cfg(test)]
#[allow(non_snake_case)]
mod tests {
use super::*;
use crate::NetworkConfig;
macro_rules! impl_resources_quantity_unit_test {
($val:literal) => {{
let resources = ResourcesBuilder::new()
.with_request_memory($val)
.build()
.unwrap();
assert_eq!(resources.request_memory().unwrap().as_str(), $val);
assert_eq!(resources.request_cpu(), None);
assert_eq!(resources.limit_cpu(), None);
assert_eq!(resources.limit_memory(), None);
}};
}
#[test]
fn converting_a_string_a_resource_quantity_without_unit_should_succeeds() {
impl_resources_quantity_unit_test!("1000");
}
#[test]
fn converting_a_str_with_m_unit_into_a_resource_quantity_should_succeeds() {
impl_resources_quantity_unit_test!("100m");
}
#[test]
fn converting_a_str_with_K_unit_into_a_resource_quantity_should_succeeds() {
impl_resources_quantity_unit_test!("50K");
}
#[test]
fn converting_a_str_with_M_unit_into_a_resource_quantity_should_succeeds() {
impl_resources_quantity_unit_test!("100M");
}
#[test]
fn converting_a_str_with_G_unit_into_a_resource_quantity_should_succeeds() {
impl_resources_quantity_unit_test!("1G");
}
#[test]
fn converting_a_str_with_T_unit_into_a_resource_quantity_should_succeeds() {
impl_resources_quantity_unit_test!("0.01T");
}
#[test]
fn converting_a_str_with_P_unit_into_a_resource_quantity_should_succeeds() {
impl_resources_quantity_unit_test!("0.00001P");
}
#[test]
fn converting_a_str_with_E_unit_into_a_resource_quantity_should_succeeds() {
impl_resources_quantity_unit_test!("0.000000001E");
}
#[test]
fn converting_a_str_with_Ki_unit_into_a_resource_quantity_should_succeeds() {
impl_resources_quantity_unit_test!("50Ki");
}
#[test]
fn converting_a_str_with_Mi_unit_into_a_resource_quantity_should_succeeds() {
impl_resources_quantity_unit_test!("100Mi");
}
#[test]
fn converting_a_str_with_Gi_unit_into_a_resource_quantity_should_succeeds() {
impl_resources_quantity_unit_test!("1Gi");
}
#[test]
fn converting_a_str_with_Ti_unit_into_a_resource_quantity_should_succeeds() {
impl_resources_quantity_unit_test!("0.01Ti");
}
#[test]
fn converting_a_str_with_Pi_unit_into_a_resource_quantity_should_succeeds() {
impl_resources_quantity_unit_test!("0.00001Pi");
}
#[test]
fn converting_a_str_with_Ei_unit_into_a_resource_quantity_should_succeeds() {
impl_resources_quantity_unit_test!("0.000000001Ei");
}
#[test]
fn resources_config_builder_should_succeeds_and_returns_a_resources_config() {
let resources = ResourcesBuilder::new()
.with_request_memory("200M")
.with_request_cpu("1G")
.with_limit_cpu("500M")
.with_limit_memory("2G")
.build()
.unwrap();
assert_eq!(resources.request_memory().unwrap().as_str(), "200M");
assert_eq!(resources.request_cpu().unwrap().as_str(), "1G");
assert_eq!(resources.limit_cpu().unwrap().as_str(), "500M");
assert_eq!(resources.limit_memory().unwrap().as_str(), "2G");
}
#[test]
fn resources_config_toml_import_should_succeeds_and_returns_a_resources_config() {
let load_from_toml =
NetworkConfig::load_from_toml("./testing/snapshots/0001-big-network.toml").unwrap();
let resources = load_from_toml.relaychain().default_resources().unwrap();
assert_eq!(resources.request_memory().unwrap().as_str(), "500M");
assert_eq!(resources.request_cpu().unwrap().as_str(), "100000");
assert_eq!(resources.limit_cpu().unwrap().as_str(), "10Gi");
assert_eq!(resources.limit_memory().unwrap().as_str(), "4000M");
}
#[test]
fn resources_config_builder_should_fails_and_returns_an_error_if_couldnt_parse_request_memory()
{
let resources_builder = ResourcesBuilder::new().with_request_memory("invalid");
let errors = resources_builder.build().err().unwrap();
assert_eq!(errors.len(), 1);
assert_eq!(
errors.first().unwrap().to_string(),
r"request_memory: 'invalid' doesn't match regex '^\d+(.\d+)?(m|K|M|G|T|P|E|Ki|Mi|Gi|Ti|Pi|Ei)?$'"
);
}
#[test]
fn resources_config_builder_should_fails_and_returns_an_error_if_couldnt_parse_request_cpu() {
let resources_builder = ResourcesBuilder::new().with_request_cpu("invalid");
let errors = resources_builder.build().err().unwrap();
assert_eq!(errors.len(), 1);
assert_eq!(
errors.first().unwrap().to_string(),
r"request_cpu: 'invalid' doesn't match regex '^\d+(.\d+)?(m|K|M|G|T|P|E|Ki|Mi|Gi|Ti|Pi|Ei)?$'"
);
}
#[test]
fn resources_config_builder_should_fails_and_returns_an_error_if_couldnt_parse_limit_memory() {
let resources_builder = ResourcesBuilder::new().with_limit_memory("invalid");
let errors = resources_builder.build().err().unwrap();
assert_eq!(errors.len(), 1);
assert_eq!(
errors.first().unwrap().to_string(),
r"limit_memory: 'invalid' doesn't match regex '^\d+(.\d+)?(m|K|M|G|T|P|E|Ki|Mi|Gi|Ti|Pi|Ei)?$'"
);
}
#[test]
fn resources_config_builder_should_fails_and_returns_an_error_if_couldnt_parse_limit_cpu() {
let resources_builder = ResourcesBuilder::new().with_limit_cpu("invalid");
let errors = resources_builder.build().err().unwrap();
assert_eq!(errors.len(), 1);
assert_eq!(
errors.first().unwrap().to_string(),
r"limit_cpu: 'invalid' doesn't match regex '^\d+(.\d+)?(m|K|M|G|T|P|E|Ki|Mi|Gi|Ti|Pi|Ei)?$'"
);
}
#[test]
fn resources_config_builder_should_fails_and_returns_multiple_error_if_couldnt_parse_multiple_fields(
) {
let resources_builder = ResourcesBuilder::new()
.with_limit_cpu("invalid")
.with_request_memory("invalid");
let errors = resources_builder.build().err().unwrap();
assert_eq!(errors.len(), 2);
assert_eq!(
errors.first().unwrap().to_string(),
r"limit_cpu: 'invalid' doesn't match regex '^\d+(.\d+)?(m|K|M|G|T|P|E|Ki|Mi|Gi|Ti|Pi|Ei)?$'"
);
assert_eq!(
errors.get(1).unwrap().to_string(),
r"request_memory: 'invalid' doesn't match regex '^\d+(.\d+)?(m|K|M|G|T|P|E|Ki|Mi|Gi|Ti|Pi|Ei)?$'"
);
}
}
@@ -0,0 +1,930 @@
use std::{
collections::{HashMap, HashSet},
error::Error,
fmt::{self, Display},
path::PathBuf,
str::FromStr,
};
use anyhow::anyhow;
use lazy_static::lazy_static;
use regex::Regex;
use serde::{
de::{self, IntoDeserializer},
Deserialize, Deserializer, Serialize,
};
use support::constants::{INFAILABLE, SHOULD_COMPILE, THIS_IS_A_BUG};
use tokio::fs;
use url::Url;
use super::{errors::ConversionError, resources::Resources};
/// An alias for a duration in seconds.
pub type Duration = u32;
/// An alias for a port.
pub type Port = u16;
/// An alias for a parachain ID.
pub type ParaId = u32;
/// Custom type wrapping u128 to add custom Serialization/Deserialization logic because it's not supported
/// issue tracking the problem: <https://github.com/toml-rs/toml/issues/540>
#[derive(Default, Debug, Clone, PartialEq)]
pub struct U128(pub(crate) u128);
impl From<u128> for U128 {
fn from(value: u128) -> Self {
Self(value)
}
}
impl TryFrom<&str> for U128 {
type Error = Box<dyn Error>;
fn try_from(value: &str) -> Result<Self, Self::Error> {
Ok(Self(value.to_string().parse::<u128>()?))
}
}
impl Serialize for U128 {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
// here we add a prefix to the string to be able to replace the wrapped
// value with "" to a value without "" in the TOML string
serializer.serialize_str(&format!("U128%{}", self.0))
}
}
struct U128Visitor;
impl de::Visitor<'_> for U128Visitor {
type Value = U128;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("an integer between 0 and 2^128 1.")
}
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: de::Error,
{
v.try_into().map_err(de::Error::custom)
}
}
impl<'de> Deserialize<'de> for U128 {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_str(U128Visitor)
}
}
/// A chain name.
/// It can be constructed for an `&str`, if it fails, it will returns a [`ConversionError`].
///
/// # Examples:
/// ```
/// use zombienet_configuration::shared::types::Chain;
///
/// let polkadot: Chain = "polkadot".try_into().unwrap();
/// let kusama: Chain = "kusama".try_into().unwrap();
/// let myparachain: Chain = "myparachain".try_into().unwrap();
///
/// assert_eq!(polkadot.as_str(), "polkadot");
/// assert_eq!(kusama.as_str(), "kusama");
/// assert_eq!(myparachain.as_str(), "myparachain");
/// ```
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Chain(String);
impl TryFrom<&str> for Chain {
type Error = ConversionError;
fn try_from(value: &str) -> Result<Self, Self::Error> {
if value.contains(char::is_whitespace) {
return Err(ConversionError::ContainsWhitespaces(value.to_string()));
}
if value.is_empty() {
return Err(ConversionError::CantBeEmpty);
}
Ok(Self(value.to_string()))
}
}
impl Chain {
pub fn as_str(&self) -> &str {
&self.0
}
}
/// A container image.
/// It can be constructed from an `&str` including a combination of name, version, IPv4 or/and hostname, if it fails, it will returns a [`ConversionError`].
///
/// # Examples:
/// ```
/// use zombienet_configuration::shared::types::Image;
///
/// let image1: Image = "name".try_into().unwrap();
/// let image2: Image = "name:version".try_into().unwrap();
/// let image3: Image = "myrepo.com/name:version".try_into().unwrap();
/// let image4: Image = "10.15.43.155/name:version".try_into().unwrap();
///
/// assert_eq!(image1.as_str(), "name");
/// assert_eq!(image2.as_str(), "name:version");
/// assert_eq!(image3.as_str(), "myrepo.com/name:version");
/// assert_eq!(image4.as_str(), "10.15.43.155/name:version");
/// ```
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Image(String);
impl TryFrom<&str> for Image {
type Error = ConversionError;
fn try_from(value: &str) -> Result<Self, Self::Error> {
static IP_PART: &str = "((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]).){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]))";
static HOSTNAME_PART: &str = "((([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9]).)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9-]*[A-Za-z0-9]))";
static TAG_NAME_PART: &str = "([a-z0-9](-*[a-z0-9])*)";
static TAG_VERSION_PART: &str = "([a-z0-9_]([-._a-z0-9])*)";
lazy_static! {
static ref RE: Regex = Regex::new(&format!(
"^({IP_PART}|{HOSTNAME_PART}/)?{TAG_NAME_PART}(:{TAG_VERSION_PART})?$",
))
.expect(&format!("{SHOULD_COMPILE}, {THIS_IS_A_BUG}"));
};
if !RE.is_match(value) {
return Err(ConversionError::DoesntMatchRegex {
value: value.to_string(),
regex: "^([ip]|[hostname]/)?[tag_name]:[tag_version]?$".to_string(),
});
}
Ok(Self(value.to_string()))
}
}
impl Image {
pub fn as_str(&self) -> &str {
&self.0
}
}
/// A command that will be executed natively (native provider) or in a container (podman/k8s).
/// It can be constructed from an `&str`, if it fails, it will returns a [`ConversionError`].
///
/// # Examples:
/// ```
/// use zombienet_configuration::shared::types::Command;
///
/// let command1: Command = "mycommand".try_into().unwrap();
/// let command2: Command = "myothercommand".try_into().unwrap();
///
/// assert_eq!(command1.as_str(), "mycommand");
/// assert_eq!(command2.as_str(), "myothercommand");
/// ```
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Command(String);
impl TryFrom<&str> for Command {
type Error = ConversionError;
fn try_from(value: &str) -> Result<Self, Self::Error> {
if value.contains(char::is_whitespace) {
return Err(ConversionError::ContainsWhitespaces(value.to_string()));
}
Ok(Self(value.to_string()))
}
}
impl Default for Command {
fn default() -> Self {
Self(String::from("polkadot"))
}
}
impl Command {
pub fn as_str(&self) -> &str {
&self.0
}
}
/// A command with optional custom arguments, the command will be executed natively (native provider) or in a container (podman/k8s).
/// It can be constructed from an `&str`, if it fails, it will returns a [`ConversionError`].
///
/// # Examples:
/// ```
/// use zombienet_configuration::shared::types::CommandWithCustomArgs;
///
/// let command1: CommandWithCustomArgs = "mycommand --demo=2 --other-flag".try_into().unwrap();
/// let command2: CommandWithCustomArgs = "my_other_cmd_without_args".try_into().unwrap();
///
/// assert_eq!(command1.cmd().as_str(), "mycommand");
/// assert_eq!(command2.cmd().as_str(), "my_other_cmd_without_args");
/// ```
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct CommandWithCustomArgs(Command, Vec<Arg>);
impl TryFrom<&str> for CommandWithCustomArgs {
type Error = ConversionError;
fn try_from(value: &str) -> Result<Self, Self::Error> {
if value.is_empty() {
return Err(ConversionError::CantBeEmpty);
}
let mut parts = value.split_whitespace().collect::<Vec<&str>>();
let cmd = parts.remove(0).try_into().unwrap();
let args = parts
.iter()
.map(|x| {
Arg::deserialize(x.into_deserializer()).map_err(|_: serde_json::Error| {
ConversionError::DeserializeError(String::from(*x))
})
})
.collect::<Result<Vec<Arg>, _>>()?;
Ok(Self(cmd, args))
}
}
impl Default for CommandWithCustomArgs {
fn default() -> Self {
Self("polkadot".try_into().unwrap(), vec![])
}
}
impl CommandWithCustomArgs {
pub fn cmd(&self) -> &Command {
&self.0
}
pub fn args(&self) -> &Vec<Arg> {
&self.1
}
}
/// A location for a locally or remotely stored asset.
/// It can be constructed from an [`url::Url`], a [`std::path::PathBuf`] or an `&str`.
///
/// # Examples:
/// ```
/// use url::Url;
/// use std::{path::PathBuf, str::FromStr};
/// use zombienet_configuration::shared::types::AssetLocation;
///
/// let url_location: AssetLocation = Url::from_str("https://mycloudstorage.com/path/to/my/file.tgz").unwrap().into();
/// let url_location2: AssetLocation = "https://mycloudstorage.com/path/to/my/file.tgz".into();
/// let path_location: AssetLocation = PathBuf::from_str("/tmp/path/to/my/file").unwrap().into();
/// let path_location2: AssetLocation = "/tmp/path/to/my/file".into();
///
/// assert!(matches!(url_location, AssetLocation::Url(value) if value.as_str() == "https://mycloudstorage.com/path/to/my/file.tgz"));
/// assert!(matches!(url_location2, AssetLocation::Url(value) if value.as_str() == "https://mycloudstorage.com/path/to/my/file.tgz"));
/// assert!(matches!(path_location, AssetLocation::FilePath(value) if value.to_str().unwrap() == "/tmp/path/to/my/file"));
/// assert!(matches!(path_location2, AssetLocation::FilePath(value) if value.to_str().unwrap() == "/tmp/path/to/my/file"));
/// ```
#[derive(Debug, Clone, PartialEq)]
pub enum AssetLocation {
Url(Url),
FilePath(PathBuf),
}
impl From<Url> for AssetLocation {
fn from(value: Url) -> Self {
Self::Url(value)
}
}
impl From<PathBuf> for AssetLocation {
fn from(value: PathBuf) -> Self {
Self::FilePath(value)
}
}
impl From<&str> for AssetLocation {
fn from(value: &str) -> Self {
if let Ok(parsed_url) = Url::parse(value) {
return Self::Url(parsed_url);
}
Self::FilePath(PathBuf::from_str(value).expect(&format!("{INFAILABLE}, {THIS_IS_A_BUG}")))
}
}
impl Display for AssetLocation {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
AssetLocation::Url(value) => write!(f, "{}", value.as_str()),
AssetLocation::FilePath(value) => write!(f, "{}", value.display()),
}
}
}
impl AssetLocation {
/// Get the current asset (from file or url) and return the content
pub async fn get_asset(&self) -> Result<Vec<u8>, anyhow::Error> {
let contents = match self {
AssetLocation::Url(location) => {
let res = reqwest::get(location.as_ref()).await.map_err(|err| {
anyhow!("Error dowinloding asset from url {location} - {err}")
})?;
res.bytes().await.unwrap().into()
},
AssetLocation::FilePath(filepath) => {
tokio::fs::read(filepath).await.map_err(|err| {
anyhow!(
"Error reading asset from path {} - {}",
filepath.to_string_lossy(),
err
)
})?
},
};
Ok(contents)
}
/// Write asset (from file or url) to the destination path.
pub async fn dump_asset(&self, dst_path: impl Into<PathBuf>) -> Result<(), anyhow::Error> {
let contents = self.get_asset().await?;
fs::write(dst_path.into(), contents).await?;
Ok(())
}
}
impl Serialize for AssetLocation {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(&self.to_string())
}
}
struct AssetLocationVisitor;
impl de::Visitor<'_> for AssetLocationVisitor {
type Value = AssetLocation;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("a string")
}
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(AssetLocation::from(v))
}
}
impl<'de> Deserialize<'de> for AssetLocation {
fn deserialize<D>(deserializer: D) -> Result<AssetLocation, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_any(AssetLocationVisitor)
}
}
/// A CLI argument passed to an executed command, can be an option with an assigned value or a simple flag to enable/disable a feature.
/// A flag arg can be constructed from a `&str` and a option arg can be constructed from a `(&str, &str)`.
///
/// # Examples:
/// ```
/// use zombienet_configuration::shared::types::Arg;
///
/// let flag_arg: Arg = "myflag".into();
/// let option_arg: Arg = ("name", "value").into();
///
/// assert!(matches!(flag_arg, Arg::Flag(value) if value == "myflag"));
/// assert!(matches!(option_arg, Arg::Option(name, value) if name == "name" && value == "value"));
/// ```
#[derive(Debug, Clone, PartialEq)]
pub enum Arg {
Flag(String),
Option(String, String),
Array(String, Vec<String>),
}
impl From<&str> for Arg {
fn from(flag: &str) -> Self {
Self::Flag(flag.to_owned())
}
}
impl From<(&str, &str)> for Arg {
fn from((option, value): (&str, &str)) -> Self {
Self::Option(option.to_owned(), value.to_owned())
}
}
impl<T> From<(&str, &[T])> for Arg
where
T: AsRef<str> + Clone,
{
fn from((option, values): (&str, &[T])) -> Self {
Self::Array(
option.to_owned(),
values.iter().map(|v| v.as_ref().to_string()).collect(),
)
}
}
impl<T> From<(&str, Vec<T>)> for Arg
where
T: AsRef<str>,
{
fn from((option, values): (&str, Vec<T>)) -> Self {
Self::Array(
option.to_owned(),
values.into_iter().map(|v| v.as_ref().to_string()).collect(),
)
}
}
impl Serialize for Arg {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
match self {
Arg::Flag(value) => serializer.serialize_str(value),
Arg::Option(option, value) => serializer.serialize_str(&format!("{option}={value}")),
Arg::Array(option, values) => {
serializer.serialize_str(&format!("{}=[{}]", option, values.join(",")))
},
}
}
}
struct ArgVisitor;
impl de::Visitor<'_> for ArgVisitor {
type Value = Arg;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("a string")
}
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: de::Error,
{
// covers the "-lruntime=debug,parachain=trace" case
// TODO: Make this more generic by adding the scenario in the regex below
if v.starts_with("-l") || v.starts_with("-log") {
return Ok(Arg::Flag(v.to_string()));
}
// Handle argument removal syntax: -:--flag-name
if v.starts_with("-:") {
return Ok(Arg::Flag(v.to_string()));
}
let re = Regex::new("^(?<name_prefix>(?<prefix>-{1,2})?(?<name>[a-zA-Z]+(-[a-zA-Z]+)*))((?<separator>=| )(?<value>\\[[^\\]]*\\]|[^ ]+))?$").unwrap();
let captures = re.captures(v);
if let Some(captures) = captures {
if let Some(value) = captures.name("value") {
let name_prefix = captures
.name("name_prefix")
.expect("BUG: name_prefix capture group missing")
.as_str()
.to_string();
let val = value.as_str();
if val.starts_with('[') && val.ends_with(']') {
// Remove brackets and split by comma
let inner = &val[1..val.len() - 1];
let items: Vec<String> = inner
.split(',')
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.collect();
return Ok(Arg::Array(name_prefix, items));
} else {
return Ok(Arg::Option(name_prefix, val.to_string()));
}
}
if let Some(name_prefix) = captures.name("name_prefix") {
return Ok(Arg::Flag(name_prefix.as_str().to_string()));
}
}
Err(de::Error::custom(
"the provided argument is invalid and doesn't match Arg::Option, Arg::Flag or Arg::Array",
))
}
}
impl<'de> Deserialize<'de> for Arg {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_any(ArgVisitor)
}
}
#[derive(Debug, Default, Clone)]
pub struct ValidationContext {
pub used_ports: Vec<Port>,
pub used_nodes_names: HashSet<String>,
// Store para_id already used
pub used_para_ids: HashMap<ParaId, u8>,
}
#[derive(Default, Debug, Clone, PartialEq, Deserialize)]
pub struct ChainDefaultContext {
pub(crate) default_command: Option<Command>,
pub(crate) default_image: Option<Image>,
pub(crate) default_resources: Option<Resources>,
pub(crate) default_db_snapshot: Option<AssetLocation>,
#[serde(default)]
pub(crate) default_args: Vec<Arg>,
}
/// Represent a runtime (.wasm) asset location and an
/// optional preset to use for chain-spec generation.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ChainSpecRuntime {
pub location: AssetLocation,
pub preset: Option<String>,
}
impl ChainSpecRuntime {
pub fn new(location: AssetLocation) -> Self {
ChainSpecRuntime {
location,
preset: None,
}
}
pub fn with_preset(location: AssetLocation, preset: impl Into<String>) -> Self {
ChainSpecRuntime {
location,
preset: Some(preset.into()),
}
}
}
/// Represents a set of JSON overrides for a configuration.
///
/// The overrides can be provided as an inline JSON object or loaded from a
/// separate file via a path or URL.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum JsonOverrides {
/// A path or URL pointing to a JSON file containing the overrides.
Location(AssetLocation),
/// An inline JSON value representing the overrides.
Json(serde_json::Value),
}
impl From<AssetLocation> for JsonOverrides {
fn from(value: AssetLocation) -> Self {
Self::Location(value)
}
}
impl From<serde_json::Value> for JsonOverrides {
fn from(value: serde_json::Value) -> Self {
Self::Json(value)
}
}
impl From<&str> for JsonOverrides {
fn from(value: &str) -> Self {
Self::Location(AssetLocation::from(value))
}
}
impl Display for JsonOverrides {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
JsonOverrides::Location(location) => write!(f, "{location}"),
JsonOverrides::Json(json) => write!(f, "{json}"),
}
}
}
impl JsonOverrides {
pub async fn get(&self) -> Result<serde_json::Value, anyhow::Error> {
let contents = match self {
Self::Location(location) => serde_json::from_slice(&location.get_asset().await?)
.map_err(|err| anyhow!("Error converting asset to json {location} - {err}")),
Self::Json(json) => Ok(json.clone()),
};
contents
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_arg_flag_roundtrip() {
let arg = Arg::from("verbose");
let serialized = serde_json::to_string(&arg).unwrap();
let deserialized: Arg = serde_json::from_str(&serialized).unwrap();
assert_eq!(arg, deserialized);
}
#[test]
fn test_arg_option_roundtrip() {
let arg = Arg::from(("mode", "fast"));
let serialized = serde_json::to_string(&arg).unwrap();
let deserialized: Arg = serde_json::from_str(&serialized).unwrap();
assert_eq!(arg, deserialized);
}
#[test]
fn test_arg_array_roundtrip() {
let arg = Arg::from(("items", ["a", "b", "c"].as_slice()));
let serialized = serde_json::to_string(&arg).unwrap();
println!("serialized = {serialized}");
let deserialized: Arg = serde_json::from_str(&serialized).unwrap();
assert_eq!(arg, deserialized);
}
#[test]
fn test_arg_option_valid_input() {
let expected = Arg::from(("--foo", "bar"));
// name and value delimited with =
let valid = "\"--foo=bar\"";
let result: Result<Arg, _> = serde_json::from_str(valid);
assert_eq!(result.unwrap(), expected);
// name and value delimited with space
let valid = "\"--foo bar\"";
let result: Result<Arg, _> = serde_json::from_str(valid);
assert_eq!(result.unwrap(), expected);
// value contains =
let expected = Arg::from(("--foo", "bar=baz"));
let valid = "\"--foo=bar=baz\"";
let result: Result<Arg, _> = serde_json::from_str(valid);
assert_eq!(result.unwrap(), expected);
}
#[test]
fn test_arg_array_valid_input() {
let expected = Arg::from(("--foo", vec!["bar", "baz"]));
// name and values delimited with =
let valid = "\"--foo=[bar,baz]\"";
let result: Result<Arg, _> = serde_json::from_str(valid);
assert_eq!(result.unwrap(), expected);
// name and values delimited with space
let valid = "\"--foo [bar,baz]\"";
let result: Result<Arg, _> = serde_json::from_str(valid);
assert_eq!(result.unwrap(), expected);
// values delimited with commas and space
let valid = "\"--foo [bar , baz]\"";
let result: Result<Arg, _> = serde_json::from_str(valid);
assert_eq!(result.unwrap(), expected);
// empty values array
let expected = Arg::from(("--foo", Vec::<&str>::new()));
let valid = "\"--foo []\"";
let result: Result<Arg, _> = serde_json::from_str(valid);
assert_eq!(result.unwrap(), expected);
}
#[test]
fn test_arg_invalid_input() {
// missing = or space
let invalid = "\"--foo[bar]\"";
let result: Result<Arg, _> = serde_json::from_str(invalid);
assert!(result.is_err());
// value contains space
let invalid = "\"--foo=bar baz\"";
let result: Result<Arg, _> = serde_json::from_str(invalid);
println!("result = {result:?}");
assert!(result.is_err());
}
#[test]
fn converting_a_str_without_whitespaces_into_a_chain_should_succeeds() {
let got: Result<Chain, ConversionError> = "mychain".try_into();
assert_eq!(got.unwrap().as_str(), "mychain");
}
#[test]
fn converting_a_str_containing_tag_name_into_an_image_should_succeeds() {
let got: Result<Image, ConversionError> = "myimage".try_into();
assert_eq!(got.unwrap().as_str(), "myimage");
}
#[test]
fn converting_a_str_containing_tag_name_and_tag_version_into_an_image_should_succeeds() {
let got: Result<Image, ConversionError> = "myimage:version".try_into();
assert_eq!(got.unwrap().as_str(), "myimage:version");
}
#[test]
fn converting_a_str_containing_hostname_and_tag_name_into_an_image_should_succeeds() {
let got: Result<Image, ConversionError> = "myrepository.com/myimage".try_into();
assert_eq!(got.unwrap().as_str(), "myrepository.com/myimage");
}
#[test]
fn converting_a_str_containing_hostname_tag_name_and_tag_version_into_an_image_should_succeeds()
{
let got: Result<Image, ConversionError> = "myrepository.com/myimage:version".try_into();
assert_eq!(got.unwrap().as_str(), "myrepository.com/myimage:version");
}
#[test]
fn converting_a_str_containing_ip_and_tag_name_into_an_image_should_succeeds() {
let got: Result<Image, ConversionError> = "myrepository.com/myimage".try_into();
assert_eq!(got.unwrap().as_str(), "myrepository.com/myimage");
}
#[test]
fn converting_a_str_containing_ip_tag_name_and_tag_version_into_an_image_should_succeeds() {
let got: Result<Image, ConversionError> = "127.0.0.1/myimage:version".try_into();
assert_eq!(got.unwrap().as_str(), "127.0.0.1/myimage:version");
}
#[test]
fn converting_a_str_without_whitespaces_into_a_command_should_succeeds() {
let got: Result<Command, ConversionError> = "mycommand".try_into();
assert_eq!(got.unwrap().as_str(), "mycommand");
}
#[test]
fn converting_an_url_into_an_asset_location_should_succeeds() {
let url = Url::from_str("https://mycloudstorage.com/path/to/my/file.tgz").unwrap();
let got: AssetLocation = url.clone().into();
assert!(matches!(got, AssetLocation::Url(value) if value == url));
}
#[test]
fn converting_a_pathbuf_into_an_asset_location_should_succeeds() {
let pathbuf = PathBuf::from_str("/tmp/path/to/my/file").unwrap();
let got: AssetLocation = pathbuf.clone().into();
assert!(matches!(got, AssetLocation::FilePath(value) if value == pathbuf));
}
#[test]
fn converting_a_str_into_an_url_asset_location_should_succeeds() {
let url = "https://mycloudstorage.com/path/to/my/file.tgz";
let got: AssetLocation = url.into();
assert!(matches!(got, AssetLocation::Url(value) if value == Url::from_str(url).unwrap()));
}
#[test]
fn converting_a_str_into_an_filepath_asset_location_should_succeeds() {
let filepath = "/tmp/path/to/my/file";
let got: AssetLocation = filepath.into();
assert!(matches!(
got,
AssetLocation::FilePath(value) if value == PathBuf::from_str(filepath).unwrap()
));
}
#[test]
fn converting_a_str_into_an_flag_arg_should_succeeds() {
let got: Arg = "myflag".into();
assert!(matches!(got, Arg::Flag(flag) if flag == "myflag"));
}
#[test]
fn converting_a_str_tuple_into_an_option_arg_should_succeeds() {
let got: Arg = ("name", "value").into();
assert!(matches!(got, Arg::Option(name, value) if name == "name" && value == "value"));
}
#[test]
fn converting_a_str_with_whitespaces_into_a_chain_should_fails() {
let got: Result<Chain, ConversionError> = "my chain".try_into();
assert!(matches!(
got.clone().unwrap_err(),
ConversionError::ContainsWhitespaces(_)
));
assert_eq!(
got.unwrap_err().to_string(),
"'my chain' shouldn't contains whitespace"
);
}
#[test]
fn converting_an_empty_str_into_a_chain_should_fails() {
let got: Result<Chain, ConversionError> = "".try_into();
assert!(matches!(
got.clone().unwrap_err(),
ConversionError::CantBeEmpty
));
assert_eq!(got.unwrap_err().to_string(), "can't be empty");
}
#[test]
fn converting_a_str_containing_only_ip_into_an_image_should_fails() {
let got: Result<Image, ConversionError> = "127.0.0.1".try_into();
assert!(matches!(
got.clone().unwrap_err(),
ConversionError::DoesntMatchRegex { value: _, regex: _ }
));
assert_eq!(
got.unwrap_err().to_string(),
"'127.0.0.1' doesn't match regex '^([ip]|[hostname]/)?[tag_name]:[tag_version]?$'"
);
}
#[test]
fn converting_a_str_containing_only_ip_and_tag_version_into_an_image_should_fails() {
let got: Result<Image, ConversionError> = "127.0.0.1:version".try_into();
assert!(matches!(
got.clone().unwrap_err(),
ConversionError::DoesntMatchRegex { value: _, regex: _ }
));
assert_eq!(got.unwrap_err().to_string(), "'127.0.0.1:version' doesn't match regex '^([ip]|[hostname]/)?[tag_name]:[tag_version]?$'");
}
#[test]
fn converting_a_str_containing_only_hostname_into_an_image_should_fails() {
let got: Result<Image, ConversionError> = "myrepository.com".try_into();
assert!(matches!(
got.clone().unwrap_err(),
ConversionError::DoesntMatchRegex { value: _, regex: _ }
));
assert_eq!(got.unwrap_err().to_string(), "'myrepository.com' doesn't match regex '^([ip]|[hostname]/)?[tag_name]:[tag_version]?$'");
}
#[test]
fn converting_a_str_containing_only_hostname_and_tag_version_into_an_image_should_fails() {
let got: Result<Image, ConversionError> = "myrepository.com:version".try_into();
assert!(matches!(
got.clone().unwrap_err(),
ConversionError::DoesntMatchRegex { value: _, regex: _ }
));
assert_eq!(got.unwrap_err().to_string(), "'myrepository.com:version' doesn't match regex '^([ip]|[hostname]/)?[tag_name]:[tag_version]?$'");
}
#[test]
fn converting_a_str_with_whitespaces_into_a_command_should_fails() {
let got: Result<Command, ConversionError> = "my command".try_into();
assert!(matches!(
got.clone().unwrap_err(),
ConversionError::ContainsWhitespaces(_)
));
assert_eq!(
got.unwrap_err().to_string(),
"'my command' shouldn't contains whitespace"
);
}
#[test]
fn test_convert_to_json_overrides() {
let url: AssetLocation = "https://example.com/overrides.json".into();
assert!(matches!(
url.into(),
JsonOverrides::Location(AssetLocation::Url(_))
));
let path: AssetLocation = "/path/to/overrides.json".into();
assert!(matches!(
path.into(),
JsonOverrides::Location(AssetLocation::FilePath(_))
));
let inline = serde_json::json!({ "para_id": 2000});
assert!(matches!(
inline.into(),
JsonOverrides::Json(serde_json::Value::Object(_))
));
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,65 @@
use std::env;
use support::constants::ZOMBIE_NODE_SPAWN_TIMEOUT_SECONDS;
use crate::types::{Chain, Command, Duration};
pub(crate) fn is_true(value: &bool) -> bool {
*value
}
pub(crate) fn is_false(value: &bool) -> bool {
!(*value)
}
pub(crate) fn default_as_true() -> bool {
true
}
pub(crate) fn default_as_false() -> bool {
false
}
pub(crate) fn default_initial_balance() -> crate::types::U128 {
2_000_000_000_000.into()
}
/// Default timeout for spawning a node (10mins)
pub(crate) fn default_node_spawn_timeout() -> Duration {
env::var(ZOMBIE_NODE_SPAWN_TIMEOUT_SECONDS)
.ok()
.and_then(|s| s.parse::<u32>().ok())
.unwrap_or(600)
}
/// Default timeout for spawning the whole network (1hr)
pub(crate) fn default_timeout() -> Duration {
3600
}
pub(crate) fn default_command_polkadot() -> Option<Command> {
TryInto::<Command>::try_into("polkadot").ok()
}
pub(crate) fn default_relaychain_chain() -> Chain {
TryInto::<Chain>::try_into("rococo-local").expect("'rococo-local' should be a valid chain")
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn default_node_spawn_timeout_works_before_and_after_env_is_set() {
// The default should be 600 seconds if the env var is not set
assert_eq!(default_node_spawn_timeout(), 600);
// If env var is set to a valid number, it should return that number
env::set_var(ZOMBIE_NODE_SPAWN_TIMEOUT_SECONDS, "123");
assert_eq!(default_node_spawn_timeout(), 123);
// If env var is set to a NOT valid number, it should return 600
env::set_var(ZOMBIE_NODE_SPAWN_TIMEOUT_SECONDS, "NOT_A_NUMBER");
assert_eq!(default_node_spawn_timeout(), 600);
}
}
@@ -0,0 +1,25 @@
[settings]
timeout = 3600
node_spawn_timeout = 600
tear_down_on_failure = true
[relaychain]
chain = "rococo-local"
default_command = "polkadot"
default_image = "docker.io/parity/polkadot:latest"
default_args = ["-lparachain=debug"]
[[relaychain.nodes]]
name = "alice"
validator = true
invulnerable = true
bootnode = false
balance = 2000000000000
[[relaychain.nodes]]
name = "bob"
args = ["--database=paritydb-experimental"]
validator = true
invulnerable = false
bootnode = true
balance = 2000000000000
@@ -0,0 +1,105 @@
[settings]
timeout = 3600
node_spawn_timeout = 600
tear_down_on_failure = true
[relaychain]
chain = "polkadot"
default_command = "polkadot"
default_image = "docker.io/parity/polkadot:latest"
[relaychain.default_resources.requests]
memory = "500M"
cpu = "100000"
[relaychain.default_resources.limits]
memory = "4000M"
cpu = "10Gi"
[[relaychain.nodes]]
name = "alice"
validator = true
invulnerable = true
bootnode = true
balance = 1000000000
[[relaychain.nodes]]
name = "bob"
validator = true
invulnerable = true
bootnode = true
balance = 2000000000000
[[teyrchains]]
id = 1000
chain = "myparachain"
register_para = true
onboard_as_teyrchain = false
balance = 2000000000000
default_db_snapshot = "https://storage.com/path/to/db_snapshot.tgz"
chain_spec_path = "/path/to/my/chain/spec.json"
cumulus_based = true
evm_based = false
[[teyrchains.collators]]
name = "john"
validator = true
invulnerable = true
bootnode = true
balance = 5000000000
[[teyrchains.collators]]
name = "charles"
validator = false
invulnerable = true
bootnode = true
balance = 0
[[teyrchains.collators]]
name = "frank"
validator = true
invulnerable = false
bootnode = true
balance = 1000000000
[[teyrchains]]
id = 2000
chain = "myotherparachain"
add_to_genesis = true
balance = 2000000000000
chain_spec_path = "/path/to/my/other/chain/spec.json"
cumulus_based = true
evm_based = false
[[teyrchains.collators]]
name = "mike"
validator = true
invulnerable = true
bootnode = true
balance = 5000000000
[[teyrchains.collators]]
name = "georges"
validator = false
invulnerable = true
bootnode = true
balance = 0
[[teyrchains.collators]]
name = "victor"
validator = true
invulnerable = false
bootnode = true
balance = 1000000000
[[hrmp_channels]]
sender = 1000
recipient = 2000
max_capacity = 150
max_message_size = 5000
[[hrmp_channels]]
sender = 2000
recipient = 1000
max_capacity = 200
max_message_size = 8000
@@ -0,0 +1,76 @@
[settings]
timeout = 3600
node_spawn_timeout = 600
tear_down_on_failure = true
[relaychain]
chain = "polkadot"
default_command = "polkadot"
default_image = "docker.io/parity/polkadot:latest"
default_db_snapshot = "https://storage.com/path/to/db_snapshot.tgz"
default_args = [
"-name=value",
"--flag",
]
[relaychain.default_resources.requests]
memory = "500M"
cpu = "100000"
[relaychain.default_resources.limits]
memory = "4000M"
cpu = "10Gi"
[[relaychain.nodes]]
name = "alice"
validator = true
invulnerable = true
bootnode = true
balance = 1000000000
[[relaychain.nodes]]
name = "bob"
image = "mycustomimage:latest"
command = "my-custom-command"
args = ["-myothername=value"]
validator = true
invulnerable = true
bootnode = true
balance = 2000000000000
db_snapshot = "https://storage.com/path/to/other/db_snapshot.tgz"
[relaychain.nodes.resources.requests]
memory = "250Mi"
cpu = "1000"
[relaychain.nodes.resources.limits]
memory = "2Gi"
cpu = "5Gi"
[[teyrchains]]
id = 1000
chain = "myparachain"
add_to_genesis = true
balance = 2000000000000
default_command = "my-default-command"
default_image = "mydefaultimage:latest"
default_db_snapshot = "https://storage.com/path/to/other_snapshot.tgz"
chain_spec_path = "/path/to/my/chain/spec.json"
cumulus_based = true
evm_based = false
[[teyrchains.collators]]
name = "john"
image = "anotherimage:latest"
command = "my-non-default-command"
validator = true
invulnerable = true
bootnode = true
balance = 5000000000
[[teyrchains.collators]]
name = "charles"
validator = false
invulnerable = true
bootnode = true
balance = 0
@@ -0,0 +1,57 @@
[settings]
timeout = 3600
node_spawn_timeout = 600
[relaychain]
chain = "rococo-local"
default_command = "polkadot"
default_image = "docker.io/parity/polkadot:latest"
default_args = ["-lparachain=debug"]
[[relaychain.nodes]]
name = "alice"
validator = true
invulnerable = true
bootnode = false
balance = 2000000000000
[[relaychain.nodes]]
name = "bob"
args = ["--database=paritydb-experimental"]
validator = true
invulnerable = false
bootnode = true
balance = 2000000000000
[[teyrchains]]
id = 1000
chain = "myparachain"
onboard_as_teyrchain = false
balance = 2000000000000
default_db_snapshot = "https://storage.com/path/to/db_snapshot.tgz"
chain_spec_path = "/path/to/my/chain/spec.json"
cumulus_based = true
[teyrchains.collator]
name = "john"
validator = true
invulnerable = true
bootnode = true
balance = 5000000000
[[teyrchains]]
id = 1000
chain = "myparachain"
onboard_as_teyrchain = false
balance = 2000000000000
default_db_snapshot = "https://storage.com/path/to/db_snapshot.tgz"
chain_spec_path = "/path/to/my/chain/spec.json"
cumulus_based = true
evm_based = true
[[teyrchains.collators]]
name = "john"
validator = true
invulnerable = true
bootnode = true
balance = 5000000000
@@ -0,0 +1,9 @@
[relaychain]
chain = "rococo-local"
default_command = "polkadot"
[[relaychain.nodes]]
name = "alice"
[[relaychain.nodes]]
name = "bob"
@@ -0,0 +1,17 @@
[relaychain]
chain = "rococo-local"
default_command = "polkadot"
wasm_override = "/some/path/runtime.wasm"
[[relaychain.nodes]]
name = "alice"
[[relaychain.nodes]]
name = "bob"
[[teyrchains]]
id = 1000
wasm_override = "https://some.com/runtime.wasm"
[teyrchains.collator]
name = "john"
@@ -0,0 +1,27 @@
[relaychain]
default_command = "polkadot"
chain_spec_path = "./rc.json"
[[relaychain.nodes]]
name = "alice"
validator = true
rpc_port = 9944
[[relaychain.nodes]]
name = "bob"
validator = true
rpc_port = 9945
args = [
"-lruntime::system=debug,runtime::session=trace,runtime::staking::ah-client=trace,runtime::ah-client=debug",
]
[[teyrchains]]
id = 1100
chain_spec_path = "./parachain.json"
[teyrchains.collator]
name = "charlie"
rpc_port = 9946
args = [
"-lruntime::system=debug,runtime::multiblock-election=trace,runtime::staking=debug,runtime::staking::rc-client=trace,runtime::rc-client=debug",
]
@@ -0,0 +1,40 @@
[settings]
timeout = 3600
node_spawn_timeout = 600
[relaychain]
chain = "rococo-local"
default_command = "polkadot"
default_image = "docker.io/parity/polkadot:latest"
default_args = ["-lparachain=debug"]
[[relaychain.nodes]]
name = "alice"
validator = true
invulnerable = true
bootnode = false
balance = 2000000000000
[[relaychain.nodes]]
name = "bob"
args = ["--database=paritydb-experimental"]
validator = true
invulnerable = false
bootnode = true
balance = 2000000000000
[[teyrchains]]
id = 1000
chain = "myparachain"
onboard_as_teyrchain = false
balance = 2000000000000
default_db_snapshot = "https://storage.com/path/to/db_snapshot.tgz"
chain_spec_path = "/path/to/my/chain/spec.json"
cumulus_based = true
[teyrchains.collator]
name = "alice"
validator = true
invulnerable = true
bootnode = true
balance = 5000000000
@@ -0,0 +1,17 @@
[relaychain]
chain = "rococo-local"
default_command = "polkadot"
raw_spec_override = "/some/path/raw_spec_override.json"
[[relaychain.nodes]]
name = "alice"
[[relaychain.nodes]]
name = "bob"
[[teyrchains]]
id = 1000
raw_spec_override = "https://some.com/raw_spec_override.json"
[teyrchains.collator]
name = "john"
@@ -0,0 +1,2 @@
/target
/Cargo.lock
@@ -0,0 +1,54 @@
[package]
name = "zombienet-orchestrator"
version.workspace = true
authors.workspace = true
edition.workspace = true
rust-version.workspace = true
publish = true
license.workspace = true
repository.workspace = true
description = "Zombienet Orchestrator, drive network spwan through providers"
keywords = ["zombienet", "orchestrator", "sdk"]
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
tokio = { workspace = true, features = ["time"] }
thiserror = { workspace = true }
multiaddr = { workspace = true }
serde_json = { workspace = true, features = ["arbitrary_precision"] }
futures = { workspace = true }
anyhow = { workspace = true }
rand = { workspace = true }
sha2 = { workspace = true, default-features = false }
hex = { workspace = true }
sp-core = { workspace = true }
libp2p = { workspace = true }
pezkuwi-subxt = { workspace = true }
pezkuwi-subxt-signer = { workspace = true }
reqwest = { workspace = true }
tracing = { workspace = true }
uuid = { workspace = true }
regex = { workspace = true }
glob-match = { workspace = true }
async-trait = { workspace = true }
serde = { workspace = true, features = ["derive"] }
libsecp256k1 = { workspace = true }
fancy-regex = { workspace = true }
# staging-chain-spec-builder = { workspace = true }
# parity-scale-codec = { version = "3.7.5", features = ["derive"] }
# sc-chain-spec = {workspace = true, default-features = false}
sc-chain-spec = { workspace = true }
erased-serde = { workspace = true }
# Zombienet deps
configuration = { workspace = true }
support = { workspace = true }
provider = { workspace = true }
prom-metrics-parser = { workspace = true }
[dev-dependencies]
toml = { workspace = true }
async-trait = { workspace = true }
lazy_static = { workspace = true }
@@ -0,0 +1,31 @@
//! Zombienet Orchestrator error definitions.
use provider::ProviderError;
use support::fs::FileSystemError;
use crate::generators;
#[derive(Debug, thiserror::Error)]
pub enum OrchestratorError {
// TODO: improve invalid config reporting
#[error("Invalid network configuration: {0}")]
InvalidConfig(String),
#[error("Invalid network config to use provider {0}: {1}")]
InvalidConfigForProvider(String, String),
#[error("Invalid configuration for node: {0}, field: {1}")]
InvalidNodeConfig(String, String),
#[error("Invariant not fulfilled {0}")]
InvariantError(&'static str),
#[error("Global network spawn timeout: {0} secs")]
GlobalTimeOut(u32),
#[error("Generator error: {0}")]
GeneratorError(#[from] generators::errors::GeneratorError),
#[error("Provider error")]
ProviderError(#[from] ProviderError),
#[error("FileSystem error")]
FileSystemError(#[from] FileSystemError),
#[error("Serialization error")]
SerializationError(#[from] serde_json::Error),
#[error(transparent)]
SpawnerError(#[from] anyhow::Error),
}
@@ -0,0 +1,22 @@
pub mod chain_spec;
pub mod errors;
pub mod key;
pub mod para_artifact;
mod arg_filter;
mod bootnode_addr;
mod command;
mod identity;
mod keystore;
mod keystore_key_types;
mod port;
pub use bootnode_addr::generate as generate_node_bootnode_addr;
pub use command::{
generate_for_cumulus_node as generate_node_command_cumulus,
generate_for_node as generate_node_command, GenCmdOptions,
};
pub use identity::generate as generate_node_identity;
pub use key::generate as generate_node_keys;
pub use keystore::generate as generate_node_keystore;
pub use port::generate as generate_node_port;
@@ -0,0 +1,138 @@
use configuration::types::Arg;
/// Parse args to extract those marked for removal (with `-:` prefix).
/// Returns a set of arg names/flags that should be removed from the final command.
///
/// # Examples
/// - `-:--insecure-validator-i-know-what-i-do` -> removes `--insecure-validator-i-know-what-i-do`
/// - `-:insecure-validator` -> removes `--insecure-validator` (normalized)
/// - `-:--prometheus-port` -> removes `--prometheus-port`
pub fn parse_removal_args(args: &[Arg]) -> Vec<String> {
args.iter()
.filter_map(|arg| match arg {
Arg::Flag(flag) if flag.starts_with("-:") => {
let mut flag_to_exclude = flag[2..].to_string();
// Normalize flag format - ensure it starts with --
if !flag_to_exclude.starts_with("--") {
flag_to_exclude = format!("--{flag_to_exclude}");
}
Some(flag_to_exclude)
},
_ => None,
})
.collect()
}
/// Apply arg removals to a vector of string arguments.
/// This filters out any args that match the removal list.
///
/// # Arguments
/// * `args` - The command arguments to filter
/// * `removals` - List of arg names/flags to remove
///
/// # Returns
/// Filtered vector with specified args removed
pub fn apply_arg_removals(args: Vec<String>, removals: &[String]) -> Vec<String> {
if removals.is_empty() {
return args;
}
let mut res = Vec::new();
let mut skip_next = false;
for (i, arg) in args.iter().enumerate() {
if skip_next {
skip_next = false;
continue;
}
let should_remove = removals
.iter()
.any(|removal| arg == removal || arg.starts_with(&format!("{removal}=")));
if should_remove {
// Only skip next if this looks like an option (starts with --) and next arg doesn't start with --
if !arg.contains("=") && i + 1 < args.len() {
let next_arg = &args[i + 1];
if !next_arg.starts_with("-") {
skip_next = true;
}
}
continue;
}
res.push(arg.clone());
}
res
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_removal_args() {
let args = vec![
Arg::Flag("-:--insecure-validator-i-know-what-i-do".to_string()),
Arg::Flag("--validator".to_string()),
Arg::Flag("-:--no-telemetry".to_string()),
];
let removals = parse_removal_args(&args);
assert_eq!(removals.len(), 2);
assert!(removals.contains(&"--insecure-validator-i-know-what-i-do".to_string()));
assert!(removals.contains(&"--no-telemetry".to_string()));
}
#[test]
fn test_apply_arg_removals_flag() {
let args = vec![
"--validator".to_string(),
"--insecure-validator-i-know-what-i-do".to_string(),
"--no-telemetry".to_string(),
];
let removals = vec!["--insecure-validator-i-know-what-i-do".to_string()];
let res = apply_arg_removals(args, &removals);
assert_eq!(res.len(), 2);
assert!(res.contains(&"--validator".to_string()));
assert!(res.contains(&"--no-telemetry".to_string()));
assert!(!res.contains(&"--insecure-validator-i-know-what-i-do".to_string()));
}
#[test]
fn test_apply_arg_removals_option_with_equals() {
let args = vec!["--name=alice".to_string(), "--port=30333".to_string()];
let removals = vec!["--port".to_string()];
let res = apply_arg_removals(args, &removals);
assert_eq!(res.len(), 1);
assert_eq!(res[0], "--name=alice");
}
#[test]
fn test_apply_arg_removals_option_with_space() {
let args = vec![
"--name".to_string(),
"alice".to_string(),
"--port".to_string(),
"30333".to_string(),
];
let removals = vec!["--port".to_string()];
let res = apply_arg_removals(args, &removals);
assert_eq!(res.len(), 2);
assert_eq!(res[0], "--name");
assert_eq!(res[1], "alice");
}
#[test]
fn test_apply_arg_removals_empty() {
let args = vec!["--validator".to_string()];
let removals = vec![];
let res = apply_arg_removals(args, &removals);
assert_eq!(res, vec!["--validator".to_string()]);
}
}
@@ -0,0 +1,111 @@
use std::{fmt::Display, net::IpAddr};
use super::errors::GeneratorError;
pub fn generate<T: AsRef<str> + Display>(
peer_id: &str,
ip: &IpAddr,
port: u16,
args: &[T],
p2p_cert: &Option<String>,
) -> Result<String, GeneratorError> {
let addr = if let Some(index) = args.iter().position(|arg| arg.as_ref().eq("--listen-addr")) {
let listen_value = args
.as_ref()
.get(index + 1)
.ok_or(GeneratorError::BootnodeAddrGeneration(
"can not generate bootnode address from args".into(),
))?
.to_string();
let ip_str = ip.to_string();
let port_str = port.to_string();
let mut parts = listen_value.split('/').collect::<Vec<&str>>();
parts[2] = &ip_str;
parts[4] = port_str.as_str();
parts.join("/")
} else {
format!("/ip4/{ip}/tcp/{port}/ws")
};
let mut addr_with_peer = format!("{addr}/p2p/{peer_id}");
if let Some(p2p_cert) = p2p_cert {
addr_with_peer.push_str("/certhash/");
addr_with_peer.push_str(p2p_cert)
}
Ok(addr_with_peer)
}
#[cfg(test)]
mod tests {
use provider::constants::LOCALHOST;
use super::*;
#[test]
fn generate_for_alice_without_args() {
let peer_id = "12D3KooWQCkBm1BYtkHpocxCwMgR8yjitEeHGx8spzcDLGt2gkBm"; // from alice as seed
let args: Vec<&str> = vec![];
let bootnode_addr = generate(peer_id, &LOCALHOST, 5678, &args, &None).unwrap();
assert_eq!(
&bootnode_addr,
"/ip4/127.0.0.1/tcp/5678/ws/p2p/12D3KooWQCkBm1BYtkHpocxCwMgR8yjitEeHGx8spzcDLGt2gkBm"
);
}
#[test]
fn generate_for_alice_with_listen_addr() {
// Should override the ip/port
let peer_id = "12D3KooWQCkBm1BYtkHpocxCwMgR8yjitEeHGx8spzcDLGt2gkBm"; // from alice as seed
let args: Vec<String> = [
"--some",
"other",
"--listen-addr",
"/ip4/192.168.100.1/tcp/30333/ws",
]
.iter()
.map(|x| x.to_string())
.collect();
let bootnode_addr =
generate(peer_id, &LOCALHOST, 5678, args.iter().as_ref(), &None).unwrap();
assert_eq!(
&bootnode_addr,
"/ip4/127.0.0.1/tcp/5678/ws/p2p/12D3KooWQCkBm1BYtkHpocxCwMgR8yjitEeHGx8spzcDLGt2gkBm"
);
}
#[test]
fn generate_for_alice_with_listen_addr_without_value_must_fail() {
// Should override the ip/port
let peer_id = "12D3KooWQCkBm1BYtkHpocxCwMgR8yjitEeHGx8spzcDLGt2gkBm"; // from alice as seed
let args: Vec<String> = ["--some", "other", "--listen-addr"]
.iter()
.map(|x| x.to_string())
.collect();
let bootnode_addr = generate(peer_id, &LOCALHOST, 5678, args.iter().as_ref(), &None);
assert!(bootnode_addr.is_err());
assert!(matches!(
bootnode_addr,
Err(GeneratorError::BootnodeAddrGeneration(_))
));
}
#[test]
fn generate_for_alice_withcert() {
let peer_id = "12D3KooWQCkBm1BYtkHpocxCwMgR8yjitEeHGx8spzcDLGt2gkBm"; // from alice as seed
let args: Vec<&str> = vec![];
let bootnode_addr = generate(
peer_id,
&LOCALHOST,
5678,
&args,
&Some(String::from("data")),
)
.unwrap();
assert_eq!(
&bootnode_addr,
"/ip4/127.0.0.1/tcp/5678/ws/p2p/12D3KooWQCkBm1BYtkHpocxCwMgR8yjitEeHGx8spzcDLGt2gkBm/certhash/data"
);
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,634 @@
use configuration::types::Arg;
use support::constants::THIS_IS_A_BUG;
use super::arg_filter::{apply_arg_removals, parse_removal_args};
use crate::{network_spec::node::NodeSpec, shared::constants::*};
pub struct GenCmdOptions<'a> {
pub relay_chain_name: &'a str,
pub cfg_path: &'a str,
pub data_path: &'a str,
pub relay_data_path: &'a str,
pub use_wrapper: bool,
pub bootnode_addr: Vec<String>,
pub use_default_ports_in_cmd: bool,
pub is_native: bool,
}
impl Default for GenCmdOptions<'_> {
fn default() -> Self {
Self {
relay_chain_name: "rococo-local",
cfg_path: "/cfg",
data_path: "/data",
relay_data_path: "/relay-data",
use_wrapper: true,
bootnode_addr: vec![],
use_default_ports_in_cmd: false,
is_native: true,
}
}
}
const FLAGS_ADDED_BY_US: [&str; 3] = ["--no-telemetry", "--collator", "--"];
const OPS_ADDED_BY_US: [&str; 6] = [
"--chain",
"--name",
"--rpc-cors",
"--rpc-methods",
"--parachain-id",
"--node-key",
];
// TODO: can we abstract this and use only one fn (or at least split and reuse in small fns)
pub fn generate_for_cumulus_node(
node: &NodeSpec,
options: GenCmdOptions,
para_id: u32,
) -> (String, Vec<String>) {
let NodeSpec {
key,
args,
is_validator,
bootnodes_addresses,
..
} = node;
let mut tmp_args: Vec<String> = vec!["--node-key".into(), key.clone()];
if !args.contains(&Arg::Flag("--prometheus-external".into())) {
tmp_args.push("--prometheus-external".into())
}
if *is_validator && !args.contains(&Arg::Flag("--validator".into())) {
tmp_args.push("--collator".into())
}
if !bootnodes_addresses.is_empty() {
tmp_args.push("--bootnodes".into());
let bootnodes = bootnodes_addresses
.iter()
.map(|m| m.to_string())
.collect::<Vec<String>>()
.join(" ");
tmp_args.push(bootnodes)
}
// ports
let (prometheus_port, rpc_port, p2p_port) =
resolve_ports(node, options.use_default_ports_in_cmd);
tmp_args.push("--prometheus-port".into());
tmp_args.push(prometheus_port.to_string());
tmp_args.push("--rpc-port".into());
tmp_args.push(rpc_port.to_string());
tmp_args.push("--listen-addr".into());
tmp_args.push(format!("/ip4/0.0.0.0/tcp/{p2p_port}/ws"));
let mut collator_args: &[Arg] = &[];
let mut full_node_args: &[Arg] = &[];
if !args.is_empty() {
if let Some(index) = args.iter().position(|arg| match arg {
Arg::Flag(flag) => flag.eq("--"),
Arg::Option(..) => false,
Arg::Array(..) => false,
}) {
(collator_args, full_node_args) = args.split_at(index);
} else {
// Assume args are those specified for collator only
collator_args = args;
}
}
// set our base path
tmp_args.push("--base-path".into());
tmp_args.push(options.data_path.into());
let node_specific_bootnodes: Vec<String> = node
.bootnodes_addresses
.iter()
.map(|b| b.to_string())
.collect();
let full_bootnodes = [node_specific_bootnodes, options.bootnode_addr].concat();
if !full_bootnodes.is_empty() {
tmp_args.push("--bootnodes".into());
tmp_args.push(full_bootnodes.join(" "));
}
let mut full_node_p2p_needs_to_be_injected = true;
let mut full_node_prometheus_needs_to_be_injected = true;
let mut full_node_args_filtered = full_node_args
.iter()
.filter_map(|arg| match arg {
Arg::Flag(flag) => {
if flag.starts_with("-:") || FLAGS_ADDED_BY_US.contains(&flag.as_str()) {
None
} else {
Some(vec![flag.to_owned()])
}
},
Arg::Option(k, v) => {
if OPS_ADDED_BY_US.contains(&k.as_str()) {
None
} else if k.eq(&"port") {
if v.eq(&"30333") {
full_node_p2p_needs_to_be_injected = true;
None
} else {
// non default
full_node_p2p_needs_to_be_injected = false;
Some(vec![k.to_owned(), v.to_owned()])
}
} else if k.eq(&"--prometheus-port") {
if v.eq(&"9616") {
full_node_prometheus_needs_to_be_injected = true;
None
} else {
// non default
full_node_prometheus_needs_to_be_injected = false;
Some(vec![k.to_owned(), v.to_owned()])
}
} else {
Some(vec![k.to_owned(), v.to_owned()])
}
},
Arg::Array(k, v) => {
let mut args = vec![k.to_owned()];
args.extend(v.to_owned());
Some(args)
},
})
.flatten()
.collect::<Vec<String>>();
let full_p2p_port = node
.full_node_p2p_port
.as_ref()
.expect(&format!(
"full node p2p_port should be specifed: {THIS_IS_A_BUG}"
))
.0;
let full_prometheus_port = node
.full_node_prometheus_port
.as_ref()
.expect(&format!(
"full node prometheus_port should be specifed: {THIS_IS_A_BUG}"
))
.0;
// full_node: change p2p port if is the default
if full_node_p2p_needs_to_be_injected {
full_node_args_filtered.push("--port".into());
full_node_args_filtered.push(full_p2p_port.to_string());
}
// full_node: change prometheus port if is the default
if full_node_prometheus_needs_to_be_injected {
full_node_args_filtered.push("--prometheus-port".into());
full_node_args_filtered.push(full_prometheus_port.to_string());
}
let mut args_filtered = collator_args
.iter()
.filter_map(|arg| match arg {
Arg::Flag(flag) => {
if flag.starts_with("-:") || FLAGS_ADDED_BY_US.contains(&flag.as_str()) {
None
} else {
Some(vec![flag.to_owned()])
}
},
Arg::Option(k, v) => {
if OPS_ADDED_BY_US.contains(&k.as_str()) {
None
} else {
Some(vec![k.to_owned(), v.to_owned()])
}
},
Arg::Array(k, v) => {
let mut args = vec![k.to_owned()];
args.extend(v.to_owned());
Some(args)
},
})
.flatten()
.collect::<Vec<String>>();
tmp_args.append(&mut args_filtered);
let parachain_spec_path = format!("{}/{}.json", options.cfg_path, para_id);
let mut final_args = vec![
node.command.as_str().to_string(),
"--chain".into(),
parachain_spec_path,
"--name".into(),
node.name.clone(),
"--rpc-cors".into(),
"all".into(),
"--rpc-methods".into(),
"unsafe".into(),
];
// The `--unsafe-rpc-external` option spawns an additional RPC server on a random port,
// which can conflict with reserved ports, causing an "Address already in use" error
// when using the `native` provider. Since this option isn't needed for `native`,
// it should be omitted in that case.
if !options.is_native {
final_args.push("--unsafe-rpc-external".into());
}
final_args.append(&mut tmp_args);
let relaychain_spec_path = format!("{}/{}.json", options.cfg_path, options.relay_chain_name);
let mut full_node_injected: Vec<String> = vec![
"--".into(),
"--base-path".into(),
options.relay_data_path.into(),
"--chain".into(),
relaychain_spec_path,
"--execution".into(),
"wasm".into(),
];
final_args.append(&mut full_node_injected);
final_args.append(&mut full_node_args_filtered);
let removals = parse_removal_args(args);
final_args = apply_arg_removals(final_args, &removals);
if options.use_wrapper {
("/cfg/zombie-wrapper.sh".to_string(), final_args)
} else {
(final_args.remove(0), final_args)
}
}
pub fn generate_for_node(
node: &NodeSpec,
options: GenCmdOptions,
para_id: Option<u32>,
) -> (String, Vec<String>) {
let NodeSpec {
key,
args,
is_validator,
bootnodes_addresses,
..
} = node;
let mut tmp_args: Vec<String> = vec![
"--node-key".into(),
key.clone(),
// TODO:(team) we should allow to set the telemetry url from config
"--no-telemetry".into(),
];
if !args.contains(&Arg::Flag("--prometheus-external".into())) {
tmp_args.push("--prometheus-external".into())
}
if let Some(para_id) = para_id {
tmp_args.push("--parachain-id".into());
tmp_args.push(para_id.to_string());
}
if *is_validator && !args.contains(&Arg::Flag("--validator".into())) {
tmp_args.push("--validator".into());
if node.supports_arg("--insecure-validator-i-know-what-i-do") {
tmp_args.push("--insecure-validator-i-know-what-i-do".into());
}
}
if !bootnodes_addresses.is_empty() {
tmp_args.push("--bootnodes".into());
let bootnodes = bootnodes_addresses
.iter()
.map(|m| m.to_string())
.collect::<Vec<String>>()
.join(" ");
tmp_args.push(bootnodes)
}
// ports
let (prometheus_port, rpc_port, p2p_port) =
resolve_ports(node, options.use_default_ports_in_cmd);
// Prometheus
tmp_args.push("--prometheus-port".into());
tmp_args.push(prometheus_port.to_string());
// RPC
// TODO (team): do we want to support old --ws-port?
tmp_args.push("--rpc-port".into());
tmp_args.push(rpc_port.to_string());
let listen_value = if let Some(listen_val) = args.iter().find_map(|arg| match arg {
Arg::Flag(_) => None,
Arg::Option(k, v) => {
if k.eq("--listen-addr") {
Some(v)
} else {
None
}
},
Arg::Array(..) => None,
}) {
let mut parts = listen_val.split('/').collect::<Vec<&str>>();
// TODO: move this to error
let port_part = parts
.get_mut(4)
.expect(&format!("should have at least 5 parts {THIS_IS_A_BUG}"));
let port_to_use = p2p_port.to_string();
*port_part = port_to_use.as_str();
parts.join("/")
} else {
format!("/ip4/0.0.0.0/tcp/{p2p_port}/ws")
};
tmp_args.push("--listen-addr".into());
tmp_args.push(listen_value);
// set our base path
tmp_args.push("--base-path".into());
tmp_args.push(options.data_path.into());
let node_specific_bootnodes: Vec<String> = node
.bootnodes_addresses
.iter()
.map(|b| b.to_string())
.collect();
let full_bootnodes = [node_specific_bootnodes, options.bootnode_addr].concat();
if !full_bootnodes.is_empty() {
tmp_args.push("--bootnodes".into());
tmp_args.push(full_bootnodes.join(" "));
}
// add the rest of the args
let mut args_filtered = args
.iter()
.filter_map(|arg| match arg {
Arg::Flag(flag) => {
if flag.starts_with("-:") || FLAGS_ADDED_BY_US.contains(&flag.as_str()) {
None
} else {
Some(vec![flag.to_owned()])
}
},
Arg::Option(k, v) => {
if OPS_ADDED_BY_US.contains(&k.as_str()) {
None
} else {
Some(vec![k.to_owned(), v.to_owned()])
}
},
Arg::Array(k, v) => {
let mut args = vec![k.to_owned()];
args.extend(v.to_owned());
Some(args)
},
})
.flatten()
.collect::<Vec<String>>();
tmp_args.append(&mut args_filtered);
let chain_spec_path = format!("{}/{}.json", options.cfg_path, options.relay_chain_name);
let mut final_args = vec![
node.command.as_str().to_string(),
"--chain".into(),
chain_spec_path,
"--name".into(),
node.name.clone(),
"--rpc-cors".into(),
"all".into(),
"--rpc-methods".into(),
"unsafe".into(),
];
// The `--unsafe-rpc-external` option spawns an additional RPC server on a random port,
// which can conflict with reserved ports, causing an "Address already in use" error
// when using the `native` provider. Since this option isn't needed for `native`,
// it should be omitted in that case.
if !options.is_native {
final_args.push("--unsafe-rpc-external".into());
}
final_args.append(&mut tmp_args);
if let Some(ref subcommand) = node.subcommand {
final_args.insert(1, subcommand.as_str().to_string());
}
let removals = parse_removal_args(args);
final_args = apply_arg_removals(final_args, &removals);
if options.use_wrapper {
("/cfg/zombie-wrapper.sh".to_string(), final_args)
} else {
(final_args.remove(0), final_args)
}
}
/// Returns (prometheus, rpc, p2p) ports to use in the command
fn resolve_ports(node: &NodeSpec, use_default_ports_in_cmd: bool) -> (u16, u16, u16) {
if use_default_ports_in_cmd {
(PROMETHEUS_PORT, RPC_PORT, P2P_PORT)
} else {
(node.prometheus_port.0, node.rpc_port.0, node.p2p_port.0)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{generators, shared::types::NodeAccounts};
fn get_node_spec(full_node_present: bool) -> NodeSpec {
let mut name = String::from("luca");
let initial_balance = 1_000_000_000_000_u128;
let seed = format!("//{}{name}", name.remove(0).to_uppercase());
let accounts = NodeAccounts {
accounts: generators::generate_node_keys(&seed).unwrap(),
seed,
};
let (full_node_p2p_port, full_node_prometheus_port) = if full_node_present {
(
Some(generators::generate_node_port(None).unwrap()),
Some(generators::generate_node_port(None).unwrap()),
)
} else {
(None, None)
};
NodeSpec {
name,
accounts,
initial_balance,
full_node_p2p_port,
full_node_prometheus_port,
..Default::default()
}
}
#[test]
fn generate_for_native_cumulus_node_works() {
let node = get_node_spec(true);
let opts = GenCmdOptions {
use_wrapper: false,
is_native: true,
..GenCmdOptions::default()
};
let (program, args) = generate_for_cumulus_node(&node, opts, 1000);
assert_eq!(program.as_str(), "polkadot");
let divider_flag = args.iter().position(|x| x == "--").unwrap();
// ensure full node ports
let i = args[divider_flag..]
.iter()
.position(|x| {
x == node
.full_node_p2p_port
.as_ref()
.unwrap()
.0
.to_string()
.as_str()
})
.unwrap();
assert_eq!(&args[divider_flag + i - 1], "--port");
let i = args[divider_flag..]
.iter()
.position(|x| {
x == node
.full_node_prometheus_port
.as_ref()
.unwrap()
.0
.to_string()
.as_str()
})
.unwrap();
assert_eq!(&args[divider_flag + i - 1], "--prometheus-port");
assert!(!args.iter().any(|arg| arg == "--unsafe-rpc-external"));
}
#[test]
fn generate_for_native_cumulus_node_rpc_external_is_not_removed_if_is_set_by_user() {
let mut node = get_node_spec(true);
node.args.push("--unsafe-rpc-external".into());
let opts = GenCmdOptions {
use_wrapper: false,
is_native: true,
..GenCmdOptions::default()
};
let (_, args) = generate_for_cumulus_node(&node, opts, 1000);
assert!(args.iter().any(|arg| arg == "--unsafe-rpc-external"));
}
#[test]
fn generate_for_non_native_cumulus_node_works() {
let node = get_node_spec(true);
let opts = GenCmdOptions {
use_wrapper: false,
is_native: false,
..GenCmdOptions::default()
};
let (program, args) = generate_for_cumulus_node(&node, opts, 1000);
assert_eq!(program.as_str(), "polkadot");
let divider_flag = args.iter().position(|x| x == "--").unwrap();
// ensure full node ports
let i = args[divider_flag..]
.iter()
.position(|x| {
x == node
.full_node_p2p_port
.as_ref()
.unwrap()
.0
.to_string()
.as_str()
})
.unwrap();
assert_eq!(&args[divider_flag + i - 1], "--port");
let i = args[divider_flag..]
.iter()
.position(|x| {
x == node
.full_node_prometheus_port
.as_ref()
.unwrap()
.0
.to_string()
.as_str()
})
.unwrap();
assert_eq!(&args[divider_flag + i - 1], "--prometheus-port");
// we expect to find this arg in collator node part
assert!(&args[0..divider_flag]
.iter()
.any(|arg| arg == "--unsafe-rpc-external"));
}
#[test]
fn generate_for_native_node_rpc_external_works() {
let node = get_node_spec(false);
let opts = GenCmdOptions {
use_wrapper: false,
is_native: true,
..GenCmdOptions::default()
};
let (program, args) = generate_for_node(&node, opts, Some(1000));
assert_eq!(program.as_str(), "polkadot");
assert!(!args.iter().any(|arg| arg == "--unsafe-rpc-external"));
}
#[test]
fn generate_for_non_native_node_rpc_external_works() {
let node = get_node_spec(false);
let opts = GenCmdOptions {
use_wrapper: false,
is_native: false,
..GenCmdOptions::default()
};
let (program, args) = generate_for_node(&node, opts, Some(1000));
assert_eq!(program.as_str(), "polkadot");
assert!(args.iter().any(|arg| arg == "--unsafe-rpc-external"));
}
#[test]
fn test_arg_removal_removes_insecure_validator_flag() {
let mut node = get_node_spec(false);
node.args
.push(Arg::Flag("-:--insecure-validator-i-know-what-i-do".into()));
node.is_validator = true;
node.available_args_output = Some("--insecure-validator-i-know-what-i-do".to_string());
let opts = GenCmdOptions {
use_wrapper: false,
is_native: true,
..GenCmdOptions::default()
};
let (program, args) = generate_for_node(&node, opts, Some(1000));
assert_eq!(program.as_str(), "polkadot");
assert!(args.iter().any(|arg| arg == "--validator"));
assert!(!args
.iter()
.any(|arg| arg == "--insecure-validator-i-know-what-i-do"));
}
}
@@ -0,0 +1,24 @@
use provider::ProviderError;
use support::fs::FileSystemError;
#[derive(Debug, thiserror::Error)]
pub enum GeneratorError {
#[error("Generating key {0} with input {1}")]
KeyGeneration(String, String),
#[error("Generating port {0}, err {1}")]
PortGeneration(u16, String),
#[error("Chain-spec build error: {0}")]
ChainSpecGeneration(String),
#[error("Provider error: {0}")]
ProviderError(#[from] ProviderError),
#[error("FileSystem error")]
FileSystemError(#[from] FileSystemError),
#[error("Generating identity, err {0}")]
IdentityGeneration(String),
#[error("Generating bootnode address, err {0}")]
BootnodeAddrGeneration(String),
#[error("Error overriding wasm on raw chain-spec, err {0}")]
OverridingWasm(String),
#[error("Error overriding raw chain-spec, err {0}")]
OverridingRawSpec(String),
}
@@ -0,0 +1,41 @@
use hex::FromHex;
use libp2p::identity::{ed25519, Keypair};
use sha2::digest::Digest;
use super::errors::GeneratorError;
// Generate p2p identity for node
// return `node-key` and `peerId`
pub fn generate(node_name: &str) -> Result<(String, String), GeneratorError> {
let key = hex::encode(sha2::Sha256::digest(node_name));
let bytes = <[u8; 32]>::from_hex(key.clone()).map_err(|_| {
GeneratorError::IdentityGeneration("can not transform hex to [u8;32]".into())
})?;
let sk = ed25519::SecretKey::try_from_bytes(bytes)
.map_err(|_| GeneratorError::IdentityGeneration("can not create sk from bytes".into()))?;
let local_identity: Keypair = ed25519::Keypair::from(sk).into();
let local_public = local_identity.public();
let local_peer_id = local_public.to_peer_id();
Ok((key, local_peer_id.to_base58()))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn generate_for_alice() {
let s = "alice";
let (key, peer_id) = generate(s).unwrap();
assert_eq!(
&key,
"2bd806c97f0e00af1a1fc3328fa763a9269723c8db8fac4f93af71db186d6e90"
);
assert_eq!(
&peer_id,
"12D3KooWQCkBm1BYtkHpocxCwMgR8yjitEeHGx8spzcDLGt2gkBm"
);
}
}
@@ -0,0 +1,151 @@
use sp_core::{crypto::SecretStringError, ecdsa, ed25519, keccak_256, sr25519, Pair, H160, H256};
use super::errors::GeneratorError;
use crate::shared::types::{Accounts, NodeAccount};
const KEYS: [&str; 5] = ["sr", "sr_stash", "ed", "ec", "eth"];
pub fn generate_pair<T: Pair>(seed: &str) -> Result<T::Pair, SecretStringError> {
let pair = T::Pair::from_string(seed, None)?;
Ok(pair)
}
pub fn generate(seed: &str) -> Result<Accounts, GeneratorError> {
let mut accounts: Accounts = Default::default();
for k in KEYS {
let (address, public_key) = match k {
"sr" => {
let pair = generate_pair::<sr25519::Pair>(seed)
.map_err(|_| GeneratorError::KeyGeneration(k.into(), seed.into()))?;
(pair.public().to_string(), hex::encode(pair.public()))
},
"sr_stash" => {
let pair = generate_pair::<sr25519::Pair>(&format!("{seed}//stash"))
.map_err(|_| GeneratorError::KeyGeneration(k.into(), seed.into()))?;
(pair.public().to_string(), hex::encode(pair.public()))
},
"ed" => {
let pair = generate_pair::<ed25519::Pair>(seed)
.map_err(|_| GeneratorError::KeyGeneration(k.into(), seed.into()))?;
(pair.public().to_string(), hex::encode(pair.public()))
},
"ec" => {
let pair = generate_pair::<ecdsa::Pair>(seed)
.map_err(|_| GeneratorError::KeyGeneration(k.into(), seed.into()))?;
(pair.public().to_string(), hex::encode(pair.public()))
},
"eth" => {
let pair = generate_pair::<ecdsa::Pair>(seed)
.map_err(|_| GeneratorError::KeyGeneration(k.into(), seed.into()))?;
let decompressed = libsecp256k1::PublicKey::parse_compressed(&pair.public().0)
.map_err(|_| GeneratorError::KeyGeneration(k.into(), seed.into()))?
.serialize();
let mut m = [0u8; 64];
m.copy_from_slice(&decompressed[1..65]);
let account = H160::from(H256::from(keccak_256(&m)));
(hex::encode(account), hex::encode(account))
},
_ => unreachable!(),
};
accounts.insert(k.into(), NodeAccount::new(address, public_key));
}
Ok(accounts)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn generate_for_alice() {
use sp_core::crypto::Ss58Codec;
let s = "Alice";
let seed = format!("//{s}");
let pair = generate_pair::<sr25519::Pair>(&seed).unwrap();
assert_eq!(
"5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY",
pair.public().to_ss58check()
);
let pair = generate_pair::<ecdsa::Pair>(&seed).unwrap();
assert_eq!(
"0x020a1091341fe5664bfa1782d5e04779689068c916b04cb365ec3153755684d9a1",
format!("0x{}", hex::encode(pair.public()))
);
let pair = generate_pair::<ed25519::Pair>(&seed).unwrap();
assert_eq!(
"5FA9nQDVg267DEd8m1ZypXLBnvN7SFxYwV7ndqSYGiN9TTpu",
pair.public().to_ss58check()
);
}
#[test]
fn generate_for_zombie() {
use sp_core::crypto::Ss58Codec;
let s = "Zombie";
let seed = format!("//{s}");
let pair = generate_pair::<sr25519::Pair>(&seed).unwrap();
assert_eq!(
"5FTcLfwFc7ctvqp3RhbEig6UuHLHcHVRujuUm8r21wy4dAR8",
pair.public().to_ss58check()
);
}
#[test]
fn generate_pair_invalid_should_fail() {
let s = "Alice";
let seed = s.to_string();
let pair = generate_pair::<sr25519::Pair>(&seed);
assert!(pair.is_err());
}
#[test]
fn generate_invalid_should_fail() {
let s = "Alice";
let seed = s.to_string();
let pair = generate(&seed);
assert!(pair.is_err());
assert!(matches!(pair, Err(GeneratorError::KeyGeneration(_, _))));
}
#[test]
fn generate_work() {
let s = "Alice";
let seed = format!("//{s}");
let pair = generate(&seed).unwrap();
let sr = pair.get("sr").unwrap();
let sr_stash = pair.get("sr_stash").unwrap();
let ed = pair.get("ed").unwrap();
let ec = pair.get("ec").unwrap();
let eth = pair.get("eth").unwrap();
assert_eq!(
sr.address,
"5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY"
);
assert_eq!(
sr_stash.address,
"5GNJqTPyNqANBkUVMN1LPPrxXnFouWXoe2wNSmmEoLctxiZY"
);
assert_eq!(
ed.address,
"5FA9nQDVg267DEd8m1ZypXLBnvN7SFxYwV7ndqSYGiN9TTpu"
);
assert_eq!(
format!("0x{}", ec.public_key),
"0x020a1091341fe5664bfa1782d5e04779689068c916b04cb365ec3153755684d9a1"
);
assert_eq!(
format!("0x{}", eth.public_key),
"0xe04cc55ebee1cbce552f250e85c57b70b2e2625b"
)
}
}
@@ -0,0 +1,290 @@
use std::{
path::{Path, PathBuf},
vec,
};
use hex::encode;
use support::{constants::THIS_IS_A_BUG, fs::FileSystem};
use super::errors::GeneratorError;
use crate::{
generators::keystore_key_types::{parse_keystore_key_types, KeystoreKeyType},
shared::types::NodeAccounts,
ScopedFilesystem,
};
/// Generates keystore files for a node.
///
/// # Arguments
/// * `acc` - The node accounts containing the seed and public keys
/// * `node_files_path` - The path where keystore files will be created
/// * `scoped_fs` - The scoped filesystem for file operations
/// * `asset_hub_polkadot` - Whether this is for asset-hub-polkadot (affects aura key scheme)
/// * `keystore_key_types` - Optional list of key type specifications
///
/// If `keystore_key_types` is empty, all default key types will be generated.
/// Otherwise, only the specified key types will be generated.
pub async fn generate<'a, T>(
acc: &NodeAccounts,
node_files_path: impl AsRef<Path>,
scoped_fs: &ScopedFilesystem<'a, T>,
asset_hub_polkadot: bool,
keystore_key_types: Vec<&str>,
) -> Result<Vec<PathBuf>, GeneratorError>
where
T: FileSystem,
{
// Create local keystore
scoped_fs.create_dir_all(node_files_path.as_ref()).await?;
let mut filenames = vec![];
// Parse the key type specifications
let key_types = parse_keystore_key_types(&keystore_key_types, asset_hub_polkadot);
let futures: Vec<_> = key_types
.iter()
.map(|key_type| {
let filename = generate_keystore_filename(key_type, acc);
let file_path = PathBuf::from(format!(
"{}/{}",
node_files_path.as_ref().to_string_lossy(),
filename
));
let content = format!("\"{}\"", acc.seed);
(filename, scoped_fs.write(file_path, content))
})
.collect();
for (filename, future) in futures {
future.await?;
filenames.push(PathBuf::from(filename));
}
Ok(filenames)
}
/// Generates the keystore filename for a given key type.
///
/// The filename format is: `{hex_encoded_key_type}{public_key}`
fn generate_keystore_filename(key_type: &KeystoreKeyType, acc: &NodeAccounts) -> String {
let account_key = key_type.scheme.account_key();
let pk = acc
.accounts
.get(account_key)
.expect(&format!(
"Key '{}' should be set for node {THIS_IS_A_BUG}",
account_key
))
.public_key
.as_str();
format!("{}{}", encode(&key_type.key_type), pk)
}
#[cfg(test)]
mod tests {
use std::{collections::HashMap, ffi::OsString, str::FromStr};
use support::fs::in_memory::{InMemoryFile, InMemoryFileSystem};
use super::*;
use crate::shared::types::{NodeAccount, NodeAccounts};
fn create_test_accounts() -> NodeAccounts {
let mut accounts = HashMap::new();
accounts.insert(
"sr".to_string(),
NodeAccount::new("sr_address", "sr_public_key"),
);
accounts.insert(
"ed".to_string(),
NodeAccount::new("ed_address", "ed_public_key"),
);
accounts.insert(
"ec".to_string(),
NodeAccount::new("ec_address", "ec_public_key"),
);
NodeAccounts {
seed: "//Alice".to_string(),
accounts,
}
}
fn create_test_fs() -> InMemoryFileSystem {
InMemoryFileSystem::new(HashMap::from([(
OsString::from_str("/").unwrap(),
InMemoryFile::dir(),
)]))
}
#[tokio::test]
async fn generate_creates_default_keystore_files_when_no_key_types_specified() {
let accounts = create_test_accounts();
let fs = create_test_fs();
let base_dir = "/tmp/test";
let scoped_fs = ScopedFilesystem { fs: &fs, base_dir };
let key_types: Vec<&str> = vec![];
let res = generate(&accounts, "node1", &scoped_fs, false, key_types).await;
assert!(res.is_ok());
let filenames = res.unwrap();
assert!(filenames.len() > 10);
let filename_strs: Vec<String> = filenames
.iter()
.map(|p| p.to_string_lossy().to_string())
.collect();
// Check that aura key is generated (hex of "aura" is 61757261)
assert!(filename_strs.iter().any(|f| f.starts_with("61757261")));
// Check that babe key is generated (hex of "babe" is 62616265)
assert!(filename_strs.iter().any(|f| f.starts_with("62616265")));
// Check that gran key is generated (hex of "gran" is 6772616e)
assert!(filename_strs.iter().any(|f| f.starts_with("6772616e")));
}
#[tokio::test]
async fn generate_creates_only_specified_keystore_files() {
let accounts = create_test_accounts();
let fs = create_test_fs();
let base_dir = "/tmp/test";
let scoped_fs = ScopedFilesystem { fs: &fs, base_dir };
let key_types = vec!["audi", "gran"];
let res = generate(&accounts, "node1", &scoped_fs, false, key_types).await;
assert!(res.is_ok());
let filenames = res.unwrap();
assert_eq!(filenames.len(), 2);
let filename_strs: Vec<String> = filenames
.iter()
.map(|p| p.to_string_lossy().to_string())
.collect();
// audi uses sr scheme by default
assert!(filename_strs
.iter()
.any(|f| f.starts_with("61756469") && f.contains("sr_public_key")));
// gran uses ed scheme by default
assert!(filename_strs
.iter()
.any(|f| f.starts_with("6772616e") && f.contains("ed_public_key")));
}
#[tokio::test]
async fn generate_produces_correct_keystore_files() {
struct TestCase {
name: &'static str,
key_types: Vec<&'static str>,
asset_hub_polkadot: bool,
expected_prefix: &'static str,
expected_public_key: &'static str,
}
let test_cases = vec![
TestCase {
name: "explicit scheme override (gran_sr)",
key_types: vec!["gran_sr"],
asset_hub_polkadot: false,
expected_prefix: "6772616e", // "gran" in hex
expected_public_key: "sr_public_key",
},
TestCase {
name: "aura with asset_hub_polkadot uses ed",
key_types: vec!["aura"],
asset_hub_polkadot: true,
expected_prefix: "61757261", // "aura" in hex
expected_public_key: "ed_public_key",
},
TestCase {
name: "aura without asset_hub_polkadot uses sr",
key_types: vec!["aura"],
asset_hub_polkadot: false,
expected_prefix: "61757261", // "aura" in hex
expected_public_key: "sr_public_key",
},
TestCase {
name: "custom key type with explicit ec scheme",
key_types: vec!["cust_ec"],
asset_hub_polkadot: false,
expected_prefix: "63757374", // "cust" in hex
expected_public_key: "ec_public_key",
},
];
for tc in test_cases {
let accounts = create_test_accounts();
let fs = create_test_fs();
let scoped_fs = ScopedFilesystem {
fs: &fs,
base_dir: "/tmp/test",
};
let key_types: Vec<&str> = tc.key_types.clone();
let res = generate(
&accounts,
"node1",
&scoped_fs,
tc.asset_hub_polkadot,
key_types,
)
.await;
assert!(
res.is_ok(),
"[{}] Expected Ok but got: {:?}",
tc.name,
res.err()
);
let filenames = res.unwrap();
assert_eq!(filenames.len(), 1, "[{}] Expected 1 file", tc.name);
let filename = filenames[0].to_string_lossy().to_string();
assert!(
filename.starts_with(tc.expected_prefix),
"[{}] Expected prefix '{}', got '{}'",
tc.name,
tc.expected_prefix,
filename
);
assert!(
filename.contains(tc.expected_public_key),
"[{}] Expected public key '{}' in '{}'",
tc.name,
tc.expected_public_key,
filename
);
}
}
#[tokio::test]
async fn generate_ignores_invalid_key_specs_and_uses_defaults() {
let accounts = create_test_accounts();
let fs = create_test_fs();
let scoped_fs = ScopedFilesystem {
fs: &fs,
base_dir: "/tmp/test",
};
let key_types = vec![
"invalid", // Too long
"xxx", // Too short
"audi_xx", // Invalid sceme
];
let res = generate(&accounts, "node1", &scoped_fs, false, key_types).await;
assert!(res.is_ok());
let filenames = res.unwrap();
// Should fall back to defaults since all specs are invalid
assert!(filenames.len() > 10);
}
}
@@ -0,0 +1,282 @@
use std::{collections::HashMap, fmt::Formatter};
use serde::{Deserialize, Serialize};
/// Supported cryptographic schemes for keystore keys.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum KeyScheme {
/// Sr25519 scheme
Sr,
/// Ed25519 scheme
Ed,
/// ECDSA scheme
Ec,
}
impl KeyScheme {
/// Returns the account key suffix used in `NodeAccounts` for this scheme.
pub fn account_key(&self) -> &'static str {
match self {
KeyScheme::Sr => "sr",
KeyScheme::Ed => "ed",
KeyScheme::Ec => "ec",
}
}
}
impl std::fmt::Display for KeyScheme {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
KeyScheme::Sr => write!(f, "sr"),
KeyScheme::Ed => write!(f, "ed"),
KeyScheme::Ec => write!(f, "ec"),
}
}
}
impl TryFrom<&str> for KeyScheme {
type Error = String;
fn try_from(value: &str) -> Result<Self, Self::Error> {
match value.to_lowercase().as_str() {
"sr" => Ok(KeyScheme::Sr),
"ed" => Ok(KeyScheme::Ed),
"ec" => Ok(KeyScheme::Ec),
_ => Err(format!("Unsupported key scheme: {}", value)),
}
}
}
/// A parsed keystore key type.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct KeystoreKeyType {
/// The 4-character key type identifier (e.g., "aura", "babe", "gran").
pub key_type: String,
/// The cryptographic scheme to use for this key type.
pub scheme: KeyScheme,
}
impl KeystoreKeyType {
pub fn new(key_type: impl Into<String>, scheme: KeyScheme) -> Self {
Self {
key_type: key_type.into(),
scheme,
}
}
}
/// Returns the default predefined key schemes for known key types.
/// Special handling for `aura` when `is_asset_hub_polkadot` is true.
fn get_predefined_schemes(is_asset_hub_polkadot: bool) -> HashMap<&'static str, KeyScheme> {
let mut schemes = HashMap::new();
// aura has special handling for asset-hub-polkadot
if is_asset_hub_polkadot {
schemes.insert("aura", KeyScheme::Ed);
} else {
schemes.insert("aura", KeyScheme::Sr);
}
schemes.insert("babe", KeyScheme::Sr);
schemes.insert("imon", KeyScheme::Sr);
schemes.insert("gran", KeyScheme::Ed);
schemes.insert("audi", KeyScheme::Sr);
schemes.insert("asgn", KeyScheme::Sr);
schemes.insert("para", KeyScheme::Sr);
schemes.insert("beef", KeyScheme::Ec);
schemes.insert("nmbs", KeyScheme::Sr); // Nimbus
schemes.insert("rand", KeyScheme::Sr); // Randomness (Moonbeam)
schemes.insert("rate", KeyScheme::Ed); // Equilibrium rate module
schemes.insert("acco", KeyScheme::Sr);
schemes.insert("bcsv", KeyScheme::Sr); // BlockchainSrvc (StorageHub)
schemes.insert("ftsv", KeyScheme::Ed); // FileTransferSrvc (StorageHub)
schemes.insert("mixn", KeyScheme::Sr); // Mixnet
schemes
}
/// Parses a single keystore key type specification string.
///
/// Supports two formats:
/// - Short: `audi` - creates key type with predefined default scheme (defaults to `sr` if not predefined)
/// - Long: `audi_sr` - creates key type with explicit scheme
///
/// Returns `None` if the spec is invalid or doesn't match the expected format.
fn parse_key_spec(spec: &str, predefined: &HashMap<&str, KeyScheme>) -> Option<KeystoreKeyType> {
let spec = spec.trim();
// Try parsing as long form first: key_type_scheme (e.g., "audi_sr")
if let Some((key_type, scheme_str)) = spec.split_once('_') {
if key_type.len() != 4 {
return None;
}
let scheme = KeyScheme::try_from(scheme_str).ok()?;
return Some(KeystoreKeyType::new(key_type, scheme));
}
// Try parsing as short form: key_type only (e.g., "audi")
if spec.len() == 4 {
// Look up predefined scheme; default to Sr if not found
let scheme = predefined.get(spec).copied().unwrap_or(KeyScheme::Sr);
return Some(KeystoreKeyType::new(spec, scheme));
}
None
}
/// Parses a list of keystore key type specifications.
///
/// Each spec can be in short form (`audi`) or long form (`audi_sr`).
/// Invalid specs are silently ignored.
///
/// If the resulting list is empty, returns the default keystore key types.
pub fn parse_keystore_key_types<T: AsRef<str>>(
specs: &[T],
is_asset_hub_polkadot: bool,
) -> Vec<KeystoreKeyType> {
let predefined_schemes = get_predefined_schemes(is_asset_hub_polkadot);
let parsed: Vec<KeystoreKeyType> = specs
.iter()
.filter_map(|spec| parse_key_spec(spec.as_ref(), &predefined_schemes))
.collect();
if parsed.is_empty() {
get_default_keystore_key_types(is_asset_hub_polkadot)
} else {
parsed
}
}
/// Returns the default keystore key types when none are specified.
pub fn get_default_keystore_key_types(is_asset_hub_polkadot: bool) -> Vec<KeystoreKeyType> {
let predefined_schemes = get_predefined_schemes(is_asset_hub_polkadot);
let default_keys = [
"aura", "babe", "imon", "gran", "audi", "asgn", "para", "beef", "nmbs", "rand", "rate",
"mixn", "bcsv", "ftsv",
];
default_keys
.iter()
.filter_map(|key_type| {
predefined_schemes
.get(*key_type)
.map(|scheme| KeystoreKeyType::new(*key_type, *scheme))
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_keystore_key_types_ignores_invalid_specs() {
let specs = vec![
"audi".to_string(),
"invalid".to_string(), // Too long - ignored
"xxx".to_string(), // Too short - ignored
"xxxx".to_string(), // Unknown key - defaults to sr
"audi_xx".to_string(), // Invalid scheme - ignored
"gran".to_string(),
];
let result = parse_keystore_key_types(&specs, false);
assert_eq!(result.len(), 3);
assert_eq!(result[1], KeystoreKeyType::new("xxxx", KeyScheme::Sr)); // Unknown defaults to sr
assert_eq!(result[2], KeystoreKeyType::new("gran", KeyScheme::Ed));
}
#[test]
fn parse_keystore_key_types_returns_specified_keys() {
let specs = vec!["audi".to_string(), "gran".to_string()];
let res = parse_keystore_key_types(&specs, false);
assert_eq!(res.len(), 2);
assert_eq!(res[0], KeystoreKeyType::new("audi", KeyScheme::Sr));
assert_eq!(res[1], KeystoreKeyType::new("gran", KeyScheme::Ed));
}
#[test]
fn parse_keystore_key_types_mixed_short_and_long_forms() {
let specs = vec![
"audi".to_string(),
"gran_sr".to_string(), // Override gran's default ed to sr
"gran".to_string(),
"beef".to_string(),
];
let res = parse_keystore_key_types(&specs, false);
assert_eq!(res.len(), 4);
assert_eq!(res[0], KeystoreKeyType::new("audi", KeyScheme::Sr));
assert_eq!(res[1], KeystoreKeyType::new("gran", KeyScheme::Sr)); // Overridden
assert_eq!(res[2], KeystoreKeyType::new("gran", KeyScheme::Ed));
assert_eq!(res[3], KeystoreKeyType::new("beef", KeyScheme::Ec));
}
#[test]
fn parse_keystore_key_types_returns_defaults_when_empty() {
let specs: Vec<String> = vec![];
let res = parse_keystore_key_types(&specs, false);
// Should return all default keys
assert!(!res.is_empty());
assert!(res.iter().any(|k| k.key_type == "aura"));
assert!(res.iter().any(|k| k.key_type == "babe"));
assert!(res.iter().any(|k| k.key_type == "gran"));
}
#[test]
fn parse_keystore_key_types_allows_custom_key_with_explicit_scheme() {
let specs = vec![
"cust_sr".to_string(), // Custom key with explicit scheme
"audi".to_string(),
];
let result = parse_keystore_key_types(&specs, false);
assert_eq!(result.len(), 2);
assert_eq!(result[0], KeystoreKeyType::new("cust", KeyScheme::Sr));
assert_eq!(result[1], KeystoreKeyType::new("audi", KeyScheme::Sr));
}
#[test]
fn full_workflow_asset_hub_polkadot() {
// For asset-hub-polkadot, aura should default to ed
let specs = vec!["aura".to_string(), "babe".to_string()];
let res = parse_keystore_key_types(&specs, true);
assert_eq!(res.len(), 2);
assert_eq!(res[0].key_type, "aura");
assert_eq!(res[0].scheme, KeyScheme::Ed); // ed for asset-hub-polkadot
assert_eq!(res[1].key_type, "babe");
assert_eq!(res[1].scheme, KeyScheme::Sr);
}
#[test]
fn full_workflow_custom_key_types() {
let specs = vec![
"aura".to_string(), // Use default scheme
"gran_sr".to_string(), // Override gran to use sr instead of ed
"cust_ec".to_string(), // Custom key type with ecdsa
];
let res = parse_keystore_key_types(&specs, false);
assert_eq!(res.len(), 3);
// aura uses default sr
assert_eq!(res[0].key_type, "aura");
assert_eq!(res[0].scheme, KeyScheme::Sr);
// gran overridden to sr
assert_eq!(res[1].key_type, "gran");
assert_eq!(res[1].scheme, KeyScheme::Sr);
// custom key with ec
assert_eq!(res[2].key_type, "cust");
assert_eq!(res[2].scheme, KeyScheme::Ec);
}
}
@@ -0,0 +1,165 @@
use std::path::{Path, PathBuf};
use configuration::types::CommandWithCustomArgs;
use provider::{
constants::NODE_CONFIG_DIR,
types::{GenerateFileCommand, GenerateFilesOptions, TransferedFile},
DynNamespace,
};
use serde::{Deserialize, Serialize};
use support::fs::FileSystem;
use uuid::Uuid;
use super::errors::GeneratorError;
use crate::ScopedFilesystem;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) enum ParaArtifactType {
Wasm,
State,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(crate) enum ParaArtifactBuildOption {
Path(String),
Command(String),
CommandWithCustomArgs(CommandWithCustomArgs),
}
/// Parachain artifact (could be either the genesis state or genesis wasm)
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ParaArtifact {
artifact_type: ParaArtifactType,
build_option: ParaArtifactBuildOption,
artifact_path: Option<PathBuf>,
// image to use for building the para artifact
image: Option<String>,
}
impl ParaArtifact {
pub(crate) fn new(
artifact_type: ParaArtifactType,
build_option: ParaArtifactBuildOption,
) -> Self {
Self {
artifact_type,
build_option,
artifact_path: None,
image: None,
}
}
pub(crate) fn image(mut self, image: Option<String>) -> Self {
self.image = image;
self
}
pub(crate) fn artifact_path(&self) -> Option<&PathBuf> {
self.artifact_path.as_ref()
}
pub(crate) async fn build<'a, T>(
&mut self,
chain_spec_path: Option<impl AsRef<Path>>,
artifact_path: impl AsRef<Path>,
ns: &DynNamespace,
scoped_fs: &ScopedFilesystem<'a, T>,
maybe_output_path: Option<PathBuf>,
) -> Result<(), GeneratorError>
where
T: FileSystem,
{
let (cmd, custom_args) = match &self.build_option {
ParaArtifactBuildOption::Path(path) => {
let t = TransferedFile::new(PathBuf::from(path), artifact_path.as_ref().into());
scoped_fs.copy_files(vec![&t]).await?;
self.artifact_path = Some(artifact_path.as_ref().into());
return Ok(()); // work done!
},
ParaArtifactBuildOption::Command(cmd) => (cmd, &vec![]),
ParaArtifactBuildOption::CommandWithCustomArgs(cmd_with_custom_args) => {
(
&cmd_with_custom_args.cmd().as_str().to_string(),
cmd_with_custom_args.args(),
)
// (cmd.cmd_as_str().to_string(), cmd.1)
},
};
let generate_subcmd = match self.artifact_type {
ParaArtifactType::Wasm => "export-genesis-wasm",
ParaArtifactType::State => "export-genesis-state",
};
// TODO: replace uuid with para_id-random
let temp_name = format!("temp-{}-{}", generate_subcmd, Uuid::new_v4());
let mut args: Vec<String> = vec![generate_subcmd.into()];
let files_to_inject = if let Some(chain_spec_path) = chain_spec_path {
// TODO: we should get the full path from the scoped filesystem
let chain_spec_path_local = format!(
"{}/{}",
ns.base_dir().to_string_lossy(),
chain_spec_path.as_ref().to_string_lossy()
);
// Remote path to be injected
let chain_spec_path_in_pod = format!(
"{}/{}",
NODE_CONFIG_DIR,
chain_spec_path.as_ref().to_string_lossy()
);
// Path in the context of the node, this can be different in the context of the providers (e.g native)
let chain_spec_path_in_args = if ns.capabilities().prefix_with_full_path {
// In native
format!(
"{}/{}{}",
ns.base_dir().to_string_lossy(),
&temp_name,
&chain_spec_path_in_pod
)
} else {
chain_spec_path_in_pod.clone()
};
args.push("--chain".into());
args.push(chain_spec_path_in_args);
for custom_arg in custom_args {
match custom_arg {
configuration::types::Arg::Flag(flag) => {
args.push(flag.into());
},
configuration::types::Arg::Option(flag, flag_value) => {
args.push(flag.into());
args.push(flag_value.into());
},
configuration::types::Arg::Array(flag, values) => {
args.push(flag.into());
values.iter().for_each(|v| args.push(v.into()));
},
}
}
vec![TransferedFile::new(
chain_spec_path_local,
chain_spec_path_in_pod,
)]
} else {
vec![]
};
let artifact_path_ref = artifact_path.as_ref();
let generate_command = GenerateFileCommand::new(cmd.as_str(), artifact_path_ref).args(args);
let options = GenerateFilesOptions::with_files(
vec![generate_command],
self.image.clone(),
&files_to_inject,
maybe_output_path,
)
.temp_name(temp_name);
ns.generate_files(options).await?;
self.artifact_path = Some(artifact_path_ref.into());
Ok(())
}
}
@@ -0,0 +1,48 @@
use std::net::TcpListener;
use configuration::shared::types::Port;
use support::constants::THIS_IS_A_BUG;
use super::errors::GeneratorError;
use crate::shared::types::ParkedPort;
// TODO: (team), we want to continue support ws_port? No
enum PortTypes {
Rpc,
P2P,
Prometheus,
}
pub fn generate(port: Option<Port>) -> Result<ParkedPort, GeneratorError> {
let port = port.unwrap_or(0);
let listener = TcpListener::bind(format!("0.0.0.0:{port}"))
.map_err(|_e| GeneratorError::PortGeneration(port, "Can't bind".into()))?;
let port = listener
.local_addr()
.expect(&format!(
"We should always get the local_addr from the listener {THIS_IS_A_BUG}"
))
.port();
Ok(ParkedPort::new(port, listener))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn generate_random() {
let port = generate(None).unwrap();
let listener = port.1.write().unwrap();
assert!(listener.is_some());
}
#[test]
fn generate_fixed_port() {
let port = generate(Some(33056)).unwrap();
let listener = port.1.write().unwrap();
assert!(listener.is_some());
assert_eq!(port.0, 33056);
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,842 @@
pub mod chain_upgrade;
pub mod node;
pub mod relaychain;
pub mod teyrchain;
use std::{cell::RefCell, collections::HashMap, path::PathBuf, rc::Rc, sync::Arc, time::Duration};
use configuration::{
para_states::{Initial, Running},
shared::{helpers::generate_unique_node_name_from_names, node::EnvVar},
types::{Arg, Command, Image, Port, ValidationContext},
ParachainConfig, ParachainConfigBuilder, RegistrationStrategy,
};
use provider::{types::TransferedFile, DynNamespace, ProviderError};
use serde::Serialize;
use support::fs::FileSystem;
use tokio::sync::RwLock;
use tracing::{error, warn};
use self::{node::NetworkNode, relaychain::Relaychain, teyrchain::Parachain};
use crate::{
generators::chain_spec::ChainSpec,
network_spec::{self, NetworkSpec},
shared::{
constants::{NODE_MONITORING_FAILURE_THRESHOLD_SECONDS, NODE_MONITORING_INTERVAL_SECONDS},
macros,
types::{ChainDefaultContext, RegisterParachainOptions},
},
spawner::{self, SpawnNodeCtx},
ScopedFilesystem, ZombieRole,
};
#[derive(Serialize)]
pub struct Network<T: FileSystem> {
#[serde(skip)]
ns: DynNamespace,
#[serde(skip)]
filesystem: T,
relay: Relaychain,
initial_spec: NetworkSpec,
parachains: HashMap<u32, Vec<Parachain>>,
#[serde(skip)]
nodes_by_name: HashMap<String, NetworkNode>,
#[serde(skip)]
nodes_to_watch: Arc<RwLock<Vec<NetworkNode>>>,
}
impl<T: FileSystem> std::fmt::Debug for Network<T> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Network")
.field("ns", &"ns_skipped")
.field("relay", &self.relay)
.field("initial_spec", &self.initial_spec)
.field("parachains", &self.parachains)
.field("nodes_by_name", &self.nodes_by_name)
.finish()
}
}
macros::create_add_options!(AddNodeOptions {
chain_spec: Option<PathBuf>,
override_eth_key: Option<String>
});
macros::create_add_options!(AddCollatorOptions {
chain_spec: Option<PathBuf>,
chain_spec_relay: Option<PathBuf>,
override_eth_key: Option<String>
});
impl<T: FileSystem> Network<T> {
pub(crate) fn new_with_relay(
relay: Relaychain,
ns: DynNamespace,
fs: T,
initial_spec: NetworkSpec,
) -> Self {
Self {
ns,
filesystem: fs,
relay,
initial_spec,
parachains: Default::default(),
nodes_by_name: Default::default(),
nodes_to_watch: Default::default(),
}
}
// Pubic API
pub fn ns_name(&self) -> String {
self.ns.name().to_string()
}
pub fn base_dir(&self) -> Option<&str> {
self.ns.base_dir().to_str()
}
pub fn relaychain(&self) -> &Relaychain {
&self.relay
}
// Teardown the network
pub async fn destroy(self) -> Result<(), ProviderError> {
self.ns.destroy().await
}
/// Add a node to the relaychain
// The new node is added to the running network instance.
/// # Example:
/// ```rust
/// # use provider::NativeProvider;
/// # use support::{fs::local::LocalFileSystem};
/// # use zombienet_orchestrator::{errors, AddNodeOptions, Orchestrator};
/// # use configuration::NetworkConfig;
/// # async fn example() -> Result<(), errors::OrchestratorError> {
/// # let provider = NativeProvider::new(LocalFileSystem {});
/// # let orchestrator = Orchestrator::new(LocalFileSystem {}, provider);
/// # let config = NetworkConfig::load_from_toml("config.toml")?;
/// let mut network = orchestrator.spawn(config).await?;
///
/// // Create the options to add the new node
/// let opts = AddNodeOptions {
/// rpc_port: Some(9444),
/// is_validator: true,
/// ..Default::default()
/// };
///
/// network.add_node("new-node", opts).await?;
/// # Ok(())
/// # }
/// ```
pub async fn add_node(
&mut self,
name: impl Into<String>,
options: AddNodeOptions,
) -> Result<(), anyhow::Error> {
let name = generate_unique_node_name_from_names(
name,
&mut self.nodes_by_name.keys().cloned().collect(),
);
let relaychain = self.relaychain();
let chain_spec_path = if let Some(chain_spec_custom_path) = &options.chain_spec {
chain_spec_custom_path.clone()
} else {
PathBuf::from(format!(
"{}/{}.json",
self.ns.base_dir().to_string_lossy(),
relaychain.chain
))
};
let chain_context = ChainDefaultContext {
default_command: self.initial_spec.relaychain.default_command.as_ref(),
default_image: self.initial_spec.relaychain.default_image.as_ref(),
default_resources: self.initial_spec.relaychain.default_resources.as_ref(),
default_db_snapshot: self.initial_spec.relaychain.default_db_snapshot.as_ref(),
default_args: self.initial_spec.relaychain.default_args.iter().collect(),
};
let mut node_spec = network_spec::node::NodeSpec::from_ad_hoc(
&name,
options.into(),
&chain_context,
false,
false,
)?;
node_spec.available_args_output = Some(
self.initial_spec
.node_available_args_output(&node_spec, self.ns.clone())
.await?,
);
let base_dir = self.ns.base_dir().to_string_lossy();
let scoped_fs = ScopedFilesystem::new(&self.filesystem, &base_dir);
let ctx = SpawnNodeCtx {
chain_id: &relaychain.chain_id,
parachain_id: None,
chain: &relaychain.chain,
role: ZombieRole::Node,
ns: &self.ns,
scoped_fs: &scoped_fs,
parachain: None,
bootnodes_addr: &vec![],
wait_ready: true,
nodes_by_name: serde_json::to_value(&self.nodes_by_name)?,
global_settings: &self.initial_spec.global_settings,
};
let global_files_to_inject = vec![TransferedFile::new(
chain_spec_path,
PathBuf::from(format!("/cfg/{}.json", relaychain.chain)),
)];
let node = spawner::spawn_node(&node_spec, global_files_to_inject, &ctx).await?;
// TODO: register the new node as validator in the relaychain
// STEPS:
// - check balance of `stash` derivation for validator account
// - call rotate_keys on the new validator
// - call setKeys on the new validator
// if node_spec.is_validator {
// let running_node = self.relay.nodes.first().unwrap();
// // tx_helper::validator_actions::register(vec![&node], &running_node.ws_uri, None).await?;
// }
// Let's make sure node is up before adding
node.wait_until_is_up(self.initial_spec.global_settings.network_spawn_timeout())
.await?;
// Add node to relaychain data
self.add_running_node(node.clone(), None).await;
Ok(())
}
/// Add a new collator to a parachain
///
/// NOTE: if more parachains with given id available (rare corner case)
/// then it adds collator to the first parachain
///
/// # Example:
/// ```rust
/// # use provider::NativeProvider;
/// # use support::{fs::local::LocalFileSystem};
/// # use zombienet_orchestrator::{errors, AddCollatorOptions, Orchestrator};
/// # use configuration::NetworkConfig;
/// # async fn example() -> Result<(), anyhow::Error> {
/// # let provider = NativeProvider::new(LocalFileSystem {});
/// # let orchestrator = Orchestrator::new(LocalFileSystem {}, provider);
/// # let config = NetworkConfig::load_from_toml("config.toml")?;
/// let mut network = orchestrator.spawn(config).await?;
///
/// let col_opts = AddCollatorOptions {
/// command: Some("polkadot-parachain".try_into()?),
/// ..Default::default()
/// };
///
/// network.add_collator("new-col-1", col_opts, 100).await?;
/// # Ok(())
/// # }
/// ```
pub async fn add_collator(
&mut self,
name: impl Into<String>,
options: AddCollatorOptions,
para_id: u32,
) -> Result<(), anyhow::Error> {
let name = generate_unique_node_name_from_names(
name,
&mut self.nodes_by_name.keys().cloned().collect(),
);
let spec = self
.initial_spec
.parachains
.iter()
.find(|para| para.id == para_id)
.ok_or(anyhow::anyhow!(format!("parachain: {para_id} not found!")))?;
let role = if spec.is_cumulus_based {
ZombieRole::CumulusCollator
} else {
ZombieRole::Collator
};
let chain_context = ChainDefaultContext {
default_command: spec.default_command.as_ref(),
default_image: spec.default_image.as_ref(),
default_resources: spec.default_resources.as_ref(),
default_db_snapshot: spec.default_db_snapshot.as_ref(),
default_args: spec.default_args.iter().collect(),
};
let parachain = self
.parachains
.get_mut(&para_id)
.ok_or(anyhow::anyhow!(format!("parachain: {para_id} not found!")))?
.get_mut(0)
.ok_or(anyhow::anyhow!(format!("parachain: {para_id} not found!")))?;
let base_dir = self.ns.base_dir().to_string_lossy();
let scoped_fs = ScopedFilesystem::new(&self.filesystem, &base_dir);
// TODO: we want to still supporting spawn a dedicated bootnode??
let ctx = SpawnNodeCtx {
chain_id: &self.relay.chain_id,
parachain_id: parachain.chain_id.as_deref(),
chain: &self.relay.chain,
role,
ns: &self.ns,
scoped_fs: &scoped_fs,
parachain: Some(spec),
bootnodes_addr: &vec![],
wait_ready: true,
nodes_by_name: serde_json::to_value(&self.nodes_by_name)?,
global_settings: &self.initial_spec.global_settings,
};
let relaychain_spec_path = if let Some(chain_spec_custom_path) = &options.chain_spec_relay {
chain_spec_custom_path.clone()
} else {
PathBuf::from(format!(
"{}/{}.json",
self.ns.base_dir().to_string_lossy(),
self.relay.chain
))
};
let mut global_files_to_inject = vec![TransferedFile::new(
relaychain_spec_path,
PathBuf::from(format!("/cfg/{}.json", self.relay.chain)),
)];
let para_chain_spec_local_path = if let Some(para_chain_spec_custom) = &options.chain_spec {
Some(para_chain_spec_custom.clone())
} else if let Some(para_spec_path) = &parachain.chain_spec_path {
Some(PathBuf::from(format!(
"{}/{}",
self.ns.base_dir().to_string_lossy(),
para_spec_path.to_string_lossy()
)))
} else {
None
};
if let Some(para_spec_path) = para_chain_spec_local_path {
global_files_to_inject.push(TransferedFile::new(
para_spec_path,
PathBuf::from(format!("/cfg/{para_id}.json")),
));
}
let mut node_spec = network_spec::node::NodeSpec::from_ad_hoc(
name,
options.into(),
&chain_context,
true,
spec.is_evm_based,
)?;
node_spec.available_args_output = Some(
self.initial_spec
.node_available_args_output(&node_spec, self.ns.clone())
.await?,
);
let node = spawner::spawn_node(&node_spec, global_files_to_inject, &ctx).await?;
// Let's make sure node is up before adding
node.wait_until_is_up(self.initial_spec.global_settings.network_spawn_timeout())
.await?;
parachain.collators.push(node.clone());
self.add_running_node(node, None).await;
Ok(())
}
/// Get a parachain config builder from a running network
///
/// This allow you to build a new parachain config to be deployed into
/// the running network.
pub fn para_config_builder(&self) -> ParachainConfigBuilder<Initial, Running> {
let used_ports = self
.nodes_iter()
.map(|node| node.spec())
.flat_map(|spec| {
[
spec.ws_port.0,
spec.rpc_port.0,
spec.prometheus_port.0,
spec.p2p_port.0,
]
})
.collect();
let used_nodes_names = self.nodes_by_name.keys().cloned().collect();
// need to inverse logic of generate_unique_para_id
let used_para_ids = self
.parachains
.iter()
.map(|(id, paras)| (*id, paras.len().saturating_sub(1) as u8))
.collect();
let context = ValidationContext {
used_ports,
used_nodes_names,
used_para_ids,
};
let context = Rc::new(RefCell::new(context));
ParachainConfigBuilder::new_with_running(context)
}
/// Add a new parachain to the running network
///
/// # Arguments
/// * `para_config` - Parachain configuration to deploy
/// * `custom_relaychain_spec` - Optional path to a custom relaychain spec to use
/// * `custom_parchain_fs_prefix` - Optional prefix to use when artifacts are created
///
///
/// # Example:
/// ```rust
/// # use anyhow::anyhow;
/// # use provider::NativeProvider;
/// # use support::{fs::local::LocalFileSystem};
/// # use zombienet_orchestrator::{errors, AddCollatorOptions, Orchestrator};
/// # use configuration::NetworkConfig;
/// # async fn example() -> Result<(), anyhow::Error> {
/// # let provider = NativeProvider::new(LocalFileSystem {});
/// # let orchestrator = Orchestrator::new(LocalFileSystem {}, provider);
/// # let config = NetworkConfig::load_from_toml("config.toml")?;
/// let mut network = orchestrator.spawn(config).await?;
/// let para_config = network
/// .para_config_builder()
/// .with_id(100)
/// .with_default_command("polkadot-parachain")
/// .with_collator(|c| c.with_name("col-100-1"))
/// .build()
/// .map_err(|_e| anyhow!("Building config"))?;
///
/// network.add_parachain(&para_config, None, None).await?;
///
/// # Ok(())
/// # }
/// ```
pub async fn add_parachain(
&mut self,
para_config: &ParachainConfig,
custom_relaychain_spec: Option<PathBuf>,
custom_parchain_fs_prefix: Option<String>,
) -> Result<(), anyhow::Error> {
let base_dir = self.ns.base_dir().to_string_lossy().to_string();
let scoped_fs = ScopedFilesystem::new(&self.filesystem, &base_dir);
let mut global_files_to_inject = vec![];
// get relaychain id
let relay_chain_id = if let Some(custom_path) = custom_relaychain_spec {
// use this file as relaychain spec
global_files_to_inject.push(TransferedFile::new(
custom_path.clone(),
PathBuf::from(format!("/cfg/{}.json", self.relaychain().chain)),
));
let content = std::fs::read_to_string(custom_path)?;
ChainSpec::chain_id_from_spec(&content)?
} else {
global_files_to_inject.push(TransferedFile::new(
PathBuf::from(format!(
"{}/{}",
scoped_fs.base_dir,
self.relaychain().chain_spec_path.to_string_lossy()
)),
PathBuf::from(format!("/cfg/{}.json", self.relaychain().chain)),
));
self.relay.chain_id.clone()
};
let mut para_spec = network_spec::teyrchain::TeyrchainSpec::from_config(
para_config,
relay_chain_id.as_str().try_into()?,
)?;
let chain_spec_raw_path = para_spec
.build_chain_spec(&relay_chain_id, &self.ns, &scoped_fs)
.await?;
// Para artifacts
let para_path_prefix = if let Some(custom_prefix) = custom_parchain_fs_prefix {
custom_prefix
} else {
para_spec.id.to_string()
};
scoped_fs.create_dir(&para_path_prefix).await?;
// create wasm/state
para_spec
.genesis_state
.build(
chain_spec_raw_path.as_ref(),
format!("{}/genesis-state", &para_path_prefix),
&self.ns,
&scoped_fs,
None,
)
.await?;
para_spec
.genesis_wasm
.build(
chain_spec_raw_path.as_ref(),
format!("{}/para_spec-wasm", &para_path_prefix),
&self.ns,
&scoped_fs,
None,
)
.await?;
let parachain =
Parachain::from_spec(&para_spec, &global_files_to_inject, &scoped_fs).await?;
let parachain_id = parachain.chain_id.clone();
// Create `ctx` for spawn the nodes
let ctx_para = SpawnNodeCtx {
parachain: Some(&para_spec),
parachain_id: parachain_id.as_deref(),
role: if para_spec.is_cumulus_based {
ZombieRole::CumulusCollator
} else {
ZombieRole::Collator
},
bootnodes_addr: &para_config
.bootnodes_addresses()
.iter()
.map(|&a| a.to_string())
.collect(),
chain_id: &self.relaychain().chain_id,
chain: &self.relaychain().chain,
ns: &self.ns,
scoped_fs: &scoped_fs,
wait_ready: false,
nodes_by_name: serde_json::to_value(&self.nodes_by_name)?,
global_settings: &self.initial_spec.global_settings,
};
// Register the parachain to the running network
let first_node_url = self
.relaychain()
.nodes
.first()
.ok_or(anyhow::anyhow!(
"At least one node of the relaychain should be running"
))?
.ws_uri();
if para_config.registration_strategy() == Some(&RegistrationStrategy::UsingExtrinsic) {
let register_para_options = RegisterParachainOptions {
id: parachain.para_id,
// This needs to resolve correctly
wasm_path: para_spec
.genesis_wasm
.artifact_path()
.ok_or(anyhow::anyhow!(
"artifact path for wasm must be set at this point",
))?
.to_path_buf(),
state_path: para_spec
.genesis_state
.artifact_path()
.ok_or(anyhow::anyhow!(
"artifact path for state must be set at this point",
))?
.to_path_buf(),
node_ws_url: first_node_url.to_string(),
onboard_as_para: para_spec.onboard_as_parachain,
seed: None, // TODO: Seed is passed by?
finalization: false,
};
Parachain::register(register_para_options, &scoped_fs).await?;
}
// Spawn the nodes
let spawning_tasks = para_spec
.collators
.iter()
.map(|node| spawner::spawn_node(node, parachain.files_to_inject.clone(), &ctx_para));
let running_nodes = futures::future::try_join_all(spawning_tasks).await?;
// Let's make sure nodes are up before adding them
let waiting_tasks = running_nodes.iter().map(|node| {
node.wait_until_is_up(self.initial_spec.global_settings.network_spawn_timeout())
});
let _ = futures::future::try_join_all(waiting_tasks).await?;
let running_para_id = parachain.para_id;
self.add_para(parachain);
for node in running_nodes {
self.add_running_node(node, Some(running_para_id)).await;
}
Ok(())
}
/// Register a parachain, which has already been added to the network (with manual registration
/// strategy)
///
/// # Arguments
/// * `para_id` - Parachain Id
///
///
/// # Example:
/// ```rust
/// # use anyhow::anyhow;
/// # use provider::NativeProvider;
/// # use support::{fs::local::LocalFileSystem};
/// # use zombienet_orchestrator::Orchestrator;
/// # use configuration::{NetworkConfig, NetworkConfigBuilder, RegistrationStrategy};
/// # async fn example() -> Result<(), anyhow::Error> {
/// # let provider = NativeProvider::new(LocalFileSystem {});
/// # let orchestrator = Orchestrator::new(LocalFileSystem {}, provider);
/// # let config = NetworkConfigBuilder::new()
/// # .with_relaychain(|r| {
/// # r.with_chain("rococo-local")
/// # .with_default_command("polkadot")
/// # .with_node(|node| node.with_name("alice"))
/// # })
/// # .with_parachain(|p| {
/// # p.with_id(100)
/// # .with_registration_strategy(RegistrationStrategy::Manual)
/// # .with_default_command("test-parachain")
/// # .with_collator(|n| n.with_name("dave").validator(false))
/// # })
/// # .build()
/// # .map_err(|_e| anyhow!("Building config"))?;
/// let mut network = orchestrator.spawn(config).await?;
///
/// network.register_parachain(100).await?;
///
/// # Ok(())
/// # }
/// ```
pub async fn register_parachain(&mut self, para_id: u32) -> Result<(), anyhow::Error> {
let para = self
.initial_spec
.parachains
.iter()
.find(|p| p.id == para_id)
.ok_or(anyhow::anyhow!(
"no parachain with id = {para_id} available",
))?;
let para_genesis_config = para.get_genesis_config()?;
let first_node_url = self
.relaychain()
.nodes
.first()
.ok_or(anyhow::anyhow!(
"At least one node of the relaychain should be running"
))?
.ws_uri();
let register_para_options: RegisterParachainOptions = RegisterParachainOptions {
id: para_id,
// This needs to resolve correctly
wasm_path: para_genesis_config.wasm_path.clone(),
state_path: para_genesis_config.state_path.clone(),
node_ws_url: first_node_url.to_string(),
onboard_as_para: para_genesis_config.as_parachain,
seed: None, // TODO: Seed is passed by?
finalization: false,
};
let base_dir = self.ns.base_dir().to_string_lossy().to_string();
let scoped_fs = ScopedFilesystem::new(&self.filesystem, &base_dir);
Parachain::register(register_para_options, &scoped_fs).await?;
Ok(())
}
// deregister and stop the collator?
// remove_parachain()
pub fn get_node(&self, name: impl Into<String>) -> Result<&NetworkNode, anyhow::Error> {
let name = name.into();
if let Some(node) = self.nodes_iter().find(|&n| n.name == name) {
return Ok(node);
}
let list = self
.nodes_iter()
.map(|n| &n.name)
.cloned()
.collect::<Vec<_>>()
.join(", ");
Err(anyhow::anyhow!(
"can't find node with name: {name:?}, should be one of {list}"
))
}
pub fn get_node_mut(
&mut self,
name: impl Into<String>,
) -> Result<&mut NetworkNode, anyhow::Error> {
let name = name.into();
self.nodes_iter_mut()
.find(|n| n.name == name)
.ok_or(anyhow::anyhow!("can't find node with name: {name:?}"))
}
pub fn nodes(&self) -> Vec<&NetworkNode> {
self.nodes_by_name.values().collect::<Vec<&NetworkNode>>()
}
pub async fn detach(&self) {
self.ns.detach().await
}
// Internal API
pub(crate) async fn add_running_node(&mut self, node: NetworkNode, para_id: Option<u32>) {
if let Some(para_id) = para_id {
if let Some(para) = self.parachains.get_mut(&para_id).and_then(|p| p.get_mut(0)) {
para.collators.push(node.clone());
} else {
// is the first node of the para, let create the entry
unreachable!()
}
} else {
self.relay.nodes.push(node.clone());
}
// TODO: we should hold a ref to the node in the vec in the future.
node.set_is_running(true);
let node_name = node.name.clone();
self.nodes_by_name.insert(node_name, node.clone());
self.nodes_to_watch.write().await.push(node);
}
pub(crate) fn add_para(&mut self, para: Parachain) {
self.parachains.entry(para.para_id).or_default().push(para);
}
pub fn name(&self) -> &str {
self.ns.name()
}
/// Get a first parachain from the list of the parachains with specified id.
/// NOTE!
/// Usually the list will contain only one parachain.
/// Multiple parachains with the same id is a corner case.
/// If this is the case then one can get such parachain with
/// `parachain_by_unique_id()` method
///
/// # Arguments
/// * `para_id` - Parachain Id
pub fn parachain(&self, para_id: u32) -> Option<&Parachain> {
self.parachains.get(&para_id)?.first()
}
/// Get a parachain by its unique id.
///
/// This is particularly useful if there are multiple parachains
/// with the same id (this is a rare corner case).
///
/// # Arguments
/// * `unique_id` - unique id of the parachain
pub fn parachain_by_unique_id(&self, unique_id: impl AsRef<str>) -> Option<&Parachain> {
self.parachains
.values()
.flat_map(|p| p.iter())
.find(|p| p.unique_id == unique_id.as_ref())
}
pub fn parachains(&self) -> Vec<&Parachain> {
self.parachains.values().flatten().collect()
}
pub(crate) fn nodes_iter(&self) -> impl Iterator<Item = &NetworkNode> {
self.relay.nodes.iter().chain(
self.parachains
.values()
.flat_map(|p| p.iter())
.flat_map(|p| &p.collators),
)
}
pub(crate) fn nodes_iter_mut(&mut self) -> impl Iterator<Item = &mut NetworkNode> {
self.relay.nodes.iter_mut().chain(
self.parachains
.values_mut()
.flat_map(|p| p.iter_mut())
.flat_map(|p| &mut p.collators),
)
}
/// Waits given number of seconds until all nodes in the network report that they are
/// up and running.
///
/// # Arguments
/// * `timeout_secs` - The number of seconds to wait.
///
/// # Returns
/// * `Ok()` if the node is up before timeout occured.
/// * `Err(e)` if timeout or other error occurred while waiting.
pub async fn wait_until_is_up(&self, timeout_secs: u64) -> Result<(), anyhow::Error> {
let handles = self
.nodes_iter()
.map(|node| node.wait_until_is_up(timeout_secs));
futures::future::try_join_all(handles).await?;
Ok(())
}
pub(crate) fn spawn_watching_task(&self) {
let nodes_to_watch = Arc::clone(&self.nodes_to_watch);
let ns = Arc::clone(&self.ns);
tokio::spawn(async move {
loop {
tokio::time::sleep(Duration::from_secs(NODE_MONITORING_INTERVAL_SECONDS)).await;
let all_running = {
let guard = nodes_to_watch.read().await;
let nodes = guard.iter().filter(|n| n.is_running()).collect::<Vec<_>>();
let all_running =
futures::future::try_join_all(nodes.iter().map(|n| {
n.wait_until_is_up(NODE_MONITORING_FAILURE_THRESHOLD_SECONDS)
}))
.await;
// Re-check `is_running` to make sure we don't kill the network unnecessarily
if nodes.iter().any(|n| !n.is_running()) {
continue;
} else {
all_running
}
};
if let Err(e) = all_running {
warn!("\n\t🧟 One of the nodes crashed: {e}. tearing the network down...");
if let Err(e) = ns.destroy().await {
error!("an error occurred during network teardown: {}", e);
}
std::process::exit(1);
}
}
});
}
pub(crate) fn set_parachains(&mut self, parachains: HashMap<u32, Vec<Parachain>>) {
self.parachains = parachains;
}
pub(crate) fn insert_node(&mut self, node: NetworkNode) {
self.nodes_by_name.insert(node.name.clone(), node);
}
}
@@ -0,0 +1,41 @@
use std::str::FromStr;
use anyhow::anyhow;
use async_trait::async_trait;
use pezkuwi_subxt_signer::{sr25519::Keypair, SecretUri};
use super::node::NetworkNode;
use crate::{shared::types::RuntimeUpgradeOptions, tx_helper};
#[async_trait]
pub trait ChainUpgrade {
/// Perform a runtime upgrade (with sudo)
///
/// This call 'System.set_code_without_checks' wrapped in
/// 'Sudo.sudo_unchecked_weight'
async fn runtime_upgrade(&self, options: RuntimeUpgradeOptions) -> Result<(), anyhow::Error>;
/// Perform a runtime upgrade (with sudo), inner call with the node pass as arg.
///
/// This call 'System.set_code_without_checks' wrapped in
/// 'Sudo.sudo_unchecked_weight'
async fn perform_runtime_upgrade(
&self,
node: &NetworkNode,
options: RuntimeUpgradeOptions,
) -> Result<(), anyhow::Error> {
let sudo = if let Some(possible_seed) = options.seed {
Keypair::from_secret_key(possible_seed)
.map_err(|_| anyhow!("seed should return a Keypair"))?
} else {
let uri = SecretUri::from_str("//Alice")?;
Keypair::from_uri(&uri).map_err(|_| anyhow!("'//Alice' should return a Keypair"))?
};
let wasm_data = options.wasm.get_asset().await?;
tx_helper::runtime_upgrade::upgrade(node, &wasm_data, &sudo).await?;
Ok(())
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,75 @@
use std::path::PathBuf;
use anyhow::anyhow;
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use super::node::NetworkNode;
use crate::{
network::chain_upgrade::ChainUpgrade, shared::types::RuntimeUpgradeOptions,
utils::default_as_empty_vec,
};
#[derive(Debug, Serialize, Deserialize)]
pub struct Relaychain {
pub(crate) chain: String,
pub(crate) chain_id: String,
pub(crate) chain_spec_path: PathBuf,
#[serde(default, deserialize_with = "default_as_empty_vec")]
pub(crate) nodes: Vec<NetworkNode>,
}
#[derive(Debug, Deserialize)]
pub(crate) struct RawRelaychain {
#[serde(flatten)]
pub(crate) inner: Relaychain,
pub(crate) nodes: serde_json::Value,
}
#[async_trait]
impl ChainUpgrade for Relaychain {
async fn runtime_upgrade(&self, options: RuntimeUpgradeOptions) -> Result<(), anyhow::Error> {
// check if the node is valid first
let node = if let Some(node_name) = &options.node_name {
if let Some(node) = self
.nodes()
.into_iter()
.find(|node| node.name() == node_name)
{
node
} else {
return Err(anyhow!("Node: {node_name} is not part of the set of nodes"));
}
} else {
// take the first node
if let Some(node) = self.nodes().first() {
node
} else {
return Err(anyhow!("chain doesn't have any node!"));
}
};
self.perform_runtime_upgrade(node, options).await
}
}
impl Relaychain {
pub(crate) fn new(chain: String, chain_id: String, chain_spec_path: PathBuf) -> Self {
Self {
chain,
chain_id,
chain_spec_path,
nodes: Default::default(),
}
}
// Public API
pub fn nodes(&self) -> Vec<&NetworkNode> {
self.nodes.iter().collect()
}
/// Get chain name
pub fn chain(&self) -> &str {
&self.chain
}
}
@@ -0,0 +1,330 @@
use std::{
path::{Path, PathBuf},
str::FromStr,
};
use anyhow::anyhow;
use async_trait::async_trait;
use pezkuwi_subxt::{dynamic::Value, tx::TxStatus, BizinikiwConfig, OnlineClient};
use pezkuwi_subxt_signer::{sr25519::Keypair, SecretUri};
use provider::types::TransferedFile;
use serde::{Deserialize, Serialize};
use support::{constants::THIS_IS_A_BUG, fs::FileSystem, net::wait_ws_ready};
use tracing::info;
use super::{chain_upgrade::ChainUpgrade, node::NetworkNode};
use crate::{
network_spec::teyrchain::TeyrchainSpec,
shared::types::{RegisterParachainOptions, RuntimeUpgradeOptions},
tx_helper::client::get_client_from_url,
utils::default_as_empty_vec,
ScopedFilesystem,
};
#[derive(Debug, Serialize, Deserialize)]
pub struct Parachain {
pub(crate) chain: Option<String>,
pub(crate) para_id: u32,
// unique_id is internally used to allow multiple parachains with the same id
// See `ParachainConfig` for more details
pub(crate) unique_id: String,
pub(crate) chain_id: Option<String>,
pub(crate) chain_spec_path: Option<PathBuf>,
#[serde(default, deserialize_with = "default_as_empty_vec")]
pub(crate) collators: Vec<NetworkNode>,
pub(crate) files_to_inject: Vec<TransferedFile>,
pub(crate) bootnodes_addresses: Vec<multiaddr::Multiaddr>,
}
#[derive(Debug, Deserialize)]
pub(crate) struct RawParachain {
#[serde(flatten)]
pub(crate) inner: Parachain,
pub(crate) collators: serde_json::Value,
}
#[async_trait]
impl ChainUpgrade for Parachain {
async fn runtime_upgrade(&self, options: RuntimeUpgradeOptions) -> Result<(), anyhow::Error> {
// check if the node is valid first
let node = if let Some(node_name) = &options.node_name {
if let Some(node) = self
.collators()
.into_iter()
.find(|node| node.name() == node_name)
{
node
} else {
return Err(anyhow!("Node: {node_name} is not part of the set of nodes"));
}
} else {
// take the first node
if let Some(node) = self.collators().first() {
node
} else {
return Err(anyhow!("chain doesn't have any node!"));
}
};
self.perform_runtime_upgrade(node, options).await
}
}
impl Parachain {
pub(crate) fn new(para_id: u32, unique_id: impl Into<String>) -> Self {
Self {
chain: None,
para_id,
unique_id: unique_id.into(),
chain_id: None,
chain_spec_path: None,
collators: Default::default(),
files_to_inject: Default::default(),
bootnodes_addresses: vec![],
}
}
pub(crate) fn with_chain_spec(
para_id: u32,
unique_id: impl Into<String>,
chain_id: impl Into<String>,
chain_spec_path: impl AsRef<Path>,
) -> Self {
Self {
para_id,
unique_id: unique_id.into(),
chain: None,
chain_id: Some(chain_id.into()),
chain_spec_path: Some(chain_spec_path.as_ref().into()),
collators: Default::default(),
files_to_inject: Default::default(),
bootnodes_addresses: vec![],
}
}
pub(crate) async fn from_spec(
para: &TeyrchainSpec,
files_to_inject: &[TransferedFile],
scoped_fs: &ScopedFilesystem<'_, impl FileSystem>,
) -> Result<Self, anyhow::Error> {
let mut para_files_to_inject = files_to_inject.to_owned();
// parachain id is used for the keystore
let mut parachain = if let Some(chain_spec) = para.chain_spec.as_ref() {
let id = chain_spec.read_chain_id(scoped_fs).await?;
// add the spec to global files to inject
let spec_name = chain_spec.chain_spec_name();
let base = PathBuf::from_str(scoped_fs.base_dir)?;
para_files_to_inject.push(TransferedFile::new(
base.join(format!("{spec_name}.json")),
PathBuf::from(format!("/cfg/{}.json", para.id)),
));
let raw_path = chain_spec
.raw_path()
.ok_or(anyhow::anyhow!("chain-spec path should be set by now.",))?;
let mut running_para =
Parachain::with_chain_spec(para.id, &para.unique_id, id, raw_path);
if let Some(chain_name) = chain_spec.chain_name() {
running_para.chain = Some(chain_name.to_string());
}
running_para
} else {
Parachain::new(para.id, &para.unique_id)
};
parachain.bootnodes_addresses = para.bootnodes_addresses().into_iter().cloned().collect();
parachain.files_to_inject = para_files_to_inject;
Ok(parachain)
}
pub async fn register(
options: RegisterParachainOptions,
scoped_fs: &ScopedFilesystem<'_, impl FileSystem>,
) -> Result<(), anyhow::Error> {
info!("Registering parachain: {:?}", options);
// get the seed
let sudo: Keypair;
if let Some(possible_seed) = options.seed {
sudo = Keypair::from_secret_key(possible_seed)
.expect(&format!("seed should return a Keypair {THIS_IS_A_BUG}"));
} else {
let uri = SecretUri::from_str("//Alice")?;
sudo = Keypair::from_uri(&uri)?;
}
let genesis_state = scoped_fs
.read_to_string(options.state_path)
.await
.expect(&format!(
"State Path should be ok by this point {THIS_IS_A_BUG}"
));
let wasm_data = scoped_fs
.read_to_string(options.wasm_path)
.await
.expect(&format!(
"Wasm Path should be ok by this point {THIS_IS_A_BUG}"
));
wait_ws_ready(options.node_ws_url.as_str())
.await
.map_err(|_| {
anyhow::anyhow!(
"Error waiting for ws to be ready, at {}",
options.node_ws_url.as_str()
)
})?;
let api: OnlineClient<BizinikiwConfig> = get_client_from_url(&options.node_ws_url).await?;
let schedule_para = pezkuwi_subxt::dynamic::tx(
"ParasSudoWrapper",
"sudo_schedule_para_initialize",
vec![
Value::primitive(options.id.into()),
Value::named_composite([
(
"genesis_head",
Value::from_bytes(hex::decode(&genesis_state[2..])?),
),
(
"validation_code",
Value::from_bytes(hex::decode(&wasm_data[2..])?),
),
("para_kind", Value::bool(options.onboard_as_para)),
]),
],
);
let sudo_call =
pezkuwi_subxt::dynamic::tx("Sudo", "sudo", vec![schedule_para.into_value()]);
// TODO: uncomment below and fix the sign and submit (and follow afterwards until
// finalized block) to register the parachain
let mut tx = api
.tx()
.sign_and_submit_then_watch_default(&sudo_call, &sudo)
.await?;
// Below we use the low level API to replicate the `wait_for_in_block` behaviour
// which was removed in subxt 0.33.0. See https://github.com/paritytech/subxt/pull/1237.
while let Some(status) = tx.next().await {
match status? {
TxStatus::InBestBlock(tx_in_block) | TxStatus::InFinalizedBlock(tx_in_block) => {
let _result = tx_in_block.wait_for_success().await?;
info!("In block: {:#?}", tx_in_block.block_hash());
},
TxStatus::Error { message }
| TxStatus::Invalid { message }
| TxStatus::Dropped { message } => {
return Err(anyhow::format_err!("Error submitting tx: {message}"));
},
_ => continue,
}
}
Ok(())
}
pub fn para_id(&self) -> u32 {
self.para_id
}
pub fn unique_id(&self) -> &str {
self.unique_id.as_str()
}
pub fn chain_id(&self) -> Option<&str> {
self.chain_id.as_deref()
}
pub fn collators(&self) -> Vec<&NetworkNode> {
self.collators.iter().collect()
}
pub fn bootnodes_addresses(&self) -> Vec<&multiaddr::Multiaddr> {
self.bootnodes_addresses.iter().collect()
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use super::*;
#[test]
fn create_with_is_works() {
let para = Parachain::new(100, "100");
// only para_id and unique_id should be set
assert_eq!(para.para_id, 100);
assert_eq!(para.unique_id, "100");
assert_eq!(para.chain_id, None);
assert_eq!(para.chain, None);
assert_eq!(para.chain_spec_path, None);
}
#[test]
fn create_with_chain_spec_works() {
let para = Parachain::with_chain_spec(100, "100", "rococo-local", "/tmp/rococo-local.json");
assert_eq!(para.para_id, 100);
assert_eq!(para.unique_id, "100");
assert_eq!(para.chain_id, Some("rococo-local".to_string()));
assert_eq!(para.chain, None);
assert_eq!(
para.chain_spec_path,
Some(PathBuf::from("/tmp/rococo-local.json"))
);
}
#[tokio::test]
async fn create_with_para_spec_works() {
use configuration::ParachainConfigBuilder;
use crate::network_spec::teyrchain::TeyrchainSpec;
let bootnode_addresses = vec!["/ip4/10.41.122.55/tcp/45421"];
let para_config = ParachainConfigBuilder::new(Default::default())
.with_id(100)
.cumulus_based(false)
.with_default_command("adder-collator")
.with_raw_bootnodes_addresses(bootnode_addresses.clone())
.with_collator(|c| c.with_name("col"))
.build()
.unwrap();
let para_spec =
TeyrchainSpec::from_config(&para_config, "rococo-local".try_into().unwrap()).unwrap();
let fs = support::fs::in_memory::InMemoryFileSystem::new(HashMap::default());
let scoped_fs = ScopedFilesystem {
fs: &fs,
base_dir: "/tmp/some",
};
let files = vec![TransferedFile::new(
PathBuf::from("/tmp/some"),
PathBuf::from("/tmp/some"),
)];
let para = Parachain::from_spec(&para_spec, &files, &scoped_fs)
.await
.unwrap();
println!("{para:#?}");
assert_eq!(para.para_id, 100);
assert_eq!(para.unique_id, "100");
assert_eq!(para.chain_id, None);
assert_eq!(para.chain, None);
// one file should be added.
assert_eq!(para.files_to_inject.len(), 1);
assert_eq!(
para.bootnodes_addresses()
.iter()
.map(|addr| addr.to_string())
.collect::<Vec<_>>(),
bootnode_addresses
);
}
}
@@ -0,0 +1,2 @@
pub mod metrics;
pub mod verifier;
@@ -0,0 +1,62 @@
use std::collections::HashMap;
use async_trait::async_trait;
use reqwest::Url;
#[async_trait]
pub trait MetricsHelper {
async fn metric(&self, metric_name: &str) -> Result<f64, anyhow::Error>;
async fn metric_with_url(
metric: impl AsRef<str> + Send,
endpoint: impl Into<Url> + Send,
) -> Result<f64, anyhow::Error>;
}
pub struct Metrics {
endpoint: Url,
}
impl Metrics {
fn new(endpoint: impl Into<Url>) -> Self {
Self {
endpoint: endpoint.into(),
}
}
async fn fetch_metrics(
endpoint: impl AsRef<str>,
) -> Result<HashMap<String, f64>, anyhow::Error> {
let response = reqwest::get(endpoint.as_ref()).await?;
Ok(prom_metrics_parser::parse(&response.text().await?)?)
}
fn get_metric(
metrics_map: HashMap<String, f64>,
metric_name: &str,
) -> Result<f64, anyhow::Error> {
let treat_not_found_as_zero = true;
if let Some(val) = metrics_map.get(metric_name) {
Ok(*val)
} else if treat_not_found_as_zero {
Ok(0_f64)
} else {
Err(anyhow::anyhow!("MetricNotFound: {metric_name}"))
}
}
}
#[async_trait]
impl MetricsHelper for Metrics {
async fn metric(&self, metric_name: &str) -> Result<f64, anyhow::Error> {
let metrics_map = Metrics::fetch_metrics(self.endpoint.as_str()).await?;
Metrics::get_metric(metrics_map, metric_name)
}
async fn metric_with_url(
metric_name: impl AsRef<str> + Send,
endpoint: impl Into<Url> + Send,
) -> Result<f64, anyhow::Error> {
let metrics_map = Metrics::fetch_metrics(endpoint.into()).await?;
Metrics::get_metric(metrics_map, metric_name.as_ref())
}
}
@@ -0,0 +1,34 @@
use std::time::Duration;
use tokio::time::timeout;
use tracing::trace;
use crate::network::node::NetworkNode;
pub(crate) async fn verify_nodes(nodes: &[&NetworkNode]) -> Result<(), anyhow::Error> {
timeout(Duration::from_secs(90), check_nodes(nodes))
.await
.map_err(|_| anyhow::anyhow!("one or more nodes are not ready!"))
}
// TODO: we should inject in someway the logic to make the request
// in order to allow us to `mock` and easily test this.
// maybe moved to the provider with a NodeStatus, and some helpers like wait_running, wait_ready, etc... ? to be discussed
async fn check_nodes(nodes: &[&NetworkNode]) {
loop {
let tasks: Vec<_> = nodes
.iter()
.map(|node| {
trace!("🔎 checking node: {} ", node.name);
reqwest::get(node.prometheus_uri.clone())
})
.collect();
let all_ready = futures::future::try_join_all(tasks).await;
if all_ready.is_ok() {
return;
}
tokio::time::sleep(Duration::from_millis(1000)).await;
}
}
@@ -0,0 +1,330 @@
use std::{
collections::{hash_map::Entry, HashMap},
sync::Arc,
};
use configuration::{GlobalSettings, HrmpChannelConfig, NetworkConfig};
use futures::future::try_join_all;
use provider::{DynNamespace, ProviderError, ProviderNamespace};
use serde::{Deserialize, Serialize};
use support::{constants::THIS_IS_A_BUG, fs::FileSystem};
use tracing::{debug, trace};
use crate::{errors::OrchestratorError, ScopedFilesystem};
pub mod node;
pub mod relaychain;
pub mod teyrchain;
use self::{node::NodeSpec, relaychain::RelaychainSpec, teyrchain::TeyrchainSpec};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NetworkSpec {
/// Relaychain configuration.
pub(crate) relaychain: RelaychainSpec,
/// Parachains configurations.
pub(crate) parachains: Vec<TeyrchainSpec>,
/// HRMP channels configurations.
pub(crate) hrmp_channels: Vec<HrmpChannelConfig>,
/// Global settings
pub(crate) global_settings: GlobalSettings,
}
impl NetworkSpec {
pub async fn from_config(
network_config: &NetworkConfig,
) -> Result<NetworkSpec, OrchestratorError> {
let mut errs = vec![];
let relaychain = RelaychainSpec::from_config(network_config.relaychain())?;
let mut parachains = vec![];
// TODO: move to `fold` or map+fold
for para_config in network_config.parachains() {
match TeyrchainSpec::from_config(para_config, relaychain.chain.clone()) {
Ok(para) => parachains.push(para),
Err(err) => errs.push(err),
}
}
if errs.is_empty() {
Ok(NetworkSpec {
relaychain,
parachains,
hrmp_channels: network_config
.hrmp_channels()
.into_iter()
.cloned()
.collect(),
global_settings: network_config.global_settings().clone(),
})
} else {
let errs_str = errs
.into_iter()
.map(|e| e.to_string())
.collect::<Vec<String>>()
.join("\n");
Err(OrchestratorError::InvalidConfig(errs_str))
}
}
pub async fn populate_nodes_available_args(
&mut self,
ns: Arc<dyn ProviderNamespace + Send + Sync>,
) -> Result<(), OrchestratorError> {
let network_nodes = self.collect_network_nodes();
let mut image_command_to_nodes_mapping =
Self::create_image_command_to_nodes_mapping(network_nodes);
let available_args_outputs =
Self::retrieve_all_nodes_available_args_output(ns, &image_command_to_nodes_mapping)
.await?;
Self::update_nodes_available_args_output(
&mut image_command_to_nodes_mapping,
available_args_outputs,
);
Ok(())
}
//
pub async fn node_available_args_output(
&self,
node_spec: &NodeSpec,
ns: Arc<dyn ProviderNamespace + Send + Sync>,
) -> Result<String, ProviderError> {
// try to find a node that use the same combination of image/cmd
let cmp_fn = |ad_hoc: &&NodeSpec| -> bool {
ad_hoc.image == node_spec.image && ad_hoc.command == node_spec.command
};
// check if we already had computed the args output for this cmd/[image]
let node = self.relaychain.nodes.iter().find(cmp_fn);
let node = if let Some(node) = node {
Some(node)
} else {
let node = self
.parachains
.iter()
.find_map(|para| para.collators.iter().find(cmp_fn));
node
};
let output = if let Some(node) = node {
node.available_args_output.clone().expect(&format!(
"args_output should be set for running nodes {THIS_IS_A_BUG}"
))
} else {
// we need to compute the args output
let image = node_spec
.image
.as_ref()
.map(|image| image.as_str().to_string());
let command = node_spec.command.as_str().to_string();
ns.get_node_available_args((command, image)).await?
};
Ok(output)
}
pub fn relaychain(&self) -> &RelaychainSpec {
&self.relaychain
}
pub fn relaychain_mut(&mut self) -> &mut RelaychainSpec {
&mut self.relaychain
}
pub fn parachains_iter(&self) -> impl Iterator<Item = &TeyrchainSpec> {
self.parachains.iter()
}
pub fn parachains_iter_mut(&mut self) -> impl Iterator<Item = &mut TeyrchainSpec> {
self.parachains.iter_mut()
}
pub fn set_global_settings(&mut self, global_settings: GlobalSettings) {
self.global_settings = global_settings;
}
pub async fn build_parachain_artifacts<'a, T: FileSystem>(
&mut self,
ns: DynNamespace,
scoped_fs: &ScopedFilesystem<'a, T>,
relaychain_id: &str,
base_dir_exists: bool,
) -> Result<(), anyhow::Error> {
for para in self.parachains.iter_mut() {
let chain_spec_raw_path = para.build_chain_spec(relaychain_id, &ns, scoped_fs).await?;
trace!("creating dirs for {}", &para.unique_id);
if base_dir_exists {
scoped_fs.create_dir_all(&para.unique_id).await?;
} else {
scoped_fs.create_dir(&para.unique_id).await?;
};
trace!("created dirs for {}", &para.unique_id);
// create wasm/state
para.genesis_state
.build(
chain_spec_raw_path.clone(),
format!("{}/genesis-state", para.unique_id),
&ns,
scoped_fs,
None,
)
.await?;
debug!("parachain genesis state built!");
para.genesis_wasm
.build(
chain_spec_raw_path,
format!("{}/genesis-wasm", para.unique_id),
&ns,
scoped_fs,
None,
)
.await?;
debug!("parachain genesis wasm built!");
}
Ok(())
}
// collect mutable references to all nodes from relaychain and parachains
fn collect_network_nodes(&mut self) -> Vec<&mut NodeSpec> {
vec![
self.relaychain.nodes.iter_mut().collect::<Vec<_>>(),
self.parachains
.iter_mut()
.flat_map(|para| para.collators.iter_mut())
.collect(),
]
.into_iter()
.flatten()
.collect::<Vec<_>>()
}
// initialize the mapping of all possible node image/commands to corresponding nodes
fn create_image_command_to_nodes_mapping(
network_nodes: Vec<&mut NodeSpec>,
) -> HashMap<(Option<String>, String), Vec<&mut NodeSpec>> {
network_nodes.into_iter().fold(
HashMap::new(),
|mut acc: HashMap<(Option<String>, String), Vec<&mut node::NodeSpec>>, node| {
// build mapping key using image and command if image is present or command only
let key = node
.image
.as_ref()
.map(|image| {
(
Some(image.as_str().to_string()),
node.command.as_str().to_string(),
)
})
.unwrap_or_else(|| (None, node.command.as_str().to_string()));
// append the node to the vector of nodes for this image/command tuple
if let Entry::Vacant(entry) = acc.entry(key.clone()) {
entry.insert(vec![node]);
} else {
acc.get_mut(&key).unwrap().push(node);
}
acc
},
)
}
async fn retrieve_all_nodes_available_args_output(
ns: Arc<dyn ProviderNamespace + Send + Sync>,
image_command_to_nodes_mapping: &HashMap<(Option<String>, String), Vec<&mut NodeSpec>>,
) -> Result<Vec<(Option<String>, String, String)>, OrchestratorError> {
try_join_all(
image_command_to_nodes_mapping
.keys()
.map(|(image, command)| {
let ns = ns.clone();
let image = image.clone();
let command = command.clone();
async move {
// get node available args output from image/command
let available_args = ns
.get_node_available_args((command.clone(), image.clone()))
.await?;
debug!(
"retrieved available args for image: {:?}, command: {}",
image, command
);
// map the result to include image and command
Ok::<_, OrchestratorError>((image, command, available_args))
}
})
.collect::<Vec<_>>(),
)
.await
}
fn update_nodes_available_args_output(
image_command_to_nodes_mapping: &mut HashMap<(Option<String>, String), Vec<&mut NodeSpec>>,
available_args_outputs: Vec<(Option<String>, String, String)>,
) {
for (image, command, available_args_output) in available_args_outputs {
let nodes = image_command_to_nodes_mapping
.get_mut(&(image, command))
.expect(&format!(
"node image/command key should exist {THIS_IS_A_BUG}"
));
for node in nodes {
node.available_args_output = Some(available_args_output.clone());
}
}
}
}
#[cfg(test)]
mod tests {
#[tokio::test]
async fn small_network_config_get_spec() {
use configuration::NetworkConfigBuilder;
use super::*;
let config = NetworkConfigBuilder::new()
.with_relaychain(|r| {
r.with_chain("rococo-local")
.with_default_command("polkadot")
.with_validator(|node| node.with_name("alice"))
.with_fullnode(|node| node.with_name("bob").with_command("polkadot1"))
})
.with_parachain(|p| {
p.with_id(100)
.with_default_command("adder-collator")
.with_collator(|c| c.with_name("collator1"))
})
.build()
.unwrap();
let network_spec = NetworkSpec::from_config(&config).await.unwrap();
let alice = network_spec.relaychain.nodes.first().unwrap();
let bob = network_spec.relaychain.nodes.get(1).unwrap();
assert_eq!(alice.command.as_str(), "polkadot");
assert_eq!(bob.command.as_str(), "polkadot1");
assert!(alice.is_validator);
assert!(!bob.is_validator);
// paras
assert_eq!(network_spec.parachains.len(), 1);
let para_100 = network_spec.parachains.first().unwrap();
assert_eq!(para_100.id, 100);
}
}
@@ -0,0 +1,356 @@
use std::path::PathBuf;
use configuration::shared::{
node::{EnvVar, NodeConfig},
resources::Resources,
types::{Arg, AssetLocation, Command, Image},
};
use multiaddr::Multiaddr;
use provider::types::Port;
use serde::{Deserialize, Serialize};
use support::constants::THIS_IS_A_BUG;
use crate::{
errors::OrchestratorError,
generators,
network::AddNodeOptions,
shared::{
macros,
types::{ChainDefaultContext, NodeAccount, NodeAccounts, ParkedPort},
},
AddCollatorOptions,
};
macros::create_add_options!(AddNodeSpecOpts {
override_eth_key: Option<String>
});
macro_rules! impl_from_for_add_node_opts {
($struct:ident) => {
impl From<$struct> for AddNodeSpecOpts {
fn from(value: $struct) -> Self {
Self {
image: value.image,
command: value.command,
subcommand: value.subcommand,
args: value.args,
env: value.env,
is_validator: value.is_validator,
rpc_port: value.rpc_port,
prometheus_port: value.prometheus_port,
p2p_port: value.p2p_port,
override_eth_key: value.override_eth_key,
}
}
}
};
}
impl_from_for_add_node_opts!(AddNodeOptions);
impl_from_for_add_node_opts!(AddCollatorOptions);
/// A node configuration, with fine-grained configuration options.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct NodeSpec {
// Node name (should be unique or an index will be appended).
pub(crate) name: String,
/// Node key, used for compute the p2p identity.
pub(crate) key: String,
// libp2p local identity
pub(crate) peer_id: String,
/// Accounts to be injected in the keystore.
pub(crate) accounts: NodeAccounts,
/// Image to run (only podman/k8s). Override the default.
pub(crate) image: Option<Image>,
/// Command to run the node. Override the default.
pub(crate) command: Command,
/// Optional subcommand for the node.
pub(crate) subcommand: Option<Command>,
/// Arguments to use for node. Appended to default.
pub(crate) args: Vec<Arg>,
// The help command output containing the available arguments.
pub(crate) available_args_output: Option<String>,
/// Wether the node is a validator.
pub(crate) is_validator: bool,
/// Whether the node keys must be added to invulnerables.
pub(crate) is_invulnerable: bool,
/// Whether the node is a bootnode.
pub(crate) is_bootnode: bool,
/// Node initial balance present in genesis.
pub(crate) initial_balance: u128,
/// Environment variables to set (inside pod for podman/k8s, inside shell for native).
pub(crate) env: Vec<EnvVar>,
/// List of node's bootnodes addresses to use. Appended to default.
pub(crate) bootnodes_addresses: Vec<Multiaddr>,
/// Default resources. Override the default.
pub(crate) resources: Option<Resources>,
/// Websocket port to use.
pub(crate) ws_port: ParkedPort,
/// RPC port to use.
pub(crate) rpc_port: ParkedPort,
/// Prometheus port to use.
pub(crate) prometheus_port: ParkedPort,
/// P2P port to use.
pub(crate) p2p_port: ParkedPort,
/// libp2p cert hash to use with `webrtc` transport.
pub(crate) p2p_cert_hash: Option<String>,
/// Database snapshot. Override the default.
pub(crate) db_snapshot: Option<AssetLocation>,
/// P2P port to use by full node if this is the case
pub(crate) full_node_p2p_port: Option<ParkedPort>,
/// Prometheus port to use by full node if this is the case
pub(crate) full_node_prometheus_port: Option<ParkedPort>,
/// Optionally specify a log path for the node
pub(crate) node_log_path: Option<PathBuf>,
/// Optionally specify a keystore path for the node
pub(crate) keystore_path: Option<PathBuf>,
/// Keystore key types to generate.
/// Supports short form (e.g., "audi") using predefined schemas,
/// or long form (e.g., "audi_sr") with explicit schema (sr, ed, ec).
pub(crate) keystore_key_types: Vec<String>,
}
impl NodeSpec {
pub fn from_config(
node_config: &NodeConfig,
chain_context: &ChainDefaultContext,
full_node_present: bool,
evm_based: bool,
) -> Result<Self, OrchestratorError> {
// Check first if the image is set at node level, then try with the default
let image = node_config.image().or(chain_context.default_image).cloned();
// Check first if the command is set at node level, then try with the default
let command = if let Some(cmd) = node_config.command() {
cmd.clone()
} else if let Some(cmd) = chain_context.default_command {
cmd.clone()
} else {
return Err(OrchestratorError::InvalidNodeConfig(
node_config.name().into(),
"command".to_string(),
));
};
let subcommand = node_config.subcommand().cloned();
// If `args` is set at `node` level use them
// otherwise use the default_args (can be empty).
let args: Vec<Arg> = if node_config.args().is_empty() {
chain_context
.default_args
.iter()
.map(|x| x.to_owned().clone())
.collect()
} else {
node_config.args().into_iter().cloned().collect()
};
let (key, peer_id) = generators::generate_node_identity(node_config.name())?;
let mut name = node_config.name().to_string();
let seed = format!("//{}{name}", name.remove(0).to_uppercase());
let accounts = generators::generate_node_keys(&seed)?;
let mut accounts = NodeAccounts { seed, accounts };
if evm_based {
if let Some(session_key) = node_config.override_eth_key() {
accounts
.accounts
.insert("eth".into(), NodeAccount::new(session_key, session_key));
}
}
let db_snapshot = match (node_config.db_snapshot(), chain_context.default_db_snapshot) {
(Some(db_snapshot), _) => Some(db_snapshot),
(None, Some(db_snapshot)) => Some(db_snapshot),
_ => None,
};
let (full_node_p2p_port, full_node_prometheus_port) = if full_node_present {
(
Some(generators::generate_node_port(None)?),
Some(generators::generate_node_port(None)?),
)
} else {
(None, None)
};
Ok(Self {
name: node_config.name().to_string(),
key,
peer_id,
image,
command,
subcommand,
args,
available_args_output: None,
is_validator: node_config.is_validator(),
is_invulnerable: node_config.is_invulnerable(),
is_bootnode: node_config.is_bootnode(),
initial_balance: node_config.initial_balance(),
env: node_config.env().into_iter().cloned().collect(),
bootnodes_addresses: node_config
.bootnodes_addresses()
.into_iter()
.cloned()
.collect(),
resources: node_config.resources().cloned(),
p2p_cert_hash: node_config.p2p_cert_hash().map(str::to_string),
db_snapshot: db_snapshot.cloned(),
accounts,
ws_port: generators::generate_node_port(node_config.ws_port())?,
rpc_port: generators::generate_node_port(node_config.rpc_port())?,
prometheus_port: generators::generate_node_port(node_config.prometheus_port())?,
p2p_port: generators::generate_node_port(node_config.p2p_port())?,
full_node_p2p_port,
full_node_prometheus_port,
node_log_path: node_config.node_log_path().cloned(),
keystore_path: node_config.keystore_path().cloned(),
keystore_key_types: node_config
.keystore_key_types()
.into_iter()
.map(str::to_string)
.collect(),
})
}
pub fn from_ad_hoc(
name: impl Into<String>,
options: AddNodeSpecOpts,
chain_context: &ChainDefaultContext,
full_node_present: bool,
evm_based: bool,
) -> Result<Self, OrchestratorError> {
// Check first if the image is set at node level, then try with the default
let image = if let Some(img) = options.image {
Some(img.clone())
} else {
chain_context.default_image.cloned()
};
let name = name.into();
// Check first if the command is set at node level, then try with the default
let command = if let Some(cmd) = options.command {
cmd.clone()
} else if let Some(cmd) = chain_context.default_command {
cmd.clone()
} else {
return Err(OrchestratorError::InvalidNodeConfig(
name,
"command".to_string(),
));
};
let subcommand = options.subcommand.clone();
// If `args` is set at `node` level use them
// otherwise use the default_args (can be empty).
let args: Vec<Arg> = if options.args.is_empty() {
chain_context
.default_args
.iter()
.map(|x| x.to_owned().clone())
.collect()
} else {
options.args
};
let (key, peer_id) = generators::generate_node_identity(&name)?;
let mut name_capitalized = name.clone();
let seed = format!(
"//{}{name_capitalized}",
name_capitalized.remove(0).to_uppercase()
);
let accounts = generators::generate_node_keys(&seed)?;
let mut accounts = NodeAccounts { seed, accounts };
if evm_based {
if let Some(session_key) = options.override_eth_key.as_ref() {
accounts
.accounts
.insert("eth".into(), NodeAccount::new(session_key, session_key));
}
}
let (full_node_p2p_port, full_node_prometheus_port) = if full_node_present {
(
Some(generators::generate_node_port(None)?),
Some(generators::generate_node_port(None)?),
)
} else {
(None, None)
};
//
Ok(Self {
name,
key,
peer_id,
image,
command,
subcommand,
args,
available_args_output: None,
is_validator: options.is_validator,
is_invulnerable: false,
is_bootnode: false,
initial_balance: 0,
env: options.env,
bootnodes_addresses: vec![],
resources: None,
p2p_cert_hash: None,
db_snapshot: None,
accounts,
// should be deprecated now!
ws_port: generators::generate_node_port(None)?,
rpc_port: generators::generate_node_port(options.rpc_port)?,
prometheus_port: generators::generate_node_port(options.prometheus_port)?,
p2p_port: generators::generate_node_port(options.p2p_port)?,
full_node_p2p_port,
full_node_prometheus_port,
node_log_path: None,
keystore_path: None,
keystore_key_types: vec![],
})
}
pub(crate) fn supports_arg(&self, arg: impl AsRef<str>) -> bool {
self.available_args_output
.as_ref()
.expect(&format!(
"available args should be present at this point {THIS_IS_A_BUG}"
))
.contains(arg.as_ref())
}
pub fn command(&self) -> &str {
self.command.as_str()
}
}
@@ -0,0 +1,181 @@
use std::collections::{HashMap, HashSet};
use configuration::{
shared::{
helpers::generate_unique_node_name_from_names,
resources::Resources,
types::{Arg, AssetLocation, Chain, Command, Image},
},
types::JsonOverrides,
NodeConfig, RelaychainConfig,
};
use serde::{Deserialize, Serialize};
use support::replacer::apply_replacements;
use super::node::NodeSpec;
use crate::{
errors::OrchestratorError,
generators::chain_spec::{ChainSpec, Context},
shared::{constants::DEFAULT_CHAIN_SPEC_TPL_COMMAND, types::ChainDefaultContext},
};
/// A relaychain configuration spec
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RelaychainSpec {
/// Chain to use (e.g. rococo-local).
pub(crate) chain: Chain,
/// Default command to run the node. Can be overridden on each node.
pub(crate) default_command: Option<Command>,
/// Default image to use (only podman/k8s). Can be overridden on each node.
pub(crate) default_image: Option<Image>,
/// Default resources. Can be overridden on each node.
pub(crate) default_resources: Option<Resources>,
/// Default database snapshot. Can be overridden on each node.
pub(crate) default_db_snapshot: Option<AssetLocation>,
/// Default arguments to use in nodes. Can be overridden on each node.
pub(crate) default_args: Vec<Arg>,
// chain_spec_path: Option<AssetLocation>,
pub(crate) chain_spec: ChainSpec,
/// Set the count of nominators to generator (used with PoS networks).
pub(crate) random_nominators_count: u32,
/// Set the max nominators value (used with PoS networks).
pub(crate) max_nominations: u8,
/// Genesis overrides as JSON value.
pub(crate) runtime_genesis_patch: Option<serde_json::Value>,
/// Wasm override path/url to use.
pub(crate) wasm_override: Option<AssetLocation>,
/// Nodes to run.
pub(crate) nodes: Vec<NodeSpec>,
/// Raw chain-spec override path, url or inline json to use.
pub(crate) raw_spec_override: Option<JsonOverrides>,
}
impl RelaychainSpec {
pub fn from_config(config: &RelaychainConfig) -> Result<RelaychainSpec, OrchestratorError> {
// Relaychain main command to use, in order:
// set as `default_command` or
// use the command of the first node.
// If non of those is set, return an error.
let main_cmd = config
.default_command()
.or(config.nodes().first().and_then(|node| node.command()))
.ok_or(OrchestratorError::InvalidConfig(
"Relaychain, either default_command or first node with a command needs to be set."
.to_string(),
))?;
// TODO: internally we use image as String
let main_image = config
.default_image()
.or(config.nodes().first().and_then(|node| node.image()))
.map(|image| image.as_str().to_string());
let replacements = HashMap::from([
("disableBootnodes", "--disable-default-bootnode"),
("mainCommand", main_cmd.as_str()),
]);
let tmpl = if let Some(tmpl) = config.chain_spec_command() {
apply_replacements(tmpl, &replacements)
} else {
apply_replacements(DEFAULT_CHAIN_SPEC_TPL_COMMAND, &replacements)
};
let chain_spec = ChainSpec::new(config.chain().as_str(), Context::Relay)
.set_chain_name(config.chain().as_str())
.command(
tmpl.as_str(),
config.chain_spec_command_is_local(),
config.chain_spec_command_output_path(),
)
.image(main_image.clone());
// Add asset location if present
let chain_spec = if let Some(chain_spec_path) = config.chain_spec_path() {
chain_spec.asset_location(chain_spec_path.clone())
} else {
chain_spec
};
// add chain-spec runtime if present
let chain_spec = if let Some(chain_spec_runtime) = config.chain_spec_runtime() {
chain_spec.runtime(chain_spec_runtime.clone())
} else {
chain_spec
};
// build the `node_specs`
let chain_context = ChainDefaultContext {
default_command: config.default_command(),
default_image: config.default_image(),
default_resources: config.default_resources(),
default_db_snapshot: config.default_db_snapshot(),
default_args: config.default_args(),
};
let mut nodes: Vec<NodeConfig> = config.nodes().into_iter().cloned().collect();
nodes.extend(
config
.group_node_configs()
.into_iter()
.flat_map(|node_group| node_group.expand_group_configs()),
);
let mut names = HashSet::new();
let (nodes, mut errs) = nodes
.iter()
.map(|node_config| NodeSpec::from_config(node_config, &chain_context, false, false))
.fold((vec![], vec![]), |(mut nodes, mut errs), result| {
match result {
Ok(mut node) => {
let unique_name =
generate_unique_node_name_from_names(node.name, &mut names);
node.name = unique_name;
nodes.push(node);
},
Err(err) => errs.push(err),
}
(nodes, errs)
});
if !errs.is_empty() {
// TODO: merge errs, maybe return something like Result<Sometype, Vec<OrchestratorError>>
return Err(errs.swap_remove(0));
}
Ok(RelaychainSpec {
chain: config.chain().clone(),
default_command: config.default_command().cloned(),
default_image: config.default_image().cloned(),
default_resources: config.default_resources().cloned(),
default_db_snapshot: config.default_db_snapshot().cloned(),
wasm_override: config.wasm_override().cloned(),
default_args: config.default_args().into_iter().cloned().collect(),
chain_spec,
random_nominators_count: config.random_nominators_count().unwrap_or(0),
max_nominations: config.max_nominations().unwrap_or(24),
runtime_genesis_patch: config.runtime_genesis_patch().cloned(),
nodes,
raw_spec_override: config.raw_spec_override().cloned(),
})
}
pub fn chain_spec(&self) -> &ChainSpec {
&self.chain_spec
}
pub fn chain_spec_mut(&mut self) -> &mut ChainSpec {
&mut self.chain_spec
}
}
@@ -0,0 +1,386 @@
use std::{
collections::{HashMap, HashSet},
path::PathBuf,
};
use configuration::{
shared::{helpers::generate_unique_node_name_from_names, resources::Resources},
types::{Arg, AssetLocation, Chain, Command, Image, JsonOverrides},
NodeConfig, ParachainConfig, RegistrationStrategy,
};
use provider::DynNamespace;
use serde::{Deserialize, Serialize};
use support::{fs::FileSystem, replacer::apply_replacements};
use tracing::debug;
use super::node::NodeSpec;
use crate::{
errors::OrchestratorError,
generators::{
chain_spec::{ChainSpec, Context, ParaGenesisConfig},
para_artifact::*,
},
shared::{constants::DEFAULT_CHAIN_SPEC_TPL_COMMAND, types::ChainDefaultContext},
ScopedFilesystem,
};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TeyrchainSpec {
// `name` of the parachain (used in some corner cases)
// name: Option<Chain>,
/// Parachain id
pub(crate) id: u32,
/// Unique id of the parachain, in the patter of <para_id>-<n>
/// where the suffix is only present if more than one parachain is set with the same id
pub(crate) unique_id: String,
/// Default command to run the node. Can be overridden on each node.
pub(crate) default_command: Option<Command>,
/// Default image to use (only podman/k8s). Can be overridden on each node.
pub(crate) default_image: Option<Image>,
/// Default resources. Can be overridden on each node.
pub(crate) default_resources: Option<Resources>,
/// Default database snapshot. Can be overridden on each node.
pub(crate) default_db_snapshot: Option<AssetLocation>,
/// Default arguments to use in nodes. Can be overridden on each node.
pub(crate) default_args: Vec<Arg>,
/// Chain-spec, only needed by cumulus based paras
pub(crate) chain_spec: Option<ChainSpec>,
/// Do not automatically assign a bootnode role if no nodes are marked as bootnodes.
pub(crate) no_default_bootnodes: bool,
/// Registration strategy to use
pub(crate) registration_strategy: RegistrationStrategy,
/// Onboard as parachain or parathread
pub(crate) onboard_as_parachain: bool,
/// Is the parachain cumulus-based
pub(crate) is_cumulus_based: bool,
/// Is the parachain evm-based
pub(crate) is_evm_based: bool,
/// Initial balance
pub(crate) initial_balance: u128,
/// Genesis state (head) to register the parachain
pub(crate) genesis_state: ParaArtifact,
/// Genesis WASM to register the parachain
pub(crate) genesis_wasm: ParaArtifact,
/// Genesis overrides as JSON value.
pub(crate) genesis_overrides: Option<serde_json::Value>,
/// Wasm override path/url to use.
pub(crate) wasm_override: Option<AssetLocation>,
/// Collators to spawn
pub(crate) collators: Vec<NodeSpec>,
/// Raw chain-spec override path, url or inline json to use.
pub(crate) raw_spec_override: Option<JsonOverrides>,
/// Bootnodes addresses to use for the parachain nodes
pub(crate) bootnodes_addresses: Vec<multiaddr::Multiaddr>,
}
impl TeyrchainSpec {
pub fn from_config(
config: &ParachainConfig,
relay_chain: Chain,
) -> Result<TeyrchainSpec, OrchestratorError> {
let main_cmd = if let Some(cmd) = config.default_command() {
cmd
} else if let Some(first_node) = config.collators().first() {
let Some(cmd) = first_node.command() else {
return Err(OrchestratorError::InvalidConfig(format!("Parachain {}, either default_command or command in the first node needs to be set.", config.id())));
};
cmd
} else {
return Err(OrchestratorError::InvalidConfig(format!(
"Parachain {}, without nodes and default_command isn't set.",
config.id()
)));
};
// TODO: internally we use image as String
let main_image = config
.default_image()
.or(config.collators().first().and_then(|node| node.image()))
.map(|image| image.as_str().to_string());
let chain_spec = if config.is_cumulus_based() {
// we need a chain-spec
let chain_name = if let Some(chain_name) = config.chain() {
chain_name.as_str()
} else {
""
};
let chain_spec_builder = if chain_name.is_empty() {
// if the chain don't have name use the unique_id for the name of the file
ChainSpec::new(
config.unique_id().to_string(),
Context::Para {
relay_chain,
para_id: config.id(),
},
)
} else {
let chain_spec_file_name = if config.unique_id().contains('-') {
&format!("{}-{}", chain_name, config.unique_id())
} else {
chain_name
};
ChainSpec::new(
chain_spec_file_name,
Context::Para {
relay_chain,
para_id: config.id(),
},
)
};
let chain_spec_builder = chain_spec_builder.set_chain_name(chain_name);
let replacements = HashMap::from([
("disableBootnodes", "--disable-default-bootnode"),
("mainCommand", main_cmd.as_str()),
]);
let tmpl = if let Some(tmpl) = config.chain_spec_command() {
apply_replacements(tmpl, &replacements)
} else {
apply_replacements(DEFAULT_CHAIN_SPEC_TPL_COMMAND, &replacements)
};
let chain_spec = chain_spec_builder
.command(
tmpl.as_str(),
config.chain_spec_command_is_local(),
config.chain_spec_command_output_path(),
)
.image(main_image.clone());
let chain_spec = if let Some(chain_spec_path) = config.chain_spec_path() {
chain_spec.asset_location(chain_spec_path.clone())
} else {
chain_spec
};
// add chain-spec runtime if present
let chain_spec = if let Some(chain_spec_runtime) = config.chain_spec_runtime() {
chain_spec.runtime(chain_spec_runtime.clone())
} else {
chain_spec
};
Some(chain_spec)
} else {
None
};
// build the `node_specs`
let chain_context = ChainDefaultContext {
default_command: config.default_command(),
default_image: config.default_image(),
default_resources: config.default_resources(),
default_db_snapshot: config.default_db_snapshot(),
default_args: config.default_args(),
};
// We want to track the errors for all the nodes and report them ones
let mut errs: Vec<OrchestratorError> = Default::default();
let mut collators: Vec<NodeSpec> = Default::default();
let mut nodes: Vec<NodeConfig> = config.collators().into_iter().cloned().collect();
nodes.extend(
config
.group_collators_configs()
.into_iter()
.flat_map(|node_group| node_group.expand_group_configs()),
);
let mut names = HashSet::new();
for node_config in nodes {
match NodeSpec::from_config(&node_config, &chain_context, true, config.is_evm_based()) {
Ok(mut node) => {
let unique_name = generate_unique_node_name_from_names(node.name, &mut names);
node.name = unique_name;
collators.push(node)
},
Err(err) => errs.push(err),
}
}
let genesis_state = if let Some(path) = config.genesis_state_path() {
ParaArtifact::new(
ParaArtifactType::State,
ParaArtifactBuildOption::Path(path.to_string()),
)
} else {
let cmd = if let Some(cmd) = config.genesis_state_generator() {
cmd.cmd()
} else {
main_cmd
};
ParaArtifact::new(
ParaArtifactType::State,
ParaArtifactBuildOption::Command(cmd.as_str().into()),
)
.image(main_image.clone())
};
let genesis_wasm = if let Some(path) = config.genesis_wasm_path() {
ParaArtifact::new(
ParaArtifactType::Wasm,
ParaArtifactBuildOption::Path(path.to_string()),
)
} else {
let cmd = if let Some(cmd) = config.genesis_wasm_generator() {
cmd.as_str()
} else {
main_cmd.as_str()
};
ParaArtifact::new(
ParaArtifactType::Wasm,
ParaArtifactBuildOption::Command(cmd.into()),
)
.image(main_image.clone())
};
let para_spec = TeyrchainSpec {
id: config.id(),
// ensure unique id is set at this point, if not just set to the para_id
unique_id: if config.unique_id().is_empty() {
config.id().to_string()
} else {
config.unique_id().to_string()
},
default_command: config.default_command().cloned(),
default_image: config.default_image().cloned(),
default_resources: config.default_resources().cloned(),
default_db_snapshot: config.default_db_snapshot().cloned(),
wasm_override: config.wasm_override().cloned(),
default_args: config.default_args().into_iter().cloned().collect(),
chain_spec,
no_default_bootnodes: config.no_default_bootnodes(),
registration_strategy: config
.registration_strategy()
.unwrap_or(&RegistrationStrategy::InGenesis)
.clone(),
onboard_as_parachain: config.onboard_as_parachain(),
is_cumulus_based: config.is_cumulus_based(),
is_evm_based: config.is_evm_based(),
initial_balance: config.initial_balance(),
genesis_state,
genesis_wasm,
genesis_overrides: config.genesis_overrides().cloned(),
collators,
raw_spec_override: config.raw_spec_override().cloned(),
bootnodes_addresses: config.bootnodes_addresses().into_iter().cloned().collect(),
};
Ok(para_spec)
}
pub fn registration_strategy(&self) -> &RegistrationStrategy {
&self.registration_strategy
}
pub fn get_genesis_config(&self) -> Result<ParaGenesisConfig<&PathBuf>, OrchestratorError> {
let genesis_config = ParaGenesisConfig {
state_path: self.genesis_state.artifact_path().ok_or(
OrchestratorError::InvariantError(
"artifact path for state must be set at this point",
),
)?,
wasm_path: self.genesis_wasm.artifact_path().ok_or(
OrchestratorError::InvariantError(
"artifact path for wasm must be set at this point",
),
)?,
id: self.id,
as_parachain: self.onboard_as_parachain,
};
Ok(genesis_config)
}
pub fn id(&self) -> u32 {
self.id
}
pub fn chain_spec(&self) -> Option<&ChainSpec> {
self.chain_spec.as_ref()
}
pub fn chain_spec_mut(&mut self) -> Option<&mut ChainSpec> {
self.chain_spec.as_mut()
}
/// Build parachain chain-spec
///
/// This function customize the chain-spec (if is possible) and build the raw version
/// of the chain-spec.
pub(crate) async fn build_chain_spec<'a, T>(
&mut self,
relay_chain_id: &str,
ns: &DynNamespace,
scoped_fs: &ScopedFilesystem<'a, T>,
) -> Result<Option<PathBuf>, anyhow::Error>
where
T: FileSystem,
{
let cloned = self.clone();
let chain_spec_raw_path = if let Some(chain_spec) = self.chain_spec.as_mut() {
debug!("parachain chain-spec building!");
chain_spec.build(ns, scoped_fs).await?;
debug!("parachain chain-spec built!");
chain_spec
.customize_para(&cloned, relay_chain_id, scoped_fs)
.await?;
debug!("parachain chain-spec customized!");
chain_spec
.build_raw(ns, scoped_fs, Some(relay_chain_id.try_into()?))
.await?;
debug!("parachain chain-spec raw built!");
// override wasm if needed
if let Some(ref wasm_override) = self.wasm_override {
chain_spec.override_code(scoped_fs, wasm_override).await?;
}
// override raw spec if needed
if let Some(ref raw_spec_override) = self.raw_spec_override {
chain_spec
.override_raw_spec(scoped_fs, raw_spec_override)
.await?;
}
let chain_spec_raw_path =
chain_spec
.raw_path()
.ok_or(OrchestratorError::InvariantError(
"chain-spec raw path should be set now",
))?;
Some(chain_spec_raw_path.to_path_buf())
} else {
None
};
Ok(chain_spec_raw_path)
}
/// Get the bootnodes addresses for the parachain spec
pub(crate) fn bootnodes_addresses(&self) -> Vec<&multiaddr::Multiaddr> {
self.bootnodes_addresses.iter().collect()
}
}
@@ -0,0 +1,3 @@
pub mod constants;
pub mod macros;
pub mod types;
@@ -0,0 +1,17 @@
/// Prometheus exporter default port
pub const PROMETHEUS_PORT: u16 = 9615;
/// Prometheus exporter default port in collator full-node
pub const FULL_NODE_PROMETHEUS_PORT: u16 = 9616;
/// JSON-RPC server (ws)
pub const RPC_PORT: u16 = 9944;
// JSON-RPC server (http, used by old versions)
pub const RPC_HTTP_PORT: u16 = 9933;
// P2P default port
pub const P2P_PORT: u16 = 30333;
// default command template to build chain-spec
pub const DEFAULT_CHAIN_SPEC_TPL_COMMAND: &str =
"{{mainCommand}} build-spec --chain {{chainName}} {{disableBootnodes}}";
// interval to determine how often to run node liveness checks
pub const NODE_MONITORING_INTERVAL_SECONDS: u64 = 15;
// how long to wait before a node is considered unresponsive
pub const NODE_MONITORING_FAILURE_THRESHOLD_SECONDS: u64 = 5;
@@ -0,0 +1,32 @@
macro_rules! create_add_options {
($struct:ident {$( $field:ident:$type:ty ),*}) =>{
#[derive(Default, Debug, Clone)]
pub struct $struct {
/// Image to run the node
pub image: Option<Image>,
/// Command to run the node
pub command: Option<Command>,
/// Subcommand for the node
pub subcommand: Option<Command>,
/// Arguments to pass to the node
pub args: Vec<Arg>,
/// Env vars to set
pub env: Vec<EnvVar>,
/// Make the node a validator
///
/// This implies `--validator` or `--collator`
pub is_validator: bool,
/// RPC port to use, if None a random one will be set
pub rpc_port: Option<Port>,
/// Prometheus port to use, if None a random one will be set
pub prometheus_port: Option<Port>,
/// P2P port to use, if None a random one will be set
pub p2p_port: Option<Port>,
$(
pub $field: $type,
)*
}
};
}
pub(crate) use create_add_options;
@@ -0,0 +1,99 @@
use std::{
collections::HashMap,
net::TcpListener,
path::PathBuf,
sync::{Arc, RwLock},
};
use configuration::shared::{
resources::Resources,
types::{Arg, AssetLocation, Command, Image, Port},
};
use serde::{Deserialize, Serialize};
pub type Accounts = HashMap<String, NodeAccount>;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct NodeAccount {
pub address: String,
pub public_key: String,
}
impl NodeAccount {
pub fn new(addr: impl Into<String>, pk: impl Into<String>) -> Self {
Self {
address: addr.into(),
public_key: pk.into(),
}
}
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct NodeAccounts {
pub seed: String,
pub accounts: Accounts,
}
#[derive(Clone, Default, Debug, Serialize, Deserialize)]
pub struct ParkedPort(
pub(crate) Port,
#[serde(skip)] pub(crate) Arc<RwLock<Option<TcpListener>>>,
);
impl ParkedPort {
pub(crate) fn new(port: u16, listener: TcpListener) -> ParkedPort {
let listener = Arc::new(RwLock::new(Some(listener)));
ParkedPort(port, listener)
}
pub(crate) fn drop_listener(&self) {
// drop the listener will allow the running node to start listenen connections
let mut l = self.1.write().unwrap();
*l = None;
}
}
#[derive(Debug, Clone, Default)]
pub struct ChainDefaultContext<'a> {
pub default_command: Option<&'a Command>,
pub default_image: Option<&'a Image>,
pub default_resources: Option<&'a Resources>,
pub default_db_snapshot: Option<&'a AssetLocation>,
pub default_args: Vec<&'a Arg>,
}
#[derive(Debug, Clone)]
pub struct RegisterParachainOptions {
pub id: u32,
pub wasm_path: PathBuf,
pub state_path: PathBuf,
pub node_ws_url: String,
pub onboard_as_para: bool,
pub seed: Option<[u8; 32]>,
pub finalization: bool,
}
pub struct RuntimeUpgradeOptions {
/// Location of the wasm file (could be either a local file or an url)
pub wasm: AssetLocation,
/// Name of the node to use as rpc endpoint
pub node_name: Option<String>,
/// Seed to use to sign and submit (default to //Alice)
pub seed: Option<[u8; 32]>,
}
impl RuntimeUpgradeOptions {
pub fn new(wasm: AssetLocation) -> Self {
Self {
wasm,
node_name: None,
seed: None,
}
}
}
#[derive(Debug, Clone)]
pub struct ParachainGenesisArgs {
pub genesis_head: String,
pub validation_code: String,
pub parachain: bool,
}
@@ -0,0 +1,305 @@
use std::{collections::HashMap, path::PathBuf};
use anyhow::Context;
use configuration::GlobalSettings;
use provider::{
constants::{LOCALHOST, NODE_CONFIG_DIR, NODE_DATA_DIR, NODE_RELAY_DATA_DIR, P2P_PORT},
shared::helpers::running_in_ci,
types::{SpawnNodeOptions, TransferedFile},
DynNamespace,
};
use support::{
constants::THIS_IS_A_BUG, fs::FileSystem, replacer::apply_running_network_replacements,
};
use tracing::info;
use crate::{
generators,
network::node::NetworkNode,
network_spec::{node::NodeSpec, teyrchain::TeyrchainSpec},
shared::constants::{FULL_NODE_PROMETHEUS_PORT, PROMETHEUS_PORT, RPC_PORT},
ScopedFilesystem, ZombieRole,
};
#[derive(Clone)]
pub struct SpawnNodeCtx<'a, T: FileSystem> {
/// Relaychain id, from the chain-spec (e.g rococo_local_testnet)
pub(crate) chain_id: &'a str,
// Parachain id, from the chain-spec (e.g local_testnet)
pub(crate) parachain_id: Option<&'a str>,
/// Relaychain chain name (e.g rococo-local)
pub(crate) chain: &'a str,
/// Role of the node in the network
pub(crate) role: ZombieRole,
/// Ref to the namespace
pub(crate) ns: &'a DynNamespace,
/// Ref to an scoped filesystem (encapsulate fs actions inside the ns directory)
pub(crate) scoped_fs: &'a ScopedFilesystem<'a, T>,
/// Ref to a parachain (used to spawn collators)
pub(crate) parachain: Option<&'a TeyrchainSpec>,
/// The string representation of the bootnode address to pass to nodes
pub(crate) bootnodes_addr: &'a Vec<String>,
/// Flag to wait node is ready or not
/// Ready state means we can query Prometheus internal server
pub(crate) wait_ready: bool,
/// A json representation of the running nodes with their names as 'key'
pub(crate) nodes_by_name: serde_json::Value,
/// A ref to the global settings
pub(crate) global_settings: &'a GlobalSettings,
}
pub async fn spawn_node<'a, T>(
node: &NodeSpec,
mut files_to_inject: Vec<TransferedFile>,
ctx: &SpawnNodeCtx<'a, T>,
) -> Result<NetworkNode, anyhow::Error>
where
T: FileSystem,
{
let mut created_paths = vec![];
// Create and inject the keystore IFF
// - The node is validator in the relaychain
// - The node is collator (encoded as validator) and the parachain is cumulus_based
// (parachain_id) should be set then.
if node.is_validator && (ctx.parachain.is_none() || ctx.parachain_id.is_some()) {
// Generate keystore for node
let node_files_path = if let Some(para) = ctx.parachain {
para.id.to_string()
} else {
node.name.clone()
};
let asset_hub_polkadot = ctx
.parachain_id
.map(|id| id.starts_with("asset-hub-polkadot"))
.unwrap_or_default();
let keystore_key_types = node.keystore_key_types.iter().map(String::as_str).collect();
let key_filenames = generators::generate_node_keystore(
&node.accounts,
&node_files_path,
ctx.scoped_fs,
asset_hub_polkadot,
keystore_key_types,
)
.await
.unwrap();
// Paths returned are relative to the base dir, we need to convert into
// fullpaths to inject them in the nodes.
let remote_keystore_chain_id = if let Some(id) = ctx.parachain_id {
id
} else {
ctx.chain_id
};
let keystore_path = node.keystore_path.clone().unwrap_or(PathBuf::from(format!(
"/data/chains/{remote_keystore_chain_id}/keystore",
)));
for key_filename in key_filenames {
let f = TransferedFile::new(
PathBuf::from(format!(
"{}/{}/{}",
ctx.ns.base_dir().to_string_lossy(),
node_files_path,
key_filename.to_string_lossy()
)),
keystore_path.join(key_filename),
);
files_to_inject.push(f);
}
created_paths.push(keystore_path);
}
let base_dir = format!("{}/{}", ctx.ns.base_dir().to_string_lossy(), &node.name);
let (cfg_path, data_path, relay_data_path) = if !ctx.ns.capabilities().prefix_with_full_path {
(
NODE_CONFIG_DIR.into(),
NODE_DATA_DIR.into(),
NODE_RELAY_DATA_DIR.into(),
)
} else {
let cfg_path = format!("{}{NODE_CONFIG_DIR}", &base_dir);
let data_path = format!("{}{NODE_DATA_DIR}", &base_dir);
let relay_data_path = format!("{}{NODE_RELAY_DATA_DIR}", &base_dir);
(cfg_path, data_path, relay_data_path)
};
let gen_opts = generators::GenCmdOptions {
relay_chain_name: ctx.chain,
cfg_path: &cfg_path, // TODO: get from provider/ns
data_path: &data_path, // TODO: get from provider
relay_data_path: &relay_data_path, // TODO: get from provider
use_wrapper: false, // TODO: get from provider
bootnode_addr: ctx.bootnodes_addr.clone(),
use_default_ports_in_cmd: ctx.ns.capabilities().use_default_ports_in_cmd,
// IFF the provider require an image (e.g k8s) we know this is not native
is_native: !ctx.ns.capabilities().requires_image,
};
let mut collator_full_node_prom_port: Option<u16> = None;
let mut collator_full_node_prom_port_external: Option<u16> = None;
let (program, args) = match ctx.role {
// Collator should be `non-cumulus` one (e.g adder/undying)
ZombieRole::Node | ZombieRole::Collator => {
let maybe_para_id = ctx.parachain.map(|para| para.id);
generators::generate_node_command(node, gen_opts, maybe_para_id)
},
ZombieRole::CumulusCollator => {
let para = ctx.parachain.expect(&format!(
"parachain must be part of the context {THIS_IS_A_BUG}"
));
collator_full_node_prom_port = node.full_node_prometheus_port.as_ref().map(|p| p.0);
generators::generate_node_command_cumulus(node, gen_opts, para.id)
},
_ => unreachable!(), /* TODO: do we need those?
* ZombieRole::Bootnode => todo!(),
* ZombieRole::Companion => todo!(), */
};
// apply running networ replacements
let args: Vec<String> = args
.iter()
.map(|arg| apply_running_network_replacements(arg, &ctx.nodes_by_name))
.collect();
info!(
"🚀 {}, spawning.... with command: {} {}",
node.name,
program,
args.join(" ")
);
let ports = if ctx.ns.capabilities().use_default_ports_in_cmd {
// should use default ports to as internal
[
(P2P_PORT, node.p2p_port.0),
(RPC_PORT, node.rpc_port.0),
(PROMETHEUS_PORT, node.prometheus_port.0),
]
} else {
[
(P2P_PORT, P2P_PORT),
(RPC_PORT, RPC_PORT),
(PROMETHEUS_PORT, PROMETHEUS_PORT),
]
};
let spawn_ops = SpawnNodeOptions::new(node.name.clone(), program)
.args(args)
.env(
node.env
.iter()
.map(|var| (var.name.clone(), var.value.clone())),
)
.injected_files(files_to_inject)
.created_paths(created_paths)
.db_snapshot(node.db_snapshot.clone())
.port_mapping(HashMap::from(ports))
.node_log_path(node.node_log_path.clone());
let spawn_ops = if let Some(image) = node.image.as_ref() {
spawn_ops.image(image.as_str())
} else {
spawn_ops
};
// Drops the port parking listeners before spawn
node.ws_port.drop_listener();
node.p2p_port.drop_listener();
node.rpc_port.drop_listener();
node.prometheus_port.drop_listener();
if let Some(port) = &node.full_node_p2p_port {
port.drop_listener();
}
if let Some(port) = &node.full_node_prometheus_port {
port.drop_listener();
}
let running_node = ctx.ns.spawn_node(&spawn_ops).await.with_context(|| {
format!(
"Failed to spawn node: {} with opts: {:#?}",
node.name, spawn_ops
)
})?;
let mut ip_to_use = if let Some(local_ip) = ctx.global_settings.local_ip() {
*local_ip
} else {
LOCALHOST
};
let (rpc_port_external, prometheus_port_external, p2p_external);
if running_in_ci() && ctx.ns.provider_name() == "k8s" {
// running kubernets in ci require to use ip and default port
(rpc_port_external, prometheus_port_external, p2p_external) =
(RPC_PORT, PROMETHEUS_PORT, P2P_PORT);
collator_full_node_prom_port_external = Some(FULL_NODE_PROMETHEUS_PORT);
ip_to_use = running_node.ip().await?;
} else {
// Create port-forward iff we are not in CI or provider doesn't use the default ports (native)
let ports = futures::future::try_join_all(vec![
running_node.create_port_forward(node.rpc_port.0, RPC_PORT),
running_node.create_port_forward(node.prometheus_port.0, PROMETHEUS_PORT),
])
.await?;
(rpc_port_external, prometheus_port_external, p2p_external) = (
ports[0].unwrap_or(node.rpc_port.0),
ports[1].unwrap_or(node.prometheus_port.0),
// p2p don't need port-fwd
node.p2p_port.0,
);
if let Some(full_node_prom_port) = collator_full_node_prom_port {
let port_fwd = running_node
.create_port_forward(full_node_prom_port, FULL_NODE_PROMETHEUS_PORT)
.await?;
collator_full_node_prom_port_external = Some(port_fwd.unwrap_or(full_node_prom_port));
}
}
let multiaddr = generators::generate_node_bootnode_addr(
&node.peer_id,
&running_node.ip().await?,
p2p_external,
running_node.args().as_ref(),
&node.p2p_cert_hash,
)?;
let ws_uri = format!("ws://{ip_to_use}:{rpc_port_external}");
let prometheus_uri = format!("http://{ip_to_use}:{prometheus_port_external}/metrics");
info!("🚀 {}, should be running now", node.name);
info!(
"💻 {}: direct link (pjs) https://polkadot.js.org/apps/?rpc={ws_uri}#/explorer",
node.name
);
info!(
"💻 {}: direct link (papi) https://dev.papi.how/explorer#networkId=custom&endpoint={ws_uri}",
node.name
);
info!("📊 {}: metrics link {prometheus_uri}", node.name);
if let Some(full_node_prom_port) = collator_full_node_prom_port_external {
info!(
"📊 {}: collator full-node metrics link http://{}:{}/metrics",
node.name, ip_to_use, full_node_prom_port
);
}
info!("📓 logs cmd: {}", running_node.log_cmd());
Ok(NetworkNode::new(
node.name.clone(),
ws_uri,
prometheus_uri,
multiaddr,
node.clone(),
running_node,
))
}
@@ -0,0 +1,2 @@
pub mod client;
pub mod runtime_upgrade;
@@ -0,0 +1,43 @@
use pezkuwi_subxt::{backend::rpc::RpcClient, OnlineClient};
#[async_trait::async_trait]
pub trait ClientFromUrl: Sized {
async fn from_secure_url(url: &str) -> Result<Self, pezkuwi_subxt::Error>;
async fn from_insecure_url(url: &str) -> Result<Self, pezkuwi_subxt::Error>;
}
#[async_trait::async_trait]
impl<Config: pezkuwi_subxt::Config + Send + Sync> ClientFromUrl for OnlineClient<Config> {
async fn from_secure_url(url: &str) -> Result<Self, pezkuwi_subxt::Error> {
Self::from_url(url).await.map_err(Into::into)
}
async fn from_insecure_url(url: &str) -> Result<Self, pezkuwi_subxt::Error> {
Self::from_insecure_url(url).await.map_err(Into::into)
}
}
#[async_trait::async_trait]
impl ClientFromUrl for RpcClient {
async fn from_secure_url(url: &str) -> Result<Self, pezkuwi_subxt::Error> {
Self::from_url(url)
.await
.map_err(pezkuwi_subxt::Error::from)
}
async fn from_insecure_url(url: &str) -> Result<Self, pezkuwi_subxt::Error> {
Self::from_insecure_url(url)
.await
.map_err(pezkuwi_subxt::Error::from)
}
}
pub async fn get_client_from_url<T: ClientFromUrl + Send>(
url: &str,
) -> Result<T, pezkuwi_subxt::Error> {
if pezkuwi_subxt::utils::url_is_secure(url)? {
T::from_secure_url(url).await
} else {
T::from_insecure_url(url).await
}
}
@@ -0,0 +1,69 @@
use pezkuwi_subxt::{dynamic::Value, tx::TxStatus, BizinikiwConfig, OnlineClient};
use pezkuwi_subxt_signer::sr25519::Keypair;
use tracing::{debug, info};
use crate::network::node::NetworkNode;
pub async fn upgrade(
node: &NetworkNode,
wasm_data: &[u8],
sudo: &Keypair,
) -> Result<(), anyhow::Error> {
debug!(
"Upgrading runtime, using node: {} with endpoting {}",
node.name, node.ws_uri
);
let api: OnlineClient<BizinikiwConfig> = node.wait_client().await?;
let upgrade = pezkuwi_subxt::dynamic::tx(
"System",
"set_code_without_checks",
vec![Value::from_bytes(wasm_data)],
);
let sudo_call = pezkuwi_subxt::dynamic::tx(
"Sudo",
"sudo_unchecked_weight",
vec![
upgrade.into_value(),
Value::named_composite([
("ref_time", Value::primitive(1.into())),
("proof_size", Value::primitive(1.into())),
]),
],
);
let mut tx = api
.tx()
.sign_and_submit_then_watch_default(&sudo_call, sudo)
.await?;
// Below we use the low level API to replicate the `wait_for_in_block` behaviour
// which was removed in subxt 0.33.0. See https://github.com/paritytech/subxt/pull/1237.
while let Some(status) = tx.next().await {
let status = status?;
match &status {
TxStatus::InBestBlock(tx_in_block) | TxStatus::InFinalizedBlock(tx_in_block) => {
let _result = tx_in_block.wait_for_success().await?;
let block_status = if status.as_finalized().is_some() {
"Finalized"
} else {
"Best"
};
info!(
"[{}] In block: {:#?}",
block_status,
tx_in_block.block_hash()
);
},
TxStatus::Error { message }
| TxStatus::Invalid { message }
| TxStatus::Dropped { message } => {
return Err(anyhow::format_err!("Error submitting tx: {message}"));
},
_ => continue,
}
}
Ok(())
}
@@ -0,0 +1,8 @@
use serde::Deserializer;
pub fn default_as_empty_vec<'de, D, T>(_deserializer: D) -> Result<Vec<T>, D::Error>
where
D: Deserializer<'de>,
{
Ok(Vec::new())
}
@@ -0,0 +1,211 @@
{
"name": "Rococo Local Testnet",
"id": "rococo_local_testnet",
"chainType": "Local",
"bootNodes": [
"/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWJcDp2Cdok4uSHz5zpjWzfduNCzis9GsMfpej1jwdaYij"
],
"telemetryEndpoints": null,
"protocolId": "dot",
"properties": null,
"forkBlocks": null,
"badBlocks": null,
"lightSyncState": null,
"codeSubstitutes": {},
"genesis": {
"runtime": {
"system": {
"code": "0x52"
},
"babe": {
"authorities": [],
"epochConfig": {
"c": [
1,
4
],
"allowed_slots": "PrimaryAndSecondaryVRFSlots"
}
},
"indices": {
"indices": []
},
"balances": {
"balances": [
[
"5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY",
1000000000000000000
],
[
"5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty",
1000000000000000000
],
[
"5FLSigC9HGRKVhB9FiEo4Y3koPsNmBmLJbpXg2mp1hXcS59Y",
1000000000000000000
],
[
"5DAAnrj7VHTznn2AWBemMuyBwZWs6FNFjdyVXUeYum3PTXFy",
1000000000000000000
],
[
"5HGjWAeFDfFCWPsjFQdVV2Msvz2XtMktvgocEZcCj68kUMaw",
1000000000000000000
],
[
"5CiPPseXPECbkjWCa6MnjNokrgYjMqmKndv2rSnekmSK2DjL",
1000000000000000000
],
[
"5GNJqTPyNqANBkUVMN1LPPrxXnFouWXoe2wNSmmEoLctxiZY",
1000000000000000000
],
[
"5HpG9w8EBLe5XCrbczpwq5TSXvedjrBGCwqxK1iQ7qUsSWFc",
1000000000000000000
],
[
"5Ck5SLSHYac6WFt5UZRSsdJjwmpSZq85fd5TRNAdZQVzEAPT",
1000000000000000000
],
[
"5HKPmK9GYtE1PSLsS1qiYU9xQ9Si1NcEhdeCq9sw5bqu4ns8",
1000000000000000000
],
[
"5FCfAonRZgTFrTd9HREEyeJjDpT397KMzizE6T3DvebLFE7n",
1000000000000000000
],
[
"5CRmqmsiNFExV6VbdmPJViVxrWmkaXXvBrSX8oqBT8R9vmWk",
1000000000000000000
]
]
},
"beefy": {
"authorities": [],
"genesisBlock": 1
},
"session": {
"keys": [
[
"5GNJqTPyNqANBkUVMN1LPPrxXnFouWXoe2wNSmmEoLctxiZY",
"5GNJqTPyNqANBkUVMN1LPPrxXnFouWXoe2wNSmmEoLctxiZY",
{
"grandpa": "5FA9nQDVg267DEd8m1ZypXLBnvN7SFxYwV7ndqSYGiN9TTpu",
"babe": "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY",
"im_online": "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY",
"para_validator": "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY",
"para_assignment": "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY",
"authority_discovery": "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY",
"beefy": "KW39r9CJjAVzmkf9zQ4YDb2hqfAVGdRqn53eRqyruqpxAP5YL"
}
],
[
"5HpG9w8EBLe5XCrbczpwq5TSXvedjrBGCwqxK1iQ7qUsSWFc",
"5HpG9w8EBLe5XCrbczpwq5TSXvedjrBGCwqxK1iQ7qUsSWFc",
{
"grandpa": "5GoNkf6WdbxCFnPdAnYYQyCjAKPJgLNxXwPjwTh6DGg6gN3E",
"babe": "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty",
"im_online": "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty",
"para_validator": "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty",
"para_assignment": "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty",
"authority_discovery": "5FHneW46xGXgs5mUiveU4sbTyGBzmstUspZC92UhjJM694ty",
"beefy": "KWByAN7WfZABWS5AoWqxriRmF5f2jnDqy3rB5pfHLGkY93ibN"
}
]
]
},
"grandpa": {
"authorities": []
},
"imOnline": {
"keys": []
},
"authorityDiscovery": {
"keys": []
},
"treasury": {},
"claims": {
"claims": [],
"vesting": []
},
"vesting": {
"vesting": []
},
"nisCounterpartBalances": {
"balances": []
},
"configuration": {
"config": {
"max_code_size": 3145728,
"max_head_data_size": 32768,
"max_upward_queue_count": 8,
"max_upward_queue_size": 1048576,
"max_upward_message_size": 51200,
"max_upward_message_num_per_candidate": 5,
"hrmp_max_message_num_per_candidate": 5,
"validation_upgrade_cooldown": 2,
"validation_upgrade_delay": 2,
"async_backing_params": {
"max_candidate_depth": 0,
"allowed_ancestry_len": 0
},
"max_pov_size": 5242880,
"max_downward_message_size": 1048576,
"hrmp_max_parachain_outbound_channels": 4,
"hrmp_sender_deposit": 0,
"hrmp_recipient_deposit": 0,
"hrmp_channel_max_capacity": 8,
"hrmp_channel_max_total_size": 8192,
"hrmp_max_parachain_inbound_channels": 4,
"hrmp_channel_max_message_size": 1048576,
"executor_params": [],
"code_retention_period": 1200,
"on_demand_cores": 0,
"on_demand_retries": 0,
"on_demand_queue_max_size": 10000,
"on_demand_target_queue_utilization": 250000000,
"on_demand_fee_variability": 30000000,
"on_demand_base_fee": 10000000,
"on_demand_ttl": 5,
"group_rotation_frequency": 20,
"paras_availability_period": 4,
"scheduling_lookahead": 1,
"max_validators_per_core": 1,
"max_validators": null,
"dispute_period": 6,
"dispute_post_conclusion_acceptance_period": 100,
"no_show_slots": 2,
"n_delay_tranches": 25,
"zeroth_delay_tranche_width": 0,
"needed_approvals": 2,
"relay_vrf_modulo_samples": 2,
"pvf_voting_ttl": 2,
"minimum_validation_upgrade_delay": 5,
"minimum_backing_votes": 2
}
},
"paras": {
"paras": []
},
"hrmp": {
"preopenHrmpChannels": []
},
"registrar": {
"nextFreeParaId": 2000
},
"xcmPallet": {
"safeXcmVersion": 3
},
"assignedSlots": {
"maxTemporarySlots": 0,
"maxPermanentSlots": 0,
"config": null
},
"sudo": {
"key": "5GrwvaEF5zXb26Fz9rcQpDWS57CtERHpNehXCPcNoHGKutQY"
}
}
}
}
@@ -0,0 +1,18 @@
[package]
name = "zombienet-prom-metrics-parser"
version.workspace = true
authors.workspace = true
edition.workspace = true
rust-version.workspace = true
publish = true
license.workspace = true
repository.workspace = true
description = "Prometheus metric parser, parse metrics provided by internal prometheus server"
keywords = ["zombienet", "prometheus"]
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
pest = { workspace = true }
pest_derive = { workspace = true }
thiserror = { workspace = true }
@@ -0,0 +1,47 @@
// Grammar taken from https://github.com/mitghi/promerge/ with
// some small modifications.
alpha = _{'a'..'z' | 'A'..'Z'}
alphanum = _{'a'..'z' | 'A'..'Z' | '0'..'9'}
number = @{
"-"?
~ ("0" | ASCII_NONZERO_DIGIT ~ ASCII_DIGIT*)
~ ("." ~ ASCII_DIGIT*)?
~ (^"e" ~ ("+" | "-")? ~ ASCII_DIGIT+)?
}
string = ${"\"" ~ inner ~ "\""}
inner = @{char*}
char = {
!("\"" | "\\") ~ ANY
| "\\" ~ ("\"" | "\\" | "/" | "b" | "f" | "n" | "r" | "t")
| "\\" ~ ("u" ~ ASCII_HEX_DIGIT{4})
}
whitespace_or_newline = _{(" "| "\n")*}
hash = _{"#"}
posInf = {"+Inf"}
negInf = {"-Inf"}
NaN = {"NaN"}
lbrace = _{"{"}
rbrace = _{"}"}
typelit = _{"TYPE"}
helplit = _{"HELP"}
comma = _{","}
countertype = {"counter"}
gaugetype = {"gauge"}
histogramtype = {"histogram"}
summarytype = {"summary"}
untyped = {"untyped"}
ident = {alphanum+}
key = @{ident ~ ("_" ~ ident)*}
label = {key ~ "=" ~ string}
labels = {label ~ (comma ~ label)*}
helpkey = {key}
helpval = {inner}
typekey = {key}
typeval = {countertype | gaugetype | histogramtype | summarytype | untyped}
commentval = @{((ASCII_DIGIT| ASCII_NONZERO_DIGIT | ASCII_BIN_DIGIT | ASCII_OCT_DIGIT | ASCII_HEX_DIGIT | ASCII_ALPHA_LOWER | ASCII_ALPHA_UPPER | ASCII_ALPHA | ASCII_ALPHANUMERIC | !"\n" ~ ANY ))*}
helpexpr = {hash ~ whitespace_or_newline ~ helplit ~ whitespace_or_newline ~ helpkey ~ whitespace_or_newline ~ commentval}
typexpr = {hash ~ whitespace_or_newline ~ typelit ~ whitespace_or_newline ~ typekey ~ whitespace_or_newline ~ typeval }
genericomment = {hash ~ whitespace_or_newline ~ commentval}
promstmt = {key ~ (lbrace ~ (labels)* ~ rbrace){0,1} ~ whitespace_or_newline ~ ((posInf | negInf | NaN | number) ~ whitespace_or_newline ){1,2}}
block = {((helpexpr | typexpr | genericomment)~ NEWLINE?)+ ~ (promstmt ~ NEWLINE?)+}
statement = {SOI ~ block+ ~ EOI}
@@ -0,0 +1,178 @@
use std::{collections::HashMap, num::ParseFloatError};
use pest::Parser;
use pest_derive::Parser;
/// An error at parsing level.
#[derive(thiserror::Error, Debug)]
pub enum ParserError {
#[error("error parsing input")]
ParseError(Box<pest::error::Error<Rule>>),
#[error("root node should be valid: {0}")]
ParseRootNodeError(String),
#[error("can't cast metric value as f64: {0}")]
CastValueError(#[from] ParseFloatError),
}
// This include forces recompiling this source file if the grammar file changes.
// Uncomment it when doing changes to the .pest file
const _GRAMMAR: &str = include_str!("grammar.pest");
#[derive(Parser)]
#[grammar = "grammar.pest"]
pub struct MetricsParser;
pub type MetricMap = HashMap<String, f64>;
pub fn parse(input: &str) -> Result<MetricMap, ParserError> {
let mut metric_map: MetricMap = Default::default();
let mut pairs = MetricsParser::parse(Rule::statement, input)
.map_err(|e| ParserError::ParseError(Box::new(e)))?;
let root = pairs
.next()
.ok_or(ParserError::ParseRootNodeError(pairs.as_str().to_string()))?;
for token in root.into_inner() {
if token.as_rule() == Rule::block {
let inner = token.into_inner();
for value in inner {
match value.as_rule() {
Rule::genericomment | Rule::typexpr | Rule::helpexpr => {
// don't need to collect comments/types/helpers blocks.
continue;
},
Rule::promstmt => {
let mut key: &str = "";
let mut labels: Vec<(&str, &str)> = Vec::new();
let mut val: f64 = 0_f64;
for v in value.clone().into_inner() {
match &v.as_rule() {
Rule::key => {
key = v.as_span().as_str();
},
Rule::NaN | Rule::posInf | Rule::negInf => {
// noop (not used in substrate metrics)
},
Rule::number => {
val = v.as_span().as_str().parse::<f64>()?;
},
Rule::labels => {
// SAFETY: use unwrap should be safe since we are just
// walking the parser struct and if are matching a label
// should have a key/vals
for p in v.into_inner() {
let mut inner = p.into_inner();
let key = inner.next().unwrap().as_span().as_str();
let value = inner
.next()
.unwrap()
.into_inner()
.next()
.unwrap()
.as_span()
.as_str();
labels.push((key, value));
}
},
_ => {
todo!("not implemented");
},
}
}
// we should store to make it compatible with zombienet v1:
// key_without_prefix
// key_without_prefix_and_without_chain
// key_with_prefix_with_chain
// key_with_prefix_and_without_chain
let key_with_out_prefix =
key.split('_').collect::<Vec<&str>>()[1..].join("_");
let (labels_without_chain, labels_with_chain) =
labels.iter().fold((vec![], vec![]), |mut acc, item| {
if item.0.eq("chain") {
acc.1.push(format!("{}=\"{}\"", item.0, item.1));
} else {
acc.0.push(format!("{}=\"{}\"", item.0, item.1));
acc.1.push(format!("{}=\"{}\"", item.0, item.1));
}
acc
});
let labels_with_chain_str = if labels_with_chain.is_empty() {
String::from("")
} else {
format!("{{{}}}", labels_with_chain.join(","))
};
let labels_without_chain_str = if labels_without_chain.is_empty() {
String::from("")
} else {
format!("{{{}}}", labels_without_chain.join(","))
};
metric_map.insert(format!("{key}{labels_without_chain_str}"), val);
metric_map.insert(
format!("{key_with_out_prefix}{labels_without_chain_str}"),
val,
);
metric_map.insert(format!("{key}{labels_with_chain_str}"), val);
metric_map
.insert(format!("{key_with_out_prefix}{labels_with_chain_str}"), val);
},
_ => {},
}
}
}
}
Ok(metric_map)
}
#[cfg(test)]
mod tests {
use std::fs;
use super::*;
#[test]
fn parse_metrics_works() {
let metrics_raw = fs::read_to_string("./testing/metrics.txt").unwrap();
let metrics = parse(&metrics_raw).unwrap();
// full key
assert_eq!(
metrics
.get("polkadot_node_is_active_validator{chain=\"rococo_local_testnet\"}")
.unwrap(),
&1_f64
);
// with prefix and no chain
assert_eq!(
metrics.get("polkadot_node_is_active_validator").unwrap(),
&1_f64
);
// no prefix with chain
assert_eq!(
metrics
.get("node_is_active_validator{chain=\"rococo_local_testnet\"}")
.unwrap(),
&1_f64
);
// no prefix without chain
assert_eq!(metrics.get("node_is_active_validator").unwrap(), &1_f64);
}
#[test]
fn parse_invalid_metrics_str_should_fail() {
let metrics_raw = r"
# HELP polkadot_node_is_active_validator Tracks if the validator is in the active set. Updates at session boundary.
# TYPE polkadot_node_is_active_validator gauge
polkadot_node_is_active_validator{chain=} 1
";
let metrics = parse(metrics_raw);
assert!(metrics.is_err());
assert!(matches!(metrics, Err(ParserError::ParseError(_))));
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,2 @@
/target
/Cargo.lock
+47
View File
@@ -0,0 +1,47 @@
[package]
name = "zombienet-provider"
version.workspace = true
authors.workspace = true
edition.workspace = true
rust-version.workspace = true
publish = true
license.workspace = true
repository.workspace = true
description = "Zombienet provider, implement the logic to run the nodes in the native provider"
keywords = ["zombienet", "provider", "native"]
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
async-trait = { workspace = true }
futures = { workspace = true }
serde = { workspace = true, features = ["derive", "rc"] }
serde_json = { workspace = true }
serde_yaml = { workspace = true }
tokio = { workspace = true, features = [
"process",
"macros",
"fs",
"time",
"rt",
] }
tokio-util = { workspace = true, features = ["compat"] }
thiserror = { workspace = true }
anyhow = { workspace = true }
uuid = { workspace = true, features = ["v4"] }
nix = { workspace = true, features = ["signal"] }
kube = { workspace = true, features = ["ws", "runtime"] }
k8s-openapi = { workspace = true, features = ["v1_27"] }
tar = { workspace = true }
sha2 = { workspace = true }
hex = { workspace = true }
tracing = { workspace = true }
reqwest = { workspace = true }
regex = { workspace = true }
url = { workspace = true }
flate2 = { workspace = true }
erased-serde = { workspace = true }
# Zomebienet deps
support = { workspace = true }
configuration = { workspace = true }
@@ -0,0 +1,6 @@
mod client;
mod namespace;
mod node;
mod provider;
pub use provider::DockerProvider;
@@ -0,0 +1,596 @@
use std::{collections::HashMap, path::Path, process::Stdio};
use anyhow::anyhow;
use futures::future::try_join_all;
use serde::{Deserialize, Deserializer};
use tokio::process::Command;
use tracing::{info, trace};
use crate::types::{ExecutionResult, Port};
#[derive(thiserror::Error, Debug)]
#[error(transparent)]
pub struct Error(#[from] anyhow::Error);
pub type Result<T> = core::result::Result<T, Error>;
#[derive(Clone)]
pub struct DockerClient {
using_podman: bool,
}
#[derive(Debug)]
pub struct ContainerRunOptions {
image: String,
command: Vec<String>,
env: Option<Vec<(String, String)>>,
volume_mounts: Option<HashMap<String, String>>,
name: Option<String>,
entrypoint: Option<String>,
port_mapping: HashMap<Port, Port>,
rm: bool,
detach: bool,
}
enum Container {
Docker(DockerContainer),
Podman(PodmanContainer),
}
// TODO: we may don't need this
#[allow(dead_code)]
#[derive(Deserialize, Debug)]
struct DockerContainer {
#[serde(alias = "Names", deserialize_with = "deserialize_list")]
names: Vec<String>,
#[serde(alias = "Ports", deserialize_with = "deserialize_list")]
ports: Vec<String>,
#[serde(alias = "State")]
state: String,
}
// TODO: we may don't need this
#[allow(dead_code)]
#[derive(Deserialize, Debug)]
struct PodmanPort {
host_ip: String,
container_port: u16,
host_port: u16,
range: u16,
protocol: String,
}
// TODO: we may don't need this
#[allow(dead_code)]
#[derive(Deserialize, Debug)]
struct PodmanContainer {
#[serde(alias = "Id")]
id: String,
#[serde(alias = "Image")]
image: String,
#[serde(alias = "Mounts")]
mounts: Vec<String>,
#[serde(alias = "Names")]
names: Vec<String>,
#[serde(alias = "Ports", deserialize_with = "deserialize_null_as_default")]
ports: Vec<PodmanPort>,
#[serde(alias = "State")]
state: String,
}
fn deserialize_list<'de, D>(deserializer: D) -> std::result::Result<Vec<String>, D::Error>
where
D: Deserializer<'de>,
{
let str_sequence = String::deserialize(deserializer)?;
Ok(str_sequence
.split(',')
.filter(|item| !item.is_empty())
.map(|item| item.to_owned())
.collect())
}
fn deserialize_null_as_default<'de, D, T>(deserializer: D) -> std::result::Result<T, D::Error>
where
T: Default + Deserialize<'de>,
D: Deserializer<'de>,
{
let opt = Option::deserialize(deserializer)?;
Ok(opt.unwrap_or_default())
}
impl ContainerRunOptions {
pub fn new<S>(image: &str, command: Vec<S>) -> Self
where
S: Into<String> + std::fmt::Debug + Send + Clone,
{
ContainerRunOptions {
image: image.to_string(),
command: command
.clone()
.into_iter()
.map(|s| s.into())
.collect::<Vec<_>>(),
env: None,
volume_mounts: None,
name: None,
entrypoint: None,
port_mapping: HashMap::default(),
rm: false,
detach: true, // add -d flag by default
}
}
pub fn env<S>(mut self, env: Vec<(S, S)>) -> Self
where
S: Into<String> + std::fmt::Debug + Send + Clone,
{
self.env = Some(
env.into_iter()
.map(|(name, value)| (name.into(), value.into()))
.collect(),
);
self
}
pub fn volume_mounts<S>(mut self, volume_mounts: HashMap<S, S>) -> Self
where
S: Into<String> + std::fmt::Debug + Send + Clone,
{
self.volume_mounts = Some(
volume_mounts
.into_iter()
.map(|(source, target)| (source.into(), target.into()))
.collect(),
);
self
}
pub fn name<S>(mut self, name: S) -> Self
where
S: Into<String> + std::fmt::Debug + Send + Clone,
{
self.name = Some(name.into());
self
}
pub fn entrypoint<S>(mut self, entrypoint: S) -> Self
where
S: Into<String> + std::fmt::Debug + Send + Clone,
{
self.entrypoint = Some(entrypoint.into());
self
}
pub fn port_mapping(mut self, port_mapping: &HashMap<Port, Port>) -> Self {
self.port_mapping.clone_from(port_mapping);
self
}
pub fn rm(mut self) -> Self {
self.rm = true;
self
}
pub fn detach(mut self, choice: bool) -> Self {
self.detach = choice;
self
}
}
impl DockerClient {
pub async fn new() -> Result<Self> {
let using_podman = Self::is_using_podman().await?;
Ok(DockerClient { using_podman })
}
pub fn client_binary(&self) -> String {
String::from(if self.using_podman {
"podman"
} else {
"docker"
})
}
async fn is_using_podman() -> Result<bool> {
if let Ok(output) = tokio::process::Command::new("docker")
.arg("version")
.output()
.await
{
// detect whether we're actually running podman with docker emulation
return Ok(String::from_utf8_lossy(&output.stdout)
.to_lowercase()
.contains("podman"));
}
tokio::process::Command::new("podman")
.arg("--version")
.output()
.await
.map_err(|err| anyhow!("Failed to detect container engine: {err}"))?;
Ok(true)
}
}
impl DockerClient {
fn client_command(&self) -> tokio::process::Command {
tokio::process::Command::new(self.client_binary())
}
pub async fn create_volume(&self, name: &str) -> Result<()> {
let result = self
.client_command()
.args(["volume", "create", name])
.output()
.await
.map_err(|err| anyhow!("Failed to create volume '{name}': {err}"))?;
if !result.status.success() {
return Err(anyhow!(
"Failed to create volume '{name}': {}",
String::from_utf8_lossy(&result.stderr)
)
.into());
}
Ok(())
}
pub async fn container_run(&self, options: ContainerRunOptions) -> Result<String> {
let mut cmd = self.client_command();
cmd.args(["run", "--platform", "linux/amd64"]);
if options.detach {
cmd.arg("-d");
}
Self::apply_cmd_options(&mut cmd, &options);
trace!("cmd: {:?}", cmd);
let result = cmd.output().await.map_err(|err| {
anyhow!(
"Failed to run container with image '{image}' and command '{command}': {err}",
image = options.image,
command = options.command.join(" "),
)
})?;
if !result.status.success() {
return Err(anyhow!(
"Failed to run container with image '{image}' and command '{command}': {err}",
image = options.image,
command = options.command.join(" "),
err = String::from_utf8_lossy(&result.stderr)
)
.into());
}
Ok(String::from_utf8_lossy(&result.stdout).to_string())
}
pub async fn container_create(&self, options: ContainerRunOptions) -> Result<String> {
let mut cmd = self.client_command();
cmd.args(["container", "create"]);
Self::apply_cmd_options(&mut cmd, &options);
trace!("cmd: {:?}", cmd);
let result = cmd.output().await.map_err(|err| {
anyhow!(
"Failed to run container with image '{image}' and command '{command}': {err}",
image = options.image,
command = options.command.join(" "),
)
})?;
if !result.status.success() {
return Err(anyhow!(
"Failed to run container with image '{image}' and command '{command}': {err}",
image = options.image,
command = options.command.join(" "),
err = String::from_utf8_lossy(&result.stderr)
)
.into());
}
Ok(String::from_utf8_lossy(&result.stdout).to_string())
}
pub async fn container_exec<S>(
&self,
name: &str,
command: Vec<S>,
env: Option<Vec<(S, S)>>,
as_user: Option<S>,
) -> Result<ExecutionResult>
where
S: Into<String> + std::fmt::Debug + Send + Clone,
{
let mut cmd = self.client_command();
cmd.arg("exec");
if let Some(env) = env {
for env_var in env {
cmd.args(["-e", &format!("{}={}", env_var.0.into(), env_var.1.into())]);
}
}
if let Some(user) = as_user {
cmd.args(["-u", user.into().as_ref()]);
}
cmd.arg(name);
cmd.args(
command
.clone()
.into_iter()
.map(|s| <S as Into<String>>::into(s)),
);
trace!("cmd is : {:?}", cmd);
let result = cmd.output().await.map_err(|err| {
anyhow!(
"Failed to exec '{}' on '{}': {err}",
command
.into_iter()
.map(|s| <S as Into<String>>::into(s))
.collect::<Vec<_>>()
.join(" "),
name,
)
})?;
if !result.status.success() {
return Ok(Err((
result.status,
String::from_utf8_lossy(&result.stderr).to_string(),
)));
}
Ok(Ok(String::from_utf8_lossy(&result.stdout).to_string()))
}
pub async fn container_cp(
&self,
name: &str,
local_path: &Path,
remote_path: &Path,
) -> Result<()> {
let result = self
.client_command()
.args([
"cp",
local_path.to_string_lossy().as_ref(),
&format!("{name}:{}", remote_path.to_string_lossy().as_ref()),
])
.output()
.await
.map_err(|err| {
anyhow!(
"Failed copy file '{file}' to container '{name}': {err}",
file = local_path.to_string_lossy(),
)
})?;
if !result.status.success() {
return Err(anyhow!(
"Failed to copy file '{file}' to container '{name}': {err}",
file = local_path.to_string_lossy(),
err = String::from_utf8_lossy(&result.stderr)
)
.into());
}
Ok(())
}
pub async fn container_rm(&self, name: &str) -> Result<()> {
let result = self
.client_command()
.args(["rm", "--force", "--volumes", name])
.output()
.await
.map_err(|err| anyhow!("Failed do remove container '{name}: {err}"))?;
if !result.status.success() {
return Err(anyhow!(
"Failed to remove container '{name}': {err}",
err = String::from_utf8_lossy(&result.stderr)
)
.into());
}
Ok(())
}
pub async fn namespaced_containers_rm(&self, namespace: &str) -> Result<()> {
let container_names: Vec<String> = self
.get_containers()
.await?
.into_iter()
.filter_map(|container| match container {
Container::Docker(container) => {
if let Some(name) = container.names.first() {
if name.starts_with(namespace) {
return Some(name.to_string());
}
}
None
},
Container::Podman(container) => {
if let Some(name) = container.names.first() {
if name.starts_with(namespace) {
return Some(name.to_string());
}
}
None
},
})
.collect();
info!("{:?}", container_names);
let futures = container_names
.iter()
.map(|name| self.container_rm(name))
.collect::<Vec<_>>();
try_join_all(futures).await?;
Ok(())
}
pub async fn container_ip(&self, container_name: &str) -> Result<String> {
let ip = if self.using_podman {
"127.0.0.1".into()
} else {
let mut cmd = tokio::process::Command::new("docker");
cmd.args(vec![
"inspect",
"-f",
"{{ .NetworkSettings.IPAddress }}",
container_name,
]);
trace!("CMD: {cmd:?}");
let res = cmd
.output()
.await
.map_err(|err| anyhow!("Failed to get docker container ip, output: {err}"))?;
String::from_utf8(res.stdout)
.map_err(|err| anyhow!("Failed to get docker container ip, output: {err}"))?
.trim()
.into()
};
trace!("IP: {ip}");
Ok(ip)
}
async fn get_containers(&self) -> Result<Vec<Container>> {
let containers = if self.using_podman {
self.get_podman_containers()
.await?
.into_iter()
.map(Container::Podman)
.collect()
} else {
self.get_docker_containers()
.await?
.into_iter()
.map(Container::Docker)
.collect()
};
Ok(containers)
}
async fn get_podman_containers(&self) -> Result<Vec<PodmanContainer>> {
let res = tokio::process::Command::new("podman")
.args(vec!["ps", "--all", "--no-trunc", "--format", "json"])
.output()
.await
.map_err(|err| anyhow!("Failed to get podman containers output: {err}"))?;
let stdout = String::from_utf8_lossy(&res.stdout);
let containers = serde_json::from_str(&stdout)
.map_err(|err| anyhow!("Failed to parse podman containers output: {err}"))?;
Ok(containers)
}
async fn get_docker_containers(&self) -> Result<Vec<DockerContainer>> {
let res = tokio::process::Command::new("docker")
.args(vec!["ps", "--all", "--no-trunc", "--format", "json"])
.output()
.await
.unwrap();
let stdout = String::from_utf8_lossy(&res.stdout);
let mut containers = vec![];
for line in stdout.lines() {
containers.push(
serde_json::from_str::<DockerContainer>(line)
.map_err(|err| anyhow!("Failed to parse docker container output: {err}"))?,
);
}
Ok(containers)
}
pub(crate) async fn container_logs(&self, container_name: &str) -> Result<String> {
let output = Command::new("sh")
.arg("-c")
.arg(format!("docker logs -t '{container_name}' 2>&1"))
.stdout(Stdio::piped())
.output()
.await
.map_err(|err| {
anyhow!(
"Failed to spawn docker logs command for container '{container_name}': {err}"
)
})?;
let logs = String::from_utf8_lossy(&output.stdout).to_string();
if !output.status.success() {
// stderr was redirected to stdout, so logs should contain the error message if any
return Err(anyhow!(
"Failed to get logs for container '{name}': {logs}",
name = container_name,
logs = &logs
)
.into());
}
Ok(logs)
}
fn apply_cmd_options(cmd: &mut Command, options: &ContainerRunOptions) {
if options.rm {
cmd.arg("--rm");
}
if let Some(entrypoint) = options.entrypoint.as_ref() {
cmd.args(["--entrypoint", entrypoint]);
}
if let Some(volume_mounts) = options.volume_mounts.as_ref() {
for (source, target) in volume_mounts {
cmd.args(["-v", &format!("{source}:{target}")]);
}
}
if let Some(env) = options.env.as_ref() {
for env_var in env {
cmd.args(["-e", &format!("{}={}", env_var.0, env_var.1)]);
}
}
// add published ports
for (container_port, host_port) in options.port_mapping.iter() {
cmd.args(["-p", &format!("{host_port}:{container_port}")]);
}
if let Some(name) = options.name.as_ref() {
cmd.args(["--name", name]);
}
cmd.arg(&options.image);
for arg in &options.command {
cmd.arg(arg);
}
}
}
@@ -0,0 +1,494 @@
use std::{
collections::HashMap,
path::{Path, PathBuf},
sync::{Arc, Weak},
thread,
};
use async_trait::async_trait;
use support::{constants::THIS_IS_A_BUG, fs::FileSystem};
use tokio::sync::{Mutex, RwLock};
use tracing::{debug, trace, warn};
use uuid::Uuid;
use super::{
client::{ContainerRunOptions, DockerClient},
node::DockerNode,
DockerProvider,
};
use crate::{
constants::NAMESPACE_PREFIX,
docker::{
node::{DeserializableDockerNodeOptions, DockerNodeOptions},
provider,
},
shared::helpers::extract_execution_result,
types::{
GenerateFileCommand, GenerateFilesOptions, ProviderCapabilities, RunCommandOptions,
SpawnNodeOptions,
},
DynNode, ProviderError, ProviderNamespace, ProviderNode,
};
pub struct DockerNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
weak: Weak<DockerNamespace<FS>>,
#[allow(dead_code)]
provider: Weak<DockerProvider<FS>>,
name: String,
base_dir: PathBuf,
capabilities: ProviderCapabilities,
docker_client: DockerClient,
filesystem: FS,
delete_on_drop: Arc<Mutex<bool>>,
pub(super) nodes: RwLock<HashMap<String, Arc<DockerNode<FS>>>>,
}
impl<FS> DockerNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) async fn new(
provider: &Weak<DockerProvider<FS>>,
tmp_dir: &PathBuf,
capabilities: &ProviderCapabilities,
docker_client: &DockerClient,
filesystem: &FS,
custom_base_dir: Option<&Path>,
) -> Result<Arc<Self>, ProviderError> {
let name = format!("{}{}", NAMESPACE_PREFIX, Uuid::new_v4());
let base_dir = if let Some(custom_base_dir) = custom_base_dir {
if !filesystem.exists(custom_base_dir).await {
filesystem.create_dir(custom_base_dir).await?;
} else {
warn!(
"⚠️ Using and existing directory {} as base dir",
custom_base_dir.to_string_lossy()
);
}
PathBuf::from(custom_base_dir)
} else {
let base_dir = PathBuf::from_iter([tmp_dir, &PathBuf::from(&name)]);
filesystem.create_dir(&base_dir).await?;
base_dir
};
let namespace = Arc::new_cyclic(|weak| DockerNamespace {
weak: weak.clone(),
provider: provider.clone(),
name,
base_dir,
capabilities: capabilities.clone(),
filesystem: filesystem.clone(),
docker_client: docker_client.clone(),
nodes: RwLock::new(HashMap::new()),
delete_on_drop: Arc::new(Mutex::new(true)),
});
namespace.initialize().await?;
Ok(namespace)
}
pub(super) async fn attach_to_live(
provider: &Weak<DockerProvider<FS>>,
capabilities: &ProviderCapabilities,
docker_client: &DockerClient,
filesystem: &FS,
custom_base_dir: &Path,
name: &str,
) -> Result<Arc<Self>, ProviderError> {
let base_dir = custom_base_dir.to_path_buf();
let namespace = Arc::new_cyclic(|weak| DockerNamespace {
weak: weak.clone(),
provider: provider.clone(),
name: name.to_owned(),
base_dir,
capabilities: capabilities.clone(),
filesystem: filesystem.clone(),
docker_client: docker_client.clone(),
nodes: RwLock::new(HashMap::new()),
delete_on_drop: Arc::new(Mutex::new(false)),
});
Ok(namespace)
}
async fn initialize(&self) -> Result<(), ProviderError> {
// let ns_scripts_shared = PathBuf::from_iter([&self.base_dir, &PathBuf::from("shared-scripts")]);
// self.filesystem.create_dir(&ns_scripts_shared).await?;
self.initialize_zombie_scripts_volume().await?;
self.initialize_helper_binaries_volume().await?;
Ok(())
}
async fn initialize_zombie_scripts_volume(&self) -> Result<(), ProviderError> {
let local_zombie_wrapper_path =
PathBuf::from_iter([&self.base_dir, &PathBuf::from("zombie-wrapper.sh")]);
self.filesystem
.write(
&local_zombie_wrapper_path,
include_str!("../shared/scripts/zombie-wrapper.sh"),
)
.await?;
let local_helper_binaries_downloader_path = PathBuf::from_iter([
&self.base_dir,
&PathBuf::from("helper-binaries-downloader.sh"),
]);
self.filesystem
.write(
&local_helper_binaries_downloader_path,
include_str!("../shared/scripts/helper-binaries-downloader.sh"),
)
.await?;
let zombie_wrapper_volume_name = format!("{}-zombie-wrapper", self.name);
let zombie_wrapper_container_name = format!("{}-scripts", self.name);
self.docker_client
.create_volume(&zombie_wrapper_volume_name)
.await
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
self.docker_client
.container_create(
ContainerRunOptions::new("alpine:latest", vec!["tail", "-f", "/dev/null"])
.volume_mounts(HashMap::from([(
zombie_wrapper_volume_name.as_str(),
"/scripts",
)]))
.name(&zombie_wrapper_container_name)
.rm(),
)
.await
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
// copy the scripts
self.docker_client
.container_cp(
&zombie_wrapper_container_name,
&local_zombie_wrapper_path,
&PathBuf::from("/scripts/zombie-wrapper.sh"),
)
.await
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
self.docker_client
.container_cp(
&zombie_wrapper_container_name,
&local_helper_binaries_downloader_path,
&PathBuf::from("/scripts/helper-binaries-downloader.sh"),
)
.await
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
// set permissions for rwx on whole volume recursively
self.docker_client
.container_run(
ContainerRunOptions::new("alpine:latest", vec!["chmod", "-R", "777", "/scripts"])
.volume_mounts(HashMap::from([(
zombie_wrapper_volume_name.as_ref(),
"/scripts",
)]))
.rm(),
)
.await
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
Ok(())
}
async fn initialize_helper_binaries_volume(&self) -> Result<(), ProviderError> {
let helper_binaries_volume_name = format!("{}-helper-binaries", self.name);
let zombie_wrapper_volume_name = format!("{}-zombie-wrapper", self.name);
self.docker_client
.create_volume(&helper_binaries_volume_name)
.await
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
// download binaries to volume
self.docker_client
.container_run(
ContainerRunOptions::new(
"alpine:latest",
vec!["ash", "/scripts/helper-binaries-downloader.sh"],
)
.volume_mounts(HashMap::from([
(
helper_binaries_volume_name.as_str(),
"/helpers",
),
(
zombie_wrapper_volume_name.as_ref(),
"/scripts",
)
]))
// wait until complete
.detach(false)
.rm(),
)
.await
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
// set permissions for rwx on whole volume recursively
self.docker_client
.container_run(
ContainerRunOptions::new("alpine:latest", vec!["chmod", "-R", "777", "/helpers"])
.volume_mounts(HashMap::from([(
helper_binaries_volume_name.as_ref(),
"/helpers",
)]))
.rm(),
)
.await
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
Ok(())
}
pub async fn set_delete_on_drop(&self, delete_on_drop: bool) {
*self.delete_on_drop.lock().await = delete_on_drop;
}
pub async fn delete_on_drop(&self) -> bool {
if let Ok(delete_on_drop) = self.delete_on_drop.try_lock() {
*delete_on_drop
} else {
// if we can't lock just remove the ns
true
}
}
}
#[async_trait]
impl<FS> ProviderNamespace for DockerNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
fn name(&self) -> &str {
&self.name
}
fn base_dir(&self) -> &PathBuf {
&self.base_dir
}
fn capabilities(&self) -> &ProviderCapabilities {
&self.capabilities
}
fn provider_name(&self) -> &str {
provider::PROVIDER_NAME
}
async fn detach(&self) {
self.set_delete_on_drop(false).await;
}
async fn is_detached(&self) -> bool {
self.delete_on_drop().await
}
async fn nodes(&self) -> HashMap<String, DynNode> {
self.nodes
.read()
.await
.iter()
.map(|(name, node)| (name.clone(), node.clone() as DynNode))
.collect()
}
async fn get_node_available_args(
&self,
(command, image): (String, Option<String>),
) -> Result<String, ProviderError> {
let node_image = image.expect(&format!("image should be present when getting node available args with docker provider {THIS_IS_A_BUG}"));
let temp_node = self
.spawn_node(
&SpawnNodeOptions::new(format!("temp-{}", Uuid::new_v4()), "cat".to_string())
.image(node_image.clone()),
)
.await?;
let available_args_output = temp_node
.run_command(RunCommandOptions::new(command.clone()).args(vec!["--help"]))
.await?
.map_err(|(_exit, status)| {
ProviderError::NodeAvailableArgsError(node_image, command, status)
})?;
temp_node.destroy().await?;
Ok(available_args_output)
}
async fn spawn_node(&self, options: &SpawnNodeOptions) -> Result<DynNode, ProviderError> {
debug!("spawn option {:?}", options);
let node = DockerNode::new(DockerNodeOptions {
namespace: &self.weak,
namespace_base_dir: &self.base_dir,
name: &options.name,
image: options.image.as_ref(),
program: &options.program,
args: &options.args,
env: &options.env,
startup_files: &options.injected_files,
db_snapshot: options.db_snapshot.as_ref(),
docker_client: &self.docker_client,
container_name: format!("{}-{}", self.name, options.name),
filesystem: &self.filesystem,
port_mapping: options.port_mapping.as_ref().unwrap_or(&HashMap::default()),
})
.await?;
self.nodes
.write()
.await
.insert(node.name().to_string(), node.clone());
Ok(node)
}
async fn spawn_node_from_json(
&self,
json_value: &serde_json::Value,
) -> Result<DynNode, ProviderError> {
let deserializable: DeserializableDockerNodeOptions =
serde_json::from_value(json_value.clone())?;
let options = DockerNodeOptions::from_deserializable(
&deserializable,
&self.weak,
&self.base_dir,
&self.docker_client,
&self.filesystem,
);
let node = DockerNode::attach_to_live(options).await?;
self.nodes
.write()
.await
.insert(node.name().to_string(), node.clone());
Ok(node)
}
async fn generate_files(&self, options: GenerateFilesOptions) -> Result<(), ProviderError> {
debug!("generate files options {options:#?}");
let node_name = options
.temp_name
.unwrap_or_else(|| format!("temp-{}", Uuid::new_v4()));
let node_image = options.image.expect(&format!(
"image should be present when generating files with docker provider {THIS_IS_A_BUG}"
));
// run dummy command in a new container
let temp_node = self
.spawn_node(
&SpawnNodeOptions::new(node_name, "cat".to_string())
.injected_files(options.injected_files)
.image(node_image),
)
.await?;
for GenerateFileCommand {
program,
args,
env,
local_output_path,
} in options.commands
{
let local_output_full_path = format!(
"{}{}{}",
self.base_dir.to_string_lossy(),
if local_output_path.starts_with("/") {
""
} else {
"/"
},
local_output_path.to_string_lossy()
);
let contents = extract_execution_result(
&temp_node,
RunCommandOptions { program, args, env },
options.expected_path.as_ref(),
)
.await?;
self.filesystem
.write(local_output_full_path, contents)
.await
.map_err(|err| ProviderError::FileGenerationFailed(err.into()))?;
}
temp_node.destroy().await
}
async fn static_setup(&self) -> Result<(), ProviderError> {
todo!()
}
async fn destroy(&self) -> Result<(), ProviderError> {
let _ = self
.docker_client
.namespaced_containers_rm(&self.name)
.await
.map_err(|err| ProviderError::DeleteNamespaceFailed(self.name.clone(), err.into()))?;
if let Some(provider) = self.provider.upgrade() {
provider.namespaces.write().await.remove(&self.name);
}
Ok(())
}
}
impl<FS> Drop for DockerNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
fn drop(&mut self) {
let ns_name = self.name.clone();
if let Ok(delete_on_drop) = self.delete_on_drop.try_lock() {
if *delete_on_drop {
let client = self.docker_client.clone();
let provider = self.provider.upgrade();
let handler = thread::spawn(move || {
let rt = tokio::runtime::Runtime::new().unwrap();
rt.block_on(async move {
trace!("🧟 deleting ns {ns_name} from cluster");
let _ = client.namespaced_containers_rm(&ns_name).await;
trace!("✅ deleted");
});
});
if handler.join().is_ok() {
if let Some(provider) = provider {
if let Ok(mut p) = provider.namespaces.try_write() {
p.remove(&self.name);
} else {
warn!(
"⚠️ Can not acquire write lock to the provider, ns {} not removed",
self.name
);
}
}
}
} else {
trace!("⚠️ leaking ns {ns_name} in cluster");
}
};
}
}
@@ -0,0 +1,659 @@
use std::{
collections::HashMap,
net::IpAddr,
path::{Component, Path, PathBuf},
sync::{Arc, Weak},
time::Duration,
};
use anyhow::anyhow;
use async_trait::async_trait;
use configuration::types::AssetLocation;
use futures::future::try_join_all;
use serde::{Deserialize, Serialize};
use support::{constants::THIS_IS_A_BUG, fs::FileSystem};
use tokio::{time::sleep, try_join};
use tracing::debug;
use super::{
client::{ContainerRunOptions, DockerClient},
namespace::DockerNamespace,
};
use crate::{
constants::{NODE_CONFIG_DIR, NODE_DATA_DIR, NODE_RELAY_DATA_DIR, NODE_SCRIPTS_DIR},
docker,
types::{ExecutionResult, Port, RunCommandOptions, RunScriptOptions, TransferedFile},
ProviderError, ProviderNamespace, ProviderNode,
};
pub(super) struct DockerNodeOptions<'a, FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) namespace: &'a Weak<DockerNamespace<FS>>,
pub(super) namespace_base_dir: &'a PathBuf,
pub(super) name: &'a str,
pub(super) image: Option<&'a String>,
pub(super) program: &'a str,
pub(super) args: &'a [String],
pub(super) env: &'a [(String, String)],
pub(super) startup_files: &'a [TransferedFile],
pub(super) db_snapshot: Option<&'a AssetLocation>,
pub(super) docker_client: &'a DockerClient,
pub(super) container_name: String,
pub(super) filesystem: &'a FS,
pub(super) port_mapping: &'a HashMap<Port, Port>,
}
impl<'a, FS> DockerNodeOptions<'a, FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub fn from_deserializable(
deserializable: &'a DeserializableDockerNodeOptions,
namespace: &'a Weak<DockerNamespace<FS>>,
namespace_base_dir: &'a PathBuf,
docker_client: &'a DockerClient,
filesystem: &'a FS,
) -> Self {
DockerNodeOptions {
namespace,
namespace_base_dir,
name: &deserializable.name,
image: deserializable.image.as_ref(),
program: &deserializable.program,
args: &deserializable.args,
env: &deserializable.env,
startup_files: &[],
db_snapshot: None,
docker_client,
container_name: deserializable.container_name.clone(),
filesystem,
port_mapping: &deserializable.port_mapping,
}
}
}
#[derive(Deserialize)]
pub(super) struct DeserializableDockerNodeOptions {
pub(super) name: String,
pub(super) image: Option<String>,
pub(super) program: String,
pub(super) args: Vec<String>,
pub(super) env: Vec<(String, String)>,
pub(super) container_name: String,
pub(super) port_mapping: HashMap<Port, Port>,
}
#[derive(Serialize)]
pub struct DockerNode<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
#[serde(skip)]
namespace: Weak<DockerNamespace<FS>>,
name: String,
image: String,
program: String,
args: Vec<String>,
env: Vec<(String, String)>,
base_dir: PathBuf,
config_dir: PathBuf,
data_dir: PathBuf,
relay_data_dir: PathBuf,
scripts_dir: PathBuf,
log_path: PathBuf,
#[serde(skip)]
docker_client: DockerClient,
container_name: String,
port_mapping: HashMap<Port, Port>,
#[allow(dead_code)]
#[serde(skip)]
filesystem: FS,
provider_tag: String,
}
impl<FS> DockerNode<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) async fn new(
options: DockerNodeOptions<'_, FS>,
) -> Result<Arc<Self>, ProviderError> {
let image = options.image.ok_or_else(|| {
ProviderError::MissingNodeInfo(options.name.to_string(), "missing image".to_string())
})?;
let filesystem = options.filesystem.clone();
let base_dir =
PathBuf::from_iter([options.namespace_base_dir, &PathBuf::from(options.name)]);
filesystem.create_dir_all(&base_dir).await?;
let base_dir_raw = base_dir.to_string_lossy();
let config_dir = PathBuf::from(format!("{base_dir_raw}{NODE_CONFIG_DIR}"));
let data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_DATA_DIR}"));
let relay_data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_RELAY_DATA_DIR}"));
let scripts_dir = PathBuf::from(format!("{base_dir_raw}{NODE_SCRIPTS_DIR}"));
let log_path = base_dir.join("node.log");
try_join!(
filesystem.create_dir_all(&config_dir),
filesystem.create_dir_all(&data_dir),
filesystem.create_dir_all(&relay_data_dir),
filesystem.create_dir_all(&scripts_dir),
)?;
let node = Arc::new(DockerNode {
namespace: options.namespace.clone(),
name: options.name.to_string(),
image: image.to_string(),
program: options.program.to_string(),
args: options.args.to_vec(),
env: options.env.to_vec(),
base_dir,
config_dir,
data_dir,
relay_data_dir,
scripts_dir,
log_path,
filesystem: filesystem.clone(),
docker_client: options.docker_client.clone(),
container_name: options.container_name,
port_mapping: options.port_mapping.clone(),
provider_tag: docker::provider::PROVIDER_NAME.to_string(),
});
node.initialize_docker().await?;
if let Some(db_snap) = options.db_snapshot {
node.initialize_db_snapshot(db_snap).await?;
}
node.initialize_startup_files(options.startup_files).await?;
node.start().await?;
Ok(node)
}
pub(super) async fn attach_to_live(
options: DockerNodeOptions<'_, FS>,
) -> Result<Arc<Self>, ProviderError> {
let image = options.image.ok_or_else(|| {
ProviderError::MissingNodeInfo(options.name.to_string(), "missing image".to_string())
})?;
let filesystem = options.filesystem.clone();
let base_dir =
PathBuf::from_iter([options.namespace_base_dir, &PathBuf::from(options.name)]);
filesystem.create_dir_all(&base_dir).await?;
let base_dir_raw = base_dir.to_string_lossy();
let config_dir = PathBuf::from(format!("{base_dir_raw}{NODE_CONFIG_DIR}"));
let data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_DATA_DIR}"));
let relay_data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_RELAY_DATA_DIR}"));
let scripts_dir = PathBuf::from(format!("{base_dir_raw}{NODE_SCRIPTS_DIR}"));
let log_path = base_dir.join("node.log");
let node = Arc::new(DockerNode {
namespace: options.namespace.clone(),
name: options.name.to_string(),
image: image.to_string(),
program: options.program.to_string(),
args: options.args.to_vec(),
env: options.env.to_vec(),
base_dir,
config_dir,
data_dir,
relay_data_dir,
scripts_dir,
log_path,
filesystem: filesystem.clone(),
docker_client: options.docker_client.clone(),
container_name: options.container_name,
port_mapping: options.port_mapping.clone(),
provider_tag: docker::provider::PROVIDER_NAME.to_string(),
});
Ok(node)
}
async fn initialize_docker(&self) -> Result<(), ProviderError> {
let command = [vec![self.program.to_string()], self.args.to_vec()].concat();
self.docker_client
.container_run(
ContainerRunOptions::new(&self.image, command)
.name(&self.container_name)
.env(self.env.clone())
.volume_mounts(HashMap::from([
(
format!("{}-zombie-wrapper", self.namespace_name(),),
"/scripts".to_string(),
),
(
format!("{}-helper-binaries", self.namespace_name()),
"/helpers".to_string(),
),
(
self.config_dir.to_string_lossy().into_owned(),
"/cfg".to_string(),
),
(
self.data_dir.to_string_lossy().into_owned(),
"/data".to_string(),
),
(
self.relay_data_dir.to_string_lossy().into_owned(),
"/relay-data".to_string(),
),
]))
.entrypoint("/scripts/zombie-wrapper.sh")
.port_mapping(&self.port_mapping),
)
.await
.map_err(|err| ProviderError::NodeSpawningFailed(self.name.clone(), err.into()))?;
// change dirs permission
let _ = self
.docker_client
.container_exec(
&self.container_name,
["chmod", "777", "/cfg", "/data", "/relay-data"].into(),
None,
Some("root"),
)
.await
.map_err(|err| ProviderError::NodeSpawningFailed(self.name.clone(), err.into()))?;
Ok(())
}
async fn initialize_db_snapshot(
&self,
_db_snapshot: &AssetLocation,
) -> Result<(), ProviderError> {
todo!()
// trace!("snap: {db_snapshot}");
// let url_of_snap = match db_snapshot {
// AssetLocation::Url(location) => location.clone(),
// AssetLocation::FilePath(filepath) => self.upload_to_fileserver(filepath).await?,
// };
// // we need to get the snapshot from a public access
// // and extract to /data
// let opts = RunCommandOptions::new("mkdir").args([
// "-p",
// "/data/",
// "&&",
// "mkdir",
// "-p",
// "/relay-data/",
// "&&",
// // Use our version of curl
// "/cfg/curl",
// url_of_snap.as_ref(),
// "--output",
// "/data/db.tgz",
// "&&",
// "cd",
// "/",
// "&&",
// "tar",
// "--skip-old-files",
// "-xzvf",
// "/data/db.tgz",
// ]);
// trace!("cmd opts: {:#?}", opts);
// let _ = self.run_command(opts).await?;
// Ok(())
}
async fn initialize_startup_files(
&self,
startup_files: &[TransferedFile],
) -> Result<(), ProviderError> {
try_join_all(
startup_files
.iter()
.map(|file| self.send_file(&file.local_path, &file.remote_path, &file.mode)),
)
.await?;
Ok(())
}
pub(super) async fn start(&self) -> Result<(), ProviderError> {
self.docker_client
.container_exec(
&self.container_name,
vec!["sh", "-c", "echo start > /tmp/zombiepipe"],
None,
None,
)
.await
.map_err(|err| {
ProviderError::NodeSpawningFailed(
format!("failed to start pod {} after spawning", self.name),
err.into(),
)
})?
.map_err(|err| {
ProviderError::NodeSpawningFailed(
format!("failed to start pod {} after spawning", self.name,),
anyhow!("command failed in container: status {}: {}", err.0, err.1),
)
})?;
Ok(())
}
fn get_remote_parent_dir(&self, remote_file_path: &Path) -> Option<PathBuf> {
if let Some(remote_parent_dir) = remote_file_path.parent() {
if matches!(
remote_parent_dir.components().rev().peekable().peek(),
Some(Component::Normal(_))
) {
return Some(remote_parent_dir.to_path_buf());
}
}
None
}
async fn create_remote_dir(&self, remote_dir: &Path) -> Result<(), ProviderError> {
let _ = self
.docker_client
.container_exec(
&self.container_name,
vec!["mkdir", "-p", &remote_dir.to_string_lossy()],
None,
None,
)
.await
.map_err(|err| {
ProviderError::NodeSpawningFailed(
format!(
"failed to create dir {} for container {}",
remote_dir.to_string_lossy(),
&self.name
),
err.into(),
)
})?;
Ok(())
}
fn namespace_name(&self) -> String {
self.namespace
.upgrade()
.map(|namespace| namespace.name().to_string())
.unwrap_or_else(|| panic!("namespace shouldn't be dropped, {THIS_IS_A_BUG}"))
}
}
#[async_trait]
impl<FS> ProviderNode for DockerNode<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
fn name(&self) -> &str {
&self.name
}
fn args(&self) -> Vec<&str> {
self.args.iter().map(|arg| arg.as_str()).collect()
}
fn base_dir(&self) -> &PathBuf {
&self.base_dir
}
fn config_dir(&self) -> &PathBuf {
&self.config_dir
}
fn data_dir(&self) -> &PathBuf {
&self.data_dir
}
fn relay_data_dir(&self) -> &PathBuf {
&self.relay_data_dir
}
fn scripts_dir(&self) -> &PathBuf {
&self.scripts_dir
}
fn log_path(&self) -> &PathBuf {
&self.log_path
}
fn log_cmd(&self) -> String {
format!(
"{} logs -f {}",
self.docker_client.client_binary(),
self.container_name
)
}
fn path_in_node(&self, file: &Path) -> PathBuf {
// here is just a noop op since we will receive the path
// for the file inside the pod
PathBuf::from(file)
}
async fn logs(&self) -> Result<String, ProviderError> {
self.docker_client
.container_logs(&self.container_name)
.await
.map_err(|err| ProviderError::GetLogsFailed(self.name.to_string(), err.into()))
}
async fn dump_logs(&self, local_dest: PathBuf) -> Result<(), ProviderError> {
let logs = self.logs().await?;
self.filesystem
.write(local_dest, logs)
.await
.map_err(|err| ProviderError::DumpLogsFailed(self.name.to_string(), err.into()))?;
Ok(())
}
async fn run_command(
&self,
options: RunCommandOptions,
) -> Result<ExecutionResult, ProviderError> {
debug!(
"running command for {} with options {:?}",
self.name, options
);
let command = [vec![options.program], options.args].concat();
self.docker_client
.container_exec(
&self.container_name,
vec!["sh", "-c", &command.join(" ")],
Some(
options
.env
.iter()
.map(|(k, v)| (k.as_ref(), v.as_ref()))
.collect(),
),
None,
)
.await
.map_err(|err| {
ProviderError::RunCommandError(
format!("sh -c {}", &command.join(" ")),
format!("in pod {}", self.name),
err.into(),
)
})
}
async fn run_script(
&self,
_options: RunScriptOptions,
) -> Result<ExecutionResult, ProviderError> {
todo!()
}
async fn send_file(
&self,
local_file_path: &Path,
remote_file_path: &Path,
mode: &str,
) -> Result<(), ProviderError> {
if let Some(remote_parent_dir) = self.get_remote_parent_dir(remote_file_path) {
self.create_remote_dir(&remote_parent_dir).await?;
}
debug!(
"starting sending file for {}: {} to {} with mode {}",
self.name,
local_file_path.to_string_lossy(),
remote_file_path.to_string_lossy(),
mode
);
let _ = self
.docker_client
.container_cp(&self.container_name, local_file_path, remote_file_path)
.await
.map_err(|err| {
ProviderError::SendFile(
local_file_path.to_string_lossy().to_string(),
self.name.clone(),
err.into(),
)
});
let _ = self
.docker_client
.container_exec(
&self.container_name,
vec!["chmod", mode, &remote_file_path.to_string_lossy()],
None,
None,
)
.await
.map_err(|err| {
ProviderError::SendFile(
self.name.clone(),
local_file_path.to_string_lossy().to_string(),
err.into(),
)
})?;
Ok(())
}
async fn receive_file(
&self,
_remote_src: &Path,
_local_dest: &Path,
) -> Result<(), ProviderError> {
Ok(())
}
async fn ip(&self) -> Result<IpAddr, ProviderError> {
let ip = self
.docker_client
.container_ip(&self.container_name)
.await
.map_err(|err| {
ProviderError::InvalidConfig(format!("Error getting container ip, err: {err}"))
})?;
Ok(ip.parse::<IpAddr>().map_err(|err| {
ProviderError::InvalidConfig(format!(
"Can not parse the container ip: {ip}, err: {err}"
))
})?)
}
async fn pause(&self) -> Result<(), ProviderError> {
self.docker_client
.container_exec(
&self.container_name,
vec!["sh", "-c", "echo pause > /tmp/zombiepipe"],
None,
None,
)
.await
.map_err(|err| ProviderError::PauseNodeFailed(self.name.to_string(), err.into()))?
.map_err(|err| {
ProviderError::PauseNodeFailed(
self.name.to_string(),
anyhow!("error when pausing node: status {}: {}", err.0, err.1),
)
})?;
Ok(())
}
async fn resume(&self) -> Result<(), ProviderError> {
self.docker_client
.container_exec(
&self.container_name,
vec!["sh", "-c", "echo resume > /tmp/zombiepipe"],
None,
None,
)
.await
.map_err(|err| ProviderError::PauseNodeFailed(self.name.to_string(), err.into()))?
.map_err(|err| {
ProviderError::PauseNodeFailed(
self.name.to_string(),
anyhow!("error when pausing node: status {}: {}", err.0, err.1),
)
})?;
Ok(())
}
async fn restart(&self, after: Option<Duration>) -> Result<(), ProviderError> {
if let Some(duration) = after {
sleep(duration).await;
}
self.docker_client
.container_exec(
&self.container_name,
vec!["sh", "-c", "echo restart > /tmp/zombiepipe"],
None,
None,
)
.await
.map_err(|err| ProviderError::PauseNodeFailed(self.name.to_string(), err.into()))?
.map_err(|err| {
ProviderError::PauseNodeFailed(
self.name.to_string(),
anyhow!("error when pausing node: status {}: {}", err.0, err.1),
)
})?;
Ok(())
}
async fn destroy(&self) -> Result<(), ProviderError> {
self.docker_client
.container_rm(&self.container_name)
.await
.map_err(|err| ProviderError::KillNodeFailed(self.name.to_string(), err.into()))?;
if let Some(namespace) = self.namespace.upgrade() {
namespace.nodes.write().await.remove(&self.name);
}
Ok(())
}
}
@@ -0,0 +1,161 @@
use std::{
collections::HashMap,
path::{Path, PathBuf},
sync::{Arc, Weak},
};
use async_trait::async_trait;
use support::fs::FileSystem;
use tokio::sync::RwLock;
use super::{client::DockerClient, namespace::DockerNamespace};
use crate::{
shared::helpers::extract_namespace_info, types::ProviderCapabilities, DynNamespace, Provider,
ProviderError, ProviderNamespace,
};
pub const PROVIDER_NAME: &str = "docker";
pub struct DockerProvider<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
weak: Weak<DockerProvider<FS>>,
capabilities: ProviderCapabilities,
tmp_dir: PathBuf,
docker_client: DockerClient,
filesystem: FS,
pub(super) namespaces: RwLock<HashMap<String, Arc<DockerNamespace<FS>>>>,
}
impl<FS> DockerProvider<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub async fn new(filesystem: FS) -> Arc<Self> {
let docker_client = DockerClient::new().await.unwrap();
let provider = Arc::new_cyclic(|weak| DockerProvider {
weak: weak.clone(),
capabilities: ProviderCapabilities {
requires_image: true,
has_resources: false,
prefix_with_full_path: false,
use_default_ports_in_cmd: true,
},
tmp_dir: std::env::temp_dir(),
docker_client,
filesystem,
namespaces: RwLock::new(HashMap::new()),
});
let cloned_provider = provider.clone();
tokio::spawn(async move {
tokio::signal::ctrl_c().await.unwrap();
for (_, ns) in cloned_provider.namespaces().await {
if ns.is_detached().await {
// best effort
let _ = ns.destroy().await;
}
}
// exit the process (130, SIGINT)
std::process::exit(130)
});
provider
}
pub fn tmp_dir(mut self, tmp_dir: impl Into<PathBuf>) -> Self {
self.tmp_dir = tmp_dir.into();
self
}
}
#[async_trait]
impl<FS> Provider for DockerProvider<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
fn name(&self) -> &str {
PROVIDER_NAME
}
fn capabilities(&self) -> &ProviderCapabilities {
&self.capabilities
}
async fn namespaces(&self) -> HashMap<String, DynNamespace> {
self.namespaces
.read()
.await
.iter()
.map(|(name, namespace)| (name.clone(), namespace.clone() as DynNamespace))
.collect()
}
async fn create_namespace(&self) -> Result<DynNamespace, ProviderError> {
let namespace = DockerNamespace::new(
&self.weak,
&self.tmp_dir,
&self.capabilities,
&self.docker_client,
&self.filesystem,
None,
)
.await?;
self.namespaces
.write()
.await
.insert(namespace.name().to_string(), namespace.clone());
Ok(namespace)
}
async fn create_namespace_with_base_dir(
&self,
base_dir: &Path,
) -> Result<DynNamespace, ProviderError> {
let namespace = DockerNamespace::new(
&self.weak,
&self.tmp_dir,
&self.capabilities,
&self.docker_client,
&self.filesystem,
Some(base_dir),
)
.await?;
self.namespaces
.write()
.await
.insert(namespace.name().to_string(), namespace.clone());
Ok(namespace)
}
async fn create_namespace_from_json(
&self,
json_value: &serde_json::Value,
) -> Result<DynNamespace, ProviderError> {
let (base_dir, name) = extract_namespace_info(json_value)?;
let namespace = DockerNamespace::attach_to_live(
&self.weak,
&self.capabilities,
&self.docker_client,
&self.filesystem,
&base_dir,
&name,
)
.await?;
self.namespaces
.write()
.await
.insert(namespace.name().to_string(), namespace.clone());
Ok(namespace)
}
}
@@ -0,0 +1,7 @@
mod client;
mod namespace;
mod node;
mod pod_spec_builder;
mod provider;
pub use provider::KubernetesProvider;
@@ -0,0 +1,602 @@
use std::{
collections::BTreeMap, fmt::Debug, os::unix::process::ExitStatusExt, process::ExitStatus,
time::Duration,
};
use anyhow::anyhow;
use futures::{StreamExt, TryStreamExt};
use k8s_openapi::api::core::v1::{
ConfigMap, Namespace, Pod, PodSpec, PodStatus, Service, ServiceSpec,
};
use kube::{
api::{AttachParams, DeleteParams, ListParams, LogParams, PostParams, WatchParams},
core::{DynamicObject, GroupVersionKind, ObjectMeta, TypeMeta, WatchEvent},
discovery::ApiResource,
runtime::{conditions, wait::await_condition},
Api, Resource,
};
use serde::de::DeserializeOwned;
use support::constants::THIS_IS_A_BUG;
use tokio::{
io::{AsyncRead, ErrorKind},
net::TcpListener,
task::JoinHandle,
};
use tokio_util::compat::FuturesAsyncReadCompatExt;
use tracing::{debug, trace};
use crate::{constants::LOCALHOST, types::ExecutionResult};
#[derive(thiserror::Error, Debug)]
#[error(transparent)]
pub struct Error(#[from] anyhow::Error);
pub type Result<T> = core::result::Result<T, Error>;
#[derive(Clone)]
pub struct KubernetesClient {
inner: kube::Client,
}
impl KubernetesClient {
pub(super) async fn new() -> Result<Self> {
Ok(Self {
// TODO: make it more flexible with path to kube config
inner: kube::Client::try_default()
.await
.map_err(|err| Error::from(anyhow!("error initializing kubers client: {err}")))?,
})
}
#[allow(dead_code)]
pub(super) async fn get_namespace(&self, name: &str) -> Result<Option<Namespace>> {
Api::<Namespace>::all(self.inner.clone())
.get_opt(name.as_ref())
.await
.map_err(|err| Error::from(anyhow!("error while getting namespace {name}: {err}")))
}
#[allow(dead_code)]
pub(super) async fn get_namespaces(&self) -> Result<Vec<Namespace>> {
Ok(Api::<Namespace>::all(self.inner.clone())
.list(&ListParams::default())
.await
.map_err(|err| Error::from(anyhow!("error while getting all namespaces: {err}")))?
.into_iter()
.filter(|ns| matches!(&ns.meta().name, Some(name) if name.starts_with("zombienet")))
.collect())
}
pub(super) async fn create_namespace(
&self,
name: &str,
labels: BTreeMap<String, String>,
) -> Result<Namespace> {
let namespaces = Api::<Namespace>::all(self.inner.clone());
let namespace = Namespace {
metadata: ObjectMeta {
name: Some(name.to_string()),
labels: Some(labels),
..Default::default()
},
..Default::default()
};
namespaces
.create(&PostParams::default(), &namespace)
.await
.map_err(|err| Error::from(anyhow!("error while created namespace {name}: {err}")))?;
self.wait_created(namespaces, name).await?;
Ok(namespace)
}
pub(super) async fn delete_namespace(&self, name: &str) -> Result<()> {
let namespaces = Api::<Namespace>::all(self.inner.clone());
namespaces
.delete(name, &DeleteParams::default())
.await
.map_err(|err| Error::from(anyhow!("error while deleting namespace {name}: {err}")))?;
Ok(())
}
pub(super) async fn create_config_map_from_file(
&self,
namespace: &str,
name: &str,
file_name: &str,
file_contents: &str,
labels: BTreeMap<String, String>,
) -> Result<ConfigMap> {
let config_maps = Api::<ConfigMap>::namespaced(self.inner.clone(), namespace);
let config_map = ConfigMap {
metadata: ObjectMeta {
name: Some(name.to_string()),
namespace: Some(namespace.to_string()),
labels: Some(labels),
..Default::default()
},
data: Some(BTreeMap::from([(
file_name.to_string(),
file_contents.to_string(),
)])),
..Default::default()
};
config_maps
.create(&PostParams::default(), &config_map)
.await
.map_err(|err| {
Error::from(anyhow!(
"error while creating config map {name} for {file_name}: {err}"
))
})?;
self.wait_created(config_maps, name).await?;
Ok(config_map)
}
pub(super) async fn create_pod(
&self,
namespace: &str,
name: &str,
spec: PodSpec,
labels: BTreeMap<String, String>,
) -> Result<Pod> {
let pods = Api::<Pod>::namespaced(self.inner.clone(), namespace);
let pod = Pod {
metadata: ObjectMeta {
name: Some(name.to_string()),
namespace: Some(namespace.to_string()),
labels: Some(labels),
..Default::default()
},
spec: Some(spec),
..Default::default()
};
pods.create(&PostParams::default(), &pod)
.await
.map_err(|err| Error::from(anyhow!("error while creating pod {name}: {err}")))?;
trace!("Pod {name} checking for ready state!");
let wait_ready = await_condition(pods, name, helpers::is_pod_ready());
// TODO: we should use the `node_spawn_timeout` from global settings here.
let _ = tokio::time::timeout(Duration::from_secs(600), wait_ready)
.await
.map_err(|err| {
Error::from(anyhow!("error while awaiting pod {name} running: {err}"))
})?;
debug!("Pod {name} is Ready!");
Ok(pod)
}
pub(super) async fn pod_logs(&self, namespace: &str, name: &str) -> Result<String> {
Api::<Pod>::namespaced(self.inner.clone(), namespace)
.logs(
name,
&LogParams {
pretty: true,
timestamps: true,
..Default::default()
},
)
.await
.map_err(|err| Error::from(anyhow!("error while getting logs for pod {name}: {err}")))
}
pub(super) async fn pod_status(&self, namespace: &str, name: &str) -> Result<PodStatus> {
let pod = Api::<Pod>::namespaced(self.inner.clone(), namespace)
.get(name)
.await
.map_err(|err| Error::from(anyhow!("error while getting pod {name}: {err}")))?;
let status = pod.status.ok_or(Error::from(anyhow!(
"error while getting status for pod {name}"
)))?;
Ok(status)
}
#[allow(dead_code)]
pub(super) async fn create_pod_logs_stream(
&self,
namespace: &str,
name: &str,
) -> Result<Box<dyn AsyncRead + Send + Unpin>> {
Ok(Box::new(
Api::<Pod>::namespaced(self.inner.clone(), namespace)
.log_stream(
name,
&LogParams {
follow: true,
pretty: true,
timestamps: true,
..Default::default()
},
)
.await
.map_err(|err| {
Error::from(anyhow!(
"error while getting a log stream for {name}: {err}"
))
})?
.compat(),
))
}
pub(super) async fn pod_exec<S>(
&self,
namespace: &str,
name: &str,
command: Vec<S>,
) -> Result<ExecutionResult>
where
S: Into<String> + std::fmt::Debug + Send,
{
trace!("running command: {command:?} on pod {name} for ns {namespace}");
let mut process = Api::<Pod>::namespaced(self.inner.clone(), namespace)
.exec(
name,
command,
&AttachParams::default().stdout(true).stderr(true),
)
.await
.map_err(|err| Error::from(anyhow!("error while exec in the pod {name}: {err}")))?;
let stdout_stream = process.stdout().expect(&format!(
"stdout shouldn't be None when true passed to exec {THIS_IS_A_BUG}"
));
let stdout = tokio_util::io::ReaderStream::new(stdout_stream)
.filter_map(|r| async { r.ok().and_then(|v| String::from_utf8(v.to_vec()).ok()) })
.collect::<Vec<_>>()
.await
.join("");
let stderr_stream = process.stderr().expect(&format!(
"stderr shouldn't be None when true passed to exec {THIS_IS_A_BUG}"
));
let stderr = tokio_util::io::ReaderStream::new(stderr_stream)
.filter_map(|r| async { r.ok().and_then(|v| String::from_utf8(v.to_vec()).ok()) })
.collect::<Vec<_>>()
.await
.join("");
let status = process
.take_status()
.expect(&format!(
"first call to status shouldn't fail {THIS_IS_A_BUG}"
))
.await;
// await process to finish
process.join().await.map_err(|err| {
Error::from(anyhow!(
"error while joining process during exec for {name}: {err}"
))
})?;
match status {
// command succeeded with stdout
Some(status) if status.status.as_ref().is_some_and(|s| s == "Success") => {
Ok(Ok(stdout))
},
// command failed
Some(status) if status.status.as_ref().is_some_and(|s| s == "Failure") => {
match status.reason {
// due to exit code
Some(reason) if reason == "NonZeroExitCode" => {
let exit_status = status
.details
.and_then(|details| {
details.causes.and_then(|causes| {
causes.first().and_then(|cause| {
cause.message.as_deref().and_then(|message| {
message.parse::<i32>().ok().map(ExitStatus::from_raw)
})
})
})
})
.expect(
&format!("command with non-zero exit code should have exit code present {THIS_IS_A_BUG}")
);
Ok(Err((exit_status, stderr)))
},
// due to other unknown reason
Some(ref reason) => Err(Error::from(anyhow!(
format!("unhandled reason while exec for {name}: {reason}, stderr: {stderr}, status: {status:?}")
))),
None => {
panic!("command had status but no reason was present, this is a bug")
},
}
},
Some(_) => {
unreachable!("command had status but it didn't matches either Success or Failure, this is a bug from the kube.rs library itself");
},
None => {
panic!("command has no status following its execution, this is a bug");
},
}
}
pub(super) async fn delete_pod(&self, namespace: &str, name: &str) -> Result<()> {
let pods = Api::<Pod>::namespaced(self.inner.clone(), namespace);
pods.delete(name, &DeleteParams::default())
.await
.map_err(|err| Error::from(anyhow!("error when deleting pod {name}: {err}")))?;
await_condition(pods, name, conditions::is_deleted(name))
.await
.map_err(|err| {
Error::from(anyhow!(
"error when waiting for pod {name} to be deleted: {err}"
))
})?;
Ok(())
}
pub(super) async fn create_service(
&self,
namespace: &str,
name: &str,
spec: ServiceSpec,
labels: BTreeMap<String, String>,
) -> Result<Service> {
let services = Api::<Service>::namespaced(self.inner.clone(), namespace);
let service = Service {
metadata: ObjectMeta {
name: Some(name.to_string()),
namespace: Some(namespace.to_string()),
labels: Some(labels),
..Default::default()
},
spec: Some(spec),
..Default::default()
};
services
.create(&PostParams::default(), &service)
.await
.map_err(|err| Error::from(anyhow!("error while creating service {name}: {err}")))?;
Ok(service)
}
pub(super) async fn create_pod_port_forward(
&self,
namespace: &str,
name: &str,
local_port: u16,
remote_port: u16,
) -> Result<(u16, JoinHandle<()>)> {
let pods = Api::<Pod>::namespaced(self.inner.clone(), namespace);
let bind = TcpListener::bind((LOCALHOST, local_port))
.await
.map_err(|err| {
Error::from(anyhow!(
"error binding port {local_port} for pod {name}: {err}"
))
})?;
let local_port = bind.local_addr().map_err(|err| Error(err.into()))?.port();
let name = name.to_string();
const MAX_FAILURES: usize = 5;
let monitor_handle = tokio::spawn(async move {
let mut consecutive_failures = 0;
loop {
let (mut client_conn, _) = match bind.accept().await {
Ok(conn) => {
consecutive_failures = 0;
conn
},
Err(e) => {
if consecutive_failures < MAX_FAILURES {
trace!("Port-forward accept error: {e:?}, retrying in 1s");
tokio::time::sleep(Duration::from_secs(1)).await;
consecutive_failures += 1;
continue;
} else {
trace!("Port-forward accept failed too many times, giving up");
break;
}
},
};
let peer = match client_conn.peer_addr() {
Ok(addr) => addr,
Err(e) => {
trace!("Failed to get peer address: {e:?}");
break;
},
};
trace!("new connection on local_port: {local_port}, peer: {peer}");
let (name, pods) = (name.clone(), pods.clone());
tokio::spawn(async move {
loop {
// Try to establish port-forward
let mut forwarder = match pods.portforward(&name, &[remote_port]).await {
Ok(f) => {
consecutive_failures = 0;
f
},
Err(e) => {
consecutive_failures += 1;
if consecutive_failures < MAX_FAILURES {
trace!("portforward failed to establish ({}/{}): {e:?}, retrying in 1s",
consecutive_failures, MAX_FAILURES);
tokio::time::sleep(Duration::from_secs(1)).await;
continue;
} else {
trace!("portforward failed to establish after {} attempts: {e:?}, closing connection",
consecutive_failures);
break;
}
},
};
let mut upstream_conn = match forwarder.take_stream(remote_port) {
Some(s) => s,
None => {
trace!("Failed to take stream for remote_port: {remote_port}, retrying in 1s");
tokio::time::sleep(Duration::from_secs(1)).await;
continue;
},
};
match tokio::io::copy_bidirectional(&mut client_conn, &mut upstream_conn)
.await
{
Ok((_n1, _n2)) => {
// EOF reached, close connection
trace!("copy_bidirectional finished (EOF), closing connection");
drop(upstream_conn);
let _ = forwarder.join().await;
break;
},
Err(e) => {
let kind = e.kind();
match kind {
ErrorKind::ConnectionReset
| ErrorKind::ConnectionAborted
| ErrorKind::ConnectionRefused
| ErrorKind::TimedOut => {
consecutive_failures += 1;
if consecutive_failures < MAX_FAILURES {
trace!("Network error ({kind:?}): {e:?}, retrying port-forward for this connection");
tokio::time::sleep(Duration::from_secs(1)).await;
continue;
} else {
trace!("portforward failed to establish after {} attempts: {e:?}, closing connection",
consecutive_failures);
break;
}
},
_ => {
trace!("Non-network error ({kind:?}): {e:?}, closing connection");
break;
},
}
},
}
}
});
trace!("finished forwarder process for local port: {local_port}, peer: {peer}");
}
});
Ok((local_port, monitor_handle))
}
/// Create resources from yamls in `static-configs` directory
pub(super) async fn create_static_resource(
&self,
namespace: &str,
raw_manifest: &str,
) -> Result<()> {
let tm: TypeMeta = serde_yaml::from_str(raw_manifest).map_err(|err| {
Error::from(anyhow!(
"error while extracting TypeMeta from manifest: {raw_manifest}: {err}"
))
})?;
let gvk = GroupVersionKind::try_from(&tm).map_err(|err| {
Error::from(anyhow!(
"error while extracting GroupVersionKind from manifest: {raw_manifest}: {err}"
))
})?;
let ar = ApiResource::from_gvk(&gvk);
let api: Api<DynamicObject> = Api::namespaced_with(self.inner.clone(), namespace, &ar);
api.create(
&PostParams::default(),
&serde_yaml::from_str(raw_manifest).unwrap(),
)
.await
.map_err(|err| {
Error::from(anyhow!(
"error while creating static-config {raw_manifest}: {err}"
))
})?;
Ok(())
}
async fn wait_created<K>(&self, api: Api<K>, name: &str) -> Result<()>
where
K: Clone + DeserializeOwned + Debug,
{
let params = &WatchParams::default().fields(&format!("metadata.name={name}"));
let mut stream = api
.watch(params, "0")
.await
.map_err(|err| {
Error::from(anyhow!(
"error while awaiting first response when resource {name} is created: {err}"
))
})?
.boxed();
while let Some(status) = stream.try_next().await.map_err(|err| {
Error::from(anyhow!(
"error while awaiting next change after resource {name} is created: {err}"
))
})? {
match status {
WatchEvent::Added(_) => break,
WatchEvent::Error(err) => Err(Error::from(anyhow!(
"error while awaiting resource {name} is created: {err}"
)))?,
WatchEvent::Bookmark(_) => {
// bookmark events are periodically sent as keep-alive/checkpoint, we should continue waiting
}
any_other_event => panic!("Unexpected event happened while creating '{name}' {THIS_IS_A_BUG}. Event: {any_other_event:?}"),
}
}
Ok(())
}
}
mod helpers {
use k8s_openapi::api::core::v1::Pod;
use kube::runtime::wait::Condition;
use tracing::trace;
/// An await condition for `Pod` that returns `true` once it is ready
/// based on [`kube::runtime::wait::conditions::is_pod_running`]
pub fn is_pod_ready() -> impl Condition<Pod> {
|obj: Option<&Pod>| {
if let Some(pod) = &obj {
if let Some(status) = &pod.status {
if let Some(conditions) = &status.conditions {
let ready = conditions
.iter()
.any(|cond| cond.status == "True" && cond.type_ == "Ready");
if ready {
trace!("{:#?}", status);
return ready;
}
}
}
}
false
}
}
}
@@ -0,0 +1,600 @@
use std::{
collections::{BTreeMap, HashMap},
env,
path::{Path, PathBuf},
sync::{Arc, Weak},
};
use async_trait::async_trait;
use k8s_openapi::{
api::core::v1::{
Container, ContainerPort, HTTPGetAction, PodSpec, Probe, ServicePort, ServiceSpec,
},
apimachinery::pkg::util::intstr::IntOrString,
};
use support::{constants::THIS_IS_A_BUG, fs::FileSystem, replacer::apply_replacements};
use tokio::sync::{Mutex, RwLock};
use tracing::{debug, trace, warn};
use uuid::Uuid;
use super::{client::KubernetesClient, node::KubernetesNode};
use crate::{
constants::NAMESPACE_PREFIX,
kubernetes::{
node::{DeserializableKubernetesNodeOptions, KubernetesNodeOptions},
provider,
},
shared::helpers::{extract_execution_result, running_in_ci},
types::{
GenerateFileCommand, GenerateFilesOptions, ProviderCapabilities, RunCommandOptions,
SpawnNodeOptions,
},
DynNode, KubernetesProvider, ProviderError, ProviderNamespace, ProviderNode,
};
const FILE_SERVER_IMAGE: &str = "europe-west3-docker.pkg.dev/parity-zombienet/zombienet-public-images/zombienet-file-server:latest";
// env var used by our internal CI to pass the namespace created and ready to use
const ZOMBIE_K8S_CI_NAMESPACE: &str = "ZOMBIE_K8S_CI_NAMESPACE";
pub(super) struct KubernetesNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
weak: Weak<KubernetesNamespace<FS>>,
provider: Weak<KubernetesProvider<FS>>,
name: String,
base_dir: PathBuf,
capabilities: ProviderCapabilities,
k8s_client: KubernetesClient,
filesystem: FS,
file_server_fw_task: RwLock<Option<tokio::task::JoinHandle<()>>>,
delete_on_drop: Arc<Mutex<bool>>,
pub(super) file_server_port: RwLock<Option<u16>>,
pub(super) nodes: RwLock<HashMap<String, Arc<KubernetesNode<FS>>>>,
}
impl<FS> KubernetesNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) async fn new(
provider: &Weak<KubernetesProvider<FS>>,
tmp_dir: &PathBuf,
capabilities: &ProviderCapabilities,
k8s_client: &KubernetesClient,
filesystem: &FS,
custom_base_dir: Option<&Path>,
) -> Result<Arc<Self>, ProviderError> {
// If the namespace is already provided
let name = if let Ok(name) = env::var(ZOMBIE_K8S_CI_NAMESPACE) {
name
} else {
format!("{}{}", NAMESPACE_PREFIX, Uuid::new_v4())
};
let base_dir = if let Some(custom_base_dir) = custom_base_dir {
if !filesystem.exists(custom_base_dir).await {
filesystem.create_dir(custom_base_dir).await?;
} else {
warn!(
"⚠️ Using and existing directory {} as base dir",
custom_base_dir.to_string_lossy()
);
}
PathBuf::from(custom_base_dir)
} else {
let base_dir = PathBuf::from_iter([tmp_dir, &PathBuf::from(&name)]);
filesystem.create_dir(&base_dir).await?;
base_dir
};
let namespace = Arc::new_cyclic(|weak| KubernetesNamespace {
weak: weak.clone(),
provider: provider.clone(),
name,
base_dir,
capabilities: capabilities.clone(),
filesystem: filesystem.clone(),
k8s_client: k8s_client.clone(),
file_server_port: RwLock::new(None),
file_server_fw_task: RwLock::new(None),
nodes: RwLock::new(HashMap::new()),
delete_on_drop: Arc::new(Mutex::new(true)),
});
namespace.initialize().await?;
Ok(namespace)
}
pub(super) async fn attach_to_live(
provider: &Weak<KubernetesProvider<FS>>,
capabilities: &ProviderCapabilities,
k8s_client: &KubernetesClient,
filesystem: &FS,
custom_base_dir: &Path,
name: &str,
) -> Result<Arc<Self>, ProviderError> {
let base_dir = custom_base_dir.to_path_buf();
let namespace = Arc::new_cyclic(|weak| KubernetesNamespace {
weak: weak.clone(),
provider: provider.clone(),
name: name.to_owned(),
base_dir,
capabilities: capabilities.clone(),
filesystem: filesystem.clone(),
k8s_client: k8s_client.clone(),
file_server_port: RwLock::new(None),
file_server_fw_task: RwLock::new(None),
nodes: RwLock::new(HashMap::new()),
delete_on_drop: Arc::new(Mutex::new(false)),
});
namespace.setup_file_server_port_fwd("fileserver").await?;
Ok(namespace)
}
async fn initialize(&self) -> Result<(), ProviderError> {
// Initialize the namespace IFF
// we are not in CI or we don't have the env `ZOMBIE_NAMESPACE` set
if env::var(ZOMBIE_K8S_CI_NAMESPACE).is_err() || !running_in_ci() {
self.initialize_k8s().await?;
}
// Ensure namespace isolation and minimal resources IFF we are running in CI
if running_in_ci() {
self.initialize_static_resources().await?
}
self.initialize_file_server().await?;
self.setup_script_config_map(
"zombie-wrapper",
include_str!("../shared/scripts/zombie-wrapper.sh"),
"zombie_wrapper_config_map_manifest.yaml",
// TODO: add correct labels
BTreeMap::new(),
)
.await?;
self.setup_script_config_map(
"helper-binaries-downloader",
include_str!("../shared/scripts/helper-binaries-downloader.sh"),
"helper_binaries_downloader_config_map_manifest.yaml",
// TODO: add correct labels
BTreeMap::new(),
)
.await?;
Ok(())
}
async fn initialize_k8s(&self) -> Result<(), ProviderError> {
// TODO (javier): check with Hamid if we are using this labels in any scheduling logic.
let labels = BTreeMap::from([
(
"jobId".to_string(),
env::var("CI_JOB_ID").unwrap_or("".to_string()),
),
(
"projectName".to_string(),
env::var("CI_PROJECT_NAME").unwrap_or("".to_string()),
),
(
"projectId".to_string(),
env::var("CI_PROJECT_ID").unwrap_or("".to_string()),
),
]);
let manifest = self
.k8s_client
.create_namespace(&self.name, labels)
.await
.map_err(|err| {
ProviderError::CreateNamespaceFailed(self.name.to_string(), err.into())
})?;
let serialized_manifest = serde_yaml::to_string(&manifest).map_err(|err| {
ProviderError::CreateNamespaceFailed(self.name.to_string(), err.into())
})?;
let dest_path =
PathBuf::from_iter([&self.base_dir, &PathBuf::from("namespace_manifest.yaml")]);
self.filesystem
.write(dest_path, serialized_manifest)
.await?;
Ok(())
}
async fn initialize_static_resources(&self) -> Result<(), ProviderError> {
let np_manifest = apply_replacements(
include_str!("./static-configs/namespace-network-policy.yaml"),
&HashMap::from([("namespace", self.name())]),
);
// Apply NetworkPolicy manifest
self.k8s_client
.create_static_resource(&self.name, &np_manifest)
.await
.map_err(|err| {
ProviderError::CreateNamespaceFailed(self.name.to_string(), err.into())
})?;
// Apply LimitRange manifest
self.k8s_client
.create_static_resource(
&self.name,
include_str!("./static-configs/baseline-resources.yaml"),
)
.await
.map_err(|err| {
ProviderError::CreateNamespaceFailed(self.name.to_string(), err.into())
})?;
Ok(())
}
async fn initialize_file_server(&self) -> Result<(), ProviderError> {
let name = "fileserver".to_string();
let labels = BTreeMap::from([
("app.kubernetes.io/name".to_string(), name.clone()),
(
"x-infra-instance".to_string(),
env::var("X_INFRA_INSTANCE").unwrap_or("ondemand".to_string()),
),
]);
let pod_spec = PodSpec {
hostname: Some(name.clone()),
containers: vec![Container {
name: name.clone(),
image: Some(FILE_SERVER_IMAGE.to_string()),
image_pull_policy: Some("Always".to_string()),
ports: Some(vec![ContainerPort {
container_port: 80,
..Default::default()
}]),
startup_probe: Some(Probe {
http_get: Some(HTTPGetAction {
path: Some("/".to_string()),
port: IntOrString::Int(80),
..Default::default()
}),
initial_delay_seconds: Some(1),
period_seconds: Some(2),
failure_threshold: Some(3),
..Default::default()
}),
..Default::default()
}],
restart_policy: Some("OnFailure".into()),
..Default::default()
};
let pod_manifest = self
.k8s_client
.create_pod(&self.name, &name, pod_spec, labels.clone())
.await
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
// TODO: remove duplication across methods
let pod_serialized_manifest = serde_yaml::to_string(&pod_manifest)
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
let pod_dest_path = PathBuf::from_iter([
&self.base_dir,
&PathBuf::from("file_server_pod_manifest.yaml"),
]);
self.filesystem
.write(pod_dest_path, pod_serialized_manifest)
.await?;
let service_spec = ServiceSpec {
selector: Some(labels.clone()),
ports: Some(vec![ServicePort {
port: 80,
..Default::default()
}]),
..Default::default()
};
let service_manifest = self
.k8s_client
.create_service(&self.name, &name, service_spec, labels)
.await
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
let serialized_service_manifest = serde_yaml::to_string(&service_manifest)
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
let service_dest_path = PathBuf::from_iter([
&self.base_dir,
&PathBuf::from("file_server_service_manifest.yaml"),
]);
self.filesystem
.write(service_dest_path, serialized_service_manifest)
.await?;
self.setup_file_server_port_fwd(&name).await?;
Ok(())
}
async fn setup_file_server_port_fwd(&self, name: &str) -> Result<(), ProviderError> {
let (port, task) = self
.k8s_client
.create_pod_port_forward(&self.name, name, 0, 80)
.await
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
*self.file_server_port.write().await = Some(port);
*self.file_server_fw_task.write().await = Some(task);
Ok(())
}
async fn setup_script_config_map(
&self,
name: &str,
script_contents: &str,
local_manifest_name: &str,
labels: BTreeMap<String, String>,
) -> Result<(), ProviderError> {
let manifest = self
.k8s_client
.create_config_map_from_file(
&self.name,
name,
&format!("{name}.sh"),
script_contents,
labels,
)
.await
.map_err(|err| {
ProviderError::CreateNamespaceFailed(self.name.to_string(), err.into())
})?;
let serializer_manifest = serde_yaml::to_string(&manifest).map_err(|err| {
ProviderError::CreateNamespaceFailed(self.name.to_string(), err.into())
})?;
let dest_path = PathBuf::from_iter([&self.base_dir, &PathBuf::from(local_manifest_name)]);
self.filesystem
.write(dest_path, serializer_manifest)
.await?;
Ok(())
}
pub async fn set_delete_on_drop(&self, delete_on_drop: bool) {
*self.delete_on_drop.lock().await = delete_on_drop;
}
pub async fn delete_on_drop(&self) -> bool {
if let Ok(delete_on_drop) = self.delete_on_drop.try_lock() {
*delete_on_drop
} else {
// if we can't lock just remove the ns
true
}
}
}
impl<FS> Drop for KubernetesNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
fn drop(&mut self) {
let ns_name = self.name.clone();
if let Ok(delete_on_drop) = self.delete_on_drop.try_lock() {
if *delete_on_drop {
let client = self.k8s_client.clone();
let provider = self.provider.upgrade();
futures::executor::block_on(async move {
trace!("🧟 deleting ns {ns_name} from cluster");
let _ = client.delete_namespace(&ns_name).await;
if let Some(provider) = provider {
provider.namespaces.write().await.remove(&ns_name);
}
trace!("✅ deleted");
});
} else {
trace!("⚠️ leaking ns {ns_name} in cluster");
}
};
}
}
#[async_trait]
impl<FS> ProviderNamespace for KubernetesNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
fn name(&self) -> &str {
&self.name
}
fn base_dir(&self) -> &PathBuf {
&self.base_dir
}
fn capabilities(&self) -> &ProviderCapabilities {
&self.capabilities
}
fn provider_name(&self) -> &str {
provider::PROVIDER_NAME
}
async fn detach(&self) {
self.set_delete_on_drop(false).await;
}
async fn is_detached(&self) -> bool {
self.delete_on_drop().await
}
async fn nodes(&self) -> HashMap<String, DynNode> {
self.nodes
.read()
.await
.iter()
.map(|(name, node)| (name.clone(), node.clone() as DynNode))
.collect()
}
async fn get_node_available_args(
&self,
(command, image): (String, Option<String>),
) -> Result<String, ProviderError> {
let node_image = image.expect(&format!("image should be present when getting node available args with kubernetes provider {THIS_IS_A_BUG}"));
// run dummy command in new pod
let temp_node = self
.spawn_node(
&SpawnNodeOptions::new(format!("temp-{}", Uuid::new_v4()), "cat".to_string())
.image(node_image.clone()),
)
.await?;
let available_args_output = temp_node
.run_command(RunCommandOptions::new(command.clone()).args(vec!["--help"]))
.await?
.map_err(|(_exit, status)| {
ProviderError::NodeAvailableArgsError(node_image, command, status)
})?;
temp_node.destroy().await?;
Ok(available_args_output)
}
async fn spawn_node(&self, options: &SpawnNodeOptions) -> Result<DynNode, ProviderError> {
trace!("spawn node options {options:?}");
let node = KubernetesNode::new(KubernetesNodeOptions {
namespace: &self.weak,
namespace_base_dir: &self.base_dir,
name: &options.name,
image: options.image.as_ref(),
program: &options.program,
args: &options.args,
env: &options.env,
startup_files: &options.injected_files,
resources: options.resources.as_ref(),
db_snapshot: options.db_snapshot.as_ref(),
k8s_client: &self.k8s_client,
filesystem: &self.filesystem,
})
.await?;
self.nodes
.write()
.await
.insert(node.name().to_string(), node.clone());
Ok(node)
}
async fn spawn_node_from_json(
&self,
json_value: &serde_json::Value,
) -> Result<DynNode, ProviderError> {
let deserializable: DeserializableKubernetesNodeOptions =
serde_json::from_value(json_value.clone())?;
let options = KubernetesNodeOptions::from_deserializable(
&deserializable,
&self.weak,
&self.base_dir,
&self.k8s_client,
&self.filesystem,
);
let node = KubernetesNode::attach_to_live(options).await?;
self.nodes
.write()
.await
.insert(node.name().to_string(), node.clone());
Ok(node)
}
async fn generate_files(&self, options: GenerateFilesOptions) -> Result<(), ProviderError> {
debug!("generate files options {options:#?}");
let node_name = options
.temp_name
.unwrap_or_else(|| format!("temp-{}", Uuid::new_v4()));
let node_image = options
.image
.expect(&format!("image should be present when generating files with kubernetes provider {THIS_IS_A_BUG}"));
// run dummy command in new pod
let temp_node = self
.spawn_node(
&SpawnNodeOptions::new(node_name, "cat".to_string())
.injected_files(options.injected_files)
.image(node_image),
)
.await?;
for GenerateFileCommand {
program,
args,
env,
local_output_path,
} in options.commands
{
let local_output_full_path = format!(
"{}{}{}",
self.base_dir.to_string_lossy(),
if local_output_path.starts_with("/") {
""
} else {
"/"
},
local_output_path.to_string_lossy()
);
let contents = extract_execution_result(
&temp_node,
RunCommandOptions { program, args, env },
options.expected_path.as_ref(),
)
.await?;
self.filesystem
.write(local_output_full_path, contents)
.await
.map_err(|err| ProviderError::FileGenerationFailed(err.into()))?;
}
temp_node.destroy().await
}
async fn static_setup(&self) -> Result<(), ProviderError> {
todo!()
}
async fn destroy(&self) -> Result<(), ProviderError> {
let _ = self
.k8s_client
.delete_namespace(&self.name)
.await
.map_err(|err| ProviderError::DeleteNamespaceFailed(self.name.clone(), err.into()))?;
if let Some(provider) = self.provider.upgrade() {
provider.namespaces.write().await.remove(&self.name);
}
Ok(())
}
}
@@ -0,0 +1,886 @@
use std::{
collections::{BTreeMap, HashMap},
env,
net::IpAddr,
path::{Component, Path, PathBuf},
sync::{Arc, Weak},
time::Duration,
};
use anyhow::anyhow;
use async_trait::async_trait;
use configuration::{shared::resources::Resources, types::AssetLocation};
use futures::future::try_join_all;
use k8s_openapi::api::core::v1::{ServicePort, ServiceSpec};
use serde::{Deserialize, Serialize};
use sha2::Digest;
use support::{constants::THIS_IS_A_BUG, fs::FileSystem};
use tokio::{sync::RwLock, task::JoinHandle, time::sleep, try_join};
use tracing::{debug, trace, warn};
use url::Url;
use super::{
client::KubernetesClient, namespace::KubernetesNamespace, pod_spec_builder::PodSpecBuilder,
};
use crate::{
constants::{
NODE_CONFIG_DIR, NODE_DATA_DIR, NODE_RELAY_DATA_DIR, NODE_SCRIPTS_DIR, P2P_PORT,
PROMETHEUS_PORT, RPC_HTTP_PORT, RPC_WS_PORT,
},
kubernetes,
types::{ExecutionResult, RunCommandOptions, RunScriptOptions, TransferedFile},
ProviderError, ProviderNamespace, ProviderNode,
};
pub(super) struct KubernetesNodeOptions<'a, FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) namespace: &'a Weak<KubernetesNamespace<FS>>,
pub(super) namespace_base_dir: &'a PathBuf,
pub(super) name: &'a str,
pub(super) image: Option<&'a String>,
pub(super) program: &'a str,
pub(super) args: &'a [String],
pub(super) env: &'a [(String, String)],
pub(super) startup_files: &'a [TransferedFile],
pub(super) resources: Option<&'a Resources>,
pub(super) db_snapshot: Option<&'a AssetLocation>,
pub(super) k8s_client: &'a KubernetesClient,
pub(super) filesystem: &'a FS,
}
impl<'a, FS> KubernetesNodeOptions<'a, FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) fn from_deserializable(
deserializable: &'a DeserializableKubernetesNodeOptions,
namespace: &'a Weak<KubernetesNamespace<FS>>,
namespace_base_dir: &'a PathBuf,
k8s_client: &'a KubernetesClient,
filesystem: &'a FS,
) -> KubernetesNodeOptions<'a, FS> {
KubernetesNodeOptions {
namespace,
namespace_base_dir,
name: &deserializable.name,
image: deserializable.image.as_ref(),
program: &deserializable.program,
args: &deserializable.args,
env: &deserializable.env,
startup_files: &[],
resources: deserializable.resources.as_ref(),
db_snapshot: None,
k8s_client,
filesystem,
}
}
}
#[derive(Deserialize)]
pub(super) struct DeserializableKubernetesNodeOptions {
pub(super) name: String,
pub(super) image: Option<String>,
pub(super) program: String,
pub(super) args: Vec<String>,
pub(super) env: Vec<(String, String)>,
pub(super) resources: Option<Resources>,
}
type FwdInfo = (u16, JoinHandle<()>);
#[derive(Serialize)]
pub(super) struct KubernetesNode<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
#[serde(skip)]
namespace: Weak<KubernetesNamespace<FS>>,
name: String,
image: String,
program: String,
args: Vec<String>,
env: Vec<(String, String)>,
resources: Option<Resources>,
base_dir: PathBuf,
config_dir: PathBuf,
data_dir: PathBuf,
relay_data_dir: PathBuf,
scripts_dir: PathBuf,
log_path: PathBuf,
#[serde(skip)]
k8s_client: KubernetesClient,
#[serde(skip)]
http_client: reqwest::Client,
#[serde(skip)]
filesystem: FS,
#[serde(skip)]
port_fwds: RwLock<HashMap<u16, FwdInfo>>,
provider_tag: String,
}
impl<FS> KubernetesNode<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) async fn new(
options: KubernetesNodeOptions<'_, FS>,
) -> Result<Arc<Self>, ProviderError> {
let image = options.image.ok_or_else(|| {
ProviderError::MissingNodeInfo(options.name.to_string(), "missing image".to_string())
})?;
let filesystem = options.filesystem.clone();
let base_dir =
PathBuf::from_iter([options.namespace_base_dir, &PathBuf::from(options.name)]);
filesystem.create_dir_all(&base_dir).await?;
let base_dir_raw = base_dir.to_string_lossy();
let config_dir = PathBuf::from(format!("{base_dir_raw}{NODE_CONFIG_DIR}"));
let data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_DATA_DIR}"));
let relay_data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_RELAY_DATA_DIR}"));
let scripts_dir = PathBuf::from(format!("{base_dir_raw}{NODE_SCRIPTS_DIR}"));
let log_path = base_dir.join("node.log");
try_join!(
filesystem.create_dir(&config_dir),
filesystem.create_dir(&data_dir),
filesystem.create_dir(&relay_data_dir),
filesystem.create_dir(&scripts_dir),
)?;
let node = Arc::new(KubernetesNode {
namespace: options.namespace.clone(),
name: options.name.to_string(),
image: image.to_string(),
program: options.program.to_string(),
args: options.args.to_vec(),
env: options.env.to_vec(),
resources: options.resources.cloned(),
base_dir,
config_dir,
data_dir,
relay_data_dir,
scripts_dir,
log_path,
filesystem: filesystem.clone(),
k8s_client: options.k8s_client.clone(),
http_client: reqwest::Client::new(),
port_fwds: Default::default(),
provider_tag: kubernetes::provider::PROVIDER_NAME.to_string(),
});
node.initialize_k8s().await?;
if let Some(db_snap) = options.db_snapshot {
node.initialize_db_snapshot(db_snap).await?;
}
node.initialize_startup_files(options.startup_files).await?;
node.start().await?;
Ok(node)
}
pub(super) async fn attach_to_live(
options: KubernetesNodeOptions<'_, FS>,
) -> Result<Arc<Self>, ProviderError> {
let image = options.image.ok_or_else(|| {
ProviderError::MissingNodeInfo(options.name.to_string(), "missing image".to_string())
})?;
let filesystem = options.filesystem.clone();
let base_dir =
PathBuf::from_iter([options.namespace_base_dir, &PathBuf::from(options.name)]);
filesystem.create_dir_all(&base_dir).await?;
let base_dir_raw = base_dir.to_string_lossy();
let config_dir = PathBuf::from(format!("{base_dir_raw}{NODE_CONFIG_DIR}"));
let data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_DATA_DIR}"));
let relay_data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_RELAY_DATA_DIR}"));
let scripts_dir = PathBuf::from(format!("{base_dir_raw}{NODE_SCRIPTS_DIR}"));
let log_path = base_dir.join("node.log");
let node = Arc::new(KubernetesNode {
namespace: options.namespace.clone(),
name: options.name.to_string(),
image: image.to_string(),
program: options.program.to_string(),
args: options.args.to_vec(),
env: options.env.to_vec(),
resources: options.resources.cloned(),
base_dir,
config_dir,
data_dir,
relay_data_dir,
scripts_dir,
log_path,
filesystem: filesystem.clone(),
k8s_client: options.k8s_client.clone(),
http_client: reqwest::Client::new(),
port_fwds: Default::default(),
provider_tag: kubernetes::provider::PROVIDER_NAME.to_string(),
});
Ok(node)
}
async fn initialize_k8s(&self) -> Result<(), ProviderError> {
let labels = BTreeMap::from([
(
"app.kubernetes.io/name".to_string(),
self.name().to_string(),
),
(
"x-infra-instance".to_string(),
env::var("X_INFRA_INSTANCE").unwrap_or("ondemand".to_string()),
),
]);
// Create pod
let pod_spec = PodSpecBuilder::build(
&self.name,
&self.image,
self.resources.as_ref(),
&self.program,
&self.args,
&self.env,
);
let manifest = self
.k8s_client
.create_pod(&self.namespace_name(), &self.name, pod_spec, labels.clone())
.await
.map_err(|err| ProviderError::NodeSpawningFailed(self.name.clone(), err.into()))?;
let serialized_manifest = serde_yaml::to_string(&manifest)
.map_err(|err| ProviderError::NodeSpawningFailed(self.name.to_string(), err.into()))?;
let dest_path = PathBuf::from_iter([
&self.base_dir,
&PathBuf::from(format!("{}_manifest.yaml", &self.name)),
]);
self.filesystem
.write(dest_path, serialized_manifest)
.await
.map_err(|err| ProviderError::NodeSpawningFailed(self.name.to_string(), err.into()))?;
// Create service for pod
let service_spec = ServiceSpec {
selector: Some(labels.clone()),
ports: Some(vec![
ServicePort {
port: P2P_PORT.into(),
name: Some("p2p".into()),
..Default::default()
},
ServicePort {
port: RPC_WS_PORT.into(),
name: Some("rpc".into()),
..Default::default()
},
ServicePort {
port: RPC_HTTP_PORT.into(),
name: Some("rpc-http".into()),
..Default::default()
},
ServicePort {
port: PROMETHEUS_PORT.into(),
name: Some("prom".into()),
..Default::default()
},
]),
..Default::default()
};
let service_manifest = self
.k8s_client
.create_service(&self.namespace_name(), &self.name, service_spec, labels)
.await
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
let serialized_service_manifest = serde_yaml::to_string(&service_manifest)
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
let service_dest_path = PathBuf::from_iter([
&self.base_dir,
&PathBuf::from(format!("{}_svc_manifest.yaml", &self.name)),
]);
self.filesystem
.write(service_dest_path, serialized_service_manifest)
.await?;
Ok(())
}
async fn initialize_db_snapshot(
&self,
db_snapshot: &AssetLocation,
) -> Result<(), ProviderError> {
trace!("snap: {db_snapshot}");
let url_of_snap = match db_snapshot {
AssetLocation::Url(location) => location.clone(),
AssetLocation::FilePath(filepath) => {
let (url, _) = self.upload_to_fileserver(filepath).await?;
url
},
};
// we need to get the snapshot from a public access
// and extract to /data
let opts = RunCommandOptions::new("mkdir").args([
"-p",
"/data/",
"&&",
"mkdir",
"-p",
"/relay-data/",
"&&",
// Use our version of curl
"/cfg/curl",
url_of_snap.as_ref(),
"--output",
"/data/db.tgz",
"&&",
"cd",
"/",
"&&",
"tar",
"--skip-old-files",
"-xzvf",
"/data/db.tgz",
]);
trace!("cmd opts: {:#?}", opts);
let _ = self.run_command(opts).await?;
Ok(())
}
async fn initialize_startup_files(
&self,
startup_files: &[TransferedFile],
) -> Result<(), ProviderError> {
try_join_all(
startup_files
.iter()
.map(|file| self.send_file(&file.local_path, &file.remote_path, &file.mode)),
)
.await?;
Ok(())
}
pub(super) async fn start(&self) -> Result<(), ProviderError> {
self.k8s_client
.pod_exec(
&self.namespace_name(),
&self.name,
vec!["sh", "-c", "echo start > /tmp/zombiepipe"],
)
.await
.map_err(|err| {
ProviderError::NodeSpawningFailed(
format!("failed to start pod {} after spawning", self.name),
err.into(),
)
})?
.map_err(|err| {
ProviderError::NodeSpawningFailed(
format!("failed to start pod {} after spawning", self.name,),
anyhow!("command failed in container: status {}: {}", err.0, err.1),
)
})?;
Ok(())
}
fn get_remote_parent_dir(&self, remote_file_path: &Path) -> Option<PathBuf> {
if let Some(remote_parent_dir) = remote_file_path.parent() {
if matches!(
remote_parent_dir.components().rev().peekable().peek(),
Some(Component::Normal(_))
) {
return Some(remote_parent_dir.to_path_buf());
}
}
None
}
async fn create_remote_dir(&self, remote_dir: &Path) -> Result<(), ProviderError> {
let _ = self
.k8s_client
.pod_exec(
&self.namespace_name(),
&self.name,
vec!["mkdir", "-p", &remote_dir.to_string_lossy()],
)
.await
.map_err(|err| {
ProviderError::NodeSpawningFailed(
format!(
"failed to create dir {} for pod {}",
remote_dir.to_string_lossy(),
&self.name
),
err.into(),
)
})?;
Ok(())
}
fn namespace_name(&self) -> String {
self.namespace
.upgrade()
.map(|namespace| namespace.name().to_string())
.unwrap_or_else(|| panic!("namespace shouldn't be dropped, {THIS_IS_A_BUG}"))
}
async fn upload_to_fileserver(&self, location: &Path) -> Result<(Url, String), ProviderError> {
let file_name = if let Some(name) = location.file_name() {
name.to_string_lossy()
} else {
"unnamed".into()
};
let data = self.filesystem.read(location).await?;
let content_hashed = hex::encode(sha2::Sha256::digest(&data));
let req = self
.http_client
.head(format!(
"http://{}/{content_hashed}__{file_name}",
self.file_server_local_host().await?
))
.build()
.map_err(|err| {
ProviderError::UploadFile(location.to_string_lossy().to_string(), err.into())
})?;
let url = req.url().clone();
let res = self.http_client.execute(req).await.map_err(|err| {
ProviderError::UploadFile(location.to_string_lossy().to_string(), err.into())
})?;
if res.status() != reqwest::StatusCode::OK {
// we need to upload the file
self.http_client
.post(url.as_ref())
.body(data)
.send()
.await
.map_err(|err| {
ProviderError::UploadFile(location.to_string_lossy().to_string(), err.into())
})?;
}
Ok((url, content_hashed))
}
async fn file_server_local_host(&self) -> Result<String, ProviderError> {
if let Some(namespace) = self.namespace.upgrade() {
if let Some(port) = *namespace.file_server_port.read().await {
return Ok(format!("localhost:{port}"));
}
}
Err(ProviderError::FileServerSetupError(anyhow!(
"file server port not bound locally"
)))
}
async fn download_file(
&self,
url: &str,
remote_file_path: &Path,
hash: Option<&str>,
) -> Result<(), ProviderError> {
let r = self
.k8s_client
.pod_exec(
&self.namespace_name(),
&self.name,
vec![
"/cfg/curl",
url,
"--output",
&remote_file_path.to_string_lossy(),
],
)
.await
.map_err(|err| {
ProviderError::DownloadFile(
remote_file_path.to_string_lossy().to_string(),
anyhow!(format!("node: {}, err: {}", self.name(), err)),
)
})?;
trace!("download url {} result: {:?}", url, r);
if r.is_err() {
return Err(ProviderError::DownloadFile(
remote_file_path.to_string_lossy().to_string(),
anyhow!(format!("node: {}, err downloading file", self.name())),
));
}
if let Some(hash) = hash {
// check if the hash of the file is correct
let res = self
.k8s_client
.pod_exec(
&self.namespace_name(),
&self.name,
vec![
"/cfg/coreutils",
"sha256sum",
&remote_file_path.to_string_lossy(),
],
)
.await
.map_err(|err| {
ProviderError::DownloadFile(
remote_file_path.to_string_lossy().to_string(),
anyhow!(format!("node: {}, err: {}", self.name(), err)),
)
})?;
if let Ok(output) = res {
if !output.contains(hash) {
return Err(ProviderError::DownloadFile(
remote_file_path.to_string_lossy().to_string(),
anyhow!(format!("node: {}, invalid sha256sum hash: {hash} for file, output was {output}", self.name())),
));
}
} else {
return Err(ProviderError::DownloadFile(
remote_file_path.to_string_lossy().to_string(),
anyhow!(format!(
"node: {}, err calculating sha256sum for file {:?}",
self.name(),
res
)),
));
}
}
Ok(())
}
}
#[async_trait]
impl<FS> ProviderNode for KubernetesNode<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
fn name(&self) -> &str {
&self.name
}
fn args(&self) -> Vec<&str> {
self.args.iter().map(|arg| arg.as_str()).collect()
}
fn base_dir(&self) -> &PathBuf {
&self.base_dir
}
fn config_dir(&self) -> &PathBuf {
&self.config_dir
}
fn data_dir(&self) -> &PathBuf {
&self.data_dir
}
fn relay_data_dir(&self) -> &PathBuf {
&self.relay_data_dir
}
fn scripts_dir(&self) -> &PathBuf {
&self.scripts_dir
}
fn log_path(&self) -> &PathBuf {
&self.log_path
}
fn log_cmd(&self) -> String {
format!("kubectl -n {} logs {}", self.namespace_name(), self.name)
}
fn path_in_node(&self, file: &Path) -> PathBuf {
// here is just a noop op since we will receive the path
// for the file inside the pod
PathBuf::from(file)
}
// TODO: handle log rotation as we do in v1
async fn logs(&self) -> Result<String, ProviderError> {
self.k8s_client
.pod_logs(&self.namespace_name(), &self.name)
.await
.map_err(|err| ProviderError::GetLogsFailed(self.name.to_string(), err.into()))
}
async fn dump_logs(&self, local_dest: PathBuf) -> Result<(), ProviderError> {
let logs = self.logs().await?;
self.filesystem
.write(local_dest, logs)
.await
.map_err(|err| ProviderError::DumpLogsFailed(self.name.to_string(), err.into()))?;
Ok(())
}
async fn create_port_forward(
&self,
local_port: u16,
remote_port: u16,
) -> Result<Option<u16>, ProviderError> {
// If the fwd exist just return the local port
if let Some(fwd_info) = self.port_fwds.read().await.get(&remote_port) {
return Ok(Some(fwd_info.0));
};
let (port, task) = self
.k8s_client
.create_pod_port_forward(&self.namespace_name(), &self.name, local_port, remote_port)
.await
.map_err(|err| ProviderError::PortForwardError(local_port, remote_port, err.into()))?;
self.port_fwds
.write()
.await
.insert(remote_port, (port, task));
Ok(Some(port))
}
async fn run_command(
&self,
options: RunCommandOptions,
) -> Result<ExecutionResult, ProviderError> {
let mut command = vec![];
for (name, value) in options.env {
command.push(format!("export {name}={value};"));
}
command.push(options.program);
for arg in options.args {
command.push(arg);
}
self.k8s_client
.pod_exec(
&self.namespace_name(),
&self.name,
vec!["sh", "-c", &command.join(" ")],
)
.await
.map_err(|err| {
ProviderError::RunCommandError(
format!("sh -c {}", &command.join(" ")),
format!("in pod {}", self.name),
err.into(),
)
})
}
async fn run_script(
&self,
options: RunScriptOptions,
) -> Result<ExecutionResult, ProviderError> {
let file_name = options
.local_script_path
.file_name()
.expect(&format!(
"file name should be present at this point {THIS_IS_A_BUG}"
))
.to_string_lossy();
self.run_command(RunCommandOptions {
program: format!("/tmp/{file_name}"),
args: options.args,
env: options.env,
})
.await
.map_err(|err| ProviderError::RunScriptError(self.name.to_string(), err.into()))
}
async fn send_file(
&self,
local_file_path: &Path,
remote_file_path: &Path,
mode: &str,
) -> Result<(), ProviderError> {
if let Some(remote_parent_dir) = self.get_remote_parent_dir(remote_file_path) {
self.create_remote_dir(&remote_parent_dir).await?;
}
debug!(
"Uploading file: {} IFF not present in the fileserver",
local_file_path.to_string_lossy()
);
// we need to override the url to use inside the pod
let (mut url, hash) = self.upload_to_fileserver(local_file_path).await?;
let _ = url.set_host(Some("fileserver"));
let _ = url.set_port(Some(80));
// Sometimes downloading the file fails (the file is corrupted)
// Add at most 5 retries
let mut last_err = None;
for i in 0..5 {
if i > 0 {
warn!("retrying number {i} download file {:?}", remote_file_path);
tokio::time::sleep(Duration::from_secs(i)).await;
}
let res = self
.download_file(url.as_ref(), remote_file_path, Some(&hash))
.await;
last_err = res.err();
if last_err.is_none() {
// ready to continue
break;
}
}
if let Some(last_err) = last_err {
return Err(last_err);
}
let _ = self
.k8s_client
.pod_exec(
&self.namespace_name(),
&self.name,
vec!["chmod", mode, &remote_file_path.to_string_lossy()],
)
.await
.map_err(|err| {
ProviderError::SendFile(
self.name.clone(),
local_file_path.to_string_lossy().to_string(),
err.into(),
)
})?;
Ok(())
}
async fn receive_file(
&self,
_remote_src: &Path,
_local_dest: &Path,
) -> Result<(), ProviderError> {
Ok(())
}
async fn ip(&self) -> Result<IpAddr, ProviderError> {
let status = self
.k8s_client
.pod_status(&self.namespace_name(), &self.name)
.await
.map_err(|_| ProviderError::MissingNode(self.name.clone()))?;
if let Some(ip) = status.pod_ip {
// Pod ip should be parseable
Ok(ip.parse::<IpAddr>().map_err(|err| {
ProviderError::InvalidConfig(format!("Can not parse the pod ip: {ip}, err: {err}"))
})?)
} else {
Err(ProviderError::InvalidConfig(format!(
"Can not find ip of pod: {}",
self.name()
)))
}
}
async fn pause(&self) -> Result<(), ProviderError> {
self.k8s_client
.pod_exec(
&self.namespace_name(),
&self.name,
vec!["sh", "-c", "echo pause > /tmp/zombiepipe"],
)
.await
.map_err(|err| ProviderError::PauseNodeFailed(self.name.to_string(), err.into()))?
.map_err(|err| {
ProviderError::PauseNodeFailed(
self.name.to_string(),
anyhow!("error when pausing node: status {}: {}", err.0, err.1),
)
})?;
Ok(())
}
async fn resume(&self) -> Result<(), ProviderError> {
self.k8s_client
.pod_exec(
&self.namespace_name(),
&self.name,
vec!["sh", "-c", "echo resume > /tmp/zombiepipe"],
)
.await
.map_err(|err| ProviderError::ResumeNodeFailed(self.name.to_string(), err.into()))?
.map_err(|err| {
ProviderError::ResumeNodeFailed(
self.name.to_string(),
anyhow!("error when pausing node: status {}: {}", err.0, err.1),
)
})?;
Ok(())
}
async fn restart(&self, after: Option<Duration>) -> Result<(), ProviderError> {
if let Some(duration) = after {
sleep(duration).await;
}
self.k8s_client
.pod_exec(
&self.namespace_name(),
&self.name,
vec!["sh", "-c", "echo restart > /tmp/zombiepipe"],
)
.await
.map_err(|err| ProviderError::RestartNodeFailed(self.name.to_string(), err.into()))?
.map_err(|err| {
ProviderError::RestartNodeFailed(
self.name.to_string(),
anyhow!("error when restarting node: status {}: {}", err.0, err.1),
)
})?;
Ok(())
}
async fn destroy(&self) -> Result<(), ProviderError> {
self.k8s_client
.delete_pod(&self.namespace_name(), &self.name)
.await
.map_err(|err| ProviderError::KillNodeFailed(self.name.to_string(), err.into()))?;
if let Some(namespace) = self.namespace.upgrade() {
namespace.nodes.write().await.remove(&self.name);
}
Ok(())
}
}
@@ -0,0 +1,188 @@
use std::collections::BTreeMap;
use configuration::shared::resources::{ResourceQuantity, Resources};
use k8s_openapi::{
api::core::v1::{
ConfigMapVolumeSource, Container, EnvVar, PodSpec, ResourceRequirements, Volume,
VolumeMount,
},
apimachinery::pkg::api::resource::Quantity,
};
pub(super) struct PodSpecBuilder;
impl PodSpecBuilder {
pub(super) fn build(
name: &str,
image: &str,
resources: Option<&Resources>,
program: &str,
args: &[String],
env: &[(String, String)],
) -> PodSpec {
PodSpec {
hostname: Some(name.to_string()),
init_containers: Some(vec![Self::build_helper_binaries_setup_container()]),
containers: vec![Self::build_main_container(
name, image, resources, program, args, env,
)],
volumes: Some(Self::build_volumes()),
..Default::default()
}
}
fn build_main_container(
name: &str,
image: &str,
resources: Option<&Resources>,
program: &str,
args: &[String],
env: &[(String, String)],
) -> Container {
Container {
name: name.to_string(),
image: Some(image.to_string()),
image_pull_policy: Some("Always".to_string()),
command: Some(
[
vec!["/zombie-wrapper.sh".to_string(), program.to_string()],
args.to_vec(),
]
.concat(),
),
env: Some(
env.iter()
.map(|(name, value)| EnvVar {
name: name.clone(),
value: Some(value.clone()),
value_from: None,
})
.collect(),
),
volume_mounts: Some(Self::build_volume_mounts(vec![VolumeMount {
name: "zombie-wrapper-volume".to_string(),
mount_path: "/zombie-wrapper.sh".to_string(),
sub_path: Some("zombie-wrapper.sh".to_string()),
..Default::default()
}])),
resources: Self::build_resources_requirements(resources),
..Default::default()
}
}
fn build_helper_binaries_setup_container() -> Container {
Container {
name: "helper-binaries-setup".to_string(),
image: Some("europe-west3-docker.pkg.dev/parity-zombienet/zombienet-public-images/alpine:latest".to_string()),
image_pull_policy: Some("IfNotPresent".to_string()),
volume_mounts: Some(Self::build_volume_mounts(vec![VolumeMount {
name: "helper-binaries-downloader-volume".to_string(),
mount_path: "/helper-binaries-downloader.sh".to_string(),
sub_path: Some("helper-binaries-downloader.sh".to_string()),
..Default::default()
}])),
command: Some(vec![
"ash".to_string(),
"/helper-binaries-downloader.sh".to_string(),
]),
..Default::default()
}
}
fn build_volumes() -> Vec<Volume> {
vec![
Volume {
name: "cfg".to_string(),
..Default::default()
},
Volume {
name: "data".to_string(),
..Default::default()
},
Volume {
name: "relay-data".to_string(),
..Default::default()
},
Volume {
name: "zombie-wrapper-volume".to_string(),
config_map: Some(ConfigMapVolumeSource {
name: Some("zombie-wrapper".to_string()),
default_mode: Some(0o755),
..Default::default()
}),
..Default::default()
},
Volume {
name: "helper-binaries-downloader-volume".to_string(),
config_map: Some(ConfigMapVolumeSource {
name: Some("helper-binaries-downloader".to_string()),
default_mode: Some(0o755),
..Default::default()
}),
..Default::default()
},
]
}
fn build_volume_mounts(non_default_mounts: Vec<VolumeMount>) -> Vec<VolumeMount> {
[
vec![
VolumeMount {
name: "cfg".to_string(),
mount_path: "/cfg".to_string(),
read_only: Some(false),
..Default::default()
},
VolumeMount {
name: "data".to_string(),
mount_path: "/data".to_string(),
read_only: Some(false),
..Default::default()
},
VolumeMount {
name: "relay-data".to_string(),
mount_path: "/relay-data".to_string(),
read_only: Some(false),
..Default::default()
},
],
non_default_mounts,
]
.concat()
}
fn build_resources_requirements(resources: Option<&Resources>) -> Option<ResourceRequirements> {
resources.map(|resources| ResourceRequirements {
limits: Self::build_resources_requirements_quantities(
resources.limit_cpu(),
resources.limit_memory(),
),
requests: Self::build_resources_requirements_quantities(
resources.request_cpu(),
resources.request_memory(),
),
..Default::default()
})
}
fn build_resources_requirements_quantities(
cpu: Option<&ResourceQuantity>,
memory: Option<&ResourceQuantity>,
) -> Option<BTreeMap<String, Quantity>> {
let mut quantities = BTreeMap::new();
if let Some(cpu) = cpu {
quantities.insert("cpu".to_string(), Quantity(cpu.as_str().to_string()));
}
if let Some(memory) = memory {
quantities.insert("memory".to_string(), Quantity(memory.as_str().to_string()));
}
if !quantities.is_empty() {
Some(quantities)
} else {
None
}
}
}
@@ -0,0 +1,145 @@
use std::{
collections::HashMap,
path::{Path, PathBuf},
sync::{Arc, Weak},
};
use async_trait::async_trait;
use support::fs::FileSystem;
use tokio::sync::RwLock;
use super::{client::KubernetesClient, namespace::KubernetesNamespace};
use crate::{
shared::helpers::extract_namespace_info, types::ProviderCapabilities, DynNamespace, Provider,
ProviderError, ProviderNamespace,
};
pub const PROVIDER_NAME: &str = "k8s";
pub struct KubernetesProvider<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
weak: Weak<KubernetesProvider<FS>>,
capabilities: ProviderCapabilities,
tmp_dir: PathBuf,
k8s_client: KubernetesClient,
filesystem: FS,
pub(super) namespaces: RwLock<HashMap<String, Arc<KubernetesNamespace<FS>>>>,
}
impl<FS> KubernetesProvider<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
pub async fn new(filesystem: FS) -> Arc<Self> {
let k8s_client = KubernetesClient::new().await.unwrap();
Arc::new_cyclic(|weak| KubernetesProvider {
weak: weak.clone(),
capabilities: ProviderCapabilities {
requires_image: true,
has_resources: true,
prefix_with_full_path: false,
use_default_ports_in_cmd: true,
},
tmp_dir: std::env::temp_dir(),
k8s_client,
filesystem,
namespaces: RwLock::new(HashMap::new()),
})
}
pub fn tmp_dir(mut self, tmp_dir: impl Into<PathBuf>) -> Self {
self.tmp_dir = tmp_dir.into();
self
}
}
#[async_trait]
impl<FS> Provider for KubernetesProvider<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
fn name(&self) -> &str {
PROVIDER_NAME
}
fn capabilities(&self) -> &ProviderCapabilities {
&self.capabilities
}
async fn namespaces(&self) -> HashMap<String, DynNamespace> {
self.namespaces
.read()
.await
.iter()
.map(|(name, namespace)| (name.clone(), namespace.clone() as DynNamespace))
.collect()
}
async fn create_namespace(&self) -> Result<DynNamespace, ProviderError> {
let namespace = KubernetesNamespace::new(
&self.weak,
&self.tmp_dir,
&self.capabilities,
&self.k8s_client,
&self.filesystem,
None,
)
.await?;
self.namespaces
.write()
.await
.insert(namespace.name().to_string(), namespace.clone());
Ok(namespace)
}
async fn create_namespace_with_base_dir(
&self,
base_dir: &Path,
) -> Result<DynNamespace, ProviderError> {
let namespace = KubernetesNamespace::new(
&self.weak,
&self.tmp_dir,
&self.capabilities,
&self.k8s_client,
&self.filesystem,
Some(base_dir),
)
.await?;
self.namespaces
.write()
.await
.insert(namespace.name().to_string(), namespace.clone());
Ok(namespace)
}
async fn create_namespace_from_json(
&self,
json_value: &serde_json::Value,
) -> Result<DynNamespace, ProviderError> {
let (base_dir, name) = extract_namespace_info(json_value)?;
let namespace = KubernetesNamespace::attach_to_live(
&self.weak,
&self.capabilities,
&self.k8s_client,
&self.filesystem,
&base_dir,
&name,
)
.await?;
self.namespaces
.write()
.await
.insert(namespace.name().to_string(), namespace.clone());
Ok(namespace)
}
}
@@ -0,0 +1,10 @@
apiVersion: v1
kind: LimitRange
metadata:
name: mem-limit-range
spec:
limits:
- defaultRequest:
memory: 1G
cpu: 0.5
type: Container
@@ -0,0 +1,23 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: internal-access
spec:
podSelector: {}
ingress:
- from:
- namespaceSelector:
matchExpressions:
- key: kubernetes.io/metadata.name
operator: In
values:
- {{namespace}}
- gitlab
- arc-runner
- loki
- tempo
- monitoring
- parachain-exporter
- default
policyTypes:
- Ingress
+264
View File
@@ -0,0 +1,264 @@
#![allow(clippy::expect_fun_call)]
mod docker;
mod kubernetes;
mod native;
pub mod shared;
use std::{
collections::HashMap,
net::IpAddr,
path::{Path, PathBuf},
sync::Arc,
time::Duration,
};
use async_trait::async_trait;
use shared::{
constants::LOCALHOST,
types::{
ExecutionResult, GenerateFilesOptions, ProviderCapabilities, RunCommandOptions,
RunScriptOptions, SpawnNodeOptions,
},
};
use support::fs::FileSystemError;
#[derive(Debug, thiserror::Error)]
#[allow(missing_docs)]
pub enum ProviderError {
#[error("Failed to create client '{0}': {1}")]
CreateClientFailed(String, anyhow::Error),
#[error("Failed to create namespace '{0}': {1}")]
CreateNamespaceFailed(String, anyhow::Error),
#[error("Failed to spawn node '{0}': {1}")]
NodeSpawningFailed(String, anyhow::Error),
#[error("Error running command '{0}' {1}: {2}")]
RunCommandError(String, String, anyhow::Error),
#[error("Error running script'{0}': {1}")]
RunScriptError(String, anyhow::Error),
#[error("Invalid network configuration field {0}")]
InvalidConfig(String),
#[error("Failed to retrieve node available args using image {0} and command {1}: {2}")]
NodeAvailableArgsError(String, String, String),
#[error("Can not recover node: {0}")]
MissingNode(String),
#[error("Can not recover node: {0} info, field: {1}")]
MissingNodeInfo(String, String),
#[error("File generation failed: {0}")]
FileGenerationFailed(anyhow::Error),
#[error(transparent)]
FileSystemError(#[from] FileSystemError),
#[error("Invalid script path for {0}")]
InvalidScriptPath(anyhow::Error),
#[error("Script with path {0} not found")]
ScriptNotFound(PathBuf),
#[error("Failed to retrieve process ID for node '{0}'")]
ProcessIdRetrievalFailed(String),
#[error("Failed to pause node '{0}': {1}")]
PauseNodeFailed(String, anyhow::Error),
#[error("Failed to resume node '{0}': {1}")]
ResumeNodeFailed(String, anyhow::Error),
#[error("Failed to kill node '{0}': {1}")]
KillNodeFailed(String, anyhow::Error),
#[error("Failed to restart node '{0}': {1}")]
RestartNodeFailed(String, anyhow::Error),
#[error("Failed to destroy node '{0}': {1}")]
DestroyNodeFailed(String, anyhow::Error),
#[error("Failed to get logs for node '{0}': {1}")]
GetLogsFailed(String, anyhow::Error),
#[error("Failed to dump logs for node '{0}': {1}")]
DumpLogsFailed(String, anyhow::Error),
#[error("Failed to copy file from node '{0}': {1}")]
CopyFileFromNodeError(String, anyhow::Error),
#[error("Failed to setup fileserver: {0}")]
FileServerSetupError(anyhow::Error),
#[error("Error uploading file: '{0}': {1}")]
UploadFile(String, anyhow::Error),
#[error("Error downloading file: '{0}': {1}")]
DownloadFile(String, anyhow::Error),
#[error("Error sending file '{0}' to {1}: {2}")]
SendFile(String, String, anyhow::Error),
#[error("Error creating port-forward '{0}:{1}': {2}")]
PortForwardError(u16, u16, anyhow::Error),
#[error("Failed to delete namespace '{0}': {1}")]
DeleteNamespaceFailed(String, anyhow::Error),
#[error("Serialization error")]
SerializationError(#[from] serde_json::Error),
#[error("Failed to acquire lock: {0}")]
FailedToAcquireLock(String),
}
#[async_trait]
pub trait Provider {
fn name(&self) -> &str;
fn capabilities(&self) -> &ProviderCapabilities;
async fn namespaces(&self) -> HashMap<String, DynNamespace>;
async fn create_namespace(&self) -> Result<DynNamespace, ProviderError>;
async fn create_namespace_with_base_dir(
&self,
base_dir: &Path,
) -> Result<DynNamespace, ProviderError>;
async fn create_namespace_from_json(
&self,
json_value: &serde_json::Value,
) -> Result<DynNamespace, ProviderError>;
}
pub type DynProvider = Arc<dyn Provider + Send + Sync>;
#[async_trait]
pub trait ProviderNamespace {
fn name(&self) -> &str;
fn base_dir(&self) -> &PathBuf;
fn capabilities(&self) -> &ProviderCapabilities;
fn provider_name(&self) -> &str;
async fn detach(&self) {
// noop by default
warn!("Detach is not implemented for {}", self.name());
}
async fn is_detached(&self) -> bool {
// false by default
false
}
async fn nodes(&self) -> HashMap<String, DynNode>;
async fn get_node_available_args(
&self,
options: (String, Option<String>),
) -> Result<String, ProviderError>;
async fn spawn_node(&self, options: &SpawnNodeOptions) -> Result<DynNode, ProviderError>;
async fn spawn_node_from_json(
&self,
json_value: &serde_json::Value,
) -> Result<DynNode, ProviderError>;
async fn generate_files(&self, options: GenerateFilesOptions) -> Result<(), ProviderError>;
async fn destroy(&self) -> Result<(), ProviderError>;
async fn static_setup(&self) -> Result<(), ProviderError>;
}
pub type DynNamespace = Arc<dyn ProviderNamespace + Send + Sync>;
#[async_trait]
pub trait ProviderNode: erased_serde::Serialize {
fn name(&self) -> &str;
fn args(&self) -> Vec<&str>;
fn base_dir(&self) -> &PathBuf;
fn config_dir(&self) -> &PathBuf;
fn data_dir(&self) -> &PathBuf;
fn relay_data_dir(&self) -> &PathBuf;
fn scripts_dir(&self) -> &PathBuf;
fn log_path(&self) -> &PathBuf;
fn log_cmd(&self) -> String;
// Return the absolute path to the file in the `node` perspective
// TODO: purpose?
fn path_in_node(&self, file: &Path) -> PathBuf;
async fn logs(&self) -> Result<String, ProviderError>;
async fn dump_logs(&self, local_dest: PathBuf) -> Result<(), ProviderError>;
// By default return localhost, should be overrided for k8s
async fn ip(&self) -> Result<IpAddr, ProviderError> {
Ok(LOCALHOST)
}
// Noop by default (native/docker provider)
async fn create_port_forward(
&self,
_local_port: u16,
_remote_port: u16,
) -> Result<Option<u16>, ProviderError> {
Ok(None)
}
async fn run_command(
&self,
options: RunCommandOptions,
) -> Result<ExecutionResult, ProviderError>;
async fn run_script(&self, options: RunScriptOptions)
-> Result<ExecutionResult, ProviderError>;
async fn send_file(
&self,
local_file_path: &Path,
remote_file_path: &Path,
mode: &str,
) -> Result<(), ProviderError>;
async fn receive_file(
&self,
remote_file_path: &Path,
local_file_path: &Path,
) -> Result<(), ProviderError>;
async fn pause(&self) -> Result<(), ProviderError>;
async fn resume(&self) -> Result<(), ProviderError>;
async fn restart(&self, after: Option<Duration>) -> Result<(), ProviderError>;
async fn destroy(&self) -> Result<(), ProviderError>;
}
pub type DynNode = Arc<dyn ProviderNode + Send + Sync>;
// re-export
pub use docker::*;
pub use kubernetes::*;
pub use native::*;
pub use shared::{constants, types};
use tracing::warn;
@@ -0,0 +1,5 @@
mod namespace;
mod node;
mod provider;
pub use provider::NativeProvider;
@@ -0,0 +1,374 @@
use std::{
collections::HashMap,
path::{Path, PathBuf},
sync::{Arc, Weak},
};
use async_trait::async_trait;
use support::fs::FileSystem;
use tokio::sync::RwLock;
use tracing::{trace, warn};
use uuid::Uuid;
use super::node::{NativeNode, NativeNodeOptions};
use crate::{
constants::NAMESPACE_PREFIX,
native::{node::DeserializableNativeNodeOptions, provider},
shared::helpers::extract_execution_result,
types::{
GenerateFileCommand, GenerateFilesOptions, ProviderCapabilities, RunCommandOptions,
SpawnNodeOptions,
},
DynNode, NativeProvider, ProviderError, ProviderNamespace, ProviderNode,
};
pub(super) struct NativeNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
weak: Weak<NativeNamespace<FS>>,
name: String,
provider: Weak<NativeProvider<FS>>,
base_dir: PathBuf,
capabilities: ProviderCapabilities,
filesystem: FS,
pub(super) nodes: RwLock<HashMap<String, Arc<NativeNode<FS>>>>,
}
impl<FS> NativeNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) async fn new(
provider: &Weak<NativeProvider<FS>>,
tmp_dir: &PathBuf,
capabilities: &ProviderCapabilities,
filesystem: &FS,
custom_base_dir: Option<&Path>,
) -> Result<Arc<Self>, ProviderError> {
let name = format!("{}{}", NAMESPACE_PREFIX, Uuid::new_v4());
let base_dir = if let Some(custom_base_dir) = custom_base_dir {
if !filesystem.exists(custom_base_dir).await {
filesystem.create_dir_all(custom_base_dir).await?;
} else {
warn!(
"⚠️ Using and existing directory {} as base dir",
custom_base_dir.to_string_lossy()
);
}
PathBuf::from(custom_base_dir)
} else {
let base_dir = PathBuf::from_iter([tmp_dir, &PathBuf::from(&name)]);
filesystem.create_dir(&base_dir).await?;
base_dir
};
Ok(Arc::new_cyclic(|weak| NativeNamespace {
weak: weak.clone(),
provider: provider.clone(),
name,
base_dir,
capabilities: capabilities.clone(),
filesystem: filesystem.clone(),
nodes: RwLock::new(HashMap::new()),
}))
}
pub(super) async fn attach_to_live(
provider: &Weak<NativeProvider<FS>>,
capabilities: &ProviderCapabilities,
filesystem: &FS,
custom_base_dir: &Path,
name: &str,
) -> Result<Arc<Self>, ProviderError> {
let base_dir = custom_base_dir.to_path_buf();
Ok(Arc::new_cyclic(|weak| NativeNamespace {
weak: weak.clone(),
provider: provider.clone(),
name: name.to_string(),
base_dir,
capabilities: capabilities.clone(),
filesystem: filesystem.clone(),
nodes: RwLock::new(HashMap::new()),
}))
}
}
#[async_trait]
impl<FS> ProviderNamespace for NativeNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
fn name(&self) -> &str {
&self.name
}
fn base_dir(&self) -> &PathBuf {
&self.base_dir
}
fn capabilities(&self) -> &ProviderCapabilities {
&self.capabilities
}
fn provider_name(&self) -> &str {
provider::PROVIDER_NAME
}
async fn nodes(&self) -> HashMap<String, DynNode> {
self.nodes
.read()
.await
.iter()
.map(|(name, node)| (name.clone(), node.clone() as DynNode))
.collect()
}
async fn get_node_available_args(
&self,
(command, _image): (String, Option<String>),
) -> Result<String, ProviderError> {
let temp_node = self
.spawn_node(
&SpawnNodeOptions::new(format!("temp-{}", Uuid::new_v4()), "bash".to_string())
.args(vec!["-c", "while :; do sleep 1; done"]),
)
.await?;
let available_args_output = temp_node
.run_command(RunCommandOptions::new(command.clone()).args(vec!["--help"]))
.await?
.map_err(|(_exit, status)| {
ProviderError::NodeAvailableArgsError("".to_string(), command, status)
})?;
temp_node.destroy().await?;
Ok(available_args_output)
}
async fn spawn_node(&self, options: &SpawnNodeOptions) -> Result<DynNode, ProviderError> {
trace!("spawn node options {options:?}");
let node = NativeNode::new(NativeNodeOptions {
namespace: &self.weak,
namespace_base_dir: &self.base_dir,
name: &options.name,
program: &options.program,
args: &options.args,
env: &options.env,
startup_files: &options.injected_files,
created_paths: &options.created_paths,
db_snapshot: options.db_snapshot.as_ref(),
filesystem: &self.filesystem,
node_log_path: options.node_log_path.as_ref(),
})
.await?;
self.nodes
.write()
.await
.insert(options.name.clone(), node.clone());
Ok(node)
}
async fn spawn_node_from_json(
&self,
json_value: &serde_json::Value,
) -> Result<DynNode, ProviderError> {
let deserializable: DeserializableNativeNodeOptions =
serde_json::from_value(json_value.clone())?;
let options = NativeNodeOptions::from_deserializable(
&deserializable,
&self.weak,
&self.base_dir,
&self.filesystem,
);
let pid = json_value
.get("process_handle")
.and_then(|v| v.as_i64())
.ok_or_else(|| ProviderError::InvalidConfig("Missing pid field".to_string()))?
as i32;
let node = NativeNode::attach_to_live(options, pid).await?;
self.nodes
.write()
.await
.insert(node.name().to_string(), node.clone());
Ok(node)
}
async fn generate_files(&self, options: GenerateFilesOptions) -> Result<(), ProviderError> {
let node_name = if let Some(name) = options.temp_name {
name
} else {
format!("temp-{}", Uuid::new_v4())
};
// we spawn a node doing nothing but looping so we can execute our commands
let temp_node = self
.spawn_node(
&SpawnNodeOptions::new(node_name, "bash".to_string())
.args(vec!["-c", "while :; do sleep 1; done"])
.injected_files(options.injected_files),
)
.await?;
for GenerateFileCommand {
program,
args,
env,
local_output_path,
} in options.commands
{
trace!(
"🏗 building file {:?} in path {} with command {} {}",
local_output_path.as_os_str(),
self.base_dir.to_string_lossy(),
program,
args.join(" ")
);
let local_output_full_path = format!(
"{}{}{}",
self.base_dir.to_string_lossy(),
if local_output_path.starts_with("/") {
""
} else {
"/"
},
local_output_path.to_string_lossy()
);
let contents = extract_execution_result(
&temp_node,
RunCommandOptions { program, args, env },
options.expected_path.as_ref(),
)
.await?;
self.filesystem
.write(local_output_full_path, contents)
.await
.map_err(|err| ProviderError::FileGenerationFailed(err.into()))?;
}
temp_node.destroy().await
}
async fn static_setup(&self) -> Result<(), ProviderError> {
// no static setup exists for native provider
todo!()
}
async fn destroy(&self) -> Result<(), ProviderError> {
let mut names = vec![];
for node in self.nodes.read().await.values() {
node.abort()
.await
.map_err(|err| ProviderError::DestroyNodeFailed(node.name().to_string(), err))?;
names.push(node.name().to_string());
}
let mut nodes = self.nodes.write().await;
for name in names {
nodes.remove(&name);
}
if let Some(provider) = self.provider.upgrade() {
provider.namespaces.write().await.remove(&self.name);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use support::fs::local::LocalFileSystem;
use super::*;
use crate::{
types::{GenerateFileCommand, GenerateFilesOptions},
NativeProvider, Provider,
};
fn unique_temp_dir() -> PathBuf {
let mut base = std::env::temp_dir();
base.push(format!("znet_native_ns_test_{}", uuid::Uuid::new_v4()));
base
}
#[tokio::test]
async fn generate_files_uses_expected_path_when_provided() {
let fs = LocalFileSystem;
let provider = NativeProvider::new(fs.clone());
let base_dir = unique_temp_dir();
// Namespace builder will create directory if needed
let ns = provider
.create_namespace_with_base_dir(&base_dir)
.await
.expect("namespace should be created");
// Create a unique on-host path that the native node will write to
let expected_path =
std::env::temp_dir().join(format!("znet_expected_{}.json", uuid::Uuid::new_v4()));
// Command will write JSON into expected_path; stdout will be something else to ensure we don't read it
let program = "bash".to_string();
let script = format!(
"echo -n '{{\"hello\":\"world\"}}' > {} && echo should_not_be_used",
expected_path.to_string_lossy()
);
let args: Vec<String> = vec!["-lc".into(), script];
let out_name = PathBuf::from("result_expected.json");
let cmd = GenerateFileCommand::new(program, out_name.clone()).args(args);
let options = GenerateFilesOptions::new(vec![cmd], None, Some(expected_path.clone()));
ns.generate_files(options)
.await
.expect("generation should succeed");
// Read produced file from namespace base_dir
let produced_path = base_dir.join(out_name);
let produced = fs
.read_to_string(&produced_path)
.await
.expect("should read produced file");
assert_eq!(produced, "{\"hello\":\"world\"}");
}
#[tokio::test]
async fn generate_files_uses_stdout_when_expected_path_absent() {
let fs = LocalFileSystem;
let provider = NativeProvider::new(fs.clone());
let base_dir = unique_temp_dir();
let ns = provider
.create_namespace_with_base_dir(&base_dir)
.await
.expect("namespace should be created");
// Command prints to stdout only
let program = "bash".to_string();
let args: Vec<String> = vec!["-lc".into(), "echo -n 42".into()];
let out_name = PathBuf::from("result_stdout.txt");
let cmd = GenerateFileCommand::new(program, out_name.clone()).args(args);
let options = GenerateFilesOptions::new(vec![cmd], None, None);
ns.generate_files(options)
.await
.expect("generation should succeed");
let produced_path = base_dir.join(out_name);
let produced = fs
.read_to_string(&produced_path)
.await
.expect("should read produced file");
assert_eq!(produced, "42");
}
}
@@ -0,0 +1,734 @@
use std::{
collections::HashMap,
env,
path::{Path, PathBuf},
process::Stdio,
sync::{Arc, Weak},
time::Duration,
};
use anyhow::anyhow;
use async_trait::async_trait;
use configuration::types::AssetLocation;
use flate2::read::GzDecoder;
use futures::future::try_join_all;
use nix::{
sys::signal::{kill, Signal},
unistd::Pid,
};
use serde::{ser::Error, Deserialize, Serialize, Serializer};
use sha2::Digest;
use support::{constants::THIS_IS_A_BUG, fs::FileSystem};
use tar::Archive;
use tokio::{
fs,
io::{AsyncRead, AsyncReadExt, BufReader},
process::{Child, ChildStderr, ChildStdout, Command},
sync::{
mpsc::{self, Sender},
RwLock,
},
task::JoinHandle,
time::sleep,
try_join,
};
use tracing::trace;
use super::namespace::NativeNamespace;
use crate::{
constants::{NODE_CONFIG_DIR, NODE_DATA_DIR, NODE_RELAY_DATA_DIR, NODE_SCRIPTS_DIR},
native,
types::{ExecutionResult, RunCommandOptions, RunScriptOptions, TransferedFile},
ProviderError, ProviderNamespace, ProviderNode,
};
pub(super) struct NativeNodeOptions<'a, FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) namespace: &'a Weak<NativeNamespace<FS>>,
pub(super) namespace_base_dir: &'a PathBuf,
pub(super) name: &'a str,
pub(super) program: &'a str,
pub(super) args: &'a [String],
pub(super) env: &'a [(String, String)],
pub(super) startup_files: &'a [TransferedFile],
pub(super) created_paths: &'a [PathBuf],
pub(super) db_snapshot: Option<&'a AssetLocation>,
pub(super) filesystem: &'a FS,
pub(super) node_log_path: Option<&'a PathBuf>,
}
impl<'a, FS> NativeNodeOptions<'a, FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) fn from_deserializable(
deserializable: &'a DeserializableNativeNodeOptions,
namespace: &'a Weak<NativeNamespace<FS>>,
namespace_base_dir: &'a PathBuf,
filesystem: &'a FS,
) -> NativeNodeOptions<'a, FS> {
NativeNodeOptions {
namespace,
namespace_base_dir,
name: &deserializable.name,
program: &deserializable.program,
args: &deserializable.args,
env: &deserializable.env,
startup_files: &[],
created_paths: &[],
db_snapshot: None,
filesystem,
node_log_path: deserializable.node_log_path.as_ref(),
}
}
}
#[derive(Deserialize)]
pub(super) struct DeserializableNativeNodeOptions {
pub name: String,
pub program: String,
pub args: Vec<String>,
pub env: Vec<(String, String)>,
pub node_log_path: Option<PathBuf>,
}
enum ProcessHandle {
Spawned(Child, Pid),
Attached(Pid),
}
#[derive(Serialize)]
pub(super) struct NativeNode<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
#[serde(skip)]
namespace: Weak<NativeNamespace<FS>>,
name: String,
program: String,
args: Vec<String>,
env: Vec<(String, String)>,
base_dir: PathBuf,
config_dir: PathBuf,
data_dir: PathBuf,
relay_data_dir: PathBuf,
scripts_dir: PathBuf,
log_path: PathBuf,
#[serde(serialize_with = "serialize_process_handle")]
// using RwLock from std to serialize properly, generally using sync locks is ok in async code as long as they
// are not held across await points
process_handle: std::sync::RwLock<Option<ProcessHandle>>,
#[serde(skip)]
stdout_reading_task: RwLock<Option<JoinHandle<()>>>,
#[serde(skip)]
stderr_reading_task: RwLock<Option<JoinHandle<()>>>,
#[serde(skip)]
log_writing_task: RwLock<Option<JoinHandle<()>>>,
#[serde(skip)]
filesystem: FS,
provider_tag: String,
}
impl<FS> NativeNode<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) async fn new(
options: NativeNodeOptions<'_, FS>,
) -> Result<Arc<Self>, ProviderError> {
let filesystem = options.filesystem.clone();
let base_dir =
PathBuf::from_iter([options.namespace_base_dir, &PathBuf::from(options.name)]);
trace!("creating base_dir {:?}", base_dir);
options.filesystem.create_dir_all(&base_dir).await?;
trace!("created base_dir {:?}", base_dir);
let base_dir_raw = base_dir.to_string_lossy();
let config_dir = PathBuf::from(format!("{base_dir_raw}{NODE_CONFIG_DIR}"));
let data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_DATA_DIR}"));
let relay_data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_RELAY_DATA_DIR}"));
let scripts_dir = PathBuf::from(format!("{base_dir_raw}{NODE_SCRIPTS_DIR}"));
let log_path = options
.node_log_path
.cloned()
.unwrap_or_else(|| base_dir.join(format!("{}.log", options.name)));
trace!("creating dirs {:?}", config_dir);
try_join!(
filesystem.create_dir_all(&config_dir),
filesystem.create_dir_all(&data_dir),
filesystem.create_dir_all(&relay_data_dir),
filesystem.create_dir_all(&scripts_dir),
)?;
trace!("created!");
let node = Arc::new(NativeNode {
namespace: options.namespace.clone(),
name: options.name.to_string(),
program: options.program.to_string(),
args: options.args.to_vec(),
env: options.env.to_vec(),
base_dir,
config_dir,
data_dir,
relay_data_dir,
scripts_dir,
log_path,
process_handle: std::sync::RwLock::new(None),
stdout_reading_task: RwLock::new(None),
stderr_reading_task: RwLock::new(None),
log_writing_task: RwLock::new(None),
filesystem: filesystem.clone(),
provider_tag: native::provider::PROVIDER_NAME.to_string(),
});
node.initialize_startup_paths(options.created_paths).await?;
node.initialize_startup_files(options.startup_files).await?;
if let Some(db_snap) = options.db_snapshot {
node.initialize_db_snapshot(db_snap).await?;
}
let (stdout, stderr) = node.initialize_process().await?;
node.initialize_log_writing(stdout, stderr).await;
Ok(node)
}
pub(super) async fn attach_to_live(
options: NativeNodeOptions<'_, FS>,
pid: i32,
) -> Result<Arc<Self>, ProviderError> {
let filesystem = options.filesystem.clone();
let base_dir =
PathBuf::from_iter([options.namespace_base_dir, &PathBuf::from(options.name)]);
trace!("creating base_dir {:?}", base_dir);
options.filesystem.create_dir_all(&base_dir).await?;
trace!("created base_dir {:?}", base_dir);
let base_dir_raw = base_dir.to_string_lossy();
let config_dir = PathBuf::from(format!("{base_dir_raw}{NODE_CONFIG_DIR}"));
let data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_DATA_DIR}"));
let relay_data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_RELAY_DATA_DIR}"));
let scripts_dir = PathBuf::from(format!("{base_dir_raw}{NODE_SCRIPTS_DIR}"));
let log_path = options
.node_log_path
.cloned()
.unwrap_or_else(|| base_dir.join(format!("{}.log", options.name)));
let pid = Pid::from_raw(pid);
let node = Arc::new(NativeNode {
namespace: options.namespace.clone(),
name: options.name.to_string(),
program: options.program.to_string(),
args: options.args.to_vec(),
env: options.env.to_vec(),
base_dir,
config_dir,
data_dir,
relay_data_dir,
scripts_dir,
log_path,
process_handle: std::sync::RwLock::new(Some(ProcessHandle::Attached(pid))),
stdout_reading_task: RwLock::new(None),
stderr_reading_task: RwLock::new(None),
log_writing_task: RwLock::new(None),
filesystem: filesystem.clone(),
provider_tag: native::provider::PROVIDER_NAME.to_string(),
});
Ok(node)
}
async fn initialize_startup_paths(&self, paths: &[PathBuf]) -> Result<(), ProviderError> {
trace!("creating paths {:?}", paths);
let base_dir_raw = self.base_dir.to_string_lossy();
try_join_all(paths.iter().map(|file| {
let full_path = format!("{base_dir_raw}{}", file.to_string_lossy());
self.filesystem.create_dir_all(full_path)
}))
.await?;
trace!("paths created!");
Ok(())
}
async fn initialize_startup_files(
&self,
startup_files: &[TransferedFile],
) -> Result<(), ProviderError> {
trace!("creating files {:?}", startup_files);
try_join_all(
startup_files
.iter()
.map(|file| self.send_file(&file.local_path, &file.remote_path, &file.mode)),
)
.await?;
trace!("files created!");
Ok(())
}
async fn initialize_db_snapshot(
&self,
db_snapshot: &AssetLocation,
) -> Result<(), ProviderError> {
trace!("snap: {db_snapshot}");
// check if we need to get the db or is already in the ns
let ns_base_dir = self.namespace_base_dir();
let hashed_location = match db_snapshot {
AssetLocation::Url(location) => hex::encode(sha2::Sha256::digest(location.to_string())),
AssetLocation::FilePath(filepath) => {
hex::encode(sha2::Sha256::digest(filepath.to_string_lossy().to_string()))
},
};
let full_path = format!("{ns_base_dir}/{hashed_location}.tgz");
trace!("db_snap fullpath in ns: {full_path}");
if !self.filesystem.exists(&full_path).await {
// needs to download/copy
self.get_db_snapshot(db_snapshot, &full_path).await?;
}
let contents = self.filesystem.read(&full_path).await.unwrap();
let gz = GzDecoder::new(&contents[..]);
let mut archive = Archive::new(gz);
archive
.unpack(self.base_dir.to_string_lossy().as_ref())
.unwrap();
if std::env::var("ZOMBIE_RM_TGZ_AFTER_EXTRACT").is_ok() {
let res = fs::remove_file(&full_path).await;
trace!("removing {}, result {:?}", full_path, res);
}
Ok(())
}
async fn get_db_snapshot(
&self,
location: &AssetLocation,
full_path: &str,
) -> Result<(), ProviderError> {
trace!("getting db_snapshot from: {:?} to: {full_path}", location);
match location {
AssetLocation::Url(location) => {
let res = reqwest::get(location.as_ref())
.await
.map_err(|err| ProviderError::DownloadFile(location.to_string(), err.into()))?;
let contents: &[u8] = &res.bytes().await.unwrap();
trace!("writing: {full_path}");
self.filesystem.write(full_path, contents).await?;
},
AssetLocation::FilePath(filepath) => {
self.filesystem.copy(filepath, full_path).await?;
},
};
Ok(())
}
async fn initialize_process(&self) -> Result<(ChildStdout, ChildStderr), ProviderError> {
let filtered_env: HashMap<String, String> = env::vars()
.filter(|(k, _)| k == "TZ" || k == "LANG" || k == "PATH")
.collect();
let mut process = Command::new(&self.program)
.args(&self.args)
.env_clear()
.envs(&filtered_env) // minimal environment
.envs(self.env.to_vec())
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true)
.current_dir(&self.base_dir)
.spawn()
.map_err(|err| ProviderError::NodeSpawningFailed(self.name.to_string(), err.into()))?;
let stdout = process
.stdout
.take()
.expect(&format!("infaillible, stdout is piped {THIS_IS_A_BUG}"));
let stderr = process
.stderr
.take()
.expect(&format!("infaillible, stderr is piped {THIS_IS_A_BUG}"));
let pid = Pid::from_raw(
process
.id()
.ok_or_else(|| ProviderError::ProcessIdRetrievalFailed(self.name.to_string()))?
as i32,
);
self.process_handle
.write()
.map_err(|_e| ProviderError::FailedToAcquireLock(self.name.clone()))?
.replace(ProcessHandle::Spawned(process, pid));
Ok((stdout, stderr))
}
async fn initialize_log_writing(&self, stdout: ChildStdout, stderr: ChildStderr) {
let (stdout_tx, mut rx) = mpsc::channel(10);
let stderr_tx = stdout_tx.clone();
self.stdout_reading_task
.write()
.await
.replace(self.create_stream_polling_task(stdout, stdout_tx));
self.stderr_reading_task
.write()
.await
.replace(self.create_stream_polling_task(stderr, stderr_tx));
let filesystem = self.filesystem.clone();
let log_path = self.log_path.clone();
self.log_writing_task
.write()
.await
.replace(tokio::spawn(async move {
loop {
while let Some(Ok(data)) = rx.recv().await {
// TODO: find a better way instead of ignoring error ?
let _ = filesystem.append(&log_path, data).await;
}
sleep(Duration::from_millis(250)).await;
}
}));
}
fn create_stream_polling_task(
&self,
stream: impl AsyncRead + Unpin + Send + 'static,
tx: Sender<Result<Vec<u8>, std::io::Error>>,
) -> JoinHandle<()> {
tokio::spawn(async move {
let mut reader = BufReader::new(stream);
let mut buffer = vec![0u8; 1024];
loop {
match reader.read(&mut buffer).await {
Ok(0) => {
let _ = tx.send(Ok(Vec::new())).await;
break;
},
Ok(n) => {
let _ = tx.send(Ok(buffer[..n].to_vec())).await;
},
Err(e) => {
let _ = tx.send(Err(e)).await;
break;
},
}
}
})
}
fn process_id(&self) -> Result<Pid, ProviderError> {
let pid = self
.process_handle
.read()
.map_err(|_e| ProviderError::FailedToAcquireLock(self.name.clone()))?
.as_ref()
.map(|handle| match handle {
ProcessHandle::Spawned(_, pid) => *pid,
ProcessHandle::Attached(pid) => *pid,
})
.ok_or_else(|| ProviderError::ProcessIdRetrievalFailed(self.name.to_string()))?;
Ok(pid)
}
pub(crate) async fn abort(&self) -> anyhow::Result<()> {
if let Some(task) = self.log_writing_task.write().await.take() {
task.abort();
}
if let Some(task) = self.stdout_reading_task.write().await.take() {
task.abort();
}
if let Some(task) = self.stderr_reading_task.write().await.take() {
task.abort();
}
let process_handle = {
let mut guard = self
.process_handle
.write()
.map_err(|_e| ProviderError::FailedToAcquireLock(self.name.clone()))?;
guard
.take()
.ok_or_else(|| anyhow!("no process was attached for the node"))?
};
match process_handle {
ProcessHandle::Spawned(mut child, _pid) => {
child.kill().await?;
},
ProcessHandle::Attached(pid) => {
kill(pid, Signal::SIGKILL)
.map_err(|err| anyhow!("Failed to kill attached process {pid}: {err}"))?;
},
}
Ok(())
}
fn namespace_base_dir(&self) -> String {
self.namespace
.upgrade()
.map(|namespace| namespace.base_dir().to_string_lossy().to_string())
.unwrap_or_else(|| panic!("namespace shouldn't be dropped, {THIS_IS_A_BUG}"))
}
}
#[async_trait]
impl<FS> ProviderNode for NativeNode<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
fn name(&self) -> &str {
&self.name
}
fn args(&self) -> Vec<&str> {
self.args.iter().map(|arg| arg.as_str()).collect()
}
fn base_dir(&self) -> &PathBuf {
&self.base_dir
}
fn config_dir(&self) -> &PathBuf {
&self.config_dir
}
fn data_dir(&self) -> &PathBuf {
&self.data_dir
}
fn relay_data_dir(&self) -> &PathBuf {
&self.relay_data_dir
}
fn scripts_dir(&self) -> &PathBuf {
&self.scripts_dir
}
fn log_path(&self) -> &PathBuf {
&self.log_path
}
fn log_cmd(&self) -> String {
format!("tail -f {}", self.log_path().to_string_lossy())
}
fn path_in_node(&self, file: &Path) -> PathBuf {
let full_path = format!(
"{}/{}",
self.base_dir.to_string_lossy(),
file.to_string_lossy()
);
PathBuf::from(full_path)
}
async fn logs(&self) -> Result<String, ProviderError> {
Ok(self.filesystem.read_to_string(&self.log_path).await?)
}
async fn dump_logs(&self, local_dest: PathBuf) -> Result<(), ProviderError> {
Ok(self.filesystem.copy(&self.log_path, local_dest).await?)
}
async fn run_command(
&self,
options: RunCommandOptions,
) -> Result<ExecutionResult, ProviderError> {
let result = Command::new(options.program.clone())
.args(options.args.clone())
.envs(options.env)
.current_dir(&self.base_dir)
.output()
.await
.map_err(|err| {
ProviderError::RunCommandError(
format!("{} {}", &options.program, &options.args.join(" ")),
"locally".to_string(),
err.into(),
)
})?;
if result.status.success() {
Ok(Ok(String::from_utf8_lossy(&result.stdout).to_string()))
} else {
Ok(Err((
result.status,
String::from_utf8_lossy(&result.stderr).to_string(),
)))
}
}
async fn run_script(
&self,
options: RunScriptOptions,
) -> Result<ExecutionResult, ProviderError> {
let local_script_path = PathBuf::from(&options.local_script_path);
if !self.filesystem.exists(&local_script_path).await {
return Err(ProviderError::ScriptNotFound(local_script_path));
}
// extract file name and build remote file path
let script_file_name = local_script_path
.file_name()
.map(|file_name| file_name.to_string_lossy().to_string())
.ok_or(ProviderError::InvalidScriptPath(anyhow!(
"Can't retrieve filename from script with path: {:?}",
options.local_script_path
)))?;
let remote_script_path = format!(
"{}/{}",
self.scripts_dir.to_string_lossy(),
script_file_name
);
// copy and set script's execute permission
self.filesystem
.copy(local_script_path, &remote_script_path)
.await?;
self.filesystem.set_mode(&remote_script_path, 0o744).await?;
// execute script
self.run_command(RunCommandOptions {
program: remote_script_path,
args: options.args,
env: options.env,
})
.await
}
async fn send_file(
&self,
local_file_path: &Path,
remote_file_path: &Path,
mode: &str,
) -> Result<(), ProviderError> {
let namespaced_remote_file_path = PathBuf::from(format!(
"{}{}",
&self.base_dir.to_string_lossy(),
remote_file_path.to_string_lossy()
));
self.filesystem
.copy(local_file_path, &namespaced_remote_file_path)
.await?;
self.run_command(
RunCommandOptions::new("chmod")
.args(vec![mode, &namespaced_remote_file_path.to_string_lossy()]),
)
.await?
.map_err(|(_, err)| {
ProviderError::SendFile(
self.name.clone(),
local_file_path.to_string_lossy().to_string(),
anyhow!("{err}"),
)
})?;
Ok(())
}
async fn receive_file(
&self,
remote_file_path: &Path,
local_file_path: &Path,
) -> Result<(), ProviderError> {
let namespaced_remote_file_path = PathBuf::from(format!(
"{}{}",
&self.base_dir.to_string_lossy(),
remote_file_path.to_string_lossy()
));
self.filesystem
.copy(namespaced_remote_file_path, local_file_path)
.await?;
Ok(())
}
async fn pause(&self) -> Result<(), ProviderError> {
let process_id = self.process_id()?;
kill(process_id, Signal::SIGSTOP)
.map_err(|err| ProviderError::PauseNodeFailed(self.name.clone(), err.into()))?;
Ok(())
}
async fn resume(&self) -> Result<(), ProviderError> {
let process_id = self.process_id()?;
nix::sys::signal::kill(process_id, Signal::SIGCONT)
.map_err(|err| ProviderError::ResumeNodeFailed(self.name.clone(), err.into()))?;
Ok(())
}
async fn restart(&self, after: Option<Duration>) -> Result<(), ProviderError> {
if let Some(duration) = after {
sleep(duration).await;
}
self.abort()
.await
.map_err(|err| ProviderError::RestartNodeFailed(self.name.clone(), err))?;
let (stdout, stderr) = self
.initialize_process()
.await
.map_err(|err| ProviderError::RestartNodeFailed(self.name.clone(), err.into()))?;
self.initialize_log_writing(stdout, stderr).await;
Ok(())
}
async fn destroy(&self) -> Result<(), ProviderError> {
self.abort()
.await
.map_err(|err| ProviderError::DestroyNodeFailed(self.name.clone(), err))?;
if let Some(namespace) = self.namespace.upgrade() {
namespace.nodes.write().await.remove(&self.name);
}
Ok(())
}
}
fn serialize_process_handle<S>(
process_handle: &std::sync::RwLock<Option<ProcessHandle>>,
serializer: S,
) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let pid = process_handle
.read()
.map_err(|_e| S::Error::custom("failed to acquire read lock"))?
.as_ref()
.map(|handle| match handle {
ProcessHandle::Spawned(_, pid) => pid.as_raw(),
ProcessHandle::Attached(pid) => pid.as_raw(),
});
pid.serialize(serializer)
}
@@ -0,0 +1,142 @@
use std::{
collections::HashMap,
path::{Path, PathBuf},
sync::{Arc, Weak},
};
use async_trait::async_trait;
use support::fs::FileSystem;
use tokio::sync::RwLock;
use super::namespace::NativeNamespace;
use crate::{
shared::helpers::extract_namespace_info, types::ProviderCapabilities, DynNamespace, Provider,
ProviderError, ProviderNamespace,
};
pub const PROVIDER_NAME: &str = "native";
pub struct NativeProvider<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
weak: Weak<NativeProvider<FS>>,
capabilities: ProviderCapabilities,
tmp_dir: PathBuf,
filesystem: FS,
pub(super) namespaces: RwLock<HashMap<String, Arc<NativeNamespace<FS>>>>,
}
impl<FS> NativeProvider<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
pub fn new(filesystem: FS) -> Arc<Self> {
Arc::new_cyclic(|weak| NativeProvider {
weak: weak.clone(),
capabilities: ProviderCapabilities {
has_resources: false,
requires_image: false,
prefix_with_full_path: true,
use_default_ports_in_cmd: false,
},
// NOTE: temp_dir in linux return `/tmp` but on mac something like
// `/var/folders/rz/1cyx7hfj31qgb98d8_cg7jwh0000gn/T/`, having
// one `trailing slash` and the other no can cause issues if
// you try to build a fullpath by concatenate. Use Pathbuf to prevent the issue.
tmp_dir: std::env::temp_dir(),
filesystem,
namespaces: RwLock::new(HashMap::new()),
})
}
pub fn tmp_dir(mut self, tmp_dir: impl Into<PathBuf>) -> Self {
self.tmp_dir = tmp_dir.into();
self
}
}
#[async_trait]
impl<FS> Provider for NativeProvider<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
fn name(&self) -> &str {
PROVIDER_NAME
}
fn capabilities(&self) -> &ProviderCapabilities {
&self.capabilities
}
async fn namespaces(&self) -> HashMap<String, DynNamespace> {
self.namespaces
.read()
.await
.iter()
.map(|(name, namespace)| (name.clone(), namespace.clone() as DynNamespace))
.collect()
}
async fn create_namespace(&self) -> Result<DynNamespace, ProviderError> {
let namespace = NativeNamespace::new(
&self.weak,
&self.tmp_dir,
&self.capabilities,
&self.filesystem,
None,
)
.await?;
self.namespaces
.write()
.await
.insert(namespace.name().to_string(), namespace.clone());
Ok(namespace)
}
async fn create_namespace_with_base_dir(
&self,
base_dir: &Path,
) -> Result<DynNamespace, ProviderError> {
let namespace = NativeNamespace::new(
&self.weak,
&self.tmp_dir,
&self.capabilities,
&self.filesystem,
Some(base_dir),
)
.await?;
self.namespaces
.write()
.await
.insert(namespace.name().to_string(), namespace.clone());
Ok(namespace)
}
async fn create_namespace_from_json(
&self,
json_value: &serde_json::Value,
) -> Result<DynNamespace, ProviderError> {
let (base_dir, name) = extract_namespace_info(json_value)?;
let namespace = NativeNamespace::attach_to_live(
&self.weak,
&self.capabilities,
&self.filesystem,
&base_dir,
&name,
)
.await?;
self.namespaces
.write()
.await
.insert(namespace.name().to_string(), namespace.clone());
Ok(namespace)
}
}
@@ -0,0 +1,3 @@
pub mod constants;
pub mod helpers;
pub mod types;
@@ -0,0 +1,22 @@
use std::net::{IpAddr, Ipv4Addr};
/// Namespace prefix
pub const NAMESPACE_PREFIX: &str = "zombie-";
/// Directory for node configuration
pub const NODE_CONFIG_DIR: &str = "/cfg";
/// Directory for node data dir
pub const NODE_DATA_DIR: &str = "/data";
/// Directory for node relay data dir
pub const NODE_RELAY_DATA_DIR: &str = "/relay-data";
/// Directory for node scripts
pub const NODE_SCRIPTS_DIR: &str = "/scripts";
/// Localhost ip
pub const LOCALHOST: IpAddr = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1));
/// The port substrate listens for p2p connections on
pub const P2P_PORT: u16 = 30333;
/// The remote port Prometheus can be accessed with
pub const PROMETHEUS_PORT: u16 = 9615;
/// The remote port websocket to access the RPC
pub const RPC_WS_PORT: u16 = 9944;
/// The remote port HTTP to access the RPC
pub const RPC_HTTP_PORT: u16 = 9933;
@@ -0,0 +1,79 @@
use std::{env, path::PathBuf};
use anyhow::anyhow;
use crate::{types::RunCommandOptions, DynNode, ProviderError};
/// Check if we are running in `CI` by checking the 'RUN_IN_CI' env var
pub fn running_in_ci() -> bool {
env::var("RUN_IN_CI").unwrap_or_default() == "1"
}
/// Executes a command on a temporary node and extracts the execution result either from the
/// standard output or a file.
pub async fn extract_execution_result(
temp_node: &DynNode,
options: RunCommandOptions,
expected_path: Option<&PathBuf>,
) -> Result<String, ProviderError> {
let output_contents = temp_node
.run_command(options)
.await?
.map_err(|(_, msg)| ProviderError::FileGenerationFailed(anyhow!("{msg}")))?;
// If an expected_path is provided, read the file contents from inside the container
if let Some(expected_path) = expected_path.as_ref() {
Ok(temp_node
.run_command(
RunCommandOptions::new("cat")
.args(vec![expected_path.to_string_lossy().to_string()]),
)
.await?
.map_err(|(_, msg)| {
ProviderError::FileGenerationFailed(anyhow!(format!(
"failed reading expected_path {}: {}",
expected_path.display(),
msg
)))
})?)
} else {
Ok(output_contents)
}
}
pub fn extract_namespace_info(
json_value: &serde_json::Value,
) -> Result<(PathBuf, String), ProviderError> {
let base_dir = json_value
.get("local_base_dir")
.and_then(|v| v.as_str())
.map(PathBuf::from)
.ok_or(ProviderError::InvalidConfig(
"`field local_base_dir` is missing from zombie.json".to_string(),
))?;
let name =
json_value
.get("ns")
.and_then(|v| v.as_str())
.ok_or(ProviderError::InvalidConfig(
"field `ns` is missing from zombie.json".to_string(),
))?;
Ok((base_dir, name.to_string()))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn check_runing_in_ci_env_var() {
assert!(!running_in_ci());
// now set the env var
env::set_var("RUN_IN_CI", "1");
assert!(running_in_ci());
// reset
env::set_var("RUN_IN_CI", "");
}
}
@@ -0,0 +1,34 @@
#!/bin/ash
log() {
echo "$(date +"%F %T") $1"
}
# used to handle the distinction where /cfg is used for k8s and /helpers for docker/podman
# to share a volume across nodes containing helper binaries and independent from /cfg
# where some node files are stored
OUTDIR=$([ -d /helpers ] && echo "/helpers" || echo "/cfg")
# Allow to use our image and just cp'd the binaries.
if [ -f /tmp/curl ]; then
cp /tmp/curl $OUTDIR/curl
log "curl copied"
else
wget github.com/moparisthebest/static-curl/releases/download/v7.83.1/curl-amd64 -O "$OUTDIR/curl"
log "curl downloaded"
fi;
chmod +x "$OUTDIR/curl"
log "curl chmoded"
if [ -f /tmp/coreutils ]; then
cp /tmp/coreutils $OUTDIR/coreutils
log "coreutils copied"
else
wget -qO- github.com/uutils/coreutils/releases/download/0.0.17/coreutils-0.0.17-x86_64-unknown-linux-musl.tar.gz | tar -xz -C $OUTDIR --strip-components=1 coreutils-0.0.17-x86_64-unknown-linux-musl/coreutils
log "coreutils downloaded"
fi;
chmod +x "$OUTDIR/coreutils"
log "coreutils chmoded"
@@ -0,0 +1,178 @@
#!/bin/bash
set -uxo pipefail
if [ -f /cfg/coreutils ]; then
RM="/cfg/coreutils rm"
MKFIFO="/cfg/coreutils mkfifo"
MKNOD="/cfg/coreutils mknod"
LS="/cfg/coreutils ls"
KILL="/cfg/coreutils kill"
SLEEP="/cfg/coreutils sleep"
ECHO="/cfg/coreutils echo"
elif [ -f /helpers/coreutils ]; then
# used for docker/podman to have a single volume sharing helper binaries
# across nodes independent from the /cfg where some files are stored
# by the node itself
RM="/helpers/coreutils rm"
MKFIFO="/helpers/coreutils mkfifo"
MKNOD="/helpers/coreutils mknod"
LS="/helpers/coreutils ls"
KILL="/helpers/coreutils kill"
SLEEP="/helpers/coreutils sleep"
ECHO="/helpers/coreutils echo"
else
RM="rm"
MKFIFO="mkfifo"
MKNOD="mknod"
LS="ls"
KILL="kill"
SLEEP="sleep"
ECHO="echo"
fi
echo "COMMANDS DEFINED"
# add /cfg as first `looking dir` to allow to overrides commands.
export PATH="/cfg":$PATH
echo "EXPORT PATH"
# setup pipe
pipe=/tmp/zombiepipe
trap "$RM -f $pipe" EXIT
# try mkfifo first and allow to fail
if [[ ! -p $pipe ]]; then
$MKFIFO $pipe
fi
# set immediately exit on any non 0 exit code
set -e
# if fails try mknod
if [[ ! -p $pipe ]]; then
$MKNOD $pipe p
fi
echo "PIPE CREATED"
# init empty
child_pid=""
# get the command to exec
CMD=($@)
# File to store CMD (and update from there)
ZOMBIE_CMD_FILE=/tmp/zombie.cmd
ZOMBIE_CMD_PID=/tmp/zombie.pid
# Store the cmd and make it available to later usage
# NOTE: echo without new line to allow to customize the cmd later
$ECHO -n "${CMD[@]}" > $ZOMBIE_CMD_FILE
echo "COMMAND TO RUN IS: $CMD"
start() {
# redirect the output to be expored to loki
"${CMD[@]}" >> /proc/1/fd/1 2>> /proc/1/fd/2 &
if [[ "$CMD" != "cat" ]]; then
child_pid="$!"
$ECHO $(cat $ZOMBIE_CMD_FILE)
# store pid
$ECHO ${child_pid} > $ZOMBIE_CMD_PID
# sleep a few secs to detect errors bootstraping the node
sleep 3
# check if the process is running
if ! $LS /proc/$child_pid > /dev/null 2>&1 ; then
echo "child process doesn't exist, quiting...";
exit 1;
else
echo "PID: $child_pid alive";
fi;
else
echo "Process not started, PID not stored, since was 'cat'";
fi;
}
restart() {
if [ ! -z "${child_pid}" ]; then
$KILL -9 "$child_pid"
fi
# check if we have timeout
if [[ "$1" ]]; then
$SLEEP "$1"
fi
start
}
pause() {
if [ ! -z "${child_pid}" ]; then
echo "send -STOP to process $child_pid"
$KILL -STOP "$child_pid"
echo "result $?"
# Wait until the process is actually stopped (state 'T')
for i in {1..10}; do
local state
state=$(awk '{print $3}' /proc/$child_pid/stat 2>/dev/null)
if [ "$state" = "T" ]; then
echo "Process $child_pid is paused (state: $state)"
return
fi
$SLEEP 0.2
done
echo "Warning: Process $child_pid not paused after SIGSTOP"
fi
}
resume() {
if [ ! -z "${child_pid}" ]; then
echo "send -CONT to process $child_pid"
$KILL -CONT "$child_pid"
echo "result $?"
# Wait until the process is actually resumed (state not 'T')
for i in {1..10}; do
local state
state=$(awk '{print $3}' /proc/$child_pid/stat 2>/dev/null)
if [ "$state" != "T" ] && [ -n "$state" ]; then
echo "Process $child_pid is resumed (state: $state)"
return
fi
$SLEEP 0.2
done
echo "Warning: Process $child_pid not resumed after SIGCONT"
fi
}
# keep listening from the pipe
while read line <$pipe
echo "read line: ${line}"
do
if [[ "$line" == "start" ]]; then
start
elif [[ "$line" == "quit" ]]; then
break
elif [[ "$line" =~ "restart" ]]; then
# check if we have timeout between restart
if [[ $line =~ [^0-9]+([0-9]+) ]]; then
restart "${BASH_REMATCH[1]}"
else
restart 0
fi;
elif [[ "$line" == "pause" ]]; then
pause
elif [[ "$line" == "resume" ]]; then
resume
fi
done
exit 0
@@ -0,0 +1,375 @@
use std::{
collections::HashMap,
path::{Path, PathBuf},
process::ExitStatus,
};
use configuration::{shared::resources::Resources, types::AssetLocation};
use serde::{Deserialize, Serialize};
pub type Port = u16;
pub type ExecutionResult = Result<String, (ExitStatus, String)>;
#[derive(Debug, Clone, PartialEq)]
pub struct ProviderCapabilities {
// default ports internal
/// Ensure that we have an image for each node (k8s/podman/docker)
pub requires_image: bool,
/// Allow to customize the resources through manifest (k8s).
pub has_resources: bool,
/// Used in native to prefix filepath with fullpath
pub prefix_with_full_path: bool,
/// Use default ports in node cmd/args.
/// NOTE: generally used in k8s/dockers since the images expose those ports.
pub use_default_ports_in_cmd: bool,
}
#[derive(Debug, Clone)]
pub struct SpawnNodeOptions {
/// Name of the node
pub name: String,
/// Image of the node (IFF is supported by the provider)
pub image: Option<String>,
/// Resources to apply to the node (IFF is supported by the provider)
pub resources: Option<Resources>,
/// Main command to execute
pub program: String,
/// Arguments to pass to the main command
pub args: Vec<String>,
/// Environment to set when running the `program`
pub env: Vec<(String, String)>,
// TODO: rename startup_files
/// Files to inject at startup
pub injected_files: Vec<TransferedFile>,
/// Paths to create before start the node (e.g keystore)
/// should be created with `create_dir_all` in order
/// to create the full path even when we have missing parts
pub created_paths: Vec<PathBuf>,
/// Database snapshot to be injected (should be a tgz file)
/// Could be a local or remote asset
pub db_snapshot: Option<AssetLocation>,
pub port_mapping: Option<HashMap<Port, Port>>,
/// Optionally specify a log path for the node
pub node_log_path: Option<PathBuf>,
}
impl SpawnNodeOptions {
pub fn new<S>(name: S, program: S) -> Self
where
S: AsRef<str>,
{
Self {
name: name.as_ref().to_string(),
image: None,
resources: None,
program: program.as_ref().to_string(),
args: vec![],
env: vec![],
injected_files: vec![],
created_paths: vec![],
db_snapshot: None,
port_mapping: None,
node_log_path: None,
}
}
pub fn image<S>(mut self, image: S) -> Self
where
S: AsRef<str>,
{
self.image = Some(image.as_ref().to_string());
self
}
pub fn resources(mut self, resources: Resources) -> Self {
self.resources = Some(resources);
self
}
pub fn db_snapshot(mut self, db_snap: Option<AssetLocation>) -> Self {
self.db_snapshot = db_snap;
self
}
pub fn args<S, I>(mut self, args: I) -> Self
where
S: AsRef<str>,
I: IntoIterator<Item = S>,
{
self.args = args.into_iter().map(|s| s.as_ref().to_string()).collect();
self
}
pub fn env<S, I>(mut self, env: I) -> Self
where
S: AsRef<str>,
I: IntoIterator<Item = (S, S)>,
{
self.env = env
.into_iter()
.map(|(name, value)| (name.as_ref().to_string(), value.as_ref().to_string()))
.collect();
self
}
pub fn injected_files<I>(mut self, injected_files: I) -> Self
where
I: IntoIterator<Item = TransferedFile>,
{
self.injected_files = injected_files.into_iter().collect();
self
}
pub fn created_paths<P, I>(mut self, created_paths: I) -> Self
where
P: AsRef<Path>,
I: IntoIterator<Item = P>,
{
self.created_paths = created_paths
.into_iter()
.map(|path| path.as_ref().into())
.collect();
self
}
pub fn port_mapping(mut self, ports: HashMap<Port, Port>) -> Self {
self.port_mapping = Some(ports);
self
}
pub fn node_log_path(mut self, path: Option<PathBuf>) -> Self {
self.node_log_path = path;
self
}
}
#[derive(Debug)]
pub struct GenerateFileCommand {
pub program: String,
pub args: Vec<String>,
pub env: Vec<(String, String)>,
pub local_output_path: PathBuf,
}
impl GenerateFileCommand {
pub fn new<S, P>(program: S, local_output_path: P) -> Self
where
S: AsRef<str>,
P: AsRef<Path>,
{
Self {
program: program.as_ref().to_string(),
args: vec![],
env: vec![],
local_output_path: local_output_path.as_ref().into(),
}
}
pub fn args<S, I>(mut self, args: I) -> Self
where
S: AsRef<str>,
I: IntoIterator<Item = S>,
{
self.args = args.into_iter().map(|s| s.as_ref().to_string()).collect();
self
}
pub fn env<S, I>(mut self, env: I) -> Self
where
S: AsRef<str>,
I: IntoIterator<Item = (S, S)>,
{
self.env = env
.into_iter()
.map(|(name, value)| (name.as_ref().to_string(), value.as_ref().to_string()))
.collect();
self
}
}
#[derive(Debug)]
pub struct GenerateFilesOptions {
pub commands: Vec<GenerateFileCommand>,
pub image: Option<String>,
pub injected_files: Vec<TransferedFile>,
// Allow to control the name of the node used to create the files.
pub temp_name: Option<String>,
pub expected_path: Option<PathBuf>,
}
impl GenerateFilesOptions {
pub fn new<I>(commands: I, image: Option<String>, expected_path: Option<PathBuf>) -> Self
where
I: IntoIterator<Item = GenerateFileCommand>,
{
Self {
commands: commands.into_iter().collect(),
injected_files: vec![],
image,
temp_name: None,
expected_path,
}
}
pub fn with_files<I>(
commands: I,
image: Option<String>,
injected_files: &[TransferedFile],
expected_path: Option<PathBuf>,
) -> Self
where
I: IntoIterator<Item = GenerateFileCommand>,
{
Self {
commands: commands.into_iter().collect(),
injected_files: injected_files.into(),
image,
temp_name: None,
expected_path,
}
}
pub fn image<S>(mut self, image: S) -> Self
where
S: AsRef<str>,
{
self.image = Some(image.as_ref().to_string());
self
}
pub fn injected_files<I>(mut self, injected_files: I) -> Self
where
I: IntoIterator<Item = TransferedFile>,
{
self.injected_files = injected_files.into_iter().collect();
self
}
pub fn temp_name(mut self, name: impl Into<String>) -> Self {
self.temp_name = Some(name.into());
self
}
}
#[derive(Debug)]
pub struct RunCommandOptions {
pub program: String,
pub args: Vec<String>,
pub env: Vec<(String, String)>,
}
impl RunCommandOptions {
pub fn new<S>(program: S) -> Self
where
S: AsRef<str>,
{
Self {
program: program.as_ref().to_string(),
args: vec![],
env: vec![],
}
}
pub fn args<S, I>(mut self, args: I) -> Self
where
S: AsRef<str>,
I: IntoIterator<Item = S>,
{
self.args = args.into_iter().map(|s| s.as_ref().to_string()).collect();
self
}
pub fn env<S, I>(mut self, env: I) -> Self
where
S: AsRef<str>,
I: IntoIterator<Item = (S, S)>,
{
self.env = env
.into_iter()
.map(|(name, value)| (name.as_ref().to_string(), value.as_ref().to_string()))
.collect();
self
}
}
pub struct RunScriptOptions {
pub local_script_path: PathBuf,
pub args: Vec<String>,
pub env: Vec<(String, String)>,
}
impl RunScriptOptions {
pub fn new<P>(local_script_path: P) -> Self
where
P: AsRef<Path>,
{
Self {
local_script_path: local_script_path.as_ref().into(),
args: vec![],
env: vec![],
}
}
pub fn args<S, I>(mut self, args: I) -> Self
where
S: AsRef<str>,
I: IntoIterator<Item = S>,
{
self.args = args.into_iter().map(|s| s.as_ref().to_string()).collect();
self
}
pub fn env<S, I>(mut self, env: I) -> Self
where
S: AsRef<str>,
I: IntoIterator<Item = (S, S)>,
{
self.env = env
.into_iter()
.map(|(name, value)| (name.as_ref().to_string(), value.as_ref().to_string()))
.collect();
self
}
}
// TODO(team): I think we can rename it to FileMap?
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TransferedFile {
pub local_path: PathBuf,
pub remote_path: PathBuf,
// TODO: Can be narrowed to have strict typing on this?
pub mode: String,
}
impl TransferedFile {
pub fn new<P>(local_path: P, remote_path: P) -> Self
where
P: AsRef<Path>,
{
Self {
local_path: local_path.as_ref().into(),
remote_path: remote_path.as_ref().into(),
mode: "0644".to_string(), // default to rw-r--r--
}
}
pub fn mode<S>(mut self, mode: S) -> Self
where
S: AsRef<str>,
{
self.mode = mode.as_ref().to_string();
self
}
}
impl std::fmt::Display for TransferedFile {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"File to transfer (local: {}, remote: {})",
self.local_path.display(),
self.remote_path.display()
)
}
}
+33
View File
@@ -0,0 +1,33 @@
[package]
name = "zombienet-sdk"
version.workspace = true
authors.workspace = true
edition.workspace = true
rust-version.workspace = true
publish = true
license.workspace = true
repository.workspace = true
description = "Zombienet SDK, entrypoint for using zombienet"
keywords = ["zombienet", "sdk"]
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
async-trait = { workspace = true }
tokio = { workspace = true }
futures = { workspace = true }
lazy_static = { workspace = true }
pezkuwi-subxt = { workspace = true }
pezkuwi-subxt-signer = { workspace = true, features = ["subxt"] }
# Zombienet deps
configuration = { workspace = true }
orchestrator = { workspace = true }
provider = { workspace = true }
support = { workspace = true }
[dev-dependencies]
tracing-subscriber = { workspace = true }
kube = { workspace = true, features = ["ws", "runtime"] }
k8s-openapi = { workspace = true, features = ["v1_27"] }
serde_json = {workspace = true }
@@ -0,0 +1,90 @@
//! Helpers functions to get configuration (e.g. Provider and images) from the env vars
use std::{env, future::Future, path::PathBuf, pin::Pin};
use crate::{
AttachToLive, AttachToLiveNetwork, LocalFileSystem, Network, NetworkConfig, NetworkConfigExt,
OrchestratorError,
};
const DEFAULT_POLKADOT_IMAGE: &str = "docker.io/parity/polkadot:latest";
const DEFAULT_CUMULUS_IMAGE: &str = "docker.io/parity/polkadot-parachain:latest";
#[derive(Debug, Default)]
pub struct Images {
pub polkadot: String,
pub cumulus: String,
}
impl Images {
/// Alias for polkadot field - returns reference to pezkuwi/polkadot image
pub fn pezkuwi(&self) -> &str {
&self.polkadot
}
/// Alias for cumulus field - returns reference to pezcumulus/cumulus image
pub fn pezcumulus(&self) -> &str {
&self.cumulus
}
}
pub enum Provider {
Native,
K8s,
Docker,
}
impl Provider {
pub fn get_spawn_fn(
&self,
) -> fn(NetworkConfig) -> Pin<Box<dyn Future<Output = SpawnResult> + Send>> {
match self {
Provider::Native => NetworkConfigExt::spawn_native,
Provider::K8s => NetworkConfigExt::spawn_k8s,
Provider::Docker => NetworkConfigExt::spawn_docker,
}
}
}
// Use `docker` as default provider
impl From<String> for Provider {
fn from(value: String) -> Self {
match value.to_ascii_lowercase().as_ref() {
"native" => Provider::Native,
"k8s" => Provider::K8s,
_ => Provider::Docker, // default provider
}
}
}
pub fn get_images_from_env() -> Images {
let polkadot = env::var("POLKADOT_IMAGE").unwrap_or(DEFAULT_POLKADOT_IMAGE.into());
let cumulus = env::var("CUMULUS_IMAGE").unwrap_or(DEFAULT_CUMULUS_IMAGE.into());
Images { polkadot, cumulus }
}
pub fn get_provider_from_env() -> Provider {
env::var("ZOMBIE_PROVIDER").unwrap_or_default().into()
}
pub type SpawnResult = Result<Network<LocalFileSystem>, OrchestratorError>;
pub fn get_spawn_fn() -> fn(NetworkConfig) -> Pin<Box<dyn Future<Output = SpawnResult> + Send>> {
let provider = get_provider_from_env();
match provider {
Provider::Native => NetworkConfigExt::spawn_native,
Provider::K8s => NetworkConfigExt::spawn_k8s,
Provider::Docker => NetworkConfigExt::spawn_docker,
}
}
pub type AttachResult = Result<Network<LocalFileSystem>, OrchestratorError>;
pub fn get_attach_fn() -> fn(PathBuf) -> Pin<Box<dyn Future<Output = AttachResult> + Send>> {
let provider = get_provider_from_env();
match provider {
Provider::Native => AttachToLiveNetwork::attach_native,
Provider::K8s => AttachToLiveNetwork::attach_k8s,
Provider::Docker => AttachToLiveNetwork::attach_docker,
}
}
+131
View File
@@ -0,0 +1,131 @@
use std::path::PathBuf;
use async_trait::async_trait;
pub use configuration::{
GlobalSettings, GlobalSettingsBuilder, NetworkConfig, NetworkConfigBuilder,
RegistrationStrategy, WithRelaychain,
};
pub use orchestrator::{
errors::OrchestratorError,
network::{node::NetworkNode, Network},
sc_chain_spec, AddCollatorOptions, AddNodeOptions, Orchestrator,
};
// Helpers used for interact with the network
pub mod tx_helper {
pub use orchestrator::{
network::chain_upgrade::ChainUpgrade, shared::types::RuntimeUpgradeOptions,
};
}
use provider::{DockerProvider, KubernetesProvider, NativeProvider};
pub use support::fs::local::LocalFileSystem;
pub mod environment;
pub const PROVIDERS: [&str; 3] = ["k8s", "native", "docker"];
// re-export pezkuwi-subxt (with subxt alias for backwards compatibility)
pub use pezkuwi_subxt;
pub use pezkuwi_subxt as subxt;
pub use pezkuwi_subxt_signer;
pub use pezkuwi_subxt_signer as subxt_signer;
#[async_trait]
pub trait NetworkConfigExt {
/// Spawns a network using the native or k8s provider.
///
/// # Example:
/// ```rust
/// # use zombienet_sdk::{NetworkConfig, NetworkConfigExt};
/// # async fn example() -> Result<(), zombienet_sdk::OrchestratorError> {
/// let network = NetworkConfig::load_from_toml("config.toml")?
/// .spawn_native()
/// .await?;
/// # Ok(())
/// # }
/// ```
async fn spawn_native(self) -> Result<Network<LocalFileSystem>, OrchestratorError>;
async fn spawn_k8s(self) -> Result<Network<LocalFileSystem>, OrchestratorError>;
async fn spawn_docker(self) -> Result<Network<LocalFileSystem>, OrchestratorError>;
}
#[async_trait]
pub trait AttachToLive {
/// Attaches to a running live network using the native, docker or k8s provider.
///
/// # Example:
/// ```rust
/// # use zombienet_sdk::{AttachToLive, AttachToLiveNetwork};
/// # use std::path::PathBuf;
/// # async fn example() -> Result<(), zombienet_sdk::OrchestratorError> {
/// let zombie_json_path = PathBuf::from("some/path/zombie.json");
/// let network = AttachToLiveNetwork::attach_native(zombie_json_path).await?;
/// # Ok(())
/// # }
/// ```
async fn attach_native(
zombie_json_path: PathBuf,
) -> Result<Network<LocalFileSystem>, OrchestratorError>;
async fn attach_k8s(
zombie_json_path: PathBuf,
) -> Result<Network<LocalFileSystem>, OrchestratorError>;
async fn attach_docker(
zombie_json_path: PathBuf,
) -> Result<Network<LocalFileSystem>, OrchestratorError>;
}
#[async_trait]
impl NetworkConfigExt for NetworkConfig {
async fn spawn_native(self) -> Result<Network<LocalFileSystem>, OrchestratorError> {
let filesystem = LocalFileSystem;
let provider = NativeProvider::new(filesystem.clone());
let orchestrator = Orchestrator::new(filesystem, provider);
orchestrator.spawn(self).await
}
async fn spawn_k8s(self) -> Result<Network<LocalFileSystem>, OrchestratorError> {
let filesystem = LocalFileSystem;
let provider = KubernetesProvider::new(filesystem.clone()).await;
let orchestrator = Orchestrator::new(filesystem, provider);
orchestrator.spawn(self).await
}
async fn spawn_docker(self) -> Result<Network<LocalFileSystem>, OrchestratorError> {
let filesystem = LocalFileSystem;
let provider = DockerProvider::new(filesystem.clone()).await;
let orchestrator = Orchestrator::new(filesystem, provider);
orchestrator.spawn(self).await
}
}
pub struct AttachToLiveNetwork;
#[async_trait]
impl AttachToLive for AttachToLiveNetwork {
async fn attach_native(
zombie_json_path: PathBuf,
) -> Result<Network<LocalFileSystem>, OrchestratorError> {
let filesystem = LocalFileSystem;
let provider = NativeProvider::new(filesystem.clone());
let orchestrator = Orchestrator::new(filesystem, provider);
orchestrator.attach_to_live(zombie_json_path.as_ref()).await
}
async fn attach_k8s(
zombie_json_path: PathBuf,
) -> Result<Network<LocalFileSystem>, OrchestratorError> {
let filesystem = LocalFileSystem;
let provider = KubernetesProvider::new(filesystem.clone()).await;
let orchestrator = Orchestrator::new(filesystem, provider);
orchestrator.attach_to_live(zombie_json_path.as_ref()).await
}
async fn attach_docker(
zombie_json_path: PathBuf,
) -> Result<Network<LocalFileSystem>, OrchestratorError> {
let filesystem = LocalFileSystem;
let provider = DockerProvider::new(filesystem.clone()).await;
let orchestrator = Orchestrator::new(filesystem, provider);
orchestrator.attach_to_live(zombie_json_path.as_ref()).await
}
}
@@ -0,0 +1,75 @@
use futures::StreamExt;
use zombienet_sdk::{environment::get_spawn_fn, NetworkConfigBuilder};
const BEST_BLOCK_METRIC: &str = "block_height{status=\"best\"}";
#[tokio::test(flavor = "multi_thread")]
async fn rococo_local_with_omni_node_and_wasm_runtime() {
let _ = tracing_subscriber::fmt::try_init();
let config = NetworkConfigBuilder::new()
.with_relaychain(|relaychain| {
relaychain
.with_chain("rococo-local")
.with_default_command("polkadot")
.with_default_image("docker.io/parity/polkadot:latest")
.with_validator(|node| node.with_name("alice"))
.with_validator(|node| node.with_name("bob"))
})
.with_parachain(|parachain| {
parachain
.with_id(100).cumulus_based(true)
.with_chain("asset-hub-rococo-local")
.with_default_command("polkadot-omni-node")
.with_default_image("docker.io/parity/polkadot-omni-node:latest")
.with_chain_spec_runtime("https://github.com/polkadot-fellows/runtimes/releases/download/v1.9.2/asset-hub-polkadot_runtime-v1009002.compact.compressed.wasm", None )
.with_collator(|collator| collator.with_name("omni-collator-1"))
.with_collator(|collator| collator.with_name("omni-collator-2"))
})
.build()
.unwrap();
let spawn_fn = get_spawn_fn();
let network = spawn_fn(config).await.unwrap();
println!("🚀🚀🚀🚀 network deployed");
// wait 2 blocks
let alice = network.get_node("alice").unwrap();
assert!(alice
.wait_metric(BEST_BLOCK_METRIC, |b| b > 2_f64)
.await
.is_ok());
// omni-collator-1
let collator = network.get_node("omni-collator-1").unwrap();
let client = collator
.wait_client::<subxt::PolkadotConfig>()
.await
.unwrap();
// wait 1 blocks
let mut blocks = client.blocks().subscribe_finalized().await.unwrap().take(1);
while let Some(block) = blocks.next().await {
println!(
"Block (omni-collator-1) #{}",
block.unwrap().header().number
);
}
// omni-collator-2
let collator = network.get_node("omni-collator-2").unwrap();
let client = collator
.wait_client::<subxt::PolkadotConfig>()
.await
.unwrap();
// wait 1 blocks
let mut blocks = client.blocks().subscribe_finalized().await.unwrap().take(1);
while let Some(block) = blocks.next().await {
println!(
"Block (omni-collator-2) #{}",
block.unwrap().header().number
);
}
}
@@ -0,0 +1,76 @@
use futures::StreamExt;
use zombienet_sdk::{environment::get_spawn_fn, NetworkConfigBuilder};
const BEST_BLOCK_METRIC: &str = "block_height{status=\"best\"}";
#[tokio::test(flavor = "multi_thread")]
async fn polkadot_local_with_chain_spec_runtime() {
let _ = tracing_subscriber::fmt::try_init();
let config = NetworkConfigBuilder::new()
.with_relaychain(|relaychain| {
relaychain
.with_chain("polkadot-local")
.with_default_command("polkadot")
.with_default_image("docker.io/parity/polkadot:latest")
.with_chain_spec_runtime("https://github.com/polkadot-fellows/runtimes/releases/download/v1.9.3/polkadot_runtime-v1009003.compact.compressed.wasm", None)
.with_validator(|node| node.with_name("alice"))
.with_validator(|node| node.with_name("bob"))
})
.with_parachain(|parachain| {
parachain
.with_id(100)
.with_chain("asset-hub-polkadot-local")
.with_default_command("polkadot-parachain")
.with_default_image("docker.io/parity/polkadot-parachain:latest")
.with_chain_spec_runtime("https://github.com/polkadot-fellows/runtimes/releases/download/v1.9.2/asset-hub-polkadot_runtime-v1009002.compact.compressed.wasm", None)
.with_collator(|collator| collator.with_name("asset-hub-collator-1"))
.with_collator(|collator| collator.with_name("asset-hub-collator-2"))
})
.build()
.unwrap();
let spawn_fn = get_spawn_fn();
let network = spawn_fn(config).await.unwrap();
println!("🚀🚀🚀🚀 network deployed");
// wait 2 blocks
let alice = network.get_node("alice").unwrap();
assert!(alice
.wait_metric(BEST_BLOCK_METRIC, |b| b > 2_f64)
.await
.is_ok());
// asset-hub-collator-1
let collator = network.get_node("asset-hub-collator-1").unwrap();
let client = collator
.wait_client::<subxt::PolkadotConfig>()
.await
.unwrap();
// wait 1 blocks
let mut blocks = client.blocks().subscribe_finalized().await.unwrap().take(1);
while let Some(block) = blocks.next().await {
println!(
"Block (asset-hub-collator-1) #{}",
block.unwrap().header().number
);
}
// asset-hub-collator-2
let collator = network.get_node("asset-hub-collator-2").unwrap();
let client = collator
.wait_client::<subxt::PolkadotConfig>()
.await
.unwrap();
// wait 1 blocks
let mut blocks = client.blocks().subscribe_finalized().await.unwrap().take(1);
while let Some(block) = blocks.next().await {
println!(
"Block (asset-hub-collator-2) #{}",
block.unwrap().header().number
);
}
}
@@ -0,0 +1,57 @@
use std::time::Instant;
use configuration::{NetworkConfig, NetworkConfigBuilder};
use zombienet_sdk::environment::get_spawn_fn;
fn small_network() -> NetworkConfig {
NetworkConfigBuilder::new()
.with_relaychain(|r| {
r.with_chain("rococo-local")
.with_default_command("polkadot")
.with_default_image("docker.io/parity/polkadot:v1.20.2")
.with_validator(|node| node.with_name("alice"))
.with_validator(|node| node.with_name("bob"))
})
.with_parachain(|p| {
p.with_id(2000)
.cumulus_based(true)
.with_default_image("docker.io/parity/polkadot-parachain:v1.20.2")
.with_collator(|n| n.with_name("collator").with_command("polkadot-parachain"))
})
.with_parachain(|p| {
p.with_id(3000)
.cumulus_based(true)
.with_default_image("docker.io/parity/polkadot-omni-node:v1.20.2")
.with_chain_spec_runtime("https://github.com/polkadot-fellows/runtimes/releases/download/v1.9.2/asset-hub-polkadot_runtime-v1009002.compact.compressed.wasm", None)
.with_collator(|n| n.with_name("collator-omni").with_command("polkadot-omni-node"))
})
.build()
.unwrap()
}
#[tokio::test(flavor = "multi_thread")]
async fn ci_native_smoke_should_works() {
tracing_subscriber::fmt::init();
const BEST_BLOCK_METRIC: &str = "block_height{status=\"best\"}";
let now = Instant::now();
let config = small_network();
let spawn_fn = get_spawn_fn();
let network = spawn_fn(config).await.unwrap();
let elapsed = now.elapsed();
println!("🚀🚀🚀🚀 network deployed in {elapsed:.2?}");
network.wait_until_is_up(20).await.unwrap();
let elapsed = now.elapsed();
println!("✅✅✅✅ network is up in {elapsed:.2?}");
// Get a ref to the node
let alice = network.get_node("alice").unwrap();
// wait 10 blocks
alice
.wait_metric(BEST_BLOCK_METRIC, |x| x > 9_f64)
.await
.unwrap();
}
+179
View File
@@ -0,0 +1,179 @@
use std::{path::PathBuf, time::Instant};
use configuration::{NetworkConfig, NetworkConfigBuilder};
use futures::{stream::StreamExt, try_join};
use orchestrator::{AddCollatorOptions, AddNodeOptions};
use zombienet_sdk::environment::{get_attach_fn, get_spawn_fn};
fn small_network() -> NetworkConfig {
NetworkConfigBuilder::new()
.with_relaychain(|r| {
r.with_chain("rococo-local")
.with_default_command("polkadot")
.with_default_image("docker.io/parity/polkadot:v1.20.2")
.with_validator(|node| node.with_name("alice"))
.with_validator(|node| node.with_name("bob"))
})
.with_parachain(|p| {
p.with_id(2000).cumulus_based(true).with_collator(|n| {
n.with_name("collator")
.with_command("polkadot-parachain")
.with_image("docker.io/parity/polkadot-parachain:1.7.0")
})
})
.with_parachain(|p| {
p.with_id(3000).cumulus_based(true).with_collator(|n| {
n.with_name("collator-new")
.with_command("polkadot-parachain")
.with_image("docker.io/parity/polkadot-parachain:v1.20.2")
})
})
.with_global_settings(|g| {
g.with_base_dir(PathBuf::from("/tmp/zombie-1"))
.with_tear_down_on_failure(false)
})
.build()
.unwrap()
}
#[tokio::test(flavor = "multi_thread")]
async fn ci_k8s_basic_functionalities_should_works() {
let _ = tracing_subscriber::fmt::try_init();
const BEST_BLOCK_METRIC: &str = "block_height{status=\"best\"}";
let now = Instant::now();
let config = small_network();
let spawn_fn = get_spawn_fn();
let network = spawn_fn(config).await.unwrap();
let elapsed = now.elapsed();
println!("🚀🚀🚀🚀 network deployed in {elapsed:.2?}");
// detach and attach to running
network.detach().await;
drop(network);
let attach_fn = get_attach_fn();
let zombie_path = PathBuf::from("/tmp/zombie-1/zombie.json");
let mut network = attach_fn(zombie_path).await.unwrap();
// Get a ref to the node
let alice = network.get_node("alice").unwrap();
let (_best_block_pass, client) = try_join!(
alice.wait_metric(BEST_BLOCK_METRIC, |x| x > 5_f64),
alice.wait_client::<subxt::PolkadotConfig>()
)
.unwrap();
alice
.wait_log_line_count("*rted #1*", true, 10)
.await
.unwrap();
// check best block through metrics with timeout
assert!(alice
.wait_metric_with_timeout(BEST_BLOCK_METRIC, |x| x > 10_f64, 45_u32)
.await
.is_ok());
// ensure timeout error
let best_block = alice.reports(BEST_BLOCK_METRIC).await.unwrap();
let res = alice
.wait_metric_with_timeout(BEST_BLOCK_METRIC, |x| x > (best_block * 2_f64), 10_u32)
.await;
assert!(res.is_err());
// get single metric
let role = alice.reports("node_roles").await.unwrap();
println!("Role is {role}");
assert_eq!(role, 4.0);
// subxt
// wait 3 blocks
let mut blocks = client.blocks().subscribe_finalized().await.unwrap().take(3);
while let Some(block) = blocks.next().await {
println!("Block #{}", block.unwrap().header().number);
}
// drop the client
drop(client);
// check best block through metrics
let best_block = alice
.reports("block_height{status=\"best\"}")
.await
.unwrap();
assert!(best_block >= 2.0, "Current best {best_block}");
// collator
let collator = network.get_node("collator").unwrap();
let client = collator
.wait_client::<subxt::PolkadotConfig>()
.await
.unwrap();
// wait 3 blocks
let mut blocks = client.blocks().subscribe_finalized().await.unwrap().take(3);
while let Some(block) = blocks.next().await {
println!("Block (para) #{}", block.unwrap().header().number);
}
// add node
let opts = AddNodeOptions {
rpc_port: Some(9444),
is_validator: true,
..Default::default()
};
network.add_node("new1", opts).await.unwrap();
// add collator
let col_opts = AddCollatorOptions {
command: Some("polkadot-parachain".try_into().unwrap()),
image: Some(
"docker.io/parity/polkadot-parachain:1.7.0"
.try_into()
.unwrap(),
),
..Default::default()
};
network
.add_collator("new-col-1", col_opts, 2000)
.await
.unwrap();
// pause / resume
let alice = network.get_node("alice").unwrap();
alice.pause().await.unwrap();
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
let res_err = alice
.wait_metric_with_timeout(BEST_BLOCK_METRIC, |x| x > 5_f64, 5_u32)
.await;
assert!(res_err.is_err());
alice.resume().await.unwrap();
alice
.wait_metric_with_timeout(BEST_BLOCK_METRIC, |x| x > 5_f64, 5_u32)
.await
.unwrap();
// timeout connecting ws
let collator = network.get_node("collator").unwrap();
collator.pause().await.unwrap();
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
let r = collator
.wait_client_with_timeout::<subxt::PolkadotConfig>(1_u32)
.await;
assert!(r.is_err());
// tear down (optional if you don't detach the network)
network.destroy().await.unwrap();
}

Some files were not shown because too many files have changed in this diff Show More