feat: Vendor pezkuwi-subxt and pezkuwi-zombienet-sdk into monorepo

- Add pezkuwi-subxt crates to vendor/pezkuwi-subxt
- Add pezkuwi-zombienet-sdk crates to vendor/pezkuwi-zombienet-sdk
- Convert git dependencies to path dependencies
- Add vendor crates to workspace members
- Remove test/example crates from vendor (not needed for SDK)
- Fix feature propagation issues detected by zepter
- Fix workspace inheritance for internal dependencies
- All 606 crates now in workspace
- All 6919 internal dependency links verified correct
- No git dependencies remaining
This commit is contained in:
2025-12-22 23:31:24 +03:00
parent 4c8f281051
commit 70ddb6516f
386 changed files with 76759 additions and 36 deletions
@@ -0,0 +1,2 @@
/target
/Cargo.lock
+47
View File
@@ -0,0 +1,47 @@
[package]
name = "zombienet-provider"
version.workspace = true
authors.workspace = true
edition.workspace = true
rust-version.workspace = true
publish = true
license.workspace = true
repository.workspace = true
description = "Zombienet provider, implement the logic to run the nodes in the native provider"
keywords = ["zombienet", "provider", "native"]
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
async-trait = { workspace = true }
futures = { workspace = true }
serde = { workspace = true, features = ["derive", "rc"] }
serde_json = { workspace = true }
serde_yaml = { workspace = true }
tokio = { workspace = true, features = [
"process",
"macros",
"fs",
"time",
"rt",
] }
tokio-util = { workspace = true, features = ["compat"] }
thiserror = { workspace = true }
anyhow = { workspace = true }
uuid = { workspace = true, features = ["v4"] }
nix = { workspace = true, features = ["signal"] }
kube = { workspace = true, features = ["ws", "runtime"] }
k8s-openapi = { workspace = true, features = ["v1_27"] }
tar = { workspace = true }
sha2 = { workspace = true }
hex = { workspace = true }
tracing = { workspace = true }
reqwest = { workspace = true }
regex = { workspace = true }
url = { workspace = true }
flate2 = { workspace = true }
erased-serde = { workspace = true }
# Zomebienet deps
support = { workspace = true }
configuration = { workspace = true }
@@ -0,0 +1,6 @@
mod client;
mod namespace;
mod node;
mod provider;
pub use provider::DockerProvider;
@@ -0,0 +1,596 @@
use std::{collections::HashMap, path::Path, process::Stdio};
use anyhow::anyhow;
use futures::future::try_join_all;
use serde::{Deserialize, Deserializer};
use tokio::process::Command;
use tracing::{info, trace};
use crate::types::{ExecutionResult, Port};
#[derive(thiserror::Error, Debug)]
#[error(transparent)]
pub struct Error(#[from] anyhow::Error);
pub type Result<T> = core::result::Result<T, Error>;
#[derive(Clone)]
pub struct DockerClient {
using_podman: bool,
}
#[derive(Debug)]
pub struct ContainerRunOptions {
image: String,
command: Vec<String>,
env: Option<Vec<(String, String)>>,
volume_mounts: Option<HashMap<String, String>>,
name: Option<String>,
entrypoint: Option<String>,
port_mapping: HashMap<Port, Port>,
rm: bool,
detach: bool,
}
enum Container {
Docker(DockerContainer),
Podman(PodmanContainer),
}
// TODO: we may don't need this
#[allow(dead_code)]
#[derive(Deserialize, Debug)]
struct DockerContainer {
#[serde(alias = "Names", deserialize_with = "deserialize_list")]
names: Vec<String>,
#[serde(alias = "Ports", deserialize_with = "deserialize_list")]
ports: Vec<String>,
#[serde(alias = "State")]
state: String,
}
// TODO: we may don't need this
#[allow(dead_code)]
#[derive(Deserialize, Debug)]
struct PodmanPort {
host_ip: String,
container_port: u16,
host_port: u16,
range: u16,
protocol: String,
}
// TODO: we may don't need this
#[allow(dead_code)]
#[derive(Deserialize, Debug)]
struct PodmanContainer {
#[serde(alias = "Id")]
id: String,
#[serde(alias = "Image")]
image: String,
#[serde(alias = "Mounts")]
mounts: Vec<String>,
#[serde(alias = "Names")]
names: Vec<String>,
#[serde(alias = "Ports", deserialize_with = "deserialize_null_as_default")]
ports: Vec<PodmanPort>,
#[serde(alias = "State")]
state: String,
}
fn deserialize_list<'de, D>(deserializer: D) -> std::result::Result<Vec<String>, D::Error>
where
D: Deserializer<'de>,
{
let str_sequence = String::deserialize(deserializer)?;
Ok(str_sequence
.split(',')
.filter(|item| !item.is_empty())
.map(|item| item.to_owned())
.collect())
}
fn deserialize_null_as_default<'de, D, T>(deserializer: D) -> std::result::Result<T, D::Error>
where
T: Default + Deserialize<'de>,
D: Deserializer<'de>,
{
let opt = Option::deserialize(deserializer)?;
Ok(opt.unwrap_or_default())
}
impl ContainerRunOptions {
pub fn new<S>(image: &str, command: Vec<S>) -> Self
where
S: Into<String> + std::fmt::Debug + Send + Clone,
{
ContainerRunOptions {
image: image.to_string(),
command: command
.clone()
.into_iter()
.map(|s| s.into())
.collect::<Vec<_>>(),
env: None,
volume_mounts: None,
name: None,
entrypoint: None,
port_mapping: HashMap::default(),
rm: false,
detach: true, // add -d flag by default
}
}
pub fn env<S>(mut self, env: Vec<(S, S)>) -> Self
where
S: Into<String> + std::fmt::Debug + Send + Clone,
{
self.env = Some(
env.into_iter()
.map(|(name, value)| (name.into(), value.into()))
.collect(),
);
self
}
pub fn volume_mounts<S>(mut self, volume_mounts: HashMap<S, S>) -> Self
where
S: Into<String> + std::fmt::Debug + Send + Clone,
{
self.volume_mounts = Some(
volume_mounts
.into_iter()
.map(|(source, target)| (source.into(), target.into()))
.collect(),
);
self
}
pub fn name<S>(mut self, name: S) -> Self
where
S: Into<String> + std::fmt::Debug + Send + Clone,
{
self.name = Some(name.into());
self
}
pub fn entrypoint<S>(mut self, entrypoint: S) -> Self
where
S: Into<String> + std::fmt::Debug + Send + Clone,
{
self.entrypoint = Some(entrypoint.into());
self
}
pub fn port_mapping(mut self, port_mapping: &HashMap<Port, Port>) -> Self {
self.port_mapping.clone_from(port_mapping);
self
}
pub fn rm(mut self) -> Self {
self.rm = true;
self
}
pub fn detach(mut self, choice: bool) -> Self {
self.detach = choice;
self
}
}
impl DockerClient {
pub async fn new() -> Result<Self> {
let using_podman = Self::is_using_podman().await?;
Ok(DockerClient { using_podman })
}
pub fn client_binary(&self) -> String {
String::from(if self.using_podman {
"podman"
} else {
"docker"
})
}
async fn is_using_podman() -> Result<bool> {
if let Ok(output) = tokio::process::Command::new("docker")
.arg("version")
.output()
.await
{
// detect whether we're actually running podman with docker emulation
return Ok(String::from_utf8_lossy(&output.stdout)
.to_lowercase()
.contains("podman"));
}
tokio::process::Command::new("podman")
.arg("--version")
.output()
.await
.map_err(|err| anyhow!("Failed to detect container engine: {err}"))?;
Ok(true)
}
}
impl DockerClient {
fn client_command(&self) -> tokio::process::Command {
tokio::process::Command::new(self.client_binary())
}
pub async fn create_volume(&self, name: &str) -> Result<()> {
let result = self
.client_command()
.args(["volume", "create", name])
.output()
.await
.map_err(|err| anyhow!("Failed to create volume '{name}': {err}"))?;
if !result.status.success() {
return Err(anyhow!(
"Failed to create volume '{name}': {}",
String::from_utf8_lossy(&result.stderr)
)
.into());
}
Ok(())
}
pub async fn container_run(&self, options: ContainerRunOptions) -> Result<String> {
let mut cmd = self.client_command();
cmd.args(["run", "--platform", "linux/amd64"]);
if options.detach {
cmd.arg("-d");
}
Self::apply_cmd_options(&mut cmd, &options);
trace!("cmd: {:?}", cmd);
let result = cmd.output().await.map_err(|err| {
anyhow!(
"Failed to run container with image '{image}' and command '{command}': {err}",
image = options.image,
command = options.command.join(" "),
)
})?;
if !result.status.success() {
return Err(anyhow!(
"Failed to run container with image '{image}' and command '{command}': {err}",
image = options.image,
command = options.command.join(" "),
err = String::from_utf8_lossy(&result.stderr)
)
.into());
}
Ok(String::from_utf8_lossy(&result.stdout).to_string())
}
pub async fn container_create(&self, options: ContainerRunOptions) -> Result<String> {
let mut cmd = self.client_command();
cmd.args(["container", "create"]);
Self::apply_cmd_options(&mut cmd, &options);
trace!("cmd: {:?}", cmd);
let result = cmd.output().await.map_err(|err| {
anyhow!(
"Failed to run container with image '{image}' and command '{command}': {err}",
image = options.image,
command = options.command.join(" "),
)
})?;
if !result.status.success() {
return Err(anyhow!(
"Failed to run container with image '{image}' and command '{command}': {err}",
image = options.image,
command = options.command.join(" "),
err = String::from_utf8_lossy(&result.stderr)
)
.into());
}
Ok(String::from_utf8_lossy(&result.stdout).to_string())
}
pub async fn container_exec<S>(
&self,
name: &str,
command: Vec<S>,
env: Option<Vec<(S, S)>>,
as_user: Option<S>,
) -> Result<ExecutionResult>
where
S: Into<String> + std::fmt::Debug + Send + Clone,
{
let mut cmd = self.client_command();
cmd.arg("exec");
if let Some(env) = env {
for env_var in env {
cmd.args(["-e", &format!("{}={}", env_var.0.into(), env_var.1.into())]);
}
}
if let Some(user) = as_user {
cmd.args(["-u", user.into().as_ref()]);
}
cmd.arg(name);
cmd.args(
command
.clone()
.into_iter()
.map(|s| <S as Into<String>>::into(s)),
);
trace!("cmd is : {:?}", cmd);
let result = cmd.output().await.map_err(|err| {
anyhow!(
"Failed to exec '{}' on '{}': {err}",
command
.into_iter()
.map(|s| <S as Into<String>>::into(s))
.collect::<Vec<_>>()
.join(" "),
name,
)
})?;
if !result.status.success() {
return Ok(Err((
result.status,
String::from_utf8_lossy(&result.stderr).to_string(),
)));
}
Ok(Ok(String::from_utf8_lossy(&result.stdout).to_string()))
}
pub async fn container_cp(
&self,
name: &str,
local_path: &Path,
remote_path: &Path,
) -> Result<()> {
let result = self
.client_command()
.args([
"cp",
local_path.to_string_lossy().as_ref(),
&format!("{name}:{}", remote_path.to_string_lossy().as_ref()),
])
.output()
.await
.map_err(|err| {
anyhow!(
"Failed copy file '{file}' to container '{name}': {err}",
file = local_path.to_string_lossy(),
)
})?;
if !result.status.success() {
return Err(anyhow!(
"Failed to copy file '{file}' to container '{name}': {err}",
file = local_path.to_string_lossy(),
err = String::from_utf8_lossy(&result.stderr)
)
.into());
}
Ok(())
}
pub async fn container_rm(&self, name: &str) -> Result<()> {
let result = self
.client_command()
.args(["rm", "--force", "--volumes", name])
.output()
.await
.map_err(|err| anyhow!("Failed do remove container '{name}: {err}"))?;
if !result.status.success() {
return Err(anyhow!(
"Failed to remove container '{name}': {err}",
err = String::from_utf8_lossy(&result.stderr)
)
.into());
}
Ok(())
}
pub async fn namespaced_containers_rm(&self, namespace: &str) -> Result<()> {
let container_names: Vec<String> = self
.get_containers()
.await?
.into_iter()
.filter_map(|container| match container {
Container::Docker(container) => {
if let Some(name) = container.names.first() {
if name.starts_with(namespace) {
return Some(name.to_string());
}
}
None
},
Container::Podman(container) => {
if let Some(name) = container.names.first() {
if name.starts_with(namespace) {
return Some(name.to_string());
}
}
None
},
})
.collect();
info!("{:?}", container_names);
let futures = container_names
.iter()
.map(|name| self.container_rm(name))
.collect::<Vec<_>>();
try_join_all(futures).await?;
Ok(())
}
pub async fn container_ip(&self, container_name: &str) -> Result<String> {
let ip = if self.using_podman {
"127.0.0.1".into()
} else {
let mut cmd = tokio::process::Command::new("docker");
cmd.args(vec![
"inspect",
"-f",
"{{ .NetworkSettings.IPAddress }}",
container_name,
]);
trace!("CMD: {cmd:?}");
let res = cmd
.output()
.await
.map_err(|err| anyhow!("Failed to get docker container ip, output: {err}"))?;
String::from_utf8(res.stdout)
.map_err(|err| anyhow!("Failed to get docker container ip, output: {err}"))?
.trim()
.into()
};
trace!("IP: {ip}");
Ok(ip)
}
async fn get_containers(&self) -> Result<Vec<Container>> {
let containers = if self.using_podman {
self.get_podman_containers()
.await?
.into_iter()
.map(Container::Podman)
.collect()
} else {
self.get_docker_containers()
.await?
.into_iter()
.map(Container::Docker)
.collect()
};
Ok(containers)
}
async fn get_podman_containers(&self) -> Result<Vec<PodmanContainer>> {
let res = tokio::process::Command::new("podman")
.args(vec!["ps", "--all", "--no-trunc", "--format", "json"])
.output()
.await
.map_err(|err| anyhow!("Failed to get podman containers output: {err}"))?;
let stdout = String::from_utf8_lossy(&res.stdout);
let containers = serde_json::from_str(&stdout)
.map_err(|err| anyhow!("Failed to parse podman containers output: {err}"))?;
Ok(containers)
}
async fn get_docker_containers(&self) -> Result<Vec<DockerContainer>> {
let res = tokio::process::Command::new("docker")
.args(vec!["ps", "--all", "--no-trunc", "--format", "json"])
.output()
.await
.unwrap();
let stdout = String::from_utf8_lossy(&res.stdout);
let mut containers = vec![];
for line in stdout.lines() {
containers.push(
serde_json::from_str::<DockerContainer>(line)
.map_err(|err| anyhow!("Failed to parse docker container output: {err}"))?,
);
}
Ok(containers)
}
pub(crate) async fn container_logs(&self, container_name: &str) -> Result<String> {
let output = Command::new("sh")
.arg("-c")
.arg(format!("docker logs -t '{container_name}' 2>&1"))
.stdout(Stdio::piped())
.output()
.await
.map_err(|err| {
anyhow!(
"Failed to spawn docker logs command for container '{container_name}': {err}"
)
})?;
let logs = String::from_utf8_lossy(&output.stdout).to_string();
if !output.status.success() {
// stderr was redirected to stdout, so logs should contain the error message if any
return Err(anyhow!(
"Failed to get logs for container '{name}': {logs}",
name = container_name,
logs = &logs
)
.into());
}
Ok(logs)
}
fn apply_cmd_options(cmd: &mut Command, options: &ContainerRunOptions) {
if options.rm {
cmd.arg("--rm");
}
if let Some(entrypoint) = options.entrypoint.as_ref() {
cmd.args(["--entrypoint", entrypoint]);
}
if let Some(volume_mounts) = options.volume_mounts.as_ref() {
for (source, target) in volume_mounts {
cmd.args(["-v", &format!("{source}:{target}")]);
}
}
if let Some(env) = options.env.as_ref() {
for env_var in env {
cmd.args(["-e", &format!("{}={}", env_var.0, env_var.1)]);
}
}
// add published ports
for (container_port, host_port) in options.port_mapping.iter() {
cmd.args(["-p", &format!("{host_port}:{container_port}")]);
}
if let Some(name) = options.name.as_ref() {
cmd.args(["--name", name]);
}
cmd.arg(&options.image);
for arg in &options.command {
cmd.arg(arg);
}
}
}
@@ -0,0 +1,494 @@
use std::{
collections::HashMap,
path::{Path, PathBuf},
sync::{Arc, Weak},
thread,
};
use async_trait::async_trait;
use support::{constants::THIS_IS_A_BUG, fs::FileSystem};
use tokio::sync::{Mutex, RwLock};
use tracing::{debug, trace, warn};
use uuid::Uuid;
use super::{
client::{ContainerRunOptions, DockerClient},
node::DockerNode,
DockerProvider,
};
use crate::{
constants::NAMESPACE_PREFIX,
docker::{
node::{DeserializableDockerNodeOptions, DockerNodeOptions},
provider,
},
shared::helpers::extract_execution_result,
types::{
GenerateFileCommand, GenerateFilesOptions, ProviderCapabilities, RunCommandOptions,
SpawnNodeOptions,
},
DynNode, ProviderError, ProviderNamespace, ProviderNode,
};
pub struct DockerNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
weak: Weak<DockerNamespace<FS>>,
#[allow(dead_code)]
provider: Weak<DockerProvider<FS>>,
name: String,
base_dir: PathBuf,
capabilities: ProviderCapabilities,
docker_client: DockerClient,
filesystem: FS,
delete_on_drop: Arc<Mutex<bool>>,
pub(super) nodes: RwLock<HashMap<String, Arc<DockerNode<FS>>>>,
}
impl<FS> DockerNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) async fn new(
provider: &Weak<DockerProvider<FS>>,
tmp_dir: &PathBuf,
capabilities: &ProviderCapabilities,
docker_client: &DockerClient,
filesystem: &FS,
custom_base_dir: Option<&Path>,
) -> Result<Arc<Self>, ProviderError> {
let name = format!("{}{}", NAMESPACE_PREFIX, Uuid::new_v4());
let base_dir = if let Some(custom_base_dir) = custom_base_dir {
if !filesystem.exists(custom_base_dir).await {
filesystem.create_dir(custom_base_dir).await?;
} else {
warn!(
"⚠️ Using and existing directory {} as base dir",
custom_base_dir.to_string_lossy()
);
}
PathBuf::from(custom_base_dir)
} else {
let base_dir = PathBuf::from_iter([tmp_dir, &PathBuf::from(&name)]);
filesystem.create_dir(&base_dir).await?;
base_dir
};
let namespace = Arc::new_cyclic(|weak| DockerNamespace {
weak: weak.clone(),
provider: provider.clone(),
name,
base_dir,
capabilities: capabilities.clone(),
filesystem: filesystem.clone(),
docker_client: docker_client.clone(),
nodes: RwLock::new(HashMap::new()),
delete_on_drop: Arc::new(Mutex::new(true)),
});
namespace.initialize().await?;
Ok(namespace)
}
pub(super) async fn attach_to_live(
provider: &Weak<DockerProvider<FS>>,
capabilities: &ProviderCapabilities,
docker_client: &DockerClient,
filesystem: &FS,
custom_base_dir: &Path,
name: &str,
) -> Result<Arc<Self>, ProviderError> {
let base_dir = custom_base_dir.to_path_buf();
let namespace = Arc::new_cyclic(|weak| DockerNamespace {
weak: weak.clone(),
provider: provider.clone(),
name: name.to_owned(),
base_dir,
capabilities: capabilities.clone(),
filesystem: filesystem.clone(),
docker_client: docker_client.clone(),
nodes: RwLock::new(HashMap::new()),
delete_on_drop: Arc::new(Mutex::new(false)),
});
Ok(namespace)
}
async fn initialize(&self) -> Result<(), ProviderError> {
// let ns_scripts_shared = PathBuf::from_iter([&self.base_dir, &PathBuf::from("shared-scripts")]);
// self.filesystem.create_dir(&ns_scripts_shared).await?;
self.initialize_zombie_scripts_volume().await?;
self.initialize_helper_binaries_volume().await?;
Ok(())
}
async fn initialize_zombie_scripts_volume(&self) -> Result<(), ProviderError> {
let local_zombie_wrapper_path =
PathBuf::from_iter([&self.base_dir, &PathBuf::from("zombie-wrapper.sh")]);
self.filesystem
.write(
&local_zombie_wrapper_path,
include_str!("../shared/scripts/zombie-wrapper.sh"),
)
.await?;
let local_helper_binaries_downloader_path = PathBuf::from_iter([
&self.base_dir,
&PathBuf::from("helper-binaries-downloader.sh"),
]);
self.filesystem
.write(
&local_helper_binaries_downloader_path,
include_str!("../shared/scripts/helper-binaries-downloader.sh"),
)
.await?;
let zombie_wrapper_volume_name = format!("{}-zombie-wrapper", self.name);
let zombie_wrapper_container_name = format!("{}-scripts", self.name);
self.docker_client
.create_volume(&zombie_wrapper_volume_name)
.await
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
self.docker_client
.container_create(
ContainerRunOptions::new("alpine:latest", vec!["tail", "-f", "/dev/null"])
.volume_mounts(HashMap::from([(
zombie_wrapper_volume_name.as_str(),
"/scripts",
)]))
.name(&zombie_wrapper_container_name)
.rm(),
)
.await
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
// copy the scripts
self.docker_client
.container_cp(
&zombie_wrapper_container_name,
&local_zombie_wrapper_path,
&PathBuf::from("/scripts/zombie-wrapper.sh"),
)
.await
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
self.docker_client
.container_cp(
&zombie_wrapper_container_name,
&local_helper_binaries_downloader_path,
&PathBuf::from("/scripts/helper-binaries-downloader.sh"),
)
.await
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
// set permissions for rwx on whole volume recursively
self.docker_client
.container_run(
ContainerRunOptions::new("alpine:latest", vec!["chmod", "-R", "777", "/scripts"])
.volume_mounts(HashMap::from([(
zombie_wrapper_volume_name.as_ref(),
"/scripts",
)]))
.rm(),
)
.await
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
Ok(())
}
async fn initialize_helper_binaries_volume(&self) -> Result<(), ProviderError> {
let helper_binaries_volume_name = format!("{}-helper-binaries", self.name);
let zombie_wrapper_volume_name = format!("{}-zombie-wrapper", self.name);
self.docker_client
.create_volume(&helper_binaries_volume_name)
.await
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
// download binaries to volume
self.docker_client
.container_run(
ContainerRunOptions::new(
"alpine:latest",
vec!["ash", "/scripts/helper-binaries-downloader.sh"],
)
.volume_mounts(HashMap::from([
(
helper_binaries_volume_name.as_str(),
"/helpers",
),
(
zombie_wrapper_volume_name.as_ref(),
"/scripts",
)
]))
// wait until complete
.detach(false)
.rm(),
)
.await
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
// set permissions for rwx on whole volume recursively
self.docker_client
.container_run(
ContainerRunOptions::new("alpine:latest", vec!["chmod", "-R", "777", "/helpers"])
.volume_mounts(HashMap::from([(
helper_binaries_volume_name.as_ref(),
"/helpers",
)]))
.rm(),
)
.await
.map_err(|err| ProviderError::CreateNamespaceFailed(self.name.clone(), err.into()))?;
Ok(())
}
pub async fn set_delete_on_drop(&self, delete_on_drop: bool) {
*self.delete_on_drop.lock().await = delete_on_drop;
}
pub async fn delete_on_drop(&self) -> bool {
if let Ok(delete_on_drop) = self.delete_on_drop.try_lock() {
*delete_on_drop
} else {
// if we can't lock just remove the ns
true
}
}
}
#[async_trait]
impl<FS> ProviderNamespace for DockerNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
fn name(&self) -> &str {
&self.name
}
fn base_dir(&self) -> &PathBuf {
&self.base_dir
}
fn capabilities(&self) -> &ProviderCapabilities {
&self.capabilities
}
fn provider_name(&self) -> &str {
provider::PROVIDER_NAME
}
async fn detach(&self) {
self.set_delete_on_drop(false).await;
}
async fn is_detached(&self) -> bool {
self.delete_on_drop().await
}
async fn nodes(&self) -> HashMap<String, DynNode> {
self.nodes
.read()
.await
.iter()
.map(|(name, node)| (name.clone(), node.clone() as DynNode))
.collect()
}
async fn get_node_available_args(
&self,
(command, image): (String, Option<String>),
) -> Result<String, ProviderError> {
let node_image = image.expect(&format!("image should be present when getting node available args with docker provider {THIS_IS_A_BUG}"));
let temp_node = self
.spawn_node(
&SpawnNodeOptions::new(format!("temp-{}", Uuid::new_v4()), "cat".to_string())
.image(node_image.clone()),
)
.await?;
let available_args_output = temp_node
.run_command(RunCommandOptions::new(command.clone()).args(vec!["--help"]))
.await?
.map_err(|(_exit, status)| {
ProviderError::NodeAvailableArgsError(node_image, command, status)
})?;
temp_node.destroy().await?;
Ok(available_args_output)
}
async fn spawn_node(&self, options: &SpawnNodeOptions) -> Result<DynNode, ProviderError> {
debug!("spawn option {:?}", options);
let node = DockerNode::new(DockerNodeOptions {
namespace: &self.weak,
namespace_base_dir: &self.base_dir,
name: &options.name,
image: options.image.as_ref(),
program: &options.program,
args: &options.args,
env: &options.env,
startup_files: &options.injected_files,
db_snapshot: options.db_snapshot.as_ref(),
docker_client: &self.docker_client,
container_name: format!("{}-{}", self.name, options.name),
filesystem: &self.filesystem,
port_mapping: options.port_mapping.as_ref().unwrap_or(&HashMap::default()),
})
.await?;
self.nodes
.write()
.await
.insert(node.name().to_string(), node.clone());
Ok(node)
}
async fn spawn_node_from_json(
&self,
json_value: &serde_json::Value,
) -> Result<DynNode, ProviderError> {
let deserializable: DeserializableDockerNodeOptions =
serde_json::from_value(json_value.clone())?;
let options = DockerNodeOptions::from_deserializable(
&deserializable,
&self.weak,
&self.base_dir,
&self.docker_client,
&self.filesystem,
);
let node = DockerNode::attach_to_live(options).await?;
self.nodes
.write()
.await
.insert(node.name().to_string(), node.clone());
Ok(node)
}
async fn generate_files(&self, options: GenerateFilesOptions) -> Result<(), ProviderError> {
debug!("generate files options {options:#?}");
let node_name = options
.temp_name
.unwrap_or_else(|| format!("temp-{}", Uuid::new_v4()));
let node_image = options.image.expect(&format!(
"image should be present when generating files with docker provider {THIS_IS_A_BUG}"
));
// run dummy command in a new container
let temp_node = self
.spawn_node(
&SpawnNodeOptions::new(node_name, "cat".to_string())
.injected_files(options.injected_files)
.image(node_image),
)
.await?;
for GenerateFileCommand {
program,
args,
env,
local_output_path,
} in options.commands
{
let local_output_full_path = format!(
"{}{}{}",
self.base_dir.to_string_lossy(),
if local_output_path.starts_with("/") {
""
} else {
"/"
},
local_output_path.to_string_lossy()
);
let contents = extract_execution_result(
&temp_node,
RunCommandOptions { program, args, env },
options.expected_path.as_ref(),
)
.await?;
self.filesystem
.write(local_output_full_path, contents)
.await
.map_err(|err| ProviderError::FileGenerationFailed(err.into()))?;
}
temp_node.destroy().await
}
async fn static_setup(&self) -> Result<(), ProviderError> {
todo!()
}
async fn destroy(&self) -> Result<(), ProviderError> {
let _ = self
.docker_client
.namespaced_containers_rm(&self.name)
.await
.map_err(|err| ProviderError::DeleteNamespaceFailed(self.name.clone(), err.into()))?;
if let Some(provider) = self.provider.upgrade() {
provider.namespaces.write().await.remove(&self.name);
}
Ok(())
}
}
impl<FS> Drop for DockerNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
fn drop(&mut self) {
let ns_name = self.name.clone();
if let Ok(delete_on_drop) = self.delete_on_drop.try_lock() {
if *delete_on_drop {
let client = self.docker_client.clone();
let provider = self.provider.upgrade();
let handler = thread::spawn(move || {
let rt = tokio::runtime::Runtime::new().unwrap();
rt.block_on(async move {
trace!("🧟 deleting ns {ns_name} from cluster");
let _ = client.namespaced_containers_rm(&ns_name).await;
trace!("✅ deleted");
});
});
if handler.join().is_ok() {
if let Some(provider) = provider {
if let Ok(mut p) = provider.namespaces.try_write() {
p.remove(&self.name);
} else {
warn!(
"⚠️ Can not acquire write lock to the provider, ns {} not removed",
self.name
);
}
}
}
} else {
trace!("⚠️ leaking ns {ns_name} in cluster");
}
};
}
}
@@ -0,0 +1,659 @@
use std::{
collections::HashMap,
net::IpAddr,
path::{Component, Path, PathBuf},
sync::{Arc, Weak},
time::Duration,
};
use anyhow::anyhow;
use async_trait::async_trait;
use configuration::types::AssetLocation;
use futures::future::try_join_all;
use serde::{Deserialize, Serialize};
use support::{constants::THIS_IS_A_BUG, fs::FileSystem};
use tokio::{time::sleep, try_join};
use tracing::debug;
use super::{
client::{ContainerRunOptions, DockerClient},
namespace::DockerNamespace,
};
use crate::{
constants::{NODE_CONFIG_DIR, NODE_DATA_DIR, NODE_RELAY_DATA_DIR, NODE_SCRIPTS_DIR},
docker,
types::{ExecutionResult, Port, RunCommandOptions, RunScriptOptions, TransferedFile},
ProviderError, ProviderNamespace, ProviderNode,
};
pub(super) struct DockerNodeOptions<'a, FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) namespace: &'a Weak<DockerNamespace<FS>>,
pub(super) namespace_base_dir: &'a PathBuf,
pub(super) name: &'a str,
pub(super) image: Option<&'a String>,
pub(super) program: &'a str,
pub(super) args: &'a [String],
pub(super) env: &'a [(String, String)],
pub(super) startup_files: &'a [TransferedFile],
pub(super) db_snapshot: Option<&'a AssetLocation>,
pub(super) docker_client: &'a DockerClient,
pub(super) container_name: String,
pub(super) filesystem: &'a FS,
pub(super) port_mapping: &'a HashMap<Port, Port>,
}
impl<'a, FS> DockerNodeOptions<'a, FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub fn from_deserializable(
deserializable: &'a DeserializableDockerNodeOptions,
namespace: &'a Weak<DockerNamespace<FS>>,
namespace_base_dir: &'a PathBuf,
docker_client: &'a DockerClient,
filesystem: &'a FS,
) -> Self {
DockerNodeOptions {
namespace,
namespace_base_dir,
name: &deserializable.name,
image: deserializable.image.as_ref(),
program: &deserializable.program,
args: &deserializable.args,
env: &deserializable.env,
startup_files: &[],
db_snapshot: None,
docker_client,
container_name: deserializable.container_name.clone(),
filesystem,
port_mapping: &deserializable.port_mapping,
}
}
}
#[derive(Deserialize)]
pub(super) struct DeserializableDockerNodeOptions {
pub(super) name: String,
pub(super) image: Option<String>,
pub(super) program: String,
pub(super) args: Vec<String>,
pub(super) env: Vec<(String, String)>,
pub(super) container_name: String,
pub(super) port_mapping: HashMap<Port, Port>,
}
#[derive(Serialize)]
pub struct DockerNode<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
#[serde(skip)]
namespace: Weak<DockerNamespace<FS>>,
name: String,
image: String,
program: String,
args: Vec<String>,
env: Vec<(String, String)>,
base_dir: PathBuf,
config_dir: PathBuf,
data_dir: PathBuf,
relay_data_dir: PathBuf,
scripts_dir: PathBuf,
log_path: PathBuf,
#[serde(skip)]
docker_client: DockerClient,
container_name: String,
port_mapping: HashMap<Port, Port>,
#[allow(dead_code)]
#[serde(skip)]
filesystem: FS,
provider_tag: String,
}
impl<FS> DockerNode<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) async fn new(
options: DockerNodeOptions<'_, FS>,
) -> Result<Arc<Self>, ProviderError> {
let image = options.image.ok_or_else(|| {
ProviderError::MissingNodeInfo(options.name.to_string(), "missing image".to_string())
})?;
let filesystem = options.filesystem.clone();
let base_dir =
PathBuf::from_iter([options.namespace_base_dir, &PathBuf::from(options.name)]);
filesystem.create_dir_all(&base_dir).await?;
let base_dir_raw = base_dir.to_string_lossy();
let config_dir = PathBuf::from(format!("{base_dir_raw}{NODE_CONFIG_DIR}"));
let data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_DATA_DIR}"));
let relay_data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_RELAY_DATA_DIR}"));
let scripts_dir = PathBuf::from(format!("{base_dir_raw}{NODE_SCRIPTS_DIR}"));
let log_path = base_dir.join("node.log");
try_join!(
filesystem.create_dir_all(&config_dir),
filesystem.create_dir_all(&data_dir),
filesystem.create_dir_all(&relay_data_dir),
filesystem.create_dir_all(&scripts_dir),
)?;
let node = Arc::new(DockerNode {
namespace: options.namespace.clone(),
name: options.name.to_string(),
image: image.to_string(),
program: options.program.to_string(),
args: options.args.to_vec(),
env: options.env.to_vec(),
base_dir,
config_dir,
data_dir,
relay_data_dir,
scripts_dir,
log_path,
filesystem: filesystem.clone(),
docker_client: options.docker_client.clone(),
container_name: options.container_name,
port_mapping: options.port_mapping.clone(),
provider_tag: docker::provider::PROVIDER_NAME.to_string(),
});
node.initialize_docker().await?;
if let Some(db_snap) = options.db_snapshot {
node.initialize_db_snapshot(db_snap).await?;
}
node.initialize_startup_files(options.startup_files).await?;
node.start().await?;
Ok(node)
}
pub(super) async fn attach_to_live(
options: DockerNodeOptions<'_, FS>,
) -> Result<Arc<Self>, ProviderError> {
let image = options.image.ok_or_else(|| {
ProviderError::MissingNodeInfo(options.name.to_string(), "missing image".to_string())
})?;
let filesystem = options.filesystem.clone();
let base_dir =
PathBuf::from_iter([options.namespace_base_dir, &PathBuf::from(options.name)]);
filesystem.create_dir_all(&base_dir).await?;
let base_dir_raw = base_dir.to_string_lossy();
let config_dir = PathBuf::from(format!("{base_dir_raw}{NODE_CONFIG_DIR}"));
let data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_DATA_DIR}"));
let relay_data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_RELAY_DATA_DIR}"));
let scripts_dir = PathBuf::from(format!("{base_dir_raw}{NODE_SCRIPTS_DIR}"));
let log_path = base_dir.join("node.log");
let node = Arc::new(DockerNode {
namespace: options.namespace.clone(),
name: options.name.to_string(),
image: image.to_string(),
program: options.program.to_string(),
args: options.args.to_vec(),
env: options.env.to_vec(),
base_dir,
config_dir,
data_dir,
relay_data_dir,
scripts_dir,
log_path,
filesystem: filesystem.clone(),
docker_client: options.docker_client.clone(),
container_name: options.container_name,
port_mapping: options.port_mapping.clone(),
provider_tag: docker::provider::PROVIDER_NAME.to_string(),
});
Ok(node)
}
async fn initialize_docker(&self) -> Result<(), ProviderError> {
let command = [vec![self.program.to_string()], self.args.to_vec()].concat();
self.docker_client
.container_run(
ContainerRunOptions::new(&self.image, command)
.name(&self.container_name)
.env(self.env.clone())
.volume_mounts(HashMap::from([
(
format!("{}-zombie-wrapper", self.namespace_name(),),
"/scripts".to_string(),
),
(
format!("{}-helper-binaries", self.namespace_name()),
"/helpers".to_string(),
),
(
self.config_dir.to_string_lossy().into_owned(),
"/cfg".to_string(),
),
(
self.data_dir.to_string_lossy().into_owned(),
"/data".to_string(),
),
(
self.relay_data_dir.to_string_lossy().into_owned(),
"/relay-data".to_string(),
),
]))
.entrypoint("/scripts/zombie-wrapper.sh")
.port_mapping(&self.port_mapping),
)
.await
.map_err(|err| ProviderError::NodeSpawningFailed(self.name.clone(), err.into()))?;
// change dirs permission
let _ = self
.docker_client
.container_exec(
&self.container_name,
["chmod", "777", "/cfg", "/data", "/relay-data"].into(),
None,
Some("root"),
)
.await
.map_err(|err| ProviderError::NodeSpawningFailed(self.name.clone(), err.into()))?;
Ok(())
}
async fn initialize_db_snapshot(
&self,
_db_snapshot: &AssetLocation,
) -> Result<(), ProviderError> {
todo!()
// trace!("snap: {db_snapshot}");
// let url_of_snap = match db_snapshot {
// AssetLocation::Url(location) => location.clone(),
// AssetLocation::FilePath(filepath) => self.upload_to_fileserver(filepath).await?,
// };
// // we need to get the snapshot from a public access
// // and extract to /data
// let opts = RunCommandOptions::new("mkdir").args([
// "-p",
// "/data/",
// "&&",
// "mkdir",
// "-p",
// "/relay-data/",
// "&&",
// // Use our version of curl
// "/cfg/curl",
// url_of_snap.as_ref(),
// "--output",
// "/data/db.tgz",
// "&&",
// "cd",
// "/",
// "&&",
// "tar",
// "--skip-old-files",
// "-xzvf",
// "/data/db.tgz",
// ]);
// trace!("cmd opts: {:#?}", opts);
// let _ = self.run_command(opts).await?;
// Ok(())
}
async fn initialize_startup_files(
&self,
startup_files: &[TransferedFile],
) -> Result<(), ProviderError> {
try_join_all(
startup_files
.iter()
.map(|file| self.send_file(&file.local_path, &file.remote_path, &file.mode)),
)
.await?;
Ok(())
}
pub(super) async fn start(&self) -> Result<(), ProviderError> {
self.docker_client
.container_exec(
&self.container_name,
vec!["sh", "-c", "echo start > /tmp/zombiepipe"],
None,
None,
)
.await
.map_err(|err| {
ProviderError::NodeSpawningFailed(
format!("failed to start pod {} after spawning", self.name),
err.into(),
)
})?
.map_err(|err| {
ProviderError::NodeSpawningFailed(
format!("failed to start pod {} after spawning", self.name,),
anyhow!("command failed in container: status {}: {}", err.0, err.1),
)
})?;
Ok(())
}
fn get_remote_parent_dir(&self, remote_file_path: &Path) -> Option<PathBuf> {
if let Some(remote_parent_dir) = remote_file_path.parent() {
if matches!(
remote_parent_dir.components().rev().peekable().peek(),
Some(Component::Normal(_))
) {
return Some(remote_parent_dir.to_path_buf());
}
}
None
}
async fn create_remote_dir(&self, remote_dir: &Path) -> Result<(), ProviderError> {
let _ = self
.docker_client
.container_exec(
&self.container_name,
vec!["mkdir", "-p", &remote_dir.to_string_lossy()],
None,
None,
)
.await
.map_err(|err| {
ProviderError::NodeSpawningFailed(
format!(
"failed to create dir {} for container {}",
remote_dir.to_string_lossy(),
&self.name
),
err.into(),
)
})?;
Ok(())
}
fn namespace_name(&self) -> String {
self.namespace
.upgrade()
.map(|namespace| namespace.name().to_string())
.unwrap_or_else(|| panic!("namespace shouldn't be dropped, {THIS_IS_A_BUG}"))
}
}
#[async_trait]
impl<FS> ProviderNode for DockerNode<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
fn name(&self) -> &str {
&self.name
}
fn args(&self) -> Vec<&str> {
self.args.iter().map(|arg| arg.as_str()).collect()
}
fn base_dir(&self) -> &PathBuf {
&self.base_dir
}
fn config_dir(&self) -> &PathBuf {
&self.config_dir
}
fn data_dir(&self) -> &PathBuf {
&self.data_dir
}
fn relay_data_dir(&self) -> &PathBuf {
&self.relay_data_dir
}
fn scripts_dir(&self) -> &PathBuf {
&self.scripts_dir
}
fn log_path(&self) -> &PathBuf {
&self.log_path
}
fn log_cmd(&self) -> String {
format!(
"{} logs -f {}",
self.docker_client.client_binary(),
self.container_name
)
}
fn path_in_node(&self, file: &Path) -> PathBuf {
// here is just a noop op since we will receive the path
// for the file inside the pod
PathBuf::from(file)
}
async fn logs(&self) -> Result<String, ProviderError> {
self.docker_client
.container_logs(&self.container_name)
.await
.map_err(|err| ProviderError::GetLogsFailed(self.name.to_string(), err.into()))
}
async fn dump_logs(&self, local_dest: PathBuf) -> Result<(), ProviderError> {
let logs = self.logs().await?;
self.filesystem
.write(local_dest, logs)
.await
.map_err(|err| ProviderError::DumpLogsFailed(self.name.to_string(), err.into()))?;
Ok(())
}
async fn run_command(
&self,
options: RunCommandOptions,
) -> Result<ExecutionResult, ProviderError> {
debug!(
"running command for {} with options {:?}",
self.name, options
);
let command = [vec![options.program], options.args].concat();
self.docker_client
.container_exec(
&self.container_name,
vec!["sh", "-c", &command.join(" ")],
Some(
options
.env
.iter()
.map(|(k, v)| (k.as_ref(), v.as_ref()))
.collect(),
),
None,
)
.await
.map_err(|err| {
ProviderError::RunCommandError(
format!("sh -c {}", &command.join(" ")),
format!("in pod {}", self.name),
err.into(),
)
})
}
async fn run_script(
&self,
_options: RunScriptOptions,
) -> Result<ExecutionResult, ProviderError> {
todo!()
}
async fn send_file(
&self,
local_file_path: &Path,
remote_file_path: &Path,
mode: &str,
) -> Result<(), ProviderError> {
if let Some(remote_parent_dir) = self.get_remote_parent_dir(remote_file_path) {
self.create_remote_dir(&remote_parent_dir).await?;
}
debug!(
"starting sending file for {}: {} to {} with mode {}",
self.name,
local_file_path.to_string_lossy(),
remote_file_path.to_string_lossy(),
mode
);
let _ = self
.docker_client
.container_cp(&self.container_name, local_file_path, remote_file_path)
.await
.map_err(|err| {
ProviderError::SendFile(
local_file_path.to_string_lossy().to_string(),
self.name.clone(),
err.into(),
)
});
let _ = self
.docker_client
.container_exec(
&self.container_name,
vec!["chmod", mode, &remote_file_path.to_string_lossy()],
None,
None,
)
.await
.map_err(|err| {
ProviderError::SendFile(
self.name.clone(),
local_file_path.to_string_lossy().to_string(),
err.into(),
)
})?;
Ok(())
}
async fn receive_file(
&self,
_remote_src: &Path,
_local_dest: &Path,
) -> Result<(), ProviderError> {
Ok(())
}
async fn ip(&self) -> Result<IpAddr, ProviderError> {
let ip = self
.docker_client
.container_ip(&self.container_name)
.await
.map_err(|err| {
ProviderError::InvalidConfig(format!("Error getting container ip, err: {err}"))
})?;
Ok(ip.parse::<IpAddr>().map_err(|err| {
ProviderError::InvalidConfig(format!(
"Can not parse the container ip: {ip}, err: {err}"
))
})?)
}
async fn pause(&self) -> Result<(), ProviderError> {
self.docker_client
.container_exec(
&self.container_name,
vec!["sh", "-c", "echo pause > /tmp/zombiepipe"],
None,
None,
)
.await
.map_err(|err| ProviderError::PauseNodeFailed(self.name.to_string(), err.into()))?
.map_err(|err| {
ProviderError::PauseNodeFailed(
self.name.to_string(),
anyhow!("error when pausing node: status {}: {}", err.0, err.1),
)
})?;
Ok(())
}
async fn resume(&self) -> Result<(), ProviderError> {
self.docker_client
.container_exec(
&self.container_name,
vec!["sh", "-c", "echo resume > /tmp/zombiepipe"],
None,
None,
)
.await
.map_err(|err| ProviderError::PauseNodeFailed(self.name.to_string(), err.into()))?
.map_err(|err| {
ProviderError::PauseNodeFailed(
self.name.to_string(),
anyhow!("error when pausing node: status {}: {}", err.0, err.1),
)
})?;
Ok(())
}
async fn restart(&self, after: Option<Duration>) -> Result<(), ProviderError> {
if let Some(duration) = after {
sleep(duration).await;
}
self.docker_client
.container_exec(
&self.container_name,
vec!["sh", "-c", "echo restart > /tmp/zombiepipe"],
None,
None,
)
.await
.map_err(|err| ProviderError::PauseNodeFailed(self.name.to_string(), err.into()))?
.map_err(|err| {
ProviderError::PauseNodeFailed(
self.name.to_string(),
anyhow!("error when pausing node: status {}: {}", err.0, err.1),
)
})?;
Ok(())
}
async fn destroy(&self) -> Result<(), ProviderError> {
self.docker_client
.container_rm(&self.container_name)
.await
.map_err(|err| ProviderError::KillNodeFailed(self.name.to_string(), err.into()))?;
if let Some(namespace) = self.namespace.upgrade() {
namespace.nodes.write().await.remove(&self.name);
}
Ok(())
}
}
@@ -0,0 +1,161 @@
use std::{
collections::HashMap,
path::{Path, PathBuf},
sync::{Arc, Weak},
};
use async_trait::async_trait;
use support::fs::FileSystem;
use tokio::sync::RwLock;
use super::{client::DockerClient, namespace::DockerNamespace};
use crate::{
shared::helpers::extract_namespace_info, types::ProviderCapabilities, DynNamespace, Provider,
ProviderError, ProviderNamespace,
};
pub const PROVIDER_NAME: &str = "docker";
pub struct DockerProvider<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
weak: Weak<DockerProvider<FS>>,
capabilities: ProviderCapabilities,
tmp_dir: PathBuf,
docker_client: DockerClient,
filesystem: FS,
pub(super) namespaces: RwLock<HashMap<String, Arc<DockerNamespace<FS>>>>,
}
impl<FS> DockerProvider<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub async fn new(filesystem: FS) -> Arc<Self> {
let docker_client = DockerClient::new().await.unwrap();
let provider = Arc::new_cyclic(|weak| DockerProvider {
weak: weak.clone(),
capabilities: ProviderCapabilities {
requires_image: true,
has_resources: false,
prefix_with_full_path: false,
use_default_ports_in_cmd: true,
},
tmp_dir: std::env::temp_dir(),
docker_client,
filesystem,
namespaces: RwLock::new(HashMap::new()),
});
let cloned_provider = provider.clone();
tokio::spawn(async move {
tokio::signal::ctrl_c().await.unwrap();
for (_, ns) in cloned_provider.namespaces().await {
if ns.is_detached().await {
// best effort
let _ = ns.destroy().await;
}
}
// exit the process (130, SIGINT)
std::process::exit(130)
});
provider
}
pub fn tmp_dir(mut self, tmp_dir: impl Into<PathBuf>) -> Self {
self.tmp_dir = tmp_dir.into();
self
}
}
#[async_trait]
impl<FS> Provider for DockerProvider<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
fn name(&self) -> &str {
PROVIDER_NAME
}
fn capabilities(&self) -> &ProviderCapabilities {
&self.capabilities
}
async fn namespaces(&self) -> HashMap<String, DynNamespace> {
self.namespaces
.read()
.await
.iter()
.map(|(name, namespace)| (name.clone(), namespace.clone() as DynNamespace))
.collect()
}
async fn create_namespace(&self) -> Result<DynNamespace, ProviderError> {
let namespace = DockerNamespace::new(
&self.weak,
&self.tmp_dir,
&self.capabilities,
&self.docker_client,
&self.filesystem,
None,
)
.await?;
self.namespaces
.write()
.await
.insert(namespace.name().to_string(), namespace.clone());
Ok(namespace)
}
async fn create_namespace_with_base_dir(
&self,
base_dir: &Path,
) -> Result<DynNamespace, ProviderError> {
let namespace = DockerNamespace::new(
&self.weak,
&self.tmp_dir,
&self.capabilities,
&self.docker_client,
&self.filesystem,
Some(base_dir),
)
.await?;
self.namespaces
.write()
.await
.insert(namespace.name().to_string(), namespace.clone());
Ok(namespace)
}
async fn create_namespace_from_json(
&self,
json_value: &serde_json::Value,
) -> Result<DynNamespace, ProviderError> {
let (base_dir, name) = extract_namespace_info(json_value)?;
let namespace = DockerNamespace::attach_to_live(
&self.weak,
&self.capabilities,
&self.docker_client,
&self.filesystem,
&base_dir,
&name,
)
.await?;
self.namespaces
.write()
.await
.insert(namespace.name().to_string(), namespace.clone());
Ok(namespace)
}
}
@@ -0,0 +1,7 @@
mod client;
mod namespace;
mod node;
mod pod_spec_builder;
mod provider;
pub use provider::KubernetesProvider;
@@ -0,0 +1,602 @@
use std::{
collections::BTreeMap, fmt::Debug, os::unix::process::ExitStatusExt, process::ExitStatus,
time::Duration,
};
use anyhow::anyhow;
use futures::{StreamExt, TryStreamExt};
use k8s_openapi::api::core::v1::{
ConfigMap, Namespace, Pod, PodSpec, PodStatus, Service, ServiceSpec,
};
use kube::{
api::{AttachParams, DeleteParams, ListParams, LogParams, PostParams, WatchParams},
core::{DynamicObject, GroupVersionKind, ObjectMeta, TypeMeta, WatchEvent},
discovery::ApiResource,
runtime::{conditions, wait::await_condition},
Api, Resource,
};
use serde::de::DeserializeOwned;
use support::constants::THIS_IS_A_BUG;
use tokio::{
io::{AsyncRead, ErrorKind},
net::TcpListener,
task::JoinHandle,
};
use tokio_util::compat::FuturesAsyncReadCompatExt;
use tracing::{debug, trace};
use crate::{constants::LOCALHOST, types::ExecutionResult};
#[derive(thiserror::Error, Debug)]
#[error(transparent)]
pub struct Error(#[from] anyhow::Error);
pub type Result<T> = core::result::Result<T, Error>;
#[derive(Clone)]
pub struct KubernetesClient {
inner: kube::Client,
}
impl KubernetesClient {
pub(super) async fn new() -> Result<Self> {
Ok(Self {
// TODO: make it more flexible with path to kube config
inner: kube::Client::try_default()
.await
.map_err(|err| Error::from(anyhow!("error initializing kubers client: {err}")))?,
})
}
#[allow(dead_code)]
pub(super) async fn get_namespace(&self, name: &str) -> Result<Option<Namespace>> {
Api::<Namespace>::all(self.inner.clone())
.get_opt(name.as_ref())
.await
.map_err(|err| Error::from(anyhow!("error while getting namespace {name}: {err}")))
}
#[allow(dead_code)]
pub(super) async fn get_namespaces(&self) -> Result<Vec<Namespace>> {
Ok(Api::<Namespace>::all(self.inner.clone())
.list(&ListParams::default())
.await
.map_err(|err| Error::from(anyhow!("error while getting all namespaces: {err}")))?
.into_iter()
.filter(|ns| matches!(&ns.meta().name, Some(name) if name.starts_with("zombienet")))
.collect())
}
pub(super) async fn create_namespace(
&self,
name: &str,
labels: BTreeMap<String, String>,
) -> Result<Namespace> {
let namespaces = Api::<Namespace>::all(self.inner.clone());
let namespace = Namespace {
metadata: ObjectMeta {
name: Some(name.to_string()),
labels: Some(labels),
..Default::default()
},
..Default::default()
};
namespaces
.create(&PostParams::default(), &namespace)
.await
.map_err(|err| Error::from(anyhow!("error while created namespace {name}: {err}")))?;
self.wait_created(namespaces, name).await?;
Ok(namespace)
}
pub(super) async fn delete_namespace(&self, name: &str) -> Result<()> {
let namespaces = Api::<Namespace>::all(self.inner.clone());
namespaces
.delete(name, &DeleteParams::default())
.await
.map_err(|err| Error::from(anyhow!("error while deleting namespace {name}: {err}")))?;
Ok(())
}
pub(super) async fn create_config_map_from_file(
&self,
namespace: &str,
name: &str,
file_name: &str,
file_contents: &str,
labels: BTreeMap<String, String>,
) -> Result<ConfigMap> {
let config_maps = Api::<ConfigMap>::namespaced(self.inner.clone(), namespace);
let config_map = ConfigMap {
metadata: ObjectMeta {
name: Some(name.to_string()),
namespace: Some(namespace.to_string()),
labels: Some(labels),
..Default::default()
},
data: Some(BTreeMap::from([(
file_name.to_string(),
file_contents.to_string(),
)])),
..Default::default()
};
config_maps
.create(&PostParams::default(), &config_map)
.await
.map_err(|err| {
Error::from(anyhow!(
"error while creating config map {name} for {file_name}: {err}"
))
})?;
self.wait_created(config_maps, name).await?;
Ok(config_map)
}
pub(super) async fn create_pod(
&self,
namespace: &str,
name: &str,
spec: PodSpec,
labels: BTreeMap<String, String>,
) -> Result<Pod> {
let pods = Api::<Pod>::namespaced(self.inner.clone(), namespace);
let pod = Pod {
metadata: ObjectMeta {
name: Some(name.to_string()),
namespace: Some(namespace.to_string()),
labels: Some(labels),
..Default::default()
},
spec: Some(spec),
..Default::default()
};
pods.create(&PostParams::default(), &pod)
.await
.map_err(|err| Error::from(anyhow!("error while creating pod {name}: {err}")))?;
trace!("Pod {name} checking for ready state!");
let wait_ready = await_condition(pods, name, helpers::is_pod_ready());
// TODO: we should use the `node_spawn_timeout` from global settings here.
let _ = tokio::time::timeout(Duration::from_secs(600), wait_ready)
.await
.map_err(|err| {
Error::from(anyhow!("error while awaiting pod {name} running: {err}"))
})?;
debug!("Pod {name} is Ready!");
Ok(pod)
}
pub(super) async fn pod_logs(&self, namespace: &str, name: &str) -> Result<String> {
Api::<Pod>::namespaced(self.inner.clone(), namespace)
.logs(
name,
&LogParams {
pretty: true,
timestamps: true,
..Default::default()
},
)
.await
.map_err(|err| Error::from(anyhow!("error while getting logs for pod {name}: {err}")))
}
pub(super) async fn pod_status(&self, namespace: &str, name: &str) -> Result<PodStatus> {
let pod = Api::<Pod>::namespaced(self.inner.clone(), namespace)
.get(name)
.await
.map_err(|err| Error::from(anyhow!("error while getting pod {name}: {err}")))?;
let status = pod.status.ok_or(Error::from(anyhow!(
"error while getting status for pod {name}"
)))?;
Ok(status)
}
#[allow(dead_code)]
pub(super) async fn create_pod_logs_stream(
&self,
namespace: &str,
name: &str,
) -> Result<Box<dyn AsyncRead + Send + Unpin>> {
Ok(Box::new(
Api::<Pod>::namespaced(self.inner.clone(), namespace)
.log_stream(
name,
&LogParams {
follow: true,
pretty: true,
timestamps: true,
..Default::default()
},
)
.await
.map_err(|err| {
Error::from(anyhow!(
"error while getting a log stream for {name}: {err}"
))
})?
.compat(),
))
}
pub(super) async fn pod_exec<S>(
&self,
namespace: &str,
name: &str,
command: Vec<S>,
) -> Result<ExecutionResult>
where
S: Into<String> + std::fmt::Debug + Send,
{
trace!("running command: {command:?} on pod {name} for ns {namespace}");
let mut process = Api::<Pod>::namespaced(self.inner.clone(), namespace)
.exec(
name,
command,
&AttachParams::default().stdout(true).stderr(true),
)
.await
.map_err(|err| Error::from(anyhow!("error while exec in the pod {name}: {err}")))?;
let stdout_stream = process.stdout().expect(&format!(
"stdout shouldn't be None when true passed to exec {THIS_IS_A_BUG}"
));
let stdout = tokio_util::io::ReaderStream::new(stdout_stream)
.filter_map(|r| async { r.ok().and_then(|v| String::from_utf8(v.to_vec()).ok()) })
.collect::<Vec<_>>()
.await
.join("");
let stderr_stream = process.stderr().expect(&format!(
"stderr shouldn't be None when true passed to exec {THIS_IS_A_BUG}"
));
let stderr = tokio_util::io::ReaderStream::new(stderr_stream)
.filter_map(|r| async { r.ok().and_then(|v| String::from_utf8(v.to_vec()).ok()) })
.collect::<Vec<_>>()
.await
.join("");
let status = process
.take_status()
.expect(&format!(
"first call to status shouldn't fail {THIS_IS_A_BUG}"
))
.await;
// await process to finish
process.join().await.map_err(|err| {
Error::from(anyhow!(
"error while joining process during exec for {name}: {err}"
))
})?;
match status {
// command succeeded with stdout
Some(status) if status.status.as_ref().is_some_and(|s| s == "Success") => {
Ok(Ok(stdout))
},
// command failed
Some(status) if status.status.as_ref().is_some_and(|s| s == "Failure") => {
match status.reason {
// due to exit code
Some(reason) if reason == "NonZeroExitCode" => {
let exit_status = status
.details
.and_then(|details| {
details.causes.and_then(|causes| {
causes.first().and_then(|cause| {
cause.message.as_deref().and_then(|message| {
message.parse::<i32>().ok().map(ExitStatus::from_raw)
})
})
})
})
.expect(
&format!("command with non-zero exit code should have exit code present {THIS_IS_A_BUG}")
);
Ok(Err((exit_status, stderr)))
},
// due to other unknown reason
Some(ref reason) => Err(Error::from(anyhow!(
format!("unhandled reason while exec for {name}: {reason}, stderr: {stderr}, status: {status:?}")
))),
None => {
panic!("command had status but no reason was present, this is a bug")
},
}
},
Some(_) => {
unreachable!("command had status but it didn't matches either Success or Failure, this is a bug from the kube.rs library itself");
},
None => {
panic!("command has no status following its execution, this is a bug");
},
}
}
pub(super) async fn delete_pod(&self, namespace: &str, name: &str) -> Result<()> {
let pods = Api::<Pod>::namespaced(self.inner.clone(), namespace);
pods.delete(name, &DeleteParams::default())
.await
.map_err(|err| Error::from(anyhow!("error when deleting pod {name}: {err}")))?;
await_condition(pods, name, conditions::is_deleted(name))
.await
.map_err(|err| {
Error::from(anyhow!(
"error when waiting for pod {name} to be deleted: {err}"
))
})?;
Ok(())
}
pub(super) async fn create_service(
&self,
namespace: &str,
name: &str,
spec: ServiceSpec,
labels: BTreeMap<String, String>,
) -> Result<Service> {
let services = Api::<Service>::namespaced(self.inner.clone(), namespace);
let service = Service {
metadata: ObjectMeta {
name: Some(name.to_string()),
namespace: Some(namespace.to_string()),
labels: Some(labels),
..Default::default()
},
spec: Some(spec),
..Default::default()
};
services
.create(&PostParams::default(), &service)
.await
.map_err(|err| Error::from(anyhow!("error while creating service {name}: {err}")))?;
Ok(service)
}
pub(super) async fn create_pod_port_forward(
&self,
namespace: &str,
name: &str,
local_port: u16,
remote_port: u16,
) -> Result<(u16, JoinHandle<()>)> {
let pods = Api::<Pod>::namespaced(self.inner.clone(), namespace);
let bind = TcpListener::bind((LOCALHOST, local_port))
.await
.map_err(|err| {
Error::from(anyhow!(
"error binding port {local_port} for pod {name}: {err}"
))
})?;
let local_port = bind.local_addr().map_err(|err| Error(err.into()))?.port();
let name = name.to_string();
const MAX_FAILURES: usize = 5;
let monitor_handle = tokio::spawn(async move {
let mut consecutive_failures = 0;
loop {
let (mut client_conn, _) = match bind.accept().await {
Ok(conn) => {
consecutive_failures = 0;
conn
},
Err(e) => {
if consecutive_failures < MAX_FAILURES {
trace!("Port-forward accept error: {e:?}, retrying in 1s");
tokio::time::sleep(Duration::from_secs(1)).await;
consecutive_failures += 1;
continue;
} else {
trace!("Port-forward accept failed too many times, giving up");
break;
}
},
};
let peer = match client_conn.peer_addr() {
Ok(addr) => addr,
Err(e) => {
trace!("Failed to get peer address: {e:?}");
break;
},
};
trace!("new connection on local_port: {local_port}, peer: {peer}");
let (name, pods) = (name.clone(), pods.clone());
tokio::spawn(async move {
loop {
// Try to establish port-forward
let mut forwarder = match pods.portforward(&name, &[remote_port]).await {
Ok(f) => {
consecutive_failures = 0;
f
},
Err(e) => {
consecutive_failures += 1;
if consecutive_failures < MAX_FAILURES {
trace!("portforward failed to establish ({}/{}): {e:?}, retrying in 1s",
consecutive_failures, MAX_FAILURES);
tokio::time::sleep(Duration::from_secs(1)).await;
continue;
} else {
trace!("portforward failed to establish after {} attempts: {e:?}, closing connection",
consecutive_failures);
break;
}
},
};
let mut upstream_conn = match forwarder.take_stream(remote_port) {
Some(s) => s,
None => {
trace!("Failed to take stream for remote_port: {remote_port}, retrying in 1s");
tokio::time::sleep(Duration::from_secs(1)).await;
continue;
},
};
match tokio::io::copy_bidirectional(&mut client_conn, &mut upstream_conn)
.await
{
Ok((_n1, _n2)) => {
// EOF reached, close connection
trace!("copy_bidirectional finished (EOF), closing connection");
drop(upstream_conn);
let _ = forwarder.join().await;
break;
},
Err(e) => {
let kind = e.kind();
match kind {
ErrorKind::ConnectionReset
| ErrorKind::ConnectionAborted
| ErrorKind::ConnectionRefused
| ErrorKind::TimedOut => {
consecutive_failures += 1;
if consecutive_failures < MAX_FAILURES {
trace!("Network error ({kind:?}): {e:?}, retrying port-forward for this connection");
tokio::time::sleep(Duration::from_secs(1)).await;
continue;
} else {
trace!("portforward failed to establish after {} attempts: {e:?}, closing connection",
consecutive_failures);
break;
}
},
_ => {
trace!("Non-network error ({kind:?}): {e:?}, closing connection");
break;
},
}
},
}
}
});
trace!("finished forwarder process for local port: {local_port}, peer: {peer}");
}
});
Ok((local_port, monitor_handle))
}
/// Create resources from yamls in `static-configs` directory
pub(super) async fn create_static_resource(
&self,
namespace: &str,
raw_manifest: &str,
) -> Result<()> {
let tm: TypeMeta = serde_yaml::from_str(raw_manifest).map_err(|err| {
Error::from(anyhow!(
"error while extracting TypeMeta from manifest: {raw_manifest}: {err}"
))
})?;
let gvk = GroupVersionKind::try_from(&tm).map_err(|err| {
Error::from(anyhow!(
"error while extracting GroupVersionKind from manifest: {raw_manifest}: {err}"
))
})?;
let ar = ApiResource::from_gvk(&gvk);
let api: Api<DynamicObject> = Api::namespaced_with(self.inner.clone(), namespace, &ar);
api.create(
&PostParams::default(),
&serde_yaml::from_str(raw_manifest).unwrap(),
)
.await
.map_err(|err| {
Error::from(anyhow!(
"error while creating static-config {raw_manifest}: {err}"
))
})?;
Ok(())
}
async fn wait_created<K>(&self, api: Api<K>, name: &str) -> Result<()>
where
K: Clone + DeserializeOwned + Debug,
{
let params = &WatchParams::default().fields(&format!("metadata.name={name}"));
let mut stream = api
.watch(params, "0")
.await
.map_err(|err| {
Error::from(anyhow!(
"error while awaiting first response when resource {name} is created: {err}"
))
})?
.boxed();
while let Some(status) = stream.try_next().await.map_err(|err| {
Error::from(anyhow!(
"error while awaiting next change after resource {name} is created: {err}"
))
})? {
match status {
WatchEvent::Added(_) => break,
WatchEvent::Error(err) => Err(Error::from(anyhow!(
"error while awaiting resource {name} is created: {err}"
)))?,
WatchEvent::Bookmark(_) => {
// bookmark events are periodically sent as keep-alive/checkpoint, we should continue waiting
}
any_other_event => panic!("Unexpected event happened while creating '{name}' {THIS_IS_A_BUG}. Event: {any_other_event:?}"),
}
}
Ok(())
}
}
mod helpers {
use k8s_openapi::api::core::v1::Pod;
use kube::runtime::wait::Condition;
use tracing::trace;
/// An await condition for `Pod` that returns `true` once it is ready
/// based on [`kube::runtime::wait::conditions::is_pod_running`]
pub fn is_pod_ready() -> impl Condition<Pod> {
|obj: Option<&Pod>| {
if let Some(pod) = &obj {
if let Some(status) = &pod.status {
if let Some(conditions) = &status.conditions {
let ready = conditions
.iter()
.any(|cond| cond.status == "True" && cond.type_ == "Ready");
if ready {
trace!("{:#?}", status);
return ready;
}
}
}
}
false
}
}
}
@@ -0,0 +1,600 @@
use std::{
collections::{BTreeMap, HashMap},
env,
path::{Path, PathBuf},
sync::{Arc, Weak},
};
use async_trait::async_trait;
use k8s_openapi::{
api::core::v1::{
Container, ContainerPort, HTTPGetAction, PodSpec, Probe, ServicePort, ServiceSpec,
},
apimachinery::pkg::util::intstr::IntOrString,
};
use support::{constants::THIS_IS_A_BUG, fs::FileSystem, replacer::apply_replacements};
use tokio::sync::{Mutex, RwLock};
use tracing::{debug, trace, warn};
use uuid::Uuid;
use super::{client::KubernetesClient, node::KubernetesNode};
use crate::{
constants::NAMESPACE_PREFIX,
kubernetes::{
node::{DeserializableKubernetesNodeOptions, KubernetesNodeOptions},
provider,
},
shared::helpers::{extract_execution_result, running_in_ci},
types::{
GenerateFileCommand, GenerateFilesOptions, ProviderCapabilities, RunCommandOptions,
SpawnNodeOptions,
},
DynNode, KubernetesProvider, ProviderError, ProviderNamespace, ProviderNode,
};
const FILE_SERVER_IMAGE: &str = "europe-west3-docker.pkg.dev/parity-zombienet/zombienet-public-images/zombienet-file-server:latest";
// env var used by our internal CI to pass the namespace created and ready to use
const ZOMBIE_K8S_CI_NAMESPACE: &str = "ZOMBIE_K8S_CI_NAMESPACE";
pub(super) struct KubernetesNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
weak: Weak<KubernetesNamespace<FS>>,
provider: Weak<KubernetesProvider<FS>>,
name: String,
base_dir: PathBuf,
capabilities: ProviderCapabilities,
k8s_client: KubernetesClient,
filesystem: FS,
file_server_fw_task: RwLock<Option<tokio::task::JoinHandle<()>>>,
delete_on_drop: Arc<Mutex<bool>>,
pub(super) file_server_port: RwLock<Option<u16>>,
pub(super) nodes: RwLock<HashMap<String, Arc<KubernetesNode<FS>>>>,
}
impl<FS> KubernetesNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) async fn new(
provider: &Weak<KubernetesProvider<FS>>,
tmp_dir: &PathBuf,
capabilities: &ProviderCapabilities,
k8s_client: &KubernetesClient,
filesystem: &FS,
custom_base_dir: Option<&Path>,
) -> Result<Arc<Self>, ProviderError> {
// If the namespace is already provided
let name = if let Ok(name) = env::var(ZOMBIE_K8S_CI_NAMESPACE) {
name
} else {
format!("{}{}", NAMESPACE_PREFIX, Uuid::new_v4())
};
let base_dir = if let Some(custom_base_dir) = custom_base_dir {
if !filesystem.exists(custom_base_dir).await {
filesystem.create_dir(custom_base_dir).await?;
} else {
warn!(
"⚠️ Using and existing directory {} as base dir",
custom_base_dir.to_string_lossy()
);
}
PathBuf::from(custom_base_dir)
} else {
let base_dir = PathBuf::from_iter([tmp_dir, &PathBuf::from(&name)]);
filesystem.create_dir(&base_dir).await?;
base_dir
};
let namespace = Arc::new_cyclic(|weak| KubernetesNamespace {
weak: weak.clone(),
provider: provider.clone(),
name,
base_dir,
capabilities: capabilities.clone(),
filesystem: filesystem.clone(),
k8s_client: k8s_client.clone(),
file_server_port: RwLock::new(None),
file_server_fw_task: RwLock::new(None),
nodes: RwLock::new(HashMap::new()),
delete_on_drop: Arc::new(Mutex::new(true)),
});
namespace.initialize().await?;
Ok(namespace)
}
pub(super) async fn attach_to_live(
provider: &Weak<KubernetesProvider<FS>>,
capabilities: &ProviderCapabilities,
k8s_client: &KubernetesClient,
filesystem: &FS,
custom_base_dir: &Path,
name: &str,
) -> Result<Arc<Self>, ProviderError> {
let base_dir = custom_base_dir.to_path_buf();
let namespace = Arc::new_cyclic(|weak| KubernetesNamespace {
weak: weak.clone(),
provider: provider.clone(),
name: name.to_owned(),
base_dir,
capabilities: capabilities.clone(),
filesystem: filesystem.clone(),
k8s_client: k8s_client.clone(),
file_server_port: RwLock::new(None),
file_server_fw_task: RwLock::new(None),
nodes: RwLock::new(HashMap::new()),
delete_on_drop: Arc::new(Mutex::new(false)),
});
namespace.setup_file_server_port_fwd("fileserver").await?;
Ok(namespace)
}
async fn initialize(&self) -> Result<(), ProviderError> {
// Initialize the namespace IFF
// we are not in CI or we don't have the env `ZOMBIE_NAMESPACE` set
if env::var(ZOMBIE_K8S_CI_NAMESPACE).is_err() || !running_in_ci() {
self.initialize_k8s().await?;
}
// Ensure namespace isolation and minimal resources IFF we are running in CI
if running_in_ci() {
self.initialize_static_resources().await?
}
self.initialize_file_server().await?;
self.setup_script_config_map(
"zombie-wrapper",
include_str!("../shared/scripts/zombie-wrapper.sh"),
"zombie_wrapper_config_map_manifest.yaml",
// TODO: add correct labels
BTreeMap::new(),
)
.await?;
self.setup_script_config_map(
"helper-binaries-downloader",
include_str!("../shared/scripts/helper-binaries-downloader.sh"),
"helper_binaries_downloader_config_map_manifest.yaml",
// TODO: add correct labels
BTreeMap::new(),
)
.await?;
Ok(())
}
async fn initialize_k8s(&self) -> Result<(), ProviderError> {
// TODO (javier): check with Hamid if we are using this labels in any scheduling logic.
let labels = BTreeMap::from([
(
"jobId".to_string(),
env::var("CI_JOB_ID").unwrap_or("".to_string()),
),
(
"projectName".to_string(),
env::var("CI_PROJECT_NAME").unwrap_or("".to_string()),
),
(
"projectId".to_string(),
env::var("CI_PROJECT_ID").unwrap_or("".to_string()),
),
]);
let manifest = self
.k8s_client
.create_namespace(&self.name, labels)
.await
.map_err(|err| {
ProviderError::CreateNamespaceFailed(self.name.to_string(), err.into())
})?;
let serialized_manifest = serde_yaml::to_string(&manifest).map_err(|err| {
ProviderError::CreateNamespaceFailed(self.name.to_string(), err.into())
})?;
let dest_path =
PathBuf::from_iter([&self.base_dir, &PathBuf::from("namespace_manifest.yaml")]);
self.filesystem
.write(dest_path, serialized_manifest)
.await?;
Ok(())
}
async fn initialize_static_resources(&self) -> Result<(), ProviderError> {
let np_manifest = apply_replacements(
include_str!("./static-configs/namespace-network-policy.yaml"),
&HashMap::from([("namespace", self.name())]),
);
// Apply NetworkPolicy manifest
self.k8s_client
.create_static_resource(&self.name, &np_manifest)
.await
.map_err(|err| {
ProviderError::CreateNamespaceFailed(self.name.to_string(), err.into())
})?;
// Apply LimitRange manifest
self.k8s_client
.create_static_resource(
&self.name,
include_str!("./static-configs/baseline-resources.yaml"),
)
.await
.map_err(|err| {
ProviderError::CreateNamespaceFailed(self.name.to_string(), err.into())
})?;
Ok(())
}
async fn initialize_file_server(&self) -> Result<(), ProviderError> {
let name = "fileserver".to_string();
let labels = BTreeMap::from([
("app.kubernetes.io/name".to_string(), name.clone()),
(
"x-infra-instance".to_string(),
env::var("X_INFRA_INSTANCE").unwrap_or("ondemand".to_string()),
),
]);
let pod_spec = PodSpec {
hostname: Some(name.clone()),
containers: vec![Container {
name: name.clone(),
image: Some(FILE_SERVER_IMAGE.to_string()),
image_pull_policy: Some("Always".to_string()),
ports: Some(vec![ContainerPort {
container_port: 80,
..Default::default()
}]),
startup_probe: Some(Probe {
http_get: Some(HTTPGetAction {
path: Some("/".to_string()),
port: IntOrString::Int(80),
..Default::default()
}),
initial_delay_seconds: Some(1),
period_seconds: Some(2),
failure_threshold: Some(3),
..Default::default()
}),
..Default::default()
}],
restart_policy: Some("OnFailure".into()),
..Default::default()
};
let pod_manifest = self
.k8s_client
.create_pod(&self.name, &name, pod_spec, labels.clone())
.await
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
// TODO: remove duplication across methods
let pod_serialized_manifest = serde_yaml::to_string(&pod_manifest)
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
let pod_dest_path = PathBuf::from_iter([
&self.base_dir,
&PathBuf::from("file_server_pod_manifest.yaml"),
]);
self.filesystem
.write(pod_dest_path, pod_serialized_manifest)
.await?;
let service_spec = ServiceSpec {
selector: Some(labels.clone()),
ports: Some(vec![ServicePort {
port: 80,
..Default::default()
}]),
..Default::default()
};
let service_manifest = self
.k8s_client
.create_service(&self.name, &name, service_spec, labels)
.await
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
let serialized_service_manifest = serde_yaml::to_string(&service_manifest)
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
let service_dest_path = PathBuf::from_iter([
&self.base_dir,
&PathBuf::from("file_server_service_manifest.yaml"),
]);
self.filesystem
.write(service_dest_path, serialized_service_manifest)
.await?;
self.setup_file_server_port_fwd(&name).await?;
Ok(())
}
async fn setup_file_server_port_fwd(&self, name: &str) -> Result<(), ProviderError> {
let (port, task) = self
.k8s_client
.create_pod_port_forward(&self.name, name, 0, 80)
.await
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
*self.file_server_port.write().await = Some(port);
*self.file_server_fw_task.write().await = Some(task);
Ok(())
}
async fn setup_script_config_map(
&self,
name: &str,
script_contents: &str,
local_manifest_name: &str,
labels: BTreeMap<String, String>,
) -> Result<(), ProviderError> {
let manifest = self
.k8s_client
.create_config_map_from_file(
&self.name,
name,
&format!("{name}.sh"),
script_contents,
labels,
)
.await
.map_err(|err| {
ProviderError::CreateNamespaceFailed(self.name.to_string(), err.into())
})?;
let serializer_manifest = serde_yaml::to_string(&manifest).map_err(|err| {
ProviderError::CreateNamespaceFailed(self.name.to_string(), err.into())
})?;
let dest_path = PathBuf::from_iter([&self.base_dir, &PathBuf::from(local_manifest_name)]);
self.filesystem
.write(dest_path, serializer_manifest)
.await?;
Ok(())
}
pub async fn set_delete_on_drop(&self, delete_on_drop: bool) {
*self.delete_on_drop.lock().await = delete_on_drop;
}
pub async fn delete_on_drop(&self) -> bool {
if let Ok(delete_on_drop) = self.delete_on_drop.try_lock() {
*delete_on_drop
} else {
// if we can't lock just remove the ns
true
}
}
}
impl<FS> Drop for KubernetesNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
fn drop(&mut self) {
let ns_name = self.name.clone();
if let Ok(delete_on_drop) = self.delete_on_drop.try_lock() {
if *delete_on_drop {
let client = self.k8s_client.clone();
let provider = self.provider.upgrade();
futures::executor::block_on(async move {
trace!("🧟 deleting ns {ns_name} from cluster");
let _ = client.delete_namespace(&ns_name).await;
if let Some(provider) = provider {
provider.namespaces.write().await.remove(&ns_name);
}
trace!("✅ deleted");
});
} else {
trace!("⚠️ leaking ns {ns_name} in cluster");
}
};
}
}
#[async_trait]
impl<FS> ProviderNamespace for KubernetesNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
fn name(&self) -> &str {
&self.name
}
fn base_dir(&self) -> &PathBuf {
&self.base_dir
}
fn capabilities(&self) -> &ProviderCapabilities {
&self.capabilities
}
fn provider_name(&self) -> &str {
provider::PROVIDER_NAME
}
async fn detach(&self) {
self.set_delete_on_drop(false).await;
}
async fn is_detached(&self) -> bool {
self.delete_on_drop().await
}
async fn nodes(&self) -> HashMap<String, DynNode> {
self.nodes
.read()
.await
.iter()
.map(|(name, node)| (name.clone(), node.clone() as DynNode))
.collect()
}
async fn get_node_available_args(
&self,
(command, image): (String, Option<String>),
) -> Result<String, ProviderError> {
let node_image = image.expect(&format!("image should be present when getting node available args with kubernetes provider {THIS_IS_A_BUG}"));
// run dummy command in new pod
let temp_node = self
.spawn_node(
&SpawnNodeOptions::new(format!("temp-{}", Uuid::new_v4()), "cat".to_string())
.image(node_image.clone()),
)
.await?;
let available_args_output = temp_node
.run_command(RunCommandOptions::new(command.clone()).args(vec!["--help"]))
.await?
.map_err(|(_exit, status)| {
ProviderError::NodeAvailableArgsError(node_image, command, status)
})?;
temp_node.destroy().await?;
Ok(available_args_output)
}
async fn spawn_node(&self, options: &SpawnNodeOptions) -> Result<DynNode, ProviderError> {
trace!("spawn node options {options:?}");
let node = KubernetesNode::new(KubernetesNodeOptions {
namespace: &self.weak,
namespace_base_dir: &self.base_dir,
name: &options.name,
image: options.image.as_ref(),
program: &options.program,
args: &options.args,
env: &options.env,
startup_files: &options.injected_files,
resources: options.resources.as_ref(),
db_snapshot: options.db_snapshot.as_ref(),
k8s_client: &self.k8s_client,
filesystem: &self.filesystem,
})
.await?;
self.nodes
.write()
.await
.insert(node.name().to_string(), node.clone());
Ok(node)
}
async fn spawn_node_from_json(
&self,
json_value: &serde_json::Value,
) -> Result<DynNode, ProviderError> {
let deserializable: DeserializableKubernetesNodeOptions =
serde_json::from_value(json_value.clone())?;
let options = KubernetesNodeOptions::from_deserializable(
&deserializable,
&self.weak,
&self.base_dir,
&self.k8s_client,
&self.filesystem,
);
let node = KubernetesNode::attach_to_live(options).await?;
self.nodes
.write()
.await
.insert(node.name().to_string(), node.clone());
Ok(node)
}
async fn generate_files(&self, options: GenerateFilesOptions) -> Result<(), ProviderError> {
debug!("generate files options {options:#?}");
let node_name = options
.temp_name
.unwrap_or_else(|| format!("temp-{}", Uuid::new_v4()));
let node_image = options
.image
.expect(&format!("image should be present when generating files with kubernetes provider {THIS_IS_A_BUG}"));
// run dummy command in new pod
let temp_node = self
.spawn_node(
&SpawnNodeOptions::new(node_name, "cat".to_string())
.injected_files(options.injected_files)
.image(node_image),
)
.await?;
for GenerateFileCommand {
program,
args,
env,
local_output_path,
} in options.commands
{
let local_output_full_path = format!(
"{}{}{}",
self.base_dir.to_string_lossy(),
if local_output_path.starts_with("/") {
""
} else {
"/"
},
local_output_path.to_string_lossy()
);
let contents = extract_execution_result(
&temp_node,
RunCommandOptions { program, args, env },
options.expected_path.as_ref(),
)
.await?;
self.filesystem
.write(local_output_full_path, contents)
.await
.map_err(|err| ProviderError::FileGenerationFailed(err.into()))?;
}
temp_node.destroy().await
}
async fn static_setup(&self) -> Result<(), ProviderError> {
todo!()
}
async fn destroy(&self) -> Result<(), ProviderError> {
let _ = self
.k8s_client
.delete_namespace(&self.name)
.await
.map_err(|err| ProviderError::DeleteNamespaceFailed(self.name.clone(), err.into()))?;
if let Some(provider) = self.provider.upgrade() {
provider.namespaces.write().await.remove(&self.name);
}
Ok(())
}
}
@@ -0,0 +1,886 @@
use std::{
collections::{BTreeMap, HashMap},
env,
net::IpAddr,
path::{Component, Path, PathBuf},
sync::{Arc, Weak},
time::Duration,
};
use anyhow::anyhow;
use async_trait::async_trait;
use configuration::{shared::resources::Resources, types::AssetLocation};
use futures::future::try_join_all;
use k8s_openapi::api::core::v1::{ServicePort, ServiceSpec};
use serde::{Deserialize, Serialize};
use sha2::Digest;
use support::{constants::THIS_IS_A_BUG, fs::FileSystem};
use tokio::{sync::RwLock, task::JoinHandle, time::sleep, try_join};
use tracing::{debug, trace, warn};
use url::Url;
use super::{
client::KubernetesClient, namespace::KubernetesNamespace, pod_spec_builder::PodSpecBuilder,
};
use crate::{
constants::{
NODE_CONFIG_DIR, NODE_DATA_DIR, NODE_RELAY_DATA_DIR, NODE_SCRIPTS_DIR, P2P_PORT,
PROMETHEUS_PORT, RPC_HTTP_PORT, RPC_WS_PORT,
},
kubernetes,
types::{ExecutionResult, RunCommandOptions, RunScriptOptions, TransferedFile},
ProviderError, ProviderNamespace, ProviderNode,
};
pub(super) struct KubernetesNodeOptions<'a, FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) namespace: &'a Weak<KubernetesNamespace<FS>>,
pub(super) namespace_base_dir: &'a PathBuf,
pub(super) name: &'a str,
pub(super) image: Option<&'a String>,
pub(super) program: &'a str,
pub(super) args: &'a [String],
pub(super) env: &'a [(String, String)],
pub(super) startup_files: &'a [TransferedFile],
pub(super) resources: Option<&'a Resources>,
pub(super) db_snapshot: Option<&'a AssetLocation>,
pub(super) k8s_client: &'a KubernetesClient,
pub(super) filesystem: &'a FS,
}
impl<'a, FS> KubernetesNodeOptions<'a, FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) fn from_deserializable(
deserializable: &'a DeserializableKubernetesNodeOptions,
namespace: &'a Weak<KubernetesNamespace<FS>>,
namespace_base_dir: &'a PathBuf,
k8s_client: &'a KubernetesClient,
filesystem: &'a FS,
) -> KubernetesNodeOptions<'a, FS> {
KubernetesNodeOptions {
namespace,
namespace_base_dir,
name: &deserializable.name,
image: deserializable.image.as_ref(),
program: &deserializable.program,
args: &deserializable.args,
env: &deserializable.env,
startup_files: &[],
resources: deserializable.resources.as_ref(),
db_snapshot: None,
k8s_client,
filesystem,
}
}
}
#[derive(Deserialize)]
pub(super) struct DeserializableKubernetesNodeOptions {
pub(super) name: String,
pub(super) image: Option<String>,
pub(super) program: String,
pub(super) args: Vec<String>,
pub(super) env: Vec<(String, String)>,
pub(super) resources: Option<Resources>,
}
type FwdInfo = (u16, JoinHandle<()>);
#[derive(Serialize)]
pub(super) struct KubernetesNode<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
#[serde(skip)]
namespace: Weak<KubernetesNamespace<FS>>,
name: String,
image: String,
program: String,
args: Vec<String>,
env: Vec<(String, String)>,
resources: Option<Resources>,
base_dir: PathBuf,
config_dir: PathBuf,
data_dir: PathBuf,
relay_data_dir: PathBuf,
scripts_dir: PathBuf,
log_path: PathBuf,
#[serde(skip)]
k8s_client: KubernetesClient,
#[serde(skip)]
http_client: reqwest::Client,
#[serde(skip)]
filesystem: FS,
#[serde(skip)]
port_fwds: RwLock<HashMap<u16, FwdInfo>>,
provider_tag: String,
}
impl<FS> KubernetesNode<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) async fn new(
options: KubernetesNodeOptions<'_, FS>,
) -> Result<Arc<Self>, ProviderError> {
let image = options.image.ok_or_else(|| {
ProviderError::MissingNodeInfo(options.name.to_string(), "missing image".to_string())
})?;
let filesystem = options.filesystem.clone();
let base_dir =
PathBuf::from_iter([options.namespace_base_dir, &PathBuf::from(options.name)]);
filesystem.create_dir_all(&base_dir).await?;
let base_dir_raw = base_dir.to_string_lossy();
let config_dir = PathBuf::from(format!("{base_dir_raw}{NODE_CONFIG_DIR}"));
let data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_DATA_DIR}"));
let relay_data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_RELAY_DATA_DIR}"));
let scripts_dir = PathBuf::from(format!("{base_dir_raw}{NODE_SCRIPTS_DIR}"));
let log_path = base_dir.join("node.log");
try_join!(
filesystem.create_dir(&config_dir),
filesystem.create_dir(&data_dir),
filesystem.create_dir(&relay_data_dir),
filesystem.create_dir(&scripts_dir),
)?;
let node = Arc::new(KubernetesNode {
namespace: options.namespace.clone(),
name: options.name.to_string(),
image: image.to_string(),
program: options.program.to_string(),
args: options.args.to_vec(),
env: options.env.to_vec(),
resources: options.resources.cloned(),
base_dir,
config_dir,
data_dir,
relay_data_dir,
scripts_dir,
log_path,
filesystem: filesystem.clone(),
k8s_client: options.k8s_client.clone(),
http_client: reqwest::Client::new(),
port_fwds: Default::default(),
provider_tag: kubernetes::provider::PROVIDER_NAME.to_string(),
});
node.initialize_k8s().await?;
if let Some(db_snap) = options.db_snapshot {
node.initialize_db_snapshot(db_snap).await?;
}
node.initialize_startup_files(options.startup_files).await?;
node.start().await?;
Ok(node)
}
pub(super) async fn attach_to_live(
options: KubernetesNodeOptions<'_, FS>,
) -> Result<Arc<Self>, ProviderError> {
let image = options.image.ok_or_else(|| {
ProviderError::MissingNodeInfo(options.name.to_string(), "missing image".to_string())
})?;
let filesystem = options.filesystem.clone();
let base_dir =
PathBuf::from_iter([options.namespace_base_dir, &PathBuf::from(options.name)]);
filesystem.create_dir_all(&base_dir).await?;
let base_dir_raw = base_dir.to_string_lossy();
let config_dir = PathBuf::from(format!("{base_dir_raw}{NODE_CONFIG_DIR}"));
let data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_DATA_DIR}"));
let relay_data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_RELAY_DATA_DIR}"));
let scripts_dir = PathBuf::from(format!("{base_dir_raw}{NODE_SCRIPTS_DIR}"));
let log_path = base_dir.join("node.log");
let node = Arc::new(KubernetesNode {
namespace: options.namespace.clone(),
name: options.name.to_string(),
image: image.to_string(),
program: options.program.to_string(),
args: options.args.to_vec(),
env: options.env.to_vec(),
resources: options.resources.cloned(),
base_dir,
config_dir,
data_dir,
relay_data_dir,
scripts_dir,
log_path,
filesystem: filesystem.clone(),
k8s_client: options.k8s_client.clone(),
http_client: reqwest::Client::new(),
port_fwds: Default::default(),
provider_tag: kubernetes::provider::PROVIDER_NAME.to_string(),
});
Ok(node)
}
async fn initialize_k8s(&self) -> Result<(), ProviderError> {
let labels = BTreeMap::from([
(
"app.kubernetes.io/name".to_string(),
self.name().to_string(),
),
(
"x-infra-instance".to_string(),
env::var("X_INFRA_INSTANCE").unwrap_or("ondemand".to_string()),
),
]);
// Create pod
let pod_spec = PodSpecBuilder::build(
&self.name,
&self.image,
self.resources.as_ref(),
&self.program,
&self.args,
&self.env,
);
let manifest = self
.k8s_client
.create_pod(&self.namespace_name(), &self.name, pod_spec, labels.clone())
.await
.map_err(|err| ProviderError::NodeSpawningFailed(self.name.clone(), err.into()))?;
let serialized_manifest = serde_yaml::to_string(&manifest)
.map_err(|err| ProviderError::NodeSpawningFailed(self.name.to_string(), err.into()))?;
let dest_path = PathBuf::from_iter([
&self.base_dir,
&PathBuf::from(format!("{}_manifest.yaml", &self.name)),
]);
self.filesystem
.write(dest_path, serialized_manifest)
.await
.map_err(|err| ProviderError::NodeSpawningFailed(self.name.to_string(), err.into()))?;
// Create service for pod
let service_spec = ServiceSpec {
selector: Some(labels.clone()),
ports: Some(vec![
ServicePort {
port: P2P_PORT.into(),
name: Some("p2p".into()),
..Default::default()
},
ServicePort {
port: RPC_WS_PORT.into(),
name: Some("rpc".into()),
..Default::default()
},
ServicePort {
port: RPC_HTTP_PORT.into(),
name: Some("rpc-http".into()),
..Default::default()
},
ServicePort {
port: PROMETHEUS_PORT.into(),
name: Some("prom".into()),
..Default::default()
},
]),
..Default::default()
};
let service_manifest = self
.k8s_client
.create_service(&self.namespace_name(), &self.name, service_spec, labels)
.await
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
let serialized_service_manifest = serde_yaml::to_string(&service_manifest)
.map_err(|err| ProviderError::FileServerSetupError(err.into()))?;
let service_dest_path = PathBuf::from_iter([
&self.base_dir,
&PathBuf::from(format!("{}_svc_manifest.yaml", &self.name)),
]);
self.filesystem
.write(service_dest_path, serialized_service_manifest)
.await?;
Ok(())
}
async fn initialize_db_snapshot(
&self,
db_snapshot: &AssetLocation,
) -> Result<(), ProviderError> {
trace!("snap: {db_snapshot}");
let url_of_snap = match db_snapshot {
AssetLocation::Url(location) => location.clone(),
AssetLocation::FilePath(filepath) => {
let (url, _) = self.upload_to_fileserver(filepath).await?;
url
},
};
// we need to get the snapshot from a public access
// and extract to /data
let opts = RunCommandOptions::new("mkdir").args([
"-p",
"/data/",
"&&",
"mkdir",
"-p",
"/relay-data/",
"&&",
// Use our version of curl
"/cfg/curl",
url_of_snap.as_ref(),
"--output",
"/data/db.tgz",
"&&",
"cd",
"/",
"&&",
"tar",
"--skip-old-files",
"-xzvf",
"/data/db.tgz",
]);
trace!("cmd opts: {:#?}", opts);
let _ = self.run_command(opts).await?;
Ok(())
}
async fn initialize_startup_files(
&self,
startup_files: &[TransferedFile],
) -> Result<(), ProviderError> {
try_join_all(
startup_files
.iter()
.map(|file| self.send_file(&file.local_path, &file.remote_path, &file.mode)),
)
.await?;
Ok(())
}
pub(super) async fn start(&self) -> Result<(), ProviderError> {
self.k8s_client
.pod_exec(
&self.namespace_name(),
&self.name,
vec!["sh", "-c", "echo start > /tmp/zombiepipe"],
)
.await
.map_err(|err| {
ProviderError::NodeSpawningFailed(
format!("failed to start pod {} after spawning", self.name),
err.into(),
)
})?
.map_err(|err| {
ProviderError::NodeSpawningFailed(
format!("failed to start pod {} after spawning", self.name,),
anyhow!("command failed in container: status {}: {}", err.0, err.1),
)
})?;
Ok(())
}
fn get_remote_parent_dir(&self, remote_file_path: &Path) -> Option<PathBuf> {
if let Some(remote_parent_dir) = remote_file_path.parent() {
if matches!(
remote_parent_dir.components().rev().peekable().peek(),
Some(Component::Normal(_))
) {
return Some(remote_parent_dir.to_path_buf());
}
}
None
}
async fn create_remote_dir(&self, remote_dir: &Path) -> Result<(), ProviderError> {
let _ = self
.k8s_client
.pod_exec(
&self.namespace_name(),
&self.name,
vec!["mkdir", "-p", &remote_dir.to_string_lossy()],
)
.await
.map_err(|err| {
ProviderError::NodeSpawningFailed(
format!(
"failed to create dir {} for pod {}",
remote_dir.to_string_lossy(),
&self.name
),
err.into(),
)
})?;
Ok(())
}
fn namespace_name(&self) -> String {
self.namespace
.upgrade()
.map(|namespace| namespace.name().to_string())
.unwrap_or_else(|| panic!("namespace shouldn't be dropped, {THIS_IS_A_BUG}"))
}
async fn upload_to_fileserver(&self, location: &Path) -> Result<(Url, String), ProviderError> {
let file_name = if let Some(name) = location.file_name() {
name.to_string_lossy()
} else {
"unnamed".into()
};
let data = self.filesystem.read(location).await?;
let content_hashed = hex::encode(sha2::Sha256::digest(&data));
let req = self
.http_client
.head(format!(
"http://{}/{content_hashed}__{file_name}",
self.file_server_local_host().await?
))
.build()
.map_err(|err| {
ProviderError::UploadFile(location.to_string_lossy().to_string(), err.into())
})?;
let url = req.url().clone();
let res = self.http_client.execute(req).await.map_err(|err| {
ProviderError::UploadFile(location.to_string_lossy().to_string(), err.into())
})?;
if res.status() != reqwest::StatusCode::OK {
// we need to upload the file
self.http_client
.post(url.as_ref())
.body(data)
.send()
.await
.map_err(|err| {
ProviderError::UploadFile(location.to_string_lossy().to_string(), err.into())
})?;
}
Ok((url, content_hashed))
}
async fn file_server_local_host(&self) -> Result<String, ProviderError> {
if let Some(namespace) = self.namespace.upgrade() {
if let Some(port) = *namespace.file_server_port.read().await {
return Ok(format!("localhost:{port}"));
}
}
Err(ProviderError::FileServerSetupError(anyhow!(
"file server port not bound locally"
)))
}
async fn download_file(
&self,
url: &str,
remote_file_path: &Path,
hash: Option<&str>,
) -> Result<(), ProviderError> {
let r = self
.k8s_client
.pod_exec(
&self.namespace_name(),
&self.name,
vec![
"/cfg/curl",
url,
"--output",
&remote_file_path.to_string_lossy(),
],
)
.await
.map_err(|err| {
ProviderError::DownloadFile(
remote_file_path.to_string_lossy().to_string(),
anyhow!(format!("node: {}, err: {}", self.name(), err)),
)
})?;
trace!("download url {} result: {:?}", url, r);
if r.is_err() {
return Err(ProviderError::DownloadFile(
remote_file_path.to_string_lossy().to_string(),
anyhow!(format!("node: {}, err downloading file", self.name())),
));
}
if let Some(hash) = hash {
// check if the hash of the file is correct
let res = self
.k8s_client
.pod_exec(
&self.namespace_name(),
&self.name,
vec![
"/cfg/coreutils",
"sha256sum",
&remote_file_path.to_string_lossy(),
],
)
.await
.map_err(|err| {
ProviderError::DownloadFile(
remote_file_path.to_string_lossy().to_string(),
anyhow!(format!("node: {}, err: {}", self.name(), err)),
)
})?;
if let Ok(output) = res {
if !output.contains(hash) {
return Err(ProviderError::DownloadFile(
remote_file_path.to_string_lossy().to_string(),
anyhow!(format!("node: {}, invalid sha256sum hash: {hash} for file, output was {output}", self.name())),
));
}
} else {
return Err(ProviderError::DownloadFile(
remote_file_path.to_string_lossy().to_string(),
anyhow!(format!(
"node: {}, err calculating sha256sum for file {:?}",
self.name(),
res
)),
));
}
}
Ok(())
}
}
#[async_trait]
impl<FS> ProviderNode for KubernetesNode<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
fn name(&self) -> &str {
&self.name
}
fn args(&self) -> Vec<&str> {
self.args.iter().map(|arg| arg.as_str()).collect()
}
fn base_dir(&self) -> &PathBuf {
&self.base_dir
}
fn config_dir(&self) -> &PathBuf {
&self.config_dir
}
fn data_dir(&self) -> &PathBuf {
&self.data_dir
}
fn relay_data_dir(&self) -> &PathBuf {
&self.relay_data_dir
}
fn scripts_dir(&self) -> &PathBuf {
&self.scripts_dir
}
fn log_path(&self) -> &PathBuf {
&self.log_path
}
fn log_cmd(&self) -> String {
format!("kubectl -n {} logs {}", self.namespace_name(), self.name)
}
fn path_in_node(&self, file: &Path) -> PathBuf {
// here is just a noop op since we will receive the path
// for the file inside the pod
PathBuf::from(file)
}
// TODO: handle log rotation as we do in v1
async fn logs(&self) -> Result<String, ProviderError> {
self.k8s_client
.pod_logs(&self.namespace_name(), &self.name)
.await
.map_err(|err| ProviderError::GetLogsFailed(self.name.to_string(), err.into()))
}
async fn dump_logs(&self, local_dest: PathBuf) -> Result<(), ProviderError> {
let logs = self.logs().await?;
self.filesystem
.write(local_dest, logs)
.await
.map_err(|err| ProviderError::DumpLogsFailed(self.name.to_string(), err.into()))?;
Ok(())
}
async fn create_port_forward(
&self,
local_port: u16,
remote_port: u16,
) -> Result<Option<u16>, ProviderError> {
// If the fwd exist just return the local port
if let Some(fwd_info) = self.port_fwds.read().await.get(&remote_port) {
return Ok(Some(fwd_info.0));
};
let (port, task) = self
.k8s_client
.create_pod_port_forward(&self.namespace_name(), &self.name, local_port, remote_port)
.await
.map_err(|err| ProviderError::PortForwardError(local_port, remote_port, err.into()))?;
self.port_fwds
.write()
.await
.insert(remote_port, (port, task));
Ok(Some(port))
}
async fn run_command(
&self,
options: RunCommandOptions,
) -> Result<ExecutionResult, ProviderError> {
let mut command = vec![];
for (name, value) in options.env {
command.push(format!("export {name}={value};"));
}
command.push(options.program);
for arg in options.args {
command.push(arg);
}
self.k8s_client
.pod_exec(
&self.namespace_name(),
&self.name,
vec!["sh", "-c", &command.join(" ")],
)
.await
.map_err(|err| {
ProviderError::RunCommandError(
format!("sh -c {}", &command.join(" ")),
format!("in pod {}", self.name),
err.into(),
)
})
}
async fn run_script(
&self,
options: RunScriptOptions,
) -> Result<ExecutionResult, ProviderError> {
let file_name = options
.local_script_path
.file_name()
.expect(&format!(
"file name should be present at this point {THIS_IS_A_BUG}"
))
.to_string_lossy();
self.run_command(RunCommandOptions {
program: format!("/tmp/{file_name}"),
args: options.args,
env: options.env,
})
.await
.map_err(|err| ProviderError::RunScriptError(self.name.to_string(), err.into()))
}
async fn send_file(
&self,
local_file_path: &Path,
remote_file_path: &Path,
mode: &str,
) -> Result<(), ProviderError> {
if let Some(remote_parent_dir) = self.get_remote_parent_dir(remote_file_path) {
self.create_remote_dir(&remote_parent_dir).await?;
}
debug!(
"Uploading file: {} IFF not present in the fileserver",
local_file_path.to_string_lossy()
);
// we need to override the url to use inside the pod
let (mut url, hash) = self.upload_to_fileserver(local_file_path).await?;
let _ = url.set_host(Some("fileserver"));
let _ = url.set_port(Some(80));
// Sometimes downloading the file fails (the file is corrupted)
// Add at most 5 retries
let mut last_err = None;
for i in 0..5 {
if i > 0 {
warn!("retrying number {i} download file {:?}", remote_file_path);
tokio::time::sleep(Duration::from_secs(i)).await;
}
let res = self
.download_file(url.as_ref(), remote_file_path, Some(&hash))
.await;
last_err = res.err();
if last_err.is_none() {
// ready to continue
break;
}
}
if let Some(last_err) = last_err {
return Err(last_err);
}
let _ = self
.k8s_client
.pod_exec(
&self.namespace_name(),
&self.name,
vec!["chmod", mode, &remote_file_path.to_string_lossy()],
)
.await
.map_err(|err| {
ProviderError::SendFile(
self.name.clone(),
local_file_path.to_string_lossy().to_string(),
err.into(),
)
})?;
Ok(())
}
async fn receive_file(
&self,
_remote_src: &Path,
_local_dest: &Path,
) -> Result<(), ProviderError> {
Ok(())
}
async fn ip(&self) -> Result<IpAddr, ProviderError> {
let status = self
.k8s_client
.pod_status(&self.namespace_name(), &self.name)
.await
.map_err(|_| ProviderError::MissingNode(self.name.clone()))?;
if let Some(ip) = status.pod_ip {
// Pod ip should be parseable
Ok(ip.parse::<IpAddr>().map_err(|err| {
ProviderError::InvalidConfig(format!("Can not parse the pod ip: {ip}, err: {err}"))
})?)
} else {
Err(ProviderError::InvalidConfig(format!(
"Can not find ip of pod: {}",
self.name()
)))
}
}
async fn pause(&self) -> Result<(), ProviderError> {
self.k8s_client
.pod_exec(
&self.namespace_name(),
&self.name,
vec!["sh", "-c", "echo pause > /tmp/zombiepipe"],
)
.await
.map_err(|err| ProviderError::PauseNodeFailed(self.name.to_string(), err.into()))?
.map_err(|err| {
ProviderError::PauseNodeFailed(
self.name.to_string(),
anyhow!("error when pausing node: status {}: {}", err.0, err.1),
)
})?;
Ok(())
}
async fn resume(&self) -> Result<(), ProviderError> {
self.k8s_client
.pod_exec(
&self.namespace_name(),
&self.name,
vec!["sh", "-c", "echo resume > /tmp/zombiepipe"],
)
.await
.map_err(|err| ProviderError::ResumeNodeFailed(self.name.to_string(), err.into()))?
.map_err(|err| {
ProviderError::ResumeNodeFailed(
self.name.to_string(),
anyhow!("error when pausing node: status {}: {}", err.0, err.1),
)
})?;
Ok(())
}
async fn restart(&self, after: Option<Duration>) -> Result<(), ProviderError> {
if let Some(duration) = after {
sleep(duration).await;
}
self.k8s_client
.pod_exec(
&self.namespace_name(),
&self.name,
vec!["sh", "-c", "echo restart > /tmp/zombiepipe"],
)
.await
.map_err(|err| ProviderError::RestartNodeFailed(self.name.to_string(), err.into()))?
.map_err(|err| {
ProviderError::RestartNodeFailed(
self.name.to_string(),
anyhow!("error when restarting node: status {}: {}", err.0, err.1),
)
})?;
Ok(())
}
async fn destroy(&self) -> Result<(), ProviderError> {
self.k8s_client
.delete_pod(&self.namespace_name(), &self.name)
.await
.map_err(|err| ProviderError::KillNodeFailed(self.name.to_string(), err.into()))?;
if let Some(namespace) = self.namespace.upgrade() {
namespace.nodes.write().await.remove(&self.name);
}
Ok(())
}
}
@@ -0,0 +1,188 @@
use std::collections::BTreeMap;
use configuration::shared::resources::{ResourceQuantity, Resources};
use k8s_openapi::{
api::core::v1::{
ConfigMapVolumeSource, Container, EnvVar, PodSpec, ResourceRequirements, Volume,
VolumeMount,
},
apimachinery::pkg::api::resource::Quantity,
};
pub(super) struct PodSpecBuilder;
impl PodSpecBuilder {
pub(super) fn build(
name: &str,
image: &str,
resources: Option<&Resources>,
program: &str,
args: &[String],
env: &[(String, String)],
) -> PodSpec {
PodSpec {
hostname: Some(name.to_string()),
init_containers: Some(vec![Self::build_helper_binaries_setup_container()]),
containers: vec![Self::build_main_container(
name, image, resources, program, args, env,
)],
volumes: Some(Self::build_volumes()),
..Default::default()
}
}
fn build_main_container(
name: &str,
image: &str,
resources: Option<&Resources>,
program: &str,
args: &[String],
env: &[(String, String)],
) -> Container {
Container {
name: name.to_string(),
image: Some(image.to_string()),
image_pull_policy: Some("Always".to_string()),
command: Some(
[
vec!["/zombie-wrapper.sh".to_string(), program.to_string()],
args.to_vec(),
]
.concat(),
),
env: Some(
env.iter()
.map(|(name, value)| EnvVar {
name: name.clone(),
value: Some(value.clone()),
value_from: None,
})
.collect(),
),
volume_mounts: Some(Self::build_volume_mounts(vec![VolumeMount {
name: "zombie-wrapper-volume".to_string(),
mount_path: "/zombie-wrapper.sh".to_string(),
sub_path: Some("zombie-wrapper.sh".to_string()),
..Default::default()
}])),
resources: Self::build_resources_requirements(resources),
..Default::default()
}
}
fn build_helper_binaries_setup_container() -> Container {
Container {
name: "helper-binaries-setup".to_string(),
image: Some("europe-west3-docker.pkg.dev/parity-zombienet/zombienet-public-images/alpine:latest".to_string()),
image_pull_policy: Some("IfNotPresent".to_string()),
volume_mounts: Some(Self::build_volume_mounts(vec![VolumeMount {
name: "helper-binaries-downloader-volume".to_string(),
mount_path: "/helper-binaries-downloader.sh".to_string(),
sub_path: Some("helper-binaries-downloader.sh".to_string()),
..Default::default()
}])),
command: Some(vec![
"ash".to_string(),
"/helper-binaries-downloader.sh".to_string(),
]),
..Default::default()
}
}
fn build_volumes() -> Vec<Volume> {
vec![
Volume {
name: "cfg".to_string(),
..Default::default()
},
Volume {
name: "data".to_string(),
..Default::default()
},
Volume {
name: "relay-data".to_string(),
..Default::default()
},
Volume {
name: "zombie-wrapper-volume".to_string(),
config_map: Some(ConfigMapVolumeSource {
name: Some("zombie-wrapper".to_string()),
default_mode: Some(0o755),
..Default::default()
}),
..Default::default()
},
Volume {
name: "helper-binaries-downloader-volume".to_string(),
config_map: Some(ConfigMapVolumeSource {
name: Some("helper-binaries-downloader".to_string()),
default_mode: Some(0o755),
..Default::default()
}),
..Default::default()
},
]
}
fn build_volume_mounts(non_default_mounts: Vec<VolumeMount>) -> Vec<VolumeMount> {
[
vec![
VolumeMount {
name: "cfg".to_string(),
mount_path: "/cfg".to_string(),
read_only: Some(false),
..Default::default()
},
VolumeMount {
name: "data".to_string(),
mount_path: "/data".to_string(),
read_only: Some(false),
..Default::default()
},
VolumeMount {
name: "relay-data".to_string(),
mount_path: "/relay-data".to_string(),
read_only: Some(false),
..Default::default()
},
],
non_default_mounts,
]
.concat()
}
fn build_resources_requirements(resources: Option<&Resources>) -> Option<ResourceRequirements> {
resources.map(|resources| ResourceRequirements {
limits: Self::build_resources_requirements_quantities(
resources.limit_cpu(),
resources.limit_memory(),
),
requests: Self::build_resources_requirements_quantities(
resources.request_cpu(),
resources.request_memory(),
),
..Default::default()
})
}
fn build_resources_requirements_quantities(
cpu: Option<&ResourceQuantity>,
memory: Option<&ResourceQuantity>,
) -> Option<BTreeMap<String, Quantity>> {
let mut quantities = BTreeMap::new();
if let Some(cpu) = cpu {
quantities.insert("cpu".to_string(), Quantity(cpu.as_str().to_string()));
}
if let Some(memory) = memory {
quantities.insert("memory".to_string(), Quantity(memory.as_str().to_string()));
}
if !quantities.is_empty() {
Some(quantities)
} else {
None
}
}
}
@@ -0,0 +1,145 @@
use std::{
collections::HashMap,
path::{Path, PathBuf},
sync::{Arc, Weak},
};
use async_trait::async_trait;
use support::fs::FileSystem;
use tokio::sync::RwLock;
use super::{client::KubernetesClient, namespace::KubernetesNamespace};
use crate::{
shared::helpers::extract_namespace_info, types::ProviderCapabilities, DynNamespace, Provider,
ProviderError, ProviderNamespace,
};
pub const PROVIDER_NAME: &str = "k8s";
pub struct KubernetesProvider<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
weak: Weak<KubernetesProvider<FS>>,
capabilities: ProviderCapabilities,
tmp_dir: PathBuf,
k8s_client: KubernetesClient,
filesystem: FS,
pub(super) namespaces: RwLock<HashMap<String, Arc<KubernetesNamespace<FS>>>>,
}
impl<FS> KubernetesProvider<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
pub async fn new(filesystem: FS) -> Arc<Self> {
let k8s_client = KubernetesClient::new().await.unwrap();
Arc::new_cyclic(|weak| KubernetesProvider {
weak: weak.clone(),
capabilities: ProviderCapabilities {
requires_image: true,
has_resources: true,
prefix_with_full_path: false,
use_default_ports_in_cmd: true,
},
tmp_dir: std::env::temp_dir(),
k8s_client,
filesystem,
namespaces: RwLock::new(HashMap::new()),
})
}
pub fn tmp_dir(mut self, tmp_dir: impl Into<PathBuf>) -> Self {
self.tmp_dir = tmp_dir.into();
self
}
}
#[async_trait]
impl<FS> Provider for KubernetesProvider<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
fn name(&self) -> &str {
PROVIDER_NAME
}
fn capabilities(&self) -> &ProviderCapabilities {
&self.capabilities
}
async fn namespaces(&self) -> HashMap<String, DynNamespace> {
self.namespaces
.read()
.await
.iter()
.map(|(name, namespace)| (name.clone(), namespace.clone() as DynNamespace))
.collect()
}
async fn create_namespace(&self) -> Result<DynNamespace, ProviderError> {
let namespace = KubernetesNamespace::new(
&self.weak,
&self.tmp_dir,
&self.capabilities,
&self.k8s_client,
&self.filesystem,
None,
)
.await?;
self.namespaces
.write()
.await
.insert(namespace.name().to_string(), namespace.clone());
Ok(namespace)
}
async fn create_namespace_with_base_dir(
&self,
base_dir: &Path,
) -> Result<DynNamespace, ProviderError> {
let namespace = KubernetesNamespace::new(
&self.weak,
&self.tmp_dir,
&self.capabilities,
&self.k8s_client,
&self.filesystem,
Some(base_dir),
)
.await?;
self.namespaces
.write()
.await
.insert(namespace.name().to_string(), namespace.clone());
Ok(namespace)
}
async fn create_namespace_from_json(
&self,
json_value: &serde_json::Value,
) -> Result<DynNamespace, ProviderError> {
let (base_dir, name) = extract_namespace_info(json_value)?;
let namespace = KubernetesNamespace::attach_to_live(
&self.weak,
&self.capabilities,
&self.k8s_client,
&self.filesystem,
&base_dir,
&name,
)
.await?;
self.namespaces
.write()
.await
.insert(namespace.name().to_string(), namespace.clone());
Ok(namespace)
}
}
@@ -0,0 +1,10 @@
apiVersion: v1
kind: LimitRange
metadata:
name: mem-limit-range
spec:
limits:
- defaultRequest:
memory: 1G
cpu: 0.5
type: Container
@@ -0,0 +1,23 @@
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: internal-access
spec:
podSelector: {}
ingress:
- from:
- namespaceSelector:
matchExpressions:
- key: kubernetes.io/metadata.name
operator: In
values:
- {{namespace}}
- gitlab
- arc-runner
- loki
- tempo
- monitoring
- parachain-exporter
- default
policyTypes:
- Ingress
+264
View File
@@ -0,0 +1,264 @@
#![allow(clippy::expect_fun_call)]
mod docker;
mod kubernetes;
mod native;
pub mod shared;
use std::{
collections::HashMap,
net::IpAddr,
path::{Path, PathBuf},
sync::Arc,
time::Duration,
};
use async_trait::async_trait;
use shared::{
constants::LOCALHOST,
types::{
ExecutionResult, GenerateFilesOptions, ProviderCapabilities, RunCommandOptions,
RunScriptOptions, SpawnNodeOptions,
},
};
use support::fs::FileSystemError;
#[derive(Debug, thiserror::Error)]
#[allow(missing_docs)]
pub enum ProviderError {
#[error("Failed to create client '{0}': {1}")]
CreateClientFailed(String, anyhow::Error),
#[error("Failed to create namespace '{0}': {1}")]
CreateNamespaceFailed(String, anyhow::Error),
#[error("Failed to spawn node '{0}': {1}")]
NodeSpawningFailed(String, anyhow::Error),
#[error("Error running command '{0}' {1}: {2}")]
RunCommandError(String, String, anyhow::Error),
#[error("Error running script'{0}': {1}")]
RunScriptError(String, anyhow::Error),
#[error("Invalid network configuration field {0}")]
InvalidConfig(String),
#[error("Failed to retrieve node available args using image {0} and command {1}: {2}")]
NodeAvailableArgsError(String, String, String),
#[error("Can not recover node: {0}")]
MissingNode(String),
#[error("Can not recover node: {0} info, field: {1}")]
MissingNodeInfo(String, String),
#[error("File generation failed: {0}")]
FileGenerationFailed(anyhow::Error),
#[error(transparent)]
FileSystemError(#[from] FileSystemError),
#[error("Invalid script path for {0}")]
InvalidScriptPath(anyhow::Error),
#[error("Script with path {0} not found")]
ScriptNotFound(PathBuf),
#[error("Failed to retrieve process ID for node '{0}'")]
ProcessIdRetrievalFailed(String),
#[error("Failed to pause node '{0}': {1}")]
PauseNodeFailed(String, anyhow::Error),
#[error("Failed to resume node '{0}': {1}")]
ResumeNodeFailed(String, anyhow::Error),
#[error("Failed to kill node '{0}': {1}")]
KillNodeFailed(String, anyhow::Error),
#[error("Failed to restart node '{0}': {1}")]
RestartNodeFailed(String, anyhow::Error),
#[error("Failed to destroy node '{0}': {1}")]
DestroyNodeFailed(String, anyhow::Error),
#[error("Failed to get logs for node '{0}': {1}")]
GetLogsFailed(String, anyhow::Error),
#[error("Failed to dump logs for node '{0}': {1}")]
DumpLogsFailed(String, anyhow::Error),
#[error("Failed to copy file from node '{0}': {1}")]
CopyFileFromNodeError(String, anyhow::Error),
#[error("Failed to setup fileserver: {0}")]
FileServerSetupError(anyhow::Error),
#[error("Error uploading file: '{0}': {1}")]
UploadFile(String, anyhow::Error),
#[error("Error downloading file: '{0}': {1}")]
DownloadFile(String, anyhow::Error),
#[error("Error sending file '{0}' to {1}: {2}")]
SendFile(String, String, anyhow::Error),
#[error("Error creating port-forward '{0}:{1}': {2}")]
PortForwardError(u16, u16, anyhow::Error),
#[error("Failed to delete namespace '{0}': {1}")]
DeleteNamespaceFailed(String, anyhow::Error),
#[error("Serialization error")]
SerializationError(#[from] serde_json::Error),
#[error("Failed to acquire lock: {0}")]
FailedToAcquireLock(String),
}
#[async_trait]
pub trait Provider {
fn name(&self) -> &str;
fn capabilities(&self) -> &ProviderCapabilities;
async fn namespaces(&self) -> HashMap<String, DynNamespace>;
async fn create_namespace(&self) -> Result<DynNamespace, ProviderError>;
async fn create_namespace_with_base_dir(
&self,
base_dir: &Path,
) -> Result<DynNamespace, ProviderError>;
async fn create_namespace_from_json(
&self,
json_value: &serde_json::Value,
) -> Result<DynNamespace, ProviderError>;
}
pub type DynProvider = Arc<dyn Provider + Send + Sync>;
#[async_trait]
pub trait ProviderNamespace {
fn name(&self) -> &str;
fn base_dir(&self) -> &PathBuf;
fn capabilities(&self) -> &ProviderCapabilities;
fn provider_name(&self) -> &str;
async fn detach(&self) {
// noop by default
warn!("Detach is not implemented for {}", self.name());
}
async fn is_detached(&self) -> bool {
// false by default
false
}
async fn nodes(&self) -> HashMap<String, DynNode>;
async fn get_node_available_args(
&self,
options: (String, Option<String>),
) -> Result<String, ProviderError>;
async fn spawn_node(&self, options: &SpawnNodeOptions) -> Result<DynNode, ProviderError>;
async fn spawn_node_from_json(
&self,
json_value: &serde_json::Value,
) -> Result<DynNode, ProviderError>;
async fn generate_files(&self, options: GenerateFilesOptions) -> Result<(), ProviderError>;
async fn destroy(&self) -> Result<(), ProviderError>;
async fn static_setup(&self) -> Result<(), ProviderError>;
}
pub type DynNamespace = Arc<dyn ProviderNamespace + Send + Sync>;
#[async_trait]
pub trait ProviderNode: erased_serde::Serialize {
fn name(&self) -> &str;
fn args(&self) -> Vec<&str>;
fn base_dir(&self) -> &PathBuf;
fn config_dir(&self) -> &PathBuf;
fn data_dir(&self) -> &PathBuf;
fn relay_data_dir(&self) -> &PathBuf;
fn scripts_dir(&self) -> &PathBuf;
fn log_path(&self) -> &PathBuf;
fn log_cmd(&self) -> String;
// Return the absolute path to the file in the `node` perspective
// TODO: purpose?
fn path_in_node(&self, file: &Path) -> PathBuf;
async fn logs(&self) -> Result<String, ProviderError>;
async fn dump_logs(&self, local_dest: PathBuf) -> Result<(), ProviderError>;
// By default return localhost, should be overrided for k8s
async fn ip(&self) -> Result<IpAddr, ProviderError> {
Ok(LOCALHOST)
}
// Noop by default (native/docker provider)
async fn create_port_forward(
&self,
_local_port: u16,
_remote_port: u16,
) -> Result<Option<u16>, ProviderError> {
Ok(None)
}
async fn run_command(
&self,
options: RunCommandOptions,
) -> Result<ExecutionResult, ProviderError>;
async fn run_script(&self, options: RunScriptOptions)
-> Result<ExecutionResult, ProviderError>;
async fn send_file(
&self,
local_file_path: &Path,
remote_file_path: &Path,
mode: &str,
) -> Result<(), ProviderError>;
async fn receive_file(
&self,
remote_file_path: &Path,
local_file_path: &Path,
) -> Result<(), ProviderError>;
async fn pause(&self) -> Result<(), ProviderError>;
async fn resume(&self) -> Result<(), ProviderError>;
async fn restart(&self, after: Option<Duration>) -> Result<(), ProviderError>;
async fn destroy(&self) -> Result<(), ProviderError>;
}
pub type DynNode = Arc<dyn ProviderNode + Send + Sync>;
// re-export
pub use docker::*;
pub use kubernetes::*;
pub use native::*;
pub use shared::{constants, types};
use tracing::warn;
@@ -0,0 +1,5 @@
mod namespace;
mod node;
mod provider;
pub use provider::NativeProvider;
@@ -0,0 +1,374 @@
use std::{
collections::HashMap,
path::{Path, PathBuf},
sync::{Arc, Weak},
};
use async_trait::async_trait;
use support::fs::FileSystem;
use tokio::sync::RwLock;
use tracing::{trace, warn};
use uuid::Uuid;
use super::node::{NativeNode, NativeNodeOptions};
use crate::{
constants::NAMESPACE_PREFIX,
native::{node::DeserializableNativeNodeOptions, provider},
shared::helpers::extract_execution_result,
types::{
GenerateFileCommand, GenerateFilesOptions, ProviderCapabilities, RunCommandOptions,
SpawnNodeOptions,
},
DynNode, NativeProvider, ProviderError, ProviderNamespace, ProviderNode,
};
pub(super) struct NativeNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
weak: Weak<NativeNamespace<FS>>,
name: String,
provider: Weak<NativeProvider<FS>>,
base_dir: PathBuf,
capabilities: ProviderCapabilities,
filesystem: FS,
pub(super) nodes: RwLock<HashMap<String, Arc<NativeNode<FS>>>>,
}
impl<FS> NativeNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) async fn new(
provider: &Weak<NativeProvider<FS>>,
tmp_dir: &PathBuf,
capabilities: &ProviderCapabilities,
filesystem: &FS,
custom_base_dir: Option<&Path>,
) -> Result<Arc<Self>, ProviderError> {
let name = format!("{}{}", NAMESPACE_PREFIX, Uuid::new_v4());
let base_dir = if let Some(custom_base_dir) = custom_base_dir {
if !filesystem.exists(custom_base_dir).await {
filesystem.create_dir_all(custom_base_dir).await?;
} else {
warn!(
"⚠️ Using and existing directory {} as base dir",
custom_base_dir.to_string_lossy()
);
}
PathBuf::from(custom_base_dir)
} else {
let base_dir = PathBuf::from_iter([tmp_dir, &PathBuf::from(&name)]);
filesystem.create_dir(&base_dir).await?;
base_dir
};
Ok(Arc::new_cyclic(|weak| NativeNamespace {
weak: weak.clone(),
provider: provider.clone(),
name,
base_dir,
capabilities: capabilities.clone(),
filesystem: filesystem.clone(),
nodes: RwLock::new(HashMap::new()),
}))
}
pub(super) async fn attach_to_live(
provider: &Weak<NativeProvider<FS>>,
capabilities: &ProviderCapabilities,
filesystem: &FS,
custom_base_dir: &Path,
name: &str,
) -> Result<Arc<Self>, ProviderError> {
let base_dir = custom_base_dir.to_path_buf();
Ok(Arc::new_cyclic(|weak| NativeNamespace {
weak: weak.clone(),
provider: provider.clone(),
name: name.to_string(),
base_dir,
capabilities: capabilities.clone(),
filesystem: filesystem.clone(),
nodes: RwLock::new(HashMap::new()),
}))
}
}
#[async_trait]
impl<FS> ProviderNamespace for NativeNamespace<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
fn name(&self) -> &str {
&self.name
}
fn base_dir(&self) -> &PathBuf {
&self.base_dir
}
fn capabilities(&self) -> &ProviderCapabilities {
&self.capabilities
}
fn provider_name(&self) -> &str {
provider::PROVIDER_NAME
}
async fn nodes(&self) -> HashMap<String, DynNode> {
self.nodes
.read()
.await
.iter()
.map(|(name, node)| (name.clone(), node.clone() as DynNode))
.collect()
}
async fn get_node_available_args(
&self,
(command, _image): (String, Option<String>),
) -> Result<String, ProviderError> {
let temp_node = self
.spawn_node(
&SpawnNodeOptions::new(format!("temp-{}", Uuid::new_v4()), "bash".to_string())
.args(vec!["-c", "while :; do sleep 1; done"]),
)
.await?;
let available_args_output = temp_node
.run_command(RunCommandOptions::new(command.clone()).args(vec!["--help"]))
.await?
.map_err(|(_exit, status)| {
ProviderError::NodeAvailableArgsError("".to_string(), command, status)
})?;
temp_node.destroy().await?;
Ok(available_args_output)
}
async fn spawn_node(&self, options: &SpawnNodeOptions) -> Result<DynNode, ProviderError> {
trace!("spawn node options {options:?}");
let node = NativeNode::new(NativeNodeOptions {
namespace: &self.weak,
namespace_base_dir: &self.base_dir,
name: &options.name,
program: &options.program,
args: &options.args,
env: &options.env,
startup_files: &options.injected_files,
created_paths: &options.created_paths,
db_snapshot: options.db_snapshot.as_ref(),
filesystem: &self.filesystem,
node_log_path: options.node_log_path.as_ref(),
})
.await?;
self.nodes
.write()
.await
.insert(options.name.clone(), node.clone());
Ok(node)
}
async fn spawn_node_from_json(
&self,
json_value: &serde_json::Value,
) -> Result<DynNode, ProviderError> {
let deserializable: DeserializableNativeNodeOptions =
serde_json::from_value(json_value.clone())?;
let options = NativeNodeOptions::from_deserializable(
&deserializable,
&self.weak,
&self.base_dir,
&self.filesystem,
);
let pid = json_value
.get("process_handle")
.and_then(|v| v.as_i64())
.ok_or_else(|| ProviderError::InvalidConfig("Missing pid field".to_string()))?
as i32;
let node = NativeNode::attach_to_live(options, pid).await?;
self.nodes
.write()
.await
.insert(node.name().to_string(), node.clone());
Ok(node)
}
async fn generate_files(&self, options: GenerateFilesOptions) -> Result<(), ProviderError> {
let node_name = if let Some(name) = options.temp_name {
name
} else {
format!("temp-{}", Uuid::new_v4())
};
// we spawn a node doing nothing but looping so we can execute our commands
let temp_node = self
.spawn_node(
&SpawnNodeOptions::new(node_name, "bash".to_string())
.args(vec!["-c", "while :; do sleep 1; done"])
.injected_files(options.injected_files),
)
.await?;
for GenerateFileCommand {
program,
args,
env,
local_output_path,
} in options.commands
{
trace!(
"🏗 building file {:?} in path {} with command {} {}",
local_output_path.as_os_str(),
self.base_dir.to_string_lossy(),
program,
args.join(" ")
);
let local_output_full_path = format!(
"{}{}{}",
self.base_dir.to_string_lossy(),
if local_output_path.starts_with("/") {
""
} else {
"/"
},
local_output_path.to_string_lossy()
);
let contents = extract_execution_result(
&temp_node,
RunCommandOptions { program, args, env },
options.expected_path.as_ref(),
)
.await?;
self.filesystem
.write(local_output_full_path, contents)
.await
.map_err(|err| ProviderError::FileGenerationFailed(err.into()))?;
}
temp_node.destroy().await
}
async fn static_setup(&self) -> Result<(), ProviderError> {
// no static setup exists for native provider
todo!()
}
async fn destroy(&self) -> Result<(), ProviderError> {
let mut names = vec![];
for node in self.nodes.read().await.values() {
node.abort()
.await
.map_err(|err| ProviderError::DestroyNodeFailed(node.name().to_string(), err))?;
names.push(node.name().to_string());
}
let mut nodes = self.nodes.write().await;
for name in names {
nodes.remove(&name);
}
if let Some(provider) = self.provider.upgrade() {
provider.namespaces.write().await.remove(&self.name);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use support::fs::local::LocalFileSystem;
use super::*;
use crate::{
types::{GenerateFileCommand, GenerateFilesOptions},
NativeProvider, Provider,
};
fn unique_temp_dir() -> PathBuf {
let mut base = std::env::temp_dir();
base.push(format!("znet_native_ns_test_{}", uuid::Uuid::new_v4()));
base
}
#[tokio::test]
async fn generate_files_uses_expected_path_when_provided() {
let fs = LocalFileSystem;
let provider = NativeProvider::new(fs.clone());
let base_dir = unique_temp_dir();
// Namespace builder will create directory if needed
let ns = provider
.create_namespace_with_base_dir(&base_dir)
.await
.expect("namespace should be created");
// Create a unique on-host path that the native node will write to
let expected_path =
std::env::temp_dir().join(format!("znet_expected_{}.json", uuid::Uuid::new_v4()));
// Command will write JSON into expected_path; stdout will be something else to ensure we don't read it
let program = "bash".to_string();
let script = format!(
"echo -n '{{\"hello\":\"world\"}}' > {} && echo should_not_be_used",
expected_path.to_string_lossy()
);
let args: Vec<String> = vec!["-lc".into(), script];
let out_name = PathBuf::from("result_expected.json");
let cmd = GenerateFileCommand::new(program, out_name.clone()).args(args);
let options = GenerateFilesOptions::new(vec![cmd], None, Some(expected_path.clone()));
ns.generate_files(options)
.await
.expect("generation should succeed");
// Read produced file from namespace base_dir
let produced_path = base_dir.join(out_name);
let produced = fs
.read_to_string(&produced_path)
.await
.expect("should read produced file");
assert_eq!(produced, "{\"hello\":\"world\"}");
}
#[tokio::test]
async fn generate_files_uses_stdout_when_expected_path_absent() {
let fs = LocalFileSystem;
let provider = NativeProvider::new(fs.clone());
let base_dir = unique_temp_dir();
let ns = provider
.create_namespace_with_base_dir(&base_dir)
.await
.expect("namespace should be created");
// Command prints to stdout only
let program = "bash".to_string();
let args: Vec<String> = vec!["-lc".into(), "echo -n 42".into()];
let out_name = PathBuf::from("result_stdout.txt");
let cmd = GenerateFileCommand::new(program, out_name.clone()).args(args);
let options = GenerateFilesOptions::new(vec![cmd], None, None);
ns.generate_files(options)
.await
.expect("generation should succeed");
let produced_path = base_dir.join(out_name);
let produced = fs
.read_to_string(&produced_path)
.await
.expect("should read produced file");
assert_eq!(produced, "42");
}
}
@@ -0,0 +1,734 @@
use std::{
collections::HashMap,
env,
path::{Path, PathBuf},
process::Stdio,
sync::{Arc, Weak},
time::Duration,
};
use anyhow::anyhow;
use async_trait::async_trait;
use configuration::types::AssetLocation;
use flate2::read::GzDecoder;
use futures::future::try_join_all;
use nix::{
sys::signal::{kill, Signal},
unistd::Pid,
};
use serde::{ser::Error, Deserialize, Serialize, Serializer};
use sha2::Digest;
use support::{constants::THIS_IS_A_BUG, fs::FileSystem};
use tar::Archive;
use tokio::{
fs,
io::{AsyncRead, AsyncReadExt, BufReader},
process::{Child, ChildStderr, ChildStdout, Command},
sync::{
mpsc::{self, Sender},
RwLock,
},
task::JoinHandle,
time::sleep,
try_join,
};
use tracing::trace;
use super::namespace::NativeNamespace;
use crate::{
constants::{NODE_CONFIG_DIR, NODE_DATA_DIR, NODE_RELAY_DATA_DIR, NODE_SCRIPTS_DIR},
native,
types::{ExecutionResult, RunCommandOptions, RunScriptOptions, TransferedFile},
ProviderError, ProviderNamespace, ProviderNode,
};
pub(super) struct NativeNodeOptions<'a, FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) namespace: &'a Weak<NativeNamespace<FS>>,
pub(super) namespace_base_dir: &'a PathBuf,
pub(super) name: &'a str,
pub(super) program: &'a str,
pub(super) args: &'a [String],
pub(super) env: &'a [(String, String)],
pub(super) startup_files: &'a [TransferedFile],
pub(super) created_paths: &'a [PathBuf],
pub(super) db_snapshot: Option<&'a AssetLocation>,
pub(super) filesystem: &'a FS,
pub(super) node_log_path: Option<&'a PathBuf>,
}
impl<'a, FS> NativeNodeOptions<'a, FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) fn from_deserializable(
deserializable: &'a DeserializableNativeNodeOptions,
namespace: &'a Weak<NativeNamespace<FS>>,
namespace_base_dir: &'a PathBuf,
filesystem: &'a FS,
) -> NativeNodeOptions<'a, FS> {
NativeNodeOptions {
namespace,
namespace_base_dir,
name: &deserializable.name,
program: &deserializable.program,
args: &deserializable.args,
env: &deserializable.env,
startup_files: &[],
created_paths: &[],
db_snapshot: None,
filesystem,
node_log_path: deserializable.node_log_path.as_ref(),
}
}
}
#[derive(Deserialize)]
pub(super) struct DeserializableNativeNodeOptions {
pub name: String,
pub program: String,
pub args: Vec<String>,
pub env: Vec<(String, String)>,
pub node_log_path: Option<PathBuf>,
}
enum ProcessHandle {
Spawned(Child, Pid),
Attached(Pid),
}
#[derive(Serialize)]
pub(super) struct NativeNode<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
#[serde(skip)]
namespace: Weak<NativeNamespace<FS>>,
name: String,
program: String,
args: Vec<String>,
env: Vec<(String, String)>,
base_dir: PathBuf,
config_dir: PathBuf,
data_dir: PathBuf,
relay_data_dir: PathBuf,
scripts_dir: PathBuf,
log_path: PathBuf,
#[serde(serialize_with = "serialize_process_handle")]
// using RwLock from std to serialize properly, generally using sync locks is ok in async code as long as they
// are not held across await points
process_handle: std::sync::RwLock<Option<ProcessHandle>>,
#[serde(skip)]
stdout_reading_task: RwLock<Option<JoinHandle<()>>>,
#[serde(skip)]
stderr_reading_task: RwLock<Option<JoinHandle<()>>>,
#[serde(skip)]
log_writing_task: RwLock<Option<JoinHandle<()>>>,
#[serde(skip)]
filesystem: FS,
provider_tag: String,
}
impl<FS> NativeNode<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
pub(super) async fn new(
options: NativeNodeOptions<'_, FS>,
) -> Result<Arc<Self>, ProviderError> {
let filesystem = options.filesystem.clone();
let base_dir =
PathBuf::from_iter([options.namespace_base_dir, &PathBuf::from(options.name)]);
trace!("creating base_dir {:?}", base_dir);
options.filesystem.create_dir_all(&base_dir).await?;
trace!("created base_dir {:?}", base_dir);
let base_dir_raw = base_dir.to_string_lossy();
let config_dir = PathBuf::from(format!("{base_dir_raw}{NODE_CONFIG_DIR}"));
let data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_DATA_DIR}"));
let relay_data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_RELAY_DATA_DIR}"));
let scripts_dir = PathBuf::from(format!("{base_dir_raw}{NODE_SCRIPTS_DIR}"));
let log_path = options
.node_log_path
.cloned()
.unwrap_or_else(|| base_dir.join(format!("{}.log", options.name)));
trace!("creating dirs {:?}", config_dir);
try_join!(
filesystem.create_dir_all(&config_dir),
filesystem.create_dir_all(&data_dir),
filesystem.create_dir_all(&relay_data_dir),
filesystem.create_dir_all(&scripts_dir),
)?;
trace!("created!");
let node = Arc::new(NativeNode {
namespace: options.namespace.clone(),
name: options.name.to_string(),
program: options.program.to_string(),
args: options.args.to_vec(),
env: options.env.to_vec(),
base_dir,
config_dir,
data_dir,
relay_data_dir,
scripts_dir,
log_path,
process_handle: std::sync::RwLock::new(None),
stdout_reading_task: RwLock::new(None),
stderr_reading_task: RwLock::new(None),
log_writing_task: RwLock::new(None),
filesystem: filesystem.clone(),
provider_tag: native::provider::PROVIDER_NAME.to_string(),
});
node.initialize_startup_paths(options.created_paths).await?;
node.initialize_startup_files(options.startup_files).await?;
if let Some(db_snap) = options.db_snapshot {
node.initialize_db_snapshot(db_snap).await?;
}
let (stdout, stderr) = node.initialize_process().await?;
node.initialize_log_writing(stdout, stderr).await;
Ok(node)
}
pub(super) async fn attach_to_live(
options: NativeNodeOptions<'_, FS>,
pid: i32,
) -> Result<Arc<Self>, ProviderError> {
let filesystem = options.filesystem.clone();
let base_dir =
PathBuf::from_iter([options.namespace_base_dir, &PathBuf::from(options.name)]);
trace!("creating base_dir {:?}", base_dir);
options.filesystem.create_dir_all(&base_dir).await?;
trace!("created base_dir {:?}", base_dir);
let base_dir_raw = base_dir.to_string_lossy();
let config_dir = PathBuf::from(format!("{base_dir_raw}{NODE_CONFIG_DIR}"));
let data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_DATA_DIR}"));
let relay_data_dir = PathBuf::from(format!("{base_dir_raw}{NODE_RELAY_DATA_DIR}"));
let scripts_dir = PathBuf::from(format!("{base_dir_raw}{NODE_SCRIPTS_DIR}"));
let log_path = options
.node_log_path
.cloned()
.unwrap_or_else(|| base_dir.join(format!("{}.log", options.name)));
let pid = Pid::from_raw(pid);
let node = Arc::new(NativeNode {
namespace: options.namespace.clone(),
name: options.name.to_string(),
program: options.program.to_string(),
args: options.args.to_vec(),
env: options.env.to_vec(),
base_dir,
config_dir,
data_dir,
relay_data_dir,
scripts_dir,
log_path,
process_handle: std::sync::RwLock::new(Some(ProcessHandle::Attached(pid))),
stdout_reading_task: RwLock::new(None),
stderr_reading_task: RwLock::new(None),
log_writing_task: RwLock::new(None),
filesystem: filesystem.clone(),
provider_tag: native::provider::PROVIDER_NAME.to_string(),
});
Ok(node)
}
async fn initialize_startup_paths(&self, paths: &[PathBuf]) -> Result<(), ProviderError> {
trace!("creating paths {:?}", paths);
let base_dir_raw = self.base_dir.to_string_lossy();
try_join_all(paths.iter().map(|file| {
let full_path = format!("{base_dir_raw}{}", file.to_string_lossy());
self.filesystem.create_dir_all(full_path)
}))
.await?;
trace!("paths created!");
Ok(())
}
async fn initialize_startup_files(
&self,
startup_files: &[TransferedFile],
) -> Result<(), ProviderError> {
trace!("creating files {:?}", startup_files);
try_join_all(
startup_files
.iter()
.map(|file| self.send_file(&file.local_path, &file.remote_path, &file.mode)),
)
.await?;
trace!("files created!");
Ok(())
}
async fn initialize_db_snapshot(
&self,
db_snapshot: &AssetLocation,
) -> Result<(), ProviderError> {
trace!("snap: {db_snapshot}");
// check if we need to get the db or is already in the ns
let ns_base_dir = self.namespace_base_dir();
let hashed_location = match db_snapshot {
AssetLocation::Url(location) => hex::encode(sha2::Sha256::digest(location.to_string())),
AssetLocation::FilePath(filepath) => {
hex::encode(sha2::Sha256::digest(filepath.to_string_lossy().to_string()))
},
};
let full_path = format!("{ns_base_dir}/{hashed_location}.tgz");
trace!("db_snap fullpath in ns: {full_path}");
if !self.filesystem.exists(&full_path).await {
// needs to download/copy
self.get_db_snapshot(db_snapshot, &full_path).await?;
}
let contents = self.filesystem.read(&full_path).await.unwrap();
let gz = GzDecoder::new(&contents[..]);
let mut archive = Archive::new(gz);
archive
.unpack(self.base_dir.to_string_lossy().as_ref())
.unwrap();
if std::env::var("ZOMBIE_RM_TGZ_AFTER_EXTRACT").is_ok() {
let res = fs::remove_file(&full_path).await;
trace!("removing {}, result {:?}", full_path, res);
}
Ok(())
}
async fn get_db_snapshot(
&self,
location: &AssetLocation,
full_path: &str,
) -> Result<(), ProviderError> {
trace!("getting db_snapshot from: {:?} to: {full_path}", location);
match location {
AssetLocation::Url(location) => {
let res = reqwest::get(location.as_ref())
.await
.map_err(|err| ProviderError::DownloadFile(location.to_string(), err.into()))?;
let contents: &[u8] = &res.bytes().await.unwrap();
trace!("writing: {full_path}");
self.filesystem.write(full_path, contents).await?;
},
AssetLocation::FilePath(filepath) => {
self.filesystem.copy(filepath, full_path).await?;
},
};
Ok(())
}
async fn initialize_process(&self) -> Result<(ChildStdout, ChildStderr), ProviderError> {
let filtered_env: HashMap<String, String> = env::vars()
.filter(|(k, _)| k == "TZ" || k == "LANG" || k == "PATH")
.collect();
let mut process = Command::new(&self.program)
.args(&self.args)
.env_clear()
.envs(&filtered_env) // minimal environment
.envs(self.env.to_vec())
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true)
.current_dir(&self.base_dir)
.spawn()
.map_err(|err| ProviderError::NodeSpawningFailed(self.name.to_string(), err.into()))?;
let stdout = process
.stdout
.take()
.expect(&format!("infaillible, stdout is piped {THIS_IS_A_BUG}"));
let stderr = process
.stderr
.take()
.expect(&format!("infaillible, stderr is piped {THIS_IS_A_BUG}"));
let pid = Pid::from_raw(
process
.id()
.ok_or_else(|| ProviderError::ProcessIdRetrievalFailed(self.name.to_string()))?
as i32,
);
self.process_handle
.write()
.map_err(|_e| ProviderError::FailedToAcquireLock(self.name.clone()))?
.replace(ProcessHandle::Spawned(process, pid));
Ok((stdout, stderr))
}
async fn initialize_log_writing(&self, stdout: ChildStdout, stderr: ChildStderr) {
let (stdout_tx, mut rx) = mpsc::channel(10);
let stderr_tx = stdout_tx.clone();
self.stdout_reading_task
.write()
.await
.replace(self.create_stream_polling_task(stdout, stdout_tx));
self.stderr_reading_task
.write()
.await
.replace(self.create_stream_polling_task(stderr, stderr_tx));
let filesystem = self.filesystem.clone();
let log_path = self.log_path.clone();
self.log_writing_task
.write()
.await
.replace(tokio::spawn(async move {
loop {
while let Some(Ok(data)) = rx.recv().await {
// TODO: find a better way instead of ignoring error ?
let _ = filesystem.append(&log_path, data).await;
}
sleep(Duration::from_millis(250)).await;
}
}));
}
fn create_stream_polling_task(
&self,
stream: impl AsyncRead + Unpin + Send + 'static,
tx: Sender<Result<Vec<u8>, std::io::Error>>,
) -> JoinHandle<()> {
tokio::spawn(async move {
let mut reader = BufReader::new(stream);
let mut buffer = vec![0u8; 1024];
loop {
match reader.read(&mut buffer).await {
Ok(0) => {
let _ = tx.send(Ok(Vec::new())).await;
break;
},
Ok(n) => {
let _ = tx.send(Ok(buffer[..n].to_vec())).await;
},
Err(e) => {
let _ = tx.send(Err(e)).await;
break;
},
}
}
})
}
fn process_id(&self) -> Result<Pid, ProviderError> {
let pid = self
.process_handle
.read()
.map_err(|_e| ProviderError::FailedToAcquireLock(self.name.clone()))?
.as_ref()
.map(|handle| match handle {
ProcessHandle::Spawned(_, pid) => *pid,
ProcessHandle::Attached(pid) => *pid,
})
.ok_or_else(|| ProviderError::ProcessIdRetrievalFailed(self.name.to_string()))?;
Ok(pid)
}
pub(crate) async fn abort(&self) -> anyhow::Result<()> {
if let Some(task) = self.log_writing_task.write().await.take() {
task.abort();
}
if let Some(task) = self.stdout_reading_task.write().await.take() {
task.abort();
}
if let Some(task) = self.stderr_reading_task.write().await.take() {
task.abort();
}
let process_handle = {
let mut guard = self
.process_handle
.write()
.map_err(|_e| ProviderError::FailedToAcquireLock(self.name.clone()))?;
guard
.take()
.ok_or_else(|| anyhow!("no process was attached for the node"))?
};
match process_handle {
ProcessHandle::Spawned(mut child, _pid) => {
child.kill().await?;
},
ProcessHandle::Attached(pid) => {
kill(pid, Signal::SIGKILL)
.map_err(|err| anyhow!("Failed to kill attached process {pid}: {err}"))?;
},
}
Ok(())
}
fn namespace_base_dir(&self) -> String {
self.namespace
.upgrade()
.map(|namespace| namespace.base_dir().to_string_lossy().to_string())
.unwrap_or_else(|| panic!("namespace shouldn't be dropped, {THIS_IS_A_BUG}"))
}
}
#[async_trait]
impl<FS> ProviderNode for NativeNode<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
fn name(&self) -> &str {
&self.name
}
fn args(&self) -> Vec<&str> {
self.args.iter().map(|arg| arg.as_str()).collect()
}
fn base_dir(&self) -> &PathBuf {
&self.base_dir
}
fn config_dir(&self) -> &PathBuf {
&self.config_dir
}
fn data_dir(&self) -> &PathBuf {
&self.data_dir
}
fn relay_data_dir(&self) -> &PathBuf {
&self.relay_data_dir
}
fn scripts_dir(&self) -> &PathBuf {
&self.scripts_dir
}
fn log_path(&self) -> &PathBuf {
&self.log_path
}
fn log_cmd(&self) -> String {
format!("tail -f {}", self.log_path().to_string_lossy())
}
fn path_in_node(&self, file: &Path) -> PathBuf {
let full_path = format!(
"{}/{}",
self.base_dir.to_string_lossy(),
file.to_string_lossy()
);
PathBuf::from(full_path)
}
async fn logs(&self) -> Result<String, ProviderError> {
Ok(self.filesystem.read_to_string(&self.log_path).await?)
}
async fn dump_logs(&self, local_dest: PathBuf) -> Result<(), ProviderError> {
Ok(self.filesystem.copy(&self.log_path, local_dest).await?)
}
async fn run_command(
&self,
options: RunCommandOptions,
) -> Result<ExecutionResult, ProviderError> {
let result = Command::new(options.program.clone())
.args(options.args.clone())
.envs(options.env)
.current_dir(&self.base_dir)
.output()
.await
.map_err(|err| {
ProviderError::RunCommandError(
format!("{} {}", &options.program, &options.args.join(" ")),
"locally".to_string(),
err.into(),
)
})?;
if result.status.success() {
Ok(Ok(String::from_utf8_lossy(&result.stdout).to_string()))
} else {
Ok(Err((
result.status,
String::from_utf8_lossy(&result.stderr).to_string(),
)))
}
}
async fn run_script(
&self,
options: RunScriptOptions,
) -> Result<ExecutionResult, ProviderError> {
let local_script_path = PathBuf::from(&options.local_script_path);
if !self.filesystem.exists(&local_script_path).await {
return Err(ProviderError::ScriptNotFound(local_script_path));
}
// extract file name and build remote file path
let script_file_name = local_script_path
.file_name()
.map(|file_name| file_name.to_string_lossy().to_string())
.ok_or(ProviderError::InvalidScriptPath(anyhow!(
"Can't retrieve filename from script with path: {:?}",
options.local_script_path
)))?;
let remote_script_path = format!(
"{}/{}",
self.scripts_dir.to_string_lossy(),
script_file_name
);
// copy and set script's execute permission
self.filesystem
.copy(local_script_path, &remote_script_path)
.await?;
self.filesystem.set_mode(&remote_script_path, 0o744).await?;
// execute script
self.run_command(RunCommandOptions {
program: remote_script_path,
args: options.args,
env: options.env,
})
.await
}
async fn send_file(
&self,
local_file_path: &Path,
remote_file_path: &Path,
mode: &str,
) -> Result<(), ProviderError> {
let namespaced_remote_file_path = PathBuf::from(format!(
"{}{}",
&self.base_dir.to_string_lossy(),
remote_file_path.to_string_lossy()
));
self.filesystem
.copy(local_file_path, &namespaced_remote_file_path)
.await?;
self.run_command(
RunCommandOptions::new("chmod")
.args(vec![mode, &namespaced_remote_file_path.to_string_lossy()]),
)
.await?
.map_err(|(_, err)| {
ProviderError::SendFile(
self.name.clone(),
local_file_path.to_string_lossy().to_string(),
anyhow!("{err}"),
)
})?;
Ok(())
}
async fn receive_file(
&self,
remote_file_path: &Path,
local_file_path: &Path,
) -> Result<(), ProviderError> {
let namespaced_remote_file_path = PathBuf::from(format!(
"{}{}",
&self.base_dir.to_string_lossy(),
remote_file_path.to_string_lossy()
));
self.filesystem
.copy(namespaced_remote_file_path, local_file_path)
.await?;
Ok(())
}
async fn pause(&self) -> Result<(), ProviderError> {
let process_id = self.process_id()?;
kill(process_id, Signal::SIGSTOP)
.map_err(|err| ProviderError::PauseNodeFailed(self.name.clone(), err.into()))?;
Ok(())
}
async fn resume(&self) -> Result<(), ProviderError> {
let process_id = self.process_id()?;
nix::sys::signal::kill(process_id, Signal::SIGCONT)
.map_err(|err| ProviderError::ResumeNodeFailed(self.name.clone(), err.into()))?;
Ok(())
}
async fn restart(&self, after: Option<Duration>) -> Result<(), ProviderError> {
if let Some(duration) = after {
sleep(duration).await;
}
self.abort()
.await
.map_err(|err| ProviderError::RestartNodeFailed(self.name.clone(), err))?;
let (stdout, stderr) = self
.initialize_process()
.await
.map_err(|err| ProviderError::RestartNodeFailed(self.name.clone(), err.into()))?;
self.initialize_log_writing(stdout, stderr).await;
Ok(())
}
async fn destroy(&self) -> Result<(), ProviderError> {
self.abort()
.await
.map_err(|err| ProviderError::DestroyNodeFailed(self.name.clone(), err))?;
if let Some(namespace) = self.namespace.upgrade() {
namespace.nodes.write().await.remove(&self.name);
}
Ok(())
}
}
fn serialize_process_handle<S>(
process_handle: &std::sync::RwLock<Option<ProcessHandle>>,
serializer: S,
) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let pid = process_handle
.read()
.map_err(|_e| S::Error::custom("failed to acquire read lock"))?
.as_ref()
.map(|handle| match handle {
ProcessHandle::Spawned(_, pid) => pid.as_raw(),
ProcessHandle::Attached(pid) => pid.as_raw(),
});
pid.serialize(serializer)
}
@@ -0,0 +1,142 @@
use std::{
collections::HashMap,
path::{Path, PathBuf},
sync::{Arc, Weak},
};
use async_trait::async_trait;
use support::fs::FileSystem;
use tokio::sync::RwLock;
use super::namespace::NativeNamespace;
use crate::{
shared::helpers::extract_namespace_info, types::ProviderCapabilities, DynNamespace, Provider,
ProviderError, ProviderNamespace,
};
pub const PROVIDER_NAME: &str = "native";
pub struct NativeProvider<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
weak: Weak<NativeProvider<FS>>,
capabilities: ProviderCapabilities,
tmp_dir: PathBuf,
filesystem: FS,
pub(super) namespaces: RwLock<HashMap<String, Arc<NativeNamespace<FS>>>>,
}
impl<FS> NativeProvider<FS>
where
FS: FileSystem + Send + Sync + Clone,
{
pub fn new(filesystem: FS) -> Arc<Self> {
Arc::new_cyclic(|weak| NativeProvider {
weak: weak.clone(),
capabilities: ProviderCapabilities {
has_resources: false,
requires_image: false,
prefix_with_full_path: true,
use_default_ports_in_cmd: false,
},
// NOTE: temp_dir in linux return `/tmp` but on mac something like
// `/var/folders/rz/1cyx7hfj31qgb98d8_cg7jwh0000gn/T/`, having
// one `trailing slash` and the other no can cause issues if
// you try to build a fullpath by concatenate. Use Pathbuf to prevent the issue.
tmp_dir: std::env::temp_dir(),
filesystem,
namespaces: RwLock::new(HashMap::new()),
})
}
pub fn tmp_dir(mut self, tmp_dir: impl Into<PathBuf>) -> Self {
self.tmp_dir = tmp_dir.into();
self
}
}
#[async_trait]
impl<FS> Provider for NativeProvider<FS>
where
FS: FileSystem + Send + Sync + Clone + 'static,
{
fn name(&self) -> &str {
PROVIDER_NAME
}
fn capabilities(&self) -> &ProviderCapabilities {
&self.capabilities
}
async fn namespaces(&self) -> HashMap<String, DynNamespace> {
self.namespaces
.read()
.await
.iter()
.map(|(name, namespace)| (name.clone(), namespace.clone() as DynNamespace))
.collect()
}
async fn create_namespace(&self) -> Result<DynNamespace, ProviderError> {
let namespace = NativeNamespace::new(
&self.weak,
&self.tmp_dir,
&self.capabilities,
&self.filesystem,
None,
)
.await?;
self.namespaces
.write()
.await
.insert(namespace.name().to_string(), namespace.clone());
Ok(namespace)
}
async fn create_namespace_with_base_dir(
&self,
base_dir: &Path,
) -> Result<DynNamespace, ProviderError> {
let namespace = NativeNamespace::new(
&self.weak,
&self.tmp_dir,
&self.capabilities,
&self.filesystem,
Some(base_dir),
)
.await?;
self.namespaces
.write()
.await
.insert(namespace.name().to_string(), namespace.clone());
Ok(namespace)
}
async fn create_namespace_from_json(
&self,
json_value: &serde_json::Value,
) -> Result<DynNamespace, ProviderError> {
let (base_dir, name) = extract_namespace_info(json_value)?;
let namespace = NativeNamespace::attach_to_live(
&self.weak,
&self.capabilities,
&self.filesystem,
&base_dir,
&name,
)
.await?;
self.namespaces
.write()
.await
.insert(namespace.name().to_string(), namespace.clone());
Ok(namespace)
}
}
@@ -0,0 +1,3 @@
pub mod constants;
pub mod helpers;
pub mod types;
@@ -0,0 +1,22 @@
use std::net::{IpAddr, Ipv4Addr};
/// Namespace prefix
pub const NAMESPACE_PREFIX: &str = "zombie-";
/// Directory for node configuration
pub const NODE_CONFIG_DIR: &str = "/cfg";
/// Directory for node data dir
pub const NODE_DATA_DIR: &str = "/data";
/// Directory for node relay data dir
pub const NODE_RELAY_DATA_DIR: &str = "/relay-data";
/// Directory for node scripts
pub const NODE_SCRIPTS_DIR: &str = "/scripts";
/// Localhost ip
pub const LOCALHOST: IpAddr = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1));
/// The port substrate listens for p2p connections on
pub const P2P_PORT: u16 = 30333;
/// The remote port Prometheus can be accessed with
pub const PROMETHEUS_PORT: u16 = 9615;
/// The remote port websocket to access the RPC
pub const RPC_WS_PORT: u16 = 9944;
/// The remote port HTTP to access the RPC
pub const RPC_HTTP_PORT: u16 = 9933;
@@ -0,0 +1,79 @@
use std::{env, path::PathBuf};
use anyhow::anyhow;
use crate::{types::RunCommandOptions, DynNode, ProviderError};
/// Check if we are running in `CI` by checking the 'RUN_IN_CI' env var
pub fn running_in_ci() -> bool {
env::var("RUN_IN_CI").unwrap_or_default() == "1"
}
/// Executes a command on a temporary node and extracts the execution result either from the
/// standard output or a file.
pub async fn extract_execution_result(
temp_node: &DynNode,
options: RunCommandOptions,
expected_path: Option<&PathBuf>,
) -> Result<String, ProviderError> {
let output_contents = temp_node
.run_command(options)
.await?
.map_err(|(_, msg)| ProviderError::FileGenerationFailed(anyhow!("{msg}")))?;
// If an expected_path is provided, read the file contents from inside the container
if let Some(expected_path) = expected_path.as_ref() {
Ok(temp_node
.run_command(
RunCommandOptions::new("cat")
.args(vec![expected_path.to_string_lossy().to_string()]),
)
.await?
.map_err(|(_, msg)| {
ProviderError::FileGenerationFailed(anyhow!(format!(
"failed reading expected_path {}: {}",
expected_path.display(),
msg
)))
})?)
} else {
Ok(output_contents)
}
}
pub fn extract_namespace_info(
json_value: &serde_json::Value,
) -> Result<(PathBuf, String), ProviderError> {
let base_dir = json_value
.get("local_base_dir")
.and_then(|v| v.as_str())
.map(PathBuf::from)
.ok_or(ProviderError::InvalidConfig(
"`field local_base_dir` is missing from zombie.json".to_string(),
))?;
let name =
json_value
.get("ns")
.and_then(|v| v.as_str())
.ok_or(ProviderError::InvalidConfig(
"field `ns` is missing from zombie.json".to_string(),
))?;
Ok((base_dir, name.to_string()))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn check_runing_in_ci_env_var() {
assert!(!running_in_ci());
// now set the env var
env::set_var("RUN_IN_CI", "1");
assert!(running_in_ci());
// reset
env::set_var("RUN_IN_CI", "");
}
}
@@ -0,0 +1,34 @@
#!/bin/ash
log() {
echo "$(date +"%F %T") $1"
}
# used to handle the distinction where /cfg is used for k8s and /helpers for docker/podman
# to share a volume across nodes containing helper binaries and independent from /cfg
# where some node files are stored
OUTDIR=$([ -d /helpers ] && echo "/helpers" || echo "/cfg")
# Allow to use our image and just cp'd the binaries.
if [ -f /tmp/curl ]; then
cp /tmp/curl $OUTDIR/curl
log "curl copied"
else
wget github.com/moparisthebest/static-curl/releases/download/v7.83.1/curl-amd64 -O "$OUTDIR/curl"
log "curl downloaded"
fi;
chmod +x "$OUTDIR/curl"
log "curl chmoded"
if [ -f /tmp/coreutils ]; then
cp /tmp/coreutils $OUTDIR/coreutils
log "coreutils copied"
else
wget -qO- github.com/uutils/coreutils/releases/download/0.0.17/coreutils-0.0.17-x86_64-unknown-linux-musl.tar.gz | tar -xz -C $OUTDIR --strip-components=1 coreutils-0.0.17-x86_64-unknown-linux-musl/coreutils
log "coreutils downloaded"
fi;
chmod +x "$OUTDIR/coreutils"
log "coreutils chmoded"
@@ -0,0 +1,178 @@
#!/bin/bash
set -uxo pipefail
if [ -f /cfg/coreutils ]; then
RM="/cfg/coreutils rm"
MKFIFO="/cfg/coreutils mkfifo"
MKNOD="/cfg/coreutils mknod"
LS="/cfg/coreutils ls"
KILL="/cfg/coreutils kill"
SLEEP="/cfg/coreutils sleep"
ECHO="/cfg/coreutils echo"
elif [ -f /helpers/coreutils ]; then
# used for docker/podman to have a single volume sharing helper binaries
# across nodes independent from the /cfg where some files are stored
# by the node itself
RM="/helpers/coreutils rm"
MKFIFO="/helpers/coreutils mkfifo"
MKNOD="/helpers/coreutils mknod"
LS="/helpers/coreutils ls"
KILL="/helpers/coreutils kill"
SLEEP="/helpers/coreutils sleep"
ECHO="/helpers/coreutils echo"
else
RM="rm"
MKFIFO="mkfifo"
MKNOD="mknod"
LS="ls"
KILL="kill"
SLEEP="sleep"
ECHO="echo"
fi
echo "COMMANDS DEFINED"
# add /cfg as first `looking dir` to allow to overrides commands.
export PATH="/cfg":$PATH
echo "EXPORT PATH"
# setup pipe
pipe=/tmp/zombiepipe
trap "$RM -f $pipe" EXIT
# try mkfifo first and allow to fail
if [[ ! -p $pipe ]]; then
$MKFIFO $pipe
fi
# set immediately exit on any non 0 exit code
set -e
# if fails try mknod
if [[ ! -p $pipe ]]; then
$MKNOD $pipe p
fi
echo "PIPE CREATED"
# init empty
child_pid=""
# get the command to exec
CMD=($@)
# File to store CMD (and update from there)
ZOMBIE_CMD_FILE=/tmp/zombie.cmd
ZOMBIE_CMD_PID=/tmp/zombie.pid
# Store the cmd and make it available to later usage
# NOTE: echo without new line to allow to customize the cmd later
$ECHO -n "${CMD[@]}" > $ZOMBIE_CMD_FILE
echo "COMMAND TO RUN IS: $CMD"
start() {
# redirect the output to be expored to loki
"${CMD[@]}" >> /proc/1/fd/1 2>> /proc/1/fd/2 &
if [[ "$CMD" != "cat" ]]; then
child_pid="$!"
$ECHO $(cat $ZOMBIE_CMD_FILE)
# store pid
$ECHO ${child_pid} > $ZOMBIE_CMD_PID
# sleep a few secs to detect errors bootstraping the node
sleep 3
# check if the process is running
if ! $LS /proc/$child_pid > /dev/null 2>&1 ; then
echo "child process doesn't exist, quiting...";
exit 1;
else
echo "PID: $child_pid alive";
fi;
else
echo "Process not started, PID not stored, since was 'cat'";
fi;
}
restart() {
if [ ! -z "${child_pid}" ]; then
$KILL -9 "$child_pid"
fi
# check if we have timeout
if [[ "$1" ]]; then
$SLEEP "$1"
fi
start
}
pause() {
if [ ! -z "${child_pid}" ]; then
echo "send -STOP to process $child_pid"
$KILL -STOP "$child_pid"
echo "result $?"
# Wait until the process is actually stopped (state 'T')
for i in {1..10}; do
local state
state=$(awk '{print $3}' /proc/$child_pid/stat 2>/dev/null)
if [ "$state" = "T" ]; then
echo "Process $child_pid is paused (state: $state)"
return
fi
$SLEEP 0.2
done
echo "Warning: Process $child_pid not paused after SIGSTOP"
fi
}
resume() {
if [ ! -z "${child_pid}" ]; then
echo "send -CONT to process $child_pid"
$KILL -CONT "$child_pid"
echo "result $?"
# Wait until the process is actually resumed (state not 'T')
for i in {1..10}; do
local state
state=$(awk '{print $3}' /proc/$child_pid/stat 2>/dev/null)
if [ "$state" != "T" ] && [ -n "$state" ]; then
echo "Process $child_pid is resumed (state: $state)"
return
fi
$SLEEP 0.2
done
echo "Warning: Process $child_pid not resumed after SIGCONT"
fi
}
# keep listening from the pipe
while read line <$pipe
echo "read line: ${line}"
do
if [[ "$line" == "start" ]]; then
start
elif [[ "$line" == "quit" ]]; then
break
elif [[ "$line" =~ "restart" ]]; then
# check if we have timeout between restart
if [[ $line =~ [^0-9]+([0-9]+) ]]; then
restart "${BASH_REMATCH[1]}"
else
restart 0
fi;
elif [[ "$line" == "pause" ]]; then
pause
elif [[ "$line" == "resume" ]]; then
resume
fi
done
exit 0
@@ -0,0 +1,375 @@
use std::{
collections::HashMap,
path::{Path, PathBuf},
process::ExitStatus,
};
use configuration::{shared::resources::Resources, types::AssetLocation};
use serde::{Deserialize, Serialize};
pub type Port = u16;
pub type ExecutionResult = Result<String, (ExitStatus, String)>;
#[derive(Debug, Clone, PartialEq)]
pub struct ProviderCapabilities {
// default ports internal
/// Ensure that we have an image for each node (k8s/podman/docker)
pub requires_image: bool,
/// Allow to customize the resources through manifest (k8s).
pub has_resources: bool,
/// Used in native to prefix filepath with fullpath
pub prefix_with_full_path: bool,
/// Use default ports in node cmd/args.
/// NOTE: generally used in k8s/dockers since the images expose those ports.
pub use_default_ports_in_cmd: bool,
}
#[derive(Debug, Clone)]
pub struct SpawnNodeOptions {
/// Name of the node
pub name: String,
/// Image of the node (IFF is supported by the provider)
pub image: Option<String>,
/// Resources to apply to the node (IFF is supported by the provider)
pub resources: Option<Resources>,
/// Main command to execute
pub program: String,
/// Arguments to pass to the main command
pub args: Vec<String>,
/// Environment to set when running the `program`
pub env: Vec<(String, String)>,
// TODO: rename startup_files
/// Files to inject at startup
pub injected_files: Vec<TransferedFile>,
/// Paths to create before start the node (e.g keystore)
/// should be created with `create_dir_all` in order
/// to create the full path even when we have missing parts
pub created_paths: Vec<PathBuf>,
/// Database snapshot to be injected (should be a tgz file)
/// Could be a local or remote asset
pub db_snapshot: Option<AssetLocation>,
pub port_mapping: Option<HashMap<Port, Port>>,
/// Optionally specify a log path for the node
pub node_log_path: Option<PathBuf>,
}
impl SpawnNodeOptions {
pub fn new<S>(name: S, program: S) -> Self
where
S: AsRef<str>,
{
Self {
name: name.as_ref().to_string(),
image: None,
resources: None,
program: program.as_ref().to_string(),
args: vec![],
env: vec![],
injected_files: vec![],
created_paths: vec![],
db_snapshot: None,
port_mapping: None,
node_log_path: None,
}
}
pub fn image<S>(mut self, image: S) -> Self
where
S: AsRef<str>,
{
self.image = Some(image.as_ref().to_string());
self
}
pub fn resources(mut self, resources: Resources) -> Self {
self.resources = Some(resources);
self
}
pub fn db_snapshot(mut self, db_snap: Option<AssetLocation>) -> Self {
self.db_snapshot = db_snap;
self
}
pub fn args<S, I>(mut self, args: I) -> Self
where
S: AsRef<str>,
I: IntoIterator<Item = S>,
{
self.args = args.into_iter().map(|s| s.as_ref().to_string()).collect();
self
}
pub fn env<S, I>(mut self, env: I) -> Self
where
S: AsRef<str>,
I: IntoIterator<Item = (S, S)>,
{
self.env = env
.into_iter()
.map(|(name, value)| (name.as_ref().to_string(), value.as_ref().to_string()))
.collect();
self
}
pub fn injected_files<I>(mut self, injected_files: I) -> Self
where
I: IntoIterator<Item = TransferedFile>,
{
self.injected_files = injected_files.into_iter().collect();
self
}
pub fn created_paths<P, I>(mut self, created_paths: I) -> Self
where
P: AsRef<Path>,
I: IntoIterator<Item = P>,
{
self.created_paths = created_paths
.into_iter()
.map(|path| path.as_ref().into())
.collect();
self
}
pub fn port_mapping(mut self, ports: HashMap<Port, Port>) -> Self {
self.port_mapping = Some(ports);
self
}
pub fn node_log_path(mut self, path: Option<PathBuf>) -> Self {
self.node_log_path = path;
self
}
}
#[derive(Debug)]
pub struct GenerateFileCommand {
pub program: String,
pub args: Vec<String>,
pub env: Vec<(String, String)>,
pub local_output_path: PathBuf,
}
impl GenerateFileCommand {
pub fn new<S, P>(program: S, local_output_path: P) -> Self
where
S: AsRef<str>,
P: AsRef<Path>,
{
Self {
program: program.as_ref().to_string(),
args: vec![],
env: vec![],
local_output_path: local_output_path.as_ref().into(),
}
}
pub fn args<S, I>(mut self, args: I) -> Self
where
S: AsRef<str>,
I: IntoIterator<Item = S>,
{
self.args = args.into_iter().map(|s| s.as_ref().to_string()).collect();
self
}
pub fn env<S, I>(mut self, env: I) -> Self
where
S: AsRef<str>,
I: IntoIterator<Item = (S, S)>,
{
self.env = env
.into_iter()
.map(|(name, value)| (name.as_ref().to_string(), value.as_ref().to_string()))
.collect();
self
}
}
#[derive(Debug)]
pub struct GenerateFilesOptions {
pub commands: Vec<GenerateFileCommand>,
pub image: Option<String>,
pub injected_files: Vec<TransferedFile>,
// Allow to control the name of the node used to create the files.
pub temp_name: Option<String>,
pub expected_path: Option<PathBuf>,
}
impl GenerateFilesOptions {
pub fn new<I>(commands: I, image: Option<String>, expected_path: Option<PathBuf>) -> Self
where
I: IntoIterator<Item = GenerateFileCommand>,
{
Self {
commands: commands.into_iter().collect(),
injected_files: vec![],
image,
temp_name: None,
expected_path,
}
}
pub fn with_files<I>(
commands: I,
image: Option<String>,
injected_files: &[TransferedFile],
expected_path: Option<PathBuf>,
) -> Self
where
I: IntoIterator<Item = GenerateFileCommand>,
{
Self {
commands: commands.into_iter().collect(),
injected_files: injected_files.into(),
image,
temp_name: None,
expected_path,
}
}
pub fn image<S>(mut self, image: S) -> Self
where
S: AsRef<str>,
{
self.image = Some(image.as_ref().to_string());
self
}
pub fn injected_files<I>(mut self, injected_files: I) -> Self
where
I: IntoIterator<Item = TransferedFile>,
{
self.injected_files = injected_files.into_iter().collect();
self
}
pub fn temp_name(mut self, name: impl Into<String>) -> Self {
self.temp_name = Some(name.into());
self
}
}
#[derive(Debug)]
pub struct RunCommandOptions {
pub program: String,
pub args: Vec<String>,
pub env: Vec<(String, String)>,
}
impl RunCommandOptions {
pub fn new<S>(program: S) -> Self
where
S: AsRef<str>,
{
Self {
program: program.as_ref().to_string(),
args: vec![],
env: vec![],
}
}
pub fn args<S, I>(mut self, args: I) -> Self
where
S: AsRef<str>,
I: IntoIterator<Item = S>,
{
self.args = args.into_iter().map(|s| s.as_ref().to_string()).collect();
self
}
pub fn env<S, I>(mut self, env: I) -> Self
where
S: AsRef<str>,
I: IntoIterator<Item = (S, S)>,
{
self.env = env
.into_iter()
.map(|(name, value)| (name.as_ref().to_string(), value.as_ref().to_string()))
.collect();
self
}
}
pub struct RunScriptOptions {
pub local_script_path: PathBuf,
pub args: Vec<String>,
pub env: Vec<(String, String)>,
}
impl RunScriptOptions {
pub fn new<P>(local_script_path: P) -> Self
where
P: AsRef<Path>,
{
Self {
local_script_path: local_script_path.as_ref().into(),
args: vec![],
env: vec![],
}
}
pub fn args<S, I>(mut self, args: I) -> Self
where
S: AsRef<str>,
I: IntoIterator<Item = S>,
{
self.args = args.into_iter().map(|s| s.as_ref().to_string()).collect();
self
}
pub fn env<S, I>(mut self, env: I) -> Self
where
S: AsRef<str>,
I: IntoIterator<Item = (S, S)>,
{
self.env = env
.into_iter()
.map(|(name, value)| (name.as_ref().to_string(), value.as_ref().to_string()))
.collect();
self
}
}
// TODO(team): I think we can rename it to FileMap?
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TransferedFile {
pub local_path: PathBuf,
pub remote_path: PathBuf,
// TODO: Can be narrowed to have strict typing on this?
pub mode: String,
}
impl TransferedFile {
pub fn new<P>(local_path: P, remote_path: P) -> Self
where
P: AsRef<Path>,
{
Self {
local_path: local_path.as_ref().into(),
remote_path: remote_path.as_ref().into(),
mode: "0644".to_string(), // default to rw-r--r--
}
}
pub fn mode<S>(mut self, mode: S) -> Self
where
S: AsRef<str>,
{
self.mode = mode.as_ref().to_string();
self
}
}
impl std::fmt::Display for TransferedFile {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"File to transfer (local: {}, remote: {})",
self.local_path.display(),
self.remote_path.display()
)
}
}